From 1afa8e94ce7e7a75d8c93dcc22018be3fe2e2334 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 26 Feb 2023 11:55:26 +0200 Subject: [PATCH 001/150] feat: deposit tx --- crates/primitives/Cargo.toml | 9 +- crates/primitives/src/lib.rs | 2 + crates/primitives/src/transaction/mod.rs | 130 +++++++++++++++++- crates/primitives/src/transaction/optimism.rs | 70 ++++++++++ crates/primitives/src/transaction/tx_type.rs | 18 ++- 5 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 crates/primitives/src/transaction/optimism.rs diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 12822404dde..f1d2a1ec08c 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -89,8 +89,13 @@ pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterio [features] default = [] -arbitrary = ["revm-primitives/arbitrary", "dep:arbitrary", "dep:proptest", "dep:proptest-derive"] -test-utils = [] +optimism = [] +arbitrary = [ + "revm-primitives/arbitrary", + "dep:arbitrary", + "dep:proptest", + "dep:proptest-derive", +] [[bench]] name = "recover_ecdsa_crit" diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index a76bcba902b..96e8c357c28 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -91,6 +91,8 @@ pub use transaction::{ TxEip1559, TxEip2930, TxLegacy, TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; +#[cfg(feature = "optimism")] +pub use transaction::{TxDeposit, DEPOSIT_TX_TYPE, DEPOSIT_VERSION}; pub use withdrawal::Withdrawal; /// A block hash. diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 055402c302f..66618ba0946 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -22,6 +22,11 @@ mod signature; mod tx_type; pub(crate) mod util; +#[cfg(feature = "optimism")] +mod optimism; +#[cfg(feature = "optimism")] +pub use optimism::{TxDeposit, DEPOSIT_TX_TYPE, DEPOSIT_VERSION}; + /// Legacy transaction. #[main_codec] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] @@ -180,6 +185,9 @@ pub enum Transaction { Eip2930(TxEip2930), /// A transaction with a priority fee ([EIP-1559](https://eips.ethereum.org/EIPS/eip-1559)). Eip1559(TxEip1559), + #[cfg(feature = "optimism")] + /// Deposit transaction. + Deposit(TxDeposit), } impl Transaction { @@ -310,6 +318,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { chain_id, .. }) => *chain_id, Transaction::Eip2930(TxEip2930 { chain_id, .. }) => Some(*chain_id), Transaction::Eip1559(TxEip1559 { chain_id, .. }) => Some(*chain_id), + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => None, } } @@ -319,6 +329,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { chain_id: ref mut c, .. }) => *c = Some(chain_id), Transaction::Eip2930(TxEip2930 { chain_id: ref mut c, .. }) => *c = chain_id, Transaction::Eip1559(TxEip1559 { chain_id: ref mut c, .. }) => *c = chain_id, + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => (), } } @@ -329,6 +341,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { to, .. }) | Transaction::Eip2930(TxEip2930 { to, .. }) | Transaction::Eip1559(TxEip1559 { to, .. }) => to, + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { to, .. }) => to, } } @@ -343,6 +357,8 @@ impl Transaction { Transaction::Legacy { .. } => TxType::Legacy, Transaction::Eip2930 { .. } => TxType::EIP2930, Transaction::Eip1559 { .. } => TxType::EIP1559, + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => TxType::DEPOSIT, } } @@ -352,6 +368,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { value, .. }) => value, Transaction::Eip2930(TxEip2930 { value, .. }) => value, Transaction::Eip1559(TxEip1559 { value, .. }) => value, + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { value, .. }) => value, } } @@ -361,6 +379,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { nonce, .. }) => *nonce, Transaction::Eip2930(TxEip2930 { nonce, .. }) => *nonce, Transaction::Eip1559(TxEip1559 { nonce, .. }) => *nonce, + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => todo!(), // TODO: } } @@ -370,6 +390,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { gas_limit, .. }) | Transaction::Eip2930(TxEip2930 { gas_limit, .. }) | Transaction::Eip1559(TxEip1559 { gas_limit, .. }) => *gas_limit, + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { gas_limit, .. }) => *gas_limit, } } @@ -381,6 +403,10 @@ impl Transaction { Transaction::Legacy(TxLegacy { gas_price, .. }) | Transaction::Eip2930(TxEip2930 { gas_price, .. }) => *gas_price, Transaction::Eip1559(TxEip1559 { max_fee_per_gas, .. }) => *max_fee_per_gas, + #[cfg(feature = "optimism")] + // Deposit transactions buy their L2 gas on L1 and, as such, the L2 gas is not + // refundable. + Transaction::Deposit(_) => 0, } } @@ -395,6 +421,8 @@ impl Transaction { Transaction::Eip1559(TxEip1559 { max_priority_fee_per_gas, .. }) => { Some(*max_priority_fee_per_gas) } + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => None, } } @@ -478,6 +506,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { input, .. }) => input, Transaction::Eip2930(TxEip2930 { input, .. }) => input, Transaction::Eip1559(TxEip1559 { input, .. }) => input, + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { input, .. }) => input, } } @@ -574,6 +604,8 @@ impl Transaction { len += access_list.length(); len } + #[cfg(feature = "optimism")] + Transaction::Deposit(deposit) => deposit.fields_len(), } } @@ -636,6 +668,8 @@ impl Transaction { input.0.encode(out); access_list.encode(out); } + #[cfg(feature = "optimism")] + Transaction::Deposit(deposit) => deposit.encode_fields(out), } } } @@ -657,6 +691,13 @@ impl Encodable for Transaction { self.encode_fields(out); self.encode_eip155_fields(out); } + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => { + out.put_u8(self.tx_type() as u8); + out.put_u8(DEPOSIT_VERSION); + Header { list: true, payload_length: self.fields_len() }.encode(out); + self.encode_fields(out); + } _ => { out.put_u8(self.tx_type() as u8); Header { list: true, payload_length: self.fields_len() }.encode(out); @@ -672,6 +713,12 @@ impl Encodable for Transaction { // 'header length' + 'payload length' length_of_length(payload_length) + payload_length } + #[cfg(feature = "optimism")] + Transaction::Deposit { .. } => { + let payload_length = self.fields_len(); + // 'tx type byte length' + 'version byte' + 'header length' + 'payload length' + 1 + 1 + length_of_length(payload_length) + payload_length + } _ => { let payload_length = self.fields_len(); // 'transaction type byte length' + 'header length' + 'payload length' @@ -934,6 +981,10 @@ impl TransactionSigned { /// /// Returns `None` if the transaction's signature is invalid, see also [Self::recover_signer]. pub fn recover_signer(&self) -> Option
{ + #[cfg(feature = "optimism")] + if let Transaction::Deposit(TxDeposit { from, .. }) = self.transaction { + return Some(from.clone()) + } let signature_hash = self.signature_hash(); self.signature.recover_signer(signature_hash) } @@ -974,7 +1025,49 @@ impl TransactionSigned { /// Inner encoding function that is used for both rlp [`Encodable`] trait and for calculating /// hash that for eip2718 does not require rlp header pub(crate) fn encode_inner(&self, out: &mut dyn bytes::BufMut, with_header: bool) { - self.transaction.encode_with_signature(&self.signature, out, with_header); + match self.transaction { + Transaction::Legacy(TxLegacy { chain_id, .. }) => { + // do nothing w/ with_header + let payload_length = self.transaction.fields_len() + + self.signature.payload_len_with_eip155_chain_id(chain_id); + let header = Header { list: true, payload_length }; + header.encode(out); + self.transaction.encode_fields(out); + self.signature.encode_with_eip155_chain_id(out, chain_id); + } + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => { + let payload_length = self.transaction.fields_len() + self.signature.payload_len(); + if with_header { + Header { + list: false, + payload_length: 1 + 1 + length_of_length(payload_length) + payload_length, + } + .encode(out); + } + out.put_u8(self.transaction.tx_type() as u8); + out.put_u8(DEPOSIT_VERSION); + let header = Header { list: true, payload_length }; + header.encode(out); + self.transaction.encode_fields(out); + self.signature.encode(out); + } + _ => { + let payload_length = self.transaction.fields_len() + self.signature.payload_len(); + if with_header { + Header { + list: false, + payload_length: 1 + length_of_length(payload_length) + payload_length, + } + .encode(out); + } + out.put_u8(self.transaction.tx_type() as u8); + let header = Header { list: true, payload_length }; + header.encode(out); + self.transaction.encode_fields(out); + self.signature.encode(out); + } + } } /// Output the length of the encode_inner(out, true). Note to assume that `with_header` is only @@ -987,6 +1080,13 @@ impl TransactionSigned { // 'header length' + 'payload length' length_of_length(payload_length) + payload_length } + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => { + let payload_length = self.transaction.fields_len() + self.signature.payload_len(); + // 'tx type byte length' + 'version byte' + 'header length' + 'payload length' + let len = 1 + 1 + length_of_length(payload_length) + payload_length; + length_of_length(len) + len + } _ => { let payload_length = self.transaction.fields_len() + self.signature.payload_len(); // 'transaction type byte length' + 'header length' + 'payload length' @@ -1052,6 +1152,16 @@ impl TransactionSigned { let tx_type = *data.first().ok_or(DecodeError::InputTooShort)?; data.advance(1); + + #[cfg(feature = "optimism")] + if tx_type == DEPOSIT_TX_TYPE { + let version = *data.first().ok_or(DecodeError::InputTooShort)?; + if version != DEPOSIT_VERSION { + return Err(DecodeError::Custom("Deposit version mismatch")) + } + data.advance(1); + } + // decode the list header for the rest of the transaction let header = Header::decode(data)?; if !header.list { @@ -1060,6 +1170,8 @@ impl TransactionSigned { // length of tx encoding = tx type byte (size = 1) + length of header + payload length let tx_length = 1 + header.length() + header.payload_length; + #[cfg(feature = "optimism")] + let tx_length = if tx_type == DEPOSIT_TX_TYPE { tx_length + 1 } else { tx_length }; // decode common fields let transaction = match tx_type { @@ -1084,6 +1196,22 @@ impl TransactionSigned { input: Bytes(Decodable::decode(data)?), access_list: Decodable::decode(data)?, }), + #[cfg(feature = "optimism")] + DEPOSIT_TX_TYPE => Transaction::Deposit(TxDeposit { + source_hash: Decodable::decode(data)?, + from: Decodable::decode(data)?, + to: Decodable::decode(data)?, + mint: if *data.first().ok_or(DecodeError::InputTooShort)? == EMPTY_STRING_CODE { + data.advance(1); + None + } else { + Some(Decodable::decode(data)?) + }, + value: Decodable::decode(data)?, + input: Decodable::decode(data)?, + gas_limit: Decodable::decode(data)?, + is_system_transaction: Decodable::decode(data)?, + }), _ => return Err(DecodeError::Custom("unsupported typed transaction type")), }; diff --git a/crates/primitives/src/transaction/optimism.rs b/crates/primitives/src/transaction/optimism.rs new file mode 100644 index 00000000000..1b48f43fd23 --- /dev/null +++ b/crates/primitives/src/transaction/optimism.rs @@ -0,0 +1,70 @@ +use crate::{Address, Bytes, TransactionKind, H256}; +use reth_codecs::{main_codec, Compact}; +use reth_rlp::{Encodable, EMPTY_STRING_CODE}; + +/// EIP-2718 transaction type selector. +pub const DEPOSIT_TX_TYPE: u8 = 126; + +/// A versioned byte sequence to enable the protocol to upgrade the deposit transaction type without +/// changing the transaction type selector. +pub const DEPOSIT_VERSION: u8 = 0; + +/// Deposited transactions, also known as deposits are transactions which are initiated on L1, and +/// executed on L2. This document outlines a new transaction type for deposits. It also describes +/// how deposits are initiated on L1, along with the authorization and validation conditions on L2. +#[main_codec] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +pub struct TxDeposit { + /// Hash that uniquely identifies the source of the deposit. + pub source_hash: H256, + /// The address of the sender account. + pub from: Address, + /// The address of the recipient account, or the null (zero-length) address if the deposited + /// transaction is a contract creation. + pub to: TransactionKind, + /// The ETH value to mint on L2. + pub mint: Option, + /// The ETH value to send to the recipient account. + pub value: u128, + /// The gas limit for the L2 transaction. + pub gas_limit: u64, + /// Field indicating if this transaction is exempt from the L2 gas limit. + pub is_system_transaction: bool, + /// Input has two uses depending if transaction is Create or Call (if `to` field is None or + /// Some). + pub input: Bytes, +} + +impl TxDeposit { + /// Outputs the length of the transaction's fields, without a RLP header or length of the + /// eip155 fields. + pub(crate) fn fields_len(&self) -> usize { + let mut len = 0; + len += self.source_hash.length(); + len += self.from.length(); + len += self.to.length(); + len += self.mint.map(|mint| mint.length()).unwrap_or(1); + len += self.value.length(); + len += self.input.0.length(); + len += self.gas_limit.length(); + len += self.is_system_transaction.length(); + len + } + + /// Encodes only the transaction's fields into the desired buffer, without a RLP header. + /// https://github.com/ethereum-optimism/optimism/blob/develop/specs/deposits.md#the-deposited-transaction-type + pub(crate) fn encode_fields(&self, out: &mut dyn bytes::BufMut) { + self.source_hash.encode(out); + self.from.encode(out); + self.to.encode(out); + if let Some(mint) = self.mint { + mint.encode(out); + } else { + out.put_u8(EMPTY_STRING_CODE); + } + self.value.encode(out); + self.input.encode(out); + self.gas_limit.encode(out); + self.is_system_transaction.encode(out); + } +} diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index 428e06f4534..1e7c9edfae5 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -23,14 +23,19 @@ pub enum TxType { EIP2930 = 1_isize, /// Transaction with Priority fee EIP1559 = 2_isize, + #[cfg(feature = "optimism")] + /// OP Deposit transaction. + DEPOSIT = 126_isize, } impl From for u8 { fn from(value: TxType) -> Self { match value { - TxType::Legacy => LEGACY_TX_TYPE_ID, - TxType::EIP2930 => EIP2930_TX_TYPE_ID, - TxType::EIP1559 => EIP1559_TX_TYPE_ID, + TxType::Legacy => 0, + TxType::EIP2930 => 1, + TxType::EIP1559 => 2, + #[cfg(feature = "optimism")] + TxType::DEPOSIT => 126, } } } @@ -50,6 +55,8 @@ impl Compact for TxType { TxType::Legacy => 0, TxType::EIP2930 => 1, TxType::EIP1559 => 2, + #[cfg(feature = "optimism")] + TxType::DEPOSIT => 126, } } @@ -58,7 +65,10 @@ impl Compact for TxType { match identifier { 0 => TxType::Legacy, 1 => TxType::EIP2930, - _ => TxType::EIP1559, + 2 => TxType::EIP1559, + #[cfg(feature = "optimism")] + 126 => TxType::DEPOSIT, + _ => panic!("unknown transaction type {identifier}"), }, buf, ) From f9a1111c9006763b5729b6626c354504e4218f38 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 26 Feb 2023 14:13:36 +0200 Subject: [PATCH 002/150] chainspec and forks --- crates/primitives/src/chain/spec.rs | 228 +++++++++++++++------------- crates/primitives/src/hardfork.rs | 22 +++ 2 files changed, 142 insertions(+), 108 deletions(-) diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 0c84dc27539..9d493d2a14c 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -16,118 +16,94 @@ use std::{ }; /// The Ethereum mainnet spec -pub static MAINNET: Lazy> = Lazy::new(|| { - ChainSpec { - chain: Chain::mainnet(), - genesis: serde_json::from_str(include_str!("../../res/genesis/mainnet.json")) - .expect("Can't deserialize Mainnet genesis json"), - genesis_hash: Some(H256(hex!( - "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" - ))), - // - paris_block_and_final_difficulty: Some(( - 15537394, - U256::from(58_750_003_716_598_352_816_469u128), - )), - fork_timestamps: ForkTimestamps::default().shanghai(1681338455), - hardforks: BTreeMap::from([ - (Hardfork::Frontier, ForkCondition::Block(0)), - (Hardfork::Homestead, ForkCondition::Block(1150000)), - (Hardfork::Dao, ForkCondition::Block(1920000)), - (Hardfork::Tangerine, ForkCondition::Block(2463000)), - (Hardfork::SpuriousDragon, ForkCondition::Block(2675000)), - (Hardfork::Byzantium, ForkCondition::Block(4370000)), - (Hardfork::Constantinople, ForkCondition::Block(7280000)), - (Hardfork::Petersburg, ForkCondition::Block(7280000)), - (Hardfork::Istanbul, ForkCondition::Block(9069000)), - (Hardfork::MuirGlacier, ForkCondition::Block(9200000)), - (Hardfork::Berlin, ForkCondition::Block(12244000)), - (Hardfork::London, ForkCondition::Block(12965000)), - (Hardfork::ArrowGlacier, ForkCondition::Block(13773000)), - (Hardfork::GrayGlacier, ForkCondition::Block(15050000)), - ( - Hardfork::Paris, - ForkCondition::TTD { - fork_block: None, - total_difficulty: U256::from(58_750_000_000_000_000_000_000_u128), - }, - ), - (Hardfork::Shanghai, ForkCondition::Timestamp(1681338455)), - ]), - } - .into() +pub static MAINNET: Lazy = Lazy::new(|| ChainSpec { + chain: Chain::mainnet(), + genesis: serde_json::from_str(include_str!("../../res/genesis/mainnet.json")) + .expect("Can't deserialize Mainnet genesis json"), + genesis_hash: Some(H256(hex!( + "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" + ))), + hardforks: BTreeMap::from([ + (Hardfork::Frontier, ForkCondition::Block(0)), + (Hardfork::Homestead, ForkCondition::Block(1150000)), + (Hardfork::Dao, ForkCondition::Block(1920000)), + (Hardfork::Tangerine, ForkCondition::Block(2463000)), + (Hardfork::SpuriousDragon, ForkCondition::Block(2675000)), + (Hardfork::Byzantium, ForkCondition::Block(4370000)), + (Hardfork::Constantinople, ForkCondition::Block(7280000)), + (Hardfork::Petersburg, ForkCondition::Block(7280000)), + (Hardfork::Istanbul, ForkCondition::Block(9069000)), + (Hardfork::MuirGlacier, ForkCondition::Block(9200000)), + (Hardfork::Berlin, ForkCondition::Block(12244000)), + (Hardfork::London, ForkCondition::Block(12965000)), + (Hardfork::ArrowGlacier, ForkCondition::Block(13773000)), + (Hardfork::GrayGlacier, ForkCondition::Block(15050000)), + ( + Hardfork::Paris, + ForkCondition::TTD { + fork_block: None, + total_difficulty: U256::from(58_750_000_000_000_000_000_000_u128), + }, + ), + ]), + #[cfg(feature = "optimism")] + optimism: None, }); /// The Goerli spec -pub static GOERLI: Lazy> = Lazy::new(|| { - ChainSpec { - chain: Chain::goerli(), - genesis: serde_json::from_str(include_str!("../../res/genesis/goerli.json")) - .expect("Can't deserialize Goerli genesis json"), - genesis_hash: Some(H256(hex!( - "bf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a" - ))), - // - paris_block_and_final_difficulty: Some((7382818, U256::from(10_790_000))), - fork_timestamps: ForkTimestamps::default().shanghai(1678832736), - hardforks: BTreeMap::from([ - (Hardfork::Frontier, ForkCondition::Block(0)), - (Hardfork::Homestead, ForkCondition::Block(0)), - (Hardfork::Dao, ForkCondition::Block(0)), - (Hardfork::Tangerine, ForkCondition::Block(0)), - (Hardfork::SpuriousDragon, ForkCondition::Block(0)), - (Hardfork::Byzantium, ForkCondition::Block(0)), - (Hardfork::Constantinople, ForkCondition::Block(0)), - (Hardfork::Petersburg, ForkCondition::Block(0)), - (Hardfork::Istanbul, ForkCondition::Block(1561651)), - (Hardfork::Berlin, ForkCondition::Block(4460644)), - (Hardfork::London, ForkCondition::Block(5062605)), - ( - Hardfork::Paris, - ForkCondition::TTD { fork_block: None, total_difficulty: U256::from(10_790_000) }, - ), - (Hardfork::Shanghai, ForkCondition::Timestamp(1678832736)), - ]), - } - .into() +pub static GOERLI: Lazy = Lazy::new(|| ChainSpec { + chain: Chain::goerli(), + genesis: serde_json::from_str(include_str!("../../res/genesis/goerli.json")) + .expect("Can't deserialize Goerli genesis json"), + genesis_hash: Some(H256(hex!( + "bf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a" + ))), + hardforks: BTreeMap::from([ + (Hardfork::Frontier, ForkCondition::Block(0)), + (Hardfork::Istanbul, ForkCondition::Block(1561651)), + (Hardfork::Berlin, ForkCondition::Block(4460644)), + (Hardfork::London, ForkCondition::Block(5062605)), + ( + Hardfork::Paris, + ForkCondition::TTD { fork_block: None, total_difficulty: U256::from(10_790_000) }, + ), + ]), + #[cfg(feature = "optimism")] + optimism: None, }); /// The Sepolia spec -pub static SEPOLIA: Lazy> = Lazy::new(|| { - ChainSpec { - chain: Chain::sepolia(), - genesis: serde_json::from_str(include_str!("../../res/genesis/sepolia.json")) - .expect("Can't deserialize Sepolia genesis json"), - genesis_hash: Some(H256(hex!( - "25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9" - ))), - // - paris_block_and_final_difficulty: Some((1450409, U256::from(17_000_018_015_853_232u128))), - fork_timestamps: ForkTimestamps::default().shanghai(1677557088), - hardforks: BTreeMap::from([ - (Hardfork::Frontier, ForkCondition::Block(0)), - (Hardfork::Homestead, ForkCondition::Block(0)), - (Hardfork::Dao, ForkCondition::Block(0)), - (Hardfork::Tangerine, ForkCondition::Block(0)), - (Hardfork::SpuriousDragon, ForkCondition::Block(0)), - (Hardfork::Byzantium, ForkCondition::Block(0)), - (Hardfork::Constantinople, ForkCondition::Block(0)), - (Hardfork::Petersburg, ForkCondition::Block(0)), - (Hardfork::Istanbul, ForkCondition::Block(0)), - (Hardfork::MuirGlacier, ForkCondition::Block(0)), - (Hardfork::Berlin, ForkCondition::Block(0)), - (Hardfork::London, ForkCondition::Block(0)), - ( - Hardfork::Paris, - ForkCondition::TTD { - fork_block: Some(1735371), - total_difficulty: U256::from(17_000_000_000_000_000u64), - }, - ), - (Hardfork::Shanghai, ForkCondition::Timestamp(1677557088)), - ]), - } - .into() +pub static SEPOLIA: Lazy = Lazy::new(|| ChainSpec { + chain: Chain::sepolia(), + genesis: serde_json::from_str(include_str!("../../res/genesis/sepolia.json")) + .expect("Can't deserialize Sepolia genesis json"), + genesis_hash: Some(H256(hex!( + "25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9" + ))), + hardforks: BTreeMap::from([ + (Hardfork::Frontier, ForkCondition::Block(0)), + (Hardfork::Homestead, ForkCondition::Block(0)), + (Hardfork::Dao, ForkCondition::Block(0)), + (Hardfork::Tangerine, ForkCondition::Block(0)), + (Hardfork::SpuriousDragon, ForkCondition::Block(0)), + (Hardfork::Byzantium, ForkCondition::Block(0)), + (Hardfork::Constantinople, ForkCondition::Block(0)), + (Hardfork::Petersburg, ForkCondition::Block(0)), + (Hardfork::Istanbul, ForkCondition::Block(0)), + (Hardfork::MuirGlacier, ForkCondition::Block(0)), + (Hardfork::Berlin, ForkCondition::Block(0)), + (Hardfork::London, ForkCondition::Block(0)), + ( + Hardfork::Paris, + ForkCondition::TTD { + fork_block: Some(1735371), + total_difficulty: U256::from(17_000_000_000_000_000u64), + }, + ), + (Hardfork::Shanghai, ForkCondition::Timestamp(1677557088)), + ]), + #[cfg(feature = "optimism")] + optimism: None, }); /// An Ethereum chain specification. @@ -164,6 +140,20 @@ pub struct ChainSpec { /// The active hard forks and their activation conditions pub hardforks: BTreeMap, + + #[cfg(feature = "optimism")] + /// Optimism configuration + pub optimism: Option, +} + +#[cfg(feature = "optimism")] +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Optimism configuration. +pub struct OptimismConfig { + /// Elasticity multiplier as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) + pub eip_1559_elasticity: u64, + /// Base fee max change denominator as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) + pub eip_1559_denominator: u64, } impl ChainSpec { @@ -386,7 +376,8 @@ impl From for ChainSpec { genesis_hash: None, fork_timestamps: ForkTimestamps::from_hardforks(&hardforks), hardforks, - paris_block_and_final_difficulty: None, + #[cfg(feature = "optimism")] + optimism: None, } } } @@ -458,6 +449,8 @@ pub struct ChainSpecBuilder { chain: Option, genesis: Option, hardforks: BTreeMap, + #[cfg(feature = "optimism")] + optimism: Option, } impl ChainSpecBuilder { @@ -467,6 +460,8 @@ impl ChainSpecBuilder { chain: Some(MAINNET.chain), genesis: Some(MAINNET.genesis.clone()), hardforks: MAINNET.hardforks.clone(), + #[cfg(feature = "optimism")] + optimism: None, } } @@ -577,6 +572,20 @@ impl ChainSpecBuilder { self } + #[cfg(feature = "optimism")] + /// Enable Bedrock at genesis + pub fn bedrock_activated(mut self) -> Self { + self.hardforks.insert(Hardfork::Bedrock, ForkCondition::Block(0)); + self + } + + #[cfg(feature = "optimism")] + /// Enable Bedrock at genesis + pub fn regolith_activated(mut self) -> Self { + self.hardforks.insert(Hardfork::Regolith, ForkCondition::Timestamp(0)); + self + } + /// Build the resulting [`ChainSpec`]. /// /// # Panics @@ -590,7 +599,8 @@ impl ChainSpecBuilder { genesis_hash: None, fork_timestamps: ForkTimestamps::from_hardforks(&self.hardforks), hardforks: self.hardforks, - paris_block_and_final_difficulty: None, + #[cfg(feature = "optimism")] + optimism: self.optimism, } } } @@ -601,6 +611,8 @@ impl From<&Arc> for ChainSpecBuilder { chain: Some(value.chain), genesis: Some(value.genesis.clone()), hardforks: value.hardforks.clone(), + #[cfg(feature = "optimism")] + optimism: value.optimism.clone(), } } } diff --git a/crates/primitives/src/hardfork.rs b/crates/primitives/src/hardfork.rs index f26ffbcc403..a2f0f7be399 100644 --- a/crates/primitives/src/hardfork.rs +++ b/crates/primitives/src/hardfork.rs @@ -39,6 +39,12 @@ pub enum Hardfork { Paris, /// Shanghai. Shanghai, + #[cfg(feature = "optimism")] + /// Bedrock. + Bedrock, + #[cfg(feature = "optimism")] + /// Regolith + Regolith, } impl Hardfork { @@ -82,6 +88,10 @@ impl FromStr for Hardfork { "grayglacier" => Hardfork::GrayGlacier, "paris" => Hardfork::Paris, "shanghai" => Hardfork::Shanghai, + #[cfg(feature = "optimism")] + "bedrock" => Hardfork::Bedrock, + #[cfg(feature = "optimism")] + "regolith" => Hardfork::Regolith, _ => return Err(format!("Unknown hardfork: {s}")), }; Ok(hardfork) @@ -146,6 +156,18 @@ mod tests { assert_eq!(hardforks, expected_hardforks); } + #[test] + #[cfg(feature = "optimism")] + fn check_op_hardfork_from_str() { + let hardfork_str = ["beDrOck", "rEgOlITH"]; + let expected_hardforks = [Hardfork::Bedrock, Hardfork::Regolith]; + + let hardforks: Vec = + hardfork_str.iter().map(|h| Hardfork::from_str(h).unwrap()).collect(); + + assert_eq!(hardforks, expected_hardforks); + } + #[test] fn check_nonexistent_hardfork_from_str() { assert!(Hardfork::from_str("not a hardfork").is_err()); From 009d1f0ccf2d22774b768bda6b8c0def378ff953 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 26 Feb 2023 14:41:49 +0200 Subject: [PATCH 003/150] feat: payload attributes op fields --- crates/rpc/rpc-types/Cargo.toml | 9 +- crates/rpc/rpc-types/src/eth/engine.rs | 246 +++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 crates/rpc/rpc-types/src/eth/engine.rs diff --git a/crates/rpc/rpc-types/Cargo.toml b/crates/rpc/rpc-types/Cargo.toml index 5db05498bac..7e56ccbe319 100644 --- a/crates/rpc/rpc-types/Cargo.toml +++ b/crates/rpc/rpc-types/Cargo.toml @@ -24,10 +24,7 @@ serde_json = { workspace = true } jsonrpsee-types = { version = "0.18" } [dev-dependencies] -# reth -reth-interfaces = { workspace = true, features = ["test-utils"] } +reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } -# misc -rand = { workspace = true } -assert_matches = "1.5" -similar-asserts = "1.4" +[features] +optimism = ["reth-primitives/optimism"] diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs new file mode 100644 index 00000000000..22c736dc1ab --- /dev/null +++ b/crates/rpc/rpc-types/src/eth/engine.rs @@ -0,0 +1,246 @@ +//! Engine API types: and following the execution specs + +#![allow(missing_docs)] + +use reth_primitives::{ + Address, Block, Bloom, Bytes, SealedBlock, Withdrawal, H256, H64, U256, U64, +}; +use reth_rlp::Encodable; +use serde::{Deserialize, Serialize}; + +/// The list of supported Engine capabilities +pub const CAPABILITIES: [&str; 9] = [ + "engine_forkchoiceUpdatedV1", + "engine_forkchoiceUpdatedV2", + "engine_exchangeTransitionConfigurationV1", + "engine_getPayloadV1", + "engine_getPayloadV2", + "engine_newPayloadV1", + "engine_newPayloadV2", + "engine_getPayloadBodiesByHashV1", + "engine_getPayloadBodiesByRangeV1", +]; + +/// This structure maps on the ExecutionPayload structure of the beacon chain spec. +/// +/// See also: +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecutionPayload { + pub parent_hash: H256, + pub fee_recipient: Address, + pub state_root: H256, + pub receipts_root: H256, + pub logs_bloom: Bloom, + pub prev_randao: H256, + pub block_number: U64, + pub gas_limit: U64, + pub gas_used: U64, + pub timestamp: U64, + pub extra_data: Bytes, + pub base_fee_per_gas: U256, + pub block_hash: H256, + pub transactions: Vec, + /// Array of [`Withdrawal`] enabled with V2 + /// See + #[serde(default, skip_serializing_if = "Option::is_none")] + pub withdrawals: Option>, +} + +impl From for ExecutionPayload { + fn from(value: SealedBlock) -> Self { + let transactions = value + .body + .iter() + .map(|tx| { + let mut encoded = Vec::new(); + tx.encode(&mut encoded); + encoded.into() + }) + .collect(); + ExecutionPayload { + parent_hash: value.parent_hash, + fee_recipient: value.beneficiary, + state_root: value.state_root, + receipts_root: value.receipts_root, + logs_bloom: value.logs_bloom, + prev_randao: value.mix_hash, + block_number: value.number.into(), + gas_limit: value.gas_limit.into(), + gas_used: value.gas_used.into(), + timestamp: value.timestamp.into(), + extra_data: value.extra_data.clone(), + base_fee_per_gas: U256::from(value.base_fee_per_gas.unwrap_or_default()), + block_hash: value.hash(), + transactions, + withdrawals: value.withdrawals, + } + } +} + +/// This structure contains a body of an execution payload. +/// +/// See also: +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExecutionPayloadBody { + pub transactions: Vec, + pub withdrawals: Vec, +} + +impl From for ExecutionPayloadBody { + fn from(value: Block) -> Self { + let transactions = value.body.into_iter().map(|tx| { + let mut out = Vec::new(); + tx.encode(&mut out); + out.into() + }); + ExecutionPayloadBody { + transactions: transactions.collect(), + withdrawals: value.withdrawals.unwrap_or_default(), + } + } +} + +/// The execution payload body response that allows for `null` values. +pub type ExecutionPayloadBodies = Vec>; + +/// This structure encapsulates the fork choice state +#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ForkchoiceState { + pub head_block_hash: H256, + pub safe_block_hash: H256, + pub finalized_block_hash: H256, +} + +/// This structure contains the attributes required to initiate a payload build process in the +/// context of an `engine_forkchoiceUpdated` call. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PayloadAttributes { + pub timestamp: U64, + pub prev_randao: H256, + pub suggested_fee_recipient: Address, + /// Array of [`Withdrawal`] enabled with V2 + /// See + #[serde(default, skip_serializing_if = "Option::is_none")] + pub withdrawals: Option>, + + #[cfg(feature = "optimism")] + /// Transactions is a field for rollups: the transactions list is forced into the block + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transactions: Option>, + + #[cfg(feature = "optimism")] + /// If true, the no transactions are taken out of the tx-pool, only transactions from the above + /// Transactions list will be included. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub no_tx_pool: Option, + + #[cfg(feature = "optimism")] + /// If set, this sets the exact gas limit the block produced with. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gas_limit: Option, +} + +/// This structure contains the result of processing a payload +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PayloadStatus { + #[serde(flatten)] + pub status: PayloadStatusEnum, + /// Hash of the most recent valid block in the branch defined by payload and its ancestors + pub latest_valid_hash: Option, +} + +impl PayloadStatus { + pub fn new(status: PayloadStatusEnum, latest_valid_hash: H256) -> Self { + Self { status, latest_valid_hash: Some(latest_valid_hash) } + } + + pub fn from_status(status: PayloadStatusEnum) -> Self { + Self { status, latest_valid_hash: None } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum PayloadStatusEnum { + Valid, + Invalid { + #[serde(rename = "validationError")] + validation_error: String, + }, + Syncing, + Accepted, + InvalidBlockHash { + #[serde(rename = "validationError")] + validation_error: String, + }, +} + +/// This structure contains configurable settings of the transition process. +#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransitionConfiguration { + /// Maps on the TERMINAL_TOTAL_DIFFICULTY parameter of EIP-3675 + pub terminal_total_difficulty: U256, + /// Maps on TERMINAL_BLOCK_HASH parameter of EIP-3675 + pub terminal_block_hash: H256, + /// Maps on TERMINAL_BLOCK_NUMBER parameter of EIP-3675 + pub terminal_block_number: U64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ForkchoiceUpdated { + pub payload_status: PayloadStatus, + pub payload_id: Option, +} + +impl ForkchoiceUpdated { + pub fn new(payload_status: PayloadStatus) -> Self { + Self { payload_status, payload_id: None } + } + + pub fn from_status(status: PayloadStatusEnum) -> Self { + Self { payload_status: PayloadStatus::from_status(status), payload_id: None } + } + + pub fn with_latest_valid_hash(mut self, hash: H256) -> Self { + self.payload_status.latest_valid_hash = Some(hash); + self + } + + pub fn with_payload_id(mut self, id: H64) -> Self { + self.payload_id = Some(id); + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_interfaces::test_utils::generators::random_block_range; + use reth_primitives::{TransactionSigned, H256}; + use reth_rlp::Decodable; + + #[test] + fn payload_body_roundtrip() { + for block in random_block_range(0..100, H256::default(), 0..2) { + let unsealed = block.clone().unseal(); + let payload_body: ExecutionPayloadBody = unsealed.into(); + + assert_eq!( + Ok(block.body), + payload_body + .transactions + .iter() + .map(|x| TransactionSigned::decode(&mut &x[..])) + .collect::, _>>(), + ); + + assert_eq!(block.withdrawals.unwrap_or_default(), payload_body.withdrawals); + } + } +} From 8b8db55f6a47dbda7de2a1dcd73b416dbf7618a3 Mon Sep 17 00:00:00 2001 From: Andreas Bigger Date: Sun, 26 Feb 2023 18:39:58 -0700 Subject: [PATCH 004/150] small fixes --- crates/primitives/src/transaction/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 66618ba0946..68031c8ecd7 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -612,6 +612,17 @@ impl Transaction { /// Encodes only the transaction's fields into the desired buffer, without a RLP header. pub(crate) fn encode_fields(&self, out: &mut dyn bytes::BufMut) { match self { + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { + tx_env.gas_limit = *gas_limit; + tx_env.gas_price = U256::from(*mint); + tx_env.gas_priority_fee = None; + tx_env.transact_to = TransactTo::Call(*to); + tx_env.value = U256::from(*value); + tx_env.data = input.0.clone(); + tx_env.chain_id = None; + tx_env.nonce = None; + } Transaction::Legacy(TxLegacy { chain_id: _, nonce, From 64b08682c7764180f2ceffc73104b85774f345ad Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 26 Feb 2023 20:36:42 -0500 Subject: [PATCH 005/150] Small changes to the deposit tx primitive --- crates/primitives/src/transaction/mod.rs | 50 +++++++++++++++++-- crates/primitives/src/transaction/optimism.rs | 2 +- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 68031c8ecd7..f1d13dcae72 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -11,7 +11,7 @@ use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, }; -use serde::{Deserialize, Serialize}; +use revm_primitives::U256; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; @@ -380,7 +380,8 @@ impl Transaction { Transaction::Eip2930(TxEip2930 { nonce, .. }) => *nonce, Transaction::Eip1559(TxEip1559 { nonce, .. }) => *nonce, #[cfg(feature = "optimism")] - Transaction::Deposit(_) => todo!(), // TODO: + // Deposit transactions don't have a nonce, so they default to zero. + Transaction::Deposit(_) => 0, } } @@ -511,6 +512,36 @@ impl Transaction { } } + #[cfg(feature = "optimism")] + /// Returns the source hash of the transaction, which uniquely identifies its source. + /// If the transaction is not a deposit transaction, this will always return `H256::zero()`. + pub fn source_hash(&self) -> H256 { + match self { + Transaction::Deposit(TxDeposit { source_hash, .. }) => *source_hash, + _ => H256::zero(), + } + } + + #[cfg(feature = "optimism")] + /// Returns the amount of ETH locked up on L1 that will be minted on L2. If the transaction + /// is not a deposit transaction, this will always return `None`. + pub fn mint(&self) -> Option { + match self { + Transaction::Deposit(TxDeposit { mint, .. }) => *mint, + _ => None, + } + } + + #[cfg(feature = "optimism")] + /// Returns whether or not the transaction is a system transaction. If the transaction + /// is not a deposit transaction, this will always return `false`. + pub fn is_system_transaction(&self) -> bool { + match self { + Transaction::Deposit(TxDeposit { is_system_transaction, .. }) => *is_system_transaction, + _ => false, + } + } + /// Encodes EIP-155 arguments into the desired buffer. Only encodes values for legacy /// transactions. pub(crate) fn encode_eip155_fields(&self, out: &mut dyn bytes::BufMut) { @@ -1061,7 +1092,16 @@ impl TransactionSigned { let header = Header { list: true, payload_length }; header.encode(out); self.transaction.encode_fields(out); - self.signature.encode(out); + // Deposit transactions do not have a signature. If the signature's values are not + // zero, then the transaction is invalid. + if self.signature().v(self.chain_id()) != 0 || + self.signature().r != U256::ZERO || + self.signature().s != U256::ZERO + { + // TODO: Ensure that this transaction may never have a non-zero signature + // higher up - we shouldn't be panicking here. + panic!("Deposit transactions must have a zero signature"); + } } _ => { let payload_length = self.transaction.fields_len() + self.signature.payload_len(); @@ -1165,6 +1205,8 @@ impl TransactionSigned { data.advance(1); #[cfg(feature = "optimism")] + // If the transaction is a deposit, we need to first ensure that the version + // byte is correct. if tx_type == DEPOSIT_TX_TYPE { let version = *data.first().ok_or(DecodeError::InputTooShort)?; if version != DEPOSIT_VERSION { @@ -1182,6 +1224,8 @@ impl TransactionSigned { // length of tx encoding = tx type byte (size = 1) + length of header + payload length let tx_length = 1 + header.length() + header.payload_length; #[cfg(feature = "optimism")] + // If the transaction is a deposit, we need to add one to the length to account for the + // version byte. let tx_length = if tx_type == DEPOSIT_TX_TYPE { tx_length + 1 } else { tx_length }; // decode common fields diff --git a/crates/primitives/src/transaction/optimism.rs b/crates/primitives/src/transaction/optimism.rs index 1b48f43fd23..31c272555fc 100644 --- a/crates/primitives/src/transaction/optimism.rs +++ b/crates/primitives/src/transaction/optimism.rs @@ -43,7 +43,7 @@ impl TxDeposit { len += self.source_hash.length(); len += self.from.length(); len += self.to.length(); - len += self.mint.map(|mint| mint.length()).unwrap_or(1); + len += self.mint.map_or(1, |mint| mint.length()); len += self.value.length(); len += self.input.0.length(); len += self.gas_limit.length(); From 09706b0e87701cd0293c92701dd6777e93e66ea3 Mon Sep 17 00:00:00 2001 From: Andreas Bigger Date: Sun, 26 Feb 2023 19:39:42 -0700 Subject: [PATCH 006/150] more small fixes --- crates/primitives/src/transaction/mod.rs | 14 ++------ crates/revm/revm-primitives/Cargo.toml | 7 ++-- crates/revm/revm-primitives/src/env.rs | 14 ++++++++ crates/transaction-pool/Cargo.toml | 1 + .../transaction-pool/src/test_utils/mock.rs | 34 +++++++++++++++++++ crates/transaction-pool/src/traits.rs | 8 +++-- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index f1d13dcae72..08cec286ef9 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -11,6 +11,7 @@ use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, }; +#[cfg(feature = "optimism")] use revm_primitives::U256; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; @@ -643,17 +644,6 @@ impl Transaction { /// Encodes only the transaction's fields into the desired buffer, without a RLP header. pub(crate) fn encode_fields(&self, out: &mut dyn bytes::BufMut) { match self { - #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { - tx_env.gas_limit = *gas_limit; - tx_env.gas_price = U256::from(*mint); - tx_env.gas_priority_fee = None; - tx_env.transact_to = TransactTo::Call(*to); - tx_env.value = U256::from(*value); - tx_env.data = input.0.clone(); - tx_env.chain_id = None; - tx_env.nonce = None; - } Transaction::Legacy(TxLegacy { chain_id: _, nonce, @@ -1025,7 +1015,7 @@ impl TransactionSigned { pub fn recover_signer(&self) -> Option
{ #[cfg(feature = "optimism")] if let Transaction::Deposit(TxDeposit { from, .. }) = self.transaction { - return Some(from.clone()) + return Some(from) } let signature_hash = self.signature_hash(); self.signature.recover_signer(signature_hash) diff --git a/crates/revm/revm-primitives/Cargo.toml b/crates/revm/revm-primitives/Cargo.toml index 96efb46ff86..8c00dc3670a 100644 --- a/crates/revm/revm-primitives/Cargo.toml +++ b/crates/revm/revm-primitives/Cargo.toml @@ -9,7 +9,8 @@ repository.workspace = true description = "core reth specific revm utilities" [dependencies] -# reth -reth-primitives = { workspace = true } +reth-primitives = { path = "../../primitives" } +revm = { version = "3.0.0" } -revm = { workspace = true } +[features] +optimism = [] diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index 4a3ee323928..27ffd532478 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -5,6 +5,9 @@ use reth_primitives::{ }; use revm::primitives::{AnalysisKind, BlockEnv, CfgEnv, SpecId, TransactTo, TxEnv}; +#[cfg(feature = "optimism")] +use reth_primitives::TxDeposit; + /// Convenience function to call both [fill_cfg_env] and [fill_block_env] pub fn fill_cfg_and_block_env( cfg: &mut CfgEnv, @@ -119,6 +122,17 @@ where { tx_env.caller = sender; match transaction.as_ref() { + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { + tx_env.gas_limit = *gas_limit; + tx_env.gas_price = U256::from(*mint); + tx_env.gas_priority_fee = None; + tx_env.transact_to = TransactTo::Call(*to); + tx_env.value = U256::from(*value); + tx_env.data = input.0.clone(); + tx_env.chain_id = None; + tx_env.nonce = None; + } Transaction::Legacy(TxLegacy { nonce, chain_id, diff --git a/crates/transaction-pool/Cargo.toml b/crates/transaction-pool/Cargo.toml index c562df71cf5..47f44068627 100644 --- a/crates/transaction-pool/Cargo.toml +++ b/crates/transaction-pool/Cargo.toml @@ -53,3 +53,4 @@ rand = "0.8" default = ["serde"] serde = ["dep:serde"] test-utils = ["rand", "paste", "serde"] +optimism = [] \ No newline at end of file diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 13ec9b5af01..9189391a815 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -22,6 +22,9 @@ pub(crate) type MockTxPool = TxPool; pub type MockValidTx = ValidPoolTransaction; +#[cfg(feature = "optimism")] +use reth_primitives::TxDeposit; + /// Create an empty `TxPool` pub(crate) fn mock_tx_pool() -> MockTxPool { MockTxPool::new(Default::default(), Default::default()) @@ -99,6 +102,17 @@ pub enum MockTransaction { to: TransactionKind, value: U256, }, + #[cfg(feature = "optimism")] + DepositTx { + source_hash: H256, + from: Address, + to: TransactionKind, + mint: Option, + value: u128, + gas_limit: u64, + is_system_transaction: bool, + input: Bytes, + }, } // === impl MockTransaction === @@ -393,6 +407,26 @@ impl FromRecoveredTransaction for MockTransaction { let transaction = tx.into_signed(); let hash = transaction.hash(); match transaction.transaction { + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { + source_hash, + from, + to, + mint, + value, + gas_limit, + is_system_transaction, + input, + }) => MockTransaction::DepositTx { + source_hash, + from, + to, + mint, + value, + gas_limit, + is_system_transaction, + input, + }, Transaction::Legacy(TxLegacy { chain_id, nonce, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index afbfb47a349..c04a2cfe7bf 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -523,9 +523,11 @@ impl PoolTransaction for PooledTransaction { /// This is also commonly referred to as the "Gas Fee Cap" (`GasFeeCap`). fn max_fee_per_gas(&self) -> u128 { match &self.transaction.transaction { - Transaction::Legacy(tx) => tx.gas_price, - Transaction::Eip2930(tx) => tx.gas_price, - Transaction::Eip1559(tx) => tx.max_fee_per_gas, + #[cfg(feature = "optimism")] + Transaction::Optimism(_) => None, + Transaction::Legacy(_) => None, + Transaction::Eip2930(_) => None, + Transaction::Eip1559(tx) => Some(tx.max_fee_per_gas), } } From 9c78320f2d78a810060101968839fee7e67f8a90 Mon Sep 17 00:00:00 2001 From: Andreas Bigger Date: Sun, 26 Feb 2023 22:26:58 -0700 Subject: [PATCH 007/150] compiles --- crates/consensus/Cargo.toml | 24 +++++++++ crates/consensus/common/src/validation.rs | 10 ++++ crates/primitives/src/hardfork.rs | 8 +-- crates/revm/revm-primitives/src/env.rs | 11 +++- crates/rpc/rpc-types/src/eth/engine.rs | 2 +- .../rpc/rpc-types/src/eth/transaction/mod.rs | 18 +++---- .../transaction-pool/src/test_utils/mock.rs | 52 ++++++++++++++++--- crates/transaction-pool/src/traits.rs | 33 ++++++++++-- 8 files changed, 126 insertions(+), 32 deletions(-) create mode 100644 crates/consensus/Cargo.toml diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml new file mode 100644 index 00000000000..bd742d0bed0 --- /dev/null +++ b/crates/consensus/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "reth-consensus" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/paradigmxyz/reth" +readme = "README.md" + +[dependencies] +# reth +reth-primitives = { path = "../primitives" } +reth-interfaces = { path = "../interfaces" } +reth-provider = { path = "../storage/provider" } + +# async +tokio = { version = "1", features = ["sync"] } + +[dev-dependencies] +reth-interfaces = { path = "../interfaces", features = ["test-utils"] } +reth-provider = { path = "../storage/provider", features = ["test-utils"] } +assert_matches = "1.5.0" + +[features] +optimism = [] \ No newline at end of file diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index f61e160f039..d54e33e80d4 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -10,6 +10,11 @@ use std::{ time::SystemTime, }; +use reth_primitives::constants; + +#[cfg(feature = "optimism")] +use reth_primitives::TxDeposit; + /// Validate header standalone pub fn validate_header_standalone( header: &SealedHeader, @@ -64,6 +69,11 @@ pub fn validate_transaction_regarding_header( base_fee: Option, ) -> Result<(), ConsensusError> { let chain_id = match transaction { + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { .. }) => { + // TODO: get the chain id + None + } Transaction::Legacy(TxLegacy { chain_id, .. }) => { // EIP-155: Simple replay attack protection: https://eips.ethereum.org/EIPS/eip-155 if chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(at_block_number) && diff --git a/crates/primitives/src/hardfork.rs b/crates/primitives/src/hardfork.rs index a2f0f7be399..cc5d384faeb 100644 --- a/crates/primitives/src/hardfork.rs +++ b/crates/primitives/src/hardfork.rs @@ -180,8 +180,8 @@ mod tests { genesis: Genesis::default(), genesis_hash: None, hardforks: BTreeMap::from([(Hardfork::Frontier, ForkCondition::Never)]), - fork_timestamps: Default::default(), - paris_block_and_final_difficulty: None, + #[cfg(feature = "optimism")] + optimism: None, }; assert_eq!(Hardfork::Frontier.fork_id(&spec), None); @@ -194,8 +194,8 @@ mod tests { genesis: Genesis::default(), genesis_hash: None, hardforks: BTreeMap::from([(Hardfork::Shanghai, ForkCondition::Never)]), - fork_timestamps: Default::default(), - paris_block_and_final_difficulty: None, + #[cfg(feature = "optimism")] + optimism: None, }; assert_eq!(Hardfork::Shanghai.fork_filter(&spec), None); diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index 27ffd532478..5351914c1e8 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -125,9 +125,16 @@ where #[cfg(feature = "optimism")] Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { tx_env.gas_limit = *gas_limit; - tx_env.gas_price = U256::from(*mint); + if let Some(m) = mint { + tx_env.gas_price = U256::from(*m); + } else { + tx_env.gas_price = U256::ZERO; + } tx_env.gas_priority_fee = None; - tx_env.transact_to = TransactTo::Call(*to); + match to { + TransactionKind::Call(to) => tx_env.transact_to = TransactTo::Call(*to), + TransactionKind::Create => tx_env.transact_to = TransactTo::create(), + } tx_env.value = U256::from(*value); tx_env.data = input.0.clone(); tx_env.chain_id = None; diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs index 22c736dc1ab..e8be77f3adc 100644 --- a/crates/rpc/rpc-types/src/eth/engine.rs +++ b/crates/rpc/rpc-types/src/eth/engine.rs @@ -140,7 +140,7 @@ pub struct PayloadAttributes { #[cfg(feature = "optimism")] /// If set, this sets the exact gas limit the block produced with. #[serde(default, skip_serializing_if = "Option::is_none")] - pub gas_limit: Option, + pub gas_limit: Option, } /// This structure contains the result of processing a payload diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index c61d6080b9b..5056a3ba341 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -111,22 +111,16 @@ impl Transaction { let (gas_price, max_fee_per_gas) = match signed_tx.tx_type() { TxType::Legacy => (Some(U128::from(signed_tx.max_fee_per_gas())), None), - TxType::EIP2930 => (Some(U128::from(signed_tx.max_fee_per_gas())), None), - TxType::EIP1559 => { - // the gas price field for EIP1559 is set to `min(tip, gasFeeCap - baseFee) + - // baseFee` - let gas_price = base_fee - .and_then(|base_fee| { - signed_tx.effective_tip_per_gas(base_fee).map(|tip| tip + base_fee as u128) - }) - .unwrap_or_else(|| signed_tx.max_fee_per_gas()); - - (Some(U128::from(gas_price)), Some(U128::from(signed_tx.max_fee_per_gas()))) - } + TxType::EIP2930 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), + TxType::EIP1559 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), + #[cfg(feature = "optimism")] + TxType::DEPOSIT => (None, None), }; let chain_id = signed_tx.chain_id().map(U64::from); let access_list = match &signed_tx.transaction { + #[cfg(feature = "optimism")] + PrimitiveTransaction::Deposit(_) => None, PrimitiveTransaction::Legacy(_) => None, PrimitiveTransaction::Eip2930(tx) => Some( tx.access_list diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 9189391a815..8531d21ab99 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -22,9 +22,15 @@ pub(crate) type MockTxPool = TxPool; pub type MockValidTx = ValidPoolTransaction; +#[cfg(feature = "optimism")] +use reth_primitives::DEPOSIT_TX_TYPE; + #[cfg(feature = "optimism")] use reth_primitives::TxDeposit; +#[cfg(feature = "optimism")] +use reth_primitives::Bytes; + /// Create an empty `TxPool` pub(crate) fn mock_tx_pool() -> MockTxPool { MockTxPool::new(Default::default(), Default::default()) @@ -35,6 +41,10 @@ macro_rules! set_value { ($this:ident => $field:ident) => { let new_value = $field; match $this { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { ref mut $field, .. } => { + *$field = new_value; + } MockTransaction::Legacy { ref mut $field, .. } => { *$field = new_value; } @@ -49,6 +59,8 @@ macro_rules! set_value { macro_rules! get_value { ($this:ident => $field:ident) => { match $this { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { $field, .. } => $field, MockTransaction::Legacy { $field, .. } => $field, MockTransaction::Eip1559 { $field, .. } => $field, } @@ -104,14 +116,15 @@ pub enum MockTransaction { }, #[cfg(feature = "optimism")] DepositTx { - source_hash: H256, - from: Address, + hash: H256, + sender: Address, + nonce: u64, to: TransactionKind, mint: Option, - value: u128, gas_limit: u64, is_system_transaction: bool, input: Bytes, + value: U256, }, } @@ -199,6 +212,8 @@ impl MockTransaction { pub fn set_gas_price(&mut self, val: u128) -> &mut Self { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => {} MockTransaction::Legacy { gas_price, .. } => { *gas_price = val; } @@ -212,6 +227,8 @@ impl MockTransaction { pub fn with_gas_price(mut self, val: u128) -> Self { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => {} MockTransaction::Legacy { ref mut gas_price, .. } => { *gas_price = val; } @@ -229,6 +246,8 @@ impl MockTransaction { pub fn get_gas_price(&self) -> u128 { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => 0, MockTransaction::Legacy { gas_price, .. } => *gas_price, MockTransaction::Eip1559 { max_fee_per_gas, .. } => *max_fee_per_gas, } @@ -313,6 +332,8 @@ impl MockTransaction { impl PoolTransaction for MockTransaction { fn hash(&self) -> &TxHash { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { hash, .. } => hash, MockTransaction::Legacy { hash, .. } => hash, MockTransaction::Eip1559 { hash, .. } => hash, } @@ -320,6 +341,8 @@ impl PoolTransaction for MockTransaction { fn sender(&self) -> Address { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { sender, .. } => *sender, MockTransaction::Legacy { sender, .. } => *sender, MockTransaction::Eip1559 { sender, .. } => *sender, } @@ -327,6 +350,8 @@ impl PoolTransaction for MockTransaction { fn nonce(&self) -> u64 { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { nonce, .. } => *nonce, MockTransaction::Legacy { nonce, .. } => *nonce, MockTransaction::Eip1559 { nonce, .. } => *nonce, } @@ -334,6 +359,8 @@ impl PoolTransaction for MockTransaction { fn cost(&self) -> U256 { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => U256::from(0), MockTransaction::Legacy { gas_price, value, gas_limit, .. } => { U256::from(*gas_limit) * U256::from(*gas_price) + *value } @@ -360,13 +387,17 @@ impl PoolTransaction for MockTransaction { fn max_fee_per_gas(&self) -> u128 { match self { - MockTransaction::Legacy { gas_price, .. } => *gas_price, - MockTransaction::Eip1559 { max_fee_per_gas, .. } => *max_fee_per_gas, + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => None, + MockTransaction::Legacy { .. } => None, + MockTransaction::Eip1559 { max_fee_per_gas, .. } => Some(*max_fee_per_gas), } } fn max_priority_fee_per_gas(&self) -> Option { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => None, MockTransaction::Legacy { .. } => None, MockTransaction::Eip1559 { max_priority_fee_per_gas, .. } => { Some(*max_priority_fee_per_gas) @@ -376,6 +407,8 @@ impl PoolTransaction for MockTransaction { fn kind(&self) -> &TransactionKind { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { to, .. } => to, MockTransaction::Legacy { to, .. } => to, MockTransaction::Eip1559 { to, .. } => to, } @@ -387,6 +420,8 @@ impl PoolTransaction for MockTransaction { fn tx_type(&self) -> u8 { match self { + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => DEPOSIT_TX_TYPE, MockTransaction::Legacy { .. } => TxType::Legacy.into(), MockTransaction::Eip1559 { .. } => TxType::EIP1559.into(), } @@ -418,11 +453,12 @@ impl FromRecoveredTransaction for MockTransaction { is_system_transaction, input, }) => MockTransaction::DepositTx { - source_hash, - from, + nonce: 0u64, + hash: source_hash, + sender: from, to, mint, - value, + value: U256::from(value), gas_limit, is_system_transaction, input, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index c04a2cfe7bf..d181966f5e8 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -524,7 +524,7 @@ impl PoolTransaction for PooledTransaction { fn max_fee_per_gas(&self) -> u128 { match &self.transaction.transaction { #[cfg(feature = "optimism")] - Transaction::Optimism(_) => None, + Transaction::Deposit(_) => None, Transaction::Legacy(_) => None, Transaction::Eip2930(_) => None, Transaction::Eip1559(tx) => Some(tx.max_fee_per_gas), @@ -536,6 +536,8 @@ impl PoolTransaction for PooledTransaction { /// This will return `None` for non-EIP1559 transactions fn max_priority_fee_per_gas(&self) -> Option { match &self.transaction.transaction { + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => None, Transaction::Legacy(_) => None, Transaction::Eip2930(_) => None, Transaction::Eip1559(tx) => Some(tx.max_priority_fee_per_gas), @@ -571,10 +573,31 @@ impl PoolTransaction for PooledTransaction { impl FromRecoveredTransaction for PooledTransaction { fn from_recovered_transaction(tx: TransactionSignedEcRecovered) -> Self { - let gas_cost = match &tx.transaction { - Transaction::Legacy(t) => U256::from(t.gas_price) * U256::from(t.gas_limit), - Transaction::Eip2930(t) => U256::from(t.gas_price) * U256::from(t.gas_limit), - Transaction::Eip1559(t) => U256::from(t.max_fee_per_gas) * U256::from(t.gas_limit), + let (cost, effective_gas_price) = match &tx.transaction { + #[cfg(feature = "optimism")] + Transaction::Deposit(t) => { + // TODO: fix this gas price estimate + let gas_price = U256::from(0); + let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); + let effective_gas_price = 0u128; + (cost, effective_gas_price) + } + Transaction::Legacy(t) => { + let cost = U256::from(t.gas_price) * U256::from(t.gas_limit) + U256::from(t.value); + let effective_gas_price = t.gas_price; + (cost, effective_gas_price) + } + Transaction::Eip2930(t) => { + let cost = U256::from(t.gas_price) * U256::from(t.gas_limit) + U256::from(t.value); + let effective_gas_price = t.gas_price; + (cost, effective_gas_price) + } + Transaction::Eip1559(t) => { + let cost = + U256::from(t.max_fee_per_gas) * U256::from(t.gas_limit) + U256::from(t.value); + let effective_gas_price = t.max_priority_fee_per_gas; + (cost, effective_gas_price) + } }; let cost = gas_cost + U256::from(tx.value()); From 8f7641b301c11a22616dca38c7f52eed54626be9 Mon Sep 17 00:00:00 2001 From: clabby Date: Mon, 27 Feb 2023 09:05:01 -0500 Subject: [PATCH 008/150] Move feature flags below comments --- crates/primitives/src/chain/spec.rs | 8 ++++---- crates/primitives/src/hardfork.rs | 4 ++-- crates/primitives/src/transaction/mod.rs | 16 ++++++++-------- crates/primitives/src/transaction/tx_type.rs | 2 +- crates/rpc/rpc-types/src/eth/engine.rs | 6 +++--- crates/transaction-pool/src/test_utils/mock.rs | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 9d493d2a14c..86d1fbbbbee 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -141,14 +141,14 @@ pub struct ChainSpec { /// The active hard forks and their activation conditions pub hardforks: BTreeMap, - #[cfg(feature = "optimism")] /// Optimism configuration + #[cfg(feature = "optimism")] pub optimism: Option, } +/// Optimism configuration. #[cfg(feature = "optimism")] #[derive(Serialize, Deserialize, Debug, Clone)] -/// Optimism configuration. pub struct OptimismConfig { /// Elasticity multiplier as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) pub eip_1559_elasticity: u64, @@ -572,15 +572,15 @@ impl ChainSpecBuilder { self } - #[cfg(feature = "optimism")] /// Enable Bedrock at genesis + #[cfg(feature = "optimism")] pub fn bedrock_activated(mut self) -> Self { self.hardforks.insert(Hardfork::Bedrock, ForkCondition::Block(0)); self } - #[cfg(feature = "optimism")] /// Enable Bedrock at genesis + #[cfg(feature = "optimism")] pub fn regolith_activated(mut self) -> Self { self.hardforks.insert(Hardfork::Regolith, ForkCondition::Timestamp(0)); self diff --git a/crates/primitives/src/hardfork.rs b/crates/primitives/src/hardfork.rs index cc5d384faeb..0c3942ebd9d 100644 --- a/crates/primitives/src/hardfork.rs +++ b/crates/primitives/src/hardfork.rs @@ -39,11 +39,11 @@ pub enum Hardfork { Paris, /// Shanghai. Shanghai, - #[cfg(feature = "optimism")] /// Bedrock. - Bedrock, #[cfg(feature = "optimism")] + Bedrock, /// Regolith + #[cfg(feature = "optimism")] Regolith, } diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 08cec286ef9..56bc10d6043 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -186,8 +186,8 @@ pub enum Transaction { Eip2930(TxEip2930), /// A transaction with a priority fee ([EIP-1559](https://eips.ethereum.org/EIPS/eip-1559)). Eip1559(TxEip1559), - #[cfg(feature = "optimism")] /// Deposit transaction. + #[cfg(feature = "optimism")] Deposit(TxDeposit), } @@ -380,8 +380,8 @@ impl Transaction { Transaction::Legacy(TxLegacy { nonce, .. }) => *nonce, Transaction::Eip2930(TxEip2930 { nonce, .. }) => *nonce, Transaction::Eip1559(TxEip1559 { nonce, .. }) => *nonce, - #[cfg(feature = "optimism")] // Deposit transactions don't have a nonce, so they default to zero. + #[cfg(feature = "optimism")] Transaction::Deposit(_) => 0, } } @@ -405,9 +405,9 @@ impl Transaction { Transaction::Legacy(TxLegacy { gas_price, .. }) | Transaction::Eip2930(TxEip2930 { gas_price, .. }) => *gas_price, Transaction::Eip1559(TxEip1559 { max_fee_per_gas, .. }) => *max_fee_per_gas, - #[cfg(feature = "optimism")] // Deposit transactions buy their L2 gas on L1 and, as such, the L2 gas is not // refundable. + #[cfg(feature = "optimism")] Transaction::Deposit(_) => 0, } } @@ -513,9 +513,9 @@ impl Transaction { } } - #[cfg(feature = "optimism")] /// Returns the source hash of the transaction, which uniquely identifies its source. /// If the transaction is not a deposit transaction, this will always return `H256::zero()`. + #[cfg(feature = "optimism")] pub fn source_hash(&self) -> H256 { match self { Transaction::Deposit(TxDeposit { source_hash, .. }) => *source_hash, @@ -523,9 +523,9 @@ impl Transaction { } } - #[cfg(feature = "optimism")] /// Returns the amount of ETH locked up on L1 that will be minted on L2. If the transaction /// is not a deposit transaction, this will always return `None`. + #[cfg(feature = "optimism")] pub fn mint(&self) -> Option { match self { Transaction::Deposit(TxDeposit { mint, .. }) => *mint, @@ -533,9 +533,9 @@ impl Transaction { } } - #[cfg(feature = "optimism")] /// Returns whether or not the transaction is a system transaction. If the transaction /// is not a deposit transaction, this will always return `false`. + #[cfg(feature = "optimism")] pub fn is_system_transaction(&self) -> bool { match self { Transaction::Deposit(TxDeposit { is_system_transaction, .. }) => *is_system_transaction, @@ -1194,9 +1194,9 @@ impl TransactionSigned { let tx_type = *data.first().ok_or(DecodeError::InputTooShort)?; data.advance(1); - #[cfg(feature = "optimism")] // If the transaction is a deposit, we need to first ensure that the version // byte is correct. + #[cfg(feature = "optimism")] if tx_type == DEPOSIT_TX_TYPE { let version = *data.first().ok_or(DecodeError::InputTooShort)?; if version != DEPOSIT_VERSION { @@ -1213,9 +1213,9 @@ impl TransactionSigned { // length of tx encoding = tx type byte (size = 1) + length of header + payload length let tx_length = 1 + header.length() + header.payload_length; - #[cfg(feature = "optimism")] // If the transaction is a deposit, we need to add one to the length to account for the // version byte. + #[cfg(feature = "optimism")] let tx_length = if tx_type == DEPOSIT_TX_TYPE { tx_length + 1 } else { tx_length }; // decode common fields diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index 1e7c9edfae5..39fa1710737 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -23,8 +23,8 @@ pub enum TxType { EIP2930 = 1_isize, /// Transaction with Priority fee EIP1559 = 2_isize, - #[cfg(feature = "optimism")] /// OP Deposit transaction. + #[cfg(feature = "optimism")] DEPOSIT = 126_isize, } diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs index e8be77f3adc..548ed16645b 100644 --- a/crates/rpc/rpc-types/src/eth/engine.rs +++ b/crates/rpc/rpc-types/src/eth/engine.rs @@ -126,19 +126,19 @@ pub struct PayloadAttributes { #[serde(default, skip_serializing_if = "Option::is_none")] pub withdrawals: Option>, - #[cfg(feature = "optimism")] /// Transactions is a field for rollups: the transactions list is forced into the block + #[cfg(feature = "optimism")] #[serde(default, skip_serializing_if = "Option::is_none")] pub transactions: Option>, - #[cfg(feature = "optimism")] /// If true, the no transactions are taken out of the tx-pool, only transactions from the above /// Transactions list will be included. + #[cfg(feature = "optimism")] #[serde(default, skip_serializing_if = "Option::is_none")] pub no_tx_pool: Option, - #[cfg(feature = "optimism")] /// If set, this sets the exact gas limit the block produced with. + #[cfg(feature = "optimism")] #[serde(default, skip_serializing_if = "Option::is_none")] pub gas_limit: Option, } diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 8531d21ab99..62cc70afc46 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -360,7 +360,7 @@ impl PoolTransaction for MockTransaction { fn cost(&self) -> U256 { match self { #[cfg(feature = "optimism")] - MockTransaction::DepositTx { .. } => U256::from(0), + MockTransaction::DepositTx { .. } => U256::ZERO, MockTransaction::Legacy { gas_price, value, gas_limit, .. } => { U256::from(*gas_limit) * U256::from(*gas_price) + *value } From 815f32b6c17482748fbd252d3016f11743232bce Mon Sep 17 00:00:00 2001 From: clabby Date: Mon, 27 Feb 2023 09:14:05 -0500 Subject: [PATCH 009/150] Doc auto link --- crates/primitives/src/transaction/optimism.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/primitives/src/transaction/optimism.rs b/crates/primitives/src/transaction/optimism.rs index 31c272555fc..faad6908d48 100644 --- a/crates/primitives/src/transaction/optimism.rs +++ b/crates/primitives/src/transaction/optimism.rs @@ -52,7 +52,7 @@ impl TxDeposit { } /// Encodes only the transaction's fields into the desired buffer, without a RLP header. - /// https://github.com/ethereum-optimism/optimism/blob/develop/specs/deposits.md#the-deposited-transaction-type + /// pub(crate) fn encode_fields(&self, out: &mut dyn bytes::BufMut) { self.source_hash.encode(out); self.from.encode(out); From 00b997a45531a7339d417f74a90181b86028c9ac Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 5 Mar 2023 13:08:35 -0700 Subject: [PATCH 010/150] WIP: OP Goerli genesis --- crates/primitives/res/genesis/goerli_op.json | 92 +++++++++++++++ crates/primitives/src/chain/mod.rs | 15 +++ crates/primitives/src/chain/spec.rs | 50 +++++++- crates/primitives/src/constants.rs | 5 + crates/primitives/src/lib.rs | 2 + crates/rpc/rpc-engine-api/Cargo.toml | 3 + crates/rpc/rpc-engine-api/src/engine_api.rs | 113 +++++++++++++++++++ crates/rpc/rpc-engine-api/src/error.rs | 49 +------- 8 files changed, 283 insertions(+), 46 deletions(-) create mode 100644 crates/primitives/res/genesis/goerli_op.json diff --git a/crates/primitives/res/genesis/goerli_op.json b/crates/primitives/res/genesis/goerli_op.json new file mode 100644 index 00000000000..0ea4638d942 --- /dev/null +++ b/crates/primitives/res/genesis/goerli_op.json @@ -0,0 +1,92 @@ +{ + "nonce": "0x00", + "timestamp": "0x00", + "extraData": "0x000000000000000000000000000000000000000000000000000000000000000027770a9694e4B4b1E130Ab91Bc327C36855f612E0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x01", + "gasLimit": "15000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x00", + "number": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "stateRoot": "0x9e6b478a1cd331a979c39e4bddf42c676bcf5a63382f898dc441fe3fe5eb0837", + "alloc": { + "0x4200000000000000000000000000000000000000": { + "balance": "00", + "storage": {}, + "code": "0x608060405234801561001057600080fd5b50600436106100365760003560e01c806382e3702d1461003b578063cafa81dc14610072575b600080fd5b61005e610049366004610112565b60006020819052908152604090205460ff1681565b604051901515815260200160405180910390f35b61008561008036600461015a565b610087565b005b6001600080833360405160200161009f929190610229565b604080518083037fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe001815291815281516020928301208352908201929092520160002080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff001691151591909117905550565b60006020828403121561012457600080fd5b5035919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b60006020828403121561016c57600080fd5b813567ffffffffffffffff8082111561018457600080fd5b818401915084601f83011261019857600080fd5b8135818111156101aa576101aa61012b565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f011681019083821181831017156101f0576101f061012b565b8160405282815287602084870101111561020957600080fd5b826020860160208301376000928101602001929092525095945050505050565b6000835160005b8181101561024a5760208187018101518583015201610230565b81811115610259576000828501525b5060609390931b7fffffffffffffffffffffffffffffffffffffffff00000000000000000000000016919092019081526014019291505056fea26469706673582212209ffc0b44ce8a27c46cae74a3b3b620a72f10aaea97ed37c15b5d36792abd2aa464736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000002": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100725760003560e01c80639b19251a116100505780639b19251a146100e9578063b1540a011461011c578063bdc7b54f1461012f57600080fd5b806308fd63221461007757806313af40351461008c5780638da5cb5b1461009f575b600080fd5b61008a610085366004610614565b610137565b005b61008a61009a366004610650565b610271565b6000546100bf9073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff90911681526020015b60405180910390f35b61010c6100f7366004610650565b60016020526000908152604090205460ff1681565b60405190151581526020016100e0565b61010c61012a366004610650565b61047c565b61008a6104cd565b60005473ffffffffffffffffffffffffffffffffffffffff1633146101e3576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603a60248201527f46756e6374696f6e2063616e206f6e6c792062652063616c6c6564206279207460448201527f6865206f776e6572206f66207468697320636f6e74726163742e00000000000060648201526084015b60405180910390fd5b73ffffffffffffffffffffffffffffffffffffffff821660008181526001602090815260409182902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00168515159081179091558251938452908301527f8daaf060c3306c38e068a75c054bf96ecd85a3db1252712c4d93632744c42e0d910160405180910390a15050565b60005473ffffffffffffffffffffffffffffffffffffffff163314610318576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603a60248201527f46756e6374696f6e2063616e206f6e6c792062652063616c6c6564206279207460448201527f6865206f776e6572206f66207468697320636f6e74726163742e00000000000060648201526084016101da565b73ffffffffffffffffffffffffffffffffffffffff81166103e1576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152605160248201527f4f564d5f4465706c6f79657257686974656c6973743a2063616e206f6e6c792060448201527f62652064697361626c65642076696120656e61626c654172626974726172794360648201527f6f6e74726163744465706c6f796d656e74000000000000000000000000000000608482015260a4016101da565b6000546040805173ffffffffffffffffffffffffffffffffffffffff928316815291831660208301527fb532073b38c83145e3e5135377a08bf9aab55bc0fd7c1179cd4fb995d2a5159c910160405180910390a1600080547fffffffffffffffffffffffff00000000000000000000000000000000000000001673ffffffffffffffffffffffffffffffffffffffff92909216919091179055565b6000805473ffffffffffffffffffffffffffffffffffffffff1615806104c7575073ffffffffffffffffffffffffffffffffffffffff821660009081526001602052604090205460ff165b92915050565b60005473ffffffffffffffffffffffffffffffffffffffff163314610574576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603a60248201527f46756e6374696f6e2063616e206f6e6c792062652063616c6c6564206279207460448201527f6865206f776e6572206f66207468697320636f6e74726163742e00000000000060648201526084016101da565b60005460405173ffffffffffffffffffffffffffffffffffffffff90911681527fc0e106cf568e50698fdbde1eff56f5a5c966cc7958e37e276918e9e4ccdf8cd49060200160405180910390a1600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000169055565b803573ffffffffffffffffffffffffffffffffffffffff8116811461060f57600080fd5b919050565b6000806040838503121561062757600080fd5b610630836105eb565b91506020830135801515811461064557600080fd5b809150509250929050565b60006020828403121561066257600080fd5b61066b826105eb565b939250505056fea264697066735822122045a02b3906eca00a51b37c2965ab13be381f71f60af681951849865fb2daa75f64736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000007": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000004": "0x000000000000000000000000000000000000000000000000000000000000dead", + "0x0000000000000000000000000000000000000000000000000000000000000005": "0x0000000000000000000000005086d1eef304eb5284a0f6720f79403b4e9be294", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x00000000000000000000000000000000000000000000000000000000000186a0" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100885760003560e01c8063a71198691161005b578063a71198691461012a578063b1b1b2091461014a578063cbd4ece91461016d578063ecc704281461018057600080fd5b806321d800ec1461008d5780633dbb202b146100c55780636e296e45146100da57806382e3702d14610107575b600080fd5b6100b061009b366004610826565b60006020819052908152604090205460ff1681565b60405190151581526020015b60405180910390f35b6100d86100d3366004610942565b610197565b005b6100e26102e2565b60405173ffffffffffffffffffffffffffffffffffffffff90911681526020016100bc565b6100b0610115366004610826565b60026020526000908152604090205460ff1681565b6005546100e29073ffffffffffffffffffffffffffffffffffffffff1681565b6100b0610158366004610826565b60016020526000908152604090205460ff1681565b6100d861017b3660046109ad565b61038b565b61018960035481565b6040519081526020016100bc565b60006101a784338560035461078d565b80516020808301919091206000908152600290915260409081902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00166001179055517fcafa81dc0000000000000000000000000000000000000000000000000000000081529091507342000000000000000000000000000000000000009063cafa81dc9061023c908490600401610a89565b600060405180830381600087803b15801561025657600080fd5b505af115801561026a573d6000803e3d6000fd5b505050508373ffffffffffffffffffffffffffffffffffffffff167fcb0f7ffd78f9aee47a248fae8db181db6eee833039123e026dcbff529522e52a3385600354866040516102bc9493929190610aa3565b60405180910390a26001600360008282546102d79190610aef565b909155505050505050565b60045460009073ffffffffffffffffffffffffffffffffffffffff1661dead141561036e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601f60248201527f78446f6d61696e4d65737361676553656e646572206973206e6f74207365740060448201526064015b60405180910390fd5b5060045473ffffffffffffffffffffffffffffffffffffffff1690565b60055473ffffffffffffffffffffffffffffffffffffffff167fffffffffffffffffffffffffeeeeffffffffffffffffffffffffffffffffeeef330173ffffffffffffffffffffffffffffffffffffffff161461046a576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602760248201527f50726f7669646564206d65737361676520636f756c64206e6f7420626520766560448201527f7269666965642e000000000000000000000000000000000000000000000000006064820152608401610365565b60006104788585858561078d565b8051602080830191909120600081815260019092526040909120549192509060ff1615610527576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602b60248201527f50726f7669646564206d6573736167652068617320616c72656164792062656560448201527f6e2072656365697665642e0000000000000000000000000000000000000000006064820152608401610365565b73ffffffffffffffffffffffffffffffffffffffff8616734200000000000000000000000000000000000000141561059957600090815260016020819052604090912080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016909117905550610787565b600480547fffffffffffffffffffffffff00000000000000000000000000000000000000001673ffffffffffffffffffffffffffffffffffffffff878116919091179091556040516000918816906105f2908790610b2e565b6000604051808303816000865af19150503d806000811461062f576040519150601f19603f3d011682016040523d82523d6000602084013e610634565b606091505b5050600480547fffffffffffffffffffffffff00000000000000000000000000000000000000001661dead1790559050801515600114156106d557600082815260016020819052604080832080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00169092179091555183917f4641df4a962071e12719d8c8c8e5ac7fc4d97b927346a3d7a335b1f7517e133c91a2610701565b60405182907f99d0e048484baa1b1540b1367cb128acd7ab2946d1ed91ec10e3c85e4bf51b8f90600090a25b600083334360405160200161071893929190610b4a565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529181528151602092830120600090815291829052902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00166001179055505050505b50505050565b6060848484846040516024016107a69493929190610b9c565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fcbd4ece9000000000000000000000000000000000000000000000000000000001790529050949350505050565b60006020828403121561083857600080fd5b5035919050565b803573ffffffffffffffffffffffffffffffffffffffff8116811461086357600080fd5b919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600082601f8301126108a857600080fd5b813567ffffffffffffffff808211156108c3576108c3610868565b604051601f83017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f0116810190828211818310171561090957610909610868565b8160405283815286602085880101111561092257600080fd5b836020870160208301376000602085830101528094505050505092915050565b60008060006060848603121561095757600080fd5b6109608461083f565b9250602084013567ffffffffffffffff81111561097c57600080fd5b61098886828701610897565b925050604084013563ffffffff811681146109a257600080fd5b809150509250925092565b600080600080608085870312156109c357600080fd5b6109cc8561083f565b93506109da6020860161083f565b9250604085013567ffffffffffffffff8111156109f657600080fd5b610a0287828801610897565b949793965093946060013593505050565b60005b83811015610a2e578181015183820152602001610a16565b838111156107875750506000910152565b60008151808452610a57816020860160208601610a13565b601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169290920160200192915050565b602081526000610a9c6020830184610a3f565b9392505050565b73ffffffffffffffffffffffffffffffffffffffff85168152608060208201526000610ad26080830186610a3f565b905083604083015263ffffffff8316606083015295945050505050565b60008219821115610b29577f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b500190565b60008251610b40818460208701610a13565b9190910192915050565b60008451610b5c818460208901610a13565b60609490941b7fffffffffffffffffffffffffffffffffffffffff0000000000000000000000001691909301908152601481019190915260340192915050565b600073ffffffffffffffffffffffffffffffffffffffff808716835280861660208401525060806040830152610bd56080830185610a3f565b90508260608301529594505050505056fea26469706673582212200afb9537c52292543adecf37773b8e4d795c984adf4e80970731e434679b4ec264736f6c63430008090033" + }, + "0x420000000000000000000000000000000000000F": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000000a693b8f8207ff043f6bbc2e2120bbe4c2251efe9", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x0000000000000000000000000000000000000000000000000000000000000001", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x0000000000000000000000000000000000000000000000000000000000000001", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x0000000000000000000000000000000000000000000000000000000000000abe", + "0x0000000000000000000000000000000000000000000000000000000000000004": "0x000000000000000000000000000000000000000000000000000000000016e360", + "0x0000000000000000000000000000000000000000000000000000000000000005": "0x0000000000000000000000000000000000000000000000000000000000000006" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100f55760003560e01c80638c8885c811610097578063de26c4a111610066578063de26c4a1146101cc578063f2fde38b146101df578063f45e65d8146101f2578063fe173b97146101fb57600080fd5b80638c8885c81461016b5780638da5cb5b1461017e578063bede39b5146101a6578063bf1fe420146101b957600080fd5b806349948e0e116100d357806349948e0e14610134578063519b4bd3146101475780637046559714610150578063715018a61461016357600080fd5b80630c18c162146100fa578063313ce567146101165780633577afc51461011f575b600080fd5b61010360035481565b6040519081526020015b60405180910390f35b61010360055481565b61013261012d3660046108d0565b610204565b005b610103610142366004610918565b6102c6565b61010360025481565b61013261015e3660046108d0565b610322565b6101326103d8565b6101326101793660046108d0565b610465565b60005460405173ffffffffffffffffffffffffffffffffffffffff909116815260200161010d565b6101326101b43660046108d0565b61051b565b6101326101c73660046108d0565b6105d1565b6101036101da366004610918565b610687565b6101326101ed3660046109e7565b61072b565b61010360045481565b61010360015481565b60005473ffffffffffffffffffffffffffffffffffffffff16331461028a576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e657260448201526064015b60405180910390fd5b60038190556040518181527f32740b35c0ea213650f60d44366b4fb211c9033b50714e4a1d34e65d5beb9bb4906020015b60405180910390a150565b6000806102d283610687565b90506000600254826102e49190610a53565b90506000600554600a6102f79190610bb2565b90506000600454836103099190610a53565b905060006103178383610bbe565b979650505050505050565b60005473ffffffffffffffffffffffffffffffffffffffff1633146103a3576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60048190556040518181527f3336cd9708eaf2769a0f0dc0679f30e80f15dcd88d1921b5a16858e8b85c591a906020016102bb565b60005473ffffffffffffffffffffffffffffffffffffffff163314610459576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b610463600061085b565b565b60005473ffffffffffffffffffffffffffffffffffffffff1633146104e6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60058190556040518181527fd68112a8707e326d08be3656b528c1bcc5bbbfc47f4177e2179b14d8640838c1906020016102bb565b60005473ffffffffffffffffffffffffffffffffffffffff16331461059c576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60028190556040518181527f351fb23757bb5ea0546c85b7996ddd7155f96b939ebaa5ff7bc49c75f27f2c44906020016102bb565b60005473ffffffffffffffffffffffffffffffffffffffff163314610652576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60018190556040518181527ffcdccc6074c6c42e4bd578aa9870c697dc976a270968452d2b8c8dc369fae396906020016102bb565b600080805b8351811015610704578381815181106106a7576106a7610bf9565b01602001517fff00000000000000000000000000000000000000000000000000000000000000166106e4576106dd600483610c28565b91506106f2565b6106ef601083610c28565b91505b806106fc81610c40565b91505061068c565b506000600354826107159190610c28565b905061072381610440610c28565b949350505050565b60005473ffffffffffffffffffffffffffffffffffffffff1633146107ac576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b73ffffffffffffffffffffffffffffffffffffffff811661084f576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602660248201527f4f776e61626c653a206e6577206f776e657220697320746865207a65726f206160448201527f64647265737300000000000000000000000000000000000000000000000000006064820152608401610281565b6108588161085b565b50565b6000805473ffffffffffffffffffffffffffffffffffffffff8381167fffffffffffffffffffffffff0000000000000000000000000000000000000000831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b6000602082840312156108e257600080fd5b5035919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b60006020828403121561092a57600080fd5b813567ffffffffffffffff8082111561094257600080fd5b818401915084601f83011261095657600080fd5b813581811115610968576109686108e9565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f011681019083821181831017156109ae576109ae6108e9565b816040528281528760208487010111156109c757600080fd5b826020860160208301376000928101602001929092525095945050505050565b6000602082840312156109f957600080fd5b813573ffffffffffffffffffffffffffffffffffffffff81168114610a1d57600080fd5b9392505050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0483118215151615610a8b57610a8b610a24565b500290565b600181815b80851115610ae957817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff04821115610acf57610acf610a24565b80851615610adc57918102915b93841c9390800290610a95565b509250929050565b600082610b0057506001610bac565b81610b0d57506000610bac565b8160018114610b235760028114610b2d57610b49565b6001915050610bac565b60ff841115610b3e57610b3e610a24565b50506001821b610bac565b5060208310610133831016604e8410600b8410161715610b6c575081810a610bac565b610b768383610a90565b807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff04821115610ba857610ba8610a24565b0290505b92915050565b6000610a1d8383610af1565b600082610bf4577f4e487b7100000000000000000000000000000000000000000000000000000000600052601260045260246000fd5b500490565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052603260045260246000fd5b60008219821115610c3b57610c3b610a24565b500190565b60007fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff821415610c7257610c72610a24565b506001019056fea2646970667358221220b949ef5f9defd6c0aab6259672d00d239cb8854c9972ba1866af1c6ec6433d4c64736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000010": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x000000000000000000000000636af16bf2f682dd3109e60102b8e1a089fedaa8", + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x0000000000000000000000004200000000000000000000000000000000000007" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100675760003560e01c80633cb747bf116100505780633cb747bf146100ca578063662a633a146100ea578063a3a79548146100fd57600080fd5b806332b7006d1461006c57806336c717c114610081575b600080fd5b61007f61007a366004610d0f565b610110565b005b6001546100a19073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff909116815260200160405180910390f35b6000546100a19073ffffffffffffffffffffffffffffffffffffffff1681565b61007f6100f8366004610d80565b610126565b61007f61010b366004610e18565b6106c1565b61011f853333878787876106d8565b5050505050565b60015473ffffffffffffffffffffffffffffffffffffffff1661015e60005473ffffffffffffffffffffffffffffffffffffffff1690565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161461021d576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602e60248201527f4f564d5f58434841494e3a206d657373656e67657220636f6e7472616374207560448201527f6e61757468656e7469636174656400000000000000000000000000000000000060648201526084015b60405180910390fd5b8073ffffffffffffffffffffffffffffffffffffffff1661025360005473ffffffffffffffffffffffffffffffffffffffff1690565b73ffffffffffffffffffffffffffffffffffffffff16636e296e456040518163ffffffff1660e01b815260040160206040518083038186803b15801561029857600080fd5b505afa1580156102ac573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906102d09190610e9b565b73ffffffffffffffffffffffffffffffffffffffff1614610373576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603060248201527f4f564d5f58434841494e3a2077726f6e672073656e646572206f662063726f7360448201527f732d646f6d61696e206d657373616765000000000000000000000000000000006064820152608401610214565b61039d877f1d1d8b6300000000000000000000000000000000000000000000000000000000610a32565b801561045357508673ffffffffffffffffffffffffffffffffffffffff1663c01e1bd66040518163ffffffff1660e01b8152600401602060405180830381600087803b1580156103ec57600080fd5b505af1158015610400573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906104249190610e9b565b73ffffffffffffffffffffffffffffffffffffffff168873ffffffffffffffffffffffffffffffffffffffff16145b15610567576040517f40c10f1900000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8681166004830152602482018690528816906340c10f1990604401600060405180830381600087803b1580156104c857600080fd5b505af11580156104dc573d6000803e3d6000fd5b505050508573ffffffffffffffffffffffffffffffffffffffff168773ffffffffffffffffffffffffffffffffffffffff168973ffffffffffffffffffffffffffffffffffffffff167fb0444523268717a02698be47d0803aa7468c00acbed2f8bd93a0459cde61dd898888888860405161055a9493929190610f08565b60405180910390a46106b7565b600063a9f9e67560e01b8989888a89898960405160240161058e9796959493929190610f3e565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fffffffff00000000000000000000000000000000000000000000000000000000909316929092179091526001549091506106339073ffffffffffffffffffffffffffffffffffffffff16600083610a57565b8673ffffffffffffffffffffffffffffffffffffffff168873ffffffffffffffffffffffffffffffffffffffff168a73ffffffffffffffffffffffffffffffffffffffff167f7ea89a4591614515571c2b51f5ea06494056f261c10ab1ed8c03c7590d87bce0898989896040516106ad9493929190610f08565b60405180910390a4505b5050505050505050565b6106d0863387878787876106d8565b505050505050565b6040517f9dc29fac0000000000000000000000000000000000000000000000000000000081523360048201526024810185905273ffffffffffffffffffffffffffffffffffffffff881690639dc29fac90604401600060405180830381600087803b15801561074657600080fd5b505af115801561075a573d6000803e3d6000fd5b5050505060008773ffffffffffffffffffffffffffffffffffffffff1663c01e1bd66040518163ffffffff1660e01b8152600401602060405180830381600087803b1580156107a857600080fd5b505af11580156107bc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906107e09190610e9b565b9050606073ffffffffffffffffffffffffffffffffffffffff891673deaddeaddeaddeaddeaddeaddeaddeaddead000014156108d5576040517f1532ec340000000000000000000000000000000000000000000000000000000090610851908a908a908a9089908990602401610f9b565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fffffffff00000000000000000000000000000000000000000000000000000000909316929092179091529050610994565b6040517fa9f9e67500000000000000000000000000000000000000000000000000000000906109149084908c908c908c908c908b908b90602401610f3e565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fffffffff000000000000000000000000000000000000000000000000000000009093169290921790915290505b6001546109b89073ffffffffffffffffffffffffffffffffffffffff168683610a57565b3373ffffffffffffffffffffffffffffffffffffffff168973ffffffffffffffffffffffffffffffffffffffff168373ffffffffffffffffffffffffffffffffffffffff167f73d170910aba9e6d50b102db522b1dbcd796216f5128b445aa2135272886497e8a8a89896040516106ad9493929190610f08565b6000610a3d83610ae8565b8015610a4e5750610a4e8383610b4c565b90505b92915050565b6000546040517f3dbb202b00000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff90911690633dbb202b90610ab190869085908790600401611016565b600060405180830381600087803b158015610acb57600080fd5b505af1158015610adf573d6000803e3d6000fd5b50505050505050565b6000610b14827f01ffc9a700000000000000000000000000000000000000000000000000000000610b4c565b8015610a515750610b45827fffffffff00000000000000000000000000000000000000000000000000000000610b4c565b1592915050565b604080517fffffffff00000000000000000000000000000000000000000000000000000000831660248083019190915282518083039091018152604490910182526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167f01ffc9a7000000000000000000000000000000000000000000000000000000001790529051600091908290819073ffffffffffffffffffffffffffffffffffffffff87169061753090610c06908690611092565b6000604051808303818686fa925050503d8060008114610c42576040519150601f19603f3d011682016040523d82523d6000602084013e610c47565b606091505b5091509150602081511015610c625760009350505050610a51565b818015610c7e575080806020019051810190610c7e91906110ae565b9695505050505050565b73ffffffffffffffffffffffffffffffffffffffff81168114610caa57600080fd5b50565b803563ffffffff81168114610cc157600080fd5b919050565b60008083601f840112610cd857600080fd5b50813567ffffffffffffffff811115610cf057600080fd5b602083019150836020828501011115610d0857600080fd5b9250929050565b600080600080600060808688031215610d2757600080fd5b8535610d3281610c88565b945060208601359350610d4760408701610cad565b9250606086013567ffffffffffffffff811115610d6357600080fd5b610d6f88828901610cc6565b969995985093965092949392505050565b600080600080600080600060c0888a031215610d9b57600080fd5b8735610da681610c88565b96506020880135610db681610c88565b95506040880135610dc681610c88565b94506060880135610dd681610c88565b93506080880135925060a088013567ffffffffffffffff811115610df957600080fd5b610e058a828b01610cc6565b989b979a50959850939692959293505050565b60008060008060008060a08789031215610e3157600080fd5b8635610e3c81610c88565b95506020870135610e4c81610c88565b945060408701359350610e6160608801610cad565b9250608087013567ffffffffffffffff811115610e7d57600080fd5b610e8989828a01610cc6565b979a9699509497509295939492505050565b600060208284031215610ead57600080fd5b8151610eb881610c88565b9392505050565b8183528181602085013750600060208284010152600060207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f840116840101905092915050565b73ffffffffffffffffffffffffffffffffffffffff85168152836020820152606060408201526000610c7e606083018486610ebf565b600073ffffffffffffffffffffffffffffffffffffffff808a1683528089166020840152808816604084015280871660608401525084608083015260c060a0830152610f8e60c083018486610ebf565b9998505050505050505050565b600073ffffffffffffffffffffffffffffffffffffffff808816835280871660208401525084604083015260806060830152610fdb608083018486610ebf565b979650505050505050565b60005b83811015611001578181015183820152602001610fe9565b83811115611010576000848401525b50505050565b73ffffffffffffffffffffffffffffffffffffffff841681526060602082015260008351806060840152611051816080850160208801610fe6565b63ffffffff93909316604083015250601f919091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0160160800192915050565b600082516110a4818460208701610fe6565b9190910192915050565b6000602082840312156110c057600080fd5b81518015158114610eb857600080fdfea2646970667358221220df892c82e9ba0fc965240aa38614ff1bd6af4d227ce2c0e9933d7f50711a886264736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000011": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000000fd1d2e729ae8eee2e146c033bf4400fe75284301" + }, + "code": "0x6080604052600436106100385760003560e01c80633ccfd60b14610044578063d3e5792b1461005b578063d4ff92181461008a57600080fd5b3661003f57005b600080fd5b34801561005057600080fd5b506100596100dc565b005b34801561006757600080fd5b5061007767d02ab486cedc000081565b6040519081526020015b60405180910390f35b34801561009657600080fd5b506000546100b79073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff9091168152602001610081565b67d02ab486cedc000047101561019e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152605760248201527f4f564d5f53657175656e6365724665655661756c743a2077697468647261776160448201527f6c20616d6f756e74206d7573742062652067726561746572207468616e206d6960648201527f6e696d756d207769746864726177616c20616d6f756e74000000000000000000608482015260a40160405180910390fd5b600080546040805160208101825283815290517fa3a795480000000000000000000000000000000000000000000000000000000081527342000000000000000000000000000000000000109363a3a79548936102309373deaddeaddeaddeaddeaddeaddeaddeaddead00009373ffffffffffffffffffffffffffffffffffffffff909216924792909190600401610264565b600060405180830381600087803b15801561024a57600080fd5b505af115801561025e573d6000803e3d6000fd5b50505050565b600073ffffffffffffffffffffffffffffffffffffffff808816835260208188168185015286604085015263ffffffff8616606085015260a06080850152845191508160a085015260005b828110156102cb5785810182015185820160c0015281016102af565b828111156102dd57600060c084870101525b5050601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169190910160c001969550505050505056fea2646970667358221220387a6116dde263ea48767352a397053c8cffa776aecb43cded2f25a4a9cfbdc264736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000012": { + "balance": "00", + "storage": {}, + "code": "0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063896f93d114610030575b600080fd5b61004361003e36600461025f565b610045565b005b73ffffffffffffffffffffffffffffffffffffffff83166100c6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601d60248201527f4d7573742070726f76696465204c3120746f6b656e2061646472657373000000604482015260640160405180910390fd5b60007342000000000000000000000000000000000000108484846040516100ec90610178565b6100f99493929190610359565b604051809103906000f080158015610115573d6000803e3d6000fd5b5090508073ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fceeb8e7d520d7f3b65fc11a262b91066940193b05d4f93df07cfdced0eb551cf60405160405180910390a350505050565b6113d7806103b083390190565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600082601f8301126101c557600080fd5b813567ffffffffffffffff808211156101e0576101e0610185565b604051601f83017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f0116810190828211818310171561022657610226610185565b8160405283815286602085880101111561023f57600080fd5b836020870160208301376000602085830101528094505050505092915050565b60008060006060848603121561027457600080fd5b833573ffffffffffffffffffffffffffffffffffffffff8116811461029857600080fd5b9250602084013567ffffffffffffffff808211156102b557600080fd5b6102c1878388016101b4565b935060408601359150808211156102d757600080fd5b506102e4868287016101b4565b9150509250925092565b6000815180845260005b81811015610314576020818501810151868301820152016102f8565b81811115610326576000602083870101525b50601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169290920160200192915050565b600073ffffffffffffffffffffffffffffffffffffffff80871683528086166020840152506080604083015261039260808301856102ee565b82810360608401526103a481856102ee565b97965050505050505056fe60806040523480156200001157600080fd5b50604051620013d7380380620013d783398101604081905262000034916200022f565b8151829082906200004d9060039060208501906200009f565b508051620000639060049060208401906200009f565b5050600580546001600160a01b039586166001600160a01b031991821617909155600680549690951695169490941790925550620002fc915050565b828054620000ad90620002bf565b90600052602060002090601f016020900481019282620000d157600085556200011c565b82601f10620000ec57805160ff19168380011785556200011c565b828001600101855582156200011c579182015b828111156200011c578251825591602001919060010190620000ff565b506200012a9291506200012e565b5090565b5b808211156200012a57600081556001016200012f565b80516001600160a01b03811681146200015d57600080fd5b919050565b634e487b7160e01b600052604160045260246000fd5b600082601f8301126200018a57600080fd5b81516001600160401b0380821115620001a757620001a762000162565b604051601f8301601f19908116603f01168101908282118183101715620001d257620001d262000162565b81604052838152602092508683858801011115620001ef57600080fd5b600091505b83821015620002135785820183015181830184015290820190620001f4565b83821115620002255760008385830101525b9695505050505050565b600080600080608085870312156200024657600080fd5b620002518562000145565b9350620002616020860162000145565b60408601519093506001600160401b03808211156200027f57600080fd5b6200028d8883890162000178565b93506060870151915080821115620002a457600080fd5b50620002b38782880162000178565b91505092959194509250565b600181811c90821680620002d457607f821691505b60208210811415620002f657634e487b7160e01b600052602260045260246000fd5b50919050565b6110cb806200030c6000396000f3fe608060405234801561001057600080fd5b50600436106101005760003560e01c806370a0823111610097578063a9059cbb11610066578063a9059cbb14610215578063ae1f6aaf14610228578063c01e1bd61461026d578063dd62ed3e1461028d57600080fd5b806370a08231146101b157806395d89b41146101e75780639dc29fac146101ef578063a457c2d71461020257600080fd5b806323b872dd116100d357806323b872dd14610167578063313ce5671461017a578063395093511461018957806340c10f191461019c57600080fd5b806301ffc9a71461010557806306fdde031461012d578063095ea7b31461014257806318160ddd14610155575b600080fd5b610118610113366004610e4a565b6102d3565b60405190151581526020015b60405180910390f35b610135610393565b6040516101249190610e93565b610118610150366004610f2f565b610425565b6002545b604051908152602001610124565b610118610175366004610f59565b61043b565b60405160128152602001610124565b610118610197366004610f2f565b61050c565b6101af6101aa366004610f2f565b610555565b005b6101596101bf366004610f95565b73ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090205490565b61013561061a565b6101af6101fd366004610f2f565b610629565b610118610210366004610f2f565b6106e2565b610118610223366004610f2f565b6107a0565b6006546102489073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff9091168152602001610124565b6005546102489073ffffffffffffffffffffffffffffffffffffffff1681565b61015961029b366004610fb0565b73ffffffffffffffffffffffffffffffffffffffff918216600090815260016020908152604080832093909416825291909152205490565b60007f01ffc9a7a5cef8baa21ed3c5c0d7e23accb804b619e9333b597f47a0d84076e27f1d1d8b63000000000000000000000000000000000000000000000000000000007fffffffff0000000000000000000000000000000000000000000000000000000084167f01ffc9a700000000000000000000000000000000000000000000000000000000148061038b57507fffffffff00000000000000000000000000000000000000000000000000000000848116908216145b949350505050565b6060600380546103a290610fe3565b80601f01602080910402602001604051908101604052809291908181526020018280546103ce90610fe3565b801561041b5780601f106103f05761010080835404028352916020019161041b565b820191906000526020600020905b8154815290600101906020018083116103fe57829003601f168201915b5050505050905090565b60006104323384846107ad565b50600192915050565b600061044884848461092d565b73ffffffffffffffffffffffffffffffffffffffff84166000908152600160209081526040808320338452909152902054828110156104f45760405162461bcd60e51b815260206004820152602860248201527f45524332303a207472616e7366657220616d6f756e742065786365656473206160448201527f6c6c6f77616e636500000000000000000000000000000000000000000000000060648201526084015b60405180910390fd5b61050185338584036107ad565b506001949350505050565b33600081815260016020908152604080832073ffffffffffffffffffffffffffffffffffffffff871684529091528120549091610432918590610550908690611066565b6107ad565b60065473ffffffffffffffffffffffffffffffffffffffff1633146105bc5760405162461bcd60e51b815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104eb565b6105c68282610b93565b8173ffffffffffffffffffffffffffffffffffffffff167f0f6798a560793a54c3bcfe86a93cde1e73087d944c0ea20544137d41213968858260405161060e91815260200190565b60405180910390a25050565b6060600480546103a290610fe3565b60065473ffffffffffffffffffffffffffffffffffffffff1633146106905760405162461bcd60e51b815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104eb565b61069a8282610c99565b8173ffffffffffffffffffffffffffffffffffffffff167fcc16f5dbb4873280815c1ee09dbd06736cffcc184412cf7a71a0fdb75d397ca58260405161060e91815260200190565b33600090815260016020908152604080832073ffffffffffffffffffffffffffffffffffffffff86168452909152812054828110156107895760405162461bcd60e51b815260206004820152602560248201527f45524332303a2064656372656173656420616c6c6f77616e63652062656c6f7760448201527f207a65726f00000000000000000000000000000000000000000000000000000060648201526084016104eb565b61079633858584036107ad565b5060019392505050565b600061043233848461092d565b73ffffffffffffffffffffffffffffffffffffffff83166108355760405162461bcd60e51b8152602060048201526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f2061646460448201527f726573730000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff82166108be5760405162461bcd60e51b815260206004820152602260248201527f45524332303a20617070726f766520746f20746865207a65726f20616464726560448201527f737300000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff83811660008181526001602090815260408083209487168084529482529182902085905590518481527f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b92591015b60405180910390a3505050565b73ffffffffffffffffffffffffffffffffffffffff83166109b65760405162461bcd60e51b815260206004820152602560248201527f45524332303a207472616e736665722066726f6d20746865207a65726f20616460448201527f647265737300000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff8216610a3f5760405162461bcd60e51b815260206004820152602360248201527f45524332303a207472616e7366657220746f20746865207a65726f206164647260448201527f657373000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604090205481811015610adb5760405162461bcd60e51b815260206004820152602660248201527f45524332303a207472616e7366657220616d6f756e742065786365656473206260448201527f616c616e6365000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff808516600090815260208190526040808220858503905591851681529081208054849290610b1f908490611066565b925050819055508273ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef84604051610b8591815260200190565b60405180910390a350505050565b73ffffffffffffffffffffffffffffffffffffffff8216610bf65760405162461bcd60e51b815260206004820152601f60248201527f45524332303a206d696e7420746f20746865207a65726f20616464726573730060448201526064016104eb565b8060026000828254610c089190611066565b909155505073ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604081208054839290610c42908490611066565b909155505060405181815273ffffffffffffffffffffffffffffffffffffffff8316906000907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9060200160405180910390a35050565b73ffffffffffffffffffffffffffffffffffffffff8216610d225760405162461bcd60e51b815260206004820152602160248201527f45524332303a206275726e2066726f6d20746865207a65726f2061646472657360448201527f730000000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604090205481811015610dbe5760405162461bcd60e51b815260206004820152602260248201527f45524332303a206275726e20616d6f756e7420657863656564732062616c616e60448201527f636500000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff83166000908152602081905260408120838303905560028054849290610dfa90849061107e565b909155505060405182815260009073ffffffffffffffffffffffffffffffffffffffff8516907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef90602001610920565b600060208284031215610e5c57600080fd5b81357fffffffff0000000000000000000000000000000000000000000000000000000081168114610e8c57600080fd5b9392505050565b600060208083528351808285015260005b81811015610ec057858101830151858201604001528201610ea4565b81811115610ed2576000604083870101525b50601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe016929092016040019392505050565b803573ffffffffffffffffffffffffffffffffffffffff81168114610f2a57600080fd5b919050565b60008060408385031215610f4257600080fd5b610f4b83610f06565b946020939093013593505050565b600080600060608486031215610f6e57600080fd5b610f7784610f06565b9250610f8560208501610f06565b9150604084013590509250925092565b600060208284031215610fa757600080fd5b610e8c82610f06565b60008060408385031215610fc357600080fd5b610fcc83610f06565b9150610fda60208401610f06565b90509250929050565b600181811c90821680610ff757607f821691505b60208210811415611031577f4e487b7100000000000000000000000000000000000000000000000000000000600052602260045260246000fd5b50919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000821982111561107957611079611037565b500190565b60008282101561109057611090611037565b50039056fea264697066735822122069a2d33039157f3f2f180571262ca2a5d0a3a24d33bf9448f3b7c2ce9ff757f964736f6c63430008090033a2646970667358221220d2e13f28319115807ec7308d1cd88642a8542d0b838e00b8769f8a85d696f26764736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000013": { + "balance": "00", + "storage": {}, + "code": "0x4B60005260206000F3" + }, + "0xDeadDeAddeAddEAddeadDEaDDEAdDeaDDeAD0000": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000006": "0x0000000000000000000000004200000000000000000000000000000000000010", + "0x0000000000000000000000000000000000000000000000000000000000000005": "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x457468657200000000000000000000000000000000000000000000000000000a", + "0x0000000000000000000000000000000000000000000000000000000000000004": "0x4554480000000000000000000000000000000000000000000000000000000006" + }, + "code": "0x608060405234801561001057600080fd5b50600436106101005760003560e01c806370a0823111610097578063a9059cbb11610066578063a9059cbb14610215578063ae1f6aaf14610228578063c01e1bd61461026d578063dd62ed3e1461028d57600080fd5b806370a08231146101b157806395d89b41146101e75780639dc29fac146101ef578063a457c2d71461020257600080fd5b806323b872dd116100d357806323b872dd14610167578063313ce5671461017a578063395093511461018957806340c10f191461019c57600080fd5b806301ffc9a71461010557806306fdde031461012d578063095ea7b31461014257806318160ddd14610155575b600080fd5b610118610113366004610c6d565b6102d3565b60405190151581526020015b60405180910390f35b610135610393565b6040516101249190610cb6565b610118610150366004610d52565b610425565b6002545b604051908152602001610124565b610118610175366004610d7c565b6104db565b60405160128152602001610124565b610118610197366004610d52565b61058c565b6101af6101aa366004610d52565b61063d565b005b6101596101bf366004610db8565b73ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090205490565b61013561071c565b6101af6101fd366004610d52565b61072b565b610118610210366004610d52565b6107fe565b610118610223366004610d52565b6108af565b6006546102489073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff9091168152602001610124565b6005546102489073ffffffffffffffffffffffffffffffffffffffff1681565b61015961029b366004610dd3565b73ffffffffffffffffffffffffffffffffffffffff918216600090815260016020908152604080832093909416825291909152205490565b60007f01ffc9a7a5cef8baa21ed3c5c0d7e23accb804b619e9333b597f47a0d84076e27f1d1d8b63000000000000000000000000000000000000000000000000000000007fffffffff0000000000000000000000000000000000000000000000000000000084167f01ffc9a700000000000000000000000000000000000000000000000000000000148061038b57507fffffffff00000000000000000000000000000000000000000000000000000000848116908216145b949350505050565b6060600380546103a290610e06565b80601f01602080910402602001604051908101604052809291908181526020018280546103ce90610e06565b801561041b5780601f106103f05761010080835404028352916020019161041b565b820191906000526020600020905b8154815290600101906020018083116103fe57829003601f168201915b5050505050905090565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604260248201527f4f564d5f4554483a20617070726f76652069732064697361626c65642070656e60448201527f64696e67206675727468657220636f6d6d756e6974792064697363757373696f60648201527f6e2e000000000000000000000000000000000000000000000000000000000000608482015260009060a4015b60405180910390fd5b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604760248201527f4f564d5f4554483a207472616e7366657246726f6d2069732064697361626c6560448201527f642070656e64696e67206675727468657220636f6d6d756e697479206469736360648201527f757373696f6e2e00000000000000000000000000000000000000000000000000608482015260009060a4016104d2565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604c60248201527f4f564d5f4554483a20696e637265617365416c6c6f77616e636520697320646960448201527f7361626c65642070656e64696e67206675727468657220636f6d6d756e69747960648201527f2064697363757373696f6e2e0000000000000000000000000000000000000000608482015260009060a4016104d2565b60065473ffffffffffffffffffffffffffffffffffffffff1633146106be576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104d2565b6106c88282610960565b8173ffffffffffffffffffffffffffffffffffffffff167f0f6798a560793a54c3bcfe86a93cde1e73087d944c0ea20544137d41213968858260405161071091815260200190565b60405180910390a25050565b6060600480546103a290610e06565b60065473ffffffffffffffffffffffffffffffffffffffff1633146107ac576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104d2565b6107b68282610a80565b8173ffffffffffffffffffffffffffffffffffffffff167fcc16f5dbb4873280815c1ee09dbd06736cffcc184412cf7a71a0fdb75d397ca58260405161071091815260200190565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604c60248201527f4f564d5f4554483a206465637265617365416c6c6f77616e636520697320646960448201527f7361626c65642070656e64696e67206675727468657220636f6d6d756e69747960648201527f2064697363757373696f6e2e0000000000000000000000000000000000000000608482015260009060a4016104d2565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604360248201527f4f564d5f4554483a207472616e736665722069732064697361626c656420706560448201527f6e64696e67206675727468657220636f6d6d756e69747920646973637573736960648201527f6f6e2e0000000000000000000000000000000000000000000000000000000000608482015260009060a4016104d2565b73ffffffffffffffffffffffffffffffffffffffff82166109dd576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601f60248201527f45524332303a206d696e7420746f20746865207a65726f20616464726573730060448201526064016104d2565b80600260008282546109ef9190610e89565b909155505073ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604081208054839290610a29908490610e89565b909155505060405181815273ffffffffffffffffffffffffffffffffffffffff8316906000907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9060200160405180910390a35050565b73ffffffffffffffffffffffffffffffffffffffff8216610b23576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602160248201527f45524332303a206275726e2066726f6d20746865207a65726f2061646472657360448201527f730000000000000000000000000000000000000000000000000000000000000060648201526084016104d2565b73ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604090205481811015610bd9576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602260248201527f45524332303a206275726e20616d6f756e7420657863656564732062616c616e60448201527f636500000000000000000000000000000000000000000000000000000000000060648201526084016104d2565b73ffffffffffffffffffffffffffffffffffffffff83166000908152602081905260408120838303905560028054849290610c15908490610ea1565b909155505060405182815260009073ffffffffffffffffffffffffffffffffffffffff8516907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9060200160405180910390a3505050565b600060208284031215610c7f57600080fd5b81357fffffffff0000000000000000000000000000000000000000000000000000000081168114610caf57600080fd5b9392505050565b600060208083528351808285015260005b81811015610ce357858101830151858201604001528201610cc7565b81811115610cf5576000604083870101525b50601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe016929092016040019392505050565b803573ffffffffffffffffffffffffffffffffffffffff81168114610d4d57600080fd5b919050565b60008060408385031215610d6557600080fd5b610d6e83610d29565b946020939093013593505050565b600080600060608486031215610d9157600080fd5b610d9a84610d29565b9250610da860208501610d29565b9150604084013590509250925092565b600060208284031215610dca57600080fd5b610caf82610d29565b60008060408385031215610de657600080fd5b610def83610d29565b9150610dfd60208401610d29565b90509250929050565b600181811c90821680610e1a57607f821691505b60208210811415610e54577f4e487b7100000000000000000000000000000000000000000000000000000000600052602260045260246000fd5b50919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b60008219821115610e9c57610e9c610e5a565b500190565b600082821015610eb357610eb3610e5a565b50039056fea2646970667358221220b71535a5111461b42945e5d842957b3a5926f7ed07d271872f6da21952b5f8b464736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000006": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x577261707065642045746865720000000000000000000000000000000000001a", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x5745544800000000000000000000000000000000000000000000000000000008", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x0000000000000000000000000000000000000000000000000000000000000012" + }, + "code": "0x6080604052600436106100bc5760003560e01c8063313ce56711610074578063a9059cbb1161004e578063a9059cbb146102cb578063d0e30db0146100bc578063dd62ed3e14610311576100bc565b8063313ce5671461024b57806370a082311461027657806395d89b41146102b6576100bc565b806318160ddd116100a557806318160ddd146101aa57806323b872dd146101d15780632e1a7d4d14610221576100bc565b806306fdde03146100c6578063095ea7b314610150575b6100c4610359565b005b3480156100d257600080fd5b506100db6103a8565b6040805160208082528351818301528351919283929083019185019080838360005b838110156101155781810151838201526020016100fd565b50505050905090810190601f1680156101425780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561015c57600080fd5b506101966004803603604081101561017357600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135169060200135610454565b604080519115158252519081900360200190f35b3480156101b657600080fd5b506101bf6104c7565b60408051918252519081900360200190f35b3480156101dd57600080fd5b50610196600480360360608110156101f457600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135811691602081013590911690604001356104cb565b34801561022d57600080fd5b506100c46004803603602081101561024457600080fd5b503561066b565b34801561025757600080fd5b50610260610700565b6040805160ff9092168252519081900360200190f35b34801561028257600080fd5b506101bf6004803603602081101561029957600080fd5b503573ffffffffffffffffffffffffffffffffffffffff16610709565b3480156102c257600080fd5b506100db61071b565b3480156102d757600080fd5b50610196600480360360408110156102ee57600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135169060200135610793565b34801561031d57600080fd5b506101bf6004803603604081101561033457600080fd5b5073ffffffffffffffffffffffffffffffffffffffff813581169160200135166107a7565b33600081815260036020908152604091829020805434908101909155825190815291517fe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c9281900390910190a2565b6000805460408051602060026001851615610100027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0190941693909304601f8101849004840282018401909252818152929183018282801561044c5780601f106104215761010080835404028352916020019161044c565b820191906000526020600020905b81548152906001019060200180831161042f57829003601f168201915b505050505081565b33600081815260046020908152604080832073ffffffffffffffffffffffffffffffffffffffff8716808552908352818420869055815186815291519394909390927f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925928290030190a350600192915050565b4790565b73ffffffffffffffffffffffffffffffffffffffff83166000908152600360205260408120548211156104fd57600080fd5b73ffffffffffffffffffffffffffffffffffffffff84163314801590610573575073ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff14155b156105ed5773ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020548211156105b557600080fd5b73ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020805483900390555b73ffffffffffffffffffffffffffffffffffffffff808516600081815260036020908152604080832080548890039055938716808352918490208054870190558351868152935191937fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef929081900390910190a35060019392505050565b3360009081526003602052604090205481111561068757600080fd5b33600081815260036020526040808220805485900390555183156108fc0291849190818181858888f193505050501580156106c6573d6000803e3d6000fd5b5060408051828152905133917f7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65919081900360200190a250565b60025460ff1681565b60036020526000908152604090205481565b60018054604080516020600284861615610100027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0190941693909304601f8101849004840282018401909252818152929183018282801561044c5780601f106104215761010080835404028352916020019161044c565b60006107a03384846104cb565b9392505050565b60046020908152600092835260408084209091529082529020548156fea265627a7a7231582091c18790e0cca5011d2518024840ee00fecc67e11f56fd746f2cf84d5b583e0064736f6c63430005110032" + } + } +} diff --git a/crates/primitives/src/chain/mod.rs b/crates/primitives/src/chain/mod.rs index 43f4647a8c9..e1c04290c88 100644 --- a/crates/primitives/src/chain/mod.rs +++ b/crates/primitives/src/chain/mod.rs @@ -14,6 +14,9 @@ pub use spec::{ ForkTimestamps, GOERLI, MAINNET, SEPOLIA, }; +#[cfg(feature = "optimism")] +pub use spec::OP_GOERLI; + // The chain info module. mod info; pub use info::ChainInfo; @@ -44,6 +47,18 @@ impl Chain { Chain::Named(ethers_core::types::Chain::Sepolia) } + /// Returns the optimism goerli chain. + #[cfg(feature = "optimism")] + pub const fn optimism_goerli() -> Self { + Chain::Named(ethers_core::types::Chain::OptimismGoerli) + } + + /// Returns the optimism mainnet chain. + #[cfg(feature = "optimism")] + pub const fn optimism_mainnet() -> Self { + Chain::Named(ethers_core::types::Chain::Optimism) + } + /// The id of the chain pub fn id(&self) -> u64 { match self { diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 86d1fbbbbee..3d44b28e474 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -106,6 +106,30 @@ pub static SEPOLIA: Lazy = Lazy::new(|| ChainSpec { optimism: None, }); +/// The Optimism Goerli spec +#[cfg(feature = "optimism")] +pub static OP_GOERLI: Lazy = Lazy::new(|| ChainSpec { + chain: Chain::optimism_goerli(), + genesis: serde_json::from_str(include_str!("../../res/genesis/goerli_op.json")) + .expect("Can't deserialize Optimism Goerli genesis json"), + genesis_hash: Some(H256(hex!( + "c1fc15cd51159b1f1e5cbc4b82e85c1447ddfa33c52cf1d98d14fba0d6354be1" + ))), + hardforks: BTreeMap::from([ + (Hardfork::Byzantium, ForkCondition::Block(0)), + (Hardfork::Constantinople, ForkCondition::Block(0)), + (Hardfork::Petersburg, ForkCondition::Block(0)), + (Hardfork::Istanbul, ForkCondition::Block(0)), + (Hardfork::MuirGlacier, ForkCondition::Block(0)), + (Hardfork::Berlin, ForkCondition::Block(0)), + (Hardfork::London, ForkCondition::Block(4061224)), + (Hardfork::ArrowGlacier, ForkCondition::Block(4061224)), + (Hardfork::GrayGlacier, ForkCondition::Block(4061224)), + (Hardfork::Bedrock, ForkCondition::Block(4061224)), + ]), + optimism: Some(OptimismConfig { eip_1559_elasticity: 10, eip_1559_denominator: 50 }), +}); + /// An Ethereum chain specification. /// /// A chain specification describes: @@ -902,9 +926,10 @@ mod tests { ForkCondition, ForkHash, ForkId, Genesis, Hardfork, Head, GOERLI, H256, MAINNET, SEPOLIA, U256, }; - use bytes::BytesMut; - use ethers_core::types as EtherType; - use reth_rlp::Encodable; + + #[cfg(feature = "optimism")] + use crate::OP_GOERLI; + fn test_fork_ids(spec: &ChainSpec, cases: &[(Head, ForkId)]) { for (block, expected_id) in cases { let computed_id = spec.fork_id(block); @@ -1200,6 +1225,25 @@ Post-merge hard forks (timestamp based): ); } + #[cfg(feature = "optimism")] + #[test] + fn optimism_goerli_forkids() { + // TODO + test_fork_ids( + &OP_GOERLI, + &[ + ( + Head { number: 0, ..Default::default() }, + ForkId { hash: ForkHash([0x00, 0x00, 0x00, 0x00]), next: 4061224 }, + ), + ( + Head { number: 4061224, ..Default::default() }, + ForkId { hash: ForkHash([0x00, 0x00, 0x00, 0x00]), next: 0 }, + ), + ], + ); + } + /// Checks that time-based forks work /// /// This is based off of the test vectors here: https://github.com/ethereum/go-ethereum/blob/5c8cc10d1e05c23ff1108022f4150749e73c0ca1/core/forkid/forkid_test.go#L155-L188 diff --git a/crates/primitives/src/constants.rs b/crates/primitives/src/constants.rs index 005d69380ab..d6f3a88a7fd 100644 --- a/crates/primitives/src/constants.rs +++ b/crates/primitives/src/constants.rs @@ -77,6 +77,11 @@ pub const GOERLI_GENESIS: H256 = pub const SEPOLIA_GENESIS: H256 = H256(hex!("25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9")); +/// Optimism goerli genesis hash. +#[cfg(feature = "optimism")] +pub const GOERLI_OP_GENESIS: H256 = + H256(hex!("c1fc15cd51159b1f1e5cbc4b82e85c1447ddfa33c52cf1d98d14fba0d6354be1")); + /// Keccak256 over empty array. pub const KECCAK_EMPTY: H256 = H256(hex!("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")); diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 96e8c357c28..e08537ea14b 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -58,6 +58,8 @@ pub use block::{ BlockWithSenders, ForkBlock, SealedBlock, SealedBlockWithSenders, }; pub use bloom::Bloom; +#[cfg(feature = "optimism")] +pub use chain::OP_GOERLI; pub use chain::{ AllGenesisFormats, Chain, ChainInfo, ChainSpec, ChainSpecBuilder, DisplayHardforks, ForkCondition, ForkTimestamps, GOERLI, MAINNET, SEPOLIA, diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index 0f417a3a65f..32059744411 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -33,3 +33,6 @@ reth-interfaces = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-payload-builder = { workspace = true, features = ["test-utils"] } assert_matches = "1.5.0" + +[features] +optimism = ["reth-primitives/optimism", "reth-rpc-types/optimism"] diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 4803c4d2d71..17bb1cec06a 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -211,6 +211,119 @@ where Ok(result) } + /// When the Consensus layer receives a new block via the consensus gossip protocol, + /// the transactions in the block are sent to the execution layer in the form of a + /// `ExecutionPayload`. The Execution layer executes the transactions and validates the + /// state in the block header, then passes validation data back to Consensus layer, that + /// adds the block to the head of its own blockchain and attests to it. The block is then + /// broadcasted over the consensus p2p network in the form of a "Beacon block". + pub fn new_payload(&mut self, payload: ExecutionPayload) -> EngineApiResult { + let block = match self.try_construct_block(payload) { + Ok(b) => b, + Err(err) => { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::InvalidBlockHash { + validation_error: err.to_string(), + })) + } + }; + let block_hash = block.header.hash(); + let parent_hash = block.parent_hash; + + // The block already exists in our database + if self.client.is_known(&block_hash)? { + return Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)) + } + + let Some(parent) = self.client.block_by_hash(parent_hash)? else { + // TODO: cache block for storing later + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Syncing)) + }; + + let parent_td = if let Some(parent_td) = self.client.header_td(&block.parent_hash)? { + parent_td + } else { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::PayloadPreMerge.to_string(), + })) + }; + + // Short circuit the check by passing parent total difficulty. + if !self.chain_spec.fork(Hardfork::Paris).active_at_ttd(parent_td, U256::ZERO) { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::PayloadPreMerge.to_string(), + })) + } + + if block.timestamp <= parent.timestamp { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::PayloadTimestamp { + invalid: block.timestamp, + latest: parent.timestamp, + } + .to_string(), + })) + } + + let state_provider = self.client.latest()?; + let total_difficulty = parent_td + block.header.difficulty; + match executor::execute_and_verify_receipt( + &block.unseal(), + total_difficulty, + None, + &self.chain_spec, + &mut SubState::new(State::new(&state_provider)), + ) { + Ok(_) => Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)), + Err(err) => Ok(PayloadStatus::new( + PayloadStatusEnum::Invalid { validation_error: err.to_string() }, + parent_hash, // The parent hash is already in our database hence it is valid + )), + } + } + + /// Called to resolve chain forks and ensure that the Execution layer is working with the latest + /// valid chain. + pub fn fork_choice_updated( + &self, + fork_choice_state: ForkchoiceState, + payload_attributes: Option, + ) -> EngineApiResult { + let ForkchoiceState { head_block_hash, finalized_block_hash, .. } = fork_choice_state; + + if head_block_hash.is_zero() { + return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::ForkchoiceEmptyHead.to_string(), + })) + } + + // Block is not known, nothing to do. + if !self.client.is_known(&head_block_hash)? { + return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) + } + + // The finalized block hash is not known, we are still syncing + if !finalized_block_hash.is_zero() && !self.client.is_known(&finalized_block_hash)? { + return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) + } + + if let Err(error) = self.forkchoice_state_tx.send(fork_choice_state) { + tracing::error!(target: "rpc::engine_api", ?error, "Failed to update forkchoice state"); + } + + if let Some(attr) = payload_attributes { + #[cfg(feature = "optimism")] + if self.chain_spec.optimism.is_some() && attr.gas_limit.is_none() { + return Err(EngineApiError::MissingGasLimitInPayloadAttributes) + } + + // TODO: optionally build the block + } + + let chain_info = self.client.chain_info()?; + Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Valid) + .with_latest_valid_hash(chain_info.best_hash)) + } + /// Called to verify network configuration parameters and ensure that Consensus and Execution /// layers are using the latest configuration. pub async fn exchange_transition_configuration( diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index 26c29916c30..8e146b07f09 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -68,47 +68,10 @@ pub enum EngineApiError { ForkChoiceUpdate(#[from] BeaconForkChoiceUpdateError), /// An error occurred while processing a new payload in the beacon consensus engine. #[error(transparent)] - NewPayload(#[from] BeaconOnNewPayloadError), - /// Encountered an internal error. - #[error(transparent)] - Internal(Box), - /// Fetching the payload failed - #[error(transparent)] - GetPayloadError(#[from] PayloadBuilderError), -} - -impl From for jsonrpsee_types::error::ErrorObject<'static> { - fn from(error: EngineApiError) -> Self { - let code = match error { - EngineApiError::InvalidBodiesRange { .. } | - EngineApiError::WithdrawalsNotSupportedInV1 | - EngineApiError::NoWithdrawalsPostShanghai | - EngineApiError::HasWithdrawalsPreShanghai => INVALID_PARAMS_CODE, - EngineApiError::UnknownPayload => UNKNOWN_PAYLOAD_CODE, - EngineApiError::PayloadRequestTooLarge { .. } => REQUEST_TOO_LARGE_CODE, - - // Error responses from the consensus engine - EngineApiError::ForkChoiceUpdate(ref err) => match err { - BeaconForkChoiceUpdateError::ForkchoiceUpdateError(err) => return (*err).into(), - BeaconForkChoiceUpdateError::EngineUnavailable | - BeaconForkChoiceUpdateError::Internal(_) => INTERNAL_ERROR_CODE, - }, - EngineApiError::NewPayload(ref err) => match err { - BeaconOnNewPayloadError::Internal(_) | - BeaconOnNewPayloadError::EngineUnavailable => INTERNAL_ERROR_CODE, - }, - // Any other server error - EngineApiError::TerminalTD { .. } | - EngineApiError::TerminalBlockHash { .. } | - EngineApiError::Internal(_) | - EngineApiError::GetPayloadError(_) => INTERNAL_ERROR_CODE, - }; - jsonrpsee_types::error::ErrorObject::owned(code, error.to_string(), None::<()>) - } -} - -impl From for jsonrpsee_core::error::Error { - fn from(error: EngineApiError) -> Self { - jsonrpsee_core::error::Error::Call(error.into()) - } + Internal(#[from] reth_interfaces::Error), + /// If the optimism feature flag is enabled, the payload attributes must have a present + /// gas limit for the forkchoice updated method. + #[cfg(feature = "optimism")] + #[error("Missing gas limit in payload attributes")] + MissingGasLimitInPayloadAttributes, } From acca206a9bf58513567e0ccedd6d89e24dce208c Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 5 Mar 2023 15:02:33 -0700 Subject: [PATCH 011/150] Signature fix --- crates/primitives/src/transaction/signature.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index faf44c2b970..1b4df1e43d2 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -61,6 +61,11 @@ impl Signature { /// Output the `v` of the signature depends on chain_id #[inline] pub fn v(&self, chain_id: Option) -> u64 { + #[cfg(feature = "optimism")] + if self.r == U256::ZERO && self.s == U256::ZERO { + return 0; + } + if let Some(chain_id) = chain_id { // EIP-155: v = {0, 1} + CHAIN_ID * 2 + 35 self.odd_y_parity as u64 + chain_id * 2 + 35 From d5398fad385cdfb493bfe48780a57548948087b3 Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 19 Mar 2023 16:34:05 -0400 Subject: [PATCH 012/150] TEMP: Expose some CLI items --- bin/reth/src/cli.rs | 6 +----- bin/reth/src/dirs.rs | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index c32b1ae17ed..06f4d3a6a2f 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -91,10 +91,6 @@ struct Cli { #[derive(Debug, Args)] #[command(next_help_heading = "Logging")] pub struct Logs { - /// The flag to enable persistent logs. - #[arg(long = "log.persistent", global = true, conflicts_with = "journald")] - persistent: bool, - /// The path to put log files in. #[arg( long = "log.directory", @@ -116,7 +112,7 @@ pub struct Logs { impl Logs { /// Builds a tracing layer from the current log options. - pub fn layer(&self) -> eyre::Result, Option)>> + pub fn layer(&self) -> (BoxedLayer, Option) where S: Subscriber, for<'a> S: LookupSpan<'a>, diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index 796377b6a16..ccdd4125750 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -86,7 +86,6 @@ impl XdgPath for LogsDir { /// A small helper trait for unit structs that represent a standard path following the XDG /// path specification. pub trait XdgPath { - /// Resolve the standard path. fn resolve() -> Option; } From b8530c979e9d1d95e6b62246d2452daeacbd542c Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 19 Mar 2023 16:53:10 -0400 Subject: [PATCH 013/150] Resolve conflicts Remove `consensus` crate `Cargo.toml` Reduce diff --- bin/reth/src/dirs.rs | 1 + crates/consensus/Cargo.toml | 24 ------------- crates/consensus/common/Cargo.toml | 3 ++ crates/consensus/common/src/validation.rs | 12 +++---- crates/primitives/src/chain/spec.rs | 10 ++++-- crates/primitives/src/transaction/mod.rs | 2 +- .../primitives/src/transaction/signature.rs | 2 +- crates/primitives/src/transaction/tx_type.rs | 20 ++++++----- crates/revm/Cargo.toml | 11 ++---- crates/revm/revm-primitives/Cargo.toml | 2 +- crates/revm/revm-primitives/src/env.rs | 36 +++++++++---------- crates/rpc/rpc-engine-api/src/engine_api.rs | 4 +-- crates/transaction-pool/Cargo.toml | 2 +- crates/transaction-pool/src/traits.rs | 24 ++++++------- 14 files changed, 68 insertions(+), 85 deletions(-) delete mode 100644 crates/consensus/Cargo.toml diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index ccdd4125750..796377b6a16 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -86,6 +86,7 @@ impl XdgPath for LogsDir { /// A small helper trait for unit structs that represent a standard path following the XDG /// path specification. pub trait XdgPath { + /// Resolve the standard path. fn resolve() -> Option; } diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml deleted file mode 100644 index bd742d0bed0..00000000000 --- a/crates/consensus/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "reth-consensus" -version = "0.1.0" -edition = "2021" -license = "MIT OR Apache-2.0" -repository = "https://github.com/paradigmxyz/reth" -readme = "README.md" - -[dependencies] -# reth -reth-primitives = { path = "../primitives" } -reth-interfaces = { path = "../interfaces" } -reth-provider = { path = "../storage/provider" } - -# async -tokio = { version = "1", features = ["sync"] } - -[dev-dependencies] -reth-interfaces = { path = "../interfaces", features = ["test-utils"] } -reth-provider = { path = "../storage/provider", features = ["test-utils"] } -assert_matches = "1.5.0" - -[features] -optimism = [] \ No newline at end of file diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml index 55253ae5ece..e09f4ba3dfc 100644 --- a/crates/consensus/common/Cargo.toml +++ b/crates/consensus/common/Cargo.toml @@ -18,3 +18,6 @@ reth-interfaces = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } assert_matches = "1.5.0" mockall = "0.11.3" + +[features] +optimism = ["reth-primitives/optimism"] diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index d54e33e80d4..adce6c26848 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -10,8 +10,6 @@ use std::{ time::SystemTime, }; -use reth_primitives::constants; - #[cfg(feature = "optimism")] use reth_primitives::TxDeposit; @@ -69,11 +67,6 @@ pub fn validate_transaction_regarding_header( base_fee: Option, ) -> Result<(), ConsensusError> { let chain_id = match transaction { - #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { .. }) => { - // TODO: get the chain id - None - } Transaction::Legacy(TxLegacy { chain_id, .. }) => { // EIP-155: Simple replay attack protection: https://eips.ethereum.org/EIPS/eip-155 if chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(at_block_number) && @@ -109,6 +102,11 @@ pub fn validate_transaction_regarding_header( Some(*chain_id) } + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { .. }) => { + // TODO: I believe the chain id should be None here, but have to confirm. + None + } }; if let Some(chain_id) = chain_id { if chain_id != chain_spec.chain().id() { diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 3d44b28e474..731d5207048 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -126,6 +126,7 @@ pub static OP_GOERLI: Lazy = Lazy::new(|| ChainSpec { (Hardfork::ArrowGlacier, ForkCondition::Block(4061224)), (Hardfork::GrayGlacier, ForkCondition::Block(4061224)), (Hardfork::Bedrock, ForkCondition::Block(4061224)), + (Hardfork::Regolith, ForkCondition::Timestamp(1679079600)), ]), optimism: Some(OptimismConfig { eip_1559_elasticity: 10, eip_1559_denominator: 50 }), }); @@ -599,14 +600,16 @@ impl ChainSpecBuilder { /// Enable Bedrock at genesis #[cfg(feature = "optimism")] pub fn bedrock_activated(mut self) -> Self { + self = self.london_activated(); self.hardforks.insert(Hardfork::Bedrock, ForkCondition::Block(0)); self } - /// Enable Bedrock at genesis + /// Enable Regolith at genesis #[cfg(feature = "optimism")] pub fn regolith_activated(mut self) -> Self { - self.hardforks.insert(Hardfork::Regolith, ForkCondition::Timestamp(0)); + self = self.bedrock_activated(); + self.hardforks.insert(Hardfork::Regolith, ForkCondition::Block(0)); self } @@ -926,6 +929,9 @@ mod tests { ForkCondition, ForkHash, ForkId, Genesis, Hardfork, Head, GOERLI, H256, MAINNET, SEPOLIA, U256, }; + use bytes::BytesMut; + use ethers_core::types as EtherType; + use reth_rlp::Encodable; #[cfg(feature = "optimism")] use crate::OP_GOERLI; diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 56bc10d6043..d92cb5a6034 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -186,7 +186,7 @@ pub enum Transaction { Eip2930(TxEip2930), /// A transaction with a priority fee ([EIP-1559](https://eips.ethereum.org/EIPS/eip-1559)). Eip1559(TxEip1559), - /// Deposit transaction. + /// Optimism deposit transaction. #[cfg(feature = "optimism")] Deposit(TxDeposit), } diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index 1b4df1e43d2..d5371450f0a 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -63,7 +63,7 @@ impl Signature { pub fn v(&self, chain_id: Option) -> u64 { #[cfg(feature = "optimism")] if self.r == U256::ZERO && self.s == U256::ZERO { - return 0; + return 0 } if let Some(chain_id) = chain_id { diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index 39fa1710737..ade0eb66d61 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -12,6 +12,10 @@ pub const EIP2930_TX_TYPE_ID: u8 = 1; /// Identifier for [TxEip1559](crate::TxEip1559) transaction. pub const EIP1559_TX_TYPE_ID: u8 = 2; +/// Identifier for [TxDeposit](crate::TxDeposit) transaction. +#[cfg(feature = "optimism")] +pub(crate) const DEPOSIT_TX_TYPE_ID: u8 = 126; + /// Transaction Type #[derive_arbitrary(compact)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] @@ -31,11 +35,11 @@ pub enum TxType { impl From for u8 { fn from(value: TxType) -> Self { match value { - TxType::Legacy => 0, - TxType::EIP2930 => 1, - TxType::EIP1559 => 2, + TxType::Legacy => LEGACY_TX_TYPE_ID, + TxType::EIP2930 => EIP2930_TX_TYPE_ID, + TxType::EIP1559 => EIP1559_TX_TYPE_ID, #[cfg(feature = "optimism")] - TxType::DEPOSIT => 126, + TxType::DEPOSIT => DEPOSIT_TX_TYPE_ID, } } } @@ -52,11 +56,11 @@ impl Compact for TxType { B: bytes::BufMut + AsMut<[u8]>, { match self { - TxType::Legacy => 0, - TxType::EIP2930 => 1, - TxType::EIP1559 => 2, + TxType::Legacy => LEGACY_TX_TYPE_ID as usize, + TxType::EIP2930 => EIP2930_TX_TYPE_ID as usize, + TxType::EIP1559 => EIP1559_TX_TYPE_ID as usize, #[cfg(feature = "optimism")] - TxType::DEPOSIT => 126, + TxType::DEPOSIT => DEPOSIT_TX_TYPE_ID as usize, } } diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index e80ab708c0f..0f375476570 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -17,12 +17,7 @@ reth-revm-primitives = { path = "./revm-primitives" } reth-revm-inspectors = { path = "./revm-inspectors" } reth-consensus-common = { path = "../consensus/common" } -# revm -revm = { workspace = true } +revm = { version = "3.0.0" } -# common -tracing = { workspace = true } - -[dev-dependencies] -reth-rlp = { workspace = true } -once_cell = "1.17.0" +[features] +optimism = ["reth-primitives/optimism", "reth-revm-primitives/optimism"] diff --git a/crates/revm/revm-primitives/Cargo.toml b/crates/revm/revm-primitives/Cargo.toml index 8c00dc3670a..c40942e6514 100644 --- a/crates/revm/revm-primitives/Cargo.toml +++ b/crates/revm/revm-primitives/Cargo.toml @@ -13,4 +13,4 @@ reth-primitives = { path = "../../primitives" } revm = { version = "3.0.0" } [features] -optimism = [] +optimism = ["reth-primitives/optimism"] diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index 5351914c1e8..e590387a7b3 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -122,24 +122,6 @@ where { tx_env.caller = sender; match transaction.as_ref() { - #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { - tx_env.gas_limit = *gas_limit; - if let Some(m) = mint { - tx_env.gas_price = U256::from(*m); - } else { - tx_env.gas_price = U256::ZERO; - } - tx_env.gas_priority_fee = None; - match to { - TransactionKind::Call(to) => tx_env.transact_to = TransactTo::Call(*to), - TransactionKind::Create => tx_env.transact_to = TransactTo::create(), - } - tx_env.value = U256::from(*value); - tx_env.data = input.0.clone(); - tx_env.chain_id = None; - tx_env.nonce = None; - } Transaction::Legacy(TxLegacy { nonce, chain_id, @@ -233,5 +215,23 @@ where }) .collect(); } + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { + tx_env.gas_limit = *gas_limit; + if let Some(m) = mint { + tx_env.gas_price = U256::from(*m); + } else { + tx_env.gas_price = U256::ZERO; + } + tx_env.gas_priority_fee = None; + match to { + TransactionKind::Call(to) => tx_env.transact_to = TransactTo::Call(*to), + TransactionKind::Create => tx_env.transact_to = TransactTo::create(), + } + tx_env.value = U256::from(*value); + tx_env.data = input.0.clone(); + tx_env.chain_id = None; + tx_env.nonce = None; + } } } diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 17bb1cec06a..9238addfe52 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -310,9 +310,9 @@ where tracing::error!(target: "rpc::engine_api", ?error, "Failed to update forkchoice state"); } - if let Some(attr) = payload_attributes { + if let Some(_attr) = payload_attributes { #[cfg(feature = "optimism")] - if self.chain_spec.optimism.is_some() && attr.gas_limit.is_none() { + if self.chain_spec.optimism.is_some() && _attr.gas_limit.is_none() { return Err(EngineApiError::MissingGasLimitInPayloadAttributes) } diff --git a/crates/transaction-pool/Cargo.toml b/crates/transaction-pool/Cargo.toml index 47f44068627..f506f751474 100644 --- a/crates/transaction-pool/Cargo.toml +++ b/crates/transaction-pool/Cargo.toml @@ -53,4 +53,4 @@ rand = "0.8" default = ["serde"] serde = ["dep:serde"] test-utils = ["rand", "paste", "serde"] -optimism = [] \ No newline at end of file +optimism = ["reth-primitives/optimism"] diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index d181966f5e8..bf452afd0de 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -523,11 +523,11 @@ impl PoolTransaction for PooledTransaction { /// This is also commonly referred to as the "Gas Fee Cap" (`GasFeeCap`). fn max_fee_per_gas(&self) -> u128 { match &self.transaction.transaction { - #[cfg(feature = "optimism")] - Transaction::Deposit(_) => None, Transaction::Legacy(_) => None, Transaction::Eip2930(_) => None, Transaction::Eip1559(tx) => Some(tx.max_fee_per_gas), + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => None, } } @@ -536,11 +536,11 @@ impl PoolTransaction for PooledTransaction { /// This will return `None` for non-EIP1559 transactions fn max_priority_fee_per_gas(&self) -> Option { match &self.transaction.transaction { - #[cfg(feature = "optimism")] - Transaction::Deposit(_) => None, Transaction::Legacy(_) => None, Transaction::Eip2930(_) => None, Transaction::Eip1559(tx) => Some(tx.max_priority_fee_per_gas), + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => None, } } @@ -574,14 +574,6 @@ impl PoolTransaction for PooledTransaction { impl FromRecoveredTransaction for PooledTransaction { fn from_recovered_transaction(tx: TransactionSignedEcRecovered) -> Self { let (cost, effective_gas_price) = match &tx.transaction { - #[cfg(feature = "optimism")] - Transaction::Deposit(t) => { - // TODO: fix this gas price estimate - let gas_price = U256::from(0); - let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); - let effective_gas_price = 0u128; - (cost, effective_gas_price) - } Transaction::Legacy(t) => { let cost = U256::from(t.gas_price) * U256::from(t.gas_limit) + U256::from(t.value); let effective_gas_price = t.gas_price; @@ -598,6 +590,14 @@ impl FromRecoveredTransaction for PooledTransaction { let effective_gas_price = t.max_priority_fee_per_gas; (cost, effective_gas_price) } + #[cfg(feature = "optimism")] + Transaction::Deposit(t) => { + // TODO: fix this gas price estimate + let gas_price = U256::from(0); + let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); + let effective_gas_price = 0u128; + (cost, effective_gas_price) + } }; let cost = gas_cost + U256::from(tx.value()); From d418af629ea699013b711c4c40e074e957a940b5 Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 19 Mar 2023 19:40:53 -0400 Subject: [PATCH 014/150] Allow gaslimit to be set by the EngineAPI caller on OP --- crates/consensus/common/src/validation.rs | 35 +++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index adce6c26848..2cca186e171 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -273,27 +273,32 @@ pub fn validate_header_regarding_parent( // TODO Check difficulty increment between parent and child // Ace age did increment it by some formula that we need to follow. - let mut parent_gas_limit = parent.gas_limit; + // Check gas limit, max diff between child/parent gas_limit should be max_diff=parent_gas/1024 + // On Optimism, the gas limit can adjust instantly, so we skip this check if the optimism + // feature flag is enabled. + #[cfg(not(feature = "optimism"))] + { + let mut parent_gas_limit = parent.gas_limit; - // By consensus, gas_limit is multiplied by elasticity (*2) on - // on exact block that hardfork happens. - if chain_spec.fork(Hardfork::London).transitions_at_block(child.number) { - parent_gas_limit = parent.gas_limit * constants::EIP1559_ELASTICITY_MULTIPLIER; - } + // By consensus, gas_limit is multiplied by elasticity (*2) on + // on exact block that hardfork happens. + if chain_spec.fork(Hardfork::London).transitions_at_block(child.number) { + parent_gas_limit = parent.gas_limit * constants::EIP1559_ELASTICITY_MULTIPLIER; + } - // Check gas limit, max diff between child/parent gas_limit should be max_diff=parent_gas/1024 - if child.gas_limit > parent_gas_limit { - if child.gas_limit - parent_gas_limit >= parent_gas_limit / 1024 { - return Err(ConsensusError::GasLimitInvalidIncrease { + if child.gas_limit > parent_gas_limit { + if child.gas_limit - parent_gas_limit >= parent_gas_limit / 1024 { + return Err(ConsensusError::GasLimitInvalidIncrease { + parent_gas_limit, + child_gas_limit: child.gas_limit, + }) + } + } else if parent_gas_limit - child.gas_limit >= parent_gas_limit / 1024 { + return Err(ConsensusError::GasLimitInvalidDecrease { parent_gas_limit, child_gas_limit: child.gas_limit, }) } - } else if parent_gas_limit - child.gas_limit >= parent_gas_limit / 1024 { - return Err(ConsensusError::GasLimitInvalidDecrease { - parent_gas_limit, - child_gas_limit: child.gas_limit, - }) } // EIP-1559 check base fee From 90af9765a3b4237d08ac340f8db1091112a6ffb0 Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 19 Mar 2023 19:53:10 -0400 Subject: [PATCH 015/150] Add Optimism EIP1559 elasticity multiplier / max base fee change denominator --- crates/primitives/src/constants.rs | 10 ++++++++++ crates/primitives/src/transaction/tx_type.rs | 8 ++++---- crates/revm/revm-primitives/Cargo.toml | 2 ++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/primitives/src/constants.rs b/crates/primitives/src/constants.rs index d6f3a88a7fd..177082ddb6f 100644 --- a/crates/primitives/src/constants.rs +++ b/crates/primitives/src/constants.rs @@ -48,11 +48,21 @@ pub const MIN_PROTOCOL_BASE_FEE_U256: U256 = U256::from_limbs([7u64, 0, 0, 0]); pub const EIP1559_INITIAL_BASE_FEE: u64 = 1_000_000_000; /// Base fee max change denominator as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) +#[cfg(not(feature = "optimism"))] pub const EIP1559_BASE_FEE_MAX_CHANGE_DENOMINATOR: u64 = 8; /// Elasticity multiplier as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) +#[cfg(not(feature = "optimism"))] pub const EIP1559_ELASTICITY_MULTIPLIER: u64 = 2; +/// Base fee max change denominator for Optimism. +#[cfg(feature = "optimism")] +pub const EIP1559_BASE_FEE_MAX_CHANGE_DENOMINATOR: u64 = 50; + +/// Elasticity multiplier for Optimism. +#[cfg(feature = "optimism")] +pub const EIP1559_ELASTICITY_MULTIPLIER: u64 = 10; + /// Multiplier for converting gwei to wei. pub const GWEI_TO_WEI: u64 = 1_000_000_000; diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index ade0eb66d61..e44bfba1dfb 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -14,7 +14,7 @@ pub const EIP1559_TX_TYPE_ID: u8 = 2; /// Identifier for [TxDeposit](crate::TxDeposit) transaction. #[cfg(feature = "optimism")] -pub(crate) const DEPOSIT_TX_TYPE_ID: u8 = 126; +use crate::DEPOSIT_TX_TYPE; /// Transaction Type #[derive_arbitrary(compact)] @@ -29,7 +29,7 @@ pub enum TxType { EIP1559 = 2_isize, /// OP Deposit transaction. #[cfg(feature = "optimism")] - DEPOSIT = 126_isize, + DEPOSIT = DEPOSIT_TX_TYPE as isize, } impl From for u8 { @@ -39,7 +39,7 @@ impl From for u8 { TxType::EIP2930 => EIP2930_TX_TYPE_ID, TxType::EIP1559 => EIP1559_TX_TYPE_ID, #[cfg(feature = "optimism")] - TxType::DEPOSIT => DEPOSIT_TX_TYPE_ID, + TxType::DEPOSIT => DEPOSIT_TX_TYPE, } } } @@ -60,7 +60,7 @@ impl Compact for TxType { TxType::EIP2930 => EIP2930_TX_TYPE_ID as usize, TxType::EIP1559 => EIP1559_TX_TYPE_ID as usize, #[cfg(feature = "optimism")] - TxType::DEPOSIT => DEPOSIT_TX_TYPE_ID as usize, + TxType::DEPOSIT => DEPOSIT_TX_TYPE as usize, } } diff --git a/crates/revm/revm-primitives/Cargo.toml b/crates/revm/revm-primitives/Cargo.toml index c40942e6514..d03fb908510 100644 --- a/crates/revm/revm-primitives/Cargo.toml +++ b/crates/revm/revm-primitives/Cargo.toml @@ -9,7 +9,9 @@ repository.workspace = true description = "core reth specific revm utilities" [dependencies] +# reth reth-primitives = { path = "../../primitives" } + revm = { version = "3.0.0" } [features] From 3eedcd56250725adc8b4688b5e612f617447b208 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 11 Jun 2023 14:46:56 +0200 Subject: [PATCH 016/150] feat: executor deposits --- .gitignore | 3 ++ crates/executor/Cargo.toml | 58 ++++++++++++++++++++++++ crates/interfaces/src/executor.rs | 4 ++ crates/primitives/src/transaction/mod.rs | 6 +++ crates/revm/src/executor.rs | 32 ++++++++++++- 5 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 crates/executor/Cargo.toml diff --git a/.gitignore b/.gitignore index c2c144f4a1c..2cebc21f55e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ target/ # MSVC Windows builds of rustc generate these, which store debugging information *.pdb +# Generated by VSCode +.vscode + # Generated by Intellij-based IDEs. .idea diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml new file mode 100644 index 00000000000..53acae3de9f --- /dev/null +++ b/crates/executor/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "reth-executor" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/paradigmxyz/reth" +readme = "README.md" + +[package.metadata.cargo-udeps.ignore] +normal = [ + # Used for diagrams in docs + "aquamarine", +] + +[dependencies] +# reth +reth-primitives = { path = "../primitives" } +reth-interfaces = { path = "../interfaces" } +reth-revm = { path = "../revm" } +reth-revm-inspectors = { path = "../revm/revm-inspectors" } +reth-rlp = { path = "../rlp" } +reth-db = { path = "../storage/db" } +reth-provider = { path = "../storage/provider" } + +# revm +revm = { version = "3.0.0" } + +# common +async-trait = "0.1.57" +thiserror = "1.0.37" +auto_impl = "1.0" +tracing = "0.1.37" +tokio = { version = "1.21.2", features = ["sync"] } + +# mics +aquamarine = "0.3.0" +parking_lot = { version = "0.12", optional = true } + +triehash = "0.8" +# See to replace hashers to simplify libraries +plain_hasher = "0.2" +hash-db = "0.15" +# todo replace with faster rlp impl +rlp = { version = "0.5", default-features = false } +# replace with tiny-keccak (it is faster hasher) +sha3 = { version = "0.10", default-features = false } + + +[dev-dependencies] +reth-db = { path = "../storage/db", features = ["test-utils"] } +reth-interfaces = { path = "../interfaces", features = ["test-utils"] } +reth-primitives = { path = "../primitives", features = ["test-utils"] } +reth-provider = { path = "../storage/provider", features = ["test-utils"] } +parking_lot = "0.12" + +[features] +test-utils = ["parking_lot"] +optimism = [] diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index cc747e29fdb..7dc8c0508f3 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -7,6 +7,10 @@ use thiserror::Error; pub enum BlockValidationError { #[error("EVM reported invalid transaction ({hash:?}): {message}")] EVM { hash: H256, message: String }, + #[error("Verification failed")] + VerificationFailed, + #[error("Fatal internal error")] + ExecutionFatalError, #[error("Failed to recover sender for transaction")] SenderRecoveryError, #[error("Receipt root {got:?} is different than expected {expected:?}.")] diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index d92cb5a6034..d34847e96bc 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -543,6 +543,12 @@ impl Transaction { } } + /// Returns whether or not the transaction is an Optimism Deposited transaction. + #[cfg(feature = "optimism")] + pub fn is_deposit(&self) -> bool { + matches!(self, Transaction::Deposit(_)) + } + /// Encodes EIP-155 arguments into the desired buffer. Only encodes values for legacy /// transactions. pub(crate) fn encode_eip155_fields(&self, out: &mut dyn bytes::BufMut) { diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 5ce714521fb..8efc3d8512b 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -17,7 +17,7 @@ use revm::{ db::{AccountState, CacheDB, DatabaseRef}, primitives::{ hash_map::{self, Entry}, - Account as RevmAccount, AccountInfo, ResultAndState, + Account as RevmAccount, AccountInfo, ExecutionResult, ResultAndState, }, EVM, }; @@ -240,9 +240,39 @@ where } .into()) } + + #[cfg(feature = "optimism")] + if let Some(m) = transaction.mint() { + // Add balance to the caller account equivalent to the minted amount + self.increment_account_balance(sender, U256::from(m), &mut post_state)?; + } + // Execute transaction. let ResultAndState { result, state } = self.transact(transaction, sender)?; + if transaction.is_deposit() && !matches!(result, ExecutionResult::Success { .. }) { + // If the deposit transaction failed, the deposit must still be included. + // In this case, we need to increment the sender nonce and disregard the + // state changes. The tx is invalid so it is also recorded as using all gas. + let mut acc = self.db().load_account(sender).map_err(|_| Error::ProviderError)?; + let old = to_reth_acc(&acc.info); + acc.info.nonce += 1; + let new = to_reth_acc(&acc.info); + + post_state.change_account(sender, old, new); + cumulative_gas_used += transaction.gas_limit(); + + post_state.add_receipt(Receipt { + tx_type: transaction.tx_type(), + success: false, + cumulative_gas_used, + bloom: Bloom::zero(), + logs: vec![], + }); + post_state.finish_transition(); + continue + } + // commit changes self.commit_changes( block.number, From 7bfd16b4a38a00c51f13d85db09be61607faac57 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 11 Jun 2023 21:43:32 +0200 Subject: [PATCH 017/150] feat: receipts, l1 cost wip --- crates/executor/src/lib.rs | 30 +++++ crates/interfaces/src/executor.rs | 26 +--- crates/net/eth-wire/Cargo.toml | 1 + crates/net/eth-wire/src/types/receipts.rs | 22 +++- crates/primitives/src/receipt.rs | 151 ++++++++++++++++------ crates/revm/src/executor.rs | 37 ++++-- crates/revm/src/optimism.rs | 58 +++++++++ crates/transaction-pool/src/traits.rs | 4 +- 8 files changed, 256 insertions(+), 73 deletions(-) create mode 100644 crates/executor/src/lib.rs create mode 100644 crates/revm/src/optimism.rs diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs new file mode 100644 index 00000000000..27be2283d23 --- /dev/null +++ b/crates/executor/src/lib.rs @@ -0,0 +1,30 @@ +#![warn(missing_docs, unreachable_pub)] +#![deny(unused_must_use, rust_2018_idioms)] +#![doc(test( + no_crate_inject, + attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) +))] + +//! Reth executor executes transaction in block of data. + +pub mod eth_dao_fork; +pub mod substate; + +/// Execution result types. +pub use reth_provider::post_state; + +pub mod blockchain_tree; + +/// Executor +pub mod executor; + +/// ExecutorFactory impl +pub mod factory; +pub use factory::Factory; + +#[cfg(any(test, feature = "test-utils"))] +/// Common test helpers for mocking out executor and executor factory +pub mod test_utils; + +#[cfg(feature = "optimism")] +pub mod optimism; diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 7dc8c0508f3..250286db602 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -46,26 +46,8 @@ pub enum BlockExecutionError { CanonicalRevert { inner: String }, #[error("Transaction error on commit: {inner:?}")] CanonicalCommit { inner: String }, - - // === tree errors === - // TODO(mattsse): move this to tree error - #[error("Block hash {block_hash} not found in blockchain tree chain")] - BlockHashNotFoundInChain { block_hash: BlockHash }, - #[error( - "Appending chain on fork (other_chain_fork:?) is not possible as the tip is {chain_tip:?}" - )] - AppendChainDoesntConnect { chain_tip: BlockNumHash, other_chain_fork: BlockNumHash }, - - /// Only used for TestExecutor - /// - /// Note: this is not feature gated for convenience. - #[error("Execution unavailable for tests")] - UnavailableForTest, -} - -impl BlockExecutionError { - /// Returns `true` if the error is fatal. - pub fn is_fatal(&self) -> bool { - matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. }) - } + #[error("Transaction error on pipeline status update: {inner:?}")] + PipelineStatusUpdate { inner: String }, + #[error("DB Error during transaction execution: {inner:?}")] + DBError { inner: String }, } diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index de43afd7130..3f6cf4702bf 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -64,6 +64,7 @@ proptest-derive = "0.3" default = ["serde"] serde = ["dep:serde", "smol_str/serde"] arbitrary = ["reth-primitives/arbitrary", "dep:arbitrary", "dep:proptest", "dep:proptest-derive"] +optimism = [] [[test]] name = "fuzz_roundtrip" diff --git a/crates/net/eth-wire/src/types/receipts.rs b/crates/net/eth-wire/src/types/receipts.rs index f1bb3bc1209..39eeae1e316 100644 --- a/crates/net/eth-wire/src/types/receipts.rs +++ b/crates/net/eth-wire/src/types/receipts.rs @@ -51,6 +51,9 @@ mod test { logs: vec![], }, bloom: Default::default(), + logs: vec![], + #[cfg(feature = "optimism")] + deposit_nonce: None, }]]); let mut out = vec![]; @@ -118,8 +121,9 @@ mod test { }, ], success: false, - },bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), - } + #[cfg(feature = "optimism")] + deposit_nonce: None, + }, ], ]), }; @@ -155,6 +159,20 @@ mod test { success: false, }, bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), + cumulative_gas_used: 0x1u64, + logs: vec![ + Log { + address: hex!("0000000000000000000000000000000000000011").into(), + topics: vec![ + hex!("000000000000000000000000000000000000000000000000000000000000dead").into(), + hex!("000000000000000000000000000000000000000000000000000000000000beef").into(), + ], + data: hex!("0100ff")[..].into(), + }, + ], + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, ], ]), diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 15a5a2308df..8360ada8faf 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -28,43 +28,42 @@ pub struct Receipt { ) )] pub logs: Vec, + /// Deposit nonce for Optimism deposit transactions + #[cfg(feature = "optimism")] + pub deposit_nonce: Option, } impl Receipt { - /// Calculates [`Log`]'s bloom filter. this is slow operation and [ReceiptWithBloom] can - /// be used to cache this value. - pub fn bloom_slow(&self) -> Bloom { - logs_bloom(self.logs.iter()) - } + /// Returns the rlp header for the receipt payload. + fn receipt_rlp_header(&self) -> reth_rlp::Header { + let mut rlp_head = reth_rlp::Header { list: true, payload_length: 0 }; - /// Calculates the bloom filter for the receipt and returns the [ReceiptWithBloom] container - /// type. - pub fn with_bloom(self) -> ReceiptWithBloom { - self.into() - } -} + rlp_head.payload_length += self.success.length(); + rlp_head.payload_length += self.cumulative_gas_used.length(); + rlp_head.payload_length += self.bloom.length(); + rlp_head.payload_length += self.logs.length(); + #[cfg(feature = "optimism")] + if self.tx_type == TxType::DEPOSIT { + // Transactions pre-Regolith don't have a deposit nonce + if let Some(nonce) = self.deposit_nonce { + rlp_head.payload_length += nonce.length(); + } + } -impl From for ReceiptWithBloom { - fn from(receipt: Receipt) -> Self { - let bloom = receipt.bloom_slow(); - ReceiptWithBloom { receipt, bloom } + rlp_head } -} - -/// [`Receipt`] with calculated bloom filter. -#[main_codec] -#[derive(Clone, Debug, PartialEq, Eq, Default)] -pub struct ReceiptWithBloom { - /// Bloom filter build from logs. - pub bloom: Bloom, - /// Main receipt body - pub receipt: Receipt, -} -impl ReceiptWithBloom { - /// Create new [ReceiptWithBloom] - pub fn new(receipt: Receipt, bloom: Bloom) -> Self { - Self { receipt, bloom } + /// Encodes the receipt data. + fn encode_fields(&self, out: &mut dyn BufMut) { + self.receipt_rlp_header().encode(out); + self.success.encode(out); + self.cumulative_gas_used.encode(out); + self.bloom.encode(out); + self.logs.encode(out); + #[cfg(feature = "optimism")] + if let Some(nonce) = self.deposit_nonce { + nonce.encode(out); + } } /// Consume the structure, returning only the receipt @@ -86,7 +85,33 @@ impl ReceiptWithBloom { impl ReceiptWithBloom { /// Encode receipt with or without the header data. pub fn encode_inner(&self, out: &mut dyn BufMut, with_header: bool) { - self.as_encoder().encode_inner(out, with_header) + if matches!(self.tx_type, TxType::Legacy) { + self.encode_fields(out); + return + } + + let mut payload = BytesMut::new(); + self.encode_fields(&mut payload); + + if with_header { + let payload_length = payload.len() + 1; + let header = reth_rlp::Header { list: false, payload_length }; + header.encode(out); + } + + match self.tx_type { + TxType::EIP2930 => out.put_u8(0x01), + TxType::EIP1559 => out.put_u8(0x02), + TxType::DEPOSIT => out.put_u8(0x7E), + TxType::Legacy => unreachable!("legacy handled; qed."), + } + out.put_slice(payload.as_ref()); + } + + /// Returns the length of the receipt data. + fn receipt_length(&self) -> usize { + let rlp_head = self.receipt_rlp_header(); + length_of_length(rlp_head.payload_length) + rlp_head.payload_length } /// Decodes the receipt payload @@ -98,12 +123,27 @@ impl ReceiptWithBloom { } let started_len = b.len(); - let success = reth_rlp::Decodable::decode(b)?; - let cumulative_gas_used = reth_rlp::Decodable::decode(b)?; - let bloom = Decodable::decode(b)?; - let logs = reth_rlp::Decodable::decode(b)?; + let this = match tx_type { + #[cfg(feature = "optimism")] + TxType::DEPOSIT => Self { + tx_type, + success: reth_rlp::Decodable::decode(b)?, + cumulative_gas_used: reth_rlp::Decodable::decode(b)?, + bloom: reth_rlp::Decodable::decode(b)?, + logs: reth_rlp::Decodable::decode(b)?, + deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), + }, + _ => Self { + tx_type, + success: reth_rlp::Decodable::decode(b)?, + cumulative_gas_used: reth_rlp::Decodable::decode(b)?, + bloom: reth_rlp::Decodable::decode(b)?, + logs: reth_rlp::Decodable::decode(b)?, + #[cfg(feature = "optimism")] + deposit_nonce: None, + }, + }; - let this = Self { receipt: Receipt { tx_type, success, cumulative_gas_used, logs }, bloom }; let consumed = started_len - b.len(); if consumed != rlp_head.payload_length { return Err(reth_rlp::DecodeError::ListLengthMismatch { @@ -147,6 +187,9 @@ impl Decodable for ReceiptWithBloom { } else if receipt_type == 0x02 { buf.advance(1); Self::decode_receipt(buf, TxType::EIP1559) + } else if receipt_type == 0x7E { + buf.advance(1); + Self::decode_receipt(buf, TxType::DEPOSIT) } else { Err(reth_rlp::DecodeError::Custom("invalid receipt type")) } @@ -315,6 +358,24 @@ mod tests { success: false, }, bloom: [0; 256].into(), + cumulative_gas_used: 0x1u64, + logs: vec![Log { + address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), + topics: vec![ + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000dead", + ) + .unwrap(), + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000beef", + ) + .unwrap(), + ], + data: Bytes::from_str("0100ff").unwrap().0.into(), + }], + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; receipt.encode(&mut data); @@ -351,6 +412,24 @@ mod tests { success: false, }, bloom: [0; 256].into(), + cumulative_gas_used: 0x1u64, + logs: vec![Log { + address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), + topics: vec![ + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000dead", + ) + .unwrap(), + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000beef", + ) + .unwrap(), + ], + data: Bytes::from_str("0100ff").unwrap().0.into(), + }], + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 8efc3d8512b..e5c2980c658 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -227,6 +227,9 @@ where self.init_env(&block.header, total_difficulty); + #[cfg(feature = "optimism")] + let mut l1_cost_oracle = crate::optimism::OptimismGasCostOracle::default(); + let mut cumulative_gas_used = 0; let mut post_state = PostState::with_tx_capacity(block.number, block.body.len()); for (transaction, sender) in block.body.iter().zip(senders) { @@ -250,6 +253,15 @@ where // Execute transaction. let ResultAndState { result, state } = self.transact(transaction, sender)?; + #[cfg(feature = "optimism")] + let l1_cost = l1_cost_oracle + .calculate_l1_cost(self.db(), block.header.number, transaction.clone()) + .map_err(|db_err| Error::DBError { inner: db_err.to_string() })?; + + // TODO: inject l1 cost into resulting state + // TODO: better manage the optimism flag in this function + + #[cfg(feature = "optimism")] if transaction.is_deposit() && !matches!(result, ExecutionResult::Success { .. }) { // If the deposit transaction failed, the deposit must still be included. // In this case, we need to increment the sender nonce and disregard the @@ -268,6 +280,7 @@ where cumulative_gas_used, bloom: Bloom::zero(), logs: vec![], + deposit_nonce: None, // TODO: add correct deposit nonce }); post_state.finish_transition(); continue @@ -285,18 +298,18 @@ where cumulative_gas_used += result.gas_used(); // Push transaction changeset and calculate header bloom filter for receipt. - post_state.add_receipt( - block.number, - Receipt { - tx_type: transaction.tx_type(), - // Success flag was added in `EIP-658: Embedding transaction status code in - // receipts`. - success: result.is_success(), - cumulative_gas_used, - // convert to reth log - logs: result.into_logs().into_iter().map(into_reth_log).collect(), - }, - ); + post_state.add_receipt(Receipt { + tx_type: transaction.tx_type(), + // Success flag was added in `EIP-658: Embedding transaction status code in + // receipts`. + success: result.is_success(), + cumulative_gas_used, + bloom: logs_bloom(logs.iter()), + logs, + #[cfg(feature = "optimism")] + deposit_nonce: None, // TODO: add correct deposit nonce + }); + post_state.finish_transition(); } Ok((post_state, cumulative_gas_used)) diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs new file mode 100644 index 00000000000..86b75716769 --- /dev/null +++ b/crates/revm/src/optimism.rs @@ -0,0 +1,58 @@ +use std::str::FromStr; + +use reth_primitives::{Address, TransactionSigned, U256}; +use revm::db::DatabaseRef; + +const L1_BLOCK_CONTRACT: &str = "0x4200000000000000000000000000000000000015"; +const L1_BASE_FEE_SLOT: u64 = 1; +const OVERHEAD_SLOT: u64 = 5; +const SCALAR_SLOT: u64 = 6; +const ZERO_BYTE_COST: u64 = 4; +const NON_ZERO_BYTE_COST: u64 = 16; + +/// OptimismGasCostOracle +/// +/// This is used to calculate the gas cost of a transaction on the L1 chain. +#[derive(Default)] +pub struct OptimismGasCostOracle { + /// The block number of the last time the L1 gas cost was updated. + pub block_num: u64, + /// The base fee of the L1 chain. + pub l1_base_fee: U256, + /// The overhead value used to calculate the gas cost. + pub overhead: U256, + /// The scalar value used to calculate the gas cost. + pub scalar: U256, +} + +impl OptimismGasCostOracle { + /// Calculate the gas cost of a transaction on the L1 chain. + pub fn calculate_l1_cost( + &mut self, + db: &mut DB, + block_num: u64, + tx: TransactionSigned, + ) -> Result, DB::Error> { + let rollup_data_gas_cost = U256::from(tx.input().iter().fold(0, |acc, byte| { + acc + if byte == &0x00 { ZERO_BYTE_COST } else { NON_ZERO_BYTE_COST } + })); + + if tx.is_deposit() || rollup_data_gas_cost == U256::ZERO { + return Ok(None) + } + + if block_num != self.block_num { + let l1_block_address = Address::from_str(L1_BLOCK_CONTRACT).unwrap(); + + self.l1_base_fee = db.storage(l1_block_address, U256::from(L1_BASE_FEE_SLOT))?; + self.overhead = db.storage(l1_block_address, U256::from(OVERHEAD_SLOT))?; + self.scalar = db.storage(l1_block_address, U256::from(SCALAR_SLOT))?; + } + + Ok(rollup_data_gas_cost + .saturating_add(self.overhead) + .saturating_mul(self.l1_base_fee) + .saturating_mul(self.scalar) + .checked_div(U256::from(1_000_000))) + } +} diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index bf452afd0de..2c847e16aa8 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -592,7 +592,9 @@ impl FromRecoveredTransaction for PooledTransaction { } #[cfg(feature = "optimism")] Transaction::Deposit(t) => { - // TODO: fix this gas price estimate + // Gas price is always set to 0 for deposits in order to zero out ETH refunds. + // The actual L1 cost varies depending on the gas oracle feed, and is calculated + // during tx execution. let gas_price = U256::from(0); let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); let effective_gas_price = 0u128; From 719eacae046df7bcc535fe9cbd16d0c5e9e29d53 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 11 Jun 2023 22:02:13 +0200 Subject: [PATCH 018/150] chore: small fixes --- crates/executor/src/lib.rs | 1 + crates/primitives/src/proofs.rs | 3 +++ crates/revm/src/executor.rs | 2 +- crates/revm/src/optimism.rs | 18 +++++++++++------- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 27be2283d23..c4e4b13664e 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -27,4 +27,5 @@ pub use factory::Factory; pub mod test_utils; #[cfg(feature = "optimism")] +/// Optimism-specific utilities for the executor pub mod optimism; diff --git a/crates/primitives/src/proofs.rs b/crates/primitives/src/proofs.rs index 95fd279d12b..8e90eeb970f 100644 --- a/crates/primitives/src/proofs.rs +++ b/crates/primitives/src/proofs.rs @@ -173,6 +173,9 @@ mod tests { logs, }, bloom, + logs, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; let receipt = vec![receipt]; let root = calculate_receipt_root(&receipt); diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index e5c2980c658..8b01d9f1b44 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -259,7 +259,7 @@ where .map_err(|db_err| Error::DBError { inner: db_err.to_string() })?; // TODO: inject l1 cost into resulting state - // TODO: better manage the optimism flag in this function + // TODO: fix the use of the optimism flag in this function #[cfg(feature = "optimism")] if transaction.is_deposit() && !matches!(result, ExecutionResult::Success { .. }) { diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 86b75716769..d27a9a4cc7d 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -10,13 +10,18 @@ const SCALAR_SLOT: u64 = 6; const ZERO_BYTE_COST: u64 = 4; const NON_ZERO_BYTE_COST: u64 = 16; -/// OptimismGasCostOracle +/// Optimism Gas Cost Oracle /// -/// This is used to calculate the gas cost of a transaction on the L1 chain. +/// This struct is used to calculate the gas cost of a transaction on the L1 chain. +/// The L1 block contract on L2 is used to propagate changes to the gas cost over time, +/// so we need to fetch it from the Reth database. +/// +/// To make this more efficient, we cache the values and only fetch them when the block +/// number changes. #[derive(Default)] pub struct OptimismGasCostOracle { - /// The block number of the last time the L1 gas cost was updated. - pub block_num: u64, + /// The cached block number + pub block_number: Option, /// The base fee of the L1 chain. pub l1_base_fee: U256, /// The overhead value used to calculate the gas cost. @@ -26,7 +31,7 @@ pub struct OptimismGasCostOracle { } impl OptimismGasCostOracle { - /// Calculate the gas cost of a transaction on the L1 chain. + /// Calculate the gas cost of a transaction based on L1 block data posted on L2 pub fn calculate_l1_cost( &mut self, db: &mut DB, @@ -41,9 +46,8 @@ impl OptimismGasCostOracle { return Ok(None) } - if block_num != self.block_num { + if self.block_number.cmp(&Some(block_num)).is_ne() { let l1_block_address = Address::from_str(L1_BLOCK_CONTRACT).unwrap(); - self.l1_base_fee = db.storage(l1_block_address, U256::from(L1_BASE_FEE_SLOT))?; self.overhead = db.storage(l1_block_address, U256::from(OVERHEAD_SLOT))?; self.scalar = db.storage(l1_block_address, U256::from(SCALAR_SLOT))?; From d0fb3e162b52145ad10486d26b58b2252a8cbe7c Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:48:03 +0200 Subject: [PATCH 019/150] wip: transact() optimism changes --- crates/interfaces/Cargo.toml | 5 +-- crates/interfaces/src/executor.rs | 5 +++ crates/revm/src/executor.rs | 55 ++++++++++++++++++++++--------- crates/revm/src/optimism.rs | 27 ++++++++------- 4 files changed, 63 insertions(+), 29 deletions(-) diff --git a/crates/interfaces/Cargo.toml b/crates/interfaces/Cargo.toml index be012c7aaf5..016330402f7 100644 --- a/crates/interfaces/Cargo.toml +++ b/crates/interfaces/Cargo.toml @@ -53,5 +53,6 @@ secp256k1 = { workspace = true, features = [ ] } [features] -test-utils = ["tokio-stream/sync", "secp256k1", "rand/std_rng"] -cli = ["clap"] \ No newline at end of file +bench = [] +optimism = [] +test-utils = ["tokio-stream/sync", "secp256k1"] diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 250286db602..242bc3ed9e5 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -48,6 +48,11 @@ pub enum BlockExecutionError { CanonicalCommit { inner: String }, #[error("Transaction error on pipeline status update: {inner:?}")] PipelineStatusUpdate { inner: String }, + + #[cfg(feature = "optimism")] #[error("DB Error during transaction execution: {inner:?}")] DBError { inner: String }, + #[cfg(feature = "optimism")] + #[error("Insufficient funds to cover transaction cost: want {want}, have {have}")] + InsufficientFunds { want: u64, have: u64 }, } diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 8b01d9f1b44..8558fbddaa3 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -245,31 +245,56 @@ where } #[cfg(feature = "optimism")] - if let Some(m) = transaction.mint() { - // Add balance to the caller account equivalent to the minted amount - self.increment_account_balance(sender, U256::from(m), &mut post_state)?; + { + if let Some(m) = transaction.mint() { + // Add balance to the caller account equivalent to the minted amount + self.increment_account_balance(sender, U256::from(m), &mut post_state)?; + } + + // Check if the sender balance can cover the total cost including L1 cost + // total cost: gas_limit * gas_price + l1_cost + let db = self.db(); + let l1_cost = l1_cost_oracle + .calculate_l1_cost(db, block.header.number, transaction.clone()) + .map_err(|db_err| Error::DBError { inner: db_err.to_string() })?; + + // TODO: check if max_fee_per_gas works for 1559 txs here, + // for legacy this is the same as gas_price and for deposit this is 0. + // If this doesn't work we can replicate revm's effective_gas_price logic here + let total_cost = U256::from(transaction.gas_limit()) + .saturating_mul(U256::from(transaction.max_fee_per_gas())) + .saturating_add(U256::from(l1_cost)); + + let sender_account = db.load_account(sender).map_err(|_| Error::ProviderError)?; + if sender_account.info.balance.cmp(&total_cost) == std::cmp::Ordering::Less { + return Err(Error::InsufficientFunds { + have: sender_account.info.balance.to::(), + want: total_cost.to::(), + }) + } + + // Safely take l1_cost from sender (the rest will be deducted by the + // EVM execution and included in result.gas_used()) + // TODO: handle calls with `disable_balance_check` flag set? + let old = to_reth_acc(&sender_account.info); + sender_account.info.balance -= l1_cost; + let new = to_reth_acc(&sender_account.info); + post_state.change_account(sender, old, new); } // Execute transaction. let ResultAndState { result, state } = self.transact(transaction, sender)?; - #[cfg(feature = "optimism")] - let l1_cost = l1_cost_oracle - .calculate_l1_cost(self.db(), block.header.number, transaction.clone()) - .map_err(|db_err| Error::DBError { inner: db_err.to_string() })?; - - // TODO: inject l1 cost into resulting state - // TODO: fix the use of the optimism flag in this function - #[cfg(feature = "optimism")] if transaction.is_deposit() && !matches!(result, ExecutionResult::Success { .. }) { // If the deposit transaction failed, the deposit must still be included. // In this case, we need to increment the sender nonce and disregard the // state changes. The tx is invalid so it is also recorded as using all gas. - let mut acc = self.db().load_account(sender).map_err(|_| Error::ProviderError)?; - let old = to_reth_acc(&acc.info); - acc.info.nonce += 1; - let new = to_reth_acc(&acc.info); + let sender_account = + self.db().load_account(sender).map_err(|_| Error::ProviderError)?; + let old = to_reth_acc(&sender_account.info); + sender_account.info.nonce += 1; + let new = to_reth_acc(&sender_account.info); post_state.change_account(sender, old, new); cumulative_gas_used += transaction.gas_limit(); diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index d27a9a4cc7d..ba07ff63fb7 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -16,18 +16,18 @@ const NON_ZERO_BYTE_COST: u64 = 16; /// The L1 block contract on L2 is used to propagate changes to the gas cost over time, /// so we need to fetch it from the Reth database. /// -/// To make this more efficient, we cache the values and only fetch them when the block -/// number changes. +/// To make this slightly more efficient, we cache the values and only fetch them again +/// when the block number changes. #[derive(Default)] pub struct OptimismGasCostOracle { /// The cached block number - pub block_number: Option, - /// The base fee of the L1 chain. - pub l1_base_fee: U256, - /// The overhead value used to calculate the gas cost. - pub overhead: U256, - /// The scalar value used to calculate the gas cost. - pub scalar: U256, + block_number: Option, + /// The base fee of the L1 chain + l1_base_fee: U256, + /// The overhead value used to calculate the gas cost + overhead: U256, + /// The scalar value used to calculate the gas cost + scalar: U256, } impl OptimismGasCostOracle { @@ -37,17 +37,19 @@ impl OptimismGasCostOracle { db: &mut DB, block_num: u64, tx: TransactionSigned, - ) -> Result, DB::Error> { + ) -> Result { let rollup_data_gas_cost = U256::from(tx.input().iter().fold(0, |acc, byte| { acc + if byte == &0x00 { ZERO_BYTE_COST } else { NON_ZERO_BYTE_COST } })); if tx.is_deposit() || rollup_data_gas_cost == U256::ZERO { - return Ok(None) + return Ok(U256::ZERO) } if self.block_number.cmp(&Some(block_num)).is_ne() { let l1_block_address = Address::from_str(L1_BLOCK_CONTRACT).unwrap(); + + // TODO: what block is the DB info last written in? Is this always up-to-date? self.l1_base_fee = db.storage(l1_block_address, U256::from(L1_BASE_FEE_SLOT))?; self.overhead = db.storage(l1_block_address, U256::from(OVERHEAD_SLOT))?; self.scalar = db.storage(l1_block_address, U256::from(SCALAR_SLOT))?; @@ -57,6 +59,7 @@ impl OptimismGasCostOracle { .saturating_add(self.overhead) .saturating_mul(self.l1_base_fee) .saturating_mul(self.scalar) - .checked_div(U256::from(1_000_000))) + .checked_div(U256::from(1_000_000)) + .unwrap_or_default()) } } From 77556daf26e728dea2f2b446c377868ce24eb459 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:55:00 +0200 Subject: [PATCH 020/150] chore: minor changes related to optimism execution --- crates/revm/src/executor.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 8558fbddaa3..fb934e12d57 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -246,11 +246,6 @@ where #[cfg(feature = "optimism")] { - if let Some(m) = transaction.mint() { - // Add balance to the caller account equivalent to the minted amount - self.increment_account_balance(sender, U256::from(m), &mut post_state)?; - } - // Check if the sender balance can cover the total cost including L1 cost // total cost: gas_limit * gas_price + l1_cost let db = self.db(); @@ -258,6 +253,13 @@ where .calculate_l1_cost(db, block.header.number, transaction.clone()) .map_err(|db_err| Error::DBError { inner: db_err.to_string() })?; + let sender_account = db.load_account(sender).map_err(|_| Error::ProviderError)?; + let old_sender_info = to_reth_acc(&sender_account.info); + if let Some(m) = transaction.mint() { + // Add balance to the caller account equivalent to the minted amount + sender_account.info.balance += U256::from(m); + } + // TODO: check if max_fee_per_gas works for 1559 txs here, // for legacy this is the same as gas_price and for deposit this is 0. // If this doesn't work we can replicate revm's effective_gas_price logic here @@ -265,7 +267,6 @@ where .saturating_mul(U256::from(transaction.max_fee_per_gas())) .saturating_add(U256::from(l1_cost)); - let sender_account = db.load_account(sender).map_err(|_| Error::ProviderError)?; if sender_account.info.balance.cmp(&total_cost) == std::cmp::Ordering::Less { return Err(Error::InsufficientFunds { have: sender_account.info.balance.to::(), @@ -275,11 +276,11 @@ where // Safely take l1_cost from sender (the rest will be deducted by the // EVM execution and included in result.gas_used()) - // TODO: handle calls with `disable_balance_check` flag set? - let old = to_reth_acc(&sender_account.info); + // TODO: need to handle calls with `disable_balance_check` flag set? sender_account.info.balance -= l1_cost; - let new = to_reth_acc(&sender_account.info); - post_state.change_account(sender, old, new); + + let new_sender_info = to_reth_acc(&sender_account.info); + post_state.change_account(sender, old_sender_info, new_sender_info); } // Execute transaction. From 9bf71657968c9dbdcc5bb3463c3db8947278f900 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sat, 17 Jun 2023 22:25:28 +0200 Subject: [PATCH 021/150] wip: removed receipts changes + improved gas handling --- crates/interfaces/src/executor.rs | 4 +-- crates/primitives/src/receipt.rs | 21 ------------ crates/revm/revm-primitives/src/env.rs | 1 + crates/revm/src/executor.rs | 47 ++++++++++++-------------- crates/revm/src/optimism.rs | 12 +++---- 5 files changed, 30 insertions(+), 55 deletions(-) diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 242bc3ed9e5..d0c35209fdb 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -53,6 +53,6 @@ pub enum BlockExecutionError { #[error("DB Error during transaction execution: {inner:?}")] DBError { inner: String }, #[cfg(feature = "optimism")] - #[error("Insufficient funds to cover transaction cost: want {want}, have {have}")] - InsufficientFunds { want: u64, have: u64 }, + #[error("Insufficient funds to cover transaction L1 cost: want {want}, have {have}")] + InsufficientFundsForL1Cost { want: u64, have: u64 }, } diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 8360ada8faf..7995ee17a92 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -28,9 +28,6 @@ pub struct Receipt { ) )] pub logs: Vec, - /// Deposit nonce for Optimism deposit transactions - #[cfg(feature = "optimism")] - pub deposit_nonce: Option, } impl Receipt { @@ -42,13 +39,6 @@ impl Receipt { rlp_head.payload_length += self.cumulative_gas_used.length(); rlp_head.payload_length += self.bloom.length(); rlp_head.payload_length += self.logs.length(); - #[cfg(feature = "optimism")] - if self.tx_type == TxType::DEPOSIT { - // Transactions pre-Regolith don't have a deposit nonce - if let Some(nonce) = self.deposit_nonce { - rlp_head.payload_length += nonce.length(); - } - } rlp_head } @@ -60,10 +50,6 @@ impl Receipt { self.cumulative_gas_used.encode(out); self.bloom.encode(out); self.logs.encode(out); - #[cfg(feature = "optimism")] - if let Some(nonce) = self.deposit_nonce { - nonce.encode(out); - } } /// Consume the structure, returning only the receipt @@ -131,7 +117,6 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, - deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), }, _ => Self { tx_type, @@ -139,8 +124,6 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, - #[cfg(feature = "optimism")] - deposit_nonce: None, }, }; @@ -374,8 +357,6 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; receipt.encode(&mut data); @@ -428,8 +409,6 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index e590387a7b3..be2702cc7c0 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -219,6 +219,7 @@ where Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { tx_env.gas_limit = *gas_limit; if let Some(m) = mint { + // TODO: why is this not zero? tx_env.gas_price = U256::from(*m); } else { tx_env.gas_price = U256::ZERO; diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index fb934e12d57..3e7d95830b6 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -26,6 +26,9 @@ use std::{ sync::Arc, }; +#[cfg(feature = "optimism")] +use crate::optimism::OptimismGasCostOracle; + /// Main block executor pub struct Executor where @@ -228,7 +231,7 @@ where self.init_env(&block.header, total_difficulty); #[cfg(feature = "optimism")] - let mut l1_cost_oracle = crate::optimism::OptimismGasCostOracle::default(); + let mut l1_cost_oracle = OptimismGasCostOracle::default(); let mut cumulative_gas_used = 0; let mut post_state = PostState::with_tx_capacity(block.number, block.body.len()); @@ -246,38 +249,33 @@ where #[cfg(feature = "optimism")] { - // Check if the sender balance can cover the total cost including L1 cost - // total cost: gas_limit * gas_price + l1_cost + // Check if the sender balance can cover the L1 cost let db = self.db(); let l1_cost = l1_cost_oracle - .calculate_l1_cost(db, block.header.number, transaction.clone()) + .calculate_l1_cost(db, block.header.number, transaction) .map_err(|db_err| Error::DBError { inner: db_err.to_string() })?; let sender_account = db.load_account(sender).map_err(|_| Error::ProviderError)?; let old_sender_info = to_reth_acc(&sender_account.info); if let Some(m) = transaction.mint() { - // Add balance to the caller account equivalent to the minted amount + // Add balance to the caller account equal to the minted amount. + // Note: this is unconditional, and will not be reverted if the tx fails. sender_account.info.balance += U256::from(m); } - // TODO: check if max_fee_per_gas works for 1559 txs here, - // for legacy this is the same as gas_price and for deposit this is 0. - // If this doesn't work we can replicate revm's effective_gas_price logic here - let total_cost = U256::from(transaction.gas_limit()) - .saturating_mul(U256::from(transaction.max_fee_per_gas())) - .saturating_add(U256::from(l1_cost)); - - if sender_account.info.balance.cmp(&total_cost) == std::cmp::Ordering::Less { - return Err(Error::InsufficientFunds { - have: sender_account.info.balance.to::(), - want: total_cost.to::(), - }) - } + if !transaction.is_deposit() { + if sender_account.info.balance.cmp(&l1_cost) == std::cmp::Ordering::Less { + return Err(Error::InsufficientFundsForL1Cost { + have: sender_account.info.balance.to::(), + want: l1_cost.to::(), + }) + } - // Safely take l1_cost from sender (the rest will be deducted by the - // EVM execution and included in result.gas_used()) - // TODO: need to handle calls with `disable_balance_check` flag set? - sender_account.info.balance -= l1_cost; + // Safely take l1_cost from sender (the rest will be deducted by the + // internal EVM execution and included in result.gas_used()) + // TODO: need to handle calls with `disable_balance_check` flag set? + sender_account.info.balance -= l1_cost; + } let new_sender_info = to_reth_acc(&sender_account.info); post_state.change_account(sender, old_sender_info, new_sender_info); @@ -288,7 +286,7 @@ where #[cfg(feature = "optimism")] if transaction.is_deposit() && !matches!(result, ExecutionResult::Success { .. }) { - // If the deposit transaction failed, the deposit must still be included. + // If the Deposited transaction failed, the deposit must still be included. // In this case, we need to increment the sender nonce and disregard the // state changes. The tx is invalid so it is also recorded as using all gas. let sender_account = @@ -306,7 +304,6 @@ where cumulative_gas_used, bloom: Bloom::zero(), logs: vec![], - deposit_nonce: None, // TODO: add correct deposit nonce }); post_state.finish_transition(); continue @@ -332,8 +329,6 @@ where cumulative_gas_used, bloom: logs_bloom(logs.iter()), logs, - #[cfg(feature = "optimism")] - deposit_nonce: None, // TODO: add correct deposit nonce }); post_state.finish_transition(); } diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index ba07ff63fb7..9eae8c8256e 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -12,12 +12,12 @@ const NON_ZERO_BYTE_COST: u64 = 16; /// Optimism Gas Cost Oracle /// -/// This struct is used to calculate the gas cost of a transaction on the L1 chain. +/// This is used to calculate the gas cost of a transaction on the L1 chain. /// The L1 block contract on L2 is used to propagate changes to the gas cost over time, -/// so we need to fetch it from the Reth database. +/// so we need to fetch this data from the DB. /// -/// To make this slightly more efficient, we cache the values and only fetch them again -/// when the block number changes. +/// To make this slightly more efficient when executing entire blocks, the values are cached and +/// only fetched again when the block number changes. #[derive(Default)] pub struct OptimismGasCostOracle { /// The cached block number @@ -36,7 +36,7 @@ impl OptimismGasCostOracle { &mut self, db: &mut DB, block_num: u64, - tx: TransactionSigned, + tx: &TransactionSigned, ) -> Result { let rollup_data_gas_cost = U256::from(tx.input().iter().fold(0, |acc, byte| { acc + if byte == &0x00 { ZERO_BYTE_COST } else { NON_ZERO_BYTE_COST } @@ -49,7 +49,7 @@ impl OptimismGasCostOracle { if self.block_number.cmp(&Some(block_num)).is_ne() { let l1_block_address = Address::from_str(L1_BLOCK_CONTRACT).unwrap(); - // TODO: what block is the DB info last written in? Is this always up-to-date? + // TODO: Check: Is the db data always up-to-date? self.l1_base_fee = db.storage(l1_block_address, U256::from(L1_BASE_FEE_SLOT))?; self.overhead = db.storage(l1_block_address, U256::from(OVERHEAD_SLOT))?; self.scalar = db.storage(l1_block_address, U256::from(SCALAR_SLOT))?; From b6e0704a34b15bfc777858c7c238c291242a85b3 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 18 Jun 2023 09:36:26 +0200 Subject: [PATCH 022/150] chore: cleanup --- crates/primitives/src/receipt.rs | 52 ++++++++++++++++++++++++-------- crates/revm/src/executor.rs | 23 +++++++++++--- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 7995ee17a92..ba0c43c5736 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -28,6 +28,9 @@ pub struct Receipt { ) )] pub logs: Vec, + /// Deposit nonce for Optimism deposited transactions + #[cfg(feature = "optimism")] + pub deposit_nonce: Option, } impl Receipt { @@ -39,6 +42,13 @@ impl Receipt { rlp_head.payload_length += self.cumulative_gas_used.length(); rlp_head.payload_length += self.bloom.length(); rlp_head.payload_length += self.logs.length(); + #[cfg(feature = "optimism")] + if self.tx_type == TxType::DEPOSIT { + // Transactions pre-Regolith don't have a deposit nonce + if let Some(nonce) = self.deposit_nonce { + rlp_head.payload_length += nonce.length(); + } + } rlp_head } @@ -50,6 +60,10 @@ impl Receipt { self.cumulative_gas_used.encode(out); self.bloom.encode(out); self.logs.encode(out); + #[cfg(feature = "optimism")] + if let Some(nonce) = self.deposit_nonce { + nonce.encode(out); + } } /// Consume the structure, returning only the receipt @@ -88,8 +102,10 @@ impl ReceiptWithBloom { match self.tx_type { TxType::EIP2930 => out.put_u8(0x01), TxType::EIP1559 => out.put_u8(0x02), - TxType::DEPOSIT => out.put_u8(0x7E), TxType::Legacy => unreachable!("legacy handled; qed."), + + #[cfg(feature = "optimism")] + TxType::DEPOSIT => out.put_u8(0x7E), } out.put_slice(payload.as_ref()); } @@ -117,6 +133,7 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, + deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), }, _ => Self { tx_type, @@ -124,6 +141,8 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, }; @@ -164,17 +183,22 @@ impl Decodable for ReceiptWithBloom { let receipt_type = *buf.first().ok_or(reth_rlp::DecodeError::Custom( "typed receipt cannot be decoded from an empty slice", ))?; - if receipt_type == 0x01 { - buf.advance(1); - Self::decode_receipt(buf, TxType::EIP2930) - } else if receipt_type == 0x02 { - buf.advance(1); - Self::decode_receipt(buf, TxType::EIP1559) - } else if receipt_type == 0x7E { - buf.advance(1); - Self::decode_receipt(buf, TxType::DEPOSIT) - } else { - Err(reth_rlp::DecodeError::Custom("invalid receipt type")) + + match receipt_type { + 0x01 => { + buf.advance(1); + Self::decode_receipt(buf, TxType::EIP2930) + } + 0x02 => { + buf.advance(1); + Self::decode_receipt(buf, TxType::EIP1559) + } + #[cfg(feature = "optimism")] + 0x7E => { + buf.advance(1); + Self::decode_receipt(buf, TxType::DEPOSIT) + } + _ => Err(reth_rlp::DecodeError::Custom("invalid receipt type")), } } Ordering::Equal => { @@ -357,6 +381,8 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; receipt.encode(&mut data); @@ -409,6 +435,8 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 3e7d95830b6..e248de5128b 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -291,11 +291,11 @@ where // state changes. The tx is invalid so it is also recorded as using all gas. let sender_account = self.db().load_account(sender).map_err(|_| Error::ProviderError)?; - let old = to_reth_acc(&sender_account.info); + let old_sender_info = to_reth_acc(&sender_account.info); sender_account.info.nonce += 1; - let new = to_reth_acc(&sender_account.info); + let new_sender_info = to_reth_acc(&sender_account.info); - post_state.change_account(sender, old, new); + post_state.change_account(sender, old_sender_info, new_sender_info); cumulative_gas_used += transaction.gas_limit(); post_state.add_receipt(Receipt { @@ -304,6 +304,8 @@ where cumulative_gas_used, bloom: Bloom::zero(), logs: vec![], + #[cfg(feature = "optimism")] + deposit_nonce: Some(transaction.nonce()), }); post_state.finish_transition(); continue @@ -317,8 +319,17 @@ where &mut post_state, ); - // append gas used - cumulative_gas_used += result.gas_used(); + #[cfg(feature = "optimism")] + if transaction.is_deposit() { + cumulative_gas_used += transaction.gas_limit() + } else { + cumulative_gas_used += result.gas_used() + } + + #[cfg(not(feature = "optimism"))] + { + cumulative_gas_used += result.gas_used(); + } // Push transaction changeset and calculate header bloom filter for receipt. post_state.add_receipt(Receipt { @@ -329,6 +340,8 @@ where cumulative_gas_used, bloom: logs_bloom(logs.iter()), logs, + #[cfg(feature = "optimism")] + deposit_nonce: Some(transaction.nonce()), }); post_state.finish_transition(); } From 8f8ae048f3986f12f1485c3dacb85ebf5839f8ec Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Mon, 19 Jun 2023 19:50:17 +0200 Subject: [PATCH 023/150] chore: system_tx and is_success method --- crates/revm/src/executor.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index e248de5128b..34cad72144f 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -17,7 +17,7 @@ use revm::{ db::{AccountState, CacheDB, DatabaseRef}, primitives::{ hash_map::{self, Entry}, - Account as RevmAccount, AccountInfo, ExecutionResult, ResultAndState, + Account as RevmAccount, AccountInfo, ResultAndState, }, EVM, }; @@ -285,7 +285,7 @@ where let ResultAndState { result, state } = self.transact(transaction, sender)?; #[cfg(feature = "optimism")] - if transaction.is_deposit() && !matches!(result, ExecutionResult::Success { .. }) { + if transaction.is_deposit() && !result.is_success() { // If the Deposited transaction failed, the deposit must still be included. // In this case, we need to increment the sender nonce and disregard the // state changes. The tx is invalid so it is also recorded as using all gas. @@ -320,9 +320,9 @@ where ); #[cfg(feature = "optimism")] - if transaction.is_deposit() { - cumulative_gas_used += transaction.gas_limit() - } else { + if !transaction.is_system_transaction() { + // After Regolith, deposits are reported as using the actual gas used instead of + // all the gas. System transactions are not reported as using any gas. cumulative_gas_used += result.gas_used() } From 96d5249a5a76e48db139c23547cccb2ab8014dba Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Tue, 20 Jun 2023 22:41:25 +0200 Subject: [PATCH 024/150] feat: refactoring: separated optimism feature + added routing fees to vaults --- crates/revm/src/executor.rs | 150 +++++++++++++++++++++++------------- crates/revm/src/optimism.rs | 33 +++++++- 2 files changed, 128 insertions(+), 55 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 34cad72144f..11dd98b1e03 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -27,7 +27,7 @@ use std::{ }; #[cfg(feature = "optimism")] -use crate::optimism::OptimismGasCostOracle; +use crate::optimism; /// Main block executor pub struct Executor @@ -231,7 +231,7 @@ where self.init_env(&block.header, total_difficulty); #[cfg(feature = "optimism")] - let mut l1_cost_oracle = OptimismGasCostOracle::default(); + let mut l1_cost_oracle = optimism::L1GasCostOracle::default(); let mut cumulative_gas_used = 0; let mut post_state = PostState::with_tx_capacity(block.number, block.body.len()); @@ -249,7 +249,6 @@ where #[cfg(feature = "optimism")] { - // Check if the sender balance can cover the L1 cost let db = self.db(); let l1_cost = l1_cost_oracle .calculate_l1_cost(db, block.header.number, transaction) @@ -259,10 +258,13 @@ where let old_sender_info = to_reth_acc(&sender_account.info); if let Some(m) = transaction.mint() { // Add balance to the caller account equal to the minted amount. - // Note: this is unconditional, and will not be reverted if the tx fails. + // Note: this is unconditional, and will not be reverted if the tx fails + // (unless the block can't be built at all due to gas limit constraints) sender_account.info.balance += U256::from(m); } + // Check if the sender balance can cover the L1 cost. + // Deposits pay for their gas directly on L1 so they are exempt from this if !transaction.is_deposit() { if sender_account.info.balance.cmp(&l1_cost) == std::cmp::Ordering::Less { return Err(Error::InsufficientFundsForL1Cost { @@ -279,71 +281,113 @@ where let new_sender_info = to_reth_acc(&sender_account.info); post_state.change_account(sender, old_sender_info, new_sender_info); - } - // Execute transaction. - let ResultAndState { result, state } = self.transact(transaction, sender)?; + // Execute transaction. + let ResultAndState { result, state } = self.transact(transaction, sender)?; + + if transaction.is_deposit() && !result.is_success() { + // If the Deposited transaction failed, the deposit must still be included. + // In this case, we need to increment the sender nonce and disregard the + // state changes. The transaction is also recorded as using all gas. + let db = self.db(); + let sender_account = + db.load_account(sender).map_err(|_| Error::ProviderError)?; + let old_sender_info = to_reth_acc(&sender_account.info); + sender_account.info.nonce += 1; + let new_sender_info = to_reth_acc(&sender_account.info); + + post_state.change_account(sender, old_sender_info, new_sender_info); + if !transaction.is_system_transaction() { + cumulative_gas_used += transaction.gas_limit(); + } - #[cfg(feature = "optimism")] - if transaction.is_deposit() && !result.is_success() { - // If the Deposited transaction failed, the deposit must still be included. - // In this case, we need to increment the sender nonce and disregard the - // state changes. The tx is invalid so it is also recorded as using all gas. - let sender_account = - self.db().load_account(sender).map_err(|_| Error::ProviderError)?; - let old_sender_info = to_reth_acc(&sender_account.info); - sender_account.info.nonce += 1; - let new_sender_info = to_reth_acc(&sender_account.info); + post_state.add_receipt(Receipt { + tx_type: transaction.tx_type(), + success: false, + cumulative_gas_used, + bloom: Bloom::zero(), + logs: vec![], + deposit_nonce: Some(transaction.nonce()), + }); + post_state.finish_transition(); + continue + } - post_state.change_account(sender, old_sender_info, new_sender_info); - cumulative_gas_used += transaction.gas_limit(); + // commit changes + self.commit_changes( + state, + self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), + &mut post_state, + ); + + if !transaction.is_system_transaction() { + // After Regolith, deposits are reported as using the actual gas used instead of + // all the gas. System transactions are not reported as using any gas. + cumulative_gas_used += result.gas_used() + } + // Route base fee and l1 cost to the appropriate optimism vaults + let db = self.db(); + optimism::route_fee_to_vault( + db, + block.base_fee_per_gas.unwrap_or_default(), + result.gas_used(), + optimism::get_base_fee_recipient(), + )?; + optimism::route_fee_to_vault( + db, + l1_cost.to::(), + result.gas_used(), + optimism::get_l1_fee_recipient(), + )?; + + // cast revm logs to reth logs + let logs: Vec = result.logs().into_iter().map(into_reth_log).collect(); + + // Push transaction changeset and calculate header bloom filter for receipt. post_state.add_receipt(Receipt { tx_type: transaction.tx_type(), - success: false, + // Success flag was added in `EIP-658: Embedding transaction status code in + // receipts`. + success: result.is_success(), cumulative_gas_used, - bloom: Bloom::zero(), - logs: vec![], - #[cfg(feature = "optimism")] + bloom: logs_bloom(logs.iter()), + logs, deposit_nonce: Some(transaction.nonce()), }); post_state.finish_transition(); - continue - } - - // commit changes - self.commit_changes( - block.number, - state, - self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), - &mut post_state, - ); - - #[cfg(feature = "optimism")] - if !transaction.is_system_transaction() { - // After Regolith, deposits are reported as using the actual gas used instead of - // all the gas. System transactions are not reported as using any gas. - cumulative_gas_used += result.gas_used() } #[cfg(not(feature = "optimism"))] { + // Execute transaction. + let ResultAndState { result, state } = self.transact(transaction, sender)?; + + // commit changes + self.commit_changes( + state, + self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), + &mut post_state, + ); + cumulative_gas_used += result.gas_used(); - } - // Push transaction changeset and calculate header bloom filter for receipt. - post_state.add_receipt(Receipt { - tx_type: transaction.tx_type(), - // Success flag was added in `EIP-658: Embedding transaction status code in - // receipts`. - success: result.is_success(), - cumulative_gas_used, - bloom: logs_bloom(logs.iter()), - logs, - #[cfg(feature = "optimism")] - deposit_nonce: Some(transaction.nonce()), - }); - post_state.finish_transition(); + // cast revm logs to reth logs + let logs: Vec = result.logs().into_iter().map(into_reth_log).collect(); + + // Push transaction changeset and calculate header bloom filter for receipt. + post_state.add_receipt(Receipt { + tx_type: transaction.tx_type(), + // Success flag was added in `EIP-658: Embedding transaction status code in + // receipts`. + success: result.is_success(), + cumulative_gas_used, + bloom: logs_bloom(logs.iter()), + logs, + deposit_nonce: Some(transaction.nonce()), + }); + post_state.finish_transition(); + } } Ok((post_state, cumulative_gas_used)) diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 9eae8c8256e..5d62a7084ce 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -1,9 +1,15 @@ use std::str::FromStr; +use reth_interfaces::executor::Error; use reth_primitives::{Address, TransactionSigned, U256}; +use reth_provider::StateProvider; +use reth_revm::database::SubState; use revm::db::DatabaseRef; +const L1_FEE_RECIPIENT: &str = "0x420000000000000000000000000000000000001A"; +const BASE_FEE_RECIPIENT: &str = "0x4200000000000000000000000000000000000019"; const L1_BLOCK_CONTRACT: &str = "0x4200000000000000000000000000000000000015"; + const L1_BASE_FEE_SLOT: u64 = 1; const OVERHEAD_SLOT: u64 = 5; const SCALAR_SLOT: u64 = 6; @@ -19,7 +25,7 @@ const NON_ZERO_BYTE_COST: u64 = 16; /// To make this slightly more efficient when executing entire blocks, the values are cached and /// only fetched again when the block number changes. #[derive(Default)] -pub struct OptimismGasCostOracle { +pub struct L1GasCostOracle { /// The cached block number block_number: Option, /// The base fee of the L1 chain @@ -30,7 +36,7 @@ pub struct OptimismGasCostOracle { scalar: U256, } -impl OptimismGasCostOracle { +impl L1GasCostOracle { /// Calculate the gas cost of a transaction based on L1 block data posted on L2 pub fn calculate_l1_cost( &mut self, @@ -63,3 +69,26 @@ impl OptimismGasCostOracle { .unwrap_or_default()) } } + +/// Get the L1 fee recipient address +pub fn get_l1_fee_recipient() -> Address { + Address::from_str(L1_FEE_RECIPIENT).unwrap() +} + +/// Get the base fee recipient address +pub fn get_base_fee_recipient() -> Address { + Address::from_str(BASE_FEE_RECIPIENT).unwrap() +} + +/// Route any fee to a recipient account +pub fn route_fee_to_vault( + db: &mut SubState, + fee: u64, + gas_used: u64, + recipient: Address, +) -> Result<(), Error> { + let mut account = db.load_account(recipient).map_err(|_| Error::ProviderError)?; + let amount_to_send = U256::from(fee.saturating_add(gas_used)); + account.info.balance = account.info.balance.saturating_add(amount_to_send); + Ok(()) +} From c9fc38b6210f79f63bea0c58945da85d58c38a54 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Tue, 20 Jun 2023 23:01:26 +0200 Subject: [PATCH 025/150] fix: fee vault calc --- crates/revm/src/executor.rs | 12 +++--------- crates/revm/src/optimism.rs | 33 +++++++++++++++++---------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 11dd98b1e03..f1f9ec1a428 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -326,19 +326,13 @@ where cumulative_gas_used += result.gas_used() } - // Route base fee and l1 cost to the appropriate optimism vaults + // Route the l1 cost and base fee to the appropriate optimism vaults let db = self.db(); - optimism::route_fee_to_vault( + optimism::route_l1_cost_to_vault(db, l1_cost)?; + optimism::route_base_fee_to_vault( db, block.base_fee_per_gas.unwrap_or_default(), result.gas_used(), - optimism::get_base_fee_recipient(), - )?; - optimism::route_fee_to_vault( - db, - l1_cost.to::(), - result.gas_used(), - optimism::get_l1_fee_recipient(), )?; // cast revm logs to reth logs diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 5d62a7084ce..12d617820e7 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -70,25 +70,26 @@ impl L1GasCostOracle { } } -/// Get the L1 fee recipient address -pub fn get_l1_fee_recipient() -> Address { - Address::from_str(L1_FEE_RECIPIENT).unwrap() -} - -/// Get the base fee recipient address -pub fn get_base_fee_recipient() -> Address { - Address::from_str(BASE_FEE_RECIPIENT).unwrap() +/// Route the base fee to the appropriate recipient account +pub fn route_base_fee_to_vault( + db: &mut SubState, + base_fee: u64, + gas_used: u64, +) -> Result<(), Error> { + let recipient_address = Address::from_str(BASE_FEE_RECIPIENT).unwrap(); + let mut recipient = db.load_account(recipient_address).map_err(|_| Error::ProviderError)?; + let amount_to_send = U256::from(base_fee.saturating_mul(gas_used)); + recipient.info.balance = recipient.info.balance.saturating_add(amount_to_send); + Ok(()) } -/// Route any fee to a recipient account -pub fn route_fee_to_vault( +/// Route the L1 cost to the appropriate recipient account +pub fn route_l1_cost_to_vault( db: &mut SubState, - fee: u64, - gas_used: u64, - recipient: Address, + l1_cost: U256, ) -> Result<(), Error> { - let mut account = db.load_account(recipient).map_err(|_| Error::ProviderError)?; - let amount_to_send = U256::from(fee.saturating_add(gas_used)); - account.info.balance = account.info.balance.saturating_add(amount_to_send); + let recipient_address = Address::from_str(L1_FEE_RECIPIENT).unwrap(); + let mut recipient = db.load_account(recipient_address).map_err(|_| Error::ProviderError)?; + recipient.info.balance = recipient.info.balance.saturating_add(l1_cost); Ok(()) } From 50ecb2aa9cab3af3b0d483387f0e72318935f53e Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Tue, 20 Jun 2023 23:47:28 +0200 Subject: [PATCH 026/150] fix: increment balance via executor function --- crates/revm/src/executor.rs | 20 ++++++++++++++------ crates/revm/src/optimism.rs | 29 ++++++----------------------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index f1f9ec1a428..dfb65ee8c6b 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -327,12 +327,20 @@ where } // Route the l1 cost and base fee to the appropriate optimism vaults - let db = self.db(); - optimism::route_l1_cost_to_vault(db, l1_cost)?; - optimism::route_base_fee_to_vault( - db, - block.base_fee_per_gas.unwrap_or_default(), - result.gas_used(), + self.increment_account_balance( + optimism::l1_cost_recipient(), + l1_cost, + &mut post_state, + )?; + self.increment_account_balance( + optimism::base_fee_recipient(), + U256::from( + block + .base_fee_per_gas + .unwrap_or_default() + .saturating_mul(result.gas_used()), + ), + &mut post_state, )?; // cast revm logs to reth logs diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 12d617820e7..6db8d5db032 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -1,9 +1,6 @@ use std::str::FromStr; -use reth_interfaces::executor::Error; use reth_primitives::{Address, TransactionSigned, U256}; -use reth_provider::StateProvider; -use reth_revm::database::SubState; use revm::db::DatabaseRef; const L1_FEE_RECIPIENT: &str = "0x420000000000000000000000000000000000001A"; @@ -70,26 +67,12 @@ impl L1GasCostOracle { } } -/// Route the base fee to the appropriate recipient account -pub fn route_base_fee_to_vault( - db: &mut SubState, - base_fee: u64, - gas_used: u64, -) -> Result<(), Error> { - let recipient_address = Address::from_str(BASE_FEE_RECIPIENT).unwrap(); - let mut recipient = db.load_account(recipient_address).map_err(|_| Error::ProviderError)?; - let amount_to_send = U256::from(base_fee.saturating_mul(gas_used)); - recipient.info.balance = recipient.info.balance.saturating_add(amount_to_send); - Ok(()) +/// Get the base fee recipient address +pub fn base_fee_recipient() -> Address { + Address::from_str(BASE_FEE_RECIPIENT).unwrap() } -/// Route the L1 cost to the appropriate recipient account -pub fn route_l1_cost_to_vault( - db: &mut SubState, - l1_cost: U256, -) -> Result<(), Error> { - let recipient_address = Address::from_str(L1_FEE_RECIPIENT).unwrap(); - let mut recipient = db.load_account(recipient_address).map_err(|_| Error::ProviderError)?; - recipient.info.balance = recipient.info.balance.saturating_add(l1_cost); - Ok(()) +/// Get the L1 cost recipient address +pub fn l1_cost_recipient() -> Address { + Address::from_str(L1_FEE_RECIPIENT).unwrap() } From 06954f339c17c333fb16bd4aed60c38af0b6b5ac Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Wed, 21 Jun 2023 09:21:36 +0200 Subject: [PATCH 027/150] chore: minor fixes --- crates/revm/src/executor.rs | 1 - crates/transaction-pool/src/traits.rs | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index dfb65ee8c6b..77d24c2fad2 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -386,7 +386,6 @@ where cumulative_gas_used, bloom: logs_bloom(logs.iter()), logs, - deposit_nonce: Some(transaction.nonce()), }); post_state.finish_transition(); } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 2c847e16aa8..72f2b3e0f28 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -592,9 +592,8 @@ impl FromRecoveredTransaction for PooledTransaction { } #[cfg(feature = "optimism")] Transaction::Deposit(t) => { - // Gas price is always set to 0 for deposits in order to zero out ETH refunds. - // The actual L1 cost varies depending on the gas oracle feed, and is calculated - // during tx execution. + // Gas price is always set to 0 for deposits in order to zero out ETH refunds, + // because they already pay for their gas on L1. let gas_price = U256::from(0); let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); let effective_gas_price = 0u128; From a04cc8d1cd0e91355ba738587686730456d5e8e0 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Thu, 29 Jun 2023 00:35:17 +0200 Subject: [PATCH 028/150] feat: parse l1 block info from l2 block --- crates/interfaces/src/executor.rs | 4 +- crates/revm/src/executor.rs | 6 +- crates/revm/src/optimism.rs | 113 +++++++++++++++++++----------- 3 files changed, 77 insertions(+), 46 deletions(-) diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index d0c35209fdb..bbb0afe16b9 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -50,8 +50,8 @@ pub enum BlockExecutionError { PipelineStatusUpdate { inner: String }, #[cfg(feature = "optimism")] - #[error("DB Error during transaction execution: {inner:?}")] - DBError { inner: String }, + #[error("Could not get L1 block info from L2 block: {message:?}")] + L1BlockInfoError { message: String }, #[cfg(feature = "optimism")] #[error("Insufficient funds to cover transaction L1 cost: want {want}, have {have}")] InsufficientFundsForL1Cost { want: u64, have: u64 }, diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 77d24c2fad2..4a9fe12184a 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -231,7 +231,7 @@ where self.init_env(&block.header, total_difficulty); #[cfg(feature = "optimism")] - let mut l1_cost_oracle = optimism::L1GasCostOracle::default(); + let mut l1_block_info = optimism::L1BlockInfo::new(block)?; let mut cumulative_gas_used = 0; let mut post_state = PostState::with_tx_capacity(block.number, block.body.len()); @@ -250,9 +250,7 @@ where #[cfg(feature = "optimism")] { let db = self.db(); - let l1_cost = l1_cost_oracle - .calculate_l1_cost(db, block.header.number, transaction) - .map_err(|db_err| Error::DBError { inner: db_err.to_string() })?; + let l1_cost = l1_block_info.calculate_tx_l1_cost(transaction); let sender_account = db.load_account(sender).map_err(|_| Error::ProviderError)?; let old_sender_info = to_reth_acc(&sender_account.info); diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 6db8d5db032..87813ecf961 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -1,69 +1,102 @@ use std::str::FromStr; -use reth_primitives::{Address, TransactionSigned, U256}; -use revm::db::DatabaseRef; +use reth_interfaces::executor; +use reth_primitives::{Address, Block, TransactionKind, TransactionSigned, U256}; const L1_FEE_RECIPIENT: &str = "0x420000000000000000000000000000000000001A"; const BASE_FEE_RECIPIENT: &str = "0x4200000000000000000000000000000000000019"; const L1_BLOCK_CONTRACT: &str = "0x4200000000000000000000000000000000000015"; -const L1_BASE_FEE_SLOT: u64 = 1; -const OVERHEAD_SLOT: u64 = 5; -const SCALAR_SLOT: u64 = 6; const ZERO_BYTE_COST: u64 = 4; const NON_ZERO_BYTE_COST: u64 = 16; -/// Optimism Gas Cost Oracle +/// L1 block info /// -/// This is used to calculate the gas cost of a transaction on the L1 chain. -/// The L1 block contract on L2 is used to propagate changes to the gas cost over time, -/// so we need to fetch this data from the DB. +/// We can extract L1 epoch data from each L2 block, by looking at the `setL1BlockValues` +/// transaction data. This data is then used to calculate the L1 cost of a transaction. /// -/// To make this slightly more efficient when executing entire blocks, the values are cached and -/// only fetched again when the block number changes. +/// Here is the format of the `setL1BlockValues` transaction data: +/// +/// setL1BlockValues(uint64 _number, uint64 _timestamp, uint256 _basefee, bytes32 _hash, +/// uint64 _sequenceNumber, bytes32 _batcherHash, uint256 _l1FeeOverhead, uint256 _l1FeeScalar) +/// +/// For now, we only care about the fields necessary for L1 cost calculation. #[derive(Default)] -pub struct L1GasCostOracle { - /// The cached block number - block_number: Option, - /// The base fee of the L1 chain +pub struct L1BlockInfo { l1_base_fee: U256, - /// The overhead value used to calculate the gas cost - overhead: U256, - /// The scalar value used to calculate the gas cost - scalar: U256, + l1_fee_overhead: U256, + l1_fee_scalar: U256, } -impl L1GasCostOracle { +impl L1BlockInfo { + /// Create a new L1 block info struct from a L2 block + pub fn new(block: &Block) -> Result { + let l1_block_contract = Address::from_str(L1_BLOCK_CONTRACT).unwrap(); + + let l1_info_tx_data = block + .body + .iter() + .find(|tx| matches!(tx.kind(), TransactionKind::Call(to) if to == &l1_block_contract)) + .ok_or(executor::Error::L1BlockInfoError { + message: "could not find l1 block info tx in the L2 block".to_string(), + }) + .and_then(|tx| { + tx.input().get(4..).ok_or(executor::Error::L1BlockInfoError { + message: "could not get l1 block info tx calldata bytes".to_string(), + }) + })?; + + // The setL1BlockValues tx calldata must be exactly 184 bytes long, considering that + // we already removed the first 4 bytes (the function selector). Detailed breakdown: + // 8 bytes for the block number + // + 8 bytes for the block timestamp + // + 32 bytes for the base fee + // + 32 bytes for the block hash + // + 8 bytes for the block sequence number + // + 32 bytes for the batcher hash + // + 32 bytes for the fee overhead + // + 32 bytes for the fee scalar + if l1_info_tx_data.len() != 184 { + return Err(executor::Error::L1BlockInfoError { + message: "unexpected l1 block info tx calldata length found".to_string(), + }) + } + + let l1_base_fee = U256::try_from_le_slice(&l1_info_tx_data[16..48]).ok_or( + executor::Error::L1BlockInfoError { + message: "could not convert l1 base fee".to_string(), + }, + )?; + let l1_fee_overhead = U256::try_from_le_slice(&l1_info_tx_data[120..152]).ok_or( + executor::Error::L1BlockInfoError { + message: "could not convert l1 fee overhead".to_string(), + }, + )?; + let l1_fee_scalar = U256::try_from_le_slice(&l1_info_tx_data[152..184]).ok_or( + executor::Error::L1BlockInfoError { + message: "could not convert l1 fee scalar".to_string(), + }, + )?; + + Ok(Self { l1_base_fee, l1_fee_overhead, l1_fee_scalar }) + } + /// Calculate the gas cost of a transaction based on L1 block data posted on L2 - pub fn calculate_l1_cost( - &mut self, - db: &mut DB, - block_num: u64, - tx: &TransactionSigned, - ) -> Result { + pub fn calculate_tx_l1_cost(&mut self, tx: &TransactionSigned) -> U256 { let rollup_data_gas_cost = U256::from(tx.input().iter().fold(0, |acc, byte| { acc + if byte == &0x00 { ZERO_BYTE_COST } else { NON_ZERO_BYTE_COST } })); if tx.is_deposit() || rollup_data_gas_cost == U256::ZERO { - return Ok(U256::ZERO) - } - - if self.block_number.cmp(&Some(block_num)).is_ne() { - let l1_block_address = Address::from_str(L1_BLOCK_CONTRACT).unwrap(); - - // TODO: Check: Is the db data always up-to-date? - self.l1_base_fee = db.storage(l1_block_address, U256::from(L1_BASE_FEE_SLOT))?; - self.overhead = db.storage(l1_block_address, U256::from(OVERHEAD_SLOT))?; - self.scalar = db.storage(l1_block_address, U256::from(SCALAR_SLOT))?; + return U256::ZERO } - Ok(rollup_data_gas_cost - .saturating_add(self.overhead) + rollup_data_gas_cost + .saturating_add(self.l1_fee_overhead) .saturating_mul(self.l1_base_fee) - .saturating_mul(self.scalar) + .saturating_mul(self.l1_fee_scalar) .checked_div(U256::from(1_000_000)) - .unwrap_or_default()) + .unwrap_or_default() } } From 4868a521855f8fdcfeec0f3ea317e5e5dece9a3e Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Thu, 29 Jun 2023 00:44:06 +0200 Subject: [PATCH 029/150] chore: removed default trait --- crates/revm/src/optimism.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 87813ecf961..9f579d8b688 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -21,7 +21,6 @@ const NON_ZERO_BYTE_COST: u64 = 16; /// uint64 _sequenceNumber, bytes32 _batcherHash, uint256 _l1FeeOverhead, uint256 _l1FeeScalar) /// /// For now, we only care about the fields necessary for L1 cost calculation. -#[derive(Default)] pub struct L1BlockInfo { l1_base_fee: U256, l1_fee_overhead: U256, From 08daf5b2a367c4c0c1e6f01672177c571e56ba4c Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:10:13 +0200 Subject: [PATCH 030/150] chore: set deposit gas to 0 always --- crates/revm/revm-primitives/src/env.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index be2702cc7c0..b1d15f2529e 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -216,14 +216,9 @@ where .collect(); } #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { + Transaction::Deposit(TxDeposit { to, value, gas_limit, input, .. }) => { tx_env.gas_limit = *gas_limit; - if let Some(m) = mint { - // TODO: why is this not zero? - tx_env.gas_price = U256::from(*m); - } else { - tx_env.gas_price = U256::ZERO; - } + tx_env.gas_price = U256::ZERO; tx_env.gas_priority_fee = None; match to { TransactionKind::Call(to) => tx_env.transact_to = TransactTo::Call(*to), From feda4dc32f7fa9dd3d8de8030ad1ca24e3a4cc78 Mon Sep 17 00:00:00 2001 From: clabby Date: Sat, 8 Jul 2023 18:47:49 -0400 Subject: [PATCH 031/150] Start resolving conflicts --- bin/reth/src/cli.rs | 6 +- crates/executor/Cargo.toml | 58 ----- crates/executor/src/lib.rs | 31 --- crates/interfaces/Cargo.toml | 4 +- crates/interfaces/src/executor.rs | 27 +- crates/net/eth-wire/src/types/receipts.rs | 42 ++- crates/primitives/Cargo.toml | 8 +- crates/primitives/src/chain/spec.rs | 244 +++++++++-------- crates/primitives/src/hardfork.rs | 4 + crates/primitives/src/proofs.rs | 5 +- crates/primitives/src/receipt.rs | 165 +++++------- crates/primitives/src/transaction/mod.rs | 54 +--- crates/primitives/src/transaction/tx_type.rs | 3 +- crates/revm/Cargo.toml | 10 +- crates/revm/revm-primitives/Cargo.toml | 4 +- crates/rpc/rpc-engine-api/src/engine_api.rs | 113 -------- crates/rpc/rpc-engine-api/src/error.rs | 47 +++- crates/rpc/rpc-types/Cargo.toml | 8 +- crates/rpc/rpc-types/src/eth/engine.rs | 246 ------------------ .../rpc/rpc-types/src/eth/engine/payload.rs | 16 ++ .../rpc/rpc-types/src/eth/transaction/mod.rs | 14 +- .../transaction-pool/src/test_utils/mock.rs | 10 +- crates/transaction-pool/src/traits.rs | 31 +-- 23 files changed, 361 insertions(+), 789 deletions(-) delete mode 100644 crates/executor/Cargo.toml delete mode 100644 crates/executor/src/lib.rs delete mode 100644 crates/rpc/rpc-types/src/eth/engine.rs diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index 06f4d3a6a2f..c32b1ae17ed 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -91,6 +91,10 @@ struct Cli { #[derive(Debug, Args)] #[command(next_help_heading = "Logging")] pub struct Logs { + /// The flag to enable persistent logs. + #[arg(long = "log.persistent", global = true, conflicts_with = "journald")] + persistent: bool, + /// The path to put log files in. #[arg( long = "log.directory", @@ -112,7 +116,7 @@ pub struct Logs { impl Logs { /// Builds a tracing layer from the current log options. - pub fn layer(&self) -> (BoxedLayer, Option) + pub fn layer(&self) -> eyre::Result, Option)>> where S: Subscriber, for<'a> S: LookupSpan<'a>, diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml deleted file mode 100644 index 53acae3de9f..00000000000 --- a/crates/executor/Cargo.toml +++ /dev/null @@ -1,58 +0,0 @@ -[package] -name = "reth-executor" -version = "0.1.0" -edition = "2021" -license = "MIT OR Apache-2.0" -repository = "https://github.com/paradigmxyz/reth" -readme = "README.md" - -[package.metadata.cargo-udeps.ignore] -normal = [ - # Used for diagrams in docs - "aquamarine", -] - -[dependencies] -# reth -reth-primitives = { path = "../primitives" } -reth-interfaces = { path = "../interfaces" } -reth-revm = { path = "../revm" } -reth-revm-inspectors = { path = "../revm/revm-inspectors" } -reth-rlp = { path = "../rlp" } -reth-db = { path = "../storage/db" } -reth-provider = { path = "../storage/provider" } - -# revm -revm = { version = "3.0.0" } - -# common -async-trait = "0.1.57" -thiserror = "1.0.37" -auto_impl = "1.0" -tracing = "0.1.37" -tokio = { version = "1.21.2", features = ["sync"] } - -# mics -aquamarine = "0.3.0" -parking_lot = { version = "0.12", optional = true } - -triehash = "0.8" -# See to replace hashers to simplify libraries -plain_hasher = "0.2" -hash-db = "0.15" -# todo replace with faster rlp impl -rlp = { version = "0.5", default-features = false } -# replace with tiny-keccak (it is faster hasher) -sha3 = { version = "0.10", default-features = false } - - -[dev-dependencies] -reth-db = { path = "../storage/db", features = ["test-utils"] } -reth-interfaces = { path = "../interfaces", features = ["test-utils"] } -reth-primitives = { path = "../primitives", features = ["test-utils"] } -reth-provider = { path = "../storage/provider", features = ["test-utils"] } -parking_lot = "0.12" - -[features] -test-utils = ["parking_lot"] -optimism = [] diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs deleted file mode 100644 index c4e4b13664e..00000000000 --- a/crates/executor/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -#![warn(missing_docs, unreachable_pub)] -#![deny(unused_must_use, rust_2018_idioms)] -#![doc(test( - no_crate_inject, - attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) -))] - -//! Reth executor executes transaction in block of data. - -pub mod eth_dao_fork; -pub mod substate; - -/// Execution result types. -pub use reth_provider::post_state; - -pub mod blockchain_tree; - -/// Executor -pub mod executor; - -/// ExecutorFactory impl -pub mod factory; -pub use factory::Factory; - -#[cfg(any(test, feature = "test-utils"))] -/// Common test helpers for mocking out executor and executor factory -pub mod test_utils; - -#[cfg(feature = "optimism")] -/// Optimism-specific utilities for the executor -pub mod optimism; diff --git a/crates/interfaces/Cargo.toml b/crates/interfaces/Cargo.toml index 016330402f7..8278bc412ce 100644 --- a/crates/interfaces/Cargo.toml +++ b/crates/interfaces/Cargo.toml @@ -53,6 +53,6 @@ secp256k1 = { workspace = true, features = [ ] } [features] -bench = [] +test-utils = ["tokio-stream/sync", "secp256k1", "rand/std_rng"] +cli = ["clap"] optimism = [] -test-utils = ["tokio-stream/sync", "secp256k1"] diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index bbb0afe16b9..7f315688a35 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -7,10 +7,6 @@ use thiserror::Error; pub enum BlockValidationError { #[error("EVM reported invalid transaction ({hash:?}): {message}")] EVM { hash: H256, message: String }, - #[error("Verification failed")] - VerificationFailed, - #[error("Fatal internal error")] - ExecutionFatalError, #[error("Failed to recover sender for transaction")] SenderRecoveryError, #[error("Receipt root {got:?} is different than expected {expected:?}.")] @@ -46,8 +42,20 @@ pub enum BlockExecutionError { CanonicalRevert { inner: String }, #[error("Transaction error on commit: {inner:?}")] CanonicalCommit { inner: String }, - #[error("Transaction error on pipeline status update: {inner:?}")] - PipelineStatusUpdate { inner: String }, + // === tree errors === + // TODO(mattsse): move this to tree error + #[error("Block hash {block_hash} not found in blockchain tree chain")] + BlockHashNotFoundInChain { block_hash: BlockHash }, + #[error( + "Appending chain on fork (other_chain_fork:?) is not possible as the tip is {chain_tip:?}" + )] + AppendChainDoesntConnect { chain_tip: BlockNumHash, other_chain_fork: BlockNumHash }, + + /// Only used for TestExecutor + /// + /// Note: this is not feature gated for convenience. + #[error("Execution unavailable for tests")] + UnavailableForTest, #[cfg(feature = "optimism")] #[error("Could not get L1 block info from L2 block: {message:?}")] @@ -56,3 +64,10 @@ pub enum BlockExecutionError { #[error("Insufficient funds to cover transaction L1 cost: want {want}, have {have}")] InsufficientFundsForL1Cost { want: u64, have: u64 }, } + +impl BlockExecutionError { + /// Returns `true` if the error is fatal. + pub fn is_fatal(&self) -> bool { + matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. }) + } +} diff --git a/crates/net/eth-wire/src/types/receipts.rs b/crates/net/eth-wire/src/types/receipts.rs index 39eeae1e316..8122ec8b437 100644 --- a/crates/net/eth-wire/src/types/receipts.rs +++ b/crates/net/eth-wire/src/types/receipts.rs @@ -49,11 +49,10 @@ mod test { success: false, cumulative_gas_used: 0, logs: vec![], + #[cfg(feature = "optimism")] + deposit_nonce: None, }, bloom: Default::default(), - logs: vec![], - #[cfg(feature = "optimism")] - deposit_nonce: None, }]]); let mut out = vec![]; @@ -105,10 +104,10 @@ mod test { let mut data = vec![]; let request = RequestPair:: { request_id: 1111, - message: Receipts(vec![ - vec![ - ReceiptWithBloom { - receipt: Receipt {tx_type: TxType::Legacy, + message: Receipts(vec![vec![ + ReceiptWithBloom { + receipt: Receipt { + tx_type: TxType::Legacy, cumulative_gas_used: 0x1u64, logs: vec![ Log { @@ -120,12 +119,13 @@ mod test { data: hex!("0100ff")[..].into(), }, ], - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, - }, - ], - ]), + bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), + } + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, + }, + ]]), }; request.encode(&mut data); assert_eq!(data, expected); @@ -157,22 +157,10 @@ mod test { }, ], success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), - cumulative_gas_used: 0x1u64, - logs: vec![ - Log { - address: hex!("0000000000000000000000000000000000000011").into(), - topics: vec![ - hex!("000000000000000000000000000000000000000000000000000000000000dead").into(), - hex!("000000000000000000000000000000000000000000000000000000000000beef").into(), - ], - data: hex!("0100ff")[..].into(), - }, - ], - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }, ], ]), diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index f1d2a1ec08c..b364683abc4 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -89,13 +89,9 @@ pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterio [features] default = [] +arbitrary = ["revm-primitives/arbitrary", "dep:arbitrary", "dep:proptest", "dep:proptest-derive"] +test-utils = [] optimism = [] -arbitrary = [ - "revm-primitives/arbitrary", - "dep:arbitrary", - "dep:proptest", - "dep:proptest-derive", -] [[bench]] name = "recover_ecdsa_crit" diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 731d5207048..d3dfec9ddb5 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -16,119 +16,155 @@ use std::{ }; /// The Ethereum mainnet spec -pub static MAINNET: Lazy = Lazy::new(|| ChainSpec { - chain: Chain::mainnet(), - genesis: serde_json::from_str(include_str!("../../res/genesis/mainnet.json")) - .expect("Can't deserialize Mainnet genesis json"), - genesis_hash: Some(H256(hex!( - "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" - ))), - hardforks: BTreeMap::from([ - (Hardfork::Frontier, ForkCondition::Block(0)), - (Hardfork::Homestead, ForkCondition::Block(1150000)), - (Hardfork::Dao, ForkCondition::Block(1920000)), - (Hardfork::Tangerine, ForkCondition::Block(2463000)), - (Hardfork::SpuriousDragon, ForkCondition::Block(2675000)), - (Hardfork::Byzantium, ForkCondition::Block(4370000)), - (Hardfork::Constantinople, ForkCondition::Block(7280000)), - (Hardfork::Petersburg, ForkCondition::Block(7280000)), - (Hardfork::Istanbul, ForkCondition::Block(9069000)), - (Hardfork::MuirGlacier, ForkCondition::Block(9200000)), - (Hardfork::Berlin, ForkCondition::Block(12244000)), - (Hardfork::London, ForkCondition::Block(12965000)), - (Hardfork::ArrowGlacier, ForkCondition::Block(13773000)), - (Hardfork::GrayGlacier, ForkCondition::Block(15050000)), - ( - Hardfork::Paris, - ForkCondition::TTD { - fork_block: None, - total_difficulty: U256::from(58_750_000_000_000_000_000_000_u128), - }, - ), - ]), - #[cfg(feature = "optimism")] - optimism: None, +pub static MAINNET: Lazy> = Lazy::new(|| { + ChainSpec { + chain: Chain::mainnet(), + genesis: serde_json::from_str(include_str!("../../res/genesis/mainnet.json")) + .expect("Can't deserialize Mainnet genesis json"), + genesis_hash: Some(H256(hex!( + "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" + ))), + // + paris_block_and_final_difficulty: Some(( + 15537394, + U256::from(58_750_003_716_598_352_816_469u128), + )), + fork_timestamps: ForkTimestamps::default().shanghai(1681338455), + hardforks: BTreeMap::from([ + (Hardfork::Frontier, ForkCondition::Block(0)), + (Hardfork::Homestead, ForkCondition::Block(1150000)), + (Hardfork::Dao, ForkCondition::Block(1920000)), + (Hardfork::Tangerine, ForkCondition::Block(2463000)), + (Hardfork::SpuriousDragon, ForkCondition::Block(2675000)), + (Hardfork::Byzantium, ForkCondition::Block(4370000)), + (Hardfork::Constantinople, ForkCondition::Block(7280000)), + (Hardfork::Petersburg, ForkCondition::Block(7280000)), + (Hardfork::Istanbul, ForkCondition::Block(9069000)), + (Hardfork::MuirGlacier, ForkCondition::Block(9200000)), + (Hardfork::Berlin, ForkCondition::Block(12244000)), + (Hardfork::London, ForkCondition::Block(12965000)), + (Hardfork::ArrowGlacier, ForkCondition::Block(13773000)), + (Hardfork::GrayGlacier, ForkCondition::Block(15050000)), + ( + Hardfork::Paris, + ForkCondition::TTD { + fork_block: None, + total_difficulty: U256::from(58_750_000_000_000_000_000_000_u128), + }, + ), + (Hardfork::Shanghai, ForkCondition::Timestamp(1681338455)), + ]), + #[cfg(feature = "optimism")] + optimism: None, + } + .into() }); /// The Goerli spec -pub static GOERLI: Lazy = Lazy::new(|| ChainSpec { - chain: Chain::goerli(), - genesis: serde_json::from_str(include_str!("../../res/genesis/goerli.json")) - .expect("Can't deserialize Goerli genesis json"), - genesis_hash: Some(H256(hex!( - "bf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a" - ))), - hardforks: BTreeMap::from([ - (Hardfork::Frontier, ForkCondition::Block(0)), - (Hardfork::Istanbul, ForkCondition::Block(1561651)), - (Hardfork::Berlin, ForkCondition::Block(4460644)), - (Hardfork::London, ForkCondition::Block(5062605)), - ( - Hardfork::Paris, - ForkCondition::TTD { fork_block: None, total_difficulty: U256::from(10_790_000) }, - ), - ]), - #[cfg(feature = "optimism")] - optimism: None, +pub static GOERLI: Lazy> = Lazy::new(|| { + ChainSpec { + chain: Chain::goerli(), + genesis: serde_json::from_str(include_str!("../../res/genesis/goerli.json")) + .expect("Can't deserialize Goerli genesis json"), + genesis_hash: Some(H256(hex!( + "bf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a" + ))), + // + paris_block_and_final_difficulty: Some((7382818, U256::from(10_790_000))), + fork_timestamps: ForkTimestamps::default().shanghai(1678832736), + hardforks: BTreeMap::from([ + (Hardfork::Frontier, ForkCondition::Block(0)), + (Hardfork::Homestead, ForkCondition::Block(0)), + (Hardfork::Dao, ForkCondition::Block(0)), + (Hardfork::Tangerine, ForkCondition::Block(0)), + (Hardfork::SpuriousDragon, ForkCondition::Block(0)), + (Hardfork::Byzantium, ForkCondition::Block(0)), + (Hardfork::Constantinople, ForkCondition::Block(0)), + (Hardfork::Petersburg, ForkCondition::Block(0)), + (Hardfork::Istanbul, ForkCondition::Block(1561651)), + (Hardfork::Berlin, ForkCondition::Block(4460644)), + (Hardfork::London, ForkCondition::Block(5062605)), + ( + Hardfork::Paris, + ForkCondition::TTD { fork_block: None, total_difficulty: U256::from(10_790_000) }, + ), + (Hardfork::Shanghai, ForkCondition::Timestamp(1678832736)), + ]), + #[cfg(feature = "optimism")] + optimism: None, + } + .into() }); /// The Sepolia spec -pub static SEPOLIA: Lazy = Lazy::new(|| ChainSpec { - chain: Chain::sepolia(), - genesis: serde_json::from_str(include_str!("../../res/genesis/sepolia.json")) - .expect("Can't deserialize Sepolia genesis json"), - genesis_hash: Some(H256(hex!( - "25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9" - ))), - hardforks: BTreeMap::from([ - (Hardfork::Frontier, ForkCondition::Block(0)), - (Hardfork::Homestead, ForkCondition::Block(0)), - (Hardfork::Dao, ForkCondition::Block(0)), - (Hardfork::Tangerine, ForkCondition::Block(0)), - (Hardfork::SpuriousDragon, ForkCondition::Block(0)), - (Hardfork::Byzantium, ForkCondition::Block(0)), - (Hardfork::Constantinople, ForkCondition::Block(0)), - (Hardfork::Petersburg, ForkCondition::Block(0)), - (Hardfork::Istanbul, ForkCondition::Block(0)), - (Hardfork::MuirGlacier, ForkCondition::Block(0)), - (Hardfork::Berlin, ForkCondition::Block(0)), - (Hardfork::London, ForkCondition::Block(0)), - ( - Hardfork::Paris, - ForkCondition::TTD { - fork_block: Some(1735371), - total_difficulty: U256::from(17_000_000_000_000_000u64), - }, - ), - (Hardfork::Shanghai, ForkCondition::Timestamp(1677557088)), - ]), - #[cfg(feature = "optimism")] - optimism: None, +pub static SEPOLIA: Lazy> = Lazy::new(|| { + ChainSpec { + chain: Chain::sepolia(), + genesis: serde_json::from_str(include_str!("../../res/genesis/sepolia.json")) + .expect("Can't deserialize Sepolia genesis json"), + genesis_hash: Some(H256(hex!( + "25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9" + ))), + // + paris_block_and_final_difficulty: Some((1450409, U256::from(17_000_018_015_853_232u128))), + fork_timestamps: ForkTimestamps::default().shanghai(1677557088), + hardforks: BTreeMap::from([ + (Hardfork::Frontier, ForkCondition::Block(0)), + (Hardfork::Homestead, ForkCondition::Block(0)), + (Hardfork::Dao, ForkCondition::Block(0)), + (Hardfork::Tangerine, ForkCondition::Block(0)), + (Hardfork::SpuriousDragon, ForkCondition::Block(0)), + (Hardfork::Byzantium, ForkCondition::Block(0)), + (Hardfork::Constantinople, ForkCondition::Block(0)), + (Hardfork::Petersburg, ForkCondition::Block(0)), + (Hardfork::Istanbul, ForkCondition::Block(0)), + (Hardfork::MuirGlacier, ForkCondition::Block(0)), + (Hardfork::Berlin, ForkCondition::Block(0)), + (Hardfork::London, ForkCondition::Block(0)), + ( + Hardfork::Paris, + ForkCondition::TTD { + fork_block: Some(1735371), + total_difficulty: U256::from(17_000_000_000_000_000u64), + }, + ), + (Hardfork::Shanghai, ForkCondition::Timestamp(1677557088)), + ]), + #[cfg(feature = "optimism")] + optimism: None, + } + .into() }); /// The Optimism Goerli spec #[cfg(feature = "optimism")] -pub static OP_GOERLI: Lazy = Lazy::new(|| ChainSpec { - chain: Chain::optimism_goerli(), - genesis: serde_json::from_str(include_str!("../../res/genesis/goerli_op.json")) - .expect("Can't deserialize Optimism Goerli genesis json"), - genesis_hash: Some(H256(hex!( - "c1fc15cd51159b1f1e5cbc4b82e85c1447ddfa33c52cf1d98d14fba0d6354be1" - ))), - hardforks: BTreeMap::from([ - (Hardfork::Byzantium, ForkCondition::Block(0)), - (Hardfork::Constantinople, ForkCondition::Block(0)), - (Hardfork::Petersburg, ForkCondition::Block(0)), - (Hardfork::Istanbul, ForkCondition::Block(0)), - (Hardfork::MuirGlacier, ForkCondition::Block(0)), - (Hardfork::Berlin, ForkCondition::Block(0)), - (Hardfork::London, ForkCondition::Block(4061224)), - (Hardfork::ArrowGlacier, ForkCondition::Block(4061224)), - (Hardfork::GrayGlacier, ForkCondition::Block(4061224)), - (Hardfork::Bedrock, ForkCondition::Block(4061224)), - (Hardfork::Regolith, ForkCondition::Timestamp(1679079600)), - ]), - optimism: Some(OptimismConfig { eip_1559_elasticity: 10, eip_1559_denominator: 50 }), +pub static OP_GOERLI: Lazy> = Lazy::new(|| { + ChainSpec { + chain: Chain::optimism_goerli(), + genesis: serde_json::from_str(include_str!("../../res/genesis/goerli_op.json")) + .expect("Can't deserialize Optimism Goerli genesis json"), + genesis_hash: Some(H256(hex!( + "c1fc15cd51159b1f1e5cbc4b82e85c1447ddfa33c52cf1d98d14fba0d6354be1" + ))), + fork_timestamps: ForkTimestamps::default(), // TODO(clabby): update this + paris_block_and_final_difficulty: None, // TODO(clabby): update this + hardforks: BTreeMap::from([ + (Hardfork::Byzantium, ForkCondition::Block(0)), + (Hardfork::Constantinople, ForkCondition::Block(0)), + (Hardfork::Petersburg, ForkCondition::Block(0)), + (Hardfork::Istanbul, ForkCondition::Block(0)), + (Hardfork::MuirGlacier, ForkCondition::Block(0)), + (Hardfork::Berlin, ForkCondition::Block(0)), + (Hardfork::London, ForkCondition::Block(0)), + ( + Hardfork::Paris, + ForkCondition::TTD { fork_block: Some(0), total_difficulty: U256::from(0) }, + ), + (Hardfork::Shanghai, ForkCondition::Timestamp(0)), + ]), + optimism: Some(OptimismConfig { eip_1559_elasticity: 10, eip_1559_denominator: 50 }), + } + .into() }); /// An Ethereum chain specification. @@ -401,6 +437,7 @@ impl From for ChainSpec { genesis_hash: None, fork_timestamps: ForkTimestamps::from_hardforks(&hardforks), hardforks, + paris_block_and_final_difficulty: None, #[cfg(feature = "optimism")] optimism: None, } @@ -626,6 +663,7 @@ impl ChainSpecBuilder { genesis_hash: None, fork_timestamps: ForkTimestamps::from_hardforks(&self.hardforks), hardforks: self.hardforks, + paris_block_and_final_difficulty: None, #[cfg(feature = "optimism")] optimism: self.optimism, } diff --git a/crates/primitives/src/hardfork.rs b/crates/primitives/src/hardfork.rs index 0c3942ebd9d..f772e8c6cc2 100644 --- a/crates/primitives/src/hardfork.rs +++ b/crates/primitives/src/hardfork.rs @@ -180,6 +180,8 @@ mod tests { genesis: Genesis::default(), genesis_hash: None, hardforks: BTreeMap::from([(Hardfork::Frontier, ForkCondition::Never)]), + fork_timestamps: Default::default(), + paris_block_and_final_difficulty: None, #[cfg(feature = "optimism")] optimism: None, }; @@ -194,6 +196,8 @@ mod tests { genesis: Genesis::default(), genesis_hash: None, hardforks: BTreeMap::from([(Hardfork::Shanghai, ForkCondition::Never)]), + fork_timestamps: Default::default(), + paris_block_and_final_difficulty: None, #[cfg(feature = "optimism")] optimism: None, }; diff --git a/crates/primitives/src/proofs.rs b/crates/primitives/src/proofs.rs index 8e90eeb970f..40740d119af 100644 --- a/crates/primitives/src/proofs.rs +++ b/crates/primitives/src/proofs.rs @@ -171,11 +171,10 @@ mod tests { success: true, cumulative_gas_used: 102068, logs, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, bloom, - logs, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; let receipt = vec![receipt]; let root = calculate_receipt_root(&receipt); diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index ba0c43c5736..537d147ebff 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -34,36 +34,40 @@ pub struct Receipt { } impl Receipt { - /// Returns the rlp header for the receipt payload. - fn receipt_rlp_header(&self) -> reth_rlp::Header { - let mut rlp_head = reth_rlp::Header { list: true, payload_length: 0 }; + /// Calculates [`Log`]'s bloom filter. this is slow operation and [ReceiptWithBloom] can + /// be used to cache this value. + pub fn bloom_slow(&self) -> Bloom { + logs_bloom(self.logs.iter()) + } - rlp_head.payload_length += self.success.length(); - rlp_head.payload_length += self.cumulative_gas_used.length(); - rlp_head.payload_length += self.bloom.length(); - rlp_head.payload_length += self.logs.length(); - #[cfg(feature = "optimism")] - if self.tx_type == TxType::DEPOSIT { - // Transactions pre-Regolith don't have a deposit nonce - if let Some(nonce) = self.deposit_nonce { - rlp_head.payload_length += nonce.length(); - } - } + /// Calculates the bloom filter for the receipt and returns the [ReceiptWithBloom] container + /// type. + pub fn with_bloom(self) -> ReceiptWithBloom { + self.into() + } +} - rlp_head +impl From for ReceiptWithBloom { + fn from(receipt: Receipt) -> Self { + let bloom = receipt.bloom_slow(); + ReceiptWithBloom { receipt, bloom } } +} - /// Encodes the receipt data. - fn encode_fields(&self, out: &mut dyn BufMut) { - self.receipt_rlp_header().encode(out); - self.success.encode(out); - self.cumulative_gas_used.encode(out); - self.bloom.encode(out); - self.logs.encode(out); - #[cfg(feature = "optimism")] - if let Some(nonce) = self.deposit_nonce { - nonce.encode(out); - } +/// [`Receipt`] with calculated bloom filter. +#[main_codec] +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct ReceiptWithBloom { + /// Bloom filter build from logs. + pub bloom: Bloom, + /// Main receipt body + pub receipt: Receipt, +} + +impl ReceiptWithBloom { + /// Create new [ReceiptWithBloom] + pub fn new(receipt: Receipt, bloom: Bloom) -> Self { + Self { receipt, bloom } } /// Consume the structure, returning only the receipt @@ -85,35 +89,7 @@ impl Receipt { impl ReceiptWithBloom { /// Encode receipt with or without the header data. pub fn encode_inner(&self, out: &mut dyn BufMut, with_header: bool) { - if matches!(self.tx_type, TxType::Legacy) { - self.encode_fields(out); - return - } - - let mut payload = BytesMut::new(); - self.encode_fields(&mut payload); - - if with_header { - let payload_length = payload.len() + 1; - let header = reth_rlp::Header { list: false, payload_length }; - header.encode(out); - } - - match self.tx_type { - TxType::EIP2930 => out.put_u8(0x01), - TxType::EIP1559 => out.put_u8(0x02), - TxType::Legacy => unreachable!("legacy handled; qed."), - - #[cfg(feature = "optimism")] - TxType::DEPOSIT => out.put_u8(0x7E), - } - out.put_slice(payload.as_ref()); - } - - /// Returns the length of the receipt data. - fn receipt_length(&self) -> usize { - let rlp_head = self.receipt_rlp_header(); - length_of_length(rlp_head.payload_length) + rlp_head.payload_length + self.as_encoder().encode_inner(out, with_header) } /// Decodes the receipt payload @@ -125,27 +101,35 @@ impl ReceiptWithBloom { } let started_len = b.len(); - let this = match tx_type { + let success = reth_rlp::Decodable::decode(b)?; + let cumulative_gas_used = reth_rlp::Decodable::decode(b)?; + let bloom = Decodable::decode(b)?; + let logs = reth_rlp::Decodable::decode(b)?; + + let receipt = match tx_type { #[cfg(feature = "optimism")] - TxType::DEPOSIT => Self { - tx_type, - success: reth_rlp::Decodable::decode(b)?, - cumulative_gas_used: reth_rlp::Decodable::decode(b)?, - bloom: reth_rlp::Decodable::decode(b)?, - logs: reth_rlp::Decodable::decode(b)?, - deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), - }, - _ => Self { + TxType::DEPOSIT => { + let deposit_nonce = reth_rlp::Decodable::decode(b)?; + Receipt { + tx_type, + success, + cumulative_gas_used, + logs, + #[cfg(feature = "optimism")] + deposit_nonce: Some(deposit_nonce), + } + } + _ => Receipt { tx_type, - success: reth_rlp::Decodable::decode(b)?, - cumulative_gas_used: reth_rlp::Decodable::decode(b)?, - bloom: reth_rlp::Decodable::decode(b)?, - logs: reth_rlp::Decodable::decode(b)?, + success, + cumulative_gas_used, + logs, #[cfg(feature = "optimism")] deposit_nonce: None, }, }; + let this = Self { receipt, bloom }; let consumed = started_len - b.len(); if consumed != rlp_head.payload_length { return Err(reth_rlp::DecodeError::ListLengthMismatch { @@ -183,7 +167,6 @@ impl Decodable for ReceiptWithBloom { let receipt_type = *buf.first().ok_or(reth_rlp::DecodeError::Custom( "typed receipt cannot be decoded from an empty slice", ))?; - match receipt_type { 0x01 => { buf.advance(1); @@ -363,26 +346,10 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, bloom: [0; 256].into(), - cumulative_gas_used: 0x1u64, - logs: vec![Log { - address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), - topics: vec![ - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000dead", - ) - .unwrap(), - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000beef", - ) - .unwrap(), - ], - data: Bytes::from_str("0100ff").unwrap().0.into(), - }], - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; receipt.encode(&mut data); @@ -417,26 +384,10 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, bloom: [0; 256].into(), - cumulative_gas_used: 0x1u64, - logs: vec![Log { - address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), - topics: vec![ - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000dead", - ) - .unwrap(), - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000beef", - ) - .unwrap(), - ], - data: Bytes::from_str("0100ff").unwrap().0.into(), - }], - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); @@ -469,6 +420,8 @@ mod tests { data: crate::Bytes::from(vec![1; 0xffffff]), }, ], + #[cfg(feature = "optimism")] + deposit_nonce: None, }; let mut data = vec![]; diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index d34847e96bc..c35162117d0 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -13,6 +13,7 @@ use reth_rlp::{ }; #[cfg(feature = "optimism")] use revm_primitives::U256; +use serde::{Deserialize, Serialize}; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; @@ -1063,58 +1064,7 @@ impl TransactionSigned { /// Inner encoding function that is used for both rlp [`Encodable`] trait and for calculating /// hash that for eip2718 does not require rlp header pub(crate) fn encode_inner(&self, out: &mut dyn bytes::BufMut, with_header: bool) { - match self.transaction { - Transaction::Legacy(TxLegacy { chain_id, .. }) => { - // do nothing w/ with_header - let payload_length = self.transaction.fields_len() + - self.signature.payload_len_with_eip155_chain_id(chain_id); - let header = Header { list: true, payload_length }; - header.encode(out); - self.transaction.encode_fields(out); - self.signature.encode_with_eip155_chain_id(out, chain_id); - } - #[cfg(feature = "optimism")] - Transaction::Deposit(_) => { - let payload_length = self.transaction.fields_len() + self.signature.payload_len(); - if with_header { - Header { - list: false, - payload_length: 1 + 1 + length_of_length(payload_length) + payload_length, - } - .encode(out); - } - out.put_u8(self.transaction.tx_type() as u8); - out.put_u8(DEPOSIT_VERSION); - let header = Header { list: true, payload_length }; - header.encode(out); - self.transaction.encode_fields(out); - // Deposit transactions do not have a signature. If the signature's values are not - // zero, then the transaction is invalid. - if self.signature().v(self.chain_id()) != 0 || - self.signature().r != U256::ZERO || - self.signature().s != U256::ZERO - { - // TODO: Ensure that this transaction may never have a non-zero signature - // higher up - we shouldn't be panicking here. - panic!("Deposit transactions must have a zero signature"); - } - } - _ => { - let payload_length = self.transaction.fields_len() + self.signature.payload_len(); - if with_header { - Header { - list: false, - payload_length: 1 + length_of_length(payload_length) + payload_length, - } - .encode(out); - } - out.put_u8(self.transaction.tx_type() as u8); - let header = Header { list: true, payload_length }; - header.encode(out); - self.transaction.encode_fields(out); - self.signature.encode(out); - } - } + self.transaction.encode_with_signature(&self.signature, out, with_header); } /// Output the length of the encode_inner(out, true). Note to assume that `with_header` is only diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index e44bfba1dfb..3fcca61a774 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -69,10 +69,9 @@ impl Compact for TxType { match identifier { 0 => TxType::Legacy, 1 => TxType::EIP2930, - 2 => TxType::EIP1559, #[cfg(feature = "optimism")] 126 => TxType::DEPOSIT, - _ => panic!("unknown transaction type {identifier}"), + _ => TxType::EIP1559, }, buf, ) diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index 0f375476570..31622ab3a1f 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -17,7 +17,15 @@ reth-revm-primitives = { path = "./revm-primitives" } reth-revm-inspectors = { path = "./revm-inspectors" } reth-consensus-common = { path = "../consensus/common" } -revm = { version = "3.0.0" } +# revm +revm = { workspace = true } + +# common +tracing = { workspace = true } + +[dev-dependencies] +reth-rlp = { workspace = true } +once_cell = "1.17.0" [features] optimism = ["reth-primitives/optimism", "reth-revm-primitives/optimism"] diff --git a/crates/revm/revm-primitives/Cargo.toml b/crates/revm/revm-primitives/Cargo.toml index d03fb908510..85c66da26fd 100644 --- a/crates/revm/revm-primitives/Cargo.toml +++ b/crates/revm/revm-primitives/Cargo.toml @@ -10,9 +10,9 @@ description = "core reth specific revm utilities" [dependencies] # reth -reth-primitives = { path = "../../primitives" } +reth-primitives = { workspace = true } -revm = { version = "3.0.0" } +revm = { workspace = true } [features] optimism = ["reth-primitives/optimism"] diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 9238addfe52..4803c4d2d71 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -211,119 +211,6 @@ where Ok(result) } - /// When the Consensus layer receives a new block via the consensus gossip protocol, - /// the transactions in the block are sent to the execution layer in the form of a - /// `ExecutionPayload`. The Execution layer executes the transactions and validates the - /// state in the block header, then passes validation data back to Consensus layer, that - /// adds the block to the head of its own blockchain and attests to it. The block is then - /// broadcasted over the consensus p2p network in the form of a "Beacon block". - pub fn new_payload(&mut self, payload: ExecutionPayload) -> EngineApiResult { - let block = match self.try_construct_block(payload) { - Ok(b) => b, - Err(err) => { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::InvalidBlockHash { - validation_error: err.to_string(), - })) - } - }; - let block_hash = block.header.hash(); - let parent_hash = block.parent_hash; - - // The block already exists in our database - if self.client.is_known(&block_hash)? { - return Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)) - } - - let Some(parent) = self.client.block_by_hash(parent_hash)? else { - // TODO: cache block for storing later - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Syncing)) - }; - - let parent_td = if let Some(parent_td) = self.client.header_td(&block.parent_hash)? { - parent_td - } else { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::PayloadPreMerge.to_string(), - })) - }; - - // Short circuit the check by passing parent total difficulty. - if !self.chain_spec.fork(Hardfork::Paris).active_at_ttd(parent_td, U256::ZERO) { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::PayloadPreMerge.to_string(), - })) - } - - if block.timestamp <= parent.timestamp { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::PayloadTimestamp { - invalid: block.timestamp, - latest: parent.timestamp, - } - .to_string(), - })) - } - - let state_provider = self.client.latest()?; - let total_difficulty = parent_td + block.header.difficulty; - match executor::execute_and_verify_receipt( - &block.unseal(), - total_difficulty, - None, - &self.chain_spec, - &mut SubState::new(State::new(&state_provider)), - ) { - Ok(_) => Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)), - Err(err) => Ok(PayloadStatus::new( - PayloadStatusEnum::Invalid { validation_error: err.to_string() }, - parent_hash, // The parent hash is already in our database hence it is valid - )), - } - } - - /// Called to resolve chain forks and ensure that the Execution layer is working with the latest - /// valid chain. - pub fn fork_choice_updated( - &self, - fork_choice_state: ForkchoiceState, - payload_attributes: Option, - ) -> EngineApiResult { - let ForkchoiceState { head_block_hash, finalized_block_hash, .. } = fork_choice_state; - - if head_block_hash.is_zero() { - return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::ForkchoiceEmptyHead.to_string(), - })) - } - - // Block is not known, nothing to do. - if !self.client.is_known(&head_block_hash)? { - return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) - } - - // The finalized block hash is not known, we are still syncing - if !finalized_block_hash.is_zero() && !self.client.is_known(&finalized_block_hash)? { - return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) - } - - if let Err(error) = self.forkchoice_state_tx.send(fork_choice_state) { - tracing::error!(target: "rpc::engine_api", ?error, "Failed to update forkchoice state"); - } - - if let Some(_attr) = payload_attributes { - #[cfg(feature = "optimism")] - if self.chain_spec.optimism.is_some() && _attr.gas_limit.is_none() { - return Err(EngineApiError::MissingGasLimitInPayloadAttributes) - } - - // TODO: optionally build the block - } - - let chain_info = self.client.chain_info()?; - Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Valid) - .with_latest_valid_hash(chain_info.best_hash)) - } - /// Called to verify network configuration parameters and ensure that Consensus and Execution /// layers are using the latest configuration. pub async fn exchange_transition_configuration( diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index 8e146b07f09..b6dfca1b68c 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -68,10 +68,55 @@ pub enum EngineApiError { ForkChoiceUpdate(#[from] BeaconForkChoiceUpdateError), /// An error occurred while processing a new payload in the beacon consensus engine. #[error(transparent)] - Internal(#[from] reth_interfaces::Error), + NewPayload(#[from] BeaconOnNewPayloadError), + /// Encountered an internal error. + #[error(transparent)] + Internal(Box), + /// Fetching the payload failed + #[error(transparent)] + GetPayloadError(#[from] PayloadBuilderError), /// If the optimism feature flag is enabled, the payload attributes must have a present /// gas limit for the forkchoice updated method. #[cfg(feature = "optimism")] #[error("Missing gas limit in payload attributes")] MissingGasLimitInPayloadAttributes, } + +impl From for jsonrpsee_types::error::ErrorObject<'static> { + fn from(error: EngineApiError) -> Self { + let code = match error { + EngineApiError::InvalidBodiesRange { .. } | + EngineApiError::WithdrawalsNotSupportedInV1 | + EngineApiError::NoWithdrawalsPostShanghai | + EngineApiError::HasWithdrawalsPreShanghai => INVALID_PARAMS_CODE, + EngineApiError::UnknownPayload => UNKNOWN_PAYLOAD_CODE, + EngineApiError::PayloadRequestTooLarge { .. } => REQUEST_TOO_LARGE_CODE, + + // Error responses from the consensus engine + EngineApiError::ForkChoiceUpdate(ref err) => match err { + BeaconForkChoiceUpdateError::ForkchoiceUpdateError(err) => return (*err).into(), + BeaconForkChoiceUpdateError::EngineUnavailable | + BeaconForkChoiceUpdateError::Internal(_) => INTERNAL_ERROR_CODE, + }, + EngineApiError::NewPayload(ref err) => match err { + BeaconOnNewPayloadError::Internal(_) | + BeaconOnNewPayloadError::EngineUnavailable => INTERNAL_ERROR_CODE, + }, + // Any other server error + EngineApiError::TerminalTD { .. } | + EngineApiError::TerminalBlockHash { .. } | + EngineApiError::Internal(_) | + EngineApiError::GetPayloadError(_) => INTERNAL_ERROR_CODE, + // Optimism errors + #[cfg(feature = "optimism")] + EngineApiError::MissingGasLimitInPayloadAttributes => INVALID_PARAMS_CODE, + }; + jsonrpsee_types::error::ErrorObject::owned(code, error.to_string(), None::<()>) + } +} + +impl From for jsonrpsee_core::error::Error { + fn from(error: EngineApiError) -> Self { + jsonrpsee_core::error::Error::Call(error.into()) + } +} diff --git a/crates/rpc/rpc-types/Cargo.toml b/crates/rpc/rpc-types/Cargo.toml index 7e56ccbe319..0e06fc3e35e 100644 --- a/crates/rpc/rpc-types/Cargo.toml +++ b/crates/rpc/rpc-types/Cargo.toml @@ -24,7 +24,13 @@ serde_json = { workspace = true } jsonrpsee-types = { version = "0.18" } [dev-dependencies] -reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } +# reth +reth-interfaces = { workspace = true, features = ["test-utils"] } + +# misc +rand = { workspace = true } +assert_matches = "1.5" +similar-asserts = "1.4" [features] optimism = ["reth-primitives/optimism"] diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs deleted file mode 100644 index 548ed16645b..00000000000 --- a/crates/rpc/rpc-types/src/eth/engine.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! Engine API types: and following the execution specs - -#![allow(missing_docs)] - -use reth_primitives::{ - Address, Block, Bloom, Bytes, SealedBlock, Withdrawal, H256, H64, U256, U64, -}; -use reth_rlp::Encodable; -use serde::{Deserialize, Serialize}; - -/// The list of supported Engine capabilities -pub const CAPABILITIES: [&str; 9] = [ - "engine_forkchoiceUpdatedV1", - "engine_forkchoiceUpdatedV2", - "engine_exchangeTransitionConfigurationV1", - "engine_getPayloadV1", - "engine_getPayloadV2", - "engine_newPayloadV1", - "engine_newPayloadV2", - "engine_getPayloadBodiesByHashV1", - "engine_getPayloadBodiesByRangeV1", -]; - -/// This structure maps on the ExecutionPayload structure of the beacon chain spec. -/// -/// See also: -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ExecutionPayload { - pub parent_hash: H256, - pub fee_recipient: Address, - pub state_root: H256, - pub receipts_root: H256, - pub logs_bloom: Bloom, - pub prev_randao: H256, - pub block_number: U64, - pub gas_limit: U64, - pub gas_used: U64, - pub timestamp: U64, - pub extra_data: Bytes, - pub base_fee_per_gas: U256, - pub block_hash: H256, - pub transactions: Vec, - /// Array of [`Withdrawal`] enabled with V2 - /// See - #[serde(default, skip_serializing_if = "Option::is_none")] - pub withdrawals: Option>, -} - -impl From for ExecutionPayload { - fn from(value: SealedBlock) -> Self { - let transactions = value - .body - .iter() - .map(|tx| { - let mut encoded = Vec::new(); - tx.encode(&mut encoded); - encoded.into() - }) - .collect(); - ExecutionPayload { - parent_hash: value.parent_hash, - fee_recipient: value.beneficiary, - state_root: value.state_root, - receipts_root: value.receipts_root, - logs_bloom: value.logs_bloom, - prev_randao: value.mix_hash, - block_number: value.number.into(), - gas_limit: value.gas_limit.into(), - gas_used: value.gas_used.into(), - timestamp: value.timestamp.into(), - extra_data: value.extra_data.clone(), - base_fee_per_gas: U256::from(value.base_fee_per_gas.unwrap_or_default()), - block_hash: value.hash(), - transactions, - withdrawals: value.withdrawals, - } - } -} - -/// This structure contains a body of an execution payload. -/// -/// See also: -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ExecutionPayloadBody { - pub transactions: Vec, - pub withdrawals: Vec, -} - -impl From for ExecutionPayloadBody { - fn from(value: Block) -> Self { - let transactions = value.body.into_iter().map(|tx| { - let mut out = Vec::new(); - tx.encode(&mut out); - out.into() - }); - ExecutionPayloadBody { - transactions: transactions.collect(), - withdrawals: value.withdrawals.unwrap_or_default(), - } - } -} - -/// The execution payload body response that allows for `null` values. -pub type ExecutionPayloadBodies = Vec>; - -/// This structure encapsulates the fork choice state -#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ForkchoiceState { - pub head_block_hash: H256, - pub safe_block_hash: H256, - pub finalized_block_hash: H256, -} - -/// This structure contains the attributes required to initiate a payload build process in the -/// context of an `engine_forkchoiceUpdated` call. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PayloadAttributes { - pub timestamp: U64, - pub prev_randao: H256, - pub suggested_fee_recipient: Address, - /// Array of [`Withdrawal`] enabled with V2 - /// See - #[serde(default, skip_serializing_if = "Option::is_none")] - pub withdrawals: Option>, - - /// Transactions is a field for rollups: the transactions list is forced into the block - #[cfg(feature = "optimism")] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub transactions: Option>, - - /// If true, the no transactions are taken out of the tx-pool, only transactions from the above - /// Transactions list will be included. - #[cfg(feature = "optimism")] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub no_tx_pool: Option, - - /// If set, this sets the exact gas limit the block produced with. - #[cfg(feature = "optimism")] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gas_limit: Option, -} - -/// This structure contains the result of processing a payload -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PayloadStatus { - #[serde(flatten)] - pub status: PayloadStatusEnum, - /// Hash of the most recent valid block in the branch defined by payload and its ancestors - pub latest_valid_hash: Option, -} - -impl PayloadStatus { - pub fn new(status: PayloadStatusEnum, latest_valid_hash: H256) -> Self { - Self { status, latest_valid_hash: Some(latest_valid_hash) } - } - - pub fn from_status(status: PayloadStatusEnum) -> Self { - Self { status, latest_valid_hash: None } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "status", rename_all = "SCREAMING_SNAKE_CASE")] -pub enum PayloadStatusEnum { - Valid, - Invalid { - #[serde(rename = "validationError")] - validation_error: String, - }, - Syncing, - Accepted, - InvalidBlockHash { - #[serde(rename = "validationError")] - validation_error: String, - }, -} - -/// This structure contains configurable settings of the transition process. -#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TransitionConfiguration { - /// Maps on the TERMINAL_TOTAL_DIFFICULTY parameter of EIP-3675 - pub terminal_total_difficulty: U256, - /// Maps on TERMINAL_BLOCK_HASH parameter of EIP-3675 - pub terminal_block_hash: H256, - /// Maps on TERMINAL_BLOCK_NUMBER parameter of EIP-3675 - pub terminal_block_number: U64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ForkchoiceUpdated { - pub payload_status: PayloadStatus, - pub payload_id: Option, -} - -impl ForkchoiceUpdated { - pub fn new(payload_status: PayloadStatus) -> Self { - Self { payload_status, payload_id: None } - } - - pub fn from_status(status: PayloadStatusEnum) -> Self { - Self { payload_status: PayloadStatus::from_status(status), payload_id: None } - } - - pub fn with_latest_valid_hash(mut self, hash: H256) -> Self { - self.payload_status.latest_valid_hash = Some(hash); - self - } - - pub fn with_payload_id(mut self, id: H64) -> Self { - self.payload_id = Some(id); - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - use reth_interfaces::test_utils::generators::random_block_range; - use reth_primitives::{TransactionSigned, H256}; - use reth_rlp::Decodable; - - #[test] - fn payload_body_roundtrip() { - for block in random_block_range(0..100, H256::default(), 0..2) { - let unsealed = block.clone().unseal(); - let payload_body: ExecutionPayloadBody = unsealed.into(); - - assert_eq!( - Ok(block.body), - payload_body - .transactions - .iter() - .map(|x| TransactionSigned::decode(&mut &x[..])) - .collect::, _>>(), - ); - - assert_eq!(block.withdrawals.unwrap_or_default(), payload_body.withdrawals); - } - } -} diff --git a/crates/rpc/rpc-types/src/eth/engine/payload.rs b/crates/rpc/rpc-types/src/eth/engine/payload.rs index d60c81b5ae2..89788ed4690 100644 --- a/crates/rpc/rpc-types/src/eth/engine/payload.rs +++ b/crates/rpc/rpc-types/src/eth/engine/payload.rs @@ -251,6 +251,22 @@ pub struct PayloadAttributes { /// See #[serde(default, skip_serializing_if = "Option::is_none")] pub withdrawals: Option>, + + /// Transactions is a field for rollups: the transactions list is forced into the block + #[cfg(feature = "optimism")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transactions: Option>, + + /// If true, the no transactions are taken out of the tx-pool, only transactions from the above + /// Transactions list will be included. + #[cfg(feature = "optimism")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub no_tx_pool: Option, + + /// If set, this sets the exact gas limit the block produced with. + #[cfg(feature = "optimism")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gas_limit: Option, } /// This structure contains the result of processing a payload or fork choice update. diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index 5056a3ba341..e42dc38f5cf 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -111,8 +111,18 @@ impl Transaction { let (gas_price, max_fee_per_gas) = match signed_tx.tx_type() { TxType::Legacy => (Some(U128::from(signed_tx.max_fee_per_gas())), None), - TxType::EIP2930 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), - TxType::EIP1559 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), + TxType::EIP2930 => (Some(U128::from(signed_tx.max_fee_per_gas())), None), + TxType::EIP1559 => { + // the gas price field for EIP1559 is set to `min(tip, gasFeeCap - baseFee) + + // baseFee` + let gas_price = base_fee + .and_then(|base_fee| { + signed_tx.effective_tip_per_gas(base_fee).map(|tip| tip + base_fee as u128) + }) + .unwrap_or_else(|| signed_tx.max_fee_per_gas()); + + (Some(U128::from(gas_price)), Some(U128::from(signed_tx.max_fee_per_gas()))) + } #[cfg(feature = "optimism")] TxType::DEPOSIT => (None, None), }; diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 62cc70afc46..081d0efbb50 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -378,6 +378,8 @@ impl PoolTransaction for MockTransaction { MockTransaction::Eip1559 { max_fee_per_gas, gas_limit, .. } => { U256::from(*gas_limit) * U256::from(*max_fee_per_gas) } + #[cfg(feature = "optimism")] + MockTransaction::DepositTx { .. } => U256::ZERO, } } @@ -387,10 +389,10 @@ impl PoolTransaction for MockTransaction { fn max_fee_per_gas(&self) -> u128 { match self { + MockTransaction::Legacy { gas_price, .. } => *gas_price, + MockTransaction::Eip1559 { max_fee_per_gas, .. } => *max_fee_per_gas, #[cfg(feature = "optimism")] - MockTransaction::DepositTx { .. } => None, - MockTransaction::Legacy { .. } => None, - MockTransaction::Eip1559 { max_fee_per_gas, .. } => Some(*max_fee_per_gas), + MockTransaction::DepositTx { .. } => 0, } } @@ -421,7 +423,7 @@ impl PoolTransaction for MockTransaction { fn tx_type(&self) -> u8 { match self { #[cfg(feature = "optimism")] - MockTransaction::DepositTx { .. } => DEPOSIT_TX_TYPE, + MockTransaction::DepositTx { .. } => TxType::DEPOSIT.into(), MockTransaction::Legacy { .. } => TxType::Legacy.into(), MockTransaction::Eip1559 { .. } => TxType::EIP1559.into(), } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 72f2b3e0f28..1765ad05120 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -523,11 +523,11 @@ impl PoolTransaction for PooledTransaction { /// This is also commonly referred to as the "Gas Fee Cap" (`GasFeeCap`). fn max_fee_per_gas(&self) -> u128 { match &self.transaction.transaction { - Transaction::Legacy(_) => None, - Transaction::Eip2930(_) => None, - Transaction::Eip1559(tx) => Some(tx.max_fee_per_gas), + Transaction::Legacy(tx) => tx.gas_price, + Transaction::Eip2930(tx) => tx.gas_price, + Transaction::Eip1559(tx) => tx.max_fee_per_gas, #[cfg(feature = "optimism")] - Transaction::Deposit(_) => None, + Transaction::Deposit(_) => 0, } } @@ -573,29 +573,16 @@ impl PoolTransaction for PooledTransaction { impl FromRecoveredTransaction for PooledTransaction { fn from_recovered_transaction(tx: TransactionSignedEcRecovered) -> Self { - let (cost, effective_gas_price) = match &tx.transaction { - Transaction::Legacy(t) => { - let cost = U256::from(t.gas_price) * U256::from(t.gas_limit) + U256::from(t.value); - let effective_gas_price = t.gas_price; - (cost, effective_gas_price) - } - Transaction::Eip2930(t) => { - let cost = U256::from(t.gas_price) * U256::from(t.gas_limit) + U256::from(t.value); - let effective_gas_price = t.gas_price; - (cost, effective_gas_price) - } - Transaction::Eip1559(t) => { - let cost = - U256::from(t.max_fee_per_gas) * U256::from(t.gas_limit) + U256::from(t.value); - let effective_gas_price = t.max_priority_fee_per_gas; - (cost, effective_gas_price) - } + let gas_cost = match &tx.transaction { + Transaction::Legacy(t) => U256::from(t.gas_price) * U256::from(t.gas_limit), + Transaction::Eip2930(t) => U256::from(t.gas_price) * U256::from(t.gas_limit), + Transaction::Eip1559(t) => U256::from(t.max_fee_per_gas) * U256::from(t.gas_limit), #[cfg(feature = "optimism")] Transaction::Deposit(t) => { // Gas price is always set to 0 for deposits in order to zero out ETH refunds, // because they already pay for their gas on L1. let gas_price = U256::from(0); - let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); + let cost = U256::from(t.value); let effective_gas_price = 0u128; (cost, effective_gas_price) } From dee3977aec5ebf92a428f5c70bb2da3e1fd6e57f Mon Sep 17 00:00:00 2001 From: clabby Date: Sat, 8 Jul 2023 20:53:30 -0400 Subject: [PATCH 032/150] More conflict resolution / updates x --- bin/reth/Cargo.toml | 10 ++ crates/consensus/beacon/Cargo.toml | 9 + .../interfaces/src/blockchain_tree/error.rs | 3 + .../interfaces/src/test_utils/generators.rs | 2 + crates/payload/basic/Cargo.toml | 8 + crates/payload/basic/src/lib.rs | 2 + crates/payload/builder/Cargo.toml | 6 + crates/primitives/src/transaction/mod.rs | 17 +- crates/revm/src/executor.rs | 156 +++--------------- crates/revm/src/lib.rs | 4 + crates/revm/src/optimism.rs | 21 ++- crates/storage/provider/Cargo.toml | 5 + .../storage/provider/src/test_utils/blocks.rs | 4 + crates/transaction-pool/src/traits.rs | 7 +- 14 files changed, 104 insertions(+), 150 deletions(-) diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 1a2fc959125..ff92e6bc444 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -95,6 +95,16 @@ min-warn-logs = ["tracing/release_max_level_warn"] min-info-logs = ["tracing/release_max_level_info"] min-debug-logs = ["tracing/release_max_level_debug"] min-trace-logs = ["tracing/release_max_level_trace"] +optimism = [ + "reth-primitives/optimism", + "reth-revm/optimism", + "reth-interfaces/optimism", + "reth-rpc-engine-api/optimism", + "reth-transaction-pool/optimism", + "reth-provider/optimism", + "reth-beacon-consensus/optimism", + "reth-basic-payload-builder/optimism", +] [build-dependencies] vergen = { version = "8.0.0", features = ["build", "cargo", "git", "gitcl"] } diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index 64845e971d8..fd5aac45c6b 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -41,3 +41,12 @@ reth-provider = { workspace = true, features = ["test-utils"] } reth-tracing = { path = "../../tracing" } assert_matches = "1.5" + +[features] +optimism = [ + "reth-consensus-common/optimism", + "reth-primitives/optimism", + "reth-interfaces/optimism", + "reth-provider/optimism", + "reth-rpc-types/optimism", +] diff --git a/crates/interfaces/src/blockchain_tree/error.rs b/crates/interfaces/src/blockchain_tree/error.rs index edcdb569bae..c2e463a7328 100644 --- a/crates/interfaces/src/blockchain_tree/error.rs +++ b/crates/interfaces/src/blockchain_tree/error.rs @@ -194,6 +194,9 @@ impl InsertBlockErrorKind { BlockExecutionError::BlockHashNotFoundInChain { .. } | BlockExecutionError::AppendChainDoesntConnect { .. } | BlockExecutionError::UnavailableForTest => false, + #[cfg(feature = "optimism")] + BlockExecutionError::L1BlockInfoError { .. } | + BlockExecutionError::InsufficientFundsForL1Cost { .. } => false, } } InsertBlockErrorKind::Tree(err) => { diff --git a/crates/interfaces/src/test_utils/generators.rs b/crates/interfaces/src/test_utils/generators.rs index dfcdf39a0c5..c5d7132a563 100644 --- a/crates/interfaces/src/test_utils/generators.rs +++ b/crates/interfaces/src/test_utils/generators.rs @@ -343,6 +343,8 @@ pub fn random_receipt( } else { vec![] }, + #[cfg(feature = "optimism")] + deposit_nonce: None, } } diff --git a/crates/payload/basic/Cargo.toml b/crates/payload/basic/Cargo.toml index a2a9f3288af..3e7efd4810e 100644 --- a/crates/payload/basic/Cargo.toml +++ b/crates/payload/basic/Cargo.toml @@ -29,3 +29,11 @@ futures-util = { workspace = true } ## misc tracing = { workspace = true } + +[features] +optimism = [ + "reth-primitives/optimism", + "reth-revm/optimism", + "reth-transaction-pool/optimism", + "reth-provider/optimism" +] diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index c598ab53509..98325e5101a 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -659,6 +659,8 @@ fn build_payload( success: result.is_success(), cumulative_gas_used, logs: result.logs().into_iter().map(into_reth_log).collect(), + #[cfg(feature = "optimism")] + deposit_nonce: None, }, ); diff --git a/crates/payload/builder/Cargo.toml b/crates/payload/builder/Cargo.toml index b83236fd9b3..7f2a80337b6 100644 --- a/crates/payload/builder/Cargo.toml +++ b/crates/payload/builder/Cargo.toml @@ -32,3 +32,9 @@ tracing = { workspace = true } [features] test-utils = [] +optimism = [ + "reth-primitives/optimism", + "reth-rpc-types/optimism", + "reth-interfaces/optimism", + "reth-revm-primitives/optimism", +] diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index c35162117d0..4622e80f24f 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -11,8 +11,6 @@ use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, }; -#[cfg(feature = "optimism")] -use revm_primitives::U256; use serde::{Deserialize, Serialize}; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; @@ -241,6 +239,8 @@ impl Transaction { Transaction::Legacy(tx) => tx.nonce = nonce, Transaction::Eip2930(tx) => tx.nonce = nonce, Transaction::Eip1559(tx) => tx.nonce = nonce, + #[cfg(feature = "optimism")] + Transaction::Deposit(_tx) => { /* noop */ } } } @@ -250,6 +250,8 @@ impl Transaction { Transaction::Legacy(tx) => tx.value = value, Transaction::Eip2930(tx) => tx.value = value, Transaction::Eip1559(tx) => tx.value = value, + #[cfg(feature = "optimism")] + Transaction::Deposit(tx) => tx.value = value, } } @@ -259,6 +261,8 @@ impl Transaction { Transaction::Legacy(tx) => tx.input = input, Transaction::Eip2930(tx) => tx.input = input, Transaction::Eip1559(tx) => tx.input = input, + #[cfg(feature = "optimism")] + Transaction::Deposit(tx) => tx.input = input, } } } @@ -281,6 +285,11 @@ impl Compact for Transaction { tx.to_compact(buf); 2 } + #[cfg(feature = "optimism")] + Transaction::Deposit(tx) => { + tx.to_compact(buf); + 126 + } } } @@ -443,6 +452,8 @@ impl Transaction { Transaction::Eip1559(TxEip1559 { max_priority_fee_per_gas, .. }) => { *max_priority_fee_per_gas } + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => 0, } } @@ -454,6 +465,8 @@ impl Transaction { Transaction::Legacy(tx) => tx.gas_price, Transaction::Eip2930(tx) => tx.gas_price, Transaction::Eip1559(dynamic_tx) => dynamic_tx.effective_gas_price(base_fee), + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => 0, } } diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 4a9fe12184a..c2ed5c7e53f 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -231,7 +231,7 @@ where self.init_env(&block.header, total_difficulty); #[cfg(feature = "optimism")] - let mut l1_block_info = optimism::L1BlockInfo::new(block)?; + let mut l1_block_info = optimism::L1BlockInfo::try_from(block)?; let mut cumulative_gas_used = 0; let mut post_state = PostState::with_tx_capacity(block.number, block.body.len()); @@ -246,147 +246,35 @@ where } .into()) } + // Execute transaction. + let ResultAndState { result, state } = self.transact(transaction, sender)?; - #[cfg(feature = "optimism")] - { - let db = self.db(); - let l1_cost = l1_block_info.calculate_tx_l1_cost(transaction); - - let sender_account = db.load_account(sender).map_err(|_| Error::ProviderError)?; - let old_sender_info = to_reth_acc(&sender_account.info); - if let Some(m) = transaction.mint() { - // Add balance to the caller account equal to the minted amount. - // Note: this is unconditional, and will not be reverted if the tx fails - // (unless the block can't be built at all due to gas limit constraints) - sender_account.info.balance += U256::from(m); - } - - // Check if the sender balance can cover the L1 cost. - // Deposits pay for their gas directly on L1 so they are exempt from this - if !transaction.is_deposit() { - if sender_account.info.balance.cmp(&l1_cost) == std::cmp::Ordering::Less { - return Err(Error::InsufficientFundsForL1Cost { - have: sender_account.info.balance.to::(), - want: l1_cost.to::(), - }) - } - - // Safely take l1_cost from sender (the rest will be deducted by the - // internal EVM execution and included in result.gas_used()) - // TODO: need to handle calls with `disable_balance_check` flag set? - sender_account.info.balance -= l1_cost; - } - - let new_sender_info = to_reth_acc(&sender_account.info); - post_state.change_account(sender, old_sender_info, new_sender_info); - - // Execute transaction. - let ResultAndState { result, state } = self.transact(transaction, sender)?; - - if transaction.is_deposit() && !result.is_success() { - // If the Deposited transaction failed, the deposit must still be included. - // In this case, we need to increment the sender nonce and disregard the - // state changes. The transaction is also recorded as using all gas. - let db = self.db(); - let sender_account = - db.load_account(sender).map_err(|_| Error::ProviderError)?; - let old_sender_info = to_reth_acc(&sender_account.info); - sender_account.info.nonce += 1; - let new_sender_info = to_reth_acc(&sender_account.info); - - post_state.change_account(sender, old_sender_info, new_sender_info); - if !transaction.is_system_transaction() { - cumulative_gas_used += transaction.gas_limit(); - } - - post_state.add_receipt(Receipt { - tx_type: transaction.tx_type(), - success: false, - cumulative_gas_used, - bloom: Bloom::zero(), - logs: vec![], - deposit_nonce: Some(transaction.nonce()), - }); - post_state.finish_transition(); - continue - } - - // commit changes - self.commit_changes( - state, - self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), - &mut post_state, - ); - - if !transaction.is_system_transaction() { - // After Regolith, deposits are reported as using the actual gas used instead of - // all the gas. System transactions are not reported as using any gas. - cumulative_gas_used += result.gas_used() - } - - // Route the l1 cost and base fee to the appropriate optimism vaults - self.increment_account_balance( - optimism::l1_cost_recipient(), - l1_cost, - &mut post_state, - )?; - self.increment_account_balance( - optimism::base_fee_recipient(), - U256::from( - block - .base_fee_per_gas - .unwrap_or_default() - .saturating_mul(result.gas_used()), - ), - &mut post_state, - )?; - - // cast revm logs to reth logs - let logs: Vec = result.logs().into_iter().map(into_reth_log).collect(); - - // Push transaction changeset and calculate header bloom filter for receipt. - post_state.add_receipt(Receipt { - tx_type: transaction.tx_type(), - // Success flag was added in `EIP-658: Embedding transaction status code in - // receipts`. - success: result.is_success(), - cumulative_gas_used, - bloom: logs_bloom(logs.iter()), - logs, - deposit_nonce: Some(transaction.nonce()), - }); - post_state.finish_transition(); - } - - #[cfg(not(feature = "optimism"))] - { - // Execute transaction. - let ResultAndState { result, state } = self.transact(transaction, sender)?; - - // commit changes - self.commit_changes( - state, - self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), - &mut post_state, - ); - - cumulative_gas_used += result.gas_used(); + // commit changes + self.commit_changes( + block.number, + state, + self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), + &mut post_state, + ); - // cast revm logs to reth logs - let logs: Vec = result.logs().into_iter().map(into_reth_log).collect(); + // append gas used + cumulative_gas_used += result.gas_used(); - // Push transaction changeset and calculate header bloom filter for receipt. - post_state.add_receipt(Receipt { + // Push transaction changeset and calculate header bloom filter for receipt. + post_state.add_receipt( + block.number, + Receipt { tx_type: transaction.tx_type(), // Success flag was added in `EIP-658: Embedding transaction status code in // receipts`. success: result.is_success(), cumulative_gas_used, - bloom: logs_bloom(logs.iter()), - logs, - }); - post_state.finish_transition(); - } + // convert to reth log + logs: result.into_logs().into_iter().map(into_reth_log).collect(), + #[cfg(feature = "optimism")] + deposit_nonce: None, + }, + ); } Ok((post_state, cumulative_gas_used)) diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index 6a99b020381..d7e4a973d71 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -33,3 +33,7 @@ pub use revm; /// Etereum DAO hardfork state change data. pub mod eth_dao_fork; + +/// Optimism-specific utilities for the executor +#[cfg(feature = "optimism")] +mod optimism; diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 9f579d8b688..6d1185e0af0 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -27,20 +27,21 @@ pub struct L1BlockInfo { l1_fee_scalar: U256, } -impl L1BlockInfo { - /// Create a new L1 block info struct from a L2 block - pub fn new(block: &Block) -> Result { +impl TryFrom<&Block> for L1BlockInfo { + type Error = executor::BlockExecutionError; + + fn try_from(block: &Block) -> Result { let l1_block_contract = Address::from_str(L1_BLOCK_CONTRACT).unwrap(); let l1_info_tx_data = block .body .iter() .find(|tx| matches!(tx.kind(), TransactionKind::Call(to) if to == &l1_block_contract)) - .ok_or(executor::Error::L1BlockInfoError { + .ok_or(executor::BlockExecutionError::L1BlockInfoError { message: "could not find l1 block info tx in the L2 block".to_string(), }) .and_then(|tx| { - tx.input().get(4..).ok_or(executor::Error::L1BlockInfoError { + tx.input().get(4..).ok_or(executor::BlockExecutionError::L1BlockInfoError { message: "could not get l1 block info tx calldata bytes".to_string(), }) })?; @@ -56,30 +57,32 @@ impl L1BlockInfo { // + 32 bytes for the fee overhead // + 32 bytes for the fee scalar if l1_info_tx_data.len() != 184 { - return Err(executor::Error::L1BlockInfoError { + return Err(executor::BlockExecutionError::L1BlockInfoError { message: "unexpected l1 block info tx calldata length found".to_string(), }) } let l1_base_fee = U256::try_from_le_slice(&l1_info_tx_data[16..48]).ok_or( - executor::Error::L1BlockInfoError { + executor::BlockExecutionError::L1BlockInfoError { message: "could not convert l1 base fee".to_string(), }, )?; let l1_fee_overhead = U256::try_from_le_slice(&l1_info_tx_data[120..152]).ok_or( - executor::Error::L1BlockInfoError { + executor::BlockExecutionError::L1BlockInfoError { message: "could not convert l1 fee overhead".to_string(), }, )?; let l1_fee_scalar = U256::try_from_le_slice(&l1_info_tx_data[152..184]).ok_or( - executor::Error::L1BlockInfoError { + executor::BlockExecutionError::L1BlockInfoError { message: "could not convert l1 fee scalar".to_string(), }, )?; Ok(Self { l1_base_fee, l1_fee_overhead, l1_fee_scalar }) } +} +impl L1BlockInfo { /// Calculate the gas cost of a transaction based on L1 block data posted on L2 pub fn calculate_tx_l1_cost(&mut self, tx: &TransactionSigned) -> U256 { let rollup_data_gas_cost = U256::from(tx.input().iter().fold(0, |acc, byte| { diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 07a2a9a02f2..0cde0e6c214 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -43,3 +43,8 @@ tempfile = "3.3" [features] test-utils = ["reth-rlp"] +optimism = [ + "reth-primitives/optimism", + "reth-interfaces/optimism", + "reth-revm-primitives/optimism" +] diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index 7df40d45e00..6752a0cfb72 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -125,6 +125,8 @@ fn block1(number: BlockNumber) -> (SealedBlockWithSenders, PostState) { topics: vec![H256::from_low_u64_be(1), H256::from_low_u64_be(2)], data: Bytes::default(), }], + #[cfg(feature = "optimism")] + deposit_nonce: None, }, ); @@ -168,6 +170,8 @@ fn block2(number: BlockNumber, parent_hash: H256) -> (SealedBlockWithSenders, Po topics: vec![H256::from_low_u64_be(3), H256::from_low_u64_be(4)], data: Bytes::default(), }], + #[cfg(feature = "optimism")] + deposit_nonce: None, }, ); diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 1765ad05120..3cfec4365aa 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -578,13 +578,10 @@ impl FromRecoveredTransaction for PooledTransaction { Transaction::Eip2930(t) => U256::from(t.gas_price) * U256::from(t.gas_limit), Transaction::Eip1559(t) => U256::from(t.max_fee_per_gas) * U256::from(t.gas_limit), #[cfg(feature = "optimism")] - Transaction::Deposit(t) => { + Transaction::Deposit(_) => { // Gas price is always set to 0 for deposits in order to zero out ETH refunds, // because they already pay for their gas on L1. - let gas_price = U256::from(0); - let cost = U256::from(t.value); - let effective_gas_price = 0u128; - (cost, effective_gas_price) + U256::ZERO } }; let cost = gas_cost + U256::from(tx.value()); From ac826c9c886a4f5e29e58881798c66744db875f0 Mon Sep 17 00:00:00 2001 From: clabby Date: Sat, 8 Jul 2023 21:19:50 -0400 Subject: [PATCH 033/150] Reimplement @merklefruit's changes --- crates/revm/src/executor.rs | 198 +++++++++++++++++++++++++++++------- crates/revm/src/lib.rs | 2 +- 2 files changed, 164 insertions(+), 36 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index c2ed5c7e53f..7930495e997 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -28,6 +28,8 @@ use std::{ #[cfg(feature = "optimism")] use crate::optimism; +#[cfg(feature = "optimism")] +use reth_primitives::Log; /// Main block executor pub struct Executor @@ -236,45 +238,171 @@ where let mut cumulative_gas_used = 0; let mut post_state = PostState::with_tx_capacity(block.number, block.body.len()); for (transaction, sender) in block.body.iter().zip(senders) { - // The sum of the transaction’s gas limit, Tg, and the gas utilised in this block prior, - // must be no greater than the block’s gasLimit. - let block_available_gas = block.header.gas_limit - cumulative_gas_used; - if transaction.gas_limit() > block_available_gas { - return Err(BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { - transaction_gas_limit: transaction.gas_limit(), - block_available_gas, + #[cfg(feature = "optimism")] + { + let db = self.db(); + let l1_cost = l1_block_info.calculate_tx_l1_cost(transaction); + + let sender_account = + db.load_account(sender).map_err(|_| BlockExecutionError::ProviderError)?; + let old_sender_info = to_reth_acc(&sender_account.info); + + if let Some(m) = transaction.mint() { + // Add balance to the caler account equal to the minted amount. + // Note: This is unconditional, and will not be reverted if the tx fails + // (unless the block can't be built at all due to gas limit constraints) + sender_account.info.balance += U256::from(m); } - .into()) - } - // Execute transaction. - let ResultAndState { result, state } = self.transact(transaction, sender)?; - // commit changes - self.commit_changes( - block.number, - state, - self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), - &mut post_state, - ); + // Check if the sender balance can cover the L1 cost. + // Deposits pay for their gas directly on L1 so they are exempt from the L2 tx fee. + if !transaction.is_deposit() { + if sender_account.info.balance.cmp(&l1_cost) == std::cmp::Ordering::Less { + return Err(BlockExecutionError::InsufficientFundsForL1Cost { + have: sender_account.info.balance.to::(), + want: l1_cost.to::(), + }) + } - // append gas used - cumulative_gas_used += result.gas_used(); + // Safely take l1_cost from sender (the rest will be deducted by the + // internal EVM execution and included in result.gas_used()) + // TODO: need to handle calls with `disable_balance_check` flag set? + sender_account.info.balance -= l1_cost; + } - // Push transaction changeset and calculate header bloom filter for receipt. - post_state.add_receipt( - block.number, - Receipt { - tx_type: transaction.tx_type(), - // Success flag was added in `EIP-658: Embedding transaction status code in - // receipts`. - success: result.is_success(), - cumulative_gas_used, - // convert to reth log - logs: result.into_logs().into_iter().map(into_reth_log).collect(), - #[cfg(feature = "optimism")] - deposit_nonce: None, - }, - ); + let new_sender_info = to_reth_acc(&sender_account.info); + post_state.change_account(block.number, sender, old_sender_info, new_sender_info); + + // Execute transaction. + let ResultAndState { result, state } = self.transact(transaction, sender)?; + + if transaction.is_deposit() && !result.is_success() { + // If the Deposited transaction failed, the deposit must still be included. + // In this case, we need to increment the sender nonce and disregard the + // state changes. The transaction is also recorded as using all gas. + let db = self.db(); + let sender_account = + db.load_account(sender).map_err(|_| BlockExecutionError::ProviderError)?; + let old_sender_info = to_reth_acc(&sender_account.info); + sender_account.info.nonce += 1; + let new_sender_info = to_reth_acc(&sender_account.info); + + post_state.change_account( + block.number, + sender, + old_sender_info, + new_sender_info, + ); + if !transaction.is_system_transaction() { + cumulative_gas_used += transaction.gas_limit(); + } + + post_state.add_receipt( + block.number, + Receipt { + tx_type: transaction.tx_type(), + success: false, + cumulative_gas_used, + logs: vec![], + deposit_nonce: Some(transaction.nonce()), + }, + ); + continue + } + + // commit changes + self.commit_changes( + block.number, + state, + self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), + &mut post_state, + ); + + if !transaction.is_system_transaction() { + // After Regolith, deposits are reported as using the actual gas used instead of + // all the gas. System transactions are not reported as using any gas. + cumulative_gas_used += result.gas_used() + } + + // Route the l1 cost and base fee to the appropriate optimism vaults + self.increment_account_balance( + block.number, + optimism::l1_cost_recipient(), + l1_cost, + &mut post_state, + )?; + self.increment_account_balance( + block.number, + optimism::base_fee_recipient(), + U256::from( + block + .base_fee_per_gas + .unwrap_or_default() + .saturating_mul(result.gas_used()), + ), + &mut post_state, + )?; + + // cast revm logs to reth logs + let logs: Vec = result.logs().into_iter().map(into_reth_log).collect(); + + // Push transaction changeset and calculate header bloom filter for receipt. + post_state.add_receipt( + block.number, + Receipt { + tx_type: transaction.tx_type(), + // Success flag was added in `EIP-658: Embedding transaction status code in + // receipts`. + success: result.is_success(), + cumulative_gas_used, + logs, + deposit_nonce: Some(transaction.nonce()), + }, + ); + } + + #[cfg(not(feature = "optimism"))] + { + // The sum of the transaction’s gas limit, Tg, and the gas utilised in this block + // prior, must be no greater than the block’s gasLimit. + let block_available_gas = block.header.gas_limit - cumulative_gas_used; + if transaction.gas_limit() > block_available_gas { + return Err(BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { + transaction_gas_limit: transaction.gas_limit(), + block_available_gas, + } + .into()) + } + // Execute transaction. + let ResultAndState { result, state } = self.transact(transaction, sender)?; + + // commit changes + self.commit_changes( + block.number, + state, + self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), + &mut post_state, + ); + + // append gas used + cumulative_gas_used += result.gas_used(); + + // Push transaction changeset and calculate header bloom filter for receipt. + post_state.add_receipt( + block.number, + Receipt { + tx_type: transaction.tx_type(), + // Success flag was added in `EIP-658: Embedding transaction status code in + // receipts`. + success: result.is_success(), + cumulative_gas_used, + // convert to reth log + logs: result.into_logs().into_iter().map(into_reth_log).collect(), + #[cfg(feature = "optimism")] + deposit_nonce: None, + }, + ); + } } Ok((post_state, cumulative_gas_used)) diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index d7e4a973d71..612a0f6482c 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -36,4 +36,4 @@ pub mod eth_dao_fork; /// Optimism-specific utilities for the executor #[cfg(feature = "optimism")] -mod optimism; +pub mod optimism; From bc46ac2c83be5a634aa72bc1427e68b119c42055 Mon Sep 17 00:00:00 2001 From: clabby Date: Sat, 8 Jul 2023 22:15:33 -0400 Subject: [PATCH 034/150] :broom: --- bin/reth/src/init.rs | 2 ++ crates/consensus/common/src/validation.rs | 5 +---- crates/net/eth-wire/src/types/receipts.rs | 10 +++++----- crates/primitives/src/chain/spec.rs | 7 ++----- crates/primitives/src/receipt.rs | 1 - crates/revm/src/executor.rs | 2 -- crates/revm/src/optimism.rs | 2 +- 7 files changed, 11 insertions(+), 18 deletions(-) diff --git a/bin/reth/src/init.rs b/bin/reth/src/init.rs index 30373b3100b..6c2e5593df9 100644 --- a/bin/reth/src/init.rs +++ b/bin/reth/src/init.rs @@ -271,6 +271,8 @@ mod tests { fork_timestamps: ForkTimestamps { shanghai: None }, genesis_hash: None, paris_block_and_final_difficulty: None, + #[cfg(feature = "optimism")] + optimism: None, }); let db = create_test_rw_db(); diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index 2cca186e171..a1907dbe9e1 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -103,10 +103,7 @@ pub fn validate_transaction_regarding_header( Some(*chain_id) } #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { .. }) => { - // TODO: I believe the chain id should be None here, but have to confirm. - None - } + Transaction::Deposit(TxDeposit { .. }) => None, }; if let Some(chain_id) = chain_id { if chain_id != chain_spec.chain().id() { diff --git a/crates/net/eth-wire/src/types/receipts.rs b/crates/net/eth-wire/src/types/receipts.rs index 8122ec8b437..af40108aa99 100644 --- a/crates/net/eth-wire/src/types/receipts.rs +++ b/crates/net/eth-wire/src/types/receipts.rs @@ -119,11 +119,11 @@ mod test { data: hex!("0100ff")[..].into(), }, ], - bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), - } - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, + }, + bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), }, ]]), }; diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index d3dfec9ddb5..732fb13abac 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -156,11 +156,8 @@ pub static OP_GOERLI: Lazy> = Lazy::new(|| { (Hardfork::MuirGlacier, ForkCondition::Block(0)), (Hardfork::Berlin, ForkCondition::Block(0)), (Hardfork::London, ForkCondition::Block(0)), - ( - Hardfork::Paris, - ForkCondition::TTD { fork_block: Some(0), total_difficulty: U256::from(0) }, - ), - (Hardfork::Shanghai, ForkCondition::Timestamp(0)), + (Hardfork::Paris, ForkCondition::Block(0)), + (Hardfork::Regolith, ForkCondition::Timestamp(1679079600)), ]), optimism: Some(OptimismConfig { eip_1559_elasticity: 10, eip_1559_denominator: 50 }), } diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 537d147ebff..85b286c1462 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -115,7 +115,6 @@ impl ReceiptWithBloom { success, cumulative_gas_used, logs, - #[cfg(feature = "optimism")] deposit_nonce: Some(deposit_nonce), } } diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 7930495e997..5d45ebc15b3 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -398,8 +398,6 @@ where cumulative_gas_used, // convert to reth log logs: result.into_logs().into_iter().map(into_reth_log).collect(), - #[cfg(feature = "optimism")] - deposit_nonce: None, }, ); } diff --git a/crates/revm/src/optimism.rs b/crates/revm/src/optimism.rs index 6d1185e0af0..deeb9dcc684 100644 --- a/crates/revm/src/optimism.rs +++ b/crates/revm/src/optimism.rs @@ -86,7 +86,7 @@ impl L1BlockInfo { /// Calculate the gas cost of a transaction based on L1 block data posted on L2 pub fn calculate_tx_l1_cost(&mut self, tx: &TransactionSigned) -> U256 { let rollup_data_gas_cost = U256::from(tx.input().iter().fold(0, |acc, byte| { - acc + if byte == &0x00 { ZERO_BYTE_COST } else { NON_ZERO_BYTE_COST } + acc + if *byte == 0x00 { ZERO_BYTE_COST } else { NON_ZERO_BYTE_COST } })); if tx.is_deposit() || rollup_data_gas_cost == U256::ZERO { From 05e20473971e282eb350364093087b164dc4dd0a Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 9 Jul 2023 22:58:44 +0200 Subject: [PATCH 035/150] chore: removed useless import --- crates/revm/src/executor.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 5d45ebc15b3..9d95d7e6123 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -28,8 +28,6 @@ use std::{ #[cfg(feature = "optimism")] use crate::optimism; -#[cfg(feature = "optimism")] -use reth_primitives::Log; /// Main block executor pub struct Executor @@ -344,7 +342,7 @@ where )?; // cast revm logs to reth logs - let logs: Vec = result.logs().into_iter().map(into_reth_log).collect(); + let logs = result.logs().into_iter().map(into_reth_log).collect(); // Push transaction changeset and calculate header bloom filter for receipt. post_state.add_receipt( From 7070af6fff656a7b1d76a58703e8df978c91431f Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Wed, 12 Jul 2023 19:31:12 +0200 Subject: [PATCH 036/150] chore: fixed broken feature dependencies --- crates/consensus/common/Cargo.toml | 7 ++++++- crates/interfaces/Cargo.toml | 2 +- crates/net/eth-wire/Cargo.toml | 2 +- crates/rpc/rpc-types/Cargo.toml | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml index e09f4ba3dfc..4090fc07050 100644 --- a/crates/consensus/common/Cargo.toml +++ b/crates/consensus/common/Cargo.toml @@ -20,4 +20,9 @@ assert_matches = "1.5.0" mockall = "0.11.3" [features] -optimism = ["reth-primitives/optimism"] +optimism = [ + "reth-primitives/optimism", + "reth-interfaces/optimism", + "reth-provider/optimism" +] + diff --git a/crates/interfaces/Cargo.toml b/crates/interfaces/Cargo.toml index 8278bc412ce..11a98f99b3c 100644 --- a/crates/interfaces/Cargo.toml +++ b/crates/interfaces/Cargo.toml @@ -55,4 +55,4 @@ secp256k1 = { workspace = true, features = [ [features] test-utils = ["tokio-stream/sync", "secp256k1", "rand/std_rng"] cli = ["clap"] -optimism = [] +optimism = ["reth-rpc-types/optimism"] diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index 3f6cf4702bf..338ec4cb6a6 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -64,7 +64,7 @@ proptest-derive = "0.3" default = ["serde"] serde = ["dep:serde", "smol_str/serde"] arbitrary = ["reth-primitives/arbitrary", "dep:arbitrary", "dep:proptest", "dep:proptest-derive"] -optimism = [] +optimism = ["reth-primitives/optimism"] [[test]] name = "fuzz_roundtrip" diff --git a/crates/rpc/rpc-types/Cargo.toml b/crates/rpc/rpc-types/Cargo.toml index 0e06fc3e35e..c7037a9ebf3 100644 --- a/crates/rpc/rpc-types/Cargo.toml +++ b/crates/rpc/rpc-types/Cargo.toml @@ -33,4 +33,4 @@ assert_matches = "1.5" similar-asserts = "1.4" [features] -optimism = ["reth-primitives/optimism"] +optimism = ["reth-primitives/optimism", "reth-interfaces/optimism"] From de47555394aed0a1dcb5d741ec6f219c70ee150c Mon Sep 17 00:00:00 2001 From: Bjerg Date: Sun, 9 Jul 2023 19:40:12 +0200 Subject: [PATCH 037/150] chore: use units on dashboard (#3684) --- etc/grafana/dashboards/overview.json | 855 +++++++++++++++------------ 1 file changed, 492 insertions(+), 363 deletions(-) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index d1855a23917..4d27f32f7e8 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -27,7 +27,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "10.0.1" + "version": "9.5.3" }, { "type": "panel", @@ -159,7 +159,7 @@ "showThresholdLabels": false, "showThresholdMarkers": true }, - "pluginVersion": "10.0.1", + "pluginVersion": "9.5.3", "targets": [ { "datasource": { @@ -226,7 +226,7 @@ "showUnfilled": true, "valueMode": "color" }, - "pluginVersion": "10.0.1", + "pluginVersion": "9.5.3", "targets": [ { "datasource": { @@ -435,7 +435,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 17 }, "id": 38, "panels": [], @@ -492,7 +492,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -504,7 +505,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 18 }, "id": 40, "options": { @@ -564,7 +565,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 18 }, "id": 42, "maxDataPoints": 25, @@ -608,7 +609,7 @@ "unit": "percentunit" } }, - "pluginVersion": "10.0.1", + "pluginVersion": "9.5.3", "targets": [ { "datasource": { @@ -655,7 +656,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 26 }, "id": 48, "options": { @@ -747,7 +748,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -763,7 +765,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 26 }, "id": 52, "options": { @@ -821,7 +823,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 34 }, "id": 50, "options": { @@ -884,7 +886,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -988,7 +991,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 34 }, "id": 58, "options": { @@ -1003,7 +1006,7 @@ }, "showHeader": true }, - "pluginVersion": "10.0.1", + "pluginVersion": "9.5.3", "targets": [ { "datasource": { @@ -1029,7 +1032,7 @@ "h": 1, "w": 24, "x": 0, - "y": 84 + "y": 42 }, "id": 46, "panels": [], @@ -1084,7 +1087,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -1096,7 +1100,7 @@ "h": 8, "w": 24, "x": 0, - "y": 85 + "y": 43 }, "id": 56, "options": { @@ -1169,7 +1173,7 @@ "h": 1, "w": 24, "x": 0, - "y": 102 + "y": 51 }, "id": 6, "panels": [], @@ -1224,7 +1228,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1239,7 +1244,7 @@ "h": 8, "w": 8, "x": 0, - "y": 103 + "y": 52 }, "id": 18, "options": { @@ -1316,7 +1321,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1331,7 +1337,7 @@ "h": 8, "w": 8, "x": 8, - "y": 103 + "y": 52 }, "id": 16, "options": { @@ -1433,14 +1439,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "cps" }, "overrides": [] }, @@ -1448,7 +1456,7 @@ "h": 8, "w": 8, "x": 16, - "y": 103 + "y": 52 }, "id": 8, "options": { @@ -1473,7 +1481,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_p2pstream_disconnected_errors{instance=~\"$instance\"}", + "expr": "rate(reth_p2pstream_disconnected_errors{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "P2P stream disconnected", "range": true, "refId": "A" @@ -1484,7 +1492,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_network_pending_session_failures{instance=~\"$instance\"}", + "expr": "rate(reth_network_pending_session_failures{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Failed pending sessions", "range": true, @@ -1496,7 +1504,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_network_invalid_messages_received{instance=~\"$instance\"}", + "expr": "rate(reth_network_invalid_messages_received{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Invalid messages", "range": true, @@ -1527,7 +1535,7 @@ "h": 8, "w": 8, "x": 0, - "y": 111 + "y": 60 }, "id": 54, "options": { @@ -1691,7 +1699,7 @@ "h": 1, "w": 24, "x": 0, - "y": 136 + "y": 68 }, "id": 24, "panels": [], @@ -1745,7 +1753,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1785,7 +1794,7 @@ "h": 8, "w": 12, "x": 0, - "y": 137 + "y": 69 }, "id": 26, "options": { @@ -1899,14 +1908,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "cps" }, "overrides": [] }, @@ -1914,7 +1925,7 @@ "h": 8, "w": 12, "x": 12, - "y": 137 + "y": 69 }, "id": 33, "options": { @@ -1936,7 +1947,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_downloaders_headers_timeout_errors{instance=~\"$instance\"}", + "expr": "rate(reth_downloaders_headers_timeout_errors{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Request timed out", "range": true, "refId": "A" @@ -1947,7 +1958,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_downloaders_headers_unexpected_errors{instance=~\"$instance\"}", + "expr": "rate(reth_downloaders_headers_unexpected_errors{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Unexpected error", "range": true, @@ -1959,7 +1970,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_downloaders_headers_validation_errors{instance=~\"$instance\"}", + "expr": "rate(reth_downloaders_headers_validation_errors{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Invalid response", "range": true, @@ -2015,7 +2026,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2030,7 +2042,7 @@ "h": 8, "w": 12, "x": 0, - "y": 145 + "y": 77 }, "id": 36, "options": { @@ -2079,7 +2091,7 @@ "h": 1, "w": 24, "x": 0, - "y": 170 + "y": 85 }, "id": 32, "panels": [], @@ -2134,14 +2146,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "locale" }, "overrides": [ { @@ -2153,6 +2167,10 @@ { "id": "custom.axisPlacement", "value": "right" + }, + { + "id": "unit", + "value": "ops" } ] }, @@ -2165,6 +2183,10 @@ { "id": "custom.axisPlacement", "value": "right" + }, + { + "id": "unit", + "value": "ops" } ] } @@ -2174,7 +2196,7 @@ "h": 8, "w": 12, "x": 0, - "y": 171 + "y": 86 }, "id": 30, "options": { @@ -2324,10 +2346,12 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] - } + }, + "unit": "cps" }, "overrides": [] }, @@ -2335,7 +2359,7 @@ "h": 8, "w": 12, "x": 12, - "y": 171 + "y": 86 }, "id": 28, "options": { @@ -2357,7 +2381,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_downloaders_bodies_timeout_errors{instance=~\"$instance\"}", + "expr": "rate(reth_downloaders_bodies_timeout_errors{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Request timed out", "range": true, "refId": "A" @@ -2368,7 +2392,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_downloaders_bodies_unexpected_errors{instance=~\"$instance\"}", + "expr": "rate(reth_downloaders_bodies_unexpected_errors{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Unexpected error", "range": true, @@ -2380,7 +2404,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_downloaders_bodies_validation_errors{instance=~\"$instance\"}", + "expr": "rate(reth_downloaders_bodies_validation_errors{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Invalid response", "range": true, @@ -2436,7 +2460,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2451,7 +2476,7 @@ "h": 8, "w": 12, "x": 0, - "y": 179 + "y": 94 }, "id": 35, "options": { @@ -2540,14 +2565,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ { @@ -2559,6 +2586,10 @@ { "id": "custom.axisPlacement", "value": "right" + }, + { + "id": "unit", + "value": "blocks" } ] } @@ -2568,7 +2599,7 @@ "h": 8, "w": 12, "x": 12, - "y": 179 + "y": 94 }, "id": 73, "options": { @@ -2592,7 +2623,7 @@ "editorMode": "builder", "expr": "reth_downloaders_bodies_buffered_blocks_size_bytes{instance=~\"$instance\"}", "hide": false, - "legendFormat": "Buffered blocks size (bytes)", + "legendFormat": "Buffered blocks size ", "range": true, "refId": "A" }, @@ -2618,7 +2649,7 @@ "h": 1, "w": 24, "x": 0, - "y": 204 + "y": 102 }, "id": 89, "panels": [], @@ -2673,14 +2704,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, @@ -2688,7 +2721,7 @@ "h": 8, "w": 12, "x": 0, - "y": 205 + "y": 103 }, "id": 91, "options": { @@ -2711,7 +2744,7 @@ }, "editorMode": "builder", "expr": "reth_transaction_pool_basefee_pool_size_bytes{instance=~\"$instance\"}", - "legendFormat": "Base fee pool size (bytes)", + "legendFormat": "Base fee pool size", "range": true, "refId": "A" }, @@ -2723,7 +2756,7 @@ "editorMode": "builder", "expr": "reth_transaction_pool_pending_pool_size_bytes{instance=~\"$instance\"}", "hide": false, - "legendFormat": "Pending pool size (bytes)", + "legendFormat": "Pending pool size", "range": true, "refId": "B" }, @@ -2735,7 +2768,7 @@ "editorMode": "builder", "expr": "reth_transaction_pool_queued_pool_size_bytes{instance=~\"$instance\"}", "hide": false, - "legendFormat": "Queued pool size (bytes)", + "legendFormat": "Queued pool size", "range": true, "refId": "C" } @@ -2789,7 +2822,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2804,7 +2838,7 @@ "h": 8, "w": 12, "x": 12, - "y": 205 + "y": 103 }, "id": 92, "options": { @@ -2871,7 +2905,7 @@ "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, + "axisCenteredZero": true, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", @@ -2905,22 +2939,53 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "cps" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 213 + "y": 111 }, "id": 93, "options": { @@ -2931,7 +2996,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -2942,7 +3007,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_transaction_pool_inserted_transactions{instance=~\"$instance\"}", + "expr": "increase(reth_transaction_pool_inserted_transactions{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Inserted transactions", "range": true, "refId": "A" @@ -2953,7 +3018,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_transaction_pool_removed_transactions{instance=~\"$instance\"}", + "expr": "increase(reth_transaction_pool_removed_transactions{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Removed transactions", "range": true, @@ -2965,7 +3030,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_transaction_pool_invalid_transactions{instance=~\"$instance\"}", + "expr": "increase(reth_transaction_pool_invalid_transactions{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Invalid transactions", "range": true, @@ -3021,7 +3086,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3036,7 +3102,7 @@ "h": 8, "w": 12, "x": 12, - "y": 213 + "y": 111 }, "id": 94, "options": { @@ -3080,7 +3146,7 @@ "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, + "axisCenteredZero": true, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", @@ -3114,22 +3180,49 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "cps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "events" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 221 + "y": 119 }, "id": 95, "options": { @@ -3150,13 +3243,37 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "reth_network_pool_transactions_messages_sent{instance=~\"$instance\"} - reth_network_pool_transactions_messages_received{instance=~\"$instance\"}", + "editorMode": "builder", + "expr": "increase(reth_network_pool_transactions_messages_sent{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "instant": false, - "legendFormat": "Total events in the channel", + "legendFormat": "Tx", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "increase(reth_network_pool_transactions_messages_received{instance=~\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Rx", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_network_pool_transactions_messages_sent{instance=~\"$instance\"} - reth_network_pool_transactions_messages_received{instance=~\"$instance\"}", + "hide": false, + "legendFormat": "Messages in channel", + "range": true, + "refId": "C" } ], "title": "Network transaction channel", @@ -3168,7 +3285,7 @@ "h": 1, "w": 24, "x": 0, - "y": 254 + "y": 127 }, "id": 79, "panels": [], @@ -3223,7 +3340,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3238,7 +3356,7 @@ "h": 8, "w": 12, "x": 0, - "y": 255 + "y": 128 }, "id": 74, "options": { @@ -3316,7 +3434,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3331,7 +3450,7 @@ "h": 8, "w": 12, "x": 12, - "y": 255 + "y": 128 }, "id": 80, "options": { @@ -3409,7 +3528,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3424,7 +3544,7 @@ "h": 8, "w": 12, "x": 0, - "y": 263 + "y": 136 }, "id": 81, "options": { @@ -3462,7 +3582,7 @@ "h": 1, "w": 24, "x": 0, - "y": 288 + "y": 144 }, "id": 87, "panels": [], @@ -3517,7 +3637,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3532,7 +3653,7 @@ "h": 8, "w": 12, "x": 0, - "y": 289 + "y": 145 }, "id": 83, "options": { @@ -3609,7 +3730,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3624,7 +3746,7 @@ "h": 8, "w": 12, "x": 12, - "y": 289 + "y": 145 }, "id": 84, "options": { @@ -3713,7 +3835,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3728,7 +3851,7 @@ "h": 8, "w": 12, "x": 0, - "y": 297 + "y": 153 }, "id": 85, "options": { @@ -3760,296 +3883,298 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 322 + "y": 161 }, "id": 68, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "panels": [], + "repeat": "instance", + "repeatDirection": "h", + "title": "Payload Builder ($instance)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of active jobs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "Number of active jobs", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 11, - "x": 0, - "y": 33 - }, - "id": 60, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "builder", - "expr": "reth_payloads_active_jobs{instance=~\"$instance\"}", - "legendFormat": "Active Jobs", - "range": true, - "refId": "A" - } - ], - "title": "Active Jobs", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 0, + "y": 162 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of initiated jobs", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "editorMode": "builder", + "expr": "reth_payloads_active_jobs{instance=~\"$instance\"}", + "legendFormat": "Active Jobs", + "range": true, + "refId": "A" + } + ], + "title": "Active Jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of initiated jobs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 13, - "x": 11, - "y": 33 - }, - "id": 62, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "single", - "sort": "none" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "builder", - "expr": "reth_payloads_initiated_jobs{instance=~\"$instance\"}", - "legendFormat": "Initiated Jobs", - "range": true, - "refId": "A" - } - ], - "title": "Initiated Jobs", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 11, + "y": 162 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of failed jobs", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "editorMode": "builder", + "expr": "reth_payloads_initiated_jobs{instance=~\"$instance\"}", + "legendFormat": "Initiated Jobs", + "range": true, + "refId": "A" + } + ], + "title": "Initiated Jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of failed jobs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 0, - "y": 41 - }, - "id": 64, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "builder", - "expr": "reth_payloads_failed_jobs{instance=~\"$instance\"}", - "legendFormat": "Failed Jobs", - "range": true, - "refId": "A" - } - ], - "title": "Failed Jobs", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 170 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_payloads_failed_jobs{instance=~\"$instance\"}", + "legendFormat": "Failed Jobs", + "range": true, + "refId": "A" } ], - "repeat": "instance", - "repeatDirection": "h", - "title": "Payload Builder ($instance)", - "type": "row" + "title": "Failed Jobs", + "type": "timeseries" }, { "collapsed": false, @@ -4057,7 +4182,7 @@ "h": 1, "w": 24, "x": 0, - "y": 324 + "y": 177 }, "id": 97, "panels": [], @@ -4109,7 +4234,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4125,7 +4251,7 @@ "h": 8, "w": 12, "x": 0, - "y": 325 + "y": 178 }, "id": 98, "options": { @@ -4268,7 +4394,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4284,7 +4411,7 @@ "h": 8, "w": 12, "x": 12, - "y": 325 + "y": 178 }, "id": 101, "options": { @@ -4362,7 +4489,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4378,7 +4506,7 @@ "h": 8, "w": 12, "x": 0, - "y": 333 + "y": 186 }, "id": 99, "options": { @@ -4456,7 +4584,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4472,7 +4601,7 @@ "h": 8, "w": 12, "x": 12, - "y": 333 + "y": 186 }, "id": 100, "options": { @@ -4544,6 +4673,6 @@ "timezone": "", "title": "reth", "uid": "2k8BXz24x", - "version": 8, + "version": 2, "weekStart": "" } \ No newline at end of file From a83444578b8238e410aa30195f992bc40d9469d3 Mon Sep 17 00:00:00 2001 From: Josh Stevens Date: Sun, 9 Jul 2023 20:57:16 +0100 Subject: [PATCH 038/150] fix: expose the revm_utils to consumer as needed structs from it (#3686) --- crates/rpc/rpc/src/eth/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 19032dd8e9b..bff4361e486 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -8,7 +8,7 @@ pub mod gas_oracle; mod id_provider; mod logs_utils; mod pubsub; -pub(crate) mod revm_utils; +pub mod revm_utils; mod signer; pub(crate) mod utils; From 97d6f2a6db4010b5f44d154b7b957a0f73f56299 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 10 Jul 2023 19:41:28 +1000 Subject: [PATCH 039/150] fix: remove txn header from getPayloadBodies (#3688) --- crates/rpc/rpc-types/src/eth/engine/payload.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/engine/payload.rs b/crates/rpc/rpc-types/src/eth/engine/payload.rs index 89788ed4690..4919396ad00 100644 --- a/crates/rpc/rpc-types/src/eth/engine/payload.rs +++ b/crates/rpc/rpc-types/src/eth/engine/payload.rs @@ -4,7 +4,7 @@ use reth_primitives::{ Address, Block, Bloom, Bytes, Header, SealedBlock, TransactionSigned, UintTryTo, Withdrawal, H256, H64, U256, U64, }; -use reth_rlp::{Decodable, Encodable}; +use reth_rlp::Decodable; use serde::{ser::SerializeMap, Deserialize, Serialize, Serializer}; /// The execution payload body response that allows for `null` values. @@ -229,7 +229,7 @@ impl From for ExecutionPayloadBody { fn from(value: Block) -> Self { let transactions = value.body.into_iter().map(|tx| { let mut out = Vec::new(); - tx.encode(&mut out); + tx.encode_enveloped(&mut out); out.into() }); ExecutionPayloadBody { From 511dde57a1682711f96fa6d56006956eb8286610 Mon Sep 17 00:00:00 2001 From: "lukebrich.eth" Date: Mon, 10 Jul 2023 06:12:45 -0400 Subject: [PATCH 040/150] feat: enforce txpool propagation setting (#3677) Co-authored-by: Matthias Seitz --- crates/transaction-pool/src/pool/mod.rs | 4 ++-- crates/transaction-pool/src/pool/txpool.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 5ae0147d643..e7f167c82a6 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -232,13 +232,13 @@ where /// Returns hashes of _all_ transactions in the pool. pub(crate) fn pooled_transactions_hashes(&self) -> Vec { let pool = self.pool.read(); - pool.all().hashes_iter().collect() + pool.all().transactions_iter().filter(|tx| tx.propagate).map(|tx| *tx.hash()).collect() } /// Returns _all_ transactions in the pool. pub(crate) fn pooled_transactions(&self) -> Vec>> { let pool = self.pool.read(); - pool.all().transactions_iter().collect() + pool.all().transactions_iter().filter(|tx| tx.propagate).collect() } /// Updates the entire pool after a new block was executed. diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 71b5590a688..9793da32a30 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -637,6 +637,7 @@ impl AllTransactions { } /// Returns an iterator over all _unique_ hashes in the pool + #[allow(unused)] pub(crate) fn hashes_iter(&self) -> impl Iterator + '_ { self.by_hash.keys().copied() } From 4c0582258921ec562f898631d4ac118a3534dec4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 10 Jul 2023 12:16:02 +0200 Subject: [PATCH 041/150] feat: add subpool transaction streams (#3668) --- crates/transaction-pool/src/lib.rs | 5 +- crates/transaction-pool/src/traits.rs | 71 ++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 8689dcc377f..f5c92b8086d 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -156,7 +156,10 @@ pub use crate::{ }, error::PoolResult, ordering::{GasCostOrdering, TransactionOrdering}, - pool::{AllTransactionsEvents, FullTransactionEvent, TransactionEvent, TransactionEvents}, + pool::{ + state::SubPool, AllTransactionsEvents, FullTransactionEvent, TransactionEvent, + TransactionEvents, + }, traits::{ AllPoolTransactions, BestTransactions, BlockInfo, CanonicalStateUpdate, ChangedAccount, NewTransactionEvent, PoolSize, PoolTransaction, PooledTransaction, PropagateKind, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 3cfec4365aa..611db300537 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -4,12 +4,19 @@ use crate::{ validate::ValidPoolTransaction, AllTransactionsEvents, }; +use futures_util::{ready, Stream}; use reth_primitives::{ Address, FromRecoveredTransaction, IntoRecoveredTransaction, PeerId, Transaction, TransactionKind, TransactionSignedEcRecovered, TxHash, EIP1559_TX_TYPE_ID, H256, U256, }; use reth_rlp::Encodable; -use std::{collections::HashMap, fmt, sync::Arc}; +use std::{ + collections::HashMap, + fmt, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; use tokio::sync::mpsc::Receiver; #[cfg(feature = "serde")] @@ -106,6 +113,34 @@ pub trait TransactionPool: Send + Sync + Clone { /// Returns a new stream that yields new valid transactions added to the pool. fn new_transactions_listener(&self) -> Receiver>; + /// Returns a new Stream that yields new transactions added to the basefee-pool. + /// + /// This is a convenience wrapper around [Self::new_transactions_listener] that filters for + /// [SubPool::Pending](crate::SubPool). + fn new_pending_pool_transactions_listener( + &self, + ) -> NewSubpoolTransactionStream { + NewSubpoolTransactionStream::new(self.new_transactions_listener(), SubPool::Pending) + } + + /// Returns a new Stream that yields new transactions added to the basefee sub-pool. + /// + /// This is a convenience wrapper around [Self::new_transactions_listener] that filters for + /// [SubPool::BaseFee](crate::SubPool). + fn new_basefee_pool_transactions_listener( + &self, + ) -> NewSubpoolTransactionStream { + NewSubpoolTransactionStream::new(self.new_transactions_listener(), SubPool::BaseFee) + } + + /// Returns a new Stream that yields new transactions added to the queued-pool. + /// + /// This is a convenience wrapper around [Self::new_transactions_listener] that filters for + /// [SubPool::Queued](crate::SubPool). + fn new_queued_transactions_listener(&self) -> NewSubpoolTransactionStream { + NewSubpoolTransactionStream::new(self.new_transactions_listener(), SubPool::Queued) + } + /// Returns the _hashes_ of all transactions in the pool. /// /// Note: This returns a `Vec` but should guarantee that all hashes are unique. @@ -630,3 +665,37 @@ pub struct BlockInfo { /// currently tracking. pub pending_basefee: u128, } + +/// A Stream that yields full transactions the subpool +#[must_use = "streams do nothing unless polled"] +#[derive(Debug)] +pub struct NewSubpoolTransactionStream { + st: Receiver>, + subpool: SubPool, +} + +// === impl NewSubpoolTransactionStream === + +impl NewSubpoolTransactionStream { + /// Create a new stream that yields full transactions from the subpool + pub fn new(st: Receiver>, subpool: SubPool) -> Self { + Self { st, subpool } + } +} + +impl Stream for NewSubpoolTransactionStream { + type Item = NewTransactionEvent; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + match ready!(self.st.poll_recv(cx)) { + Some(event) => { + if event.subpool == self.subpool { + return Poll::Ready(Some(event)) + } + } + None => return Poll::Ready(None), + } + } + } +} From ccf6700e3a0f617f9236edc8b4da7c74ace46bc9 Mon Sep 17 00:00:00 2001 From: Georgios Konstantopoulos Date: Mon, 10 Jul 2023 14:21:54 +0300 Subject: [PATCH 042/150] release: 0.1.0-alpha.3 (#3691) --- Cargo.lock | 90 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f968525ed50..53fd07832be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -918,7 +918,7 @@ checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "codecs-derive" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "convert_case 0.6.0", "parity-scale-codec", @@ -1748,7 +1748,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "reth-db", "reth-interfaces", @@ -4957,7 +4957,7 @@ dependencies = [ [[package]] name = "reth" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "backon", "clap 4.1.8", @@ -5022,7 +5022,7 @@ dependencies = [ [[package]] name = "reth-auto-seal-consensus" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "futures-util", "reth-beacon-consensus", @@ -5039,7 +5039,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "futures-core", "futures-util", @@ -5058,7 +5058,7 @@ dependencies = [ [[package]] name = "reth-beacon-consensus" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "assert_matches", "futures", @@ -5083,7 +5083,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "aquamarine", "assert_matches", @@ -5102,7 +5102,7 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "arbitrary", "bytes", @@ -5117,7 +5117,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "confy", "reth-discv4", @@ -5134,7 +5134,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "assert_matches", "mockall", @@ -5145,7 +5145,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "arbitrary", "assert_matches", @@ -5186,7 +5186,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "discv5", "enr", @@ -5209,7 +5209,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "async-trait", "data-encoding", @@ -5233,7 +5233,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "assert_matches", "futures", @@ -5258,7 +5258,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "aes 0.8.2", "block-padding", @@ -5289,7 +5289,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "arbitrary", "async-trait", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "reth-interfaces" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "arbitrary", "async-trait", @@ -5350,7 +5350,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "async-trait", "bytes", @@ -5370,7 +5370,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "bitflags 2.3.2", "byteorder", @@ -5390,7 +5390,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "bindgen 0.65.1", "cc", @@ -5399,7 +5399,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "futures", "metrics", @@ -5409,7 +5409,7 @@ dependencies = [ [[package]] name = "reth-metrics-derive" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "metrics", "once_cell", @@ -5423,7 +5423,7 @@ dependencies = [ [[package]] name = "reth-net-common" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "pin-project", "reth-primitives", @@ -5432,7 +5432,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "igd", "pin-project-lite", @@ -5446,7 +5446,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "aquamarine", "async-trait", @@ -5496,7 +5496,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "async-trait", "reth-eth-wire", @@ -5509,7 +5509,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "futures-util", "reth-interfaces", @@ -5528,7 +5528,7 @@ dependencies = [ [[package]] name = "reth-primitives" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "arbitrary", "assert_matches", @@ -5575,7 +5575,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "auto_impl", "derive_more", @@ -5596,7 +5596,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "once_cell", "reth-consensus-common", @@ -5612,7 +5612,7 @@ dependencies = [ [[package]] name = "reth-revm-inspectors" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "boa_engine", "boa_gc", @@ -5628,7 +5628,7 @@ dependencies = [ [[package]] name = "reth-revm-primitives" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "reth-primitives", "revm", @@ -5636,7 +5636,7 @@ dependencies = [ [[package]] name = "reth-rlp" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "arrayvec", "auto_impl", @@ -5654,7 +5654,7 @@ dependencies = [ [[package]] name = "reth-rlp-derive" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "proc-macro2 1.0.63", "quote 1.0.28", @@ -5663,7 +5663,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "assert_matches", "async-trait", @@ -5709,7 +5709,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "jsonrpsee", "reth-primitives", @@ -5719,7 +5719,7 @@ dependencies = [ [[package]] name = "reth-rpc-api-testing-util" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "async-trait", "futures", @@ -5733,7 +5733,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "hyper", "jsonrpsee", @@ -5763,7 +5763,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "assert_matches", "async-trait", @@ -5783,7 +5783,7 @@ dependencies = [ [[package]] name = "reth-rpc-types" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "assert_matches", "jsonrpsee-types", @@ -5799,7 +5799,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "aquamarine", "assert_matches", @@ -5835,7 +5835,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "dyn-clone", "futures-util", @@ -5848,7 +5848,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "tracing", "tracing-appender", @@ -5858,7 +5858,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "aquamarine", "async-trait", @@ -5884,7 +5884,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" dependencies = [ "criterion", "derive_more", diff --git a/Cargo.toml b/Cargo.toml index 56028251849..c73f2f87aba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ default-members = ["bin/reth"] resolver = "2" [workspace.package] -version = "0.1.0-alpha.2" +version = "0.1.0-alpha.3" edition = "2021" rust-version = "1.70" # Remember to update .clippy.toml and README.md license = "MIT OR Apache-2.0" From 72eb9a05f07d034dd2cc6c17058ce6eb17a06e7e Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 10 Jul 2023 12:07:13 +0100 Subject: [PATCH 043/150] feat(config, primitives): validate Receipts prune part (#3587) --- crates/config/src/config.rs | 7 ++- crates/primitives/src/serde_helper/mod.rs | 2 + crates/primitives/src/serde_helper/prune.rs | 49 +++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 crates/primitives/src/serde_helper/prune.rs diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index f0dfb4ac2bb..40836b8c02c 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -5,7 +5,7 @@ use reth_downloaders::{ headers::reverse_headers::ReverseHeadersDownloaderBuilder, }; use reth_network::{NetworkConfigBuilder, PeersConfig, SessionsConfig}; -use reth_primitives::PruneMode; +use reth_primitives::{serde_helper::deserialize_opt_prune_mode_with_min_distance, PruneMode}; use secp256k1::SecretKey; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -305,7 +305,10 @@ pub struct PruneParts { #[serde(skip_serializing_if = "Option::is_none")] pub transaction_lookup: Option, /// Receipts pruning configuration. - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_opt_prune_mode_with_min_distance::<64, _>" + )] pub receipts: Option, /// Account History pruning configuration. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/primitives/src/serde_helper/mod.rs b/crates/primitives/src/serde_helper/mod.rs index c63e7f62756..7cd85f892f9 100644 --- a/crates/primitives/src/serde_helper/mod.rs +++ b/crates/primitives/src/serde_helper/mod.rs @@ -10,6 +10,8 @@ use crate::H256; pub use jsonu256::*; pub mod num; +mod prune; +pub use prune::deserialize_opt_prune_mode_with_min_distance; /// serde functions for handling primitive `u64` as [U64](crate::U64) pub mod u64_hex { diff --git a/crates/primitives/src/serde_helper/prune.rs b/crates/primitives/src/serde_helper/prune.rs new file mode 100644 index 00000000000..7dffafdf8f7 --- /dev/null +++ b/crates/primitives/src/serde_helper/prune.rs @@ -0,0 +1,49 @@ +use crate::PruneMode; +use serde::{Deserialize, Deserializer}; + +/// Deserializes [`Option`] and validates that the value contained in +/// [PruneMode::Distance] (if any) is not less than the const generic parameter `MIN_DISTANCE`. +pub fn deserialize_opt_prune_mode_with_min_distance< + 'de, + const MIN_DISTANCE: u64, + D: Deserializer<'de>, +>( + deserializer: D, +) -> Result, D::Error> { + let prune_mode = Option::::deserialize(deserializer)?; + + match prune_mode { + Some(PruneMode::Distance(distance)) if distance < MIN_DISTANCE => { + Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Unsigned(distance), + // This message should have "expected" wording, so we say "not less than" + &format!("prune mode distance not less than {MIN_DISTANCE} blocks").as_str(), + )) + } + _ => Ok(prune_mode), + } +} + +#[cfg(test)] +mod test { + use crate::PruneMode; + use assert_matches::assert_matches; + use serde::Deserialize; + + #[test] + fn deserialize_opt_prune_mode_with_min_distance() { + #[derive(Debug, Deserialize, PartialEq, Eq)] + struct V( + #[serde( + deserialize_with = "super::deserialize_opt_prune_mode_with_min_distance::<10, _>" + )] + Option, + ); + + assert!(serde_json::from_str::(r#"{"distance": 10}"#).is_ok()); + assert_matches!( + serde_json::from_str::(r#"{"distance": 9}"#), + Err(err) if err.to_string() == "invalid value: integer `9`, expected prune mode distance not less than 10 blocks" + ); + } +} From a9fc15fcf5328e1cd3f1343bba59ee15d7d7e0ef Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 10 Jul 2023 07:23:24 -0400 Subject: [PATCH 044/150] feat: run pipeline if latest finalized is far from pipeline progress (#3662) --- crates/consensus/beacon/src/engine/mod.rs | 71 ++++++++++++++++------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 175f6c5dca7..f8b95d4682c 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -335,7 +335,9 @@ where } /// Returns true if the distance from the local tip to the block is greater than the configured - /// threshold + /// threshold. + /// + /// If the `local_tip` is greater than the `block`, then this will return false. #[inline] fn exceeds_pipeline_run_threshold(&self, local_tip: u64, block: u64) -> bool { block > local_tip && block - local_tip > self.pipeline_run_threshold @@ -1311,27 +1313,56 @@ where ) .is_none() { - // Update the state and hashes of the blockchain tree if possible. - match self - .update_tree_on_finished_pipeline(sync_target_state.finalized_block_hash) + let newest_finalized = self + .forkchoice_state_tracker + .sync_target_state() + .map(|s| s.finalized_block_hash) + .and_then(|h| self.blockchain.buffered_header_by_hash(h)) + .map(|header| header.number); + + // The block number that the pipeline finished at - if the progress or newest + // finalized is None then we can't check the distance anyways. + // + // If both are Some, we perform another distance check and return the desired + // pipeline target + let pipeline_target = if let (Some(progress), Some(finalized_number)) = + (ctrl.progress(), newest_finalized) { - Ok(synced) => { - if synced { - // we're consider this synced and transition to live sync - self.sync_state_updater.update_sync_state(SyncState::Idle); - } else { - // We don't have the finalized block in the database, so - // we need to run another pipeline. - self.sync.set_pipeline_sync_target( - sync_target_state.finalized_block_hash, - ); - } - } - Err(error) => { - error!(target: "consensus::engine", ?error, "Error restoring blockchain tree state"); - return Some(Err(error.into())) - } + // Determines whether or not we should run the pipeline again, in case the + // new gap is large enough to warrant running the pipeline. + self.can_pipeline_sync_to_finalized(progress, finalized_number, None) + } else { + None }; + + // If the distance is large enough, we should run the pipeline again to prevent + // the tree update from executing too many blocks and blocking. + if let Some(target) = pipeline_target { + // run the pipeline to the target since the distance is sufficient + self.sync.set_pipeline_sync_target(target); + } else { + // Update the state and hashes of the blockchain tree if possible. + match self.update_tree_on_finished_pipeline( + sync_target_state.finalized_block_hash, + ) { + Ok(synced) => { + if synced { + // we're consider this synced and transition to live sync + self.sync_state_updater.update_sync_state(SyncState::Idle); + } else { + // We don't have the finalized block in the database, so + // we need to run another pipeline. + self.sync.set_pipeline_sync_target( + sync_target_state.finalized_block_hash, + ); + } + } + Err(error) => { + error!(target: "consensus::engine", ?error, "Error restoring blockchain tree state"); + return Some(Err(error.into())) + } + }; + } } } // Any pipeline error at this point is fatal. From 74788a5fbe6e8df5d89dfd79c465904db9d20f97 Mon Sep 17 00:00:00 2001 From: fomotrader <82184770+fomotrader@users.noreply.github.com> Date: Mon, 10 Jul 2023 07:34:12 -0400 Subject: [PATCH 045/150] feat: add full pending txs to stream (#3649) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-types/src/eth/pubsub.rs | 20 ++++++++--- crates/rpc/rpc/src/eth/pubsub.rs | 46 ++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/pubsub.rs b/crates/rpc/rpc-types/src/eth/pubsub.rs index 63f152b6628..f027037c176 100644 --- a/crates/rpc/rpc-types/src/eth/pubsub.rs +++ b/crates/rpc/rpc-types/src/eth/pubsub.rs @@ -1,6 +1,10 @@ //! Ethereum types for pub-sub -use crate::{eth::Filter, Log, RichHeader}; +use crate::{ + eth::{Filter, Transaction}, + Log, RichHeader, +}; + use reth_primitives::H256; use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; @@ -14,6 +18,8 @@ pub enum SubscriptionResult { Log(Box), /// Transaction hash TransactionHash(H256), + /// Full Transaction + FullTransaction(Box), /// SyncStatus SyncState(PubSubSyncStatus), } @@ -49,6 +55,7 @@ impl Serialize for SubscriptionResult { SubscriptionResult::Header(ref header) => header.serialize(serializer), SubscriptionResult::Log(ref log) => log.serialize(serializer), SubscriptionResult::TransactionHash(ref hash) => hash.serialize(serializer), + SubscriptionResult::FullTransaction(ref tx) => tx.serialize(serializer), SubscriptionResult::SyncState(ref sync) => sync.serialize(serializer), } } @@ -76,10 +83,10 @@ pub enum SubscriptionKind { Logs, /// New Pending Transactions subscription. /// - /// Returns the hash for all transactions that are added to the pending state and are signed - /// with a key that is available in the node. When a transaction that was previously part of - /// the canonical chain isn't part of the new canonical chain after a reorganization its again - /// emitted. + /// Returns the hash or full tx for all transactions that are added to the pending state and + /// are signed with a key that is available in the node. When a transaction that was + /// previously part of the canonical chain isn't part of the new canonical chain after a + /// reorganization its again emitted. NewPendingTransactions, /// Node syncing status subscription. /// @@ -97,6 +104,8 @@ pub enum Params { None, /// Log parameters. Logs(Box), + /// New pending transaction parameters. + NewPendingTransactions(bool), } impl Serialize for Params { @@ -107,6 +116,7 @@ impl Serialize for Params { match self { Params::None => (&[] as &[serde_json::Value]).serialize(serializer), Params::Logs(logs) => logs.serialize(serializer), + Params::NewPendingTransactions(full) => full.serialize(serializer), } } } diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index f474426037e..aa5dd7363f5 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -3,21 +3,22 @@ use crate::eth::logs_utils; use futures::StreamExt; use jsonrpsee::{server::SubscriptionMessage, PendingSubscriptionSink, SubscriptionSink}; use reth_network_api::NetworkInfo; -use reth_primitives::TxHash; +use reth_primitives::{IntoRecoveredTransaction, TxHash}; use reth_provider::{BlockReader, CanonStateSubscriptions, EvmEnvProvider}; use reth_rpc_api::EthPubSubApiServer; use reth_rpc_types::FilteredParams; use std::sync::Arc; +use crate::result::invalid_params_rpc_err; use reth_rpc_types::{ pubsub::{ Params, PubSubSyncStatus, SubscriptionKind, SubscriptionResult as EthSubscriptionResult, SyncStatusMetadata, }, - Header, Log, + Header, Log, Transaction, }; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; -use reth_transaction_pool::TransactionPool; +use reth_transaction_pool::{NewTransactionEvent, TransactionPool}; use serde::Serialize; use tokio_stream::{ wrappers::{BroadcastStream, ReceiverStream}, @@ -121,8 +122,34 @@ where pipe_from_stream(accepted_sink, stream).await } SubscriptionKind::NewPendingTransactions => { - let stream = - pubsub.pending_transaction_stream().map(EthSubscriptionResult::TransactionHash); + if let Some(params) = params { + match params { + Params::NewPendingTransactions(true) => { + // full transaction objects requested + let stream = pubsub.full_pending_transaction_stream().map(|tx| { + EthSubscriptionResult::FullTransaction(Box::new( + Transaction::from_recovered( + tx.transaction.to_recovered_transaction(), + ), + )) + }); + return pipe_from_stream(accepted_sink, stream).await + } + Params::NewPendingTransactions(false) | Params::None => { + // only hashes requested + } + Params::Logs(_) => { + return Err(invalid_params_rpc_err( + "Invalid params for newPendingTransactions", + ) + .into()) + } + } + } + + let stream = pubsub + .pending_transaction_hashes_stream() + .map(EthSubscriptionResult::TransactionHash); pipe_from_stream(accepted_sink, stream).await } SubscriptionKind::Syncing => { @@ -241,9 +268,16 @@ where Pool: TransactionPool + 'static, { /// Returns a stream that yields all transactions emitted by the txpool. - fn pending_transaction_stream(&self) -> impl Stream { + fn pending_transaction_hashes_stream(&self) -> impl Stream { ReceiverStream::new(self.pool.pending_transactions_listener()) } + + /// Returns a stream that yields all transactions emitted by the txpool. + fn full_pending_transaction_stream( + &self, + ) -> impl Stream::Transaction>> { + self.pool.new_pending_pool_transactions_listener() + } } impl EthPubSubInner From c4b4ca8b07c57cd5976bab611090b56496127cf0 Mon Sep 17 00:00:00 2001 From: Sanket Shanbhag Date: Mon, 10 Jul 2023 17:53:41 +0530 Subject: [PATCH 046/150] test: add payload status error serde tests (#2803) Co-authored-by: Georgios Konstantopoulos Co-authored-by: Matthias Seitz --- .../rpc/rpc-types/src/eth/engine/payload.rs | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-types/src/eth/engine/payload.rs b/crates/rpc/rpc-types/src/eth/engine/payload.rs index 4919396ad00..ecc12334122 100644 --- a/crates/rpc/rpc-types/src/eth/engine/payload.rs +++ b/crates/rpc/rpc-types/src/eth/engine/payload.rs @@ -429,7 +429,7 @@ pub enum PayloadValidationError { #[error("invalid block number")] InvalidBlockNumber, /// Thrown when a new payload contains a wrong state root - #[error("invalid merkle root (remote: {remote:?} local: {local:?})")] + #[error("invalid merkle root: (remote: {remote:?} local: {local:?})")] InvalidStateRoot { /// The state root of the payload we received from remote (CL) remote: H256, @@ -584,6 +584,54 @@ mod tests { assert_eq!(serde_json::to_string(&status).unwrap(), full); } + #[test] + fn serde_payload_status_error_deserialize() { + let s = r#"{"status":"INVALID","latestValidHash":null,"validationError":"Failed to decode block"}"#; + let q = PayloadStatus { + latest_valid_hash: None, + status: PayloadStatusEnum::Invalid { + validation_error: "Failed to decode block".to_string(), + }, + }; + assert_eq!(q, serde_json::from_str(s).unwrap()); + + let s = r#"{"status":"INVALID","latestValidHash":null,"validationError":"links to previously rejected block"}"#; + let q = PayloadStatus { + latest_valid_hash: None, + status: PayloadStatusEnum::Invalid { + validation_error: PayloadValidationError::LinksToRejectedPayload.to_string(), + }, + }; + assert_eq!(q, serde_json::from_str(s).unwrap()); + + let s = r#"{"status":"INVALID","latestValidHash":null,"validationError":"invalid block number"}"#; + let q = PayloadStatus { + latest_valid_hash: None, + status: PayloadStatusEnum::Invalid { + validation_error: PayloadValidationError::InvalidBlockNumber.to_string(), + }, + }; + assert_eq!(q, serde_json::from_str(s).unwrap()); + + let s = r#"{"status":"INVALID","latestValidHash":null,"validationError": + "invalid merkle root: (remote: 0x3f77fb29ce67436532fee970e1add8f5cc80e8878c79b967af53b1fd92a0cab7 local: 0x603b9628dabdaadb442a3bb3d7e0360efc110e1948472909230909f1690fed17)"}"#; + let q = PayloadStatus { + latest_valid_hash: None, + status: PayloadStatusEnum::Invalid { + validation_error: PayloadValidationError::InvalidStateRoot { + remote: "0x3f77fb29ce67436532fee970e1add8f5cc80e8878c79b967af53b1fd92a0cab7" + .parse() + .unwrap(), + local: "0x603b9628dabdaadb442a3bb3d7e0360efc110e1948472909230909f1690fed17" + .parse() + .unwrap(), + } + .to_string(), + }, + }; + similar_asserts::assert_eq!(q, serde_json::from_str(s).unwrap()); + } + #[test] fn serde_roundtrip_legacy_txs_payload() { // pulled from hive tests From 4b06fbbb8a1b85570d3cc8c754c0cc9256b536f0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 10 Jul 2023 15:47:29 +0200 Subject: [PATCH 047/150] chore(deps): bump smolstr hex-literal (#3693) --- Cargo.lock | 6 +++--- crates/net/eth-wire/Cargo.toml | 2 +- crates/rlp/Cargo.toml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53fd07832be..fd96bf6bdbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5644,7 +5644,7 @@ dependencies = [ "criterion", "ethereum-types", "ethnum", - "hex-literal 0.3.4", + "hex-literal 0.4.1", "pprof", "reth-rlp", "reth-rlp-derive", @@ -6693,9 +6693,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "smol_str" -version = "0.1.24" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9" +checksum = "74212e6bbe9a4352329b2f68ba3130c15a3f26fe88ff22dbdc6cdd58fa85e99c" dependencies = [ "serde", ] diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index 338ec4cb6a6..7aba85c7a88 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -36,7 +36,7 @@ tokio-stream = { workspace = true } pin-project = { workspace = true } tracing = { workspace = true } snap = "1.0.5" -smol_str = "0.1" +smol_str = "0.2" async-trait = { workspace = true } # arbitrary utils diff --git a/crates/rlp/Cargo.toml b/crates/rlp/Cargo.toml index 234e9a52c98..bae5cf28827 100644 --- a/crates/rlp/Cargo.toml +++ b/crates/rlp/Cargo.toml @@ -13,7 +13,7 @@ arrayvec = { version = "0.7", default-features = false } auto_impl = "1" bytes.workspace = true ethnum = { version = "1", default-features = false, optional = true } -smol_str = { version = "0.1", default-features = false, optional = true } +smol_str = { version = "0.2", default-features = false, optional = true } ethereum-types = { version = "0.14", features = ["codec"], optional = true } revm-primitives = { workspace = true, features = ["serde"] } reth-rlp-derive = { path = "./rlp-derive", optional = true } @@ -27,7 +27,7 @@ reth-rlp = { workspace = true, features = [ "smol_str", ] } criterion = "0.4.0" -hex-literal = "0.3" +hex-literal = "0.4" pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } [features] From 63c0c9d7504809d2b1a92b2a1e7ceee38ae86609 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 10 Jul 2023 17:31:43 +0200 Subject: [PATCH 048/150] chore(deps): bump pprof criterion (#3698) --- Cargo.lock | 87 +++++++--------------------- crates/primitives/Cargo.toml | 4 +- crates/rlp/Cargo.toml | 4 +- crates/stages/Cargo.toml | 4 +- crates/storage/db/Cargo.toml | 4 +- crates/storage/libmdbx-rs/Cargo.toml | 4 +- crates/trie/Cargo.toml | 2 +- 7 files changed, 31 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd96bf6bdbe..8c6da07c281 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,17 +268,6 @@ dependencies = [ "wildmatch", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "auto_impl" version = "1.1.0" @@ -852,18 +841,6 @@ dependencies = [ "libloading", ] -[[package]] -name = "clap" -version = "3.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" -dependencies = [ - "bitflags 1.3.2", - "clap_lex 0.2.4", - "indexmap 1.9.3", - "textwrap", -] - [[package]] name = "clap" version = "4.1.8" @@ -872,7 +849,7 @@ checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" dependencies = [ "bitflags 1.3.2", "clap_derive", - "clap_lex 0.3.2", + "clap_lex", "is-terminal", "once_cell", "strsim 0.10.0", @@ -892,15 +869,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - [[package]] name = "clap_lex" version = "0.3.2" @@ -1075,9 +1043,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpp_demangle" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b446fd40bcc17eddd6a4a78f24315eb90afdb3334999ddfd4909985c47722442" +checksum = "ee34052ee3d93d6d8f3e6f81d85c47921f6653a19a7b70e939e3e602d893a674" dependencies = [ "cfg-if", ] @@ -1117,20 +1085,20 @@ dependencies = [ [[package]] name = "criterion" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", - "atty", "cast", "ciborium", - "clap 3.2.23", + "clap", "criterion-plot", "futures", + "is-terminal", "itertools", - "lazy_static", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", @@ -2691,15 +2659,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -3194,14 +3153,14 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", - "rustix 0.36.11", - "windows-sys 0.45.0", + "rustix 0.37.11", + "windows-sys 0.48.0", ] [[package]] @@ -4442,9 +4401,9 @@ dependencies = [ [[package]] name = "pprof" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" +checksum = "6b90f8560ad8bd57b207b8293bc5226e48e89039a6e590c12a297d91b84c7e60" dependencies = [ "backtrace", "cfg-if", @@ -4960,7 +4919,7 @@ name = "reth" version = "0.1.0-alpha.3" dependencies = [ "backon", - "clap 4.1.8", + "clap", "comfy-table", "confy", "const-str", @@ -5327,7 +5286,7 @@ dependencies = [ "arbitrary", "async-trait", "auto_impl", - "clap 4.1.8", + "clap", "futures", "hex-literal 0.3.4", "modular-bitfield", @@ -6855,9 +6814,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "10.2.1" +version = "12.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" +checksum = "38f7afd8bcd36190409e6b71d89928f7f09d918a7aa3460d847bc49a538d672e" dependencies = [ "debugid", "memmap2", @@ -6867,9 +6826,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "10.2.1" +version = "12.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" +checksum = "ec64922563a36e3fe686b6d99f06f25dacad2a202ac7502ed642930a188fb20a" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -7024,12 +6983,6 @@ dependencies = [ "test-fuzz-internal", ] -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - [[package]] name = "thin-vec" version = "0.2.12" diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index b364683abc4..330a046d58d 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -84,8 +84,8 @@ toml = "0.7.4" # necessary so we don't hit a "undeclared 'std'": # https://github.com/paradigmxyz/reth/pull/177#discussion_r1021172198 secp256k1 = { workspace = true } -criterion = "0.4.0" -pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } +criterion = "0.5" +pprof = { version = "0.12", features = ["flamegraph", "frame-pointer", "criterion"] } [features] default = [] diff --git a/crates/rlp/Cargo.toml b/crates/rlp/Cargo.toml index bae5cf28827..653543b51b9 100644 --- a/crates/rlp/Cargo.toml +++ b/crates/rlp/Cargo.toml @@ -26,9 +26,9 @@ reth-rlp = { workspace = true, features = [ "ethereum-types", "smol_str", ] } -criterion = "0.4.0" hex-literal = "0.4" -pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } +criterion = "0.5.0" +pprof = { version = "0.12", features = ["flamegraph", "frame-pointer", "criterion"] } [features] alloc = [] diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index bfe2b62884e..3bfa592616d 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -63,8 +63,8 @@ rand = { workspace = true } paste = "1.0" # Stage benchmarks -pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } -criterion = { version = "0.4.0", features = ["async_futures"] } +pprof = { version = "0.12", features = ["flamegraph", "frame-pointer", "criterion"] } +criterion = { version = "0.5", features = ["async_futures"] } # io serde_json = { workspace = true } diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index 5b5987b3156..4141cabef87 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -54,8 +54,8 @@ reth-interfaces = { workspace = true } tempfile = "3.3.0" test-fuzz = "4" -pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } -criterion = "0.4.0" +pprof = { version = "0.12", features = ["flamegraph", "frame-pointer", "criterion"] } +criterion = "0.5" iai = "0.1.1" tokio = { workspace = true, features = ["full"] } reth-db = { path = ".", features = ["test-utils", "bench"] } diff --git a/crates/storage/libmdbx-rs/Cargo.toml b/crates/storage/libmdbx-rs/Cargo.toml index bf67c037f96..669b63a54ea 100644 --- a/crates/storage/libmdbx-rs/Cargo.toml +++ b/crates/storage/libmdbx-rs/Cargo.toml @@ -29,8 +29,8 @@ default = [] return-borrowed = [] [dev-dependencies] -pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } -criterion = "0.4" +pprof = { version = "0.12", features = ["flamegraph", "frame-pointer", "criterion"] } +criterion = "0.5" rand = { workspace = true } rand_xorshift = "0.3" tempfile = "3" diff --git a/crates/trie/Cargo.toml b/crates/trie/Cargo.toml index 1c9369413bd..9664d3192c2 100644 --- a/crates/trie/Cargo.toml +++ b/crates/trie/Cargo.toml @@ -44,7 +44,7 @@ triehash = "0.8" proptest = "1.0" tokio = { workspace = true, default-features = false, features = ["sync", "rt", "macros"] } tokio-stream = { workspace = true } -criterion = "0.4" +criterion = "0.5" [features] test-utils = ["triehash"] From 20c1e129d9767eca6df84b9f5e71bad0ac170830 Mon Sep 17 00:00:00 2001 From: ChosunOne Date: Tue, 11 Jul 2023 01:15:40 +0900 Subject: [PATCH 049/150] style: replace next_sync_target Receiver loop with call to `wait_for` (#3618) --- crates/stages/src/stages/headers.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/crates/stages/src/stages/headers.rs b/crates/stages/src/stages/headers.rs index a1b7e26f05a..a56e408ab99 100644 --- a/crates/stages/src/stages/headers.rs +++ b/crates/stages/src/stages/headers.rs @@ -119,27 +119,25 @@ where // reverse from there. Else, it should use whatever the forkchoice state reports. let target = match next_header { Some(header) if checkpoint + 1 != header.number => SyncTarget::Gap(header), - None => self.next_sync_target(head_num).await, + None => self + .next_sync_target(head_num) + .await + .ok_or(StageError::StageCheckpoint(checkpoint))?, _ => return Err(StageError::StageCheckpoint(checkpoint)), }; Ok(SyncGap { local_head, target }) } - async fn next_sync_target(&mut self, head: BlockNumber) -> SyncTarget { + async fn next_sync_target(&mut self, head: BlockNumber) -> Option { match self.mode { HeaderSyncMode::Tip(ref mut rx) => { - loop { - let _ = rx.changed().await; // TODO: remove this await? - let tip = rx.borrow(); - if !tip.is_zero() { - return SyncTarget::Tip(*tip) - } - } + let tip = rx.wait_for(|tip| !tip.is_zero()).await.ok()?; + Some(SyncTarget::Tip(*tip)) } HeaderSyncMode::Continuous => { - tracing::trace!(target: "sync::stages::headers", head, "No next header found, using continuous sync strategy"); - SyncTarget::TipNum(head + 1) + trace!(target: "sync::stages::headers", head, "No next header found, using continuous sync strategy"); + Some(SyncTarget::TipNum(head + 1)) } } } From e7a733809fdeca7b47b31f68fdb13c9496f877ba Mon Sep 17 00:00:00 2001 From: "Supernovahs.eth" <91280922+supernovahs@users.noreply.github.com> Date: Mon, 10 Jul 2023 23:00:22 +0530 Subject: [PATCH 050/150] test: `eth_getProof` without storage proof (#3643) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-types/src/eth/account.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/rpc/rpc-types/src/eth/account.rs b/crates/rpc/rpc-types/src/eth/account.rs index a863d1c4d35..205e3c41f5a 100644 --- a/crates/rpc/rpc-types/src/eth/account.rs +++ b/crates/rpc/rpc-types/src/eth/account.rs @@ -61,3 +61,26 @@ pub struct RecoveredAccount { /// matches the current chain this would be true, otherwise false. pub is_valid_for_current_chain: bool, } + +#[test] +fn test_eip_1186_account_without_storage_proof() { + let response = r#"{ + "address":"0xc36442b4a4522e871399cd717abdd847ab11fe88", + "accountProof":["0xf90211a0a3deb2d4417de23e3c64a80ab58fa1cf4b62d7f193e36e507c8cf3794477b5fba0fc7ce8769dcfa9ae8d9d9537098c5cc5477b5920ed494e856049f5783c843c50a0f7d083f1e79a4c0ba1686b97a0e27c79c3a49432d333dc3574d5879cad1ca897a0cd36cf391201df64a786187d99013bdbaf5f0da6bfb8f5f2d6f0f60504f76ad9a03a9f09c92c3cefe87840938dc15fe68a3586d3b28b0f47c7037b6413c95a9feda0decb7e1969758d401af2d1cab14c0951814c094a3da108dd9f606a96840bae2ba060bf0c44ccc3ccbb5ab674841858cc5ea16495529442061295f1cecefd436659a039f8b307e0a295d6d03df089ee8211b52c5ae510d071f17ae5734a7055858002a0508040aef23dfe9c8ab16813258d95c4e765b4a557c2987fb7f3751693f34f4fa0c07e58aa6cd257695cdf147acd800c6197c235e2b5242c22e9da5d86b169d56aa00f2e89ddd874d28e62326ba365fd4f26a86cbd9f867ec0b3de69441ef8870f4ea06c1eb5455e43a36ec41a0372bde915f889cee070b8c8b8a78173d4d7df3ccebaa0cee4848c4119ed28e165e963c5b46ffa6dbeb0b14c8c51726124e7d26ff3f27aa0fc5b82dce2ee5a1691aa92b91dbeec7b2ba94df8116ea985dd7d3f4d5b8292c0a03675e148c987494e22a9767b931611fb1b7c7c287af128ea23aa70b88a1c458ba04f269f556f0f8d9cb2a9a6de52d35cf5a9098f7bb8badb1dc1d496096236aed880", + "0xf90211a0715ed9b0b002d050084eaecb878f457a348ccd47c7a597134766a7d705303de9a0c49f0fe23b0ca61892d75aebaf7277f00fdfd2022e746bab94de5d049a96edfca0b01f9c91f2bc1373862d7936198a5d11efaf370e2b9bb1dac2134b8e256ecdafa0888395aa7e0f699bb632215f08cdf92840b01e5d8e9a61d18355098cdfd50283a0ba748d609b0018667d311527a2302267209a38b08378f7d833fdead048de0defa098878e5d1461ceddeddf62bd8277586b120b5097202aa243607bc3fc8f30fc0ba0ad4111ee1952b6db0939a384986ee3fb34e0a5fc522955588fc22e159949196fa00fc948964dff427566bad468d62b0498c59df7ca7ae799ab29555d5d829d3742a0766922a88ebc6db7dfb06b03a5b17d0773094e46e42e7f2ba6a0b8567d9f1000a0db25676c4a36591f37c5e16f7199ab16559d82a2bed8c0c6a35f528a3c166bfda0149a5d50d238722e7d44c555169ed32a7f182fcb487ea378b4410a46a63a4e66a06b2298bbfe4972113e7e18cac0a8a39792c1a940ea128218343b8f88057d90aea096b2adb84105ae2aca8a7edf937e91e40872070a8641a74891e64db94d059df0a0ddbb162125ecfbd42edad8d8ef5d5e97ca7c72f54ddc404a61ae318bad0d2108a00e9a68f3e2b0c793d5fcd607edc5c55226d53fdfacd713077d6e01cb38d00d5ba05dc099f1685b2a4b7308e063e8e7905994f5c36969b1c6bfe3780c9878a4d85c80", + "0xf90211a05fc921be4d63ee07fe47a509e1abf2d69b00b6ea582a755467bf4371c2d2bd1fa0d552faa477e95f4631e2f7247aeb58693d90b03b2eee57e3fe8a9ddbd19ee42da028682c15041aa6ced1a5306aff311f5dbb8bbf7e77615994305ab3132e7842b5a0e5e0316b5046bde22d09676210885c5bea6a71703bf3b4dbac2a7199910f54faa0527fccccef17df926ccfb608f76d3c259848ed43cd24857a59c2a9352b6f1fa4a02b3863355b927b78c80ca379a4f7165bbe1644aaefed8a0bfa2001ae6284b392a09964c73eccc3d12e44dba112e31d8bd3eacbc6a42b4f17985d5b99dff968f24ea0cc426479c7ff0573629dcb2872e57f7438a28bd112a5c3fb2241bdda8031432ba04987fe755f260c2f7218640078af5f6ac4d98c2d0c001e398debc30221b14668a0e811d046c21c6cbaee464bf55553cbf88e70c2bda6951800c75c3896fdeb8e13a04aa8d0ab4946ac86e784e29000a0842cd6eebddaf8a82ece8aa69b72c98cfff5a0dfc010051ddceeec55e4146027c0eb4c72d7c242a103bf1977033ebe00a57b5da039e4da79576281284bf46ce6ca90d47832e4aefea4846615d7a61a7b976c8e3ea0dad1dfff731f7dcf37c499f4afbd5618247289c2e8c14525534b826a13b0a5a6a025f356cbc0469cb4dc326d98479e3b756e4418a67cbbb8ffb2d1abab6b1910e9a03f4082bf1da27b2a76f6bdc930eaaaf1e3f0e4d3135c2a9fb85e301f47f5174d80", + "0xf90211a0df6448f21c4e19da33f9c64c90bbcc02a499866d344c73576f63e3b4cbd4c000a010efb3b0f1d6365e2e4a389965e114e2a508ef8901f7d6c7564ba88793ff974aa0295bef2313a4f603614a5d5af3c659f63edfaa5b59a6ea2ac1da05f69ff4657ba0d8f16d5ddf4ba09616008148d2993dc50658accc2edf9111b6f464112db5d369a084604d9e06ddb53aeb7b13bb70fbe91f60df6bdc30f59bc7dc57ff37b6fe3325a04c64bd1dbeaecc54f18b23ab1ade2200970757f437e75e285f79a8c405315a14a0868075fc7f73b13863fc653c806f9a20f8e52dce44c15d2c4f94d6711021b985a01e85c49da7a8c91068468779e79b267d93d4fad01f44183353a381207304723ea05fcf186d55c53413f6988b16aa34721f0539f1cf0917f02e9d1a6ec8d3e191ffa00ad581842eab665351913e0afb3bfc070b9e4fad4d354c073f44c4f2a0c425c9a0000cb2066d81bf07f80703a40a5c5012e2c4b387bc53d381d37ee1d0f0a6643ba061f221d01c98721e79c525af5fc2eb9cc648c2ca54bb70520b868e2bdc037967a0e580f297c477df46362eb8e20371d8f0528091454bb5ad00d40368ca3ffdbd1fa079a13d35f79699f9e51d4fa07d03cd9b9dec4de9906559c0470629a663181652a0dbb402183633dbaa73e6e6a6b66bfffc4570763b264d3a702de165032298b858a065d5321015531309bb3abe0235f825d5be4270d2e511dca3b984d1e70ef308d880", + "0xf90211a06d0adafe89896724704275a42a8a63f0910dce83188add0073f621b8ca1167aaa00de7d4efad36d08f5a0320cdfd964484eba803d9933efae12c292d3ff2d06a20a083341fc12fffccf4b11df314b14f7bcead154525a097493fdf15dde4ec0c0d2aa088b7759fe3aef617828e7abd9e554add2e84ef3e2e024b1a0e2f537fce7d37f9a01e73c28722d825063304c6b51be3a8c7b6312ba8be4c6e99602e623993c014c0a0e50fbe12ddbaf184f3ba0cda971675a55abbf44c73f771bc5824b393262e5255a0b1a937d4c50528cb6aeb80aa5fe83bcfa8c294124a086302caf42cead1f99f96a04c4376b13859af218b5b09ffb33e3465288837c37fa254a46f8d0e75afecae10a0f158c0171bdb454eab6bb6dc5e276e749b6aa550f53b497492c0a392425035c3a0ac496050db1fbb1d34180ee7fd7bed18efa4cf43299390a72dcf530cc3422630a02cacb30ac3b4bab293d31833be4865cd1d1de8db8630edac4af056979cc903aea090cbb538f0f4601289db4cf49485ab3a178044daeae325c525bc3978714a7219a0542021427adbe890896fcc888418a747a555b2a7121fe3c683e07dcf5012e96ca006569c5e3715f52f62dd856dec2136e60c49bbadc1cf9fb625930da3e8f1c16ea0a2539ebb66a2c10c3809626181a2389f043e0b54867cd356eb5f20daaeb521b4a0ab49972dced10010275f2604e6182722dbc426ca1b0ae128defe80c0baefd3c080", + "0xf90211a006c1d8a7c5deeb435ea0b080aea8b7acb58d2d898e12e3560d399594a77863a1a088105243bc96e1f10baa73d670929a834c51eb7f695cf43f4fab94e73c9a5b8da0fce3a21f09b62d65607bbdabb8d675d58a5f3bfb19ae46510a4ea2205070aa03a0039ae7a999ed83bfdb49b6df7074589059ba6c2eed22bfc6dac8ff5241c71bd7a09feca6f7331b6c147f4fd7bd94de496144b85543d868f47be6345330b3f8ccd3a00e55c30d16438567979c92d387a2b99e51a4026192ccfda2ac87a190c3aee511a0a86c5bb52651e490203c63670b569b2337e838e4d80d455cc83e64571e2552f1a0cfb31ae59b691c15ffd97658bab646ff4b90dbc72a81ec52731b3fbd38d0dd5ba0d83936fc4143cc885be5fa420ef22fb97f6a8dd24e9ece9af965792565a7b2c8a0abb179481f4b29578adb8768aa4f6ba6ed6bd43c7572d7c3405c879a362f1ab1a0506651daa07d44901dfd76c12d302b2242e5ceac385f95ea928f20a0336eccf6a010e8a7f461231438987fb26adc4c5004721dc401dc2b77e9b79d26b1308d0079a09174afa82e6d27dfdde74f556d0e782ae6222dc66104d84ea0f1e21e093578c4a0391e24ed0033cc58f149af753b485de3c8b9e4b3c8e145c308db60e51cabbefca03b0991359019197dd53e3798e55a14c8795d655b0693efd37404cf8f8d979cfba0594d95bbfe8e2ea5040b571010549a233bc33bf959792e1e41c515c65abac14480", + "0xf90151a0e8ed81735d358657020dd6bc4bc58cf751cc037fa57e1d0c668bf24049e720d280a03e8bf7abdd8a4190a0ee5f92a78bf1dba529312ed66dd7ead7c9be55c81a2db480a006312425a007cda585740355f52db74d0ae43c21d562c599112546e3ffe22f01a023bbbb0ffb33c7a5477ab514c0f4f3c94ba1748a5ea1dc3edc7c4b5330cd70fe80a03ed45ab6045a10fa00b2fba662914f4dedbf3f3a5f2ce1e6e53a12ee3ea21235a01e02c98684cea92a7c0b04a01658530a09d268b395840a66263923e44b93d2b5a0a585db4a911fe6452a4540bf7dc143981ca31035ccb2c51d02eccd021a6163a480a06032919dcb44e22852b6367473bbc3f43311226ac28991a90b9c9da669f9e08a80a0146aee58a46c30bc84f6e99cd76bf29b3bd238053102679498a3ea15d4ff6d53a04cf57cfdc046c135004b9579059c84b2d902a51fb6feaed51ea272f0ca1cdc648080", + "0xf871a059ce2e1f470580853d88511bf8672f9ffaefadd80bc07b2e3d5a18c3d7812007a0867e978faf3461d2238ccf8d6a138406cb6d8bd36dfa60caddb62af14447a6f880808080a0fc6209fdaa57d224ee35f73e96469a7f95760a54d5de3da07953430b001aee6980808080808080808080", + "0xf8669d20852b2b985cd8c252fddae2acb4f798d0fecdcb1e2da53726332eb559b846f8440180a079fe22fe88fc4b45db10ce94d975e02e8a42b57dc190f8ae15e321f72bbc08eaa0692e658b31cbe3407682854806658d315d61a58c7e4933a2f91d383dc00736c6"], + "balance":"0x0", + "codeHash":"0x692e658b31cbe3407682854806658d315d61a58c7e4933a2f91d383dc00736c6", + "nonce":"0x1", + "storageHash":"0x79fe22fe88fc4b45db10ce94d975e02e8a42b57dc190f8ae15e321f72bbc08ea", + "storageProof":[] + }"#; + let val = serde_json::from_str::(response).unwrap(); + serde_json::to_value(val).unwrap(); +} From 4d8c97ea2521d60fde0e52cecba63ad289789f81 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 10 Jul 2023 20:45:21 +0100 Subject: [PATCH 051/150] chore(prometheus): add host.docker.internal:9001 scrape target (#3689) --- etc/prometheus/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/prometheus/prometheus.yml b/etc/prometheus/prometheus.yml index 483dcaa050e..4906c397967 100644 --- a/etc/prometheus/prometheus.yml +++ b/etc/prometheus/prometheus.yml @@ -3,7 +3,7 @@ scrape_configs: metrics_path: "/" scrape_interval: 5s static_configs: - - targets: ['reth:9001', 'localhost:9001'] + - targets: ['reth:9001', 'localhost:9001', 'host.docker.internal:9001'] - job_name: ethereum-metrics-exporter metrics_path: "/metrics" scrape_interval: 5s From 67c6f77ff3bd877d697b3a9635843de2d60d96ae Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Jul 2023 02:55:09 +0200 Subject: [PATCH 052/150] fix: check if value is bool (#3708) --- crates/rpc/rpc-types/src/eth/pubsub.rs | 10 +++++++--- crates/rpc/rpc/src/eth/pubsub.rs | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/pubsub.rs b/crates/rpc/rpc-types/src/eth/pubsub.rs index f027037c176..e363cb754c3 100644 --- a/crates/rpc/rpc-types/src/eth/pubsub.rs +++ b/crates/rpc/rpc-types/src/eth/pubsub.rs @@ -104,8 +104,8 @@ pub enum Params { None, /// Log parameters. Logs(Box), - /// New pending transaction parameters. - NewPendingTransactions(bool), + /// Boolean parameter for new pending transactions. + Bool(bool), } impl Serialize for Params { @@ -116,7 +116,7 @@ impl Serialize for Params { match self { Params::None => (&[] as &[serde_json::Value]).serialize(serializer), Params::Logs(logs) => logs.serialize(serializer), - Params::NewPendingTransactions(full) => full.serialize(serializer), + Params::Bool(full) => full.serialize(serializer), } } } @@ -132,6 +132,10 @@ impl<'a> Deserialize<'a> for Params { return Ok(Params::None) } + if let Some(val) = v.as_bool() { + return Ok(Params::Bool(val)) + } + serde_json::from_value(v) .map(|f| Params::Logs(Box::new(f))) .map_err(|e| D::Error::custom(format!("Invalid Pub-Sub parameters: {e}"))) diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index aa5dd7363f5..a6693001136 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -124,7 +124,7 @@ where SubscriptionKind::NewPendingTransactions => { if let Some(params) = params { match params { - Params::NewPendingTransactions(true) => { + Params::Bool(true) => { // full transaction objects requested let stream = pubsub.full_pending_transaction_stream().map(|tx| { EthSubscriptionResult::FullTransaction(Box::new( @@ -135,7 +135,7 @@ where }); return pipe_from_stream(accepted_sink, stream).await } - Params::NewPendingTransactions(false) | Params::None => { + Params::Bool(false) | Params::None => { // only hashes requested } Params::Logs(_) => { From bc84e561009255fb22bd250773bdedcbdb6dd638 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:51:34 +0100 Subject: [PATCH 053/150] feat(pruning): prune `Receipts` during pipeline (#3585) --- Cargo.lock | 1 + bin/reth/src/chain/import.rs | 4 +- bin/reth/src/debug_cmd/execution.rs | 1 + bin/reth/src/debug_cmd/merkle.rs | 3 +- bin/reth/src/init.rs | 2 +- bin/reth/src/node/mod.rs | 1 + bin/reth/src/stage/dump/merkle.rs | 3 +- bin/reth/src/stage/run.rs | 1 + crates/config/src/config.rs | 30 +---- crates/primitives/Cargo.toml | 1 + crates/primitives/src/lib.rs | 2 +- crates/primitives/src/prune/mod.rs | 2 + crates/primitives/src/prune/mode.rs | 2 +- crates/primitives/src/prune/target.rs | 82 ++++++++++++++ crates/stages/src/stage.rs | 30 +---- crates/stages/src/stages/execution.rs | 104 +++++++++++++++++- crates/storage/provider/src/post_state/mod.rs | 56 +++++++--- .../src/providers/database/provider.rs | 2 +- 18 files changed, 241 insertions(+), 86 deletions(-) create mode 100644 crates/primitives/src/prune/target.rs diff --git a/Cargo.lock b/Cargo.lock index 8c6da07c281..e8b62e26467 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5504,6 +5504,7 @@ dependencies = [ "impl-serde", "modular-bitfield", "once_cell", + "paste", "plain_hasher", "pprof", "proptest", diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs index 38bed8996f8..e08f7164b58 100644 --- a/bin/reth/src/chain/import.rs +++ b/bin/reth/src/chain/import.rs @@ -159,10 +159,11 @@ impl ImportCommand { let (tip_tx, tip_rx) = watch::channel(H256::zero()); let factory = reth_revm::Factory::new(self.chain.clone()); + let max_block = file_client.max_block().unwrap_or(0); let mut pipeline = Pipeline::builder() .with_tip_sender(tip_tx) // we want to sync all blocks the file client provides or 0 if empty - .with_max_block(file_client.max_block().unwrap_or(0)) + .with_max_block(max_block) .add_stages( DefaultStages::new( HeaderSyncMode::Tip(tip_rx), @@ -184,6 +185,7 @@ impl ImportCommand { max_blocks: config.stages.execution.max_blocks, max_changes: config.stages.execution.max_changes, }, + config.prune.map(|prune| prune.parts).unwrap_or_default(), )), ) .build(db, self.chain.clone()); diff --git a/bin/reth/src/debug_cmd/execution.rs b/bin/reth/src/debug_cmd/execution.rs index 7a6f6133462..88481b1cc14 100644 --- a/bin/reth/src/debug_cmd/execution.rs +++ b/bin/reth/src/debug_cmd/execution.rs @@ -142,6 +142,7 @@ impl Command { .set(ExecutionStage::new( factory, ExecutionStageThresholds { max_blocks: None, max_changes: None }, + config.prune.map(|prune| prune.parts).unwrap_or_default(), )), ) .build(db, self.chain.clone()); diff --git a/bin/reth/src/debug_cmd/merkle.rs b/bin/reth/src/debug_cmd/merkle.rs index 1188dbfcb67..e672ec3e027 100644 --- a/bin/reth/src/debug_cmd/merkle.rs +++ b/bin/reth/src/debug_cmd/merkle.rs @@ -8,7 +8,7 @@ use reth_db::{cursor::DbCursorRO, init_db, tables, transaction::DbTx}; use reth_primitives::{ fs, stage::{StageCheckpoint, StageId}, - ChainSpec, + ChainSpec, PruneTargets, }; use reth_provider::{ProviderFactory, StageCheckpointReader}; use reth_stages::{ @@ -96,6 +96,7 @@ impl Command { let mut execution_stage = ExecutionStage::new( factory, ExecutionStageThresholds { max_blocks: Some(1), max_changes: None }, + PruneTargets::all(), ); let mut account_hashing_stage = AccountHashingStage::default(); diff --git a/bin/reth/src/init.rs b/bin/reth/src/init.rs index 6c2e5593df9..37efd40817d 100644 --- a/bin/reth/src/init.rs +++ b/bin/reth/src/init.rs @@ -109,7 +109,7 @@ pub fn insert_genesis_state( state.change_storage(0, *address, storage_changes); } } - state.write_to_db(tx)?; + state.write_to_db(tx, 0)?; Ok(()) } diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index b7bf59466be..529d67713ac 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -719,6 +719,7 @@ impl Command { max_blocks: stage_config.execution.max_blocks, max_changes: stage_config.execution.max_changes, }, + config.prune.map(|prune| prune.parts).unwrap_or_default(), ) .with_metrics_tx(metrics_tx), ) diff --git a/bin/reth/src/stage/dump/merkle.rs b/bin/reth/src/stage/dump/merkle.rs index 601abd569cf..dd5c8f75809 100644 --- a/bin/reth/src/stage/dump/merkle.rs +++ b/bin/reth/src/stage/dump/merkle.rs @@ -2,7 +2,7 @@ use super::setup; use crate::utils::DbTool; use eyre::Result; use reth_db::{database::Database, table::TableImporter, tables, DatabaseEnv}; -use reth_primitives::{stage::StageCheckpoint, BlockNumber, ChainSpec}; +use reth_primitives::{stage::StageCheckpoint, BlockNumber, ChainSpec, PruneTargets}; use reth_provider::ProviderFactory; use reth_stages::{ stages::{ @@ -70,6 +70,7 @@ async fn unwind_and_copy( let mut exec_stage = ExecutionStage::new( reth_revm::Factory::new(db_tool.chain.clone()), ExecutionStageThresholds { max_blocks: Some(u64::MAX), max_changes: None }, + PruneTargets::all(), ); exec_stage diff --git a/bin/reth/src/stage/run.rs b/bin/reth/src/stage/run.rs index 23166e595a2..f53412c3dbb 100644 --- a/bin/reth/src/stage/run.rs +++ b/bin/reth/src/stage/run.rs @@ -202,6 +202,7 @@ impl Command { max_blocks: Some(batch_size), max_changes: None, }, + config.prune.map(|prune| prune.parts).unwrap_or_default(), )), None, ) diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index 40836b8c02c..b622ed47ee3 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -5,7 +5,7 @@ use reth_downloaders::{ headers::reverse_headers::ReverseHeadersDownloaderBuilder, }; use reth_network::{NetworkConfigBuilder, PeersConfig, SessionsConfig}; -use reth_primitives::{serde_helper::deserialize_opt_prune_mode_with_min_distance, PruneMode}; +use reth_primitives::PruneTargets; use secp256k1::SecretKey; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -285,39 +285,15 @@ pub struct PruneConfig { /// Minimum pruning interval measured in blocks. pub block_interval: u64, /// Pruning configuration for every part of the data that can be pruned. - pub parts: PruneParts, + pub parts: PruneTargets, } impl Default for PruneConfig { fn default() -> Self { - Self { block_interval: 10, parts: PruneParts::default() } + Self { block_interval: 10, parts: PruneTargets::default() } } } -/// Pruning configuration for every part of the data that can be pruned. -#[derive(Debug, Clone, Default, Copy, Deserialize, PartialEq, Serialize)] -#[serde(default)] -pub struct PruneParts { - /// Sender Recovery pruning configuration. - #[serde(skip_serializing_if = "Option::is_none")] - pub sender_recovery: Option, - /// Transaction Lookup pruning configuration. - #[serde(skip_serializing_if = "Option::is_none")] - pub transaction_lookup: Option, - /// Receipts pruning configuration. - #[serde( - skip_serializing_if = "Option::is_none", - deserialize_with = "deserialize_opt_prune_mode_with_min_distance::<64, _>" - )] - pub receipts: Option, - /// Account History pruning configuration. - #[serde(skip_serializing_if = "Option::is_none")] - pub account_history: Option, - /// Storage History pruning configuration. - #[serde(skip_serializing_if = "Option::is_none")] - pub storage_history: Option, -} - #[cfg(test)] mod tests { use super::Config; diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 330a046d58d..f75c681bc4e 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -56,6 +56,7 @@ url = "2.3" impl-serde = "0.4.0" once_cell = "1.17.0" zstd = { version = "0.12", features = ["experimental"] } +paste = "1.0" # proof related triehash = "0.8" diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index e08537ea14b..4d0cbc0240f 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -80,7 +80,7 @@ pub use net::{ SEPOLIA_BOOTNODES, }; pub use peer::{PeerId, WithPeerId}; -pub use prune::{PruneCheckpoint, PruneMode}; +pub use prune::{PruneCheckpoint, PruneMode, PruneTargets}; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef}; pub use revm_primitives::JumpMap; pub use serde_helper::JsonU256; diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index 2814f8bc4b7..18cb943fb97 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -1,5 +1,7 @@ mod checkpoint; mod mode; +mod target; pub use checkpoint::PruneCheckpoint; pub use mode::PruneMode; +pub use target::PruneTargets; diff --git a/crates/primitives/src/prune/mode.rs b/crates/primitives/src/prune/mode.rs index 47e6778e801..3fba10fe59a 100644 --- a/crates/primitives/src/prune/mode.rs +++ b/crates/primitives/src/prune/mode.rs @@ -8,7 +8,7 @@ use reth_codecs::{main_codec, Compact}; pub enum PruneMode { /// Prune all blocks. Full, - /// Prune blocks before the `head-N` block number. In other words, keep last N blocks. + /// Prune blocks before the `head-N` block number. In other words, keep last N + 1 blocks. Distance(u64), /// Prune blocks before the specified block number. The specified block number is not pruned. Before(BlockNumber), diff --git a/crates/primitives/src/prune/target.rs b/crates/primitives/src/prune/target.rs new file mode 100644 index 00000000000..987d0883a30 --- /dev/null +++ b/crates/primitives/src/prune/target.rs @@ -0,0 +1,82 @@ +use crate::{serde_helper::deserialize_opt_prune_mode_with_min_distance, BlockNumber, PruneMode}; +use paste::paste; +use serde::{Deserialize, Serialize}; + +/// Pruning configuration for every part of the data that can be pruned. +#[derive(Debug, Clone, Default, Copy, Deserialize, Eq, PartialEq, Serialize)] +#[serde(default)] +pub struct PruneTargets { + /// Sender Recovery pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub sender_recovery: Option, + /// Transaction Lookup pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub transaction_lookup: Option, + /// Receipts pruning configuration. + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_opt_prune_mode_with_min_distance::<64, _>" + )] + pub receipts: Option, + /// Account History pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub account_history: Option, + /// Storage History pruning configuration. + #[serde(skip_serializing_if = "Option::is_none")] + pub storage_history: Option, +} + +macro_rules! should_prune_method { + ($($config:ident),+) => { + $( + paste! { + #[allow(missing_docs)] + pub fn [](&self, block: BlockNumber, tip: BlockNumber) -> bool { + if let Some(config) = &self.$config { + return self.should_prune(config, block, tip) + } + false + } + } + )+ + + /// Sets pruning to all targets. + pub fn all() -> Self { + PruneTargets { + $( + $config: Some(PruneMode::Full), + )+ + } + } + + }; +} + +impl PruneTargets { + /// Sets pruning to no target. + pub fn none() -> Self { + PruneTargets::default() + } + + /// Check if target block should be pruned + pub fn should_prune(&self, target: &PruneMode, block: BlockNumber, tip: BlockNumber) -> bool { + match target { + PruneMode::Full => true, + PruneMode::Distance(distance) => { + if *distance > tip { + return false + } + block < tip - *distance + } + PruneMode::Before(n) => *n > block, + } + } + + should_prune_method!( + sender_recovery, + transaction_lookup, + receipts, + account_history, + storage_history + ); +} diff --git a/crates/stages/src/stage.rs b/crates/stages/src/stage.rs index 71c5f1ad8c2..7580c6bab42 100644 --- a/crates/stages/src/stage.rs +++ b/crates/stages/src/stage.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx}; use reth_primitives::{ stage::{StageCheckpoint, StageId}, - BlockNumber, PruneMode, TxNumber, + BlockNumber, TxNumber, }; use reth_provider::{BlockReader, DatabaseProviderRW, ProviderError}; use std::{ @@ -194,31 +194,3 @@ pub trait Stage: Send + Sync { input: UnwindInput, ) -> Result; } - -/// Prune target. -#[derive(Debug, Clone, Copy)] -pub enum PruneTarget { - /// Prune all blocks, i.e. not save any data. - All, - /// Prune blocks up to the specified block number, inclusive. - Block(BlockNumber), -} - -impl PruneTarget { - /// Returns new target to prune towards, according to stage prune mode [PruneMode] - /// and current head [BlockNumber]. - pub fn new(prune_mode: PruneMode, head: BlockNumber) -> Self { - match prune_mode { - PruneMode::Full => PruneTarget::All, - PruneMode::Distance(distance) => { - Self::Block(head.saturating_sub(distance).saturating_sub(1)) - } - PruneMode::Before(before_block) => Self::Block(before_block.saturating_sub(1)), - } - } - - /// Returns true if the target is [PruneTarget::All], i.e. prune all blocks. - pub fn is_all(&self) -> bool { - matches!(self, Self::All) - } -} diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 38284b1d125..70d07f8d736 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -14,7 +14,7 @@ use reth_primitives::{ stage::{ CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint, StageCheckpoint, StageId, }, - BlockNumber, Header, U256, + BlockNumber, Header, PruneTargets, U256, }; use reth_provider::{ post_state::PostState, BlockExecutor, BlockReader, DatabaseProviderRW, ExecutorFactory, @@ -59,19 +59,25 @@ pub struct ExecutionStage { executor_factory: EF, /// The commit thresholds of the execution stage. thresholds: ExecutionStageThresholds, + /// Pruning configuration. + prune_targets: PruneTargets, } impl ExecutionStage { /// Create new execution stage with specified config. - pub fn new(executor_factory: EF, thresholds: ExecutionStageThresholds) -> Self { - Self { metrics_tx: None, executor_factory, thresholds } + pub fn new( + executor_factory: EF, + thresholds: ExecutionStageThresholds, + prune_targets: PruneTargets, + ) -> Self { + Self { metrics_tx: None, executor_factory, thresholds, prune_targets } } /// Create an execution stage with the provided executor factory. /// /// The commit threshold will be set to 10_000. pub fn new_with_factory(executor_factory: EF) -> Self { - Self::new(executor_factory, ExecutionStageThresholds::default()) + Self::new(executor_factory, ExecutionStageThresholds::default(), PruneTargets::default()) } /// Set the metric events sender. @@ -104,6 +110,8 @@ impl ExecutionStage { // Execute block range let mut state = PostState::default(); + state.add_prune_targets(self.prune_targets); + for block_number in start_block..=max_block { let td = provider .header_td_by_number(block_number)? @@ -145,7 +153,7 @@ impl ExecutionStage { // Write remaining changes trace!(target: "sync::stages::execution", accounts = state.accounts().len(), "Writing updated state to database"); let start = Instant::now(); - state.write_to_db(provider.tx_ref())?; + state.write_to_db(provider.tx_ref(), max_block)?; trace!(target: "sync::stages::execution", took = ?start.elapsed(), "Wrote state"); let done = stage_progress == max_block; @@ -417,7 +425,8 @@ mod tests { use reth_db::{models::AccountBeforeTx, test_utils::create_test_rw_db}; use reth_primitives::{ hex_literal::hex, keccak256, stage::StageUnitCheckpoint, Account, Bytecode, - ChainSpecBuilder, SealedBlock, StorageEntry, H160, H256, MAINNET, U256, + ChainSpecBuilder, PruneMode, PruneTargets, SealedBlock, StorageEntry, H160, H256, MAINNET, + U256, }; use reth_provider::{AccountReader, BlockWriter, ProviderFactory, ReceiptProvider}; use reth_revm::Factory; @@ -430,6 +439,7 @@ mod tests { ExecutionStage::new( factory, ExecutionStageThresholds { max_blocks: Some(100), max_changes: None }, + PruneTargets::none(), ) } @@ -884,4 +894,86 @@ mod tests { ] ); } + + #[tokio::test] + async fn test_prune() { + let test_tx = TestTransaction::default(); + let factory = Arc::new(ProviderFactory::new(test_tx.tx.as_ref(), MAINNET.clone())); + + let provider = factory.provider_rw().unwrap(); + let input = ExecInput { + target: Some(1), + /// The progress of this stage the last time it was executed. + checkpoint: None, + }; + let mut genesis_rlp = hex!("f901faf901f5a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa045571b40ae66ca7480791bbb2887286e4e4c4b1b298b191c889d6959023a32eda056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000808502540be400808000a00000000000000000000000000000000000000000000000000000000000000000880000000000000000c0c0").as_slice(); + let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); + let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); + let block = SealedBlock::decode(&mut block_rlp).unwrap(); + provider.insert_block(genesis, None).unwrap(); + provider.insert_block(block.clone(), None).unwrap(); + provider.commit().unwrap(); + + // insert pre state + let provider = factory.provider_rw().unwrap(); + let code = hex!("5a465a905090036002900360015500"); + let code_hash = keccak256(hex!("5a465a905090036002900360015500")); + provider + .tx_ref() + .put::( + H160(hex!("1000000000000000000000000000000000000000")), + Account { nonce: 0, balance: U256::ZERO, bytecode_hash: Some(code_hash) }, + ) + .unwrap(); + provider + .tx_ref() + .put::( + H160(hex!("a94f5374fce5edbc8e2a8697c15331677e6ebf0b")), + Account { + nonce: 0, + balance: U256::from(0x3635c9adc5dea00000u128), + bytecode_hash: None, + }, + ) + .unwrap(); + provider + .tx_ref() + .put::(code_hash, Bytecode::new_raw(code.to_vec().into())) + .unwrap(); + provider.commit().unwrap(); + + let check_pruning = |factory: Arc>, + prune_targets: PruneTargets, + expect_num_receipts: usize| async move { + let provider = factory.provider_rw().unwrap(); + + let mut execution_stage = ExecutionStage::new( + Factory::new(Arc::new(ChainSpecBuilder::mainnet().berlin_activated().build())), + ExecutionStageThresholds { max_blocks: Some(100), max_changes: None }, + prune_targets, + ); + + execution_stage.execute(&provider, input).await.unwrap(); + assert_eq!( + provider.receipts_by_block(1.into()).unwrap().unwrap().len(), + expect_num_receipts + ); + }; + + let mut prune = PruneTargets::none(); + + check_pruning(factory.clone(), prune, 1).await; + + prune.receipts = Some(PruneMode::Full); + check_pruning(factory.clone(), prune, 0).await; + + prune.receipts = Some(PruneMode::Before(1)); + check_pruning(factory.clone(), prune, 1).await; + + prune.receipts = Some(PruneMode::Before(2)); + check_pruning(factory.clone(), prune, 0).await; + + prune.receipts = Some(PruneMode::Distance(0)); + check_pruning(factory.clone(), prune, 1).await; + } } diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index 6fb6b02989f..9e8aa189ef1 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -8,7 +8,7 @@ use reth_db::{ }; use reth_primitives::{ bloom::logs_bloom, keccak256, proofs::calculate_receipt_root_ref, Account, Address, - BlockNumber, Bloom, Bytecode, Log, Receipt, StorageEntry, H256, U256, + BlockNumber, Bloom, Bytecode, Log, PruneMode, PruneTargets, Receipt, StorageEntry, H256, U256, }; use reth_trie::{ hashed_cursor::{HashedPostState, HashedPostStateCursorFactory, HashedStorage}, @@ -78,6 +78,8 @@ pub struct PostState { bytecode: BTreeMap, /// The receipt(s) of the executed transaction(s). receipts: BTreeMap>, + /// Pruning configuration. + prune_targets: PruneTargets, } impl PostState { @@ -91,6 +93,11 @@ impl PostState { Self { receipts: BTreeMap::from([(block, Vec::with_capacity(txs))]), ..Default::default() } } + /// Add a pruning configuration. + pub fn add_prune_targets(&mut self, prune_targets: PruneTargets) { + self.prune_targets = prune_targets; + } + /// Return the current size of the poststate. /// /// Size is the sum of individual changes to accounts, storage, bytecode and receipts. @@ -319,6 +326,7 @@ impl PostState { } self.receipts.extend(other.receipts); + self.bytecode.extend(other.bytecode); } @@ -579,7 +587,11 @@ impl PostState { } /// Write the post state to the database. - pub fn write_to_db<'a, TX: DbTxMut<'a> + DbTx<'a>>(mut self, tx: &TX) -> Result<(), DbError> { + pub fn write_to_db<'a, TX: DbTxMut<'a> + DbTx<'a>>( + mut self, + tx: &TX, + tip: BlockNumber, + ) -> Result<(), DbError> { self.write_history_to_db(tx)?; // Write new storage state @@ -630,16 +642,24 @@ impl PostState { bytecodes_cursor.upsert(hash, bytecode)?; } - // Write the receipts of the transactions + // Write the receipts of the transactions if not pruned tracing::trace!(target: "provider::post_state", len = self.receipts.len(), "Writing receipts"); - let mut bodies_cursor = tx.cursor_read::()?; - let mut receipts_cursor = tx.cursor_write::()?; - for (block, receipts) in self.receipts { - let (_, body_indices) = bodies_cursor.seek_exact(block)?.expect("body indices exist"); - let tx_range = body_indices.tx_num_range(); - assert_eq!(receipts.len(), tx_range.clone().count(), "Receipt length mismatch"); - for (tx_num, receipt) in tx_range.zip(receipts) { - receipts_cursor.append(tx_num, receipt)?; + if !self.receipts.is_empty() && self.prune_targets.receipts != Some(PruneMode::Full) { + let mut bodies_cursor = tx.cursor_read::()?; + let mut receipts_cursor = tx.cursor_write::()?; + + for (block, receipts) in self.receipts { + if self.prune_targets.should_prune_receipts(block, tip) { + continue + } + + let (_, body_indices) = + bodies_cursor.seek_exact(block)?.expect("body indices exist"); + let tx_range = body_indices.tx_num_range(); + assert_eq!(receipts.len(), tx_range.clone().count(), "Receipt length mismatch"); + for (tx_num, receipt) in tx_range.zip(receipts) { + receipts_cursor.append(tx_num, receipt)?; + } } } @@ -1091,7 +1111,7 @@ mod tests { post_state.create_account(1, address_a, account_a); // 0x11.. is changed (balance + 1, nonce + 1) post_state.change_account(1, address_b, account_b, account_b_changed); - post_state.write_to_db(provider.tx_ref()).expect("Could not write post state to DB"); + post_state.write_to_db(provider.tx_ref(), 0).expect("Could not write post state to DB"); // Check plain state assert_eq!( @@ -1124,7 +1144,9 @@ mod tests { let mut post_state = PostState::new(); // 0x11.. is destroyed post_state.destroy_account(2, address_b, account_b_changed); - post_state.write_to_db(provider.tx_ref()).expect("Could not write second post state to DB"); + post_state + .write_to_db(provider.tx_ref(), 0) + .expect("Could not write second post state to DB"); // Check new plain state for account B assert_eq!( @@ -1163,7 +1185,7 @@ mod tests { post_state.change_storage(1, address_a, storage_a_changeset); post_state.change_storage(1, address_b, storage_b_changeset); - post_state.write_to_db(&tx).expect("Could not write post state to DB"); + post_state.write_to_db(&tx, 0).expect("Could not write post state to DB"); // Check plain storage state let mut storage_cursor = tx @@ -1246,7 +1268,7 @@ mod tests { // Delete account A let mut post_state = PostState::new(); post_state.destroy_account(2, address_a, Account::default()); - post_state.write_to_db(&tx).expect("Could not write post state to DB"); + post_state.write_to_db(&tx, 0).expect("Could not write post state to DB"); assert_eq!( storage_cursor.seek_exact(address_a).unwrap(), @@ -1296,7 +1318,7 @@ mod tests { (U256::from(1), (U256::ZERO, U256::from(2))), ]), ); - init_state.write_to_db(&tx).expect("Could not write init state to DB"); + init_state.write_to_db(&tx, 0).expect("Could not write init state to DB"); let mut post_state = PostState::new(); post_state.change_storage( @@ -1339,7 +1361,7 @@ mod tests { BTreeMap::from([(U256::from(0), (U256::ZERO, U256::from(9)))]), ); - post_state.write_to_db(&tx).expect("Could not write post state to DB"); + post_state.write_to_db(&tx, 0).expect("Could not write post state to DB"); let mut storage_changeset_cursor = tx .cursor_dup_read::() diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 5650b85e3dd..2e0fe5a4b9f 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1774,7 +1774,7 @@ impl<'this, TX: DbTxMut<'this> + DbTx<'this>> BlockWriter for DatabaseProvider<' // Write state and changesets to the database. // Must be written after blocks because of the receipt lookup. - state.write_to_db(self.tx_ref())?; + state.write_to_db(self.tx_ref(), new_tip_number)?; self.insert_hashes(first_number..=last_block_number, last_block_hash, expected_state_root)?; From 5854c976bbbb9f583c1408e7de4293135b42d831 Mon Sep 17 00:00:00 2001 From: Josh Stevens Date: Tue, 11 Jul 2023 12:17:46 +0100 Subject: [PATCH 054/150] fix: transaction calls on the reth-provider should not generate a hash by default (#3675) --- crates/storage/provider/src/providers/database/mod.rs | 4 ++++ crates/storage/provider/src/providers/database/provider.rs | 4 ++++ crates/storage/provider/src/providers/mod.rs | 4 ++++ crates/storage/provider/src/test_utils/mock.rs | 7 ++++++- crates/storage/provider/src/test_utils/noop.rs | 7 ++++++- crates/storage/provider/src/traits/transactions.rs | 5 ++++- 6 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 686257c1036..fde0ff2865c 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -228,6 +228,10 @@ impl TransactionsProvider for ProviderFactory { self.provider()?.transaction_by_id(id) } + fn transaction_by_id_no_hash(&self, id: TxNumber) -> Result> { + self.provider()?.transaction_by_id_no_hash(id) + } + fn transaction_by_hash(&self, hash: TxHash) -> Result> { self.provider()?.transaction_by_hash(hash) } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 2e0fe5a4b9f..28a00f45e20 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -934,6 +934,10 @@ impl<'this, TX: DbTx<'this>> TransactionsProvider for DatabaseProvider<'this, TX Ok(self.tx.get::(id)?.map(Into::into)) } + fn transaction_by_id_no_hash(&self, id: TxNumber) -> Result> { + Ok(self.tx.get::(id)?.map(Into::into)) + } + fn transaction_by_hash(&self, hash: TxHash) -> Result> { if let Some(id) = self.transaction_id(hash)? { Ok(self.transaction_by_id(id)?) diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 7dfeae75f21..81c123323fa 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -271,6 +271,10 @@ where self.database.provider()?.transaction_by_id(id) } + fn transaction_by_id_no_hash(&self, id: TxNumber) -> Result> { + self.database.provider()?.transaction_by_id_no_hash(id) + } + fn transaction_by_hash(&self, hash: TxHash) -> Result> { self.database.provider()?.transaction_by_hash(hash) } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 98a7ee7f173..d679a7ed544 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -11,7 +11,8 @@ use reth_interfaces::{provider::ProviderError, Result}; use reth_primitives::{ keccak256, Account, Address, Block, BlockHash, BlockHashOrNumber, BlockId, BlockNumber, BlockWithSenders, Bytecode, Bytes, ChainInfo, Header, Receipt, SealedBlock, SealedHeader, - StorageKey, StorageValue, TransactionMeta, TransactionSigned, TxHash, TxNumber, H256, U256, + StorageKey, StorageValue, TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, + TxNumber, H256, U256, }; use reth_revm_primitives::primitives::{BlockEnv, CfgEnv}; use std::{ @@ -168,6 +169,10 @@ impl TransactionsProvider for MockEthProvider { Ok(None) } + fn transaction_by_id_no_hash(&self, _id: TxNumber) -> Result> { + Ok(None) + } + fn transaction_by_hash(&self, hash: TxHash) -> Result> { Ok(self .blocks diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index b330697b8f8..8fa349b11b5 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -11,7 +11,8 @@ use reth_primitives::{ stage::{StageCheckpoint, StageId}, Account, Address, Block, BlockHash, BlockHashOrNumber, BlockId, BlockNumber, Bytecode, Bytes, ChainInfo, ChainSpec, Header, Receipt, SealedBlock, SealedHeader, StorageKey, StorageValue, - TransactionMeta, TransactionSigned, TxHash, TxNumber, H256, KECCAK_EMPTY, MAINNET, U256, + TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber, H256, + KECCAK_EMPTY, MAINNET, U256, }; use reth_revm_primitives::primitives::{BlockEnv, CfgEnv}; use std::{ops::RangeBounds, sync::Arc}; @@ -130,6 +131,10 @@ impl TransactionsProvider for NoopProvider { Ok(None) } + fn transaction_by_id_no_hash(&self, _id: TxNumber) -> Result> { + Ok(None) + } + fn transaction_by_hash(&self, _hash: TxHash) -> Result> { Ok(None) } diff --git a/crates/storage/provider/src/traits/transactions.rs b/crates/storage/provider/src/traits/transactions.rs index 6bbd1745444..d9da859534d 100644 --- a/crates/storage/provider/src/traits/transactions.rs +++ b/crates/storage/provider/src/traits/transactions.rs @@ -15,9 +15,12 @@ pub trait TransactionsProvider: BlockNumReader + Send + Sync { /// Returns None if the transaction is not found. fn transaction_id(&self, tx_hash: TxHash) -> Result>; - /// Get transaction by id. + /// Get transaction by id, computes hash everytime so more expensive. fn transaction_by_id(&self, id: TxNumber) -> Result>; + /// Get transaction by id without computing the hash. + fn transaction_by_id_no_hash(&self, id: TxNumber) -> Result>; + /// Get transaction by transaction hash. fn transaction_by_hash(&self, hash: TxHash) -> Result>; From d9176c296a1f87344789f46dea7b4b99be8724e5 Mon Sep 17 00:00:00 2001 From: fomotrader <82184770+fomotrader@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:05:58 -0400 Subject: [PATCH 055/150] docs: how to enable JSON-RPC endpoints for http/ws (#3711) --- book/jsonrpc/intro.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/book/jsonrpc/intro.md b/book/jsonrpc/intro.md index 4f3e72bb3e1..1c602f6d2b5 100644 --- a/book/jsonrpc/intro.md +++ b/book/jsonrpc/intro.md @@ -58,6 +58,16 @@ To enable JSON-RPC namespaces on the HTTP server, pass each namespace separated reth node --http --http.api eth,net,trace ``` +You can pass the `all` option, which is a convenient wrapper for the all the JSON-RPC namespaces `admin,debug,eth,net,trace,txpool,web3,rpc` on the HTTP server: + +```bash +reth node --http --http.api all +``` + +```bash +reth node --http --http.api All +``` + You can also restrict who can access the HTTP server by specifying a domain for Cross-Origin requests. This is important, since any application local to your node will be able to access the RPC server: ```bash From 4cac26ee5bdf00fe0f871136e13806733286bd63 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Jul 2023 15:56:24 +0200 Subject: [PATCH 056/150] feat: add txpool maintain metrics (#3715) --- crates/transaction-pool/src/maintain.rs | 16 +++++++++++----- crates/transaction-pool/src/metrics.rs | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 0c619f6d2bc..a55340ed55a 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -1,6 +1,7 @@ //! Support for maintaining the state of the transaction pool use crate::{ + metrics::MaintainPoolMetrics, traits::{CanonicalStateUpdate, ChangedAccount, TransactionPoolExt}, BlockInfo, TransactionPool, }; @@ -45,6 +46,7 @@ where P: TransactionPoolExt + 'static, St: Stream + Send + Unpin + 'static, { + let mut metrics = MaintainPoolMetrics::default(); // ensure the pool points to latest state if let Ok(Some(latest)) = client.block_by_number_or_tag(BlockNumberOrTag::Latest) { let latest = latest.seal_slow(); @@ -63,7 +65,11 @@ where let mut maintained_state = MaintainedPoolState::InSync; // Listen for new chain events and derive the update action for the pool - while let Some(event) = events.next().await { + loop { + metrics.set_dirty_accounts_len(dirty_addresses.len()); + + let Some(event) = events.next().await else { break }; + let pool_info = pool.block_info(); // TODO from time to time re-check the unique accounts in the pool and remove and resync @@ -136,7 +142,7 @@ where .filter(|tx| !new_mined_transactions.contains(&tx.hash)) .filter_map(|tx| tx.clone().into_ecrecovered()) .map(

::Transaction::from_recovered_transaction) - .collect(); + .collect::>(); // update the pool first let update = CanonicalStateUpdate { @@ -153,8 +159,8 @@ where // to be re-injected // // Note: we no longer know if the tx was local or external + metrics.inc_reinserted_transactions(pruned_old_transactions.len()); let _ = pool.add_external_transactions(pruned_old_transactions).await; - // TODO: metrics } CanonStateNotification::Revert { old } => { // this similar to the inverse of a commit where we need to insert the transactions @@ -193,13 +199,13 @@ where .transactions() .filter_map(|tx| tx.clone().into_ecrecovered()) .map(

::Transaction::from_recovered_transaction) - .collect(); + .collect::>(); // all transactions that were mined in the old chain need to be re-injected // // Note: we no longer know if the tx was local or external + metrics.inc_reinserted_transactions(pruned_old_transactions.len()); let _ = pool.add_external_transactions(pruned_old_transactions).await; - // TODO: metrics } CanonStateNotification::Commit { new } => { let (blocks, state) = new.inner(); diff --git a/crates/transaction-pool/src/metrics.rs b/crates/transaction-pool/src/metrics.rs index 91e31992bf2..5be12634e57 100644 --- a/crates/transaction-pool/src/metrics.rs +++ b/crates/transaction-pool/src/metrics.rs @@ -34,3 +34,26 @@ pub struct TxPoolMetrics { /// Number of all transactions of all sub-pools: pending + basefee + queued pub(crate) total_transactions: Gauge, } + +/// Transaction pool maintenance metrics +#[derive(Metrics)] +#[metrics(scope = "transaction_pool")] +pub struct MaintainPoolMetrics { + /// Number of currently dirty addresses that need to be updated in the pool by fetching account + /// info + pub(crate) dirty_accounts: Gauge, + /// Number of transaction reinserted into the pool after reorg. + pub(crate) reinserted_transactions: Counter, +} + +impl MaintainPoolMetrics { + #[inline] + pub(crate) fn set_dirty_accounts_len(&self, count: usize) { + self.dirty_accounts.set(count as f64); + } + + #[inline] + pub(crate) fn inc_reinserted_transactions(&self, count: usize) { + self.reinserted_transactions.increment(count as u64); + } +} From 95cfc2b72c06a0f7b8dbc9c4f0b00a943e8a688b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Jul 2023 15:56:33 +0200 Subject: [PATCH 057/150] feat: add performed pool state updates metric (#3714) --- crates/transaction-pool/src/metrics.rs | 3 +++ crates/transaction-pool/src/pool/txpool.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/crates/transaction-pool/src/metrics.rs b/crates/transaction-pool/src/metrics.rs index 5be12634e57..1a29e4b75f3 100644 --- a/crates/transaction-pool/src/metrics.rs +++ b/crates/transaction-pool/src/metrics.rs @@ -33,6 +33,9 @@ pub struct TxPoolMetrics { /// Number of all transactions of all sub-pools: pending + basefee + queued pub(crate) total_transactions: Gauge, + + /// How often the pool was updated after the canonical state changed + pub(crate) performed_state_updates: Counter, } /// Transaction pool maintenance metrics diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 9793da32a30..6b1bad86442 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -261,6 +261,8 @@ impl TxPool { // update the metrics after the update self.update_size_metrics(); + self.metrics.performed_state_updates.increment(1); + OnNewCanonicalStateOutcome { block_hash, mined: mined_transactions, promoted, discarded } } From 485cda5844aa8d265ec73ab2e727127bffb6b781 Mon Sep 17 00:00:00 2001 From: Jay Miller <3744812+jaylmiller@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:07:13 -0400 Subject: [PATCH 058/150] perf: handle engine API range request in a new task (#3685) Co-authored-by: Matthias Seitz --- Cargo.lock | 1 + bin/reth/src/node/mod.rs | 1 + crates/rpc/rpc-builder/tests/it/utils.rs | 1 + crates/rpc/rpc-engine-api/Cargo.toml | 1 + crates/rpc/rpc-engine-api/src/engine_api.rs | 95 ++++++++++++++------- 5 files changed, 70 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8b62e26467..8bb5dd01053 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5736,6 +5736,7 @@ dependencies = [ "reth-provider", "reth-rpc-api", "reth-rpc-types", + "reth-tasks", "thiserror", "tokio", "tracing", diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 529d67713ac..13fd00d78e8 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -400,6 +400,7 @@ impl Command { self.chain.clone(), beacon_engine_handle, payload_builder.into(), + Box::new(ctx.task_executor.clone()), ); info!(target: "reth::cli", "Engine API handler initialized"); diff --git a/crates/rpc/rpc-builder/tests/it/utils.rs b/crates/rpc/rpc-builder/tests/it/utils.rs index e7c4a4eb88a..0b111b33cc8 100644 --- a/crates/rpc/rpc-builder/tests/it/utils.rs +++ b/crates/rpc/rpc-builder/tests/it/utils.rs @@ -30,6 +30,7 @@ pub async fn launch_auth(secret: JwtSecret) -> AuthServerHandle { MAINNET.clone(), beacon_engine_handle, spawn_test_payload_service().into(), + Box::new(TokioTaskExecutor::default()), ); let module = AuthRpcModule::new(engine_api); module.start_server(config).await.unwrap() diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index 32059744411..162c801e19a 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -17,6 +17,7 @@ reth-rpc-types = { workspace = true } reth-rpc-api = { path = "../rpc-api" } reth-beacon-consensus = { path = "../../consensus/beacon" } reth-payload-builder = { workspace = true } +reth-tasks = { workspace = true } # async tokio = { workspace = true, features = ["sync"] } diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 4803c4d2d71..f461f4928dc 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -11,6 +11,7 @@ use reth_rpc_types::engine::{ ExecutionPayload, ExecutionPayloadBodies, ExecutionPayloadEnvelope, ForkchoiceUpdated, PayloadAttributes, PayloadId, PayloadStatus, TransitionConfiguration, CAPABILITIES, }; +use reth_tasks::TaskSpawner; use std::sync::Arc; use tokio::sync::oneshot; use tracing::trace; @@ -24,6 +25,10 @@ const MAX_PAYLOAD_BODIES_LIMIT: u64 = 1024; /// The Engine API implementation that grants the Consensus layer access to data and /// functions in the Execution layer that are crucial for the consensus process. pub struct EngineApi { + inner: Arc>, +} + +struct EngineApiInner { /// The provider to interact with the chain. provider: Provider, /// Consensus configuration @@ -32,6 +37,8 @@ pub struct EngineApi { beacon_consensus: BeaconConsensusEngineHandle, /// The type that can communicate with the payload service to retrieve payloads. payload_store: PayloadStore, + /// For spawning and executing async tasks + task_spawner: Box, } impl EngineApi @@ -44,8 +51,16 @@ where chain_spec: Arc, beacon_consensus: BeaconConsensusEngineHandle, payload_store: PayloadStore, + task_spawner: Box, ) -> Self { - Self { provider, chain_spec, beacon_consensus, payload_store } + let inner = Arc::new(EngineApiInner { + provider, + chain_spec, + beacon_consensus, + payload_store, + task_spawner, + }); + Self { inner } } /// See also @@ -59,7 +74,7 @@ where payload.timestamp.as_u64(), payload.withdrawals.is_some(), )?; - Ok(self.beacon_consensus.new_payload(payload).await?) + Ok(self.inner.beacon_consensus.new_payload(payload).await?) } /// See also @@ -72,7 +87,7 @@ where payload.timestamp.as_u64(), payload.withdrawals.is_some(), )?; - Ok(self.beacon_consensus.new_payload(payload).await?) + Ok(self.inner.beacon_consensus.new_payload(payload).await?) } /// Sends a message to the beacon consensus engine to update the fork choice _without_ @@ -93,7 +108,7 @@ where attrs.withdrawals.is_some(), )?; } - Ok(self.beacon_consensus.fork_choice_updated(state, payload_attrs).await?) + Ok(self.inner.beacon_consensus.fork_choice_updated(state, payload_attrs).await?) } /// Sends a message to the beacon consensus engine to update the fork choice _with_ withdrawals, @@ -112,7 +127,7 @@ where attrs.withdrawals.is_some(), )?; } - Ok(self.beacon_consensus.fork_choice_updated(state, payload_attrs).await?) + Ok(self.inner.beacon_consensus.fork_choice_updated(state, payload_attrs).await?) } /// Returns the most recent version of the payload that is available in the corresponding @@ -126,6 +141,7 @@ where /// > Provider software MAY stop the corresponding build process after serving this call. pub async fn get_payload_v1(&self, payload_id: PayloadId) -> EngineApiResult { Ok(self + .inner .payload_store .resolve(payload_id) .await @@ -145,6 +161,7 @@ where payload_id: PayloadId, ) -> EngineApiResult { Ok(self + .inner .payload_store .resolve(payload_id) .await @@ -162,31 +179,44 @@ where /// Implementors should take care when acting on the input to this method, specifically /// ensuring that the range is limited properly, and that the range boundaries are computed /// correctly and without panics. - pub fn get_payload_bodies_by_range( + pub async fn get_payload_bodies_by_range( &self, start: BlockNumber, count: u64, ) -> EngineApiResult { - if count > MAX_PAYLOAD_BODIES_LIMIT { - return Err(EngineApiError::PayloadRequestTooLarge { len: count }) - } + let (tx, rx) = oneshot::channel(); + let inner = self.inner.clone(); - if start == 0 || count == 0 { - return Err(EngineApiError::InvalidBodiesRange { start, count }) - } + self.inner.task_spawner.spawn_blocking(Box::pin(async move { + if count > MAX_PAYLOAD_BODIES_LIMIT { + tx.send(Err(EngineApiError::PayloadRequestTooLarge { len: count })).ok(); + return + } - let mut result = Vec::with_capacity(count as usize); + if start == 0 || count == 0 { + tx.send(Err(EngineApiError::InvalidBodiesRange { start, count })).ok(); + return + } - let end = start.saturating_add(count); - for num in start..end { - let block = self - .provider - .block(BlockHashOrNumber::Number(num)) - .map_err(|err| EngineApiError::Internal(Box::new(err)))?; - result.push(block.map(Into::into)); - } + let mut result = Vec::with_capacity(count as usize); - Ok(result) + let end = start.saturating_add(count); + for num in start..end { + let block_result = inner.provider.block(BlockHashOrNumber::Number(num)); + match block_result { + Ok(block) => { + result.push(block.map(Into::into)); + } + Err(err) => { + tx.send(Err(EngineApiError::Internal(Box::new(err)))).ok(); + return + } + }; + } + tx.send(Ok(result)).ok(); + })); + + rx.await.map_err(|err| EngineApiError::Internal(Box::new(err)))? } /// Called to retrieve execution payload bodies by hashes. @@ -202,6 +232,7 @@ where let mut result = Vec::with_capacity(hashes.len()); for hash in hashes { let block = self + .inner .provider .block(BlockHashOrNumber::Hash(hash)) .map_err(|err| EngineApiError::Internal(Box::new(err)))?; @@ -224,6 +255,7 @@ where } = config; let merge_terminal_td = self + .inner .chain_spec .fork(Hardfork::Paris) .ttd() @@ -237,7 +269,7 @@ where }) } - self.beacon_consensus.transition_configuration_exchanged().await; + self.inner.beacon_consensus.transition_configuration_exchanged().await; // Short circuit if communicated block hash is zero if terminal_block_hash.is_zero() { @@ -249,6 +281,7 @@ where // Attempt to look up terminal block hash let local_hash = self + .inner .provider .block_hash(terminal_block_number.as_u64()) .map_err(|err| EngineApiError::Internal(Box::new(err)))?; @@ -276,7 +309,8 @@ where timestamp: u64, has_withdrawals: bool, ) -> EngineApiResult<()> { - let is_shanghai = self.chain_spec.fork(Hardfork::Shanghai).active_at_timestamp(timestamp); + let is_shanghai = + self.inner.chain_spec.fork(Hardfork::Shanghai).active_at_timestamp(timestamp); match version { EngineApiMessageVersion::V1 => { @@ -404,7 +438,7 @@ where count: U64, ) -> RpcResult { trace!(target: "rpc::engine", "Serving engine_getPayloadBodiesByRangeV1"); - Ok(EngineApi::get_payload_bodies_by_range(self, start.as_u64(), count.as_u64())?) + Ok(EngineApi::get_payload_bodies_by_range(self, start.as_u64(), count.as_u64()).await?) } /// Handler for `engine_exchangeTransitionConfigurationV1` @@ -439,6 +473,7 @@ mod tests { use reth_payload_builder::test_utils::spawn_test_payload_service; use reth_primitives::{SealedBlock, H256, MAINNET}; use reth_provider::test_utils::MockEthProvider; + use reth_tasks::TokioTaskExecutor; use std::sync::Arc; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; @@ -447,11 +482,13 @@ mod tests { let provider = Arc::new(MockEthProvider::default()); let payload_store = spawn_test_payload_service(); let (to_engine, engine_rx) = unbounded_channel(); + let task_executor = Box::new(TokioTaskExecutor::default()); let api = EngineApi::new( provider.clone(), chain_spec.clone(), BeaconConsensusEngineHandle::new(to_engine), payload_store.into(), + task_executor, ); let handle = EngineApiTestHandle { chain_spec, provider, from_api: engine_rx }; (handle, api) @@ -491,7 +528,7 @@ mod tests { // test [EngineApiMessage::GetPayloadBodiesByRange] for (start, count) in by_range_tests { - let res = api.get_payload_bodies_by_range(start, count); + let res = api.get_payload_bodies_by_range(start, count).await; assert_matches!(res, Err(EngineApiError::InvalidBodiesRange { .. })); } } @@ -501,7 +538,7 @@ mod tests { let (_, api) = setup_engine_api(); let request_count = MAX_PAYLOAD_BODIES_LIMIT + 1; - let res = api.get_payload_bodies_by_range(0, request_count); + let res = api.get_payload_bodies_by_range(0, request_count).await; assert_matches!(res, Err(EngineApiError::PayloadRequestTooLarge { .. })); } @@ -518,7 +555,7 @@ mod tests { let expected = blocks.iter().cloned().map(|b| Some(b.unseal().into())).collect::>(); - let res = api.get_payload_bodies_by_range(start, count).unwrap(); + let res = api.get_payload_bodies_by_range(start, count).await.unwrap(); assert_eq!(res, expected); } @@ -558,7 +595,7 @@ mod tests { }) .collect::>(); - let res = api.get_payload_bodies_by_range(start, count).unwrap(); + let res = api.get_payload_bodies_by_range(start, count).await.unwrap(); assert_eq!(res, expected); let hashes = blocks.iter().map(|b| b.hash()).collect(); From c8a8e450375478e0c41712f1e1781b189acd078c Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 11 Jul 2023 16:12:20 +0100 Subject: [PATCH 059/150] feat(primitives, storage): save prune checkpoints in database (#3628) --- crates/primitives/src/lib.rs | 2 +- crates/primitives/src/prune/mod.rs | 2 ++ crates/primitives/src/prune/mode.rs | 2 +- crates/primitives/src/prune/part.rs | 24 ++++++++++++++++++++++ crates/storage/db/src/abstraction/table.rs | 2 +- crates/storage/db/src/tables/mod.rs | 9 ++++++-- crates/storage/db/src/tables/models/mod.rs | 19 ++++++++++++++++- 7 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 crates/primitives/src/prune/part.rs diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 4d0cbc0240f..541289b3d30 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -80,7 +80,7 @@ pub use net::{ SEPOLIA_BOOTNODES, }; pub use peer::{PeerId, WithPeerId}; -pub use prune::{PruneCheckpoint, PruneMode, PruneTargets}; +pub use prune::{PruneCheckpoint, PruneMode, PrunePart, PruneTargets}; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef}; pub use revm_primitives::JumpMap; pub use serde_helper::JsonU256; diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index 18cb943fb97..510bc40b6e5 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -1,7 +1,9 @@ mod checkpoint; mod mode; +mod part; mod target; pub use checkpoint::PruneCheckpoint; pub use mode::PruneMode; +pub use part::PrunePart; pub use target::PruneTargets; diff --git a/crates/primitives/src/prune/mode.rs b/crates/primitives/src/prune/mode.rs index 3fba10fe59a..b62a39041b8 100644 --- a/crates/primitives/src/prune/mode.rs +++ b/crates/primitives/src/prune/mode.rs @@ -17,7 +17,7 @@ pub enum PruneMode { #[cfg(test)] impl Default for PruneMode { fn default() -> Self { - Self::Distance(0) + Self::Full } } diff --git a/crates/primitives/src/prune/part.rs b/crates/primitives/src/prune/part.rs new file mode 100644 index 00000000000..caa176b86a2 --- /dev/null +++ b/crates/primitives/src/prune/part.rs @@ -0,0 +1,24 @@ +use reth_codecs::{main_codec, Compact}; + +/// Part of the data that can be pruned. +#[main_codec] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] +pub enum PrunePart { + /// Prune part responsible for the `TxSenders` table. + SenderRecovery, + /// Prune part responsible for the `TxHashNumber` table. + TransactionLookup, + /// Prune part responsible for the `Receipts` table. + Receipts, + /// Prune part responsible for the `AccountChangeSet` and `AccountHistory` tables. + AccountHistory, + /// Prune part responsible for the `StorageChangeSet` and `StorageHistory` tables. + StorageHistory, +} + +#[cfg(test)] +impl Default for PrunePart { + fn default() -> Self { + Self::SenderRecovery + } +} diff --git a/crates/storage/db/src/abstraction/table.rs b/crates/storage/db/src/abstraction/table.rs index 65d611f8685..18e66fe0e17 100644 --- a/crates/storage/db/src/abstraction/table.rs +++ b/crates/storage/db/src/abstraction/table.rs @@ -49,7 +49,7 @@ pub trait Encode: Send + Sync + Sized + Debug { /// Trait that will transform the data to be read from the DB. pub trait Decode: Send + Sync + Sized + Debug { /// Decodes data coming from the database. - fn decode>(key: B) -> Result; + fn decode>(value: B) -> Result; } /// Generic trait that enforces the database key to implement [`Encode`] and [`Decode`]. diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index ba44ce1ea03..924095ca71d 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -37,8 +37,8 @@ use crate::{ use reth_primitives::{ stage::StageCheckpoint, trie::{BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey}, - Account, Address, BlockHash, BlockNumber, Bytecode, Header, IntegerList, Receipt, StorageEntry, - TransactionSignedNoHash, TxHash, TxNumber, H256, + Account, Address, BlockHash, BlockNumber, Bytecode, Header, IntegerList, PruneCheckpoint, + PrunePart, Receipt, StorageEntry, TransactionSignedNoHash, TxHash, TxNumber, H256, }; /// Enum for the types of tables present in libmdbx. @@ -415,6 +415,11 @@ table!( ( SyncStageProgress ) StageId | Vec ); +table!( + /// Stores the highest pruned block number and prune mode of each prune part. + ( PruneParts ) PrunePart | PruneCheckpoint +); + /// Alias Types /// List with transaction numbers. diff --git a/crates/storage/db/src/tables/models/mod.rs b/crates/storage/db/src/tables/models/mod.rs index da746efda68..3bfd0fbfe95 100644 --- a/crates/storage/db/src/tables/models/mod.rs +++ b/crates/storage/db/src/tables/models/mod.rs @@ -6,7 +6,7 @@ use crate::{ use reth_codecs::Compact; use reth_primitives::{ trie::{StoredNibbles, StoredNibblesSubKey}, - Address, H256, + Address, PrunePart, H256, }; pub mod accounts; @@ -135,3 +135,20 @@ impl Decode for StoredNibblesSubKey { Ok(Self::from_compact(buf, buf.len()).0) } } + +impl Encode for PrunePart { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + let mut buf = [0u8]; + self.to_compact(&mut buf.as_mut()); + buf + } +} + +impl Decode for PrunePart { + fn decode>(value: B) -> Result { + let buf = value.as_ref(); + Ok(Self::from_compact(buf, buf.len()).0) + } +} From 9fe39cc3ff5f1e46b5a5906955d7aa7d566ece18 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Jul 2023 17:13:06 +0200 Subject: [PATCH 060/150] fix: treat bool params as invalid in logs subscription (#3716) --- crates/rpc/rpc-types/src/eth/pubsub.rs | 29 +++++++++++++++++++++++++- crates/rpc/rpc/src/eth/pubsub.rs | 3 +++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-types/src/eth/pubsub.rs b/crates/rpc/rpc-types/src/eth/pubsub.rs index e363cb754c3..d54f6f849a8 100644 --- a/crates/rpc/rpc-types/src/eth/pubsub.rs +++ b/crates/rpc/rpc-types/src/eth/pubsub.rs @@ -96,7 +96,7 @@ pub enum SubscriptionKind { Syncing, } -/// Subscription kind. +/// Any additional parameters for a subscription. #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub enum Params { /// No parameters passed. @@ -108,6 +108,20 @@ pub enum Params { Bool(bool), } +impl Params { + /// Returns true if it's a bool parameter. + #[inline] + pub fn is_bool(&self) -> bool { + matches!(self, Params::Bool(_)) + } + + /// Returns true if it's a log parameter. + #[inline] + pub fn is_logs(&self) -> bool { + matches!(self, Params::Logs(_)) + } +} + impl Serialize for Params { fn serialize(&self, serializer: S) -> Result where @@ -141,3 +155,16 @@ impl<'a> Deserialize<'a> for Params { .map_err(|e| D::Error::custom(format!("Invalid Pub-Sub parameters: {e}"))) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn params_serde() { + let s: Params = serde_json::from_str("true").unwrap(); + assert_eq!(s, Params::Bool(true)); + let s: Params = serde_json::from_str("null").unwrap(); + assert_eq!(s, Params::None); + } +} diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index a6693001136..5b823ea40cb 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -115,6 +115,9 @@ where // if no params are provided, used default filter params let filter = match params { Some(Params::Logs(filter)) => FilteredParams::new(Some(*filter)), + Some(Params::Bool(_)) => { + return Err(invalid_params_rpc_err("Invalid params for logs").into()) + } _ => FilteredParams::default(), }; let stream = From 4e0a61ad23baa1ca303a52aa3b2d34eeb2250c07 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 11 Jul 2023 16:49:33 +0100 Subject: [PATCH 061/150] chore: use `transaction_by_id_no_hash` to avoid hash computation (#3718) --- crates/stages/src/stages/sender_recovery.rs | 10 ++++++++-- .../provider/src/providers/database/provider.rs | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/stages/src/stages/sender_recovery.rs b/crates/stages/src/stages/sender_recovery.rs index 482ed1eb43f..e6294b3a861 100644 --- a/crates/stages/src/stages/sender_recovery.rs +++ b/crates/stages/src/stages/sender_recovery.rs @@ -453,8 +453,14 @@ mod tests { while let Some((_, body)) = body_cursor.next()? { for tx_id in body.tx_num_range() { - let transaction: TransactionSigned = - provider.transaction_by_id(tx_id)?.expect("no transaction entry"); + let transaction: TransactionSigned = provider + .transaction_by_id_no_hash(tx_id)? + .map(|tx| TransactionSigned { + hash: Default::default(), // we don't require the hash + signature: tx.signature, + transaction: tx.transaction, + }) + .expect("no transaction entry"); let signer = transaction.recover_signer().expect("failed to recover signer"); assert_eq!(Some(signer), provider.transaction_sender(tx_id)?) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 28a00f45e20..2d494c40e26 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -935,12 +935,16 @@ impl<'this, TX: DbTx<'this>> TransactionsProvider for DatabaseProvider<'this, TX } fn transaction_by_id_no_hash(&self, id: TxNumber) -> Result> { - Ok(self.tx.get::(id)?.map(Into::into)) + Ok(self.tx.get::(id)?) } fn transaction_by_hash(&self, hash: TxHash) -> Result> { if let Some(id) = self.transaction_id(hash)? { - Ok(self.transaction_by_id(id)?) + Ok(self.transaction_by_id_no_hash(id)?.map(|tx| TransactionSigned { + hash, + signature: tx.signature, + transaction: tx.transaction, + })) } else { Ok(None) } @@ -953,7 +957,12 @@ impl<'this, TX: DbTx<'this>> TransactionsProvider for DatabaseProvider<'this, TX ) -> Result> { let mut transaction_cursor = self.tx.cursor_read::()?; if let Some(transaction_id) = self.transaction_id(tx_hash)? { - if let Some(transaction) = self.transaction_by_id(transaction_id)? { + if let Some(tx) = self.transaction_by_id_no_hash(transaction_id)? { + let transaction = TransactionSigned { + hash: tx_hash, + signature: tx.signature, + transaction: tx.transaction, + }; if let Some(block_number) = transaction_cursor.seek(transaction_id).map(|b| b.map(|(_, bn)| bn))? { From a9b7e5b9422c9b0dfda39a0bbfd810fadeb09875 Mon Sep 17 00:00:00 2001 From: Siyuan Han <47173566+hsyodyssey@users.noreply.github.com> Date: Wed, 12 Jul 2023 00:48:33 +0800 Subject: [PATCH 062/150] Chore(book): correct the grafana dashboards json path (#3724) --- book/run/observability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book/run/observability.md b/book/run/observability.md index 43a24199917..9eb3e99a5e8 100644 --- a/book/run/observability.md +++ b/book/run/observability.md @@ -51,7 +51,7 @@ Once you've logged in, click on the gear icon in the lower left, and select "Dat As this might be a point of confusion, `localhost:9001`, which we supplied to `--metrics`, is the endpoint that Reth exposes, from which Prometheus collects metrics. Prometheus then exposes `localhost:9090` (by default) for other services (such as Grafana) to consume Prometheus metrics. -To configure the dashboard in Grafana, click on the squares icon in the upper left, and click on "New", then "Import". From there, click on "Upload JSON file", and select the example file in [`reth/etc/grafana/overview.json`](https://github.com/paradigmxyz/reth/blob/main/etc/grafana/dashboards/overview.json). Finally, select the Prometheus data source you just created, and click "Import". +To configure the dashboard in Grafana, click on the squares icon in the upper left, and click on "New", then "Import". From there, click on "Upload JSON file", and select the example file in [`reth/etc/grafana/dashboards/overview.json`](https://github.com/paradigmxyz/reth/blob/main/etc/grafana/dashboards/overview.json). Finally, select the Prometheus data source you just created, and click "Import". And voilá, you should see your dashboard! If you're not yet connected to any peers, the dashboard will look like it's in an empty state, but once you are, you should see it start populating with data. From eab681eb730cfd1f92279157bef7fc3da3bbbfc5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Jul 2023 19:38:13 +0200 Subject: [PATCH 063/150] perf: only update finalized safe if changed (#3725) --- .../consensus/beacon/src/engine/forkchoice.rs | 2 + crates/consensus/beacon/src/engine/mod.rs | 83 ++++++++++++++----- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/crates/consensus/beacon/src/engine/forkchoice.rs b/crates/consensus/beacon/src/engine/forkchoice.rs index f41200e3961..51c67f7eef3 100644 --- a/crates/consensus/beacon/src/engine/forkchoice.rs +++ b/crates/consensus/beacon/src/engine/forkchoice.rs @@ -39,6 +39,8 @@ impl ForkchoiceStateTracker { } /// Returns the [ForkchoiceStatus] of the latest received FCU. + /// + /// Caution: this can be invalid. pub(crate) fn latest_status(&self) -> Option { self.latest.as_ref().map(|s| s.status) } diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index f8b95d4682c..b69d3bfee70 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -25,7 +25,8 @@ use reth_primitives::{ Head, Header, SealedBlock, SealedHeader, H256, U256, }; use reth_provider::{ - BlockReader, BlockSource, CanonChainTracker, ProviderError, StageCheckpointReader, + BlockIdReader, BlockReader, BlockSource, CanonChainTracker, ProviderError, + StageCheckpointReader, }; use reth_rpc_types::engine::{ ExecutionPayload, PayloadAttributes, PayloadStatus, PayloadStatusEnum, PayloadValidationError, @@ -147,7 +148,11 @@ pub struct BeaconConsensusEngine where DB: Database, Client: HeadersClient + BodiesClient, - BT: BlockchainTreeEngine + BlockReader + CanonChainTracker + StageCheckpointReader, + BT: BlockchainTreeEngine + + BlockReader + + BlockIdReader + + CanonChainTracker + + StageCheckpointReader, { /// Controls syncing triggered by engine updates. sync: EngineSyncController, @@ -187,7 +192,12 @@ where impl BeaconConsensusEngine where DB: Database + Unpin + 'static, - BT: BlockchainTreeEngine + BlockReader + CanonChainTracker + StageCheckpointReader + 'static, + BT: BlockchainTreeEngine + + BlockReader + + BlockIdReader + + CanonChainTracker + + StageCheckpointReader + + 'static, Client: HeadersClient + BodiesClient + Clone + Unpin + 'static, { /// Create a new instance of the [BeaconConsensusEngine]. @@ -657,25 +667,8 @@ where // we update the the tracked header first self.blockchain.set_canonical_head(head); - if !update.finalized_block_hash.is_zero() { - let finalized = self - .blockchain - .find_block_by_hash(update.finalized_block_hash, BlockSource::Any)? - .ok_or_else(|| { - Error::Provider(ProviderError::UnknownBlockHash(update.finalized_block_hash)) - })?; - self.blockchain.set_finalized(finalized.header.seal(update.finalized_block_hash)); - } - - if !update.safe_block_hash.is_zero() { - let safe = self - .blockchain - .find_block_by_hash(update.safe_block_hash, BlockSource::Any)? - .ok_or_else(|| { - Error::Provider(ProviderError::UnknownBlockHash(update.safe_block_hash)) - })?; - self.blockchain.set_safe(safe.header.seal(update.safe_block_hash)); - } + self.update_finalized_block(update.finalized_block_hash)?; + self.update_safe_block(update.safe_block_hash)?; head_block.total_difficulty = self.blockchain.header_td_by_number(head_block.number)?.ok_or_else(|| { @@ -688,6 +681,51 @@ where Ok(()) } + /// Updates the tracked safe block if we have it + /// + /// Returns an error if the block is not found. + #[inline] + fn update_safe_block(&self, safe_block_hash: H256) -> Result<(), reth_interfaces::Error> { + if !safe_block_hash.is_zero() { + if self.blockchain.safe_block_hash()? == Some(safe_block_hash) { + // nothing to update + return Ok(()) + } + + let safe = self + .blockchain + .find_block_by_hash(safe_block_hash, BlockSource::Any)? + .ok_or_else(|| Error::Provider(ProviderError::UnknownBlockHash(safe_block_hash)))?; + self.blockchain.set_safe(safe.header.seal(safe_block_hash)); + } + Ok(()) + } + + /// Updates the tracked finalized block if we have it + /// + /// Returns an error if the block is not found. + #[inline] + fn update_finalized_block( + &self, + finalized_block_hash: H256, + ) -> Result<(), reth_interfaces::Error> { + if !finalized_block_hash.is_zero() { + if self.blockchain.finalized_block_hash()? == Some(finalized_block_hash) { + // nothing to update + return Ok(()) + } + + let finalized = self + .blockchain + .find_block_by_hash(finalized_block_hash, BlockSource::Any)? + .ok_or_else(|| { + Error::Provider(ProviderError::UnknownBlockHash(finalized_block_hash)) + })?; + self.blockchain.set_finalized(finalized.header.seal(finalized_block_hash)); + } + Ok(()) + } + /// Handler for a failed a forkchoice update due to a canonicalization error. /// /// This will determine if the state's head is invalid, and if so, return immediately. @@ -1386,6 +1424,7 @@ where Client: HeadersClient + BodiesClient + Clone + Unpin + 'static, BT: BlockchainTreeEngine + BlockReader + + BlockIdReader + CanonChainTracker + StageCheckpointReader + Unpin From 3c02b309cc358ece2b728f4de56d1bb6ef26afb7 Mon Sep 17 00:00:00 2001 From: int88 <106391185+int88@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:15:05 +0800 Subject: [PATCH 064/150] test: fix engine hive of `Invalid Transition Payload Sync` (#3710) --- crates/consensus/beacon/src/engine/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index b69d3bfee70..ad134622073 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -887,8 +887,15 @@ where let block_hash = block.hash(); let block_num_hash = block.num_hash(); + let mut lowest_buffered_ancestor = self.lowest_buffered_ancestor_or(block.hash); + if lowest_buffered_ancestor == block.hash { + lowest_buffered_ancestor = block.parent_hash; + } + // now check the block itself - if let Some(status) = self.check_invalid_ancestor_with_head(block.parent_hash, block.hash) { + if let Some(status) = + self.check_invalid_ancestor_with_head(lowest_buffered_ancestor, block.hash) + { return Ok(status) } From 3aba2a617b14049c2b78c11fe43e4f24fa98e4df Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Jul 2023 22:39:51 +0200 Subject: [PATCH 065/150] docs: add a few more docs to ControlFlow (#3603) --- crates/consensus/beacon/src/engine/mod.rs | 4 ++-- crates/stages/src/pipeline/ctrl.rs | 8 +++++--- crates/stages/src/pipeline/mod.rs | 5 +++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index ad134622073..10e53323291 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1306,7 +1306,7 @@ where // update the canon chain if continuous is enabled if self.sync.run_pipeline_continuously() { - let max_block = ctrl.progress().unwrap_or_default(); + let max_block = ctrl.block_number().unwrap_or_default(); let max_header = match self.blockchain.sealed_header(max_block) { Ok(header) => match header { Some(header) => header, @@ -1371,7 +1371,7 @@ where // If both are Some, we perform another distance check and return the desired // pipeline target let pipeline_target = if let (Some(progress), Some(finalized_number)) = - (ctrl.progress(), newest_finalized) + (ctrl.block_number(), newest_finalized) { // Determines whether or not we should run the pipeline again, in case the // new gap is large enough to warrant running the pipeline. diff --git a/crates/stages/src/pipeline/ctrl.rs b/crates/stages/src/pipeline/ctrl.rs index deece92d25f..820df423b63 100644 --- a/crates/stages/src/pipeline/ctrl.rs +++ b/crates/stages/src/pipeline/ctrl.rs @@ -1,6 +1,8 @@ use reth_primitives::{BlockNumber, SealedHeader}; /// Determines the control flow during pipeline execution. +/// +/// See [Pipeline::run_loop](crate::Pipeline::run_loop) for more information. #[derive(Debug, Eq, PartialEq)] pub enum ControlFlow { /// An unwind was requested and must be performed before continuing. @@ -10,7 +12,7 @@ pub enum ControlFlow { /// The block that caused the unwind. bad_block: SealedHeader, }, - /// The pipeline is allowed to continue executing stages. + /// The pipeline made progress. Continue { /// Block number reached by the stage. block_number: BlockNumber, @@ -33,8 +35,8 @@ impl ControlFlow { matches!(self, ControlFlow::Unwind { .. }) } - /// Returns the pipeline progress, if the state is not `Unwind`. - pub fn progress(&self) -> Option { + /// Returns the pipeline block number the stage reached, if the state is not `Unwind`. + pub fn block_number(&self) -> Option { match self { ControlFlow::Unwind { .. } => None, ControlFlow::Continue { block_number } => Some(*block_number), diff --git a/crates/stages/src/pipeline/mod.rs b/crates/stages/src/pipeline/mod.rs index fa2ec285e0a..a108695cfad 100644 --- a/crates/stages/src/pipeline/mod.rs +++ b/crates/stages/src/pipeline/mod.rs @@ -206,6 +206,11 @@ where /// If any stage is unsuccessful at execution, we proceed to /// unwind. This will undo the progress across the entire pipeline /// up to the block that caused the error. + /// + /// Returns the control flow after it ran the pipeline. + /// This will be [ControlFlow::Continue] or [ControlFlow::NoProgress] of the _last_ stage in the + /// pipeline (for example the `Finish` stage). Or [ControlFlow::Unwind] of the stage that caused + /// the unwind. pub async fn run_loop(&mut self) -> Result { let mut previous_stage = None; for stage_index in 0..self.stages.len() { From 22ab8f126663318306577574895d8273b81351a4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Jul 2023 22:59:21 +0200 Subject: [PATCH 066/150] fix: use tx gas limit for root trace (#3719) --- .../revm/revm-inspectors/src/tracing/mod.rs | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/revm/revm-inspectors/src/tracing/mod.rs b/crates/revm/revm-inspectors/src/tracing/mod.rs index b064e0dd692..bf3e746c90f 100644 --- a/crates/revm/revm-inspectors/src/tracing/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/mod.rs @@ -146,15 +146,15 @@ impl TracingInspector { /// /// Invoked on [Inspector::call]. #[allow(clippy::too_many_arguments)] - fn start_trace_on_call( + fn start_trace_on_call( &mut self, - depth: usize, + data: &EVMData<'_, DB>, address: Address, - data: Bytes, + input_data: Bytes, value: U256, kind: CallKind, caller: Address, - gas_limit: u64, + mut gas_limit: u64, maybe_precompile: Option, ) { // This will only be true if the inspector is configured to exclude precompiles and the call @@ -166,14 +166,20 @@ impl TracingInspector { PushTraceKind::PushAndAttachToParent }; + if self.trace_stack.is_empty() { + // this is the root call which should get the original gas limit of the transaction, + // because initialization costs are already subtracted from gas_limit + gas_limit = data.env.tx.gas_limit; + } + self.trace_stack.push(self.traces.push_trace( 0, push_kind, CallTrace { - depth, + depth: data.journaled_state.depth() as usize, address, kind, - data, + data: input_data, value, status: InstructionResult::Continue, caller, @@ -421,7 +427,7 @@ where self.config.exclude_precompile_calls.then(|| self.is_precompile_call(data, &to, value)); self.start_trace_on_call( - data.journaled_state.depth() as usize, + data, to, inputs.input.clone(), value, @@ -460,7 +466,7 @@ where let _ = data.journaled_state.load_account(inputs.caller, data.db); let nonce = data.journaled_state.account(inputs.caller).info.nonce; self.start_trace_on_call( - data.journaled_state.depth() as usize, + data, get_create_address(inputs, nonce), inputs.init_code.clone(), inputs.value, From bec46e8d2f09be04d0510369591f3468c32272db Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Wed, 12 Jul 2023 11:19:43 +0300 Subject: [PATCH 067/150] fix(provider): update checkpoints only for known stages (#3624) --- crates/storage/provider/src/providers/database/provider.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 2d494c40e26..597909db4ce 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1227,14 +1227,15 @@ impl<'this, TX: DbTxMut<'this>> StageCheckpointWriter for DatabaseProvider<'this ) -> Result<()> { // iterate over all existing stages in the table and update its progress. let mut cursor = self.tx.cursor_write::()?; - while let Some((stage_name, checkpoint)) = cursor.next()? { + for stage_id in StageId::ALL { + let (_, checkpoint) = cursor.seek_exact(stage_id.to_string())?.unwrap_or_default(); cursor.upsert( - stage_name, + stage_id.to_string(), StageCheckpoint { block_number, ..if drop_stage_checkpoint { Default::default() } else { checkpoint } }, - )? + )?; } Ok(()) From 656af38f4b9d537cbe9baa6c2dfb3a36d873c4a0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 12 Jul 2023 12:13:07 +0200 Subject: [PATCH 068/150] fix: register precompiles correctly (#3720) --- .../revm-inspectors/src/tracing/js/mod.rs | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/revm/revm-inspectors/src/tracing/js/mod.rs b/crates/revm/revm-inspectors/src/tracing/js/mod.rs index e9bc8f39004..b79b41f648f 100644 --- a/crates/revm/revm-inspectors/src/tracing/js/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/js/mod.rs @@ -16,6 +16,7 @@ use revm::{ interpreter::{ return_revert, CallInputs, CallScheme, CreateInputs, Gas, InstructionResult, Interpreter, }, + precompile::Precompiles, primitives::{Env, ExecutionResult, Output, ResultAndState, TransactTo, B160, B256}, Database, EVMData, Inspector, }; @@ -48,6 +49,8 @@ pub struct JsInspector { call_stack: Vec, /// sender half of a channel to communicate with the database service. to_db_service: mpsc::Sender, + /// Marker to track whether the precompiles have been registered. + precompiles_registered: bool, } impl JsInspector { @@ -130,6 +133,7 @@ impl JsInspector { step_fn, call_stack: Default::default(), to_db_service, + precompiles_registered: false, }) } @@ -264,26 +268,25 @@ impl JsInspector { fn pop_call(&mut self) { self.call_stack.pop(); } -} -impl Inspector for JsInspector -where - DB: Database, -{ - fn initialize_interp( - &mut self, - _interp: &mut Interpreter, - data: &mut EVMData<'_, DB>, - _is_static: bool, - ) -> InstructionResult { + /// Registers the precompiles in the JS context + fn register_precompiles(&mut self, precompiles: &Precompiles) { + if !self.precompiles_registered { + return + } let precompiles = - PrecompileList(data.precompiles.addresses().into_iter().map(Into::into).collect()); + PrecompileList(precompiles.addresses().into_iter().map(Into::into).collect()); let _ = precompiles.register_callable(&mut self.ctx); - InstructionResult::Continue + self.precompiles_registered = true } +} +impl Inspector for JsInspector +where + DB: Database, +{ fn step( &mut self, interp: &mut Interpreter, @@ -361,10 +364,12 @@ where fn call( &mut self, - _data: &mut EVMData<'_, DB>, + data: &mut EVMData<'_, DB>, inputs: &mut CallInputs, _is_static: bool, ) -> (InstructionResult, Gas, Bytes) { + self.register_precompiles(&data.precompiles); + // determine correct `from` and `to` based on the call scheme let (from, to) = match inputs.context.scheme { CallScheme::DelegateCall | CallScheme::CallCode => { @@ -425,6 +430,8 @@ where data: &mut EVMData<'_, DB>, inputs: &mut CreateInputs, ) -> (InstructionResult, Option, Gas, Bytes) { + self.register_precompiles(&data.precompiles); + let _ = data.journaled_state.load_account(inputs.caller, data.db); let nonce = data.journaled_state.account(inputs.caller).info.nonce; let address = get_create_address(inputs, nonce); From de573cb9e78936479213512469d539f61f9e256f Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 12 Jul 2023 13:03:08 +0100 Subject: [PATCH 069/150] feat(bin, engine, prune): spawn pruning task from the engine (#3566) --- Cargo.lock | 11 + Cargo.toml | 1 + bin/reth/Cargo.toml | 1 + bin/reth/src/node/mod.rs | 6 + crates/blockchain-tree/src/blockchain_tree.rs | 13 +- crates/blockchain-tree/src/config.rs | 2 +- crates/blockchain-tree/src/shareable.rs | 15 +- crates/consensus/beacon/Cargo.toml | 1 + crates/consensus/beacon/src/engine/error.rs | 7 + crates/consensus/beacon/src/engine/metrics.rs | 2 + crates/consensus/beacon/src/engine/mod.rs | 194 ++++++++++++++---- crates/consensus/beacon/src/engine/prune.rs | 146 +++++++++++++ crates/consensus/beacon/src/engine/sync.rs | 8 +- crates/interfaces/src/blockchain_tree/mod.rs | 12 +- crates/primitives/src/stage/checkpoints.rs | 2 +- crates/prune/Cargo.toml | 20 ++ crates/prune/src/error.rs | 4 + crates/prune/src/lib.rs | 5 + crates/prune/src/pruner.rs | 83 ++++++++ crates/storage/provider/src/providers/mod.rs | 11 +- 20 files changed, 494 insertions(+), 50 deletions(-) create mode 100644 crates/consensus/beacon/src/engine/prune.rs create mode 100644 crates/prune/Cargo.toml create mode 100644 crates/prune/src/error.rs create mode 100644 crates/prune/src/lib.rs create mode 100644 crates/prune/src/pruner.rs diff --git a/Cargo.lock b/Cargo.lock index 8bb5dd01053..48cec8dd50f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4956,6 +4956,7 @@ dependencies = [ "reth-payload-builder", "reth-primitives", "reth-provider", + "reth-prune", "reth-revm", "reth-revm-inspectors", "reth-rlp", @@ -5029,6 +5030,7 @@ dependencies = [ "reth-payload-builder", "reth-primitives", "reth-provider", + "reth-prune", "reth-rpc-types", "reth-stages", "reth-tasks", @@ -5554,6 +5556,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-prune" +version = "0.1.0-alpha.3" +dependencies = [ + "reth-primitives", + "thiserror", + "tracing", +] + [[package]] name = "reth-revm" version = "0.1.0-alpha.3" diff --git a/Cargo.toml b/Cargo.toml index c73f2f87aba..94e6ca26fa3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/net/downloaders", "crates/payload/basic", "crates/primitives", + "crates/prune", "crates/revm", "crates/revm/revm-primitives", "crates/revm/revm-inspectors", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index ff92e6bc444..a1539dc59d9 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -36,6 +36,7 @@ reth-payload-builder = { workspace = true } reth-basic-payload-builder = { path = "../../crates/payload/basic" } reth-discv4 = { path = "../../crates/net/discv4" } reth-metrics = { workspace = true } +reth-prune = { path = "../../crates/prune" } jemallocator = { version = "0.5.0", optional = true } jemalloc-ctl = { version = "0.5.0", optional = true } diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 13fd00d78e8..d2ed2825610 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -360,6 +360,11 @@ impl Command { None }; + let pruner = config.prune.map(|prune_config| { + info!(target: "reth::cli", "Pruner initialized"); + reth_prune::Pruner::new(prune_config.block_interval, tree_config.max_reorg_depth()) + }); + // Configure the consensus engine let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel( client, @@ -374,6 +379,7 @@ impl Command { MIN_BLOCKS_FOR_PIPELINE_RUN, consensus_engine_tx, consensus_engine_rx, + pruner, )?; info!(target: "reth::cli", "Consensus engine initialized"); diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 92c821064c1..43fe1be6fa5 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -750,12 +750,21 @@ impl BlockchainTree /// /// This finalizes `last_finalized_block` prior to reading the canonical hashes (using /// [`BlockchainTree::finalize_block`]). - pub fn restore_canonical_hashes( + pub fn restore_canonical_hashes_and_finalize( &mut self, last_finalized_block: BlockNumber, ) -> Result<(), Error> { self.finalize_block(last_finalized_block); + self.restore_canonical_hashes() + } + + /// Reads the last `N` canonical hashes from the database and updates the block indices of the + /// tree. + /// + /// `N` is the `max_reorg_depth` plus the number of block hashes needed to satisfy the + /// `BLOCKHASH` opcode in the EVM. + pub fn restore_canonical_hashes(&mut self) -> Result<(), Error> { let num_of_canonical_hashes = self.config.max_reorg_depth() + self.config.num_of_additional_canonical_block_hashes(); @@ -1578,7 +1587,7 @@ mod tests { .assert(&tree); // update canonical block to b2, this would make b2a be removed - assert_eq!(tree.restore_canonical_hashes(12), Ok(())); + assert_eq!(tree.restore_canonical_hashes_and_finalize(12), Ok(())); assert_eq!(tree.is_block_known(block2.num_hash()).unwrap(), Some(BlockStatus::Valid)); diff --git a/crates/blockchain-tree/src/config.rs b/crates/blockchain-tree/src/config.rs index 681c8f61b3d..3c56acc5657 100644 --- a/crates/blockchain-tree/src/config.rs +++ b/crates/blockchain-tree/src/config.rs @@ -1,7 +1,7 @@ //! Blockchain tree configuration /// The configuration for the blockchain tree. -#[derive(Clone, Debug)] +#[derive(Clone, Copy, Debug)] pub struct BlockchainTreeConfig { /// Number of blocks after the last finalized block that we are storing. /// diff --git a/crates/blockchain-tree/src/shareable.rs b/crates/blockchain-tree/src/shareable.rs index a63b18dd265..d99901dc8e0 100644 --- a/crates/blockchain-tree/src/shareable.rs +++ b/crates/blockchain-tree/src/shareable.rs @@ -66,10 +66,21 @@ impl BlockchainTreeEngine tree.update_chains_metrics(); } - fn restore_canonical_hashes(&self, last_finalized_block: BlockNumber) -> Result<(), Error> { + fn restore_canonical_hashes_and_finalize( + &self, + last_finalized_block: BlockNumber, + ) -> Result<(), Error> { trace!(target: "blockchain_tree", ?last_finalized_block, "Restoring canonical hashes for last finalized block"); let mut tree = self.tree.write(); - let res = tree.restore_canonical_hashes(last_finalized_block); + let res = tree.restore_canonical_hashes_and_finalize(last_finalized_block); + tree.update_chains_metrics(); + res + } + + fn restore_canonical_hashes(&self) -> Result<(), Error> { + trace!(target: "blockchain_tree", "Restoring canonical hashes"); + let mut tree = self.tree.write(); + let res = tree.restore_canonical_hashes(); tree.update_chains_metrics(); res } diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index fd5aac45c6b..239afd997b3 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -19,6 +19,7 @@ reth-rpc-types = { workspace = true } reth-tasks = { workspace = true } reth-payload-builder = { workspace = true } reth-metrics = { workspace = true } +reth-prune = { path = "../../prune" } # async tokio = { workspace = true, features = ["sync"] } diff --git a/crates/consensus/beacon/src/engine/error.rs b/crates/consensus/beacon/src/engine/error.rs index 355197576a5..b78b3828bdb 100644 --- a/crates/consensus/beacon/src/engine/error.rs +++ b/crates/consensus/beacon/src/engine/error.rs @@ -1,3 +1,4 @@ +use reth_prune::PrunerError; use reth_rpc_types::engine::ForkchoiceUpdateError; use reth_stages::PipelineError; @@ -16,6 +17,12 @@ pub enum BeaconConsensusEngineError { /// Pipeline error. #[error(transparent)] Pipeline(#[from] Box), + /// Pruner channel closed. + #[error("Pruner channel closed")] + PrunerChannelClosed, + /// Pruner error. + #[error(transparent)] + Pruner(#[from] PrunerError), /// Common error. Wrapper around [reth_interfaces::Error]. #[error(transparent)] Common(#[from] reth_interfaces::Error), diff --git a/crates/consensus/beacon/src/engine/metrics.rs b/crates/consensus/beacon/src/engine/metrics.rs index 14d68f4dd7a..04080e93be5 100644 --- a/crates/consensus/beacon/src/engine/metrics.rs +++ b/crates/consensus/beacon/src/engine/metrics.rs @@ -13,6 +13,8 @@ pub(crate) struct EngineMetrics { pub(crate) forkchoice_updated_messages: Counter, /// The total count of new payload messages received. pub(crate) new_payload_messages: Counter, + /// The number of times the pruner was run. + pub(crate) pruner_runs: Counter, } /// Metrics for the `EngineSyncController`. diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 10e53323291..9403597278f 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -3,6 +3,7 @@ use crate::{ forkchoice::{ForkchoiceStateHash, ForkchoiceStateTracker}, message::OnForkChoiceUpdated, metrics::EngineMetrics, + prune::{EnginePruneController, EnginePruneEvent}, }, sync::{EngineSyncController, EngineSyncEvent}, }; @@ -28,6 +29,7 @@ use reth_provider::{ BlockIdReader, BlockReader, BlockSource, CanonChainTracker, ProviderError, StageCheckpointReader, }; +use reth_prune::Pruner; use reth_rpc_types::engine::{ ExecutionPayload, PayloadAttributes, PayloadStatus, PayloadStatusEnum, PayloadValidationError, }; @@ -60,13 +62,15 @@ use invalid_headers::InvalidHeaderCache; mod event; pub use event::BeaconConsensusEngineEvent; -mod forkchoice; -mod metrics; -pub(crate) mod sync; mod handle; pub use handle::BeaconConsensusEngineHandle; +mod forkchoice; +mod metrics; +pub(crate) mod prune; +pub(crate) mod sync; + /// The maximum number of invalid headers that can be tracked by the engine. const MAX_INVALID_HEADERS: u32 = 512u32; @@ -187,6 +191,8 @@ where /// blocks using the pipeline. Otherwise, the engine, sync controller, and blockchain tree will /// be used to download and execute the missing blocks. pipeline_run_threshold: u64, + /// Controls pruning triggered by engine updates. + prune: Option, } impl BeaconConsensusEngine @@ -213,7 +219,8 @@ where payload_builder: PayloadBuilderHandle, target: Option, pipeline_run_threshold: u64, - ) -> Result<(Self, BeaconConsensusEngineHandle), reth_interfaces::Error> { + pruner: Option, + ) -> Result<(Self, BeaconConsensusEngineHandle), Error> { let (to_engine, rx) = mpsc::unbounded_channel(); Self::with_channel( client, @@ -228,6 +235,7 @@ where pipeline_run_threshold, to_engine, rx, + pruner, ) } @@ -257,15 +265,17 @@ where pipeline_run_threshold: u64, to_engine: UnboundedSender, rx: UnboundedReceiver, - ) -> Result<(Self, BeaconConsensusEngineHandle), reth_interfaces::Error> { + pruner: Option, + ) -> Result<(Self, BeaconConsensusEngineHandle), Error> { let handle = BeaconConsensusEngineHandle { to_engine }; let sync = EngineSyncController::new( pipeline, client, - task_spawner, + task_spawner.clone(), run_pipeline_continuously, max_block, ); + let prune = pruner.map(|pruner| EnginePruneController::new(pruner, task_spawner)); let mut this = Self { sync, blockchain, @@ -278,6 +288,7 @@ where invalid_headers: InvalidHeaderCache::new(MAX_INVALID_HEADERS), metrics: EngineMetrics::default(), pipeline_run_threshold, + prune, }; let maybe_pipeline_target = match target { @@ -304,7 +315,7 @@ where /// # Returns /// /// A target block hash if the pipeline is inconsistent, otherwise `None`. - fn check_pipeline_consistency(&self) -> Result, reth_interfaces::Error> { + fn check_pipeline_consistency(&self) -> Result, Error> { // If no target was provided, check if the stages are congruent - check if the // checkpoint of the last stage matches the checkpoint of the first. let first_stage_checkpoint = self @@ -532,7 +543,7 @@ where &mut self, state: ForkchoiceState, attrs: Option, - tx: oneshot::Sender>, + tx: oneshot::Sender>, ) -> bool { self.metrics.forkchoice_updated_messages.increment(1); self.blockchain.on_forkchoice_update_received(&state); @@ -583,7 +594,7 @@ where &mut self, state: ForkchoiceState, attrs: Option, - ) -> Result { + ) -> Result { trace!(target: "consensus::engine", ?state, "Received new forkchoice state update"); if state.head_block_hash.is_zero() { return Ok(OnForkChoiceUpdated::invalid_state()) @@ -602,6 +613,17 @@ where return Ok(OnForkChoiceUpdated::syncing()) } + if self.is_prune_active() { + // We can only process new forkchoice updates if the pruner is idle, since it requires + // exclusive access to the database + warn!( + target: "consensus::engine", + "Pruning is in progress, skipping forkchoice update. \ + This may affect the performance of your node as a validator." + ); + return Ok(OnForkChoiceUpdated::syncing()) + } + let status = match self.blockchain.make_canonical(&state.head_block_hash) { Ok(outcome) => { if !outcome.is_already_canonical() { @@ -654,7 +676,7 @@ where &self, head: SealedHeader, update: &ForkchoiceState, - ) -> Result<(), reth_interfaces::Error> { + ) -> Result<(), Error> { let mut head_block = Head { number: head.number, hash: head.hash, @@ -899,11 +921,14 @@ where return Ok(status) } - let res = if self.sync.is_pipeline_idle() { - // we can only insert new payloads if the pipeline is _not_ running, because it holds - // exclusive access to the database + let res = if self.sync.is_pipeline_idle() && self.is_prune_idle() { + // we can only insert new payloads if the pipeline and the pruner are _not_ running, + // because they hold exclusive access to the database self.try_insert_new_payload(block) } else { + if self.is_prune_active() { + warn!(target: "consensus::engine", "Pruning is in progress, buffering new payload."); + } self.try_buffer_payload(block) }; @@ -964,12 +989,12 @@ where Ok(block) } - /// When the pipeline is actively syncing the tree is unable to commit any additional blocks - /// since the pipeline holds exclusive access to the database. + /// When the pipeline or the pruner is active, the tree is unable to commit any additional + /// blocks since the pipeline holds exclusive access to the database. /// /// In this scenario we buffer the payload in the tree if the payload is valid, once the - /// pipeline finished syncing the tree is then able to also use the buffered payloads to commit - /// to a (newer) canonical chain. + /// pipeline or pruner is finished, the tree is then able to also use the buffered payloads to + /// commit to a (newer) canonical chain. /// /// This will return `SYNCING` if the block was buffered successfully, and an error if an error /// occurred while buffering the block. @@ -984,7 +1009,7 @@ where /// Attempts to insert a new payload into the tree. /// - /// Caution: This expects that the pipeline is idle. + /// Caution: This expects that the pipeline and the pruner are idle. #[instrument(level = "trace", skip_all, target = "consensus::engine", ret)] fn try_insert_new_payload( &mut self, @@ -1063,14 +1088,11 @@ where /// /// If the given block is missing from the database, this will return `false`. Otherwise, `true` /// is returned: the database contains the hash and the tree was updated. - fn update_tree_on_finished_pipeline( - &mut self, - block_hash: H256, - ) -> Result { + fn update_tree_on_finished_pipeline(&mut self, block_hash: H256) -> Result { let synced_to_finalized = match self.blockchain.block_number(block_hash)? { Some(number) => { // Attempt to restore the tree. - self.blockchain.restore_canonical_hashes(number)?; + self.blockchain.restore_canonical_hashes_and_finalize(number)?; true } None => false, @@ -1078,6 +1100,14 @@ where Ok(synced_to_finalized) } + /// Attempt to restore the tree. + /// + /// This is invoked after a pruner run to update the tree with the most recent canonical + /// hashes. + fn update_tree_on_finished_pruner(&mut self) -> Result<(), Error> { + self.blockchain.restore_canonical_hashes() + } + /// Invoked if we successfully downloaded a new block from the network. /// /// This will attempt to insert the block into the tree. @@ -1226,9 +1256,7 @@ where // it's part of the canonical chain: if it's the safe or the finalized block if matches!( err, - reth_interfaces::Error::Execution( - BlockExecutionError::BlockHashNotFoundInChain { .. } - ) + Error::Execution(BlockExecutionError::BlockHashNotFoundInChain { .. }) ) { // if the inserted block is the currently targeted `finalized` or `safe` // block, we will attempt to make them canonical, @@ -1250,9 +1278,9 @@ where /// This returns a result to indicate whether the engine future should resolve (fatal error). fn on_sync_event( &mut self, - ev: EngineSyncEvent, + event: EngineSyncEvent, ) -> Option> { - match ev { + match event { EngineSyncEvent::FetchedFullBlock(block) => { self.on_downloaded_block(block); } @@ -1416,6 +1444,55 @@ where None } + + /// Event handler for events emitted by the [EnginePruneController]. + /// + /// This returns a result to indicate whether the engine future should resolve (fatal error). + fn on_prune_event( + &mut self, + event: EnginePruneEvent, + ) -> Option> { + match event { + EnginePruneEvent::NotReady => {} + EnginePruneEvent::Started(tip_block_number) => { + trace!(target: "consensus::engine", %tip_block_number, "Pruner started"); + self.metrics.pruner_runs.increment(1); + } + EnginePruneEvent::TaskDropped => { + error!(target: "consensus::engine", "Failed to receive spawned pruner"); + return Some(Err(BeaconConsensusEngineError::PrunerChannelClosed)) + } + EnginePruneEvent::Finished { result } => { + trace!(target: "consensus::engine", ?result, "Pruner finished"); + match result { + Ok(_) => { + // Update the state and hashes of the blockchain tree if possible. + match self.update_tree_on_finished_pruner() { + Ok(()) => {} + Err(error) => { + error!(target: "consensus::engine", ?error, "Error restoring blockchain tree state"); + return Some(Err(error.into())) + } + }; + } + // Any pruner error at this point is fatal. + Err(error) => return Some(Err(error.into())), + }; + } + }; + + None + } + + /// Returns `true` if the prune controller's pruner is idle. + fn is_prune_idle(&self) -> bool { + self.prune.as_ref().map(|prune| prune.is_pruner_idle()).unwrap_or(true) + } + + /// Returns `true` if the prune controller's pruner is active. + fn is_prune_active(&self) -> bool { + !self.is_prune_idle() + } } /// On initialization, the consensus engine will poll the message receiver and return @@ -1446,6 +1523,7 @@ where // SyncController, hence they are polled first, and they're also time sensitive. loop { let mut engine_messages_pending = false; + let mut sync_pending = false; // handle next engine message match this.engine_message_rx.poll_next_unpin(cx) { @@ -1484,10 +1562,28 @@ where } } Poll::Pending => { - if engine_messages_pending { - // both the sync and the engine message receiver are pending - return Poll::Pending + // no more sync events to process + sync_pending = true; + } + } + + // check prune events if pipeline is idle AND (pruning is running and we need to + // prioritize checking its events OR no engine and sync messages are pending and we may + // start pruning) + if this.sync.is_pipeline_idle() && + (this.is_prune_active() || engine_messages_pending & sync_pending) + { + if let Some(ref mut prune) = this.prune { + match prune.poll(cx, this.blockchain.canonical_tip().number) { + Poll::Ready(prune_event) => { + if let Some(res) = this.on_prune_event(prune_event) { + return Poll::Ready(res) + } + } + Poll::Pending => return Poll::Pending, } + } else { + return Poll::Pending } } } @@ -1680,6 +1776,9 @@ mod tests { let shareable_db = ProviderFactory::new(db.clone(), self.chain_spec.clone()); let latest = self.chain_spec.genesis_header().seal_slow(); let blockchain_provider = BlockchainProvider::with_latest(shareable_db, tree, latest); + + let pruner = Pruner::new(5, 0); + let (mut engine, handle) = BeaconConsensusEngine::new( NoopFullBlockClient::default(), pipeline, @@ -1691,6 +1790,7 @@ mod tests { payload_builder, None, self.pipeline_run_threshold.unwrap_or(MIN_BLOCKS_FOR_PIPELINE_RUN), + Some(pruner), ) .expect("failed to create consensus engine"); @@ -1767,21 +1867,41 @@ mod tests { std::thread::sleep(Duration::from_millis(100)); assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); - // consensus engine is still idle + // consensus engine is still idle because no FCUs were received let _ = env.send_new_payload(SealedBlock::default().into()).await; assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); - // consensus engine receives a forkchoice state and triggers the pipeline + // consensus engine is still idle because pruning is running let _ = env .send_forkchoice_updated(ForkchoiceState { head_block_hash: H256::random(), ..Default::default() }) .await; - assert_matches!( - rx.await, - Ok(Err(BeaconConsensusEngineError::Pipeline(n))) if matches!(*n.as_ref(),PipelineError::Stage(StageError::ChannelClosed)) - ); + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); + + // consensus engine receives a forkchoice state and triggers the pipeline when pruning is + // finished + loop { + match rx.try_recv() { + Ok(result) => { + assert_matches!( + result, + Err(BeaconConsensusEngineError::Pipeline(n)) if matches!(*n.as_ref(), PipelineError::Stage(StageError::ChannelClosed)) + ); + break + } + Err(TryRecvError::Empty) => { + let _ = env + .send_forkchoice_updated(ForkchoiceState { + head_block_hash: H256::random(), + ..Default::default() + }) + .await; + } + Err(err) => panic!("receive error: {err}"), + } + } } // Test that the consensus engine runs the pipeline again if the tree cannot be restored. diff --git a/crates/consensus/beacon/src/engine/prune.rs b/crates/consensus/beacon/src/engine/prune.rs new file mode 100644 index 00000000000..855456f3543 --- /dev/null +++ b/crates/consensus/beacon/src/engine/prune.rs @@ -0,0 +1,146 @@ +//! Prune management for the engine implementation. + +use futures::FutureExt; +use reth_primitives::BlockNumber; +use reth_prune::{Pruner, PrunerError, PrunerWithResult}; +use reth_tasks::TaskSpawner; +use std::task::{ready, Context, Poll}; +use tokio::sync::oneshot; + +/// Manages pruning under the control of the engine. +/// +/// This type controls the [Pruner]. +pub(crate) struct EnginePruneController { + /// The current state of the pruner. + pruner_state: PrunerState, + /// The type that can spawn the pruner task. + pruner_task_spawner: Box, +} + +impl EnginePruneController { + /// Create a new instance + pub(crate) fn new(pruner: Pruner, pruner_task_spawner: Box) -> Self { + Self { pruner_state: PrunerState::Idle(Some(pruner)), pruner_task_spawner } + } + + /// Returns `true` if the pruner is idle. + pub(crate) fn is_pruner_idle(&self) -> bool { + self.pruner_state.is_idle() + } + + /// Advances the pruner state. + /// + /// This checks for the result in the channel, or returns pending if the pruner is idle. + fn poll_pruner(&mut self, cx: &mut Context<'_>) -> Poll { + let res = match self.pruner_state { + PrunerState::Idle(_) => return Poll::Pending, + PrunerState::Running(ref mut fut) => { + ready!(fut.poll_unpin(cx)) + } + }; + let ev = match res { + Ok((pruner, result)) => { + self.pruner_state = PrunerState::Idle(Some(pruner)); + EnginePruneEvent::Finished { result } + } + Err(_) => { + // failed to receive the pruner + EnginePruneEvent::TaskDropped + } + }; + Poll::Ready(ev) + } + + /// This will try to spawn the pruner if it is idle: + /// 1. Check if pruning is needed through [Pruner::is_pruning_needed]. + /// 2a. If pruning is needed, pass tip block number to the [Pruner::run] and spawn it in a + /// separate task. Set pruner state to [PrunerState::Running]. + /// 2b. If pruning is not needed, set pruner state back to [PrunerState::Idle]. + /// + /// If pruner is already running, do nothing. + fn try_spawn_pruner(&mut self, tip_block_number: BlockNumber) -> Option { + match &mut self.pruner_state { + PrunerState::Idle(pruner) => { + let mut pruner = pruner.take()?; + + // Check tip for pruning + if pruner.is_pruning_needed(tip_block_number) { + let (tx, rx) = oneshot::channel(); + self.pruner_task_spawner.spawn_critical_blocking( + "pruner task", + Box::pin(async move { + let result = pruner.run(tip_block_number); + let _ = tx.send((pruner, result)); + }), + ); + self.pruner_state = PrunerState::Running(rx); + + Some(EnginePruneEvent::Started(tip_block_number)) + } else { + self.pruner_state = PrunerState::Idle(Some(pruner)); + Some(EnginePruneEvent::NotReady) + } + } + PrunerState::Running(_) => None, + } + } + + /// Advances the prune process with the tip block number. + pub(crate) fn poll( + &mut self, + cx: &mut Context<'_>, + tip_block_number: BlockNumber, + ) -> Poll { + // Try to spawn a pruner + match self.try_spawn_pruner(tip_block_number) { + Some(EnginePruneEvent::NotReady) => return Poll::Pending, + Some(event) => return Poll::Ready(event), + None => (), + } + + // Poll pruner and check its status + self.poll_pruner(cx) + } +} + +/// The event type emitted by the [EnginePruneController]. +#[derive(Debug)] +pub(crate) enum EnginePruneEvent { + /// Pruner is not ready + NotReady, + /// Pruner started with tip block number + Started(BlockNumber), + /// Pruner finished + /// + /// If this is returned, the pruner is idle. + Finished { + /// Final result of the pruner run. + result: Result<(), PrunerError>, + }, + /// Pruner task was dropped after it was started, unable to receive it because channel + /// closed. This would indicate a panicked pruner task + TaskDropped, +} + +/// The possible pruner states within the sync controller. +/// +/// [PrunerState::Idle] means that the pruner is currently idle. +/// [PrunerState::Running] means that the pruner is currently running. +/// +/// NOTE: The differentiation between these two states is important, because when the pruner is +/// running, it acquires the write lock over the database. This means that we cannot forward to the +/// blockchain tree any messages that would result in database writes, since it would result in a +/// deadlock. +enum PrunerState { + /// Pruner is idle. + Idle(Option), + /// Pruner is running and waiting for a response + Running(oneshot::Receiver), +} + +impl PrunerState { + /// Returns `true` if the state matches idle. + fn is_idle(&self) -> bool { + matches!(self, PrunerState::Idle(_)) + } +} diff --git a/crates/consensus/beacon/src/engine/sync.rs b/crates/consensus/beacon/src/engine/sync.rs index b422583f2db..02da63d5aec 100644 --- a/crates/consensus/beacon/src/engine/sync.rs +++ b/crates/consensus/beacon/src/engine/sync.rs @@ -166,9 +166,9 @@ where return false } trace!( - target: "consensus::engine", + target: "consensus::engine::sync", ?hash, - "start downloading full block." + "Start downloading full block" ); let request = self.full_block_client.get_full_block(hash); self.inflight_full_block_requests.push(request); @@ -191,10 +191,10 @@ where self.max_block.map(|target| progress >= target).unwrap_or_default(); if has_reached_max_block { trace!( - target: "consensus::engine", + target: "consensus::engine::sync", ?progress, max_block = ?self.max_block, - "Consensus engine reached max block." + "Consensus engine reached max block" ); } has_reached_max_block diff --git a/crates/interfaces/src/blockchain_tree/mod.rs b/crates/interfaces/src/blockchain_tree/mod.rs index 63f628c49df..ce844ee0b37 100644 --- a/crates/interfaces/src/blockchain_tree/mod.rs +++ b/crates/interfaces/src/blockchain_tree/mod.rs @@ -62,7 +62,17 @@ pub trait BlockchainTreeEngine: BlockchainTreeViewer + Send + Sync { /// /// This finalizes `last_finalized_block` prior to reading the canonical hashes (using /// [`BlockchainTreeEngine::finalize_block`]). - fn restore_canonical_hashes(&self, last_finalized_block: BlockNumber) -> Result<(), Error>; + fn restore_canonical_hashes_and_finalize( + &self, + last_finalized_block: BlockNumber, + ) -> Result<(), Error>; + + /// Reads the last `N` canonical hashes from the database and updates the block indices of the + /// tree. + /// + /// `N` is the `max_reorg_depth` plus the number of block hashes needed to satisfy the + /// `BLOCKHASH` opcode in the EVM. + fn restore_canonical_hashes(&self) -> Result<(), Error>; /// Make a block and its parent chain part of the canonical chain by committing it to the /// database. diff --git a/crates/primitives/src/stage/checkpoints.rs b/crates/primitives/src/stage/checkpoints.rs index c6de28ff63d..4d7311dd06d 100644 --- a/crates/primitives/src/stage/checkpoints.rs +++ b/crates/primitives/src/stage/checkpoints.rs @@ -220,7 +220,7 @@ impl StageCheckpoint { /// Get the underlying [`EntitiesCheckpoint`], if any, to determine the number of entities /// processed, and the number of total entities to process. pub fn entities(&self) -> Option { - let Some(stage_checkpoint) = self.stage_checkpoint else { return None }; + let stage_checkpoint = self.stage_checkpoint?; match stage_checkpoint { StageUnitCheckpoint::Account(AccountHashingCheckpoint { diff --git a/crates/prune/Cargo.toml b/crates/prune/Cargo.toml new file mode 100644 index 00000000000..56b0c49a017 --- /dev/null +++ b/crates/prune/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "reth-prune" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = """ +Pruning implementation +""" + +[dependencies] +# reth +reth-primitives = { workspace = true } + +# misc +tracing = { workspace = true } +thiserror = { workspace = true } + diff --git a/crates/prune/src/error.rs b/crates/prune/src/error.rs new file mode 100644 index 00000000000..96f0a25b7d7 --- /dev/null +++ b/crates/prune/src/error.rs @@ -0,0 +1,4 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PrunerError {} diff --git a/crates/prune/src/lib.rs b/crates/prune/src/lib.rs new file mode 100644 index 00000000000..6d133030fcd --- /dev/null +++ b/crates/prune/src/lib.rs @@ -0,0 +1,5 @@ +mod error; +mod pruner; + +pub use error::PrunerError; +pub use pruner::{Pruner, PrunerResult, PrunerWithResult}; diff --git a/crates/prune/src/pruner.rs b/crates/prune/src/pruner.rs new file mode 100644 index 00000000000..9f3a60b86cc --- /dev/null +++ b/crates/prune/src/pruner.rs @@ -0,0 +1,83 @@ +//! Support for pruning. + +use crate::PrunerError; +use reth_primitives::BlockNumber; +use tracing::debug; + +/// Result of [Pruner::run] execution +pub type PrunerResult = Result<(), PrunerError>; + +/// The pipeline type itself with the result of [Pruner::run] +pub type PrunerWithResult = (Pruner, PrunerResult); + +/// Pruning routine. Main pruning logic happens in [Pruner::run]. +pub struct Pruner { + /// Minimum pruning interval measured in blocks. All prune parts are checked and, if needed, + /// pruned, when the chain advances by the specified number of blocks. + min_block_interval: u64, + /// Maximum prune depth. Used to determine the pruning target for parts that are needed during + /// the reorg, e.g. changesets. + #[allow(dead_code)] + max_prune_depth: u64, + /// Last pruned block number. Used in conjunction with `min_block_interval` to determine + /// when the pruning needs to be initiated. + last_pruned_block_number: Option, +} + +impl Pruner { + /// Creates a new [Pruner]. + pub fn new(min_block_interval: u64, max_prune_depth: u64) -> Self { + Self { min_block_interval, max_prune_depth, last_pruned_block_number: None } + } + + /// Run the pruner + pub fn run(&mut self, tip_block_number: BlockNumber) -> PrunerResult { + // Pruning logic + + self.last_pruned_block_number = Some(tip_block_number); + Ok(()) + } + + /// Returns `true` if the pruning is needed at the provided tip block number. + /// This determined by the check against minimum pruning interval and last pruned block number. + pub fn is_pruning_needed(&self, tip_block_number: BlockNumber) -> bool { + if self.last_pruned_block_number.map_or(true, |last_pruned_block_number| { + // Saturating subtraction is needed for the case when the chain was reverted, meaning + // current block number might be less than the previously pruned block number. If + // that's the case, no pruning is needed as outdated data is also reverted. + tip_block_number.saturating_sub(last_pruned_block_number) >= self.min_block_interval + }) { + debug!( + target: "pruner", + last_pruned_block_number = ?self.last_pruned_block_number, + %tip_block_number, + "Minimum pruning interval reached" + ); + true + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use crate::Pruner; + + #[test] + fn pruner_is_pruning_needed() { + let pruner = Pruner::new(5, 0); + + // No last pruned block number was set before + let first_block_number = 1; + assert!(pruner.is_pruning_needed(first_block_number)); + + // Delta is not less than min block interval + let second_block_number = first_block_number + pruner.min_block_interval; + assert!(pruner.is_pruning_needed(second_block_number)); + + // Delta is less than min block interval + let third_block_number = second_block_number; + assert!(pruner.is_pruning_needed(third_block_number)); + } +} diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 81c123323fa..a7c3ecbe4fe 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -594,8 +594,15 @@ where self.tree.finalize_block(finalized_block) } - fn restore_canonical_hashes(&self, last_finalized_block: BlockNumber) -> Result<()> { - self.tree.restore_canonical_hashes(last_finalized_block) + fn restore_canonical_hashes_and_finalize( + &self, + last_finalized_block: BlockNumber, + ) -> Result<()> { + self.tree.restore_canonical_hashes_and_finalize(last_finalized_block) + } + + fn restore_canonical_hashes(&self) -> Result<()> { + self.tree.restore_canonical_hashes() } fn make_canonical(&self, block_hash: &BlockHash) -> Result { From 7a5016356c0eda3b7aa559d4fc53209e7d08a3e6 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 12 Jul 2023 10:03:40 -0400 Subject: [PATCH 070/150] fix: use engine responses to progress autoseal mining task (#3727) Co-authored-by: Matthias Seitz --- crates/consensus/auto-seal/src/task.rs | 63 ++++++++++--------- crates/consensus/beacon/src/engine/message.rs | 2 +- crates/consensus/beacon/src/engine/mod.rs | 1 + 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/crates/consensus/auto-seal/src/task.rs b/crates/consensus/auto-seal/src/task.rs index 06f637d3890..5217661c4cd 100644 --- a/crates/consensus/auto-seal/src/task.rs +++ b/crates/consensus/auto-seal/src/task.rs @@ -1,12 +1,10 @@ use crate::{mode::MiningMode, Storage}; -use futures_util::{future::BoxFuture, FutureExt, StreamExt}; -use reth_beacon_consensus::BeaconEngineMessage; +use futures_util::{future::BoxFuture, FutureExt}; +use reth_beacon_consensus::{BeaconEngineMessage, ForkchoiceStatus}; use reth_interfaces::consensus::ForkchoiceState; use reth_primitives::{ constants::{EMPTY_RECEIPTS, EMPTY_TRANSACTIONS, ETHEREUM_BLOCK_GAS_LIMIT}, - proofs, - stage::StageId, - Block, BlockBody, ChainSpec, Header, IntoRecoveredTransaction, ReceiptWithBloom, + proofs, Block, BlockBody, ChainSpec, Header, IntoRecoveredTransaction, ReceiptWithBloom, SealedBlockWithSenders, EMPTY_OMMER_ROOT, U256, }; use reth_provider::{CanonChainTracker, CanonStateNotificationSender, Chain, StateProviderFactory}; @@ -26,7 +24,7 @@ use std::{ }; use tokio::sync::{mpsc::UnboundedSender, oneshot}; use tokio_stream::wrappers::UnboundedReceiverStream; -use tracing::{debug, trace, warn}; +use tracing::{debug, error, trace, warn}; /// A Future that listens for new ready transactions and puts new blocks into storage pub struct MiningTask { @@ -117,7 +115,7 @@ where let client = this.client.clone(); let chain_spec = Arc::clone(&this.chain_spec); let pool = this.pool.clone(); - let mut events = this.pipe_line_events.take(); + let events = this.pipe_line_events.take(); let canon_state_notification = this.canon_state_notification.clone(); // Create the mining future that creates a block, notifies the engine that drives @@ -226,29 +224,38 @@ where }; drop(storage); - // send the new update to the engine, this will trigger the pipeline to - // download the block, execute it and store it in the database. - let (tx, _rx) = oneshot::channel(); - let _ = to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { - state, - payload_attrs: None, - tx, - }); - debug!(target: "consensus::auto", ?state, "sent fork choice update"); - - // wait for the pipeline to finish - if let Some(events) = events.as_mut() { - debug!(target: "consensus::auto", "waiting for finish stage event..."); - // wait for the finish stage to - loop { - if let Some(PipelineEvent::Running { stage_id, .. }) = - events.next().await - { - if stage_id == StageId::Finish { - debug!(target: "consensus::auto", "received finish stage event"); - break + // TODO: make this a future + // await the fcu call rx for SYNCING, then wait for a VALID response + loop { + // send the new update to the engine, this will trigger the engine + // to download and execute the block we just inserted + let (tx, rx) = oneshot::channel(); + let _ = to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs: None, + tx, + }); + debug!(target: "consensus::auto", ?state, "Sent fork choice update"); + + match rx.await.unwrap() { + Ok(fcu_response) => { + match fcu_response.forkchoice_status() { + ForkchoiceStatus::Valid => break, + ForkchoiceStatus::Invalid => { + error!(target: "consensus::auto", ?fcu_response, "Forkchoice update returned invalid response"); + return None + } + ForkchoiceStatus::Syncing => { + debug!(target: "consensus::auto", ?fcu_response, "Forkchoice update returned SYNCING, waiting for VALID"); + // wait for the next fork choice update + continue + } } } + Err(err) => { + error!(target: "consensus::auto", ?err, "Autoseal fork choice update failed"); + return None + } } } diff --git a/crates/consensus/beacon/src/engine/message.rs b/crates/consensus/beacon/src/engine/message.rs index 7223cb621f7..2e5e542aab1 100644 --- a/crates/consensus/beacon/src/engine/message.rs +++ b/crates/consensus/beacon/src/engine/message.rs @@ -40,7 +40,7 @@ impl OnForkChoiceUpdated { } /// Returns the determined status of the received ForkchoiceState. - pub(crate) fn forkchoice_status(&self) -> ForkchoiceStatus { + pub fn forkchoice_status(&self) -> ForkchoiceStatus { self.forkchoice_status } diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 9403597278f..f1f38cb0fb8 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -67,6 +67,7 @@ mod handle; pub use handle::BeaconConsensusEngineHandle; mod forkchoice; +pub use forkchoice::ForkchoiceStatus; mod metrics; pub(crate) mod prune; pub(crate) mod sync; From 7f72ea66d776326f793f451f5d48002207855395 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 12 Jul 2023 10:12:33 -0400 Subject: [PATCH 071/150] fix: perform forkchoice update consistency checks (#3730) Co-authored-by: Matthias Seitz --- crates/blockchain-tree/src/shareable.rs | 5 + crates/consensus/beacon/src/engine/mod.rs | 112 ++++++++++++++++++- crates/interfaces/src/blockchain_tree/mod.rs | 3 + crates/storage/provider/src/providers/mod.rs | 4 + 4 files changed, 119 insertions(+), 5 deletions(-) diff --git a/crates/blockchain-tree/src/shareable.rs b/crates/blockchain-tree/src/shareable.rs index d99901dc8e0..ab191385ec9 100644 --- a/crates/blockchain-tree/src/shareable.rs +++ b/crates/blockchain-tree/src/shareable.rs @@ -158,6 +158,11 @@ impl BlockchainTreeViewer self.tree.read().block_indices().canonical_tip() } + fn is_canonical(&self, hash: BlockHash) -> Result { + trace!(target: "blockchain_tree", ?hash, "Checking if block is canonical"); + self.tree.read().is_block_hash_canonical(&hash) + } + fn pending_blocks(&self) -> (BlockNumber, Vec) { trace!(target: "blockchain_tree", "Returning all pending blocks"); self.tree.read().block_indices().pending_blocks() diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index f1f38cb0fb8..76a4abaa912 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -631,12 +631,20 @@ where debug!(target: "consensus::engine", hash=?state.head_block_hash, number=outcome.header().number, "canonicalized new head"); // new VALID update that moved the canonical chain forward - let _ = self.update_canon_chain(outcome.header().clone(), &state); + let _ = self.update_head(outcome.header().clone()); } else { debug!(target: "consensus::engine", fcu_head_num=?outcome.header().number, current_head_num=?self.blockchain.canonical_tip().number, "Ignoring beacon update to old head"); } if let Some(attrs) = attrs { + // if we return early then we wouldn't perform these consistency checks, so we + // need to do them here, and should do them before we process any payload + // attributes + if let Some(invalid_fcu_response) = self.ensure_consistent_state(state)? { + trace!(target: "consensus::engine", ?state, head=?state.head_block_hash, "Forkchoice state is inconsistent, returning invalid response"); + return Ok(invalid_fcu_response) + } + // the CL requested to build a new payload on top of this new VALID head let payload_response = self.process_payload_attributes( attrs, @@ -662,22 +670,119 @@ where } }; + if let Some(invalid_fcu_response) = + self.ensure_consistent_state_with_status(state, &status)? + { + trace!(target: "consensus::engine", ?status, ?state, "Forkchoice state is inconsistent, returning invalid response"); + return Ok(invalid_fcu_response) + } + trace!(target: "consensus::engine", ?status, ?state, "Returning forkchoice status"); Ok(OnForkChoiceUpdated::valid(status)) } + /// Ensures that the given forkchoice state is consistent, assuming the head block has been + /// made canonical. This takes a status as input, and will only perform consistency checks if + /// the input status is VALID. + /// + /// If the forkchoice state is consistent, this will return Ok(None). Otherwise, this will + /// return an instance of [OnForkChoiceUpdated] that is INVALID. + /// + /// This also updates the safe and finalized blocks in the [CanonChainTracker], if they are + /// consistent with the head block. + fn ensure_consistent_state_with_status( + &mut self, + state: ForkchoiceState, + status: &PayloadStatus, + ) -> Result, reth_interfaces::Error> { + // We only perform consistency checks if the status is VALID because if the status is + // INVALID, we want to return the correct _type_ of error to the CL so we can properly + // describe the reason it is invalid. For example, it's possible that the status is invalid + // because the safe block has an invalid state root. In that case, we want to preserve the + // correct `latestValidHash`, instead of returning a generic "invalid state" error that + // does not contain a `latestValidHash`. + // + // We also should not perform these checks if the status is SYNCING, because in that case + // we likely do not have the finalized or safe blocks, and would return an incorrect + // INVALID status instead. + if status.is_valid() { + return self.ensure_consistent_state(state) + } + + Ok(None) + } + + /// Ensures that the given forkchoice state is consistent, assuming the head block has been + /// made canonical. + /// + /// If the forkchoice state is consistent, this will return Ok(None). Otherwise, this will + /// return an instance of [OnForkChoiceUpdated] that is INVALID. + /// + /// This also updates the safe and finalized blocks in the [CanonChainTracker], if they are + /// consistent with the head block. + fn ensure_consistent_state( + &mut self, + state: ForkchoiceState, + ) -> Result, reth_interfaces::Error> { + // Ensure that the finalized block, if not zero, is known and in the canonical chain + // after the head block is canonicalized. + // + // This ensures that the finalized block is consistent with the head block, i.e. the + // finalized block is an ancestor of the head block. + if !state.finalized_block_hash.is_zero() && + !self.blockchain.is_canonical(state.finalized_block_hash)? + { + return Ok(Some(OnForkChoiceUpdated::invalid_state())) + } + + // Finalized block is consistent, so update it in the canon chain tracker. + self.update_finalized_block(state.finalized_block_hash)?; + + // Also ensure that the safe block, if not zero, is known and in the canonical chain + // after the head block is canonicalized. + // + // This ensures that the safe block is consistent with the head block, i.e. the safe + // block is an ancestor of the head block. + if !state.safe_block_hash.is_zero() && + !self.blockchain.is_canonical(state.safe_block_hash)? + { + return Ok(Some(OnForkChoiceUpdated::invalid_state())) + } + + // Safe block is consistent, so update it in the canon chain tracker. + self.update_safe_block(state.safe_block_hash)?; + + Ok(None) + } + /// Sets the state of the canon chain tracker based to the given head. /// /// This expects the given head to be the new canonical head. /// /// Additionally, updates the head used for p2p handshakes. /// - /// This should be called before issuing a VALID forkchoice update. + /// This also updates the tracked safe and finalized blocks, and should be called before + /// returning a VALID forkchoice update response fn update_canon_chain( &self, head: SealedHeader, update: &ForkchoiceState, ) -> Result<(), Error> { + self.update_head(head)?; + self.update_finalized_block(update.finalized_block_hash)?; + self.update_safe_block(update.safe_block_hash)?; + + Ok(()) + } + + /// Updates the state of the canon chain tracker based on the given head. + /// + /// This expects the given head to be the new canonical head. + /// Additionally, updates the head used for p2p handshakes. + /// + /// This should be called before returning a VALID forkchoice update response + #[inline] + fn update_head(&self, head: SealedHeader) -> Result<(), reth_interfaces::Error> { let mut head_block = Head { number: head.number, hash: head.hash, @@ -690,9 +795,6 @@ where // we update the the tracked header first self.blockchain.set_canonical_head(head); - self.update_finalized_block(update.finalized_block_hash)?; - self.update_safe_block(update.safe_block_hash)?; - head_block.total_difficulty = self.blockchain.header_td_by_number(head_block.number)?.ok_or_else(|| { Error::Provider(ProviderError::TotalDifficultyNotFound { diff --git a/crates/interfaces/src/blockchain_tree/mod.rs b/crates/interfaces/src/blockchain_tree/mod.rs index ce844ee0b37..8025c790112 100644 --- a/crates/interfaces/src/blockchain_tree/mod.rs +++ b/crates/interfaces/src/blockchain_tree/mod.rs @@ -217,6 +217,9 @@ pub trait BlockchainTreeViewer: Send + Sync { /// Note: this could be the given `parent_hash` if it's already canonical. fn find_canonical_ancestor(&self, parent_hash: BlockHash) -> Option; + /// Return whether or not the block is known and in the canonical chain. + fn is_canonical(&self, hash: BlockHash) -> Result; + /// Given the hash of a block, this checks the buffered blocks for the lowest ancestor in the /// buffer. /// diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index a7c3ecbe4fe..083f15f9887 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -655,6 +655,10 @@ where self.tree.canonical_tip() } + fn is_canonical(&self, hash: BlockHash) -> std::result::Result { + self.tree.is_canonical(hash) + } + fn pending_blocks(&self) -> (BlockNumber, Vec) { self.tree.pending_blocks() } From 8ded15f87227f90aee15e725ab5e5ba11d75c9fd Mon Sep 17 00:00:00 2001 From: Sabnock <24715302+Sabnock01@users.noreply.github.com> Date: Wed, 12 Jul 2023 12:39:08 -0500 Subject: [PATCH 072/150] fix(consensus): call `finalize_block` when finalized block changes. (#3731) --- crates/consensus/beacon/src/engine/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 76a4abaa912..63d0a014d04 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -846,6 +846,7 @@ where .ok_or_else(|| { Error::Provider(ProviderError::UnknownBlockHash(finalized_block_hash)) })?; + self.blockchain.finalize_block(finalized.number); self.blockchain.set_finalized(finalized.header.seal(finalized_block_hash)); } Ok(()) From e8fb67ef6d32b9f1b6100021880d4a00fed4dfa3 Mon Sep 17 00:00:00 2001 From: N Date: Wed, 12 Jul 2023 14:35:00 -0400 Subject: [PATCH 073/150] feat: complete vm and statediff tracers (#3529) Co-authored-by: N Co-authored-by: Matthias Seitz --- .../src/tracing/builder/mod.rs | 3 + .../src/tracing/builder/parity.rs | 173 ++++++++++++++++-- .../src/tracing/builder/walker.rs | 39 ++++ .../revm/revm-inspectors/src/tracing/mod.rs | 5 + .../revm/revm-inspectors/src/tracing/types.rs | 4 +- 5 files changed, 211 insertions(+), 13 deletions(-) create mode 100644 crates/revm/revm-inspectors/src/tracing/builder/walker.rs diff --git a/crates/revm/revm-inspectors/src/tracing/builder/mod.rs b/crates/revm/revm-inspectors/src/tracing/builder/mod.rs index 677ae88da5e..e6e58d8c2d9 100644 --- a/crates/revm/revm-inspectors/src/tracing/builder/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/builder/mod.rs @@ -5,3 +5,6 @@ pub mod geth; /// Parity style trace builders for `trace_` namespace pub mod parity; + +/// Walker types used for traversing various callgraphs +mod walker; diff --git a/crates/revm/revm-inspectors/src/tracing/builder/parity.rs b/crates/revm/revm-inspectors/src/tracing/builder/parity.rs index 3798d19f869..ba73f6cef0b 100644 --- a/crates/revm/revm-inspectors/src/tracing/builder/parity.rs +++ b/crates/revm/revm-inspectors/src/tracing/builder/parity.rs @@ -1,11 +1,16 @@ -use crate::tracing::{types::CallTraceNode, TracingInspectorConfig}; +use super::walker::CallTraceNodeWalkerBF; +use crate::tracing::{ + types::{CallTraceNode, CallTraceStep}, + TracingInspectorConfig, +}; use reth_primitives::{Address, U64}; use reth_rpc_types::{trace::parity::*, TransactionInfo}; use revm::{ db::DatabaseRef, - primitives::{AccountInfo, ExecutionResult, ResultAndState}, + interpreter::opcode, + primitives::{AccountInfo, ExecutionResult, ResultAndState, KECCAK_EMPTY}, }; -use std::collections::HashSet; +use std::collections::{HashSet, VecDeque}; /// A type for creating parity style traces /// @@ -14,6 +19,7 @@ use std::collections::HashSet; pub struct ParityTraceBuilder { /// Recorded trace nodes nodes: Vec, + /// How the traces were recorded _config: TracingInspectorConfig, } @@ -154,7 +160,18 @@ impl ParityTraceBuilder { DB: DatabaseRef, { let ResultAndState { result, state } = res; + + let breadth_first_addresses = if trace_types.contains(&TraceType::VmTrace) { + CallTraceNodeWalkerBF::new(&self.nodes) + .map(|node| node.trace.address) + .collect::>() + } else { + vec![] + }; + let mut trace_res = self.into_trace_results(result, trace_types); + + // check the state diff case if let Some(ref mut state_diff) = trace_res.state_diff { populate_account_balance_nonce_diffs( state_diff, @@ -162,6 +179,12 @@ impl ParityTraceBuilder { state.into_iter().map(|(addr, acc)| (addr, acc.info)), )?; } + + // check the vm trace case + if let Some(ref mut vm_trace) = trace_res.vm_trace { + populate_vm_trace_bytecodes(&db, vm_trace, breadth_first_addresses)?; + } + Ok(trace_res) } @@ -177,11 +200,8 @@ impl ParityTraceBuilder { let with_traces = trace_types.contains(&TraceType::Trace); let with_diff = trace_types.contains(&TraceType::StateDiff); - let vm_trace = if trace_types.contains(&TraceType::VmTrace) { - Some(vm_trace(&self.nodes)) - } else { - None - }; + let vm_trace = + if trace_types.contains(&TraceType::VmTrace) { Some(self.vm_trace()) } else { None }; let mut traces = Vec::with_capacity(if with_traces { self.nodes.len() } else { 0 }); let mut diff = StateDiff::default(); @@ -218,13 +238,142 @@ impl ParityTraceBuilder { pub fn into_transaction_traces(self) -> Vec { self.into_transaction_traces_iter().collect() } + + /// Creates a VM trace by walking over `CallTraceNode`s + /// + /// does not have the code fields filled in + pub fn vm_trace(&self) -> VmTrace { + match self.nodes.get(0) { + Some(current) => self.make_vm_trace(current), + None => VmTrace { code: Default::default(), ops: Vec::new() }, + } + } + + /// returns a VM trace without the code filled in + /// + /// iteratively creaters a VM trace by traversing an arena + fn make_vm_trace(&self, start: &CallTraceNode) -> VmTrace { + let mut child_idx_stack: Vec = Vec::with_capacity(self.nodes.len()); + let mut sub_stack: VecDeque> = VecDeque::with_capacity(self.nodes.len()); + + let mut current = start; + let mut child_idx: usize = 0; + + // finds the deepest nested calls of each call frame and fills them up bottom to top + let instructions = loop { + match current.children.get(child_idx) { + Some(child) => { + child_idx_stack.push(child_idx + 1); + + child_idx = 0; + current = self.nodes.get(*child).expect("there should be a child"); + } + None => { + let mut instructions: Vec = + Vec::with_capacity(current.trace.steps.len()); + + for step in ¤t.trace.steps { + let maybe_sub = match step.op.u8() { + opcode::CALL | + opcode::CALLCODE | + opcode::DELEGATECALL | + opcode::STATICCALL | + opcode::CREATE | + opcode::CREATE2 => { + sub_stack.pop_front().expect("there should be a sub trace") + } + _ => None, + }; + + instructions.push(Self::make_instruction(step, maybe_sub)); + } + + match current.parent { + Some(parent) => { + sub_stack.push_back(Some(VmTrace { + code: Default::default(), + ops: instructions, + })); + + child_idx = child_idx_stack.pop().expect("there should be a child idx"); + + current = self.nodes.get(parent).expect("there should be a parent"); + } + None => break instructions, + } + } + } + }; + + VmTrace { code: Default::default(), ops: instructions } + } + + /// Creates a VM instruction from a [CallTraceStep] and a [VmTrace] for the subcall if there is + /// one + fn make_instruction(step: &CallTraceStep, maybe_sub: Option) -> VmInstruction { + let maybe_storage = step.storage_change.map(|storage_change| StorageDelta { + key: storage_change.key, + val: storage_change.value, + }); + + let maybe_memory = match step.memory.len() { + 0 => None, + _ => { + Some(MemoryDelta { off: step.memory_size, data: step.memory.data().clone().into() }) + } + }; + + let maybe_execution = Some(VmExecutedOperation { + used: step.gas_cost, + push: step.new_stack.map(|new_stack| new_stack.into()), + mem: maybe_memory, + store: maybe_storage, + }); + + VmInstruction { + pc: step.pc, + cost: 0, // TODO: use op gas cost + ex: maybe_execution, + sub: maybe_sub, + } + } } -/// Construct the vmtrace for the entire callgraph -fn vm_trace(nodes: &[CallTraceNode]) -> VmTrace { - // TODO: populate vm trace +/// addresses are presorted via breadth first walk thru [CallTraceNode]s, this can be done by a +/// walker in [crate::tracing::builder::walker] +/// +/// iteratively fill the [VmTrace] code fields +pub(crate) fn populate_vm_trace_bytecodes( + db: &DB, + trace: &mut VmTrace, + breadth_first_addresses: I, +) -> Result<(), DB::Error> +where + DB: DatabaseRef, + I: IntoIterator, +{ + let mut stack: VecDeque<&mut VmTrace> = VecDeque::new(); + stack.push_back(trace); + + let mut addrs = breadth_first_addresses.into_iter(); + + while let Some(curr_ref) = stack.pop_front() { + for op in curr_ref.ops.iter_mut() { + if let Some(sub) = op.sub.as_mut() { + stack.push_back(sub); + } + } + + let addr = addrs.next().expect("there should be an address"); - VmTrace { code: nodes[0].trace.data.clone().into(), ops: vec![] } + let db_acc = db.basic(addr)?.unwrap_or_default(); + + let code_hash = if db_acc.code_hash != KECCAK_EMPTY { db_acc.code_hash } else { continue }; + + curr_ref.code = db.code_by_hash(code_hash)?.bytecode.into(); + } + + Ok(()) } /// Loops over all state accounts in the accounts diff that contains all accounts that are included diff --git a/crates/revm/revm-inspectors/src/tracing/builder/walker.rs b/crates/revm/revm-inspectors/src/tracing/builder/walker.rs new file mode 100644 index 00000000000..4d88a2af468 --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/builder/walker.rs @@ -0,0 +1,39 @@ +use crate::tracing::types::CallTraceNode; +use std::collections::VecDeque; + +/// Traverses Reths internal tracing structure breadth-first +/// +/// This is a lazy iterator +pub(crate) struct CallTraceNodeWalkerBF<'trace> { + /// the entire arena + nodes: &'trace Vec, + + /// holds indexes of nodes to visit as we traverse + queue: VecDeque, +} + +impl<'trace> CallTraceNodeWalkerBF<'trace> { + pub(crate) fn new(nodes: &'trace Vec) -> Self { + let mut queue = VecDeque::with_capacity(nodes.len()); + queue.push_back(0); + + Self { nodes, queue } + } +} + +impl<'trace> Iterator for CallTraceNodeWalkerBF<'trace> { + type Item = &'trace CallTraceNode; + + fn next(&mut self) -> Option { + match self.queue.pop_front() { + Some(idx) => { + let curr = self.nodes.get(idx).expect("there should be a node"); + + self.queue.extend(curr.children.iter()); + + Some(curr) + } + None => None, + } + } +} diff --git a/crates/revm/revm-inspectors/src/tracing/mod.rs b/crates/revm/revm-inspectors/src/tracing/mod.rs index bf3e746c90f..53a26f765a3 100644 --- a/crates/revm/revm-inspectors/src/tracing/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/mod.rs @@ -265,6 +265,7 @@ impl TracingInspector { op, contract: interp.contract.address, stack, + new_stack: None, memory, memory_size: interp.memory.len(), gas_remaining: self.gas_inspector.gas_remaining(), @@ -290,6 +291,10 @@ impl TracingInspector { self.step_stack.pop().expect("can't fill step without starting a step first"); let step = &mut self.traces.arena[trace_idx].trace.steps[step_idx]; + if interp.stack.len() > step.stack.len() { + step.new_stack = interp.stack.data().last().copied(); + } + if self.config.record_memory_snapshots { // resize memory so opcodes that allocated memory is correctly displayed if interp.memory.len() > step.memory.len() { diff --git a/crates/revm/revm-inspectors/src/tracing/types.rs b/crates/revm/revm-inspectors/src/tracing/types.rs index b66bd67261b..548ae72b878 100644 --- a/crates/revm/revm-inspectors/src/tracing/types.rs +++ b/crates/revm/revm-inspectors/src/tracing/types.rs @@ -463,11 +463,13 @@ pub(crate) struct CallTraceStep { pub(crate) contract: Address, /// Stack before step execution pub(crate) stack: Stack, + /// The new stack item placed by this step if any + pub(crate) new_stack: Option, /// All allocated memory in a step /// /// This will be empty if memory capture is disabled pub(crate) memory: Memory, - /// Size of memory + /// Size of memory at the beginning of the step pub(crate) memory_size: usize, /// Remaining gas before step execution pub(crate) gas_remaining: u64, From 045daf21fb932b468ee78b07a9c31abe245181dd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 12 Jul 2023 20:54:50 +0200 Subject: [PATCH 074/150] fix: serialize selfdestruct as suicide (#3736) --- crates/rpc/rpc-types/src/eth/trace/parity.rs | 64 ++++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/trace/parity.rs b/crates/rpc/rpc-types/src/eth/trace/parity.rs index 3c4b0bc57c7..1f9e73f5fed 100644 --- a/crates/rpc/rpc-types/src/eth/trace/parity.rs +++ b/crates/rpc/rpc-types/src/eth/trace/parity.rs @@ -99,11 +99,38 @@ impl DerefMut for StateDiff { pub enum Action { Call(CallAction), Create(CreateAction), + /// Parity style traces never renamed suicide to selfdestruct: + /// + /// For compatibility reasons, this is serialized as `suicide`: + #[serde(rename = "suicide", alias = "selfdestruct")] Selfdestruct(SelfdestructAction), Reward(RewardAction), } +impl Action { + /// Returns true if this is a call action + pub fn is_call(&self) -> bool { + matches!(self, Action::Call(_)) + } + + /// Returns true if this is a create action + pub fn is_create(&self) -> bool { + matches!(self, Action::Call(_)) + } + + /// Returns true if this is a selfdestruct action + pub fn is_selfdestruct(&self) -> bool { + matches!(self, Action::Selfdestruct(_)) + } + /// Returns true if this is a reward action + pub fn is_reward(&self) -> bool { + matches!(self, Action::Reward(_)) + } +} + /// An external action type. +/// +/// Used as enum identifier for [Action] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum ActionType { @@ -112,6 +139,7 @@ pub enum ActionType { /// Contract creation. Create, /// Contract suicide/selfdestruct. + #[serde(rename = "suicide", alias = "selfdestruct")] Selfdestruct, /// A block reward. Reward, @@ -318,8 +346,6 @@ mod tests { "to": "0x160f5f00288e9e1cc8655b327e081566e580a71d", "value": "0x244b" }, - "blockHash": "0xbca9ee244882bd00a19737a66f24002a4562a949c4d5ebd03c32e04111cff536", - "blockNumber": 17600209, "error": "Reverted", "result": { "gasUsed": "0x9daf", @@ -327,11 +353,41 @@ mod tests { }, "subtraces": 3, "traceAddress": [], - "transactionHash": "0x0e48a8d4419efaa2d3a9b8f625a1c559a4179fd19ddd10c02842965f3a7e7b63", - "transactionPosition": 0, "type": "call" }"#; let val = serde_json::from_str::(s).unwrap(); serde_json::to_value(val).unwrap(); } + + #[test] + fn test_selfdestruct_suicide() { + let input = r#"{ + "action": { + "address": "0x66e29f0b6b1b07071f2fde4345d512386cb66f5f", + "refundAddress": "0x66e29f0b6b1b07071f2fde4345d512386cb66f5f", + "balance": "0x244b" + }, + "error": "Reverted", + "result": { + "gasUsed": "0x9daf", + "output": "0x000000000000000000000000000000000000000000000000011c37937e080000" + }, + "subtraces": 3, + "traceAddress": [], + "type": "suicide" + }"#; + let val = serde_json::from_str::(input).unwrap(); + assert!(val.action.is_selfdestruct()); + + let json = serde_json::to_value(val.clone()).unwrap(); + let expect = serde_json::from_str::(input).unwrap(); + similar_asserts::assert_eq!(json, expect); + let s = serde_json::to_string(&val).unwrap(); + let json = serde_json::from_str::(&s).unwrap(); + similar_asserts::assert_eq!(json, expect); + + let input = input.replace("suicide", "selfdestruct"); + let val = serde_json::from_str::(&input).unwrap(); + assert!(val.action.is_selfdestruct()); + } } From de54c97d8a9f8ce006faeb32931d5bf371850844 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 12 Jul 2023 22:26:51 +0100 Subject: [PATCH 075/150] chore(storage): transactions -> receipts in `receipts_by_block` (#3744) --- .../storage/provider/src/providers/database/provider.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 597909db4ce..16f25d9792f 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1090,12 +1090,12 @@ impl<'this, TX: DbTx<'this>> ReceiptProvider for DatabaseProvider<'this, TX> { return if tx_range.is_empty() { Ok(Some(Vec::new())) } else { - let mut tx_cursor = self.tx.cursor_read::()?; - let transactions = tx_cursor + let mut receipts_cursor = self.tx.cursor_read::()?; + let receipts = receipts_cursor .walk_range(tx_range)? - .map(|result| result.map(|(_, tx)| tx)) + .map(|result| result.map(|(_, receipt)| receipt)) .collect::, _>>()?; - Ok(Some(transactions)) + Ok(Some(receipts)) } } } From 7f9845bfbd23330b536fe61a4282d0ace1c27e30 Mon Sep 17 00:00:00 2001 From: Max Wolff Date: Wed, 12 Jul 2023 15:51:43 -0700 Subject: [PATCH 076/150] #3667 Add Dial Success Metric (#3729) --- crates/net/network/src/metrics.rs | 8 ++++++++ crates/net/network/src/session/mod.rs | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/crates/net/network/src/metrics.rs b/crates/net/network/src/metrics.rs index d7969da80af..085b1f093b2 100644 --- a/crates/net/network/src/metrics.rs +++ b/crates/net/network/src/metrics.rs @@ -45,6 +45,14 @@ pub struct NetworkMetrics { pub(crate) total_dropped_eth_requests_at_full_capacity: Counter, } +/// Metrics for SessionManager +#[derive(Metrics)] +#[metrics(scope = "network")] +pub struct SesssionManagerMetrics { + /// Number of dials that resulted in a peer being added to the peerset + pub(crate) total_dial_successes: Counter, +} + /// Metrics for the TransactionsManager #[derive(Metrics)] #[metrics(scope = "network")] diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index 7dda07cdd9c..b0d628d48c4 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -1,6 +1,7 @@ //! Support for handling peer sessions. use crate::{ message::PeerMessage, + metrics::SesssionManagerMetrics, session::{ active::ActiveSession, config::SessionCounter, @@ -101,6 +102,8 @@ pub(crate) struct SessionManager { active_session_rx: ReceiverStream, /// Used to measure inbound & outbound bandwidth across all managed streams bandwidth_meter: BandwidthMeter, + /// Metrics for the session manager. + metrics: SesssionManagerMetrics, } // === impl SessionManager === @@ -137,6 +140,7 @@ impl SessionManager { active_session_tx: MeteredSender::new(active_session_tx, "network_active_session"), active_session_rx: ReceiverStream::new(active_session_rx), bandwidth_meter, + metrics: Default::default(), } } @@ -473,6 +477,10 @@ impl SessionManager { self.active_sessions.insert(peer_id, handle); self.counter.inc_active(&direction); + if direction.is_outgoing() { + self.metrics.total_dial_successes.increment(1); + } + Poll::Ready(SessionEvent::SessionEstablished { peer_id, remote_addr, @@ -695,6 +703,11 @@ impl Direction { pub(crate) fn is_incoming(&self) -> bool { matches!(self, Direction::Incoming) } + + /// Returns `true` if this an outgoing connection. + pub(crate) fn is_outgoing(&self) -> bool { + matches!(self, Direction::Outgoing(_)) + } } impl std::fmt::Display for Direction { From f49dc63bc43ed890a25f5e9e36eb8248bc8607ab Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Jul 2023 15:56:41 +0200 Subject: [PATCH 077/150] test: ignore another flaky geth test (#3757) --- crates/net/network/tests/it/connect.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/net/network/tests/it/connect.rs b/crates/net/network/tests/it/connect.rs index 5669a5acdc9..b6156ea7188 100644 --- a/crates/net/network/tests/it/connect.rs +++ b/crates/net/network/tests/it/connect.rs @@ -409,6 +409,7 @@ async fn test_incoming_connect_with_single_geth() { #[tokio::test(flavor = "multi_thread")] #[serial_test::serial] #[cfg_attr(not(feature = "geth-tests"), ignore)] +#[ignore] // TODO: Re-enable once we figure out why this test is flakey async fn test_outgoing_connect_with_single_geth() { reth_tracing::init_test_tracing(); tokio::time::timeout(GETH_TIMEOUT, async move { @@ -455,6 +456,7 @@ async fn test_outgoing_connect_with_single_geth() { #[tokio::test(flavor = "multi_thread")] #[serial_test::serial] #[cfg_attr(not(feature = "geth-tests"), ignore)] +#[ignore] // TODO: Re-enable once we figure out why this test is flakey async fn test_geth_disconnect() { reth_tracing::init_test_tracing(); tokio::time::timeout(GETH_TIMEOUT, async move { From c91170d7bcd9a24afcac3d596ef9c8df9874f5d6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Jul 2023 17:09:39 +0200 Subject: [PATCH 078/150] chore: reorder call action fields (#3758) --- crates/rpc/rpc-types/src/eth/trace/parity.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/trace/parity.rs b/crates/rpc/rpc-types/src/eth/trace/parity.rs index 1f9e73f5fed..7fbea1bbac8 100644 --- a/crates/rpc/rpc-types/src/eth/trace/parity.rs +++ b/crates/rpc/rpc-types/src/eth/trace/parity.rs @@ -168,16 +168,16 @@ pub enum CallType { pub struct CallAction { /// Address of the sending account. pub from: Address, - /// Address of the destination/target account. - pub to: Address, - /// Value transferred to the destination account. - pub value: U256, + /// The type of the call. + pub call_type: CallType, /// The gas available for executing the call. pub gas: U64, /// The input data provided to the call. pub input: Bytes, - /// The type of the call. - pub call_type: CallType, + /// Address of the destination/target account. + pub to: Address, + /// Value transferred to the destination account. + pub value: U256, } /// Represents a _create_ action, either a `CREATE` operation or a CREATE transaction. From 46aaf2c6e428dd4bb7af12ab449b85701cc07eff Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Jul 2023 17:26:02 +0200 Subject: [PATCH 079/150] chore: add network example (#3753) --- Cargo.lock | 1 + Cargo.toml | 1 + crates/net/network/src/config.rs | 21 +++++++++++++++- crates/net/network/src/state.rs | 3 +++ examples/Cargo.toml | 31 ++++++++++++++---------- examples/network.rs | 41 ++++++++++++++++++++++++++++++++ 6 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 examples/network.rs diff --git a/Cargo.lock b/Cargo.lock index 48cec8dd50f..0ee5b47fcf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2162,6 +2162,7 @@ dependencies = [ "reth-beacon-consensus", "reth-blockchain-tree", "reth-db", + "reth-network", "reth-network-api", "reth-primitives", "reth-provider", diff --git a/Cargo.toml b/Cargo.toml index 94e6ca26fa3..c6b79a2c216 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,6 +101,7 @@ reth-revm = { path = "./crates/revm" } reth-payload-builder = { path = "./crates/payload/builder" } reth-transaction-pool = { path = "./crates/transaction-pool" } reth-tasks = { path = "./crates/tasks" } +reth-network = { path = "./crates/net/network" } reth-network-api = { path = "./crates/net/network-api" } ## eth diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index dbfc570873a..9da1eed9345 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -11,7 +11,9 @@ use reth_discv4::{Discv4Config, Discv4ConfigBuilder, DEFAULT_DISCOVERY_PORT}; use reth_dns_discovery::DnsDiscoveryConfig; use reth_ecies::util::pk2id; use reth_eth_wire::{HelloMessage, Status}; -use reth_primitives::{ChainSpec, ForkFilter, Head, NodeRecord, PeerId, MAINNET}; +use reth_primitives::{ + mainnet_nodes, sepolia_nodes, ChainSpec, ForkFilter, Head, NodeRecord, PeerId, MAINNET, +}; use reth_provider::{BlockReader, HeaderProvider}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use secp256k1::SECP256K1; @@ -31,6 +33,9 @@ pub fn rng_secret_key() -> SecretKey { /// All network related initialization settings. pub struct NetworkConfig { /// The client type that can interact with the chain. + /// + /// This type is used to fetch the block number after we established a session and received the + /// [Status] block hash. pub client: C, /// The node's secret key, from which the node's identity is derived. pub secret_key: SecretKey, @@ -278,6 +283,16 @@ impl NetworkConfigBuilder { self } + /// Convenience function for setting [Self::boot_nodes] to the mainnet boot nodes. + pub fn mainnet_boot_nodes(self) -> Self { + self.boot_nodes(mainnet_nodes()) + } + + /// Convenience function for setting [Self::boot_nodes] to the sepolia boot nodes. + pub fn sepolia_boot_nodes(self) -> Self { + self.boot_nodes(sepolia_nodes()) + } + /// Sets the boot nodes. pub fn boot_nodes(mut self, nodes: impl IntoIterator) -> Self { self.boot_nodes = nodes.into_iter().collect(); @@ -330,6 +345,10 @@ impl NetworkConfigBuilder { /// Consumes the type and creates the actual [`NetworkConfig`] /// for the given client type that can interact with the chain. + /// + /// The given client is to be used for interacting with the chain, for example fetching the + /// corresponding block for a given block hash we receive from a peer in the status message when + /// establishing a connection. pub fn build(self, client: C) -> NetworkConfig { let peer_id = self.get_peer_id(); let Self { diff --git a/crates/net/network/src/state.rs b/crates/net/network/src/state.rs index 05f423a7384..104f263f4b7 100644 --- a/crates/net/network/src/state.rs +++ b/crates/net/network/src/state.rs @@ -51,6 +51,9 @@ pub struct NetworkState { /// Buffered messages until polled. queued_messages: VecDeque, /// The client type that can interact with the chain. + /// + /// This type is used to fetch the block number after we established a session and received the + /// [Status] block hash. client: C, /// Network discovery. discovery: Discovery, diff --git a/examples/Cargo.toml b/examples/Cargo.toml index c77a2d6879d..6114a573277 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -6,25 +6,26 @@ edition.workspace = true license.workspace = true [dev-dependencies] -reth-primitives = { workspace = true } +reth-primitives.workspace = true -reth-db = { workspace = true } -reth-provider = { workspace = true } +reth-db.workspace = true +reth-provider.workspace = true -reth-rpc-builder = { workspace = true } -reth-rpc-types = { workspace = true } +reth-rpc-builder.workspace = true +reth-rpc-types.workspace = true -reth-revm = { workspace = true } -reth-blockchain-tree = { workspace = true } -reth-beacon-consensus = { workspace = true } -reth-network-api = { workspace = true } -reth-transaction-pool = { workspace = true } -reth-tasks = { workspace = true } +reth-revm.workspace = true +reth-blockchain-tree.workspace = true +reth-beacon-consensus.workspace = true +reth-network-api.workspace = true +reth-network.workspace = true +reth-transaction-pool.workspace = true +reth-tasks.workspace = true eyre = "0.6.8" -futures = "0.3.0" -tokio = { workspace = true } +futures.workspace = true +tokio.workspace = true [[example]] name = "rpc-db" @@ -33,3 +34,7 @@ path = "rpc-db.rs" [[example]] name = "db-access" path = "db-access.rs" + +[[example]] +name = "network" +path = "network.rs" diff --git a/examples/network.rs b/examples/network.rs new file mode 100644 index 00000000000..09a20f3118e --- /dev/null +++ b/examples/network.rs @@ -0,0 +1,41 @@ +//! Example of how to use the network as a standalone component +//! +//! Run with +//! +//! ```not_rust +//! cargo run --example network +//! ``` + +use futures::StreamExt; +use reth_network::{config::rng_secret_key, NetworkConfig, NetworkManager}; +use reth_provider::test_utils::NoopProvider; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + // This block provider implementation is used for testing purposes. + let client = NoopProvider::default(); + + // The key that's used for encrypting sessions and to identify our node. + let local_key = rng_secret_key(); + + // Configure the network + let config = + NetworkConfig::::builder(local_key).mainnet_boot_nodes().build(client); + + // create the network instance + let network = NetworkManager::new(config).await?; + + // get a handle to the network to interact with it + let handle = network.handle().clone(); + + // spawn the network + tokio::task::spawn(network); + + // interact with the network + let mut events = handle.event_listener(); + while let Some(event) = events.next().await { + println!("Received event: {:?}", event); + } + + Ok(()) +} From 8ded51639c9b7aeaea2a80eeecc785e0ee39c2c4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Jul 2023 18:32:09 +0200 Subject: [PATCH 080/150] fix: poll logic when pipeline active (#3761) --- crates/consensus/beacon/src/engine/mod.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 63d0a014d04..e42bc3c7170 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1671,12 +1671,13 @@ where } } + // we're pending if both engine messages and sync events are pending (fully drained) + let is_pending = engine_messages_pending && sync_pending; + // check prune events if pipeline is idle AND (pruning is running and we need to // prioritize checking its events OR no engine and sync messages are pending and we may // start pruning) - if this.sync.is_pipeline_idle() && - (this.is_prune_active() || engine_messages_pending & sync_pending) - { + if this.sync.is_pipeline_idle() && (this.is_prune_active() || is_pending) { if let Some(ref mut prune) = this.prune { match prune.poll(cx, this.blockchain.canonical_tip().number) { Poll::Ready(prune_event) => { @@ -1684,12 +1685,16 @@ where return Poll::Ready(res) } } - Poll::Pending => return Poll::Pending, + Poll::Pending => {} } - } else { - return Poll::Pending } } + + if is_pending { + // incoming engine messages and sync events are drained, so we can yield back + // control + return Poll::Pending + } } } } From 985a6ca588be7c2e5a2cb8fb2150dcb923a6d892 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 13 Jul 2023 20:22:01 +0200 Subject: [PATCH 081/150] fix: return null withdrawals (#3762) --- crates/rpc/rpc-api/src/engine.rs | 6 +++--- crates/rpc/rpc-engine-api/src/engine_api.rs | 14 +++++++------ .../rpc/rpc-types/src/eth/engine/payload.rs | 20 +++++++++++-------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index be4825a5b6f..756cb5473c7 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -2,7 +2,7 @@ use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use reth_primitives::{Address, BlockHash, BlockId, BlockNumberOrTag, Bytes, H256, U256, U64}; use reth_rpc_types::{ engine::{ - ExecutionPayload, ExecutionPayloadBodies, ExecutionPayloadEnvelope, ForkchoiceState, + ExecutionPayload, ExecutionPayloadBodiesV1, ExecutionPayloadEnvelope, ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadId, PayloadStatus, TransitionConfiguration, }, state::StateOverride, @@ -64,7 +64,7 @@ pub trait EngineApi { async fn get_payload_bodies_by_hash_v1( &self, block_hashes: Vec, - ) -> RpcResult; + ) -> RpcResult; /// See also /// @@ -83,7 +83,7 @@ pub trait EngineApi { &self, start: U64, count: U64, - ) -> RpcResult; + ) -> RpcResult; /// See also #[method(name = "exchangeTransitionConfigurationV1")] diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index f461f4928dc..2a0581595f4 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -8,7 +8,7 @@ use reth_primitives::{BlockHash, BlockHashOrNumber, BlockNumber, ChainSpec, Hard use reth_provider::{BlockReader, EvmEnvProvider, HeaderProvider, StateProviderFactory}; use reth_rpc_api::EngineApiServer; use reth_rpc_types::engine::{ - ExecutionPayload, ExecutionPayloadBodies, ExecutionPayloadEnvelope, ForkchoiceUpdated, + ExecutionPayload, ExecutionPayloadBodiesV1, ExecutionPayloadEnvelope, ForkchoiceUpdated, PayloadAttributes, PayloadId, PayloadStatus, TransitionConfiguration, CAPABILITIES, }; use reth_tasks::TaskSpawner; @@ -183,7 +183,7 @@ where &self, start: BlockNumber, count: u64, - ) -> EngineApiResult { + ) -> EngineApiResult { let (tx, rx) = oneshot::channel(); let inner = self.inner.clone(); @@ -223,7 +223,7 @@ where pub fn get_payload_bodies_by_hash( &self, hashes: Vec, - ) -> EngineApiResult { + ) -> EngineApiResult { let len = hashes.len() as u64; if len > MAX_PAYLOAD_BODIES_LIMIT { return Err(EngineApiError::PayloadRequestTooLarge { len }) @@ -414,7 +414,7 @@ where async fn get_payload_bodies_by_hash_v1( &self, block_hashes: Vec, - ) -> RpcResult { + ) -> RpcResult { trace!(target: "rpc::engine", "Serving engine_getPayloadBodiesByHashV1"); Ok(EngineApi::get_payload_bodies_by_hash(self, block_hashes)?) } @@ -432,11 +432,13 @@ where /// Implementors should take care when acting on the input to this method, specifically /// ensuring that the range is limited properly, and that the range boundaries are computed /// correctly and without panics. + /// + /// Note: If a block is pre shanghai, `withdrawals` field will be `null async fn get_payload_bodies_by_range_v1( &self, start: U64, count: U64, - ) -> RpcResult { + ) -> RpcResult { trace!(target: "rpc::engine", "Serving engine_getPayloadBodiesByRangeV1"); Ok(EngineApi::get_payload_bodies_by_range(self, start.as_u64(), count.as_u64()).await?) } @@ -482,7 +484,7 @@ mod tests { let provider = Arc::new(MockEthProvider::default()); let payload_store = spawn_test_payload_service(); let (to_engine, engine_rx) = unbounded_channel(); - let task_executor = Box::new(TokioTaskExecutor::default()); + let task_executor = Box::::default(); let api = EngineApi::new( provider.clone(), chain_spec.clone(), diff --git a/crates/rpc/rpc-types/src/eth/engine/payload.rs b/crates/rpc/rpc-types/src/eth/engine/payload.rs index ecc12334122..3a0fcc4b9ec 100644 --- a/crates/rpc/rpc-types/src/eth/engine/payload.rs +++ b/crates/rpc/rpc-types/src/eth/engine/payload.rs @@ -8,7 +8,7 @@ use reth_rlp::Decodable; use serde::{ser::SerializeMap, Deserialize, Serialize, Serializer}; /// The execution payload body response that allows for `null` values. -pub type ExecutionPayloadBodies = Vec>; +pub type ExecutionPayloadBodiesV1 = Vec>; /// And 8-byte identifier for an execution payload. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] @@ -220,21 +220,25 @@ impl PayloadError { /// /// See also: #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ExecutionPayloadBody { +pub struct ExecutionPayloadBodyV1 { + /// Enveloped encoded transactions. pub transactions: Vec, - pub withdrawals: Vec, + /// All withdrawals in the block. + /// + /// Will always be `None` if pre shanghai. + pub withdrawals: Option>, } -impl From for ExecutionPayloadBody { +impl From for ExecutionPayloadBodyV1 { fn from(value: Block) -> Self { let transactions = value.body.into_iter().map(|tx| { let mut out = Vec::new(); tx.encode_enveloped(&mut out); out.into() }); - ExecutionPayloadBody { + ExecutionPayloadBodyV1 { transactions: transactions.collect(), - withdrawals: value.withdrawals.unwrap_or_default(), + withdrawals: value.withdrawals, } } } @@ -472,7 +476,7 @@ mod tests { let mut rng = generators::rng(); for block in random_block_range(&mut rng, 0..=99, H256::default(), 0..2) { let unsealed = block.clone().unseal(); - let payload_body: ExecutionPayloadBody = unsealed.into(); + let payload_body: ExecutionPayloadBodyV1 = unsealed.into(); assert_eq!( Ok(block.body), @@ -483,7 +487,7 @@ mod tests { .collect::, _>>(), ); - assert_eq!(block.withdrawals.unwrap_or_default(), payload_body.withdrawals); + assert_eq!(block.withdrawals, payload_body.withdrawals); } } From 1003fe0ad6b2d9380228e397f29113710d7c7d8d Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 13 Jul 2023 14:46:56 -0400 Subject: [PATCH 082/150] chore: fix typo in RlpDecodableWrapper derive (#3763) --- crates/rlp/rlp-derive/src/de.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rlp/rlp-derive/src/de.rs b/crates/rlp/rlp-derive/src/de.rs index 8ab670cafa7..aab545fd86d 100644 --- a/crates/rlp/rlp-derive/src/de.rs +++ b/crates/rlp/rlp-derive/src/de.rs @@ -72,12 +72,12 @@ pub(crate) fn impl_decodable(ast: &syn::DeriveInput) -> Result { } pub(crate) fn impl_decodable_wrapper(ast: &syn::DeriveInput) -> Result { - let body = parse_struct(ast, "RlpEncodableWrapper")?; + let body = parse_struct(ast, "RlpDecodableWrapper")?; assert_eq!( body.fields.iter().count(), 1, - "#[derive(RlpEncodableWrapper)] is only defined for structs with one field." + "#[derive(RlpDecodableWrapper)] is only defined for structs with one field." ); let name = &ast.ident; From f9dd37123ed1705d9529fa5aa5f6ae5a5739fdfd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Jul 2023 02:09:26 +0200 Subject: [PATCH 083/150] chore: add some txs helpers (#3767) --- crates/primitives/src/transaction/mod.rs | 25 ++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 4622e80f24f..6bfc0489f40 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -879,6 +879,8 @@ impl Decodable for TransactionKind { } /// Signed transaction without its Hash. Used type for inserting into the DB. +/// +/// This can by converted to [`TransactionSigned`] by calling [`TransactionSignedNoHash::hash`]. #[derive_arbitrary(compact)] #[derive(Debug, Clone, PartialEq, Eq, Hash, AsRef, Deref, Default, Serialize, Deserialize)] pub struct TransactionSignedNoHash { @@ -899,6 +901,14 @@ impl TransactionSignedNoHash { keccak256(&buf) } + /// Recover signer from signature and hash. + /// + /// Returns `None` if the transaction's signature is invalid, see also [Self::recover_signer]. + pub fn recover_signer(&self) -> Option

{ + let signature_hash = self.signature_hash(); + self.signature.recover_signer(signature_hash) + } + /// Converts into a transaction type with its hash: [`TransactionSigned`]. pub fn with_hash(self) -> TransactionSigned { self.into() @@ -1041,7 +1051,7 @@ impl TransactionSigned { self.signature.recover_signer(signature_hash) } - /// Devour Self, recover signer and return [`TransactionSignedEcRecovered`] + /// Consumes the type, recover signer and return [`TransactionSignedEcRecovered`] /// /// Returns `None` if the transaction's signature is invalid, see also [Self::recover_signer]. pub fn into_ecrecovered(self) -> Option { @@ -1049,12 +1059,23 @@ impl TransactionSigned { Some(TransactionSignedEcRecovered { signed_transaction: self, signer }) } - /// try to recover signer and return [`TransactionSignedEcRecovered`] + /// Tries to recover signer and return [`TransactionSignedEcRecovered`] by cloning the type. pub fn try_ecrecovered(&self) -> Option { let signer = self.recover_signer()?; Some(TransactionSignedEcRecovered { signed_transaction: self.clone(), signer }) } + /// Tries to recover signer and return [`TransactionSignedEcRecovered`]. + /// + /// Returns `Err(Self)` if the transaction's signature is invalid, see also + /// [Self::recover_signer]. + pub fn try_into_ecrecovered(self) -> Result { + match self.recover_signer() { + None => Err(self), + Some(signer) => Ok(TransactionSignedEcRecovered { signed_transaction: self, signer }), + } + } + /// Returns the enveloped encoded transactions. /// /// See also [TransactionSigned::encode_enveloped] From d30bf17f10a4d01a948007c112485306e910d25d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Jul 2023 09:59:57 +0200 Subject: [PATCH 084/150] fix(rpc): serialize traces always as vec (#3770) --- .../revm/revm-inspectors/src/tracing/builder/parity.rs | 7 ++++++- crates/rpc/rpc-types/src/eth/trace/parity.rs | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/revm/revm-inspectors/src/tracing/builder/parity.rs b/crates/revm/revm-inspectors/src/tracing/builder/parity.rs index ba73f6cef0b..cce86c43445 100644 --- a/crates/revm/revm-inspectors/src/tracing/builder/parity.rs +++ b/crates/revm/revm-inspectors/src/tracing/builder/parity.rs @@ -138,7 +138,12 @@ impl ParityTraceBuilder { let (trace, vm_trace, state_diff) = self.into_trace_type_traces(trace_types); - TraceResults { output: output.into(), trace, vm_trace, state_diff } + TraceResults { + output: output.into(), + trace: trace.unwrap_or_default(), + vm_trace, + state_diff, + } } /// Consumes the inspector and returns the trace results according to the configured trace diff --git a/crates/rpc/rpc-types/src/eth/trace/parity.rs b/crates/rpc/rpc-types/src/eth/trace/parity.rs index 7fbea1bbac8..fcc5e54bbbb 100644 --- a/crates/rpc/rpc-types/src/eth/trace/parity.rs +++ b/crates/rpc/rpc-types/src/eth/trace/parity.rs @@ -30,12 +30,13 @@ pub enum TraceType { pub struct TraceResults { /// Output of the trace pub output: Bytes, - /// Enabled if [TraceType::Trace] is provided - pub trace: Option>, - /// Enabled if [TraceType::VmTrace] is provided - pub vm_trace: Option, /// Enabled if [TraceType::StateDiff] is provided pub state_diff: Option, + /// Enabled if [TraceType::Trace] is provided, otherwise an empty vec + #[serde(default)] + pub trace: Vec, + /// Enabled if [TraceType::VmTrace] is provided + pub vm_trace: Option, } /// A `FullTrace` with an additional transaction hash From 89074bc68b6080acf31cd5d59a3981a6fbe49671 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Jul 2023 10:13:56 +0200 Subject: [PATCH 085/150] fix(rpc): make trace filter req field hex or decimal (#3772) --- crates/primitives/src/serde_helper/num.rs | 27 ++++++++++++++++++++ crates/rpc/rpc-types/src/eth/trace/filter.rs | 25 ++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/crates/primitives/src/serde_helper/num.rs b/crates/primitives/src/serde_helper/num.rs index 312f287124b..82017498513 100644 --- a/crates/primitives/src/serde_helper/num.rs +++ b/crates/primitives/src/serde_helper/num.rs @@ -88,6 +88,33 @@ pub mod u64_hex_or_decimal { U64HexOrNumber::from(*value).serialize(s) } } + +/// serde functions for handling primitive optional `u64` as [U64](crate::U64) +pub mod u64_hex_or_decimal_opt { + use crate::serde_helper::num::U64HexOrNumber; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + /// Deserializes an `u64` accepting a hex quantity string with optional 0x prefix or + /// a number + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + match Option::::deserialize(deserializer)? { + Some(val) => Ok(Some(val.into())), + None => Ok(None), + } + } + + /// Serializes u64 as hex string + pub fn serialize(value: &Option, s: S) -> Result { + match value { + Some(val) => U64HexOrNumber::from(*val).serialize(s), + None => s.serialize_none(), + } + } +} + /// Deserializes the input into an `Option`, using [`from_int_or_hex`] to deserialize the /// inner value. pub fn from_int_or_hex_opt<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/crates/rpc/rpc-types/src/eth/trace/filter.rs b/crates/rpc/rpc-types/src/eth/trace/filter.rs index bde828cab5a..3121335daeb 100644 --- a/crates/rpc/rpc-types/src/eth/trace/filter.rs +++ b/crates/rpc/rpc-types/src/eth/trace/filter.rs @@ -1,5 +1,5 @@ //! `trace_filter` types and support -use reth_primitives::{Address, BlockNumber}; +use reth_primitives::{serde_helper::num::u64_hex_or_decimal_opt, Address}; use serde::{Deserialize, Serialize}; /// Trace filter. @@ -8,15 +8,30 @@ use serde::{Deserialize, Serialize}; #[serde(rename_all = "camelCase")] pub struct TraceFilter { /// From block - pub from_block: Option, + #[serde(with = "u64_hex_or_decimal_opt")] + pub from_block: Option, /// To block - pub to_block: Option, + #[serde(with = "u64_hex_or_decimal_opt")] + pub to_block: Option, /// From address pub from_address: Option>, /// To address pub to_address: Option>, /// Output offset - pub after: Option, + pub after: Option, /// Output amount - pub count: Option, + pub count: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_filter() { + let s = r#"{"fromBlock": "0x3","toBlock": "0x5"}"#; + let filter: TraceFilter = serde_json::from_str(s).unwrap(); + assert_eq!(filter.from_block, Some(3)); + assert_eq!(filter.to_block, Some(5)); + } } From 892ec6c98b570a707ee1679e3da9228b84521fef Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Jul 2023 11:06:23 +0200 Subject: [PATCH 086/150] perf: only lookup in db in cache layer (#3773) --- crates/rpc/rpc/src/eth/cache/mod.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc/src/eth/cache/mod.rs b/crates/rpc/rpc/src/eth/cache/mod.rs index f5e758f86b7..8209a77ec30 100644 --- a/crates/rpc/rpc/src/eth/cache/mod.rs +++ b/crates/rpc/rpc/src/eth/cache/mod.rs @@ -3,7 +3,9 @@ use futures::{future::Either, Stream, StreamExt}; use reth_interfaces::{provider::ProviderError, Result}; use reth_primitives::{Block, Receipt, SealedBlock, TransactionSigned, H256}; -use reth_provider::{BlockReader, CanonStateNotification, EvmEnvProvider, StateProviderFactory}; +use reth_provider::{ + BlockReader, BlockSource, CanonStateNotification, EvmEnvProvider, StateProviderFactory, +}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use revm::primitives::{BlockEnv, CfgEnv}; use schnellru::{ByLength, Limiter}; @@ -307,7 +309,10 @@ where let provider = this.provider.clone(); let action_tx = this.action_tx.clone(); this.action_task_spawner.spawn_blocking(Box::pin(async move { - let res = provider.block_by_hash(block_hash); + // Only look in the database to prevent situations where we + // looking up the tree is blocking + let res = provider + .find_block_by_hash(block_hash, BlockSource::Database); let _ = action_tx .send(CacheAction::BlockResult { block_hash, res }); })); @@ -325,7 +330,10 @@ where let provider = this.provider.clone(); let action_tx = this.action_tx.clone(); this.action_task_spawner.spawn_blocking(Box::pin(async move { - let res = provider.block_by_hash(block_hash); + // Only look in the database to prevent situations where we + // looking up the tree is blocking + let res = provider + .find_block_by_hash(block_hash, BlockSource::Database); let _ = action_tx .send(CacheAction::BlockResult { block_hash, res }); })); From 91549da7c715ef4906102d5c6380accd4398a552 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Jul 2023 13:38:24 +0200 Subject: [PATCH 087/150] fix: add missing null check (#3766) --- crates/rpc/rpc-types/src/eth/trace/geth/mod.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/rpc/rpc-types/src/eth/trace/geth/mod.rs b/crates/rpc/rpc-types/src/eth/trace/geth/mod.rs index 092efa43855..eb9a33acd28 100644 --- a/crates/rpc/rpc-types/src/eth/trace/geth/mod.rs +++ b/crates/rpc/rpc-types/src/eth/trace/geth/mod.rs @@ -209,6 +209,9 @@ impl GethDebugTracerConfig { /// Returns the [CallConfig] if it is a call config. pub fn into_call_config(self) -> Result { + if self.0.is_null() { + return Ok(Default::default()) + } self.from_value() } @@ -219,6 +222,9 @@ impl GethDebugTracerConfig { /// Returns the [PreStateConfig] if it is a call config. pub fn into_pre_state_config(self) -> Result { + if self.0.is_null() { + return Ok(Default::default()) + } self.from_value() } } @@ -370,6 +376,18 @@ fn serialize_string_storage_map_opt( mod tests { use super::*; + #[test] + fn test_tracer_config() { + let s = "{\"tracer\": \"callTracer\"}"; + let opts = serde_json::from_str::(s).unwrap(); + assert_eq!( + opts.tracer, + Some(GethDebugTracerType::BuiltInTracer(GethDebugBuiltInTracerType::CallTracer)) + ); + let _call_config = opts.tracer_config.clone().into_call_config().unwrap(); + let _prestate_config = opts.tracer_config.into_pre_state_config().unwrap(); + } + #[test] fn test_memory_capture() { let mut config = GethDefaultTracingOptions::default(); From 70fca620d3548f2ac7a6a8ac5579fc5c50d9b56a Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Fri, 14 Jul 2023 18:14:33 +0530 Subject: [PATCH 088/150] adding row for total db size in db stats command (#3777) Co-authored-by: Matthias Seitz --- bin/reth/src/db/mod.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bin/reth/src/db/mod.rs b/bin/reth/src/db/mod.rs index ee2e0d12ab5..4670683c43d 100644 --- a/bin/reth/src/db/mod.rs +++ b/bin/reth/src/db/mod.rs @@ -104,6 +104,7 @@ impl Command { let mut tables = Tables::ALL.iter().map(|table| table.name()).collect::>(); tables.sort(); + let mut total_size = 0; for table in tables { let table_db = tx.inner.open_db(Some(table)).wrap_err("Could not open db.")?; @@ -123,6 +124,7 @@ impl Command { let num_pages = leaf_pages + branch_pages + overflow_pages; let table_size = page_size * num_pages; + total_size += table_size; let mut row = Row::new(); row.add_cell(Cell::new(table)) .add_cell(Cell::new(stats.entries())) @@ -132,6 +134,24 @@ impl Command { .add_cell(Cell::new(human_bytes(table_size as f64))); stats_table.add_row(row); } + + let max_widths = stats_table.column_max_content_widths(); + + let mut seperator = Row::new(); + for width in max_widths { + seperator.add_cell(Cell::new("-".repeat(width as usize))); + } + stats_table.add_row(seperator); + + let mut row = Row::new(); + row.add_cell(Cell::new("Total DB size")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new(human_bytes(total_size as f64))); + stats_table.add_row(row); + Ok::<(), eyre::Report>(()) })??; From 60f92f062c9d9eeb853b0154dd380f37958d33f2 Mon Sep 17 00:00:00 2001 From: suneal Date: Fri, 14 Jul 2023 22:23:37 +0800 Subject: [PATCH 089/150] fix: add unknown block error (#3779) Co-authored-by: suneal --- crates/rpc/rpc-types/src/eth/error.rs | 4 ++++ crates/rpc/rpc/src/eth/error.rs | 12 ++++++++++-- crates/storage/provider/src/traits/block_id.rs | 12 +++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/error.rs b/crates/rpc/rpc-types/src/eth/error.rs index c20e61f0c40..f580a4dbe8d 100644 --- a/crates/rpc/rpc-types/src/eth/error.rs +++ b/crates/rpc/rpc-types/src/eth/error.rs @@ -13,6 +13,9 @@ pub enum EthRpcErrorCode { /// > If the block is not found, the callee SHOULD raise a JSON-RPC error (the recommended /// > error code is -32001: Resource not found). ResourceNotFound, + /// Thrown when querying for `finalized` or `safe` block before the merge transition is + /// finalized, + UnknownBlock, } impl EthRpcErrorCode { @@ -23,6 +26,7 @@ impl EthRpcErrorCode { EthRpcErrorCode::ExecutionError => 3, EthRpcErrorCode::InvalidInput => -32000, EthRpcErrorCode::ResourceNotFound => -32001, + EthRpcErrorCode::UnknownBlock => -39001, } } } diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 1818f4aea87..023e7674061 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -26,6 +26,10 @@ pub enum EthApiError { PoolError(RpcPoolError), #[error("Unknown block number")] UnknownBlockNumber, + /// Thrown when querying for `finalized` or `safe` block before the merge transition is + /// finalized, + #[error("Unknown block")] + UnknownSafeOrFinalizedBlock, #[error("Unknown block or tx index")] UnknownBlockOrTxIndex, #[error("Invalid block range")] @@ -101,6 +105,9 @@ impl From for ErrorObject<'static> { EthApiError::UnknownBlockNumber | EthApiError::UnknownBlockOrTxIndex => { rpc_error_with_code(EthRpcErrorCode::ResourceNotFound.code(), error.to_string()) } + EthApiError::UnknownSafeOrFinalizedBlock => { + rpc_error_with_code(EthRpcErrorCode::UnknownBlock.code(), error.to_string()) + } EthApiError::Unsupported(msg) => internal_rpc_err(msg), EthApiError::InternalJsTracerError(msg) => internal_rpc_err(msg), EthApiError::InvalidParams(msg) => invalid_params_rpc_err(msg), @@ -143,11 +150,12 @@ impl From for EthApiError { ProviderError::HeaderNotFound(_) | ProviderError::BlockHashNotFound(_) | ProviderError::BestBlockNotFound | - ProviderError::FinalizedBlockNotFound | - ProviderError::SafeBlockNotFound | ProviderError::BlockNumberForTransactionIndexNotFound | ProviderError::TotalDifficultyNotFound { .. } | ProviderError::UnknownBlockHash(_) => EthApiError::UnknownBlockNumber, + ProviderError::FinalizedBlockNotFound | ProviderError::SafeBlockNotFound => { + EthApiError::UnknownSafeOrFinalizedBlock + } err => EthApiError::Internal(err.into()), } } diff --git a/crates/storage/provider/src/traits/block_id.rs b/crates/storage/provider/src/traits/block_id.rs index 4fd3ae4a1fd..296d7e1c189 100644 --- a/crates/storage/provider/src/traits/block_id.rs +++ b/crates/storage/provider/src/traits/block_id.rs @@ -1,5 +1,5 @@ use super::BlockHashReader; -use reth_interfaces::Result; +use reth_interfaces::{provider::ProviderError, Result}; use reth_primitives::{BlockHashOrNumber, BlockId, BlockNumber, BlockNumberOrTag, ChainInfo, H256}; /// Client trait for getting important block numbers (such as the latest block number), converting @@ -61,8 +61,14 @@ pub trait BlockIdReader: BlockNumReader + Send + Sync { .map(|res_opt| res_opt.map(|num_hash| num_hash.number)) } BlockNumberOrTag::Number(num) => num, - BlockNumberOrTag::Finalized => return self.finalized_block_number(), - BlockNumberOrTag::Safe => return self.safe_block_number(), + BlockNumberOrTag::Finalized => match self.finalized_block_number()? { + Some(block_number) => block_number, + None => return Err(ProviderError::FinalizedBlockNotFound.into()), + }, + BlockNumberOrTag::Safe => match self.safe_block_number()? { + Some(block_number) => block_number, + None => return Err(ProviderError::SafeBlockNotFound.into()), + }, }; Ok(Some(num)) } From 9bb52087590b2f3753164ff7c984f6228d3967e5 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 14 Jul 2023 19:35:18 +0100 Subject: [PATCH 090/150] chore: enable jemalloc by default on unix (#3735) --- bin/reth/Cargo.toml | 7 +++++-- bin/reth/src/lib.rs | 2 +- bin/reth/src/main.rs | 2 +- bin/reth/src/prometheus_exporter.rs | 8 ++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index a1539dc59d9..143200ab809 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -37,8 +37,6 @@ reth-basic-payload-builder = { path = "../../crates/payload/basic" } reth-discv4 = { path = "../../crates/net/discv4" } reth-metrics = { workspace = true } reth-prune = { path = "../../crates/prune" } -jemallocator = { version = "0.5.0", optional = true } -jemalloc-ctl = { version = "0.5.0", optional = true } # crypto secp256k1 = { workspace = true, features = ["global-context", "rand-std", "recovery"] } @@ -88,7 +86,12 @@ pretty_assertions = "1.3.0" humantime = "2.1.0" const-str = "0.5.6" +[target.'cfg(not(windows))'.dependencies] +jemallocator = { version = "0.5.0", optional = true } +jemalloc-ctl = { version = "0.5.0", optional = true } + [features] +default = ["jemalloc"] jemalloc = ["dep:jemallocator", "dep:jemalloc-ctl"] jemalloc-prof = ["jemalloc", "jemallocator?/profiling"] min-error-logs = ["tracing/release_max_level_error"] diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 46d2c76a16e..60fb98df12c 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -40,5 +40,5 @@ pub mod test_vectors; pub mod utils; pub mod version; -#[cfg(feature = "jemalloc")] +#[cfg(all(feature = "jemalloc", unix))] use jemallocator as _; diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index d7832b51c5d..220b04d9d0e 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -1,5 +1,5 @@ // We use jemalloc for performance reasons -#[cfg(feature = "jemalloc")] +#[cfg(all(feature = "jemalloc", unix))] #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; diff --git a/bin/reth/src/prometheus_exporter.rs b/bin/reth/src/prometheus_exporter.rs index f5635c88d28..c1cea799d91 100644 --- a/bin/reth/src/prometheus_exporter.rs +++ b/bin/reth/src/prometheus_exporter.rs @@ -121,7 +121,7 @@ pub(crate) async fn initialize( Ok(()) } -#[cfg(feature = "jemalloc")] +#[cfg(all(feature = "jemalloc", unix))] fn collect_memory_stats() { use jemalloc_ctl::{epoch, stats}; use reth_metrics::metrics::gauge; @@ -169,7 +169,7 @@ fn collect_memory_stats() { } } -#[cfg(feature = "jemalloc")] +#[cfg(all(feature = "jemalloc", unix))] fn describe_memory_stats() { use reth_metrics::metrics::describe_gauge; @@ -206,8 +206,8 @@ fn describe_memory_stats() { ); } -#[cfg(not(feature = "jemalloc"))] +#[cfg(not(all(feature = "jemalloc", unix)))] fn collect_memory_stats() {} -#[cfg(not(feature = "jemalloc"))] +#[cfg(not(all(feature = "jemalloc", unix)))] fn describe_memory_stats() {} From 4238cd566fb03fa4b5a878aa91060455bb284dc3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Jul 2023 20:38:09 +0200 Subject: [PATCH 091/150] fix: remove single body response check (#3783) --- crates/net/downloaders/src/bodies/request.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/net/downloaders/src/bodies/request.rs b/crates/net/downloaders/src/bodies/request.rs index a82b216cc32..a71b81f33e7 100644 --- a/crates/net/downloaders/src/bodies/request.rs +++ b/crates/net/downloaders/src/bodies/request.rs @@ -119,11 +119,11 @@ where // Increment total downloaded metric self.metrics.total_downloaded.increment(response_len as u64); - // Malicious peers often return a single block. Mark responses with single - // block when more than 1 were requested invalid. - // TODO: Instead of marking single block responses invalid, calculate - // soft response size lower limit and use that for filtering. - if bodies.is_empty() || (request_len != 1 && response_len == 1) { + // TODO: Malicious peers often return a single block even if it does not exceed the soft + // response limit (2MB). this could be penalized by checking if this block and the + // next one exceed the soft response limit, if not then peer either does not have the next + // block or deliberately sent a single block. + if bodies.is_empty() { return Err(DownloadError::EmptyResponse) } From 74352eece19bc3e26b7ce811abee4d29a99c22bd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Jul 2023 20:50:59 +0200 Subject: [PATCH 092/150] feat: remove peers after several unsuccessful attempts (#3780) --- crates/config/src/config.rs | 2 +- crates/net/network/src/error.rs | 18 ++++- crates/net/network/src/peers/manager.rs | 100 ++++++++++++++++++++++-- 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index b622ed47ee3..659df006b5e 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -313,7 +313,7 @@ mod tests { fn test_store_config() { with_tempdir("config-store-test", |config_path| { let config = Config::default(); - confy::store_path(config_path, config).unwrap(); + confy::store_path(config_path, config).expect("Failed to store config"); }) } diff --git a/crates/net/network/src/error.rs b/crates/net/network/src/error.rs index 55d5e14a8df..7362f709418 100644 --- a/crates/net/network/src/error.rs +++ b/crates/net/network/src/error.rs @@ -177,8 +177,18 @@ impl SessionError for EthStreamError { return match err { DisconnectReason::TooManyPeers | DisconnectReason::AlreadyConnected | + DisconnectReason::PingTimeout | + DisconnectReason::DisconnectRequested | DisconnectReason::TcpSubsystemError => Some(BackoffKind::Low), - _ => { + + DisconnectReason::ProtocolBreach | + DisconnectReason::UselessPeer | + DisconnectReason::IncompatibleP2PProtocolVersion | + DisconnectReason::NullNodeIdentity | + DisconnectReason::ClientQuitting | + DisconnectReason::UnexpectedHandshakeIdentity | + DisconnectReason::ConnectedToSelf | + DisconnectReason::SubprotocolSpecific => { // These are considered fatal, and are handled by the // [`SessionError::is_fatal_protocol_error`] Some(BackoffKind::High) @@ -245,8 +255,10 @@ impl SessionError for io::Error { // these usually happen when the remote instantly drops the connection, for example // if the previous connection isn't properly cleaned up yet and the peer is temp. // banned. - ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset | ErrorKind::BrokenPipe => { - Some(BackoffKind::Low) + ErrorKind::ConnectionReset | ErrorKind::BrokenPipe => Some(BackoffKind::Low), + ErrorKind::ConnectionRefused => { + // peer is unreachable, e.g. port not open or down + Some(BackoffKind::High) } _ => Some(BackoffKind::Medium), } diff --git a/crates/net/network/src/peers/manager.rs b/crates/net/network/src/peers/manager.rs index 63a89a24682..8b192f1d79a 100644 --- a/crates/net/network/src/peers/manager.rs +++ b/crates/net/network/src/peers/manager.rs @@ -110,6 +110,8 @@ pub(crate) struct PeersManager { connect_trusted_nodes_only: bool, /// Timestamp of the last time [Self::tick] was called. last_tick: Instant, + /// Maximum number of backoff attempts before we give up on a peer and dropping. + max_backoff_count: u32, } impl PeersManager { @@ -125,7 +127,7 @@ impl PeersManager { trusted_nodes, connect_trusted_nodes_only, basic_nodes, - .. + max_backoff_count, } = config; let (manager_tx, handle_rx) = mpsc::unbounded_channel(); let now = Instant::now(); @@ -161,6 +163,7 @@ impl PeersManager { backoff_durations, connect_trusted_nodes_only, last_tick: Instant::now(), + max_backoff_count, } } @@ -294,7 +297,7 @@ impl PeersManager { self.ban_list.ban_ip_until(ip, std::time::Instant::now() + self.ban_duration); } - /// Temporarily puts the peer in timeout + /// Temporarily puts the peer in timeout by inserting it into the backedoff peers set fn backoff_peer_until(&mut self, peer_id: PeerId, until: std::time::Instant) { trace!(target: "net::peers", ?peer_id, "backing off"); @@ -448,9 +451,9 @@ impl PeersManager { trace!(target: "net::peers", ?remote_addr, ?peer_id, ?err, "fatal connection error"); // remove the peer to which we can't establish a connection due to protocol related // issues. - if let Some(peer) = self.peers.remove(peer_id) { + if let Some((peer_id, peer)) = self.peers.remove_entry(peer_id) { self.connection_info.decr_state(peer.state); - self.queued_actions.push_back(PeerAction::PeerRemoved(*peer_id)); + self.queued_actions.push_back(PeerAction::PeerRemoved(peer_id)); } // ban the peer @@ -465,6 +468,7 @@ impl PeersManager { } } else { let mut backoff_until = None; + let mut remove_peer = false; if let Some(peer) = self.peers.get_mut(peer_id) { if let Some(kind) = err.should_backoff() { @@ -488,8 +492,20 @@ impl PeersManager { self.connection_info.decr_state(peer.state); peer.state = PeerConnectionState::Idle; + + if peer.severe_backoff_counter > self.max_backoff_count && !peer.is_trusted() { + // mark peer for removal if it has been backoff too many times and is _not_ + // trusted + remove_peer = true; + } } - if let Some(backoff_until) = backoff_until { + + // remove peer if it has been marked for removal + if remove_peer { + let (peer_id, _) = self.peers.remove_entry(peer_id).expect("peer must exist"); + self.queued_actions.push_back(PeerAction::PeerRemoved(peer_id)); + } else if let Some(backoff_until) = backoff_until { + // otherwise, backoff the peer if marked as such self.backoff_peer_until(*peer_id, backoff_until); } } @@ -1052,6 +1068,16 @@ pub struct PeersConfig { pub trusted_nodes: HashSet, /// Connect to trusted nodes only? pub connect_trusted_nodes_only: bool, + /// Maximum number of backoff attempts before we give up on a peer and dropping. + /// + /// The max time spent of a peer before it's removed from the set is determined by the + /// configured backoff duration and the max backoff count. + /// + /// With a backoff counter of 5 and a backoff duration of 1h, the minimum time spent of the + /// peer in the table is the sum of all backoffs (1h + 2h + 3h + 4h + 5h = 15h). + /// + /// Note: this does not apply to trusted peers. + pub max_backoff_count: u32, /// Basic nodes to connect to. #[cfg_attr(feature = "serde", serde(skip))] pub basic_nodes: HashSet, @@ -1067,6 +1093,8 @@ pub struct PeersConfig { pub reputation_weights: ReputationChangeWeights, /// How long to backoff peers that are we failed to connect to for non-fatal reasons, such as /// [`DisconnectReason::TooManyPeers`]. + /// + /// The backoff duration increases with number of backoff attempts. pub backoff_durations: PeerBackoffDurations, } @@ -1083,6 +1111,7 @@ impl Default for PeersConfig { trusted_nodes: Default::default(), connect_trusted_nodes_only: false, basic_nodes: Default::default(), + max_backoff_count: 5, } } } @@ -1134,6 +1163,12 @@ impl PeersConfig { self } + /// Configures the max allowed backoff count. + pub fn with_max_backoff_count(mut self, max_backoff_count: u32) -> Self { + self.max_backoff_count = max_backoff_count; + self + } + /// Read from file nodes available at launch. Ignored if None. pub fn with_basic_nodes_from_file( self, @@ -1575,6 +1610,61 @@ mod test { assert!(peers.peers.get(&peer).is_none()); } + #[tokio::test] + async fn test_remove_on_max_backoff_count() { + let peer = PeerId::random(); + let socket_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 2)), 8008); + let config = PeersConfig::default(); + let mut peers = PeersManager::new(config.clone()); + peers.add_peer(peer, socket_addr, None); + let peer_struct = peers.peers.get_mut(&peer).unwrap(); + + // Simulate a peer that was already backed off once + peer_struct.severe_backoff_counter = config.max_backoff_count; + + match event!(peers) { + PeerAction::PeerAdded(peer_id) => { + assert_eq!(peer_id, peer); + } + _ => unreachable!(), + } + match event!(peers) { + PeerAction::Connect { peer_id, .. } => { + assert_eq!(peer_id, peer); + } + _ => unreachable!(), + } + + poll_fn(|cx| { + assert!(peers.poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + peers.on_pending_session_dropped( + &socket_addr, + &peer, + &PendingSessionHandshakeError::Eth( + io::Error::new(io::ErrorKind::ConnectionRefused, "peer unreachable").into(), + ), + ); + + match event!(peers) { + PeerAction::PeerRemoved(peer_id) => { + assert_eq!(peer_id, peer); + } + _ => unreachable!(), + } + + poll_fn(|cx| { + assert!(peers.poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + assert!(peers.peers.get(&peer).is_none()); + } + #[tokio::test] async fn test_ban_on_pending_drop() { let peer = PeerId::random(); From 258f9b8aa0d55f70f45220019778c6fb44a73e90 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 14 Jul 2023 16:23:00 -0400 Subject: [PATCH 093/150] chore: make some session types pub (#3666) --- crates/net/network/src/lib.rs | 6 +- crates/net/network/src/session/handle.rs | 104 ++++++++++++++++++++--- crates/net/network/src/session/mod.rs | 72 ++++++++++------ 3 files changed, 147 insertions(+), 35 deletions(-) diff --git a/crates/net/network/src/lib.rs b/crates/net/network/src/lib.rs index 766ac668499..25733cc1100 100644 --- a/crates/net/network/src/lib.rs +++ b/crates/net/network/src/lib.rs @@ -151,6 +151,10 @@ pub use manager::{NetworkEvent, NetworkManager}; pub use message::PeerRequest; pub use network::NetworkHandle; pub use peers::PeersConfig; -pub use session::{PeerInfo, SessionsConfig}; +pub use session::{ + ActiveSessionHandle, ActiveSessionMessage, Direction, PeerInfo, PendingSessionEvent, + PendingSessionHandle, PendingSessionHandshakeError, SessionCommand, SessionEvent, SessionId, + SessionLimits, SessionManager, SessionsConfig, +}; pub use reth_eth_wire::{DisconnectReason, HelloBuilder, HelloMessage}; diff --git a/crates/net/network/src/session/handle.rs b/crates/net/network/src/session/handle.rs index 8d233039ded..1b4f893b6ee 100644 --- a/crates/net/network/src/session/handle.rs +++ b/crates/net/network/src/session/handle.rs @@ -14,7 +14,10 @@ use reth_primitives::PeerId; use std::{io, net::SocketAddr, sync::Arc, time::Instant}; use tokio::{ net::TcpStream, - sync::{mpsc, oneshot}, + sync::{ + mpsc::{self, error::SendError}, + oneshot, + }, }; /// A handler attached to a peer session that's not authenticated yet, pending Handshake and hello @@ -22,7 +25,7 @@ use tokio::{ /// /// This session needs to wait until it is authenticated. #[derive(Debug)] -pub(crate) struct PendingSessionHandle { +pub struct PendingSessionHandle { /// Can be used to tell the session to disconnect the connection/abort the handshake process. pub(crate) disconnect_tx: Option>, /// The direction of the session @@ -33,11 +36,16 @@ pub(crate) struct PendingSessionHandle { impl PendingSessionHandle { /// Sends a disconnect command to the pending session. - pub(crate) fn disconnect(&mut self) { + pub fn disconnect(&mut self) { if let Some(tx) = self.disconnect_tx.take() { let _ = tx.send(()); } } + + /// Returns the direction of the pending session (inbound or outbound). + pub fn direction(&self) -> Direction { + self.direction + } } /// An established session with a remote peer. @@ -46,7 +54,7 @@ impl PendingSessionHandle { /// be performed: chain synchronization, block propagation and transaction exchange. #[derive(Debug)] #[allow(unused)] -pub(crate) struct ActiveSessionHandle { +pub struct ActiveSessionHandle { /// The direction of the session pub(crate) direction: Direction, /// The assigned id for this session @@ -71,10 +79,59 @@ pub(crate) struct ActiveSessionHandle { impl ActiveSessionHandle { /// Sends a disconnect command to the session. - pub(crate) fn disconnect(&self, reason: Option) { + pub fn disconnect(&self, reason: Option) { // Note: we clone the sender which ensures the channel has capacity to send the message let _ = self.commands_to_session.clone().try_send(SessionCommand::Disconnect { reason }); } + + /// Sends a disconnect command to the session, awaiting the command channel for available + /// capacity. + pub async fn try_disconnect( + &self, + reason: Option, + ) -> Result<(), SendError> { + self.commands_to_session.clone().send(SessionCommand::Disconnect { reason }).await + } + + /// Returns the direction of the active session (inbound or outbound). + pub fn direction(&self) -> Direction { + self.direction + } + + /// Returns the assigned session id for this session. + pub fn session_id(&self) -> SessionId { + self.session_id + } + + /// Returns the negotiated eth version for this session. + pub fn version(&self) -> EthVersion { + self.version + } + + /// Returns the identifier of the remote peer. + pub fn remote_id(&self) -> PeerId { + self.remote_id + } + + /// Returns the timestamp when the session has been established. + pub fn established(&self) -> Instant { + self.established + } + + /// Returns the announced capabilities of the peer. + pub fn capabilities(&self) -> Arc { + self.capabilities.clone() + } + + /// Returns the client's name and version. + pub fn client_version(&self) -> Arc { + self.client_version.clone() + } + + /// Returns the address we're connected to. + pub fn remote_addr(&self) -> SocketAddr { + self.remote_addr + } } /// Info about an active peer session. @@ -98,46 +155,66 @@ pub struct PeerInfo { /// /// A session starts with a `Handshake`, followed by a `Hello` message which #[derive(Debug)] -pub(crate) enum PendingSessionEvent { +pub enum PendingSessionEvent { /// Represents a successful `Hello` and `Status` exchange: Established { + /// An internal identifier for the established session session_id: SessionId, + /// The remote node's socket address remote_addr: SocketAddr, /// The remote node's public key peer_id: PeerId, + /// All capabilities the peer announced capabilities: Arc, + /// The Status message the peer sent for the `eth` handshake status: Status, + /// The actual connection stream which can be used to send and receive `eth` protocol + /// messages conn: EthStream>>>, + /// The direction of the session, either `Inbound` or `Outgoing` direction: Direction, + /// The remote node's user agent, usually containing the client name and version client_id: String, }, /// Handshake unsuccessful, session was disconnected. Disconnected { + /// The remote node's socket address remote_addr: SocketAddr, + /// The internal identifier for the disconnected session session_id: SessionId, + /// The direction of the session, either `Inbound` or `Outgoing` direction: Direction, + /// The error that caused the disconnect error: Option, }, /// Thrown when unable to establish a [`TcpStream`]. OutgoingConnectionError { + /// The remote node's socket address remote_addr: SocketAddr, + /// The internal identifier for the disconnected session session_id: SessionId, + /// The remote node's public key peer_id: PeerId, + /// The error that caused the outgoing connection failure error: io::Error, }, - /// Thrown when authentication via Ecies failed. + /// Thrown when authentication via ECIES failed. EciesAuthError { + /// The remote node's socket address remote_addr: SocketAddr, + /// The internal identifier for the disconnected session session_id: SessionId, + /// The error that caused the ECIES session to fail error: ECIESError, + /// The direction of the session, either `Inbound` or `Outgoing` direction: Direction, }, } /// Commands that can be sent to the spawned session. #[derive(Debug)] -pub(crate) enum SessionCommand { +pub enum SessionCommand { /// Disconnect the connection Disconnect { /// Why the disconnect was initiated @@ -150,12 +227,19 @@ pub(crate) enum SessionCommand { /// Message variants an active session can produce and send back to the /// [`SessionManager`](crate::session::SessionManager) #[derive(Debug)] -pub(crate) enum ActiveSessionMessage { +pub enum ActiveSessionMessage { /// Session was gracefully disconnected. - Disconnected { peer_id: PeerId, remote_addr: SocketAddr }, + Disconnected { + /// The remote node's public key + peer_id: PeerId, + /// The remote node's socket address + remote_addr: SocketAddr, + }, /// Session was closed due an error ClosedOnConnectionError { + /// The remote node's public key peer_id: PeerId, + /// The remote node's socket address remote_addr: SocketAddr, /// The error that caused the session to close error: EthStreamError, diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index b0d628d48c4..d31d2c635bc 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -2,14 +2,7 @@ use crate::{ message::PeerMessage, metrics::SesssionManagerMetrics, - session::{ - active::ActiveSession, - config::SessionCounter, - handle::{ - ActiveSessionHandle, ActiveSessionMessage, PendingSessionEvent, PendingSessionHandle, - SessionCommand, - }, - }, + session::{active::ActiveSession, config::SessionCounter}, }; pub use crate::{message::PeerRequestSender, session::handle::PeerInfo}; use fnv::FnvHashMap; @@ -47,7 +40,11 @@ use tracing::{instrument, trace}; mod active; mod config; mod handle; -pub use config::SessionsConfig; +pub use config::{SessionLimits, SessionsConfig}; +pub use handle::{ + ActiveSessionHandle, ActiveSessionMessage, PendingSessionEvent, PendingSessionHandle, + SessionCommand, +}; /// Internal identifier for active sessions. #[derive(Debug, Clone, Copy, PartialOrd, PartialEq, Eq, Hash)] @@ -56,7 +53,7 @@ pub struct SessionId(usize); /// Manages a set of sessions. #[must_use = "Session Manager must be polled to process session events."] #[derive(Debug)] -pub(crate) struct SessionManager { +pub struct SessionManager { /// Tracks the identifier for the next session. next_id: usize, /// Keeps track of all sessions @@ -110,7 +107,7 @@ pub(crate) struct SessionManager { impl SessionManager { /// Creates a new empty [`SessionManager`]. - pub(crate) fn new( + pub fn new( secret_key: SecretKey, config: SessionsConfig, executor: Box, @@ -146,7 +143,7 @@ impl SessionManager { /// Check whether the provided [`ForkId`] is compatible based on the validation rules in /// `EIP-2124`. - pub(crate) fn is_valid_fork_id(&self, fork_id: ForkId) -> bool { + pub fn is_valid_fork_id(&self, fork_id: ForkId) -> bool { self.fork_filter.validate(fork_id).is_ok() } @@ -158,12 +155,12 @@ impl SessionManager { } /// Returns the current status of the session. - pub(crate) fn status(&self) -> Status { + pub fn status(&self) -> Status { self.status } /// Returns the session hello message. - pub(crate) fn hello_message(&self) -> HelloMessage { + pub fn hello_message(&self) -> HelloMessage { self.hello_message.clone() } @@ -235,7 +232,7 @@ impl SessionManager { } /// Starts a new pending session from the local node to the given remote node. - pub(crate) fn dial_outbound(&mut self, remote_addr: SocketAddr, remote_peer_id: PeerId) { + pub fn dial_outbound(&mut self, remote_addr: SocketAddr, remote_peer_id: PeerId) { // The error can be dropped because no dial will be made if it would exceed the limit if self.counter.ensure_pending_outbound().is_ok() { let session_id = self.next_id(); @@ -272,7 +269,7 @@ impl SessionManager { /// /// This will trigger the disconnect on the session task to gracefully terminate. The result /// will be picked up by the receiver. - pub(crate) fn disconnect(&self, node: PeerId, reason: Option) { + pub fn disconnect(&self, node: PeerId, reason: Option) { if let Some(session) = self.active_sessions.get(&node) { session.disconnect(reason); } @@ -297,21 +294,21 @@ impl SessionManager { /// /// It will trigger the disconnect on all the session tasks to gracefully terminate. The result /// will be picked by the receiver. - pub(crate) fn disconnect_all(&self, reason: Option) { + pub fn disconnect_all(&self, reason: Option) { for (_, session) in self.active_sessions.iter() { session.disconnect(reason); } } /// Disconnects all pending sessions. - pub(crate) fn disconnect_all_pending(&mut self) { + pub fn disconnect_all_pending(&mut self) { for (_, session) in self.pending_sessions.iter_mut() { session.disconnect(); } } /// Sends a message to the peer's session - pub(crate) fn send_message(&mut self, peer_id: &PeerId, msg: PeerMessage) { + pub fn send_message(&mut self, peer_id: &PeerId, msg: PeerMessage) { if let Some(session) = self.active_sessions.get_mut(peer_id) { let _ = session.commands_to_session.try_send(SessionCommand::Message(msg)); } @@ -565,7 +562,7 @@ impl SessionManager { } /// Returns [`PeerInfo`] for all connected peers - pub(crate) fn get_peer_info(&self) -> Vec { + pub fn get_peer_info(&self) -> Vec { self.active_sessions .values() .map(|session| PeerInfo { @@ -581,7 +578,7 @@ impl SessionManager { /// Returns [`PeerInfo`] for a given peer. /// /// Returns `None` if there's no active session to the peer. - pub(crate) fn get_peer_info_by_id(&self, peer_id: PeerId) -> Option { + pub fn get_peer_info_by_id(&self, peer_id: PeerId) -> Option { self.active_sessions.get(&peer_id).map(|session| PeerInfo { remote_id: session.remote_id, direction: session.direction, @@ -594,35 +591,50 @@ impl SessionManager { /// Events produced by the [`SessionManager`] #[derive(Debug)] -pub(crate) enum SessionEvent { +pub enum SessionEvent { /// A new session was successfully authenticated. /// /// This session is now able to exchange data. SessionEstablished { + /// The remote node's public key peer_id: PeerId, + /// The remote node's socket address remote_addr: SocketAddr, + /// The user agent of the remote node, usually containing the client name and version client_version: Arc, + /// The capabilities the remote node has announced capabilities: Arc, /// negotiated eth version version: EthVersion, + /// The Status message the peer sent during the `eth` handshake status: Status, + /// The channel for sending messages to the peer with the session messages: PeerRequestSender, + /// The direction of the session, either `Inbound` or `Outgoing` direction: Direction, + /// The maximum time that the session waits for a response from the peer before timing out + /// the connection timeout: Arc, }, + /// The peer was already connected with another session. AlreadyConnected { + /// The remote node's public key peer_id: PeerId, + /// The remote node's socket address remote_addr: SocketAddr, + /// The direction of the session, either `Inbound` or `Outgoing` direction: Direction, }, /// A session received a valid message via RLPx. ValidMessage { + /// The remote node's public key peer_id: PeerId, /// Message received from the peer. message: PeerMessage, }, /// Received a message that does not match the announced capabilities of the peer. InvalidMessage { + /// The remote node's public key peer_id: PeerId, /// Announced capabilities of the remote peer. capabilities: Arc, @@ -641,19 +653,27 @@ pub(crate) enum SessionEvent { }, /// Closed an incoming pending session during handshaking. IncomingPendingSessionClosed { + /// The remote node's socket address remote_addr: SocketAddr, + /// The pending handshake session error that caused the session to close error: Option, }, /// Closed an outgoing pending session during handshaking. OutgoingPendingSessionClosed { + /// The remote node's socket address remote_addr: SocketAddr, + /// The remote node's public key peer_id: PeerId, + /// The pending handshake session error that caused the session to close error: Option, }, /// Failed to establish a tcp stream OutgoingConnectionError { + /// The remote node's socket address remote_addr: SocketAddr, + /// The remote node's public key peer_id: PeerId, + /// The error that caused the outgoing connection to fail error: io::Error, }, /// Session was closed due to an error @@ -667,15 +687,19 @@ pub(crate) enum SessionEvent { }, /// Active session was gracefully disconnected. Disconnected { + /// The remote node's public key peer_id: PeerId, + /// The remote node's socket address that we were connected to remote_addr: SocketAddr, }, } /// Errors that can occur during handshaking/authenticating the underlying streams. #[derive(Debug)] -pub(crate) enum PendingSessionHandshakeError { +pub enum PendingSessionHandshakeError { + /// The pending session failed due to an error while establishing the `eth` stream Eth(EthStreamError), + /// The pending session failed due to an error while establishing the ECIES stream Ecies(ECIESError), } @@ -700,7 +724,7 @@ pub enum Direction { impl Direction { /// Returns `true` if this an incoming connection. - pub(crate) fn is_incoming(&self) -> bool { + pub fn is_incoming(&self) -> bool { matches!(self, Direction::Incoming) } From 001c35731c790187ba4dbbbc608899fc50c3a8e0 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sat, 15 Jul 2023 12:02:14 +0300 Subject: [PATCH 094/150] chore(downloader): simplify bodies task polling (#3788) --- crates/net/downloaders/src/bodies/task.rs | 45 +++++++++-------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/crates/net/downloaders/src/bodies/task.rs b/crates/net/downloaders/src/bodies/task.rs index 30aabe3a6ab..456f8a637cc 100644 --- a/crates/net/downloaders/src/bodies/task.rs +++ b/crates/net/downloaders/src/bodies/task.rs @@ -115,35 +115,26 @@ impl Future for SpawnedDownloader { let this = self.get_mut(); loop { - loop { - match this.updates.poll_next_unpin(cx) { - Poll::Pending => break, - Poll::Ready(None) => { - // channel closed, this means [TaskDownloader] was dropped, so we can also - // exit - return Poll::Ready(()) - } - Poll::Ready(Some(range)) => { - if let Err(err) = this.downloader.set_download_range(range) { - tracing::error!(target: "downloaders::bodies", ?err, "Failed to set bodies download range"); - - match ready!(this.bodies_tx.poll_reserve(cx)) { - Ok(()) => { - if this.bodies_tx.send_item(Err(err)).is_err() { - // channel closed, this means [TaskDownloader] was dropped, - // so we can also - // exit - return Poll::Ready(()) - } - } - Err(_) => { - // channel closed, this means [TaskDownloader] was dropped, so - // we can also exit - return Poll::Ready(()) - } - } + while let Poll::Ready(update) = this.updates.poll_next_unpin(cx) { + if let Some(range) = update { + if let Err(err) = this.downloader.set_download_range(range) { + tracing::error!(target: "downloaders::bodies", ?err, "Failed to set bodies download range"); + + // Clone the sender ensure its availability. See [PollSender::clone]. + let mut bodies_tx = this.bodies_tx.clone(); + + let forward_error_result = ready!(bodies_tx.poll_reserve(cx)) + .and_then(|_| bodies_tx.send_item(Err(err))); + if forward_error_result.is_err() { + // channel closed, this means [TaskDownloader] was dropped, + // so we can also exit + return Poll::Ready(()) } } + } else { + // channel closed, this means [TaskDownloader] was dropped, so we can also + // exit + return Poll::Ready(()) } } From 28969f661cbfb34ff676da61cc0c51417431afae Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sat, 15 Jul 2023 12:27:19 +0300 Subject: [PATCH 095/150] chore(tree): remove revert notification (#3645) --- crates/storage/provider/src/traits/chain.rs | 18 ++------- crates/transaction-pool/src/maintain.rs | 45 --------------------- 2 files changed, 4 insertions(+), 59 deletions(-) diff --git a/crates/storage/provider/src/traits/chain.rs b/crates/storage/provider/src/traits/chain.rs index dc5c8a168d7..93f5a61a4b7 100644 --- a/crates/storage/provider/src/traits/chain.rs +++ b/crates/storage/provider/src/traits/chain.rs @@ -64,14 +64,11 @@ impl Stream for CanonStateNotificationStream { #[derive(Clone, Debug)] #[allow(missing_docs)] pub enum CanonStateNotification { - /// Chain reorgs and both old and new chain are returned. - Reorg { old: Arc, new: Arc }, - /// Chain got reverted without reorg and only old chain is returned. - /// - /// This reverts the chain's tip to the first block of the chain. - Revert { old: Arc }, /// Chain got extended without reorg and only new chain is returned. Commit { new: Arc }, + /// Chain reorgs and both old and new chain are returned. + /// Revert is just a subset of reorg where the new chain is empty. + Reorg { old: Arc, new: Arc }, } // For one reason or another, the compiler can't derive PartialEq for CanonStateNotification. @@ -82,7 +79,6 @@ impl PartialEq for CanonStateNotification { (Self::Reorg { old: old1, new: new1 }, Self::Reorg { old: old2, new: new2 }) => { old1 == old2 && new1 == new2 } - (Self::Revert { old: old1 }, Self::Revert { old: old2 }) => old1 == old2, (Self::Commit { new: new1 }, Self::Commit { new: new2 }) => new1 == new2, _ => false, } @@ -94,7 +90,6 @@ impl CanonStateNotification { pub fn reverted(&self) -> Option> { match self { Self::Reorg { old, .. } => Some(old.clone()), - Self::Revert { old } => Some(old.clone()), Self::Commit { .. } => None, } } @@ -102,12 +97,9 @@ impl CanonStateNotification { /// Get the new chain if any. /// /// Returns the new committed [Chain] for [Self::Reorg] and [Self::Commit] variants. - /// - /// Returns None for [Self::Revert] variant. pub fn committed(&self) -> Option> { match self { Self::Reorg { new, .. } => Some(new.clone()), - Self::Revert { .. } => None, Self::Commit { new } => Some(new.clone()), } } @@ -115,12 +107,10 @@ impl CanonStateNotification { /// Returns the new tip of the chain. /// /// Returns the new tip for [Self::Reorg] and [Self::Commit] variants which commit at least 1 - /// new block. Returns the first block of the chain for [Self::Revert] variant, which is the - /// block that the chain reverted to. + /// new block. pub fn tip(&self) -> &SealedBlockWithSenders { match self { Self::Reorg { new, .. } => new.tip(), - Self::Revert { old } => old.first(), Self::Commit { new } => new.tip(), } } diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index a55340ed55a..11d9871f4a1 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -162,51 +162,6 @@ where metrics.inc_reinserted_transactions(pruned_old_transactions.len()); let _ = pool.add_external_transactions(pruned_old_transactions).await; } - CanonStateNotification::Revert { old } => { - // this similar to the inverse of a commit where we need to insert the transactions - // back into the pool and update the pool's state accordingly - - let (blocks, state) = old.inner(); - let first_block = blocks.first(); - - if first_block.hash == pool_info.last_seen_block_hash { - // nothing to update - continue - } - - // base fee for the next block: `first_block+1` - let pending_block_base_fee = - first_block.next_block_base_fee().unwrap_or_default() as u128; - - let mut changed_accounts = Vec::with_capacity(state.accounts().len()); - for acc in changed_accounts_iter(state) { - // we can always clear the dirty flag for this account - dirty_addresses.remove(&acc.address); - changed_accounts.push(acc); - } - - let update = CanonicalStateUpdate { - hash: first_block.hash, - number: first_block.number, - pending_block_base_fee, - changed_accounts, - // no tx to prune in the reverted chain - mined_transactions: vec![], - }; - pool.on_canonical_state_change(update); - - let pruned_old_transactions = blocks - .transactions() - .filter_map(|tx| tx.clone().into_ecrecovered()) - .map(

::Transaction::from_recovered_transaction) - .collect::>(); - - // all transactions that were mined in the old chain need to be re-injected - // - // Note: we no longer know if the tx was local or external - metrics.inc_reinserted_transactions(pruned_old_transactions.len()); - let _ = pool.add_external_transactions(pruned_old_transactions).await; - } CanonStateNotification::Commit { new } => { let (blocks, state) = new.inner(); let tip = blocks.tip(); From d3ced8bc94e71fb39a5b24662f9567eb27c2b1bb Mon Sep 17 00:00:00 2001 From: Panagiotis Ganelis <50522617+PanGan21@users.noreply.github.com> Date: Sat, 15 Jul 2023 13:33:33 +0300 Subject: [PATCH 096/150] feat: trim cmd args in parser (#3789) --- bin/reth/src/args/rpc_server_args.rs | 19 +++++++++++++++++++ crates/rpc/rpc-builder/src/lib.rs | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 9e7b890e0ca..230741885bc 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -526,6 +526,25 @@ mod tests { ); } + #[test] + fn test_transport_rpc_module_trim_config() { + let args = CommandParser::::parse_from([ + "reth", + "--http.api", + " eth, admin, debug", + "--http", + "--ws", + ]) + .args; + let config = args.transport_rpc_module_config(); + let expected = vec![RethRpcModule::Eth, RethRpcModule::Admin, RethRpcModule::Debug]; + assert_eq!(config.http().cloned().unwrap().into_selection(), expected); + assert_eq!( + config.ws().cloned().unwrap().into_selection(), + RpcModuleSelection::standard_modules() + ); + } + #[test] fn test_rpc_server_config() { let args = CommandParser::::parse_from([ diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index eb9f0d1fd39..7013131ec5c 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -604,7 +604,7 @@ impl FromStr for RpcModuleSelection { type Err = ParseError; fn from_str(s: &str) -> Result { - let mut modules = s.split(',').peekable(); + let mut modules = s.split(',').map(str::trim).peekable(); let first = modules.peek().copied().ok_or(ParseError::VariantNotFound)?; match first { "all" | "All" => Ok(RpcModuleSelection::All), From a5aa751db072ea248117e0c0347e94df2f75cd4b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 15 Jul 2023 16:27:06 +0200 Subject: [PATCH 097/150] feat: txpool block building fallback (#3755) --- crates/rpc/rpc/src/eth/api/block.rs | 10 ++- crates/rpc/rpc/src/eth/api/mod.rs | 82 ++++++++++++++++++++- crates/rpc/rpc/src/eth/api/pending_block.rs | 67 +++++++++++++++++ crates/rpc/rpc/src/eth/api/transactions.rs | 31 ++------ 4 files changed, 160 insertions(+), 30 deletions(-) create mode 100644 crates/rpc/rpc/src/eth/api/pending_block.rs diff --git a/crates/rpc/rpc/src/eth/api/block.rs b/crates/rpc/rpc/src/eth/api/block.rs index d6d12438543..5220f907a5a 100644 --- a/crates/rpc/rpc/src/eth/api/block.rs +++ b/crates/rpc/rpc/src/eth/api/block.rs @@ -10,10 +10,13 @@ use crate::{ use reth_primitives::{BlockId, BlockNumberOrTag, TransactionMeta}; use reth_provider::{BlockReaderIdExt, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{Block, Index, RichBlock, TransactionReceipt}; +use reth_transaction_pool::TransactionPool; impl EthApi where Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, + Pool: TransactionPool + Clone + 'static, + Network: Send + Sync + 'static, { /// Returns the uncle headers of the given block /// @@ -121,7 +124,12 @@ where if block_id.is_pending() { // Pending block can be fetched directly without need for caching - return Ok(self.provider().pending_block()?) + let maybe_pending = self.provider().pending_block()?; + return if maybe_pending.is_some() { + return Ok(maybe_pending) + } else { + self.local_pending_block().await + } } let block_hash = match self.provider().block_hash_for_id(block_id)? { diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 227e14cefb6..b53a80993ae 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -12,22 +12,27 @@ use crate::eth::{ use async_trait::async_trait; use reth_interfaces::Result; use reth_network_api::NetworkInfo; -use reth_primitives::{Address, BlockId, BlockNumberOrTag, ChainInfo, H256, U256, U64}; +use reth_primitives::{ + Address, BlockId, BlockNumberOrTag, ChainInfo, SealedBlock, H256, U256, U64, +}; use reth_provider::{BlockReaderIdExt, EvmEnvProvider, StateProviderBox, StateProviderFactory}; use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use reth_transaction_pool::TransactionPool; -use std::{future::Future, sync::Arc}; -use tokio::sync::oneshot; +use revm_primitives::{BlockEnv, CfgEnv}; +use std::{future::Future, sync::Arc, time::Instant}; +use tokio::sync::{oneshot, Mutex}; mod block; mod call; mod fees; +mod pending_block; mod server; mod sign; mod state; mod transactions; +use crate::eth::api::pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; pub use transactions::{EthTransactions, TransactionSource}; /// `Eth` API trait. @@ -115,6 +120,7 @@ where gas_oracle, starting_block: U256::from(latest_block), task_spawner, + pending_block: Default::default(), }; Self { inner: Arc::new(inner) } } @@ -201,6 +207,74 @@ where } } +impl EthApi +where + Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, + Pool: TransactionPool + Clone + 'static, + Network: Send + Sync + 'static, +{ + /// Configures the [CfgEnv] and [BlockEnv] for the pending block + /// + /// If no pending block is available, this will derive it from the `latest` block + pub(crate) fn pending_block_env_and_cfg(&self) -> EthResult { + let origin = if let Some(pending) = self.provider().pending_block()? { + PendingBlockEnvOrigin::ActualPending(pending) + } else { + // no pending block from the CL yet, so we use the latest block and modify the env + // values that we can + let mut latest = + self.provider().latest_header()?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; + + // child block + latest.number += 1; + // assumed child block is in the next slot + latest.timestamp += 12; + // base fee of the child block + latest.base_fee_per_gas = latest.next_block_base_fee(); + + PendingBlockEnvOrigin::DerivedFromLatest(latest) + }; + + let mut cfg = CfgEnv::default(); + let mut block_env = BlockEnv::default(); + self.provider().fill_block_env_with_header(&mut block_env, origin.header())?; + self.provider().fill_cfg_env_with_header(&mut cfg, origin.header())?; + + Ok(PendingBlockEnv { cfg, block_env, origin }) + } + + /// Returns the locally built pending block + pub(crate) async fn local_pending_block(&self) -> EthResult> { + let pending = self.pending_block_env_and_cfg()?; + if pending.origin.is_actual_pending() { + return Ok(pending.origin.into_actual_pending()) + } + + // no pending block from the CL yet, so we need to build it ourselves via txpool + self.on_blocking_task(|this| async move { + let PendingBlockEnv { cfg: _, block_env, origin } = pending; + let lock = this.inner.pending_block.lock().await; + let now = Instant::now(); + // this is guaranteed to be the `latest` header + let parent_header = origin.into_header(); + + // check if the block is still good + if let Some(pending) = lock.as_ref() { + if block_env.number.to::() == pending.block.number && + pending.block.parent_hash == parent_header.parent_hash && + now <= pending.expires_at + { + return Ok(Some(pending.block.clone())) + } + } + + // TODO(mattsse): actually build the pending block + Ok(None) + }) + .await + } +} + impl std::fmt::Debug for EthApi { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("EthApi").finish_non_exhaustive() @@ -284,4 +358,6 @@ struct EthApiInner { starting_block: U256, /// The type that can spawn tasks which would otherwise block. task_spawner: Box, + /// Cached pending block if any + pending_block: Mutex>, } diff --git a/crates/rpc/rpc/src/eth/api/pending_block.rs b/crates/rpc/rpc/src/eth/api/pending_block.rs new file mode 100644 index 00000000000..8e57f893dca --- /dev/null +++ b/crates/rpc/rpc/src/eth/api/pending_block.rs @@ -0,0 +1,67 @@ +//! Support for building a pending block via local txpool. + +use reth_primitives::{SealedBlock, SealedHeader}; +use revm_primitives::{BlockEnv, CfgEnv}; +use std::time::Instant; + +/// Configured [BlockEnv] and [CfgEnv] for a pending block +#[derive(Debug, Clone)] +pub(crate) struct PendingBlockEnv { + /// Configured [CfgEnv] for the pending block. + pub(crate) cfg: CfgEnv, + /// Configured [BlockEnv] for the pending block. + pub(crate) block_env: BlockEnv, + /// Origin block for the config + pub(crate) origin: PendingBlockEnvOrigin, +} + +/// The origin for a configured [PendingBlockEnv] +#[derive(Clone, Debug)] +pub(crate) enum PendingBlockEnvOrigin { + /// The pending block as received from the CL. + ActualPending(SealedBlock), + /// The header of the latest block + DerivedFromLatest(SealedHeader), +} + +impl PendingBlockEnvOrigin { + /// Returns true if the origin is the actual pending block as received from the CL. + pub(crate) fn is_actual_pending(&self) -> bool { + matches!(self, PendingBlockEnvOrigin::ActualPending(_)) + } + + /// Consumes the type and returns the actual pending block. + pub(crate) fn into_actual_pending(self) -> Option { + match self { + PendingBlockEnvOrigin::ActualPending(block) => Some(block), + _ => None, + } + } + + /// Returns the header this pending block is based on. + pub(crate) fn header(&self) -> &SealedHeader { + match self { + PendingBlockEnvOrigin::ActualPending(block) => &block.header, + PendingBlockEnvOrigin::DerivedFromLatest(header) => header, + } + } + + /// Consumes the type and returns the header this pending block is based on. + pub(crate) fn into_header(self) -> SealedHeader { + match self { + PendingBlockEnvOrigin::ActualPending(block) => block.header, + PendingBlockEnvOrigin::DerivedFromLatest(header) => header, + } + } +} + +/// In memory pending block for `pending` tag +#[derive(Debug)] +pub(crate) struct PendingBlock { + /// The cached pending block + pub(crate) block: SealedBlock, + /// Timestamp when the pending block is considered outdated + pub(crate) expires_at: Instant, +} + +impl PendingBlock {} diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 1cbd5df2ba2..c932ec825bc 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -1,6 +1,7 @@ //! Contains RPC handler implementations specific to transactions use crate::{ eth::{ + api::pending_block::PendingBlockEnv, error::{EthApiError, EthResult, SignError}, revm_utils::{ inspect, inspect_and_return_db, prepare_call_env, replay_transactions_until, transact, @@ -239,31 +240,8 @@ where async fn evm_env_at(&self, at: BlockId) -> EthResult<(CfgEnv, BlockEnv, BlockId)> { if at.is_pending() { - let header = if let Some(pending) = self.provider().pending_header()? { - pending - } else { - // no pending block from the CL yet, so we use the latest block and modify the env - // values that we can - let mut latest = self - .provider() - .latest_header()? - .ok_or_else(|| EthApiError::UnknownBlockNumber)?; - - // child block - latest.number += 1; - // assumed child block is in the next slot - latest.timestamp += 12; - // base fee of the child block - latest.base_fee_per_gas = latest.next_block_base_fee(); - - latest - }; - - let mut cfg = CfgEnv::default(); - let mut block_env = BlockEnv::default(); - self.provider().fill_block_env_with_header(&mut block_env, &header)?; - self.provider().fill_cfg_env_with_header(&mut cfg, &header)?; - return Ok((cfg, block_env, header.hash.into())) + let PendingBlockEnv { cfg, block_env, origin } = self.pending_block_env_and_cfg()?; + Ok((cfg, block_env, origin.header().hash.into())) } else { // Use cached values if there is no pending block let block_hash = self @@ -652,6 +630,7 @@ where impl EthApi where Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, + Network: 'static, { /// Helper function for `eth_getTransactionReceipt` /// @@ -675,7 +654,7 @@ impl EthApi where Pool: TransactionPool + 'static, Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, - Network: 'static, + Network: Send + Sync + 'static, { pub(crate) fn sign_request( &self, From 792b78220ee0bb4c7407071fea87d91fd3b4acef Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 15 Jul 2023 16:28:26 +0200 Subject: [PATCH 098/150] fix: concurrent ipc driver impl (#3790) --- Cargo.lock | 24 -------- crates/rpc/ipc/Cargo.toml | 1 - crates/rpc/ipc/src/server/connection.rs | 82 ++++++++++++++++++++++++- crates/rpc/ipc/src/server/mod.rs | 54 +++++----------- 4 files changed, 96 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ee5b47fcf7..3bf50b5dcaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5327,7 +5327,6 @@ dependencies = [ "tokio-util", "tower", "tracing", - "tracing-test", ] [[package]] @@ -7400,29 +7399,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "tracing-test" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" -dependencies = [ - "lazy_static", - "tracing-core", - "tracing-subscriber", - "tracing-test-macro", -] - -[[package]] -name = "tracing-test-macro" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" -dependencies = [ - "lazy_static", - "quote 1.0.28", - "syn 1.0.109", -] - [[package]] name = "triehash" version = "0.8.4" diff --git a/crates/rpc/ipc/Cargo.toml b/crates/rpc/ipc/Cargo.toml index 417dfd8239d..3c6d3832b53 100644 --- a/crates/rpc/ipc/Cargo.toml +++ b/crates/rpc/ipc/Cargo.toml @@ -30,5 +30,4 @@ bytes = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -tracing-test = "0.2" tokio-stream = { workspace = true, features = ["sync"] } diff --git a/crates/rpc/ipc/src/server/connection.rs b/crates/rpc/ipc/src/server/connection.rs index ff0bd4c00ab..e502a27de7a 100644 --- a/crates/rpc/ipc/src/server/connection.rs +++ b/crates/rpc/ipc/src/server/connection.rs @@ -1,8 +1,10 @@ //! A IPC connection. use crate::stream_codec::StreamCodec; -use futures::{ready, Sink, Stream, StreamExt}; +use futures::{ready, stream::FuturesUnordered, Sink, Stream, StreamExt}; use std::{ + collections::VecDeque, + future::Future, io, marker::PhantomData, pin::Pin, @@ -10,6 +12,7 @@ use std::{ }; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio_util::codec::Framed; +use tower::Service; pub(crate) type JsonRpcStream = Framed; @@ -113,3 +116,80 @@ where self.project().0.poll_close(cx) } } + +/// Drives an [IpcConn] forward. +/// +/// This forwards received requests from the connection to the service and sends responses to the +/// connection. +/// +/// This future terminates when the connection is closed. +#[pin_project::pin_project] +#[must_use = "futures do nothing unless you `.await` or poll them"] +pub(crate) struct IpcConnDriver { + #[pin] + pub(crate) conn: IpcConn>, + pub(crate) service: S, + #[pin] + pub(crate) pending_calls: FuturesUnordered, + pub(crate) items: VecDeque, +} + +impl IpcConnDriver { + /// Add a new item to the send queue. + pub(crate) fn push_back(&mut self, item: String) { + self.items.push_back(item); + } +} + +impl Future for IpcConnDriver +where + S: Service> + Send + 'static, + S::Error: Into>, + S::Future: Send, + T: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + + loop { + // process calls + if !this.pending_calls.is_empty() { + while let Poll::Ready(Some(res)) = this.pending_calls.as_mut().poll_next(cx) { + let item = match res { + Ok(Some(resp)) => resp, + Ok(None) => continue, + Err(err) => err.into().to_string(), + }; + this.items.push_back(item); + } + } + + // write to the sink + while this.conn.as_mut().poll_ready(cx).is_ready() { + if let Some(item) = this.items.pop_front() { + if let Err(err) = this.conn.as_mut().start_send(item) { + tracing::warn!("IPC response failed: {:?}", err); + return Poll::Ready(()) + } + } else { + break + } + } + + // read from the stream + match ready!(this.conn.as_mut().poll_next(cx)) { + Some(Ok(item)) => { + let call = this.service.call(item); + this.pending_calls.push(call); + } + Some(Err(err)) => { + tracing::warn!("IPC request failed: {:?}", err); + return Poll::Ready(()) + } + None => return Poll::Ready(()), + } + } + } +} diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index d30a5ae7679..7911f037d99 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -4,7 +4,7 @@ use crate::server::{ connection::{Incoming, IpcConn, JsonRpcStream}, future::{ConnectionGuard, FutureDriver, StopHandle}, }; -use futures::{FutureExt, SinkExt, Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt}; use jsonrpsee::{ core::{Error, TEN_MB_SIZE_BYTES}, server::{logger::Logger, IdProvider, RandomIntegerIdProvider, ServerHandle}, @@ -25,6 +25,7 @@ use tower::{layer::util::Identity, Service}; use tracing::{debug, trace, warn}; // re-export so can be used during builder setup +use crate::server::connection::IpcConnDriver; pub use parity_tokio_ipc::Endpoint; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; @@ -285,7 +286,7 @@ impl Service for TowerService { /// Spawns the IPC connection onto a new task async fn spawn_connection( conn: IpcConn>, - mut service: S, + service: S, mut stop_handle: StopHandle, rx: mpsc::Receiver, ) where @@ -296,51 +297,29 @@ async fn spawn_connection( { let task = tokio::task::spawn(async move { let rx_item = ReceiverStream::new(rx); + let conn = IpcConnDriver { + conn, + service, + pending_calls: Default::default(), + items: Default::default(), + }; tokio::pin!(conn, rx_item); loop { - let item = tokio::select! { - res = conn.next() => { - match res { - Some(Ok(request)) => { - // handle the RPC request - match service.call(request).await { - Ok(Some(resp)) => { - resp - }, - Ok(None) => { - continue - }, - Err(err) => err.into().to_string(), - } - }, - Some(Err(e)) => { - tracing::warn!("IPC request failed: {:?}", e); - break - } - None => { - return - } - } + tokio::select! { + _ = &mut conn => { + break } item = rx_item.next() => { - match item { - Some(item) => item, - None => { - continue - } + if let Some(item) = item { + conn.push_back(item); } } _ = stop_handle.shutdown() => { + // shutdown break } }; - - // send item over ipc - if let Err(err) = conn.send(item).await { - warn!("Failed to send IPC response: {:?}", err); - break - } } }); @@ -593,7 +572,6 @@ mod tests { use parity_tokio_ipc::dummy_endpoint; use tokio::sync::broadcast; use tokio_stream::wrappers::BroadcastStream; - use tracing_test::traced_test; async fn pipe_from_stream_with_bounded_buffer( pending: PendingSubscriptionSink, @@ -641,7 +619,6 @@ mod tests { } #[tokio::test] - #[traced_test] async fn test_rpc_request() { let endpoint = dummy_endpoint(); let server = Builder::default().build(&endpoint).unwrap(); @@ -672,7 +649,6 @@ mod tests { } #[tokio::test(flavor = "multi_thread")] - #[traced_test] async fn test_rpc_subscription() { let endpoint = dummy_endpoint(); let server = Builder::default().build(&endpoint).unwrap(); From 216f925166e172857ab25884b391fa7fa09a96d6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 15 Jul 2023 16:48:18 +0200 Subject: [PATCH 099/150] fix: rpc cap block range correctly (#3791) Co-authored-by: Georgios Konstantopoulos --- crates/rpc/rpc/src/eth/api/fees.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/fees.rs b/crates/rpc/rpc/src/eth/api/fees.rs index c9a1513535a..49b0291a119 100644 --- a/crates/rpc/rpc/src/eth/api/fees.rs +++ b/crates/rpc/rpc/src/eth/api/fees.rs @@ -67,9 +67,11 @@ where return Err(EthApiError::UnknownBlockNumber) }; - // Check that we would not be querying outside of genesis - if end_block < block_count { - return Err(EthApiError::InvalidBlockRange) + // need to add 1 to the end block to get the correct (inclusive) range + let end_block_plus = end_block + 1; + // Ensure that we would not be querying outside of genesis + if end_block_plus < block_count { + block_count = end_block_plus; } // If reward percentiles were specified, we need to validate that they are monotonically @@ -86,7 +88,8 @@ where // // Treat a request for 1 block as a request for `newest_block..=newest_block`, // otherwise `newest_block - 2 - let start_block = end_block - block_count + 1; + // SAFETY: We ensured that block count is capped + let start_block = end_block_plus - block_count; let headers = self.provider().sealed_headers_range(start_block..=end_block)?; if headers.len() != block_count as usize { return Err(EthApiError::InvalidBlockRange) From 0795b033500fc94d3396dce0fa231309053045bb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 15 Jul 2023 17:26:44 +0200 Subject: [PATCH 100/150] feat: reload dirty accounts if pool drifts (#3732) Co-authored-by: Georgios Konstantopoulos --- Cargo.lock | 4 +- bin/reth/src/node/mod.rs | 2 + crates/transaction-pool/src/lib.rs | 15 +- crates/transaction-pool/src/maintain.rs | 182 ++++++++++++++++++--- crates/transaction-pool/src/noop.rs | 6 +- crates/transaction-pool/src/pool/mod.rs | 22 ++- crates/transaction-pool/src/pool/txpool.rs | 43 +++-- crates/transaction-pool/src/traits.rs | 15 +- 8 files changed, 239 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bf50b5dcaa..3aa656a4cdd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4296,9 +4296,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index d2ed2825610..1c5cd3a7fce 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -241,6 +241,8 @@ impl Command { client, pool, chain_events, + ctx.task_executor.clone(), + Default::default(), ), ); debug!(target: "reth::cli", "Spawned txpool maintenance task"); diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index f5c92b8086d..a87e0600752 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -132,7 +132,7 @@ //! ); //! //! // spawn a task that listens for new blocks and updates the pool's transactions, mined transactions etc.. -//! tokio::task::spawn( maintain_transaction_pool_future(client, pool, stream)); +//! tokio::task::spawn( maintain_transaction_pool_future(client, pool, stream, TokioTaskExecutor::default(), Default::default())); //! //! # } //! ``` @@ -145,7 +145,10 @@ use crate::pool::PoolInner; use aquamarine as _; use reth_primitives::{Address, TxHash, U256}; use reth_provider::StateProviderFactory; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use tokio::sync::mpsc::Receiver; use tracing::{instrument, trace}; @@ -438,6 +441,10 @@ where ) -> Vec>> { self.pool.get_transactions_by_sender(sender) } + + fn unique_senders(&self) -> HashSet

{ + self.pool.unique_senders() + } } impl TransactionPoolExt for Pool @@ -454,6 +461,10 @@ where fn on_canonical_state_change(&self, update: CanonicalStateUpdate) { self.pool.on_canonical_state_change(update); } + + fn update_accounts(&self, accounts: Vec) { + self.pool.update_accounts(accounts); + } } impl Clone for Pool { diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 11d9871f4a1..e8b8a8f1ed8 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -5,33 +5,57 @@ use crate::{ traits::{CanonicalStateUpdate, ChangedAccount, TransactionPoolExt}, BlockInfo, TransactionPool, }; -use futures_util::{future::BoxFuture, FutureExt, Stream, StreamExt}; +use futures_util::{ + future::{BoxFuture, Fuse, FusedFuture}, + FutureExt, Stream, StreamExt, +}; use reth_primitives::{Address, BlockHash, BlockNumberOrTag, FromRecoveredTransaction}; use reth_provider::{BlockReaderIdExt, CanonStateNotification, PostState, StateProviderFactory}; +use reth_tasks::TaskSpawner; use std::{ borrow::Borrow, collections::HashSet, hash::{Hash, Hasher}, }; -use tracing::debug; +use tokio::sync::oneshot; +use tracing::{debug, trace}; + +/// Additional settings for maintaining the transaction pool +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MaintainPoolConfig { + /// Maximum (reorg) depth we handle when updating the transaction pool: `new.number - + /// last_seen.number` + /// + /// Default: 64 (2 epochs) + pub max_update_depth: u64, + /// Maximum number of accounts to reload from state at once when updating the transaction pool. + /// + /// Default: 250 + pub max_reload_accounts: usize, +} -/// Maximum (reorg) depth we handle when updating the transaction pool: `new.number - -/// last_seen.number` -const MAX_UPDATE_DEPTH: u64 = 64; +impl Default for MaintainPoolConfig { + fn default() -> Self { + Self { max_update_depth: 64, max_reload_accounts: 250 } + } +} /// Returns a spawnable future for maintaining the state of the transaction pool. -pub fn maintain_transaction_pool_future( +pub fn maintain_transaction_pool_future( client: Client, pool: P, events: St, + task_spawner: Tasks, + config: MaintainPoolConfig, ) -> BoxFuture<'static, ()> where - Client: StateProviderFactory + BlockReaderIdExt + Send + 'static, + Client: StateProviderFactory + BlockReaderIdExt + Clone + Send + 'static, P: TransactionPoolExt + 'static, St: Stream + Send + Unpin + 'static, + Tasks: TaskSpawner + 'static, { async move { - maintain_transaction_pool(client, pool, events).await; + maintain_transaction_pool(client, pool, events, task_spawner, config).await; } .boxed() } @@ -39,14 +63,20 @@ where /// Maintains the state of the transaction pool by handling new blocks and reorgs. /// /// This listens for any new blocks and reorgs and updates the transaction pool's state accordingly -#[allow(unused)] -pub async fn maintain_transaction_pool(client: Client, pool: P, mut events: St) -where - Client: StateProviderFactory + BlockReaderIdExt + Send + 'static, +pub async fn maintain_transaction_pool( + client: Client, + pool: P, + mut events: St, + task_spawner: Tasks, + config: MaintainPoolConfig, +) where + Client: StateProviderFactory + BlockReaderIdExt + Clone + Send + 'static, P: TransactionPoolExt + 'static, St: Stream + Send + Unpin + 'static, + Tasks: TaskSpawner + 'static, { - let mut metrics = MaintainPoolMetrics::default(); + let metrics = MaintainPoolMetrics::default(); + let MaintainPoolConfig { max_update_depth, max_reload_accounts } = config; // ensure the pool points to latest state if let Ok(Some(latest)) = client.block_by_number_or_tag(BlockNumberOrTag::Latest) { let latest = latest.seal_slow(); @@ -64,17 +94,100 @@ where // keeps track of the state of the pool wrt to blocks let mut maintained_state = MaintainedPoolState::InSync; + // the future that reloads accounts from state + let mut reload_accounts_fut = Fuse::terminated(); + + // The update loop that waits for new blocks and reorgs and performs pool updated // Listen for new chain events and derive the update action for the pool loop { + trace!(target = "txpool", state=?maintained_state, "awaiting new block or reorg"); + metrics.set_dirty_accounts_len(dirty_addresses.len()); + let pool_info = pool.block_info(); - let Some(event) = events.next().await else { break }; + // after performing a pool update after a new block we have some time to properly update + // dirty accounts and correct if the pool drifted from current state, for example after + // restart or a pipeline run + if maintained_state.is_drifted() { + // assuming all senders are dirty + dirty_addresses = pool.unique_senders(); + maintained_state = MaintainedPoolState::InSync; + } - let pool_info = pool.block_info(); + // if we have accounts that are out of sync with the pool, we reload them in chunks + if !dirty_addresses.is_empty() && reload_accounts_fut.is_terminated() { + let (tx, rx) = oneshot::channel(); + let c = client.clone(); + let at = pool_info.last_seen_block_hash; + let fut = if dirty_addresses.len() > max_reload_accounts { + // need to chunk accounts to reload + let accs_to_reload = + dirty_addresses.iter().copied().take(max_reload_accounts).collect::>(); + for acc in &accs_to_reload { + // make sure we remove them from the dirty set + dirty_addresses.remove(acc); + } + async move { + let res = load_accounts(c, at, accs_to_reload.into_iter()); + let _ = tx.send(res); + } + .boxed() + } else { + // can fetch all dirty accounts at once + let accs_to_reload = std::mem::take(&mut dirty_addresses); + async move { + let res = load_accounts(c, at, accs_to_reload.into_iter()); + let _ = tx.send(res); + } + .boxed() + }; + reload_accounts_fut = rx.fuse(); + task_spawner.spawn_blocking(fut); + } + + // outcomes of the futures we are waiting on + let mut event = None; + let mut reloaded = None; - // TODO from time to time re-check the unique accounts in the pool and remove and resync - // based on the tracked state + // select of account reloads and new canonical state updates which should arrive at the rate + // of the block time (12s) + tokio::select! { + res = &mut reload_accounts_fut => { + reloaded = Some(res); + } + ev = events.next() => { + if ev.is_none() { + // the stream ended, we are done + break; + } + event = ev; + } + } + // handle the result of the account reload + match reloaded { + Some(Ok(Ok(LoadedAccounts { accounts, failed_to_load }))) => { + // reloaded accounts successfully + // extend accounts we failed to load from database + dirty_addresses.extend(failed_to_load); + // update the pool with the loaded accounts + pool.update_accounts(accounts); + } + Some(Ok(Err(res))) => { + // Failed to load accounts from state + let (accs, err) = *res; + debug!(target = "txpool", ?err, "failed to load accounts"); + dirty_addresses.extend(accs); + } + Some(Err(_)) => { + // failed to receive the accounts, sender dropped, only possible if task panicked + maintained_state = MaintainedPoolState::Drifted; + } + None => {} + } + + // handle the new block or reorg + let Some(event) = event else { continue }; match event { CanonStateNotification::Reorg { old, new } => { let (old_blocks, old_state) = old.inner(); @@ -88,7 +201,7 @@ where new_first.parent_hash == pool_info.last_seen_block_hash) { // the new block points to a higher block than the oldest block in the old chain - maintained_state = MaintainedPoolState::Drift; + maintained_state = MaintainedPoolState::Drifted; } // base fee for the next block: `new_tip+1` @@ -108,7 +221,7 @@ where // for these we need to fetch the nonce+balance from the db at the new tip let mut changed_accounts = - match load_accounts(&client, new_tip.hash, missing_changed_acc) { + match load_accounts(client.clone(), new_tip.hash, missing_changed_acc) { Ok(LoadedAccounts { accounts, failed_to_load }) => { // extend accounts we failed to load from database dirty_addresses.extend(failed_to_load); @@ -118,6 +231,7 @@ where Err(err) => { let (addresses, err) = *err; debug!( + target = "txpool", ?err, "failed to load missing changed accounts at new tip: {:?}", new_tip.hash @@ -170,13 +284,20 @@ where let pending_block_base_fee = tip.next_block_base_fee().unwrap_or_default() as u128; let first_block = blocks.first(); + trace!( + target = "txpool", + first = first_block.number, + tip = tip.number, + pool_block = pool_info.last_seen_block_number, + "update pool on new commit" + ); // check if the depth is too large and should be skipped, this could happen after // initial sync or long re-sync let depth = tip.number.abs_diff(pool_info.last_seen_block_number); - if depth > MAX_UPDATE_DEPTH { - maintained_state = MaintainedPoolState::Drift; - debug!(?depth, "skipping deep canonical update"); + if depth > max_update_depth { + maintained_state = MaintainedPoolState::Drifted; + debug!(target = "txpool", ?depth, "skipping deep canonical update"); let info = BlockInfo { last_seen_block_hash: tip.hash, last_seen_block_number: tip.number, @@ -200,7 +321,7 @@ where // we received a new canonical chain commit but the commit is not canonical with // the pool's block, this could happen after initial sync or // long re-sync - maintained_state = MaintainedPoolState::Drift; + maintained_state = MaintainedPoolState::Drifted; } // Canonical update @@ -219,12 +340,19 @@ where /// Keeps track of the pool's state, whether the accounts in the pool are in sync with the actual /// state. -#[derive(Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq)] enum MaintainedPoolState { - /// Pool is assumed to be in sync with the state + /// Pool is assumed to be in sync with the current state InSync, /// Pool could be out of sync with the state - Drift, + Drifted, +} + +impl MaintainedPoolState { + /// Returns `true` if the pool is assumed to be out of sync with the current state. + fn is_drifted(&self) -> bool { + matches!(self, MaintainedPoolState::Drifted) + } } /// A unique ChangedAccount identified by its address that can be used for deduplication @@ -263,7 +391,7 @@ struct LoadedAccounts { /// /// Note: this expects _unique_ addresses fn load_accounts( - client: &Client, + client: Client, at: BlockHash, addresses: I, ) -> Result, reth_interfaces::Error)>> diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 0ccec210c13..a45d71662a7 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -10,7 +10,7 @@ use crate::{ TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction, }; use reth_primitives::{Address, TxHash}; -use std::{marker::PhantomData, sync::Arc}; +use std::{collections::HashSet, marker::PhantomData, sync::Arc}; use tokio::sync::{mpsc, mpsc::Receiver}; /// A [`TransactionPool`] implementation that does nothing. @@ -150,6 +150,10 @@ impl TransactionPool for NoopTransactionPool { ) -> Vec>> { vec![] } + + fn unique_senders(&self) -> HashSet
{ + Default::default() + } } /// A [`TransactionValidator`] that does nothing. diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index e7f167c82a6..560c6ee9948 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -90,12 +90,13 @@ use std::{ time::Instant, }; use tokio::sync::mpsc; -use tracing::debug; +use tracing::{debug, trace}; mod events; pub use events::{FullTransactionEvent, TransactionEvent}; mod listener; +use crate::pool::txpool::UpdateOutcome; pub use listener::{AllTransactionsEvents, TransactionEvents}; mod best; @@ -163,6 +164,11 @@ where self.identifiers.write().sender_id_or_create(addr) } + /// Returns all senders in the pool + pub(crate) fn unique_senders(&self) -> HashSet
{ + self.pool.read().unique_senders() + } + /// Converts the changed accounts to a map of sender ids to sender info (internal identifier /// used for accounts) fn changed_senders( @@ -243,6 +249,8 @@ where /// Updates the entire pool after a new block was executed. pub(crate) fn on_canonical_state_change(&self, update: CanonicalStateUpdate) { + trace!(target: "txpool", %update, "updating pool on canonical state change"); + let CanonicalStateUpdate { hash, number, @@ -264,6 +272,18 @@ where self.notify_on_new_state(outcome); } + /// Performs account updates on the pool. + /// + /// This will either promote or discard transactions based on the new account state. + pub(crate) fn update_accounts(&self, accounts: Vec) { + let changed_senders = self.changed_senders(accounts.into_iter()); + let UpdateOutcome { promoted, discarded } = + self.pool.write().update_accounts(changed_senders); + let mut listener = self.event_listener.write(); + promoted.iter().for_each(|tx| listener.pending(tx, None)); + discarded.iter().for_each(|tx| listener.discarded(tx)); + } + /// Add a single validated transaction into the pool. /// /// Note: this is only used internally by [`Self::add_transactions()`], all new transaction(s) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 6b1bad86442..b6a1bc174da 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -19,11 +19,11 @@ use crate::{ use fnv::FnvHashMap; use reth_primitives::{ constants::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}, - TxHash, H256, + Address, TxHash, H256, }; use std::{ cmp::Ordering, - collections::{btree_map::Entry, hash_map, BTreeMap, HashMap}, + collections::{btree_map::Entry, hash_map, BTreeMap, HashMap, HashSet}, fmt, ops::Bound::{Excluded, Unbounded}, sync::Arc, @@ -111,6 +111,11 @@ impl TxPool { &self.all_transactions } + /// Returns all senders in the pool + pub(crate) fn unique_senders(&self) -> HashSet
{ + self.all_transactions.txs.values().map(|tx| tx.transaction.sender()).collect() + } + /// Returns stats about the size of pool. pub(crate) fn size(&self) -> PoolSize { PoolSize { @@ -227,6 +232,22 @@ impl TxPool { self.all_transactions.txs_iter(sender).map(|(_, tx)| Arc::clone(&tx.transaction)).collect() } + /// Updates the transactions for the changed senders. + pub(crate) fn update_accounts( + &mut self, + changed_senders: HashMap, + ) -> UpdateOutcome { + // track changed accounts + self.sender_info.extend(changed_senders.clone()); + // Apply the state changes to the total set of transactions which triggers sub-pool updates. + let updates = self.all_transactions.update(changed_senders); + // Process the sub-pool updates + let update = self.process_updates(updates); + // update the metrics after the update + self.update_size_metrics(); + update + } + /// Updates the entire pool after a new block was mined. /// /// This removes all mined transactions, updates according to the new base fee and rechecks @@ -237,9 +258,6 @@ impl TxPool { mined_transactions: Vec, changed_senders: HashMap, ) -> OnNewCanonicalStateOutcome { - // track changed accounts - self.sender_info.extend(changed_senders.clone()); - // update block info let block_hash = block_info.last_seen_block_hash; self.all_transactions.set_block_info(block_info); @@ -252,14 +270,7 @@ impl TxPool { } } - // Apply the state changes to the total set of transactions which triggers sub-pool updates. - let updates = self.all_transactions.update(changed_senders); - - // Process the sub-pool updates - let UpdateOutcome { promoted, discarded } = self.process_updates(updates); - - // update the metrics after the update - self.update_size_metrics(); + let UpdateOutcome { promoted, discarded } = self.update_accounts(changed_senders); self.metrics.performed_state_updates.increment(1); @@ -1281,11 +1292,11 @@ impl PoolInternalTransaction { /// Tracks the result after updating the pool #[derive(Default, Debug)] -pub struct UpdateOutcome { +pub(crate) struct UpdateOutcome { /// transactions promoted to the ready queue - promoted: Vec, + pub(crate) promoted: Vec, /// transaction that failed and became discarded - discarded: Vec, + pub(crate) discarded: Vec, } /// Represents the outcome of a prune diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 611db300537..6b1dc062a36 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -11,7 +11,7 @@ use reth_primitives::{ }; use reth_rlp::Encodable; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fmt, pin::Pin, sync::Arc, @@ -242,6 +242,9 @@ pub trait TransactionPool: Send + Sync + Clone { &self, sender: Address, ) -> Vec>>; + + /// Returns a set of all senders of transactions in the pool + fn unique_senders(&self) -> HashSet
; } /// Extension for [TransactionPool] trait that allows to set the current block info. @@ -256,6 +259,9 @@ pub trait TransactionPoolExt: TransactionPool { /// For example the base fee of the pending block is determined after a block is mined which /// affects the dynamic fee requirement of pending transactions in the pool. fn on_canonical_state_change(&self, update: CanonicalStateUpdate); + + /// Updates the accounts in the pool + fn update_accounts(&self, accounts: Vec); } /// A Helper type that bundles all transactions in the pool. @@ -389,6 +395,13 @@ pub struct CanonicalStateUpdate { pub mined_transactions: Vec, } +impl fmt::Display for CanonicalStateUpdate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{{ hash: {}, number: {}, pending_block_base_fee: {}, changed_accounts: {}, mined_transactions: {} }}", + self.hash, self.number, self.pending_block_base_fee, self.changed_accounts.len(), self.mined_transactions.len()) + } +} + /// Represents a changed account #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub struct ChangedAccount { From 0d66f43d6ffec14c84f0423b7bc348b155c1432c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 15 Jul 2023 18:48:48 +0200 Subject: [PATCH 101/150] feat(txpool): add best_with_base_fee (#3737) --- crates/transaction-pool/src/lib.rs | 7 ++++ crates/transaction-pool/src/noop.rs | 7 ++++ crates/transaction-pool/src/ordering.rs | 6 +++ crates/transaction-pool/src/pool/best.rs | 44 ++++++++++++++++++++- crates/transaction-pool/src/pool/mod.rs | 10 +++++ crates/transaction-pool/src/pool/parked.rs | 32 ++++++++++++--- crates/transaction-pool/src/pool/pending.rs | 38 ++++++++++++++++++ crates/transaction-pool/src/pool/txpool.rs | 25 ++++++++++++ crates/transaction-pool/src/traits.rs | 9 +++++ 9 files changed, 171 insertions(+), 7 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index a87e0600752..93ebe6fd1a0 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -397,6 +397,13 @@ where Box::new(self.pool.best_transactions()) } + fn best_transactions_with_base_fee( + &self, + base_fee: u128, + ) -> Box>>> { + self.pool.best_transactions_with_base_fee(base_fee) + } + fn pending_transactions(&self) -> Vec>> { self.pool.pending_transactions() } diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index a45d71662a7..c03a475e50a 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -110,6 +110,13 @@ impl TransactionPool for NoopTransactionPool { Box::new(std::iter::empty()) } + fn best_transactions_with_base_fee( + &self, + _: u128, + ) -> Box>>> { + Box::new(std::iter::empty()) + } + fn pending_transactions(&self) -> Vec>> { vec![] } diff --git a/crates/transaction-pool/src/ordering.rs b/crates/transaction-pool/src/ordering.rs index 34044588de2..40ee1700661 100644 --- a/crates/transaction-pool/src/ordering.rs +++ b/crates/transaction-pool/src/ordering.rs @@ -45,3 +45,9 @@ impl Default for GasCostOrdering { Self(Default::default()) } } + +impl Clone for GasCostOrdering { + fn clone(&self) -> Self { + Self::default() + } +} diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index c0ed8786a31..fc6ff559566 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -1,7 +1,7 @@ use crate::{ identifier::TransactionId, pool::pending::{PendingTransaction, PendingTransactionRef}, - TransactionOrdering, ValidPoolTransaction, + PoolTransaction, TransactionOrdering, ValidPoolTransaction, }; use reth_primitives::H256 as TxHash; use std::{ @@ -10,6 +10,40 @@ use std::{ }; use tracing::debug; +/// An iterator that returns transactions that can be executed on the current state (*best* +/// transactions). +/// +/// This is a wrapper around [`BestTransactions`] that also enforces a specific basefee. +/// +/// This iterator guarantees that all transaction it returns satisfy the base fee. +pub(crate) struct BestTransactionsWithBasefee { + pub(crate) best: BestTransactions, + pub(crate) base_fee: u128, +} + +impl crate::traits::BestTransactions for BestTransactionsWithBasefee { + fn mark_invalid(&mut self, tx: &Self::Item) { + BestTransactions::mark_invalid(&mut self.best, tx) + } +} + +impl Iterator for BestTransactionsWithBasefee { + type Item = Arc>; + + fn next(&mut self) -> Option { + // find the next transaction that satisfies the base fee + loop { + let best = self.best.next()?; + if best.transaction.max_fee_per_gas() < self.base_fee { + // tx violates base fee, mark it as invalid and continue + crate::traits::BestTransactions::mark_invalid(self, &best); + } else { + return Some(best) + } + } + } +} + /// An iterator that returns transactions that can be executed on the current state (*best* /// transactions). /// @@ -35,6 +69,14 @@ impl BestTransactions { pub(crate) fn mark_invalid(&mut self, tx: &Arc>) { self.invalid.insert(*tx.hash()); } + + /// Returns the ancestor the given transaction, the transaction with `nonce - 1`. + /// + /// Note: for a transaction with nonce higher than the current on chain nonce this will always + /// return an ancestor since all transaction in this pool are gapless. + pub(crate) fn ancestor(&self, id: &TransactionId) -> Option<&Arc>> { + self.all.get(&id.unchecked_ancestor()?) + } } impl crate::traits::BestTransactions for BestTransactions { diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 560c6ee9948..2c055bf9f75 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -462,6 +462,16 @@ where self.pool.read().best_transactions() } + /// Returns an iterator that yields transactions that are ready to be included in the block with + /// the given base fee. + pub(crate) fn best_transactions_with_base_fee( + &self, + base_fee: u128, + ) -> Box>>> + { + self.pool.read().best_transactions_with_base_fee(base_fee) + } + /// Returns all transactions from the pending sub-pool pub(crate) fn pending_transactions(&self) -> Vec>> { self.pool.read().pending_transactions() diff --git a/crates/transaction-pool/src/pool/parked.rs b/crates/transaction-pool/src/pool/parked.rs index d6a62685cfb..0916886d130 100644 --- a/crates/transaction-pool/src/pool/parked.rs +++ b/crates/transaction-pool/src/pool/parked.rs @@ -110,13 +110,24 @@ impl ParkedPool { } impl ParkedPool> { - /// Removes all transactions and their dependent transaction from the subpool that no longer - /// satisfy the given basefee. + /// Returns all transactions that satisfy the given basefee. /// - /// Note: the transactions are not returned in a particular order. - pub(crate) fn enforce_basefee(&mut self, basefee: u128) -> Vec>> { - let mut to_remove = Vec::new(); + /// Note: this does _not_ remove the transactions + pub(crate) fn satisfy_base_fee_transactions( + &self, + basefee: u128, + ) -> Vec>> { + let ids = self.satisfy_base_fee_ids(basefee); + let mut txs = Vec::with_capacity(ids.len()); + for id in ids { + txs.push(self.by_id.get(&id).expect("transaction exists").transaction.clone().into()); + } + txs + } + /// Returns all transactions that satisfy the given basefee. + fn satisfy_base_fee_ids(&self, basefee: u128) -> Vec { + let mut transactions = Vec::new(); { let mut iter = self.by_id.iter().peekable(); @@ -130,10 +141,19 @@ impl ParkedPool> { iter.next(); } } else { - to_remove.push(*id); + transactions.push(*id); } } } + transactions + } + + /// Removes all transactions and their dependent transaction from the subpool that no longer + /// satisfy the given basefee. + /// + /// Note: the transactions are not returned in a particular order. + pub(crate) fn enforce_basefee(&mut self, basefee: u128) -> Vec>> { + let to_remove = self.satisfy_base_fee_ids(basefee); let mut removed = Vec::with_capacity(to_remove.len()); for id in to_remove { diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index 1d6d841b8f0..71b873a3022 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -4,6 +4,7 @@ use crate::{ TransactionOrdering, ValidPoolTransaction, }; +use crate::pool::best::BestTransactionsWithBasefee; use std::{ cmp::Ordering, collections::{BTreeMap, BTreeSet}, @@ -84,6 +85,43 @@ impl PendingPool { } } + /// Same as `best` but only returns transactions that satisfy the given basefee. + pub(crate) fn best_with_basefee(&self, base_fee: u128) -> BestTransactionsWithBasefee { + BestTransactionsWithBasefee { best: self.best(), base_fee } + } + + /// Same as `best` but also includes the given unlocked transactions. + /// + /// This mimics the [Self::add_transaction] method, but does not insert the transactions into + /// pool but only into the returned iterator. + /// + /// Note: this does not insert the unlocked transactions into the pool. + /// + /// # Panics + /// + /// if the transaction is already included + pub(crate) fn best_with_unlocked( + &self, + unlocked: Vec>>, + ) -> BestTransactions { + let mut best = self.best(); + let mut submission_id = self.submission_id; + for tx in unlocked { + submission_id += 1; + debug_assert!(!best.all.contains_key(tx.id()), "transaction already included"); + let priority = self.ordering.priority(&tx.transaction); + let tx_id = *tx.id(); + let transaction = PendingTransactionRef { submission_id, transaction: tx, priority }; + if best.ancestor(&tx_id).is_none() { + best.independent.insert(transaction.clone()); + } + let transaction = Arc::new(PendingTransaction { transaction }); + best.all.insert(tx_id, transaction); + } + + best + } + /// Returns an iterator over all transactions in the pool pub(crate) fn all( &self, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index b6a1bc174da..e3820ee6557 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -191,6 +191,31 @@ impl TxPool { self.pending_pool.best() } + /// Returns an iterator that yields transactions that are ready to be included in the block with + /// the given base fee. + pub(crate) fn best_transactions_with_base_fee( + &self, + basefee: u128, + ) -> Box>>> + { + match basefee.cmp(&self.all_transactions.pending_basefee) { + Ordering::Equal => { + // fee unchanged, nothing to shift + Box::new(self.best_transactions()) + } + Ordering::Greater => { + // base fee increased, we only need to enforces this on the pending pool + Box::new(self.pending_pool.best_with_basefee(basefee)) + } + Ordering::Less => { + // base fee decreased, we need to move transactions from the basefee pool to the + // pending pool + let unlocked = self.basefee_pool.satisfy_base_fee_transactions(basefee); + Box::new(self.pending_pool.best_with_unlocked(unlocked)) + } + } + } + /// Returns all transactions from the pending sub-pool pub(crate) fn pending_transactions(&self) -> Vec>> { self.pending_pool.all().collect() diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 6b1dc062a36..830971c9d9a 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -175,6 +175,15 @@ pub trait TransactionPool: Send + Sync + Clone { &self, ) -> Box>>>; + /// Returns an iterator that yields transactions that are ready for block production with the + /// given base fee. + /// + /// Consumer: Block production + fn best_transactions_with_base_fee( + &self, + base_fee: u128, + ) -> Box>>>; + /// Returns all transactions that can be included in the next block. /// /// This is primarily used for the `txpool_` RPC namespace: which distinguishes between `pending` and `queued` transactions, where `pending` are transactions ready for inclusion in the next block and `queued` are transactions that are ready for inclusion in future blocks. From ec162dfd4f3d79f095cfe1e0bedc3e74c450a57b Mon Sep 17 00:00:00 2001 From: rakita Date: Sat, 15 Jul 2023 21:36:27 +0200 Subject: [PATCH 102/150] bug(stages): TxLookup/Sender stages fix range if there is no tx (#3479) --- bin/reth/src/stage/drop.rs | 8 +++ crates/stages/src/stage.rs | 56 +++++++++++++-------- crates/stages/src/stages/sender_recovery.rs | 2 +- crates/stages/src/test_utils/test_db.rs | 20 +++++--- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/bin/reth/src/stage/drop.rs b/bin/reth/src/stage/drop.rs index 771dff1d204..33298d287fd 100644 --- a/bin/reth/src/stage/drop.rs +++ b/bin/reth/src/stage/drop.rs @@ -155,6 +155,14 @@ impl Command { )?; insert_genesis_header::(tx, self.chain)?; } + StageEnum::TxLookup => { + tx.clear::()?; + tx.put::( + StageId::TransactionLookup.to_string(), + Default::default(), + )?; + insert_genesis_header::(tx, self.chain)?; + } _ => { info!("Nothing to do for stage {:?}", self.stage); return Ok(()) diff --git a/crates/stages/src/stage.rs b/crates/stages/src/stage.rs index 7580c6bab42..e369af06023 100644 --- a/crates/stages/src/stage.rs +++ b/crates/stages/src/stage.rs @@ -1,14 +1,14 @@ use crate::error::StageError; use async_trait::async_trait; -use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx}; +use reth_db::database::Database; use reth_primitives::{ stage::{StageCheckpoint, StageId}, BlockNumber, TxNumber, }; -use reth_provider::{BlockReader, DatabaseProviderRW, ProviderError}; +use reth_provider::{BlockReader, DatabaseProviderRW, ProviderError, TransactionsProvider}; use std::{ cmp::{max, min}, - ops::RangeInclusive, + ops::{Range, RangeInclusive}, }; /// Stage execution input, see [Stage::execute]. @@ -77,30 +77,46 @@ impl ExecInput { &self, provider: &DatabaseProviderRW<'_, DB>, tx_threshold: u64, - ) -> Result<(RangeInclusive, RangeInclusive, bool), StageError> { + ) -> Result<(Range, RangeInclusive, bool), StageError> { let start_block = self.next_block(); + let target_block = self.target(); + let start_block_body = provider .block_body_indices(start_block)? .ok_or(ProviderError::BlockBodyIndicesNotFound(start_block))?; + let first_tx_num = start_block_body.first_tx_num(); - let target_block = self.target(); + let target_block_body = provider + .block_body_indices(target_block)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(target_block))?; - let first_tx_number = start_block_body.first_tx_num(); - let mut last_tx_number = start_block_body.last_tx_num(); - let mut end_block_number = start_block; - let mut body_indices_cursor = - provider.tx_ref().cursor_read::()?; - for entry in body_indices_cursor.walk_range(start_block..=target_block)? { - let (block, body) = entry?; - last_tx_number = body.last_tx_num(); - end_block_number = block; - let tx_count = (first_tx_number..=last_tx_number).count() as u64; - if tx_count > tx_threshold { - break - } + // number of transactions left to execute. + let all_tx_cnt = target_block_body.next_tx_num() - first_tx_num; + + if all_tx_cnt == 0 { + // if there is no more transaction return back. + return Ok((first_tx_num..first_tx_num, start_block..=target_block, true)) } - let is_final_range = end_block_number >= target_block; - Ok((first_tx_number..=last_tx_number, start_block..=end_block_number, is_final_range)) + + // get block of this tx + let (end_block, is_final_range, next_tx_num) = if all_tx_cnt <= tx_threshold { + (target_block, true, target_block_body.next_tx_num()) + } else { + // get tx block number. next_tx_num in this case will be less thean all_tx_cnt. + // So we are sure that transaction must exist. + let end_block_number = provider + .transaction_block(first_tx_num + tx_threshold)? + .expect("block of tx must exist"); + // we want to get range of all transactions of this block, so we are fetching block + // body. + let end_block_body = provider + .block_body_indices(end_block_number)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(target_block))?; + (end_block_number, false, end_block_body.next_tx_num()) + }; + + let tx_range = first_tx_num..next_tx_num; + Ok((tx_range, start_block..=end_block, is_final_range)) } } diff --git a/crates/stages/src/stages/sender_recovery.rs b/crates/stages/src/stages/sender_recovery.rs index e6294b3a861..8872c8138ff 100644 --- a/crates/stages/src/stages/sender_recovery.rs +++ b/crates/stages/src/stages/sender_recovery.rs @@ -85,7 +85,7 @@ impl Stage for SenderRecoveryStage { // Acquire the cursor over the transactions let mut tx_cursor = tx.cursor_read::>()?; // Walk the transactions from start to end index (inclusive) - let raw_tx_range = RawKey::new(*tx_range.start())..=RawKey::new(*tx_range.end()); + let raw_tx_range = RawKey::new(tx_range.start)..RawKey::new(tx_range.end); let tx_walker = tx_cursor.walk_range(raw_tx_range)?; // Iterate over transactions in chunks diff --git a/crates/stages/src/test_utils/test_db.rs b/crates/stages/src/test_utils/test_db.rs index e3ad10d15c3..ddab5504073 100644 --- a/crates/stages/src/test_utils/test_db.rs +++ b/crates/stages/src/test_utils/test_db.rs @@ -246,13 +246,19 @@ impl TestTransaction { blocks.into_iter().try_for_each(|block| { Self::insert_header(tx, &block.header)?; // Insert into body tables. - tx.put::( - block.number, - StoredBlockBodyIndices { - first_tx_num: next_tx_num, - tx_count: block.body.len() as u64, - }, - )?; + let block_body_indices = StoredBlockBodyIndices { + first_tx_num: next_tx_num, + tx_count: block.body.len() as u64, + }; + + if !block.body.is_empty() { + tx.put::( + block_body_indices.last_tx_num(), + block.number, + )?; + } + tx.put::(block.number, block_body_indices)?; + block.body.iter().try_for_each(|body_tx| { tx.put::(next_tx_num, body_tx.clone().into())?; next_tx_num += 1; From e98e9675d5bdec55bcc845bd0db6b923678a178e Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Sat, 15 Jul 2023 15:36:39 -0400 Subject: [PATCH 103/150] feat: better blocksize heuristics (#3748) --- book/run/config.md | 6 +- crates/config/src/config.rs | 8 +- crates/interfaces/src/p2p/bodies/response.rs | 9 +- crates/net/downloaders/src/bodies/bodies.rs | 10 +- crates/net/downloaders/src/bodies/request.rs | 27 +++- crates/net/downloaders/src/metrics.rs | 13 ++ crates/primitives/src/block.rs | 36 +++++ crates/primitives/src/header.rs | 33 ++++- .../primitives/src/transaction/access_list.rs | 18 +++ crates/primitives/src/transaction/mod.rs | 69 +++++++++ .../primitives/src/transaction/signature.rs | 23 +++ crates/primitives/src/withdrawal.rs | 8 + etc/grafana/dashboards/overview.json | 137 ++++++++++++++++++ 13 files changed, 374 insertions(+), 23 deletions(-) diff --git a/book/run/config.md b/book/run/config.md index 43319d9d2fa..3d5c6ce5d6b 100644 --- a/book/run/config.md +++ b/book/run/config.md @@ -89,7 +89,7 @@ downloader_request_limit = 200 # # A lower value means more frequent disk I/O (writes), but also # lowers memory usage. -downloader_stream_batch_size = 10000 +downloader_stream_batch_size = 1000 # The size of the internal block buffer in bytes. # # A bigger buffer means that bandwidth can be saturated for longer periods, @@ -98,8 +98,8 @@ downloader_stream_batch_size = 10000 # If the buffer is full, no more requests will be made to peers until # space is made for new blocks in the buffer. # -# Defaults to around 4GB. -downloader_max_buffered_blocks_size_bytes = 4294967296 +# Defaults to around 2GB. +downloader_max_buffered_blocks_size_bytes = 2147483648 # The minimum and maximum number of concurrent requests to have in flight at a time. # # The downloader uses these as best effort targets, which means that the number diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index 659df006b5e..d46e665430c 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -146,11 +146,11 @@ pub struct BodiesConfig { pub downloader_request_limit: u64, /// The maximum number of block bodies returned at once from the stream /// - /// Default: 10_000 + /// Default: 1_000 pub downloader_stream_batch_size: usize, /// The size of the internal block buffer in bytes. /// - /// Default: 4GB + /// Default: 2GB pub downloader_max_buffered_blocks_size_bytes: usize, /// The minimum number of requests to send concurrently. /// @@ -167,8 +167,8 @@ impl Default for BodiesConfig { fn default() -> Self { Self { downloader_request_limit: 200, - downloader_stream_batch_size: 10_000, - downloader_max_buffered_blocks_size_bytes: 4 * 1024 * 1024 * 1024, // ~4GB + downloader_stream_batch_size: 1_000, + downloader_max_buffered_blocks_size_bytes: 2 * 1024 * 1024 * 1024, // ~2GB downloader_min_concurrent_requests: 5, downloader_max_concurrent_requests: 100, } diff --git a/crates/interfaces/src/p2p/bodies/response.rs b/crates/interfaces/src/p2p/bodies/response.rs index a3033bd4ddf..2b32b70097f 100644 --- a/crates/interfaces/src/p2p/bodies/response.rs +++ b/crates/interfaces/src/p2p/bodies/response.rs @@ -18,13 +18,12 @@ impl BlockResponse { } } - /// Returns the total number of bytes of all transactions input data in the block + /// Calculates a heuristic for the in-memory size of the [BlockResponse]. + #[inline] pub fn size(&self) -> usize { match self { - BlockResponse::Full(block) => { - block.body.iter().map(|tx| tx.transaction.input().len()).sum() - } - BlockResponse::Empty(_) => 0, + BlockResponse::Full(block) => SealedBlock::size(block), + BlockResponse::Empty(header) => SealedHeader::size(header), } } diff --git a/crates/net/downloaders/src/bodies/bodies.rs b/crates/net/downloaders/src/bodies/bodies.rs index 7a0c3aa4b74..e615520e33e 100644 --- a/crates/net/downloaders/src/bodies/bodies.rs +++ b/crates/net/downloaders/src/bodies/bodies.rs @@ -19,6 +19,7 @@ use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use std::{ cmp::Ordering, collections::BinaryHeap, + mem, ops::RangeInclusive, pin::Pin, sync::Arc, @@ -225,13 +226,16 @@ where self.metrics.buffered_responses.decrement(1.); self.buffered_blocks_size_bytes -= resp.size(); self.metrics.buffered_blocks.decrement(resp.len() as f64); - self.metrics.buffered_blocks_size_bytes.set(resp.size() as f64); + self.metrics.buffered_blocks_size_bytes.set(self.buffered_blocks_size_bytes as f64); Some(resp) } /// Adds a new response to the internal buffer fn buffer_bodies_response(&mut self, response: Vec) { - let size = response.iter().map(|b| b.size()).sum::(); + // take into account capacity + let size = response.iter().map(BlockResponse::size).sum::() + + response.capacity() * mem::size_of::(); + let response = OrderedBodiesResponse { resp: response, size }; let response_len = response.len(); @@ -516,7 +520,7 @@ impl Default for BodiesDownloaderBuilder { Self { request_limit: 200, stream_batch_size: 10_000, - max_buffered_blocks_size_bytes: 4 * 1024 * 1024 * 1024, // ~4GB + max_buffered_blocks_size_bytes: 2 * 1024 * 1024 * 1024, // ~2GB concurrent_requests_range: 5..=100, } } diff --git a/crates/net/downloaders/src/bodies/request.rs b/crates/net/downloaders/src/bodies/request.rs index a71b81f33e7..8698e881ebb 100644 --- a/crates/net/downloaders/src/bodies/request.rs +++ b/crates/net/downloaders/src/bodies/request.rs @@ -1,4 +1,4 @@ -use crate::metrics::BodyDownloaderMetrics; +use crate::metrics::{BodyDownloaderMetrics, ResponseMetrics}; use futures::{Future, FutureExt}; use reth_interfaces::{ consensus::{Consensus as ConsensusTrait, Consensus}, @@ -11,6 +11,7 @@ use reth_interfaces::{ use reth_primitives::{BlockBody, PeerId, SealedBlock, SealedHeader, WithPeerId, H256}; use std::{ collections::VecDeque, + mem, pin::Pin, sync::Arc, task::{ready, Context, Poll}, @@ -39,6 +40,9 @@ pub(crate) struct BodiesRequestFuture { client: Arc, consensus: Arc, metrics: BodyDownloaderMetrics, + /// Metrics for individual responses. This can be used to observe how the size (in bytes) of + /// responses change while bodies are being downloaded. + response_metrics: ResponseMetrics, // Headers to download. The collection is shrunk as responses are buffered. pending_headers: VecDeque, /// Internal buffer for all blocks @@ -62,6 +66,7 @@ where client, consensus, metrics, + response_metrics: Default::default(), pending_headers: Default::default(), buffer: Default::default(), last_request_len: None, @@ -153,8 +158,11 @@ where /// This method removes headers from the internal collection. /// If the response fails validation, then the header will be put back. fn try_buffer_blocks(&mut self, bodies: Vec) -> DownloadResult<()> { + let bodies_capacity = bodies.capacity(); + let bodies_len = bodies.len(); let mut bodies = bodies.into_iter().peekable(); + let mut total_size = bodies_capacity * mem::size_of::(); while bodies.peek().is_some() { let next_header = match self.pending_headers.pop_front() { Some(header) => header, @@ -162,15 +170,16 @@ where }; if next_header.is_empty() { + // increment empty block body metric + total_size += mem::size_of::(); self.buffer.push(BlockResponse::Empty(next_header)); } else { let next_body = bodies.next().unwrap(); - let block = SealedBlock { - header: next_header, - body: next_body.transactions, - ommers: next_body.ommers, - withdrawals: next_body.withdrawals, - }; + + // increment full block body metric + total_size += next_body.size(); + + let block = SealedBlock::new(next_header, next_body); if let Err(error) = self.consensus.validate_block(&block) { // Body is invalid, put the header back and return an error @@ -183,6 +192,10 @@ where } } + // Increment per-response metric + self.response_metrics.response_size_bytes.set(total_size as f64); + self.response_metrics.response_length.set(bodies_len as f64); + Ok(()) } } diff --git a/crates/net/downloaders/src/metrics.rs b/crates/net/downloaders/src/metrics.rs index 38fc642eb31..a227f38f8b7 100644 --- a/crates/net/downloaders/src/metrics.rs +++ b/crates/net/downloaders/src/metrics.rs @@ -62,6 +62,19 @@ impl BodyDownloaderMetrics { } } +/// Metrics for an individual response, i.e. the size in bytes, and length (number of bodies) in the +/// response. +/// +/// These metrics will be initialized with the `downloaders.bodies.response` scope. +#[derive(Clone, Metrics)] +#[metrics(scope = "downloaders.bodies.response")] +pub struct ResponseMetrics { + /// The size (in bytes) of an individual bodies response received by the downloader. + pub response_size_bytes: Gauge, + /// The number of bodies in an individual bodies response received by the downloader. + pub response_length: Gauge, +} + /// Common header downloader metrics. /// /// These metrics will be initialized with the `downloaders.headers` scope. diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index 6d652f92e47..0c9866ba28e 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -59,6 +59,16 @@ impl Block { BlockWithSenders { block: self, senders } } + + /// Calculates a heuristic for the in-memory size of the [Block]. + #[inline] + pub fn size(&self) -> usize { + self.header.size() + + // take into account capacity + self.body.iter().map(TransactionSigned::size).sum::() + self.body.capacity() * std::mem::size_of::() + + self.ommers.iter().map(Header::size).sum::() + self.ommers.capacity() * std::mem::size_of::
() + + self.withdrawals.as_ref().map(|w| w.iter().map(Withdrawal::size).sum::() + w.capacity() * std::mem::size_of::()).unwrap_or(std::mem::size_of::>>()) + } } impl Deref for Block { @@ -178,6 +188,16 @@ impl SealedBlock { withdrawals: self.withdrawals, } } + + /// Calculates a heuristic for the in-memory size of the [SealedBlock]. + #[inline] + pub fn size(&self) -> usize { + self.header.size() + + // take into account capacity + self.body.iter().map(TransactionSigned::size).sum::() + self.body.capacity() * std::mem::size_of::() + + self.ommers.iter().map(Header::size).sum::() + self.ommers.capacity() * std::mem::size_of::
() + + self.withdrawals.as_ref().map(|w| w.iter().map(Withdrawal::size).sum::() + w.capacity() * std::mem::size_of::()).unwrap_or(std::mem::size_of::>>()) + } } impl From for Block { @@ -819,6 +839,22 @@ impl BlockBody { withdrawals_root: self.calculate_withdrawals_root(), } } + + /// Calculates a heuristic for the in-memory size of the [BlockBody]. + #[inline] + pub fn size(&self) -> usize { + self.transactions.iter().map(TransactionSigned::size).sum::() + + self.transactions.capacity() * std::mem::size_of::() + + self.ommers.iter().map(Header::size).sum::() + + self.ommers.capacity() * std::mem::size_of::
() + + self.withdrawals + .as_ref() + .map(|w| { + w.iter().map(Withdrawal::size).sum::() + + w.capacity() * std::mem::size_of::() + }) + .unwrap_or(std::mem::size_of::>>()) + } } /// A struct that represents roots associated with a block body. This can be used to correlate diff --git a/crates/primitives/src/header.rs b/crates/primitives/src/header.rs index 54b81649b84..ba8e8f31deb 100644 --- a/crates/primitives/src/header.rs +++ b/crates/primitives/src/header.rs @@ -9,7 +9,10 @@ use bytes::{Buf, BufMut, BytesMut}; use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{length_of_length, Decodable, Encodable, EMPTY_STRING_CODE}; use serde::{Deserialize, Serialize}; -use std::ops::{Deref, DerefMut}; +use std::{ + mem, + ops::{Deref, DerefMut}, +}; /// Describes the current head block. /// @@ -180,6 +183,28 @@ impl Header { self.seal(hash) } + /// Calculate a heuristic for the in-memory size of the [Header]. + #[inline] + pub fn size(&self) -> usize { + mem::size_of::() + // parent hash + mem::size_of::() + // ommers hash + mem::size_of::() + // beneficiary + mem::size_of::() + // state root + mem::size_of::() + // transactions root + mem::size_of::() + // receipts root + mem::size_of::>() + // withdrawals root + mem::size_of::() + // logs bloom + mem::size_of::() + // difficulty + mem::size_of::() + // number + mem::size_of::() + // gas limit + mem::size_of::() + // gas used + mem::size_of::() + // timestamp + mem::size_of::() + // mix hash + mem::size_of::() + // nonce + mem::size_of::>() + // base fee per gas + self.extra_data.len() // extra data + } + fn header_payload_length(&self) -> usize { let mut length = 0; length += self.parent_hash.length(); @@ -331,6 +356,12 @@ impl SealedHeader { pub fn num_hash(&self) -> BlockNumHash { BlockNumHash::new(self.number, self.hash) } + + /// Calculates a heuristic for the in-memory size of the [SealedHeader]. + #[inline] + pub fn size(&self) -> usize { + self.header.size() + mem::size_of::() + } } #[cfg(any(test, feature = "arbitrary"))] diff --git a/crates/primitives/src/transaction/access_list.rs b/crates/primitives/src/transaction/access_list.rs index eaa60b2603f..a86b33b69eb 100644 --- a/crates/primitives/src/transaction/access_list.rs +++ b/crates/primitives/src/transaction/access_list.rs @@ -1,3 +1,5 @@ +use std::mem; + use crate::{Address, H256}; use reth_codecs::{main_codec, Compact}; use reth_rlp::{RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper}; @@ -22,6 +24,14 @@ pub struct AccessListItem { pub storage_keys: Vec, } +impl AccessListItem { + /// Calculates a heuristic for the in-memory size of the [AccessListItem]. + #[inline] + pub fn size(&self) -> usize { + mem::size_of::
() + self.storage_keys.capacity() * mem::size_of::() + } +} + /// AccessList as defined in EIP-2930 #[main_codec(rlp)] #[derive(Clone, Debug, PartialEq, Eq, Hash, Default, RlpDecodableWrapper, RlpEncodableWrapper)] @@ -48,6 +58,14 @@ impl AccessList { }) .collect() } + + /// Calculates a heuristic for the in-memory size of the [AccessList]. + #[inline] + pub fn size(&self) -> usize { + // take into account capacity + self.0.iter().map(AccessListItem::size).sum::() + + self.0.capacity() * mem::size_of::() + } } /// Access list with gas used appended. diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 6bfc0489f40..4e02a42499e 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1,3 +1,5 @@ +use std::mem; + use crate::{ compression::{TRANSACTION_COMPRESSOR, TRANSACTION_DECOMPRESSOR}, keccak256, Address, Bytes, ChainId, TxHash, H256, @@ -69,6 +71,20 @@ pub struct TxLegacy { pub input: Bytes, } +impl TxLegacy { + /// Calculates a heuristic for the in-memory size of the [TxLegacy] transaction. + #[inline] + fn size(&self) -> usize { + mem::size_of::>() + // chain_id + mem::size_of::() + // nonce + mem::size_of::() + // gas_price + mem::size_of::() + // gas_limit + self.to.size() + // to + mem::size_of::() + // value + self.input.len() // input + } +} + /// Transaction with an [`AccessList`] ([EIP-2930](https://eips.ethereum.org/EIPS/eip-2930)). #[main_codec] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] @@ -117,6 +133,21 @@ pub struct TxEip2930 { pub input: Bytes, } +impl TxEip2930 { + /// Calculates a heuristic for the in-memory size of the [TxEip2930] transaction. + #[inline] + pub fn size(&self) -> usize { + mem::size_of::() + // chain_id + mem::size_of::() + // nonce + mem::size_of::() + // gas_price + mem::size_of::() + // gas_limit + self.to.size() + // to + mem::size_of::() + // value + self.access_list.size() + // access_list + self.input.len() // input + } +} + /// A transaction with a priority fee ([EIP-1559](https://eips.ethereum.org/EIPS/eip-1559)). #[main_codec] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] @@ -173,6 +204,22 @@ pub struct TxEip1559 { pub input: Bytes, } +impl TxEip1559 { + /// Calculates a heuristic for the in-memory size of the [TxEip1559] transaction. + #[inline] + pub fn size(&self) -> usize { + mem::size_of::() + // chain_id + mem::size_of::() + // nonce + mem::size_of::() + // gas_limit + mem::size_of::() + // max_fee_per_gas + mem::size_of::() + // max_priority_fee_per_gas + self.to.size() + // to + mem::size_of::() + // value + self.access_list.size() + // access_list + self.input.len() // input + } +} + /// A raw transaction. /// /// Transaction types were introduced in [EIP-2718](https://eips.ethereum.org/EIPS/eip-2718). @@ -265,6 +312,16 @@ impl Transaction { Transaction::Deposit(tx) => tx.input = input, } } + + /// Calculates a heuristic for the in-memory size of the [Transaction]. + #[inline] + fn size(&self) -> usize { + match self { + Transaction::Legacy(tx) => tx.size(), + Transaction::Eip2930(tx) => tx.size(), + Transaction::Eip1559(tx) => tx.size(), + } + } } impl Compact for Transaction { @@ -819,6 +876,12 @@ impl TransactionKind { TransactionKind::Call(to) => Some(to), } } + + /// Calculates a heuristic for the in-memory size of the [TransactionKind]. + #[inline] + fn size(self) -> usize { + mem::size_of::() + } } impl Compact for TransactionKind { @@ -1143,6 +1206,12 @@ impl TransactionSigned { initial_tx } + /// Calculate a heuristic for the in-memory size of the [TransactionSigned]. + #[inline] + pub fn size(&self) -> usize { + mem::size_of::() + self.transaction.size() + self.signature.size() + } + /// Decodes legacy transaction from the data buffer. /// /// This expects `rlp(legacy_tx)` diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index d5371450f0a..eff0c56b392 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -141,6 +141,12 @@ impl Signature { sig[64] = v; sig } + + /// Calculates a heuristic for the in-memory size of the [Signature]. + #[inline] + pub fn size(&self) -> usize { + std::mem::size_of::() + } } #[cfg(test)] @@ -225,4 +231,21 @@ mod tests { let expected = Address::from_str("0x9d8a62f656a8d1615c1294fd71e9cfb3e4855a4f").unwrap(); assert_eq!(expected, signer); } + + #[test] + fn ensure_size_equals_sum_of_fields() { + let signature = Signature { + r: U256::from_str( + "18515461264373351373200002665853028612451056578545711640558177340181847433846", + ) + .unwrap(), + s: U256::from_str( + "46948507304638947509940763649030358759909902576025900602547168820602576006531", + ) + .unwrap(), + odd_y_parity: false, + }; + + assert!(signature.size() >= 65); + } } diff --git a/crates/primitives/src/withdrawal.rs b/crates/primitives/src/withdrawal.rs index d8d0145b4a2..098110803c4 100644 --- a/crates/primitives/src/withdrawal.rs +++ b/crates/primitives/src/withdrawal.rs @@ -1,3 +1,5 @@ +use std::mem; + use crate::{constants::GWEI_TO_WEI, serde_helper::u64_hex, Address, U256}; use reth_codecs::{main_codec, Compact}; use reth_rlp::{RlpDecodable, RlpEncodable}; @@ -24,6 +26,12 @@ impl Withdrawal { pub fn amount_wei(&self) -> U256 { U256::from(self.amount) * U256::from(GWEI_TO_WEI) } + + /// Calculate a heuristic for the in-memory size of the [Withdrawal]. + #[inline] + pub fn size(&self) -> usize { + mem::size_of::() + } } #[cfg(test)] diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 4d27f32f7e8..4e11ec203aa 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -2643,6 +2643,143 @@ "title": "Downloader buffer", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of blocks in a request and size in bytes of those block responses", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "auto", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "blocks" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 5, + "y": 110 + }, + "id": 102, + "options": { + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_response_response_size_bytes{instance=~\"$instance\"}", + "hide": false, + "legendFormat": "Response size", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_response_response_length{instance=~\"$instance\"}", + "hide": false, + "legendFormat": "Individual response length", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_response_response_size_bytes / reth_downloaders_bodies_response_response_length", + "hide": false, + "instant": false, + "legendFormat": "Mean body size in response", + "range": true, + "refId": "C" + } + ], + "title": "Block body response sizes", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { From d9ec3db195e20ea5d9497377bb199baa43ffab47 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Sat, 15 Jul 2023 21:59:03 +0100 Subject: [PATCH 104/150] fix(stages): update entities metrics on `SyncHeight` event (#3796) --- crates/stages/src/metrics/listener.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/stages/src/metrics/listener.rs b/crates/stages/src/metrics/listener.rs index 8e8102d3f34..18bd4547727 100644 --- a/crates/stages/src/metrics/listener.rs +++ b/crates/stages/src/metrics/listener.rs @@ -59,8 +59,14 @@ impl MetricsListener { match event { MetricEvent::SyncHeight { height } => { for stage_id in StageId::ALL { - let stage_metrics = self.sync_metrics.get_stage_metrics(stage_id); - stage_metrics.checkpoint.set(height as f64); + self.handle_event(MetricEvent::StageCheckpoint { + stage_id, + checkpoint: StageCheckpoint { + block_number: height, + stage_checkpoint: None, + }, + max_block_number: Some(height), + }); } } MetricEvent::StageCheckpoint { stage_id, checkpoint, max_block_number } => { From f9bbcfa876121a35176402aa33615364387060ac Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 17 Jul 2023 10:25:00 +0200 Subject: [PATCH 105/150] chore: use best with base fee (#3804) --- crates/payload/basic/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index 98325e5101a..f49c3afa65d 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -578,12 +578,12 @@ fn build_payload( let mut cumulative_gas_used = 0; let block_gas_limit: u64 = initialized_block_env.gas_limit.try_into().unwrap_or(u64::MAX); + let base_fee = initialized_block_env.basefee.to::(); let mut executed_txs = Vec::new(); - let mut best_txs = pool.best_transactions(); + let mut best_txs = pool.best_transactions_with_base_fee(base_fee as u128); let mut total_fees = U256::ZERO; - let base_fee = initialized_block_env.basefee.to::(); let block_number = initialized_block_env.number.to::(); From 29f605e09308bb07199f2de3fcab929f54098097 Mon Sep 17 00:00:00 2001 From: "Supernovahs.eth" <91280922+supernovahs@users.noreply.github.com> Date: Mon, 17 Jul 2023 15:12:28 +0530 Subject: [PATCH 106/150] feat:new discovered node record event stream (#3707) Co-authored-by: Matthias Seitz --- crates/net/network/src/discovery.rs | 43 ++++++++++++++++++++++------- crates/net/network/src/manager.rs | 10 ++++++- crates/net/network/src/network.rs | 15 ++++++++-- crates/net/network/src/state.rs | 13 ++++++++- 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/crates/net/network/src/discovery.rs b/crates/net/network/src/discovery.rs index 0bf64e7c028..5af2579567e 100644 --- a/crates/net/network/src/discovery.rs +++ b/crates/net/network/src/discovery.rs @@ -1,6 +1,9 @@ //! Discovery support for the network. -use crate::error::{NetworkError, ServiceKind}; +use crate::{ + error::{NetworkError, ServiceKind}, + manager::DiscoveredEvent, +}; use futures::StreamExt; use reth_discv4::{DiscoveryUpdate, Discv4, Discv4Config, EnrForkIdEntry}; use reth_dns_discovery::{ @@ -14,12 +17,14 @@ use std::{ sync::Arc, task::{Context, Poll}, }; -use tokio::task::JoinHandle; +use tokio::{sync::mpsc, task::JoinHandle}; use tokio_stream::wrappers::ReceiverStream; /// An abstraction over the configured discovery protocol. /// -/// Listens for new discovered nodes and emits events for discovered nodes and their address. +/// Listens for new discovered nodes and emits events for discovered nodes and their +/// address.#[derive(Debug, Clone)] + pub struct Discovery { /// All nodes discovered via discovery protocol. /// @@ -41,6 +46,8 @@ pub struct Discovery { _dns_disc_service: Option>, /// Events buffered until polled. queued_events: VecDeque, + /// List of listeners subscribed to discovery events. + discovery_listeners: Vec>, } impl Discovery { @@ -84,6 +91,7 @@ impl Discovery { }; Ok(Self { + discovery_listeners: Default::default(), local_enr, discv4, discv4_updates, @@ -96,6 +104,17 @@ impl Discovery { }) } + /// Registers a listener for receiving [DiscoveryEvent] updates. + pub(crate) fn add_listener(&mut self, tx: mpsc::UnboundedSender) { + self.discovery_listeners.push(tx); + } + + /// Notifies all registered listeners with the provided `event`. + #[inline] + fn notify_listeners(&mut self, event: &DiscoveryEvent) { + self.discovery_listeners.retain_mut(|listener| listener.send(event.clone()).is_ok()); + } + /// Updates the `eth:ForkId` field in discv4. #[allow(unused)] pub(crate) fn update_fork_id(&self, fork_id: ForkId) { @@ -139,11 +158,9 @@ impl Discovery { Entry::Occupied(_entry) => {} Entry::Vacant(entry) => { entry.insert(addr); - self.queued_events.push_back(DiscoveryEvent::Discovered { - peer_id: id, - socket_addr: addr, - fork_id, - }); + self.queued_events.push_back(DiscoveryEvent::NewNode( + DiscoveredEvent::EventQueued { peer_id: id, socket_addr: addr, fork_id }, + )); } } } @@ -174,6 +191,7 @@ impl Discovery { loop { // Drain all buffered events first if let Some(event) = self.queued_events.pop_front() { + self.notify_listeners(&event); return Poll::Ready(event) } @@ -204,6 +222,9 @@ impl Discovery { /// /// NOTE: This instance does nothing pub(crate) fn noop() -> Self { + let (_discovery_listeners, _): (mpsc::UnboundedSender, _) = + mpsc::unbounded_channel(); + Self { discovered_nodes: Default::default(), local_enr: NodeRecord { @@ -219,14 +240,16 @@ impl Discovery { _dns_discovery: None, dns_discovery_updates: None, _dns_disc_service: None, + discovery_listeners: Default::default(), } } } /// Events produced by the [`Discovery`] manager. +#[derive(Debug, Clone)] pub enum DiscoveryEvent { - /// A new node was discovered - Discovered { peer_id: PeerId, socket_addr: SocketAddr, fork_id: Option }, + /// Discovered a node + NewNode(DiscoveredEvent), /// Retrieved a [`ForkId`] from the peer via ENR request, See EnrForkId(PeerId, ForkId), } diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index 0a997d85a60..78ec6d33211 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -41,7 +41,7 @@ use reth_eth_wire::{ use reth_metrics::common::mpsc::UnboundedMeteredSender; use reth_net_common::bandwidth_meter::BandwidthMeter; use reth_network_api::ReputationChangeKind; -use reth_primitives::{listener::EventListeners, NodeRecord, PeerId, H256}; +use reth_primitives::{listener::EventListeners, ForkId, NodeRecord, PeerId, H256}; use reth_provider::BlockReader; use reth_rpc_types::{EthProtocolInfo, NetworkStatus}; use std::{ @@ -515,6 +515,9 @@ where NetworkHandleMessage::EventListener(tx) => { self.event_listeners.push_listener(tx); } + NetworkHandleMessage::DiscoveryListener(tx) => { + self.swarm.state_mut().discovery_mut().add_listener(tx); + } NetworkHandleMessage::AnnounceBlock(block, hash) => { if self.handle.mode().is_stake() { // See [EIP-3675](https://eips.ethereum.org/EIPS/eip-3675#devp2p) @@ -919,3 +922,8 @@ pub enum NetworkEvent { /// Event emitted when a new peer is removed PeerRemoved(PeerId), } + +#[derive(Debug, Clone)] +pub enum DiscoveredEvent { + EventQueued { peer_id: PeerId, socket_addr: SocketAddr, fork_id: Option }, +} diff --git a/crates/net/network/src/network.rs b/crates/net/network/src/network.rs index 44337e1555e..9a3c8926caf 100644 --- a/crates/net/network/src/network.rs +++ b/crates/net/network/src/network.rs @@ -1,6 +1,6 @@ use crate::{ - config::NetworkMode, manager::NetworkEvent, message::PeerRequest, peers::PeersHandle, - session::PeerInfo, FetchClient, + config::NetworkMode, discovery::DiscoveryEvent, manager::NetworkEvent, message::PeerRequest, + peers::PeersHandle, session::PeerInfo, FetchClient, }; use async_trait::async_trait; use parking_lot::Mutex; @@ -83,6 +83,15 @@ impl NetworkHandle { UnboundedReceiverStream::new(rx) } + /// Returns a new [`DiscoveryEvent`] stream. + /// + /// This stream yields [`DiscoveryEvent`]s for each peer that is discovered. + pub fn discovery_listener(&self) -> UnboundedReceiverStream { + let (tx, rx) = mpsc::unbounded_channel(); + let _ = self.manager().send(NetworkHandleMessage::DiscoveryListener(tx)); + UnboundedReceiverStream::new(rx) + } + /// Returns a new [`FetchClient`] that can be cloned and shared. /// /// The [`FetchClient`] is the entrypoint for sending requests to the network. @@ -320,4 +329,6 @@ pub(crate) enum NetworkHandleMessage { GetReputationById(PeerId, oneshot::Sender>), /// Gracefully shutdown network Shutdown(oneshot::Sender<()>), + /// Add a new listener for `DiscoveryEvent`. + DiscoveryListener(UnboundedSender), } diff --git a/crates/net/network/src/state.rs b/crates/net/network/src/state.rs index 104f263f4b7..446a6796292 100644 --- a/crates/net/network/src/state.rs +++ b/crates/net/network/src/state.rs @@ -4,6 +4,7 @@ use crate::{ cache::LruCache, discovery::{Discovery, DiscoveryEvent}, fetch::{BlockResponseOutcome, FetchAction, StateFetcher}, + manager::DiscoveredEvent, message::{ BlockRequest, NewBlockMessage, PeerRequest, PeerRequestSender, PeerResponse, PeerResponseResult, @@ -11,6 +12,7 @@ use crate::{ peers::{PeerAction, PeersManager}, FetchClient, }; + use reth_eth_wire::{ capability::Capabilities, BlockHashNumber, DisconnectReason, NewBlockHashes, Status, }; @@ -95,6 +97,11 @@ where &mut self.peers_manager } + /// Returns mutable access to the [`Discovery`] + pub(crate) fn discovery_mut(&mut self) -> &mut Discovery { + &mut self.discovery + } + /// Returns access to the [`PeersManager`] pub(crate) fn peers(&self) -> &PeersManager { &self.peers_manager @@ -277,7 +284,11 @@ where /// Event hook for events received from the discovery service. fn on_discovery_event(&mut self, event: DiscoveryEvent) { match event { - DiscoveryEvent::Discovered { peer_id, socket_addr, fork_id } => { + DiscoveryEvent::NewNode(DiscoveredEvent::EventQueued { + peer_id, + socket_addr, + fork_id, + }) => { self.queued_messages.push_back(StateAction::DiscoveredNode { peer_id, socket_addr, From 4187b3a0fe7bf384e5b5bc5051ec05f412f4efc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Vincent?= <28714795+leovct@users.noreply.github.com> Date: Mon, 17 Jul 2023 12:16:25 +0200 Subject: [PATCH 107/150] feat: make rpc gas cap configurable (#3458) Co-authored-by: Alexey Shekhirin Co-authored-by: Matthias Seitz --- bin/reth/src/args/rpc_server_args.rs | 32 +++++++++++++++++++++- book/cli/node.md | 3 ++ crates/rpc/rpc-builder/src/auth.rs | 2 ++ crates/rpc/rpc-builder/src/eth.rs | 10 +++++++ crates/rpc/rpc-builder/src/lib.rs | 1 + crates/rpc/rpc/src/eth/api/mod.rs | 12 ++++++++ crates/rpc/rpc/src/eth/api/server.rs | 5 ++-- crates/rpc/rpc/src/eth/api/state.rs | 4 ++- crates/rpc/rpc/src/eth/api/transactions.rs | 3 +- 9 files changed, 67 insertions(+), 5 deletions(-) diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 230741885bc..4a1f8d2fb89 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -2,11 +2,12 @@ use crate::args::GasPriceOracleArgs; use clap::{ - builder::{PossibleValue, TypedValueParser}, + builder::{PossibleValue, RangedU64ValueParser, TypedValueParser}, Arg, Args, Command, }; use futures::TryFutureExt; use reth_network_api::{NetworkInfo, Peers}; +use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_provider::{ BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, EvmEnvProvider, HeaderProvider, StateProviderFactory, @@ -48,6 +49,9 @@ pub(crate) const RPC_DEFAULT_MAX_CONNECTIONS: u32 = 100; /// Default number of incoming connections. pub(crate) const RPC_DEFAULT_MAX_TRACING_REQUESTS: u32 = 25; +/// Default max gas limit for `eth_call` and call tracing RPC methods. +pub(crate) const RPC_DEFAULT_GAS_CAP: u64 = ETHEREUM_BLOCK_GAS_LIMIT; + /// Parameters for configuring the rpc more granularity via CLI #[derive(Debug, Args, PartialEq, Eq, Default)] #[command(next_help_heading = "RPC")] @@ -132,6 +136,16 @@ pub struct RpcServerArgs { #[arg(long, value_name = "COUNT", default_value_t = RPC_DEFAULT_MAX_TRACING_REQUESTS)] pub rpc_max_tracing_requests: u32, + /// Maximum gas limit for `eth_call` and call tracing RPC methods. + #[arg( + long, + alias = "rpc.gascap", + value_name = "GAS_CAP", + value_parser = RangedU64ValueParser::::new().range(1..), + default_value_t = RPC_DEFAULT_GAS_CAP + )] + pub rpc_gas_cap: u64, + /// Gas price oracle configuration. #[clap(flatten)] pub gas_price_oracle: GasPriceOracleArgs, @@ -174,6 +188,7 @@ impl RpcServerArgs { pub fn eth_config(&self) -> EthConfig { EthConfig::default() .max_tracing_requests(self.rpc_max_tracing_requests) + .rpc_gas_cap(self.rpc_gas_cap) .gpo_config(self.gas_price_oracle_config()) } @@ -495,6 +510,21 @@ mod tests { args: T, } + #[test] + fn test_rpc_gas_cap() { + let args = CommandParser::::parse_from(["reth"]).args; + let config = args.eth_config(); + assert_eq!(config.rpc_gas_cap, RPC_DEFAULT_GAS_CAP); + + let args = + CommandParser::::parse_from(["reth", "--rpc.gascap", "1000"]).args; + let config = args.eth_config(); + assert_eq!(config.rpc_gas_cap, 1000); + + let args = CommandParser::::try_parse_from(["reth", "--rpc.gascap", "0"]); + assert!(args.is_err()); + } + #[test] fn test_rpc_server_args_parser() { let args = diff --git a/book/cli/node.md b/book/cli/node.md index 4d447428a4c..78d9b40dba2 100644 --- a/book/cli/node.md +++ b/book/cli/node.md @@ -187,6 +187,9 @@ Gas Price Oracle: The percentile of gas prices to use for the estimate [default: 60] + + --rpc.gascap + Maximum gas limit for `eth_call` and call tracing RPC methods --block-cache-len Maximum number of block cache entries diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index 567f10bfca4..758713bbdeb 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -2,6 +2,7 @@ use crate::{ constants, error::{RpcError, ServerKind}, eth::DEFAULT_MAX_LOGS_PER_RESPONSE, + EthConfig, }; use hyper::header::AUTHORIZATION; pub use jsonrpsee::server::ServerBuilder; @@ -61,6 +62,7 @@ where network, eth_cache.clone(), gas_oracle, + EthConfig::default().rpc_gas_cap, Box::new(executor.clone()), ); let eth_filter = EthFilter::new( diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs index 3968eef6f83..767660249e2 100644 --- a/crates/rpc/rpc-builder/src/eth.rs +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -1,3 +1,4 @@ +use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_rpc::{ eth::{ cache::{EthStateCache, EthStateCacheConfig}, @@ -37,6 +38,8 @@ pub struct EthConfig { pub max_tracing_requests: u32, /// Maximum number of logs that can be returned in a single response in `eth_getLogs` calls. pub max_logs_per_response: usize, + /// Maximum gas limit for `eth_call` and call tracing RPC methods. + pub rpc_gas_cap: u64, } impl Default for EthConfig { @@ -46,6 +49,7 @@ impl Default for EthConfig { gas_oracle: GasPriceOracleConfig::default(), max_tracing_requests: DEFAULT_MAX_TRACING_REQUESTS, max_logs_per_response: DEFAULT_MAX_LOGS_PER_RESPONSE, + rpc_gas_cap: ETHEREUM_BLOCK_GAS_LIMIT, } } } @@ -74,4 +78,10 @@ impl EthConfig { self.max_logs_per_response = max_logs; self } + + /// Configures the maximum gas limit for `eth_call` and call tracing RPC methods + pub fn rpc_gas_cap(mut self, rpc_gas_cap: u64) -> Self { + self.rpc_gas_cap = rpc_gas_cap; + self + } } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 7013131ec5c..d01036f8d4b 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -967,6 +967,7 @@ where self.network.clone(), cache.clone(), gas_oracle, + self.config.eth.rpc_gas_cap, executor.clone(), ); let filter = EthFilter::new( diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index b53a80993ae..f291bf2566f 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -83,6 +83,7 @@ where network: Network, eth_cache: EthStateCache, gas_oracle: GasPriceOracle, + gas_cap: u64, ) -> Self { Self::with_spawner( provider, @@ -90,6 +91,7 @@ where network, eth_cache, gas_oracle, + gas_cap, Box::::default(), ) } @@ -101,6 +103,7 @@ where network: Network, eth_cache: EthStateCache, gas_oracle: GasPriceOracle, + gas_cap: u64, task_spawner: Box, ) -> Self { // get the block number of the latest block @@ -118,6 +121,7 @@ where signers: Default::default(), eth_cache, gas_oracle, + gas_cap, starting_block: U256::from(latest_block), task_spawner, pending_block: Default::default(), @@ -155,6 +159,12 @@ where &self.inner.gas_oracle } + /// Returns the configured gas limit cap for `eth_call` and tracing related calls + #[allow(unused)] + pub(crate) fn gas_cap(&self) -> u64 { + self.inner.gas_cap + } + /// Returns the inner `Provider` pub fn provider(&self) -> &Provider { &self.inner.provider @@ -354,6 +364,8 @@ struct EthApiInner { eth_cache: EthStateCache, /// The async gas oracle frontend for gas price suggestions gas_oracle: GasPriceOracle, + /// Maximum gas limit for `eth_call` and call tracing RPC methods. + gas_cap: u64, /// The block number at which the node started starting_block: U256, /// The type that can spawn tasks which would otherwise block. diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index b1643e89671..1cca7addc90 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -398,8 +398,8 @@ mod tests { use reth_interfaces::test_utils::{generators, generators::Rng}; use reth_network_api::noop::NoopNetwork; use reth_primitives::{ - basefee::calculate_next_block_base_fee, Block, BlockNumberOrTag, Header, TransactionSigned, - H256, U256, + basefee::calculate_next_block_base_fee, constants::ETHEREUM_BLOCK_GAS_LIMIT, Block, + BlockNumberOrTag, Header, TransactionSigned, H256, U256, }; use reth_provider::{ test_utils::{MockEthProvider, NoopProvider}, @@ -427,6 +427,7 @@ mod tests { NoopNetwork::default(), cache.clone(), GasPriceOracle::new(provider, Default::default(), cache), + ETHEREUM_BLOCK_GAS_LIMIT, ) } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index 281198be079..0930bf0b6c5 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -147,7 +147,7 @@ where mod tests { use super::*; use crate::eth::{cache::EthStateCache, gas_oracle::GasPriceOracle}; - use reth_primitives::{StorageKey, StorageValue}; + use reth_primitives::{constants::ETHEREUM_BLOCK_GAS_LIMIT, StorageKey, StorageValue}; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}; use reth_transaction_pool::test_utils::testing_pool; use std::collections::HashMap; @@ -164,6 +164,7 @@ mod tests { (), cache.clone(), GasPriceOracle::new(NoopProvider::default(), Default::default(), cache), + ETHEREUM_BLOCK_GAS_LIMIT, ); let address = Address::random(); let storage = eth_api.storage_at(address, U256::ZERO.into(), None).unwrap(); @@ -184,6 +185,7 @@ mod tests { (), cache.clone(), GasPriceOracle::new(mock_provider, Default::default(), cache), + ETHEREUM_BLOCK_GAS_LIMIT, ); let storage_key: U256 = storage_key.into(); diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index c932ec825bc..6a5e21d5e16 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -872,7 +872,7 @@ mod tests { EthApi, }; use reth_network_api::noop::NoopNetwork; - use reth_primitives::{hex_literal::hex, Bytes}; + use reth_primitives::{constants::ETHEREUM_BLOCK_GAS_LIMIT, hex_literal::hex, Bytes}; use reth_provider::test_utils::NoopProvider; use reth_transaction_pool::{test_utils::testing_pool, TransactionPool}; @@ -890,6 +890,7 @@ mod tests { noop_network_provider, cache.clone(), GasPriceOracle::new(noop_provider, Default::default(), cache), + ETHEREUM_BLOCK_GAS_LIMIT, ); // https://etherscan.io/tx/0xa694b71e6c128a2ed8e2e0f6770bddbe52e3bb8f10e8472f9a79ab81497a8b5d From e8a5207e04ea00a05d5e4dd8d7520c4913df523f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 17 Jul 2023 12:27:14 +0200 Subject: [PATCH 108/150] chore: bump version .alpha4 (#3808) --- Cargo.lock | 92 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3aa656a4cdd..a4a7b44b51b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,7 +886,7 @@ checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "codecs-derive" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "convert_case 0.6.0", "parity-scale-codec", @@ -1716,7 +1716,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "reth-db", "reth-interfaces", @@ -4917,7 +4917,7 @@ dependencies = [ [[package]] name = "reth" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "backon", "clap", @@ -4983,7 +4983,7 @@ dependencies = [ [[package]] name = "reth-auto-seal-consensus" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "futures-util", "reth-beacon-consensus", @@ -5000,7 +5000,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "futures-core", "futures-util", @@ -5019,7 +5019,7 @@ dependencies = [ [[package]] name = "reth-beacon-consensus" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "assert_matches", "futures", @@ -5045,7 +5045,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "aquamarine", "assert_matches", @@ -5064,7 +5064,7 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "arbitrary", "bytes", @@ -5079,7 +5079,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "confy", "reth-discv4", @@ -5096,7 +5096,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "assert_matches", "mockall", @@ -5107,7 +5107,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "arbitrary", "assert_matches", @@ -5148,7 +5148,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "discv5", "enr", @@ -5171,7 +5171,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "async-trait", "data-encoding", @@ -5195,7 +5195,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "assert_matches", "futures", @@ -5220,7 +5220,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "aes 0.8.2", "block-padding", @@ -5251,7 +5251,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "arbitrary", "async-trait", @@ -5284,7 +5284,7 @@ dependencies = [ [[package]] name = "reth-interfaces" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "arbitrary", "async-trait", @@ -5312,7 +5312,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "async-trait", "bytes", @@ -5331,7 +5331,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "bitflags 2.3.2", "byteorder", @@ -5351,7 +5351,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "bindgen 0.65.1", "cc", @@ -5360,7 +5360,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "futures", "metrics", @@ -5370,7 +5370,7 @@ dependencies = [ [[package]] name = "reth-metrics-derive" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "metrics", "once_cell", @@ -5384,7 +5384,7 @@ dependencies = [ [[package]] name = "reth-net-common" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "pin-project", "reth-primitives", @@ -5393,7 +5393,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "igd", "pin-project-lite", @@ -5407,7 +5407,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "aquamarine", "async-trait", @@ -5457,7 +5457,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "async-trait", "reth-eth-wire", @@ -5470,7 +5470,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "futures-util", "reth-interfaces", @@ -5489,7 +5489,7 @@ dependencies = [ [[package]] name = "reth-primitives" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "arbitrary", "assert_matches", @@ -5537,7 +5537,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "auto_impl", "derive_more", @@ -5558,7 +5558,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "reth-primitives", "thiserror", @@ -5567,7 +5567,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "once_cell", "reth-consensus-common", @@ -5583,7 +5583,7 @@ dependencies = [ [[package]] name = "reth-revm-inspectors" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "boa_engine", "boa_gc", @@ -5599,7 +5599,7 @@ dependencies = [ [[package]] name = "reth-revm-primitives" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "reth-primitives", "revm", @@ -5607,7 +5607,7 @@ dependencies = [ [[package]] name = "reth-rlp" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "arrayvec", "auto_impl", @@ -5625,7 +5625,7 @@ dependencies = [ [[package]] name = "reth-rlp-derive" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "proc-macro2 1.0.63", "quote 1.0.28", @@ -5634,7 +5634,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "assert_matches", "async-trait", @@ -5680,7 +5680,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "jsonrpsee", "reth-primitives", @@ -5690,7 +5690,7 @@ dependencies = [ [[package]] name = "reth-rpc-api-testing-util" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "async-trait", "futures", @@ -5704,7 +5704,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "hyper", "jsonrpsee", @@ -5734,7 +5734,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "assert_matches", "async-trait", @@ -5755,7 +5755,7 @@ dependencies = [ [[package]] name = "reth-rpc-types" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "assert_matches", "jsonrpsee-types", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "aquamarine", "assert_matches", @@ -5807,7 +5807,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "dyn-clone", "futures-util", @@ -5820,7 +5820,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "tracing", "tracing-appender", @@ -5830,7 +5830,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "aquamarine", "async-trait", @@ -5856,7 +5856,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" dependencies = [ "criterion", "derive_more", diff --git a/Cargo.toml b/Cargo.toml index c6b79a2c216..523be9caa9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ default-members = ["bin/reth"] resolver = "2" [workspace.package] -version = "0.1.0-alpha.3" +version = "0.1.0-alpha.4" edition = "2021" rust-version = "1.70" # Remember to update .clippy.toml and README.md license = "MIT OR Apache-2.0" From 4af8b77147d09780b84e17919e7c1a41dd984dc0 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 17 Jul 2023 07:06:26 -0400 Subject: [PATCH 109/150] feat: add `yParity` to rpc signatures (#3800) --- .../rpc/rpc-types/src/eth/transaction/mod.rs | 44 ++++- .../src/eth/transaction/signature.rs | 184 +++++++++++++++++- 2 files changed, 225 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index e42dc38f5cf..109257909a1 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -186,6 +186,8 @@ impl Transaction { #[cfg(test)] mod tests { + use crate::eth::transaction::signature::Parity; + use super::*; #[test] @@ -202,7 +204,12 @@ mod tests { gas_price: Some(U128::from(9)), gas: U256::from(10), input: Bytes::from(vec![11, 12, 13]), - signature: Some(Signature { v: U256::from(14), r: U256::from(14), s: U256::from(14) }), + signature: Some(Signature { + v: U256::from(14), + r: U256::from(14), + s: U256::from(14), + y_parity: None, + }), chain_id: Some(U64::from(17)), access_list: None, transaction_type: Some(U64::from(20)), @@ -217,4 +224,39 @@ mod tests { let deserialized: Transaction = serde_json::from_str(&serialized).unwrap(); assert_eq!(transaction, deserialized); } + + #[test] + fn serde_transaction_with_parity_bit() { + let transaction = Transaction { + hash: H256::from_low_u64_be(1), + nonce: U256::from(2), + block_hash: Some(H256::from_low_u64_be(3)), + block_number: Some(U256::from(4)), + transaction_index: Some(U256::from(5)), + from: Address::from_low_u64_be(6), + to: Some(Address::from_low_u64_be(7)), + value: U256::from(8), + gas_price: Some(U128::from(9)), + gas: U256::from(10), + input: Bytes::from(vec![11, 12, 13]), + signature: Some(Signature { + v: U256::from(14), + r: U256::from(14), + s: U256::from(14), + y_parity: Some(Parity(true)), + }), + chain_id: Some(U64::from(17)), + access_list: None, + transaction_type: Some(U64::from(20)), + max_fee_per_gas: Some(U128::from(21)), + max_priority_fee_per_gas: Some(U128::from(22)), + }; + let serialized = serde_json::to_string(&transaction).unwrap(); + assert_eq!( + serialized, + r#"{"hash":"0x0000000000000000000000000000000000000000000000000000000000000001","nonce":"0x2","blockHash":"0x0000000000000000000000000000000000000000000000000000000000000003","blockNumber":"0x4","transactionIndex":"0x5","from":"0x0000000000000000000000000000000000000006","to":"0x0000000000000000000000000000000000000007","value":"0x8","gasPrice":"0x9","gas":"0xa","maxFeePerGas":"0x15","maxPriorityFeePerGas":"0x16","input":"0x0b0c0d","r":"0xe","s":"0xe","v":"0xe","yParity":"0x1","chainId":"0x11","type":"0x14"}"# + ); + let deserialized: Transaction = serde_json::from_str(&serialized).unwrap(); + assert_eq!(transaction, deserialized); + } } diff --git a/crates/rpc/rpc-types/src/eth/transaction/signature.rs b/crates/rpc/rpc-types/src/eth/transaction/signature.rs index 6538636b40b..3c31a126002 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/signature.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/signature.rs @@ -9,6 +9,8 @@ pub struct Signature { pub r: U256, /// The S field of the signature; the point on the curve. pub s: U256, + // TODO: change these fields to an untagged enum for `v` XOR `y_parity` if/when CLs support it. + // See for more information /// For EIP-155, EIP-2930 and Blob transactions this is set to the parity (0 for even, 1 for /// odd) of the y-value of the secp256k1 signature. /// @@ -16,6 +18,9 @@ pub struct Signature { /// /// See also and pub v: U256, + /// The y parity of the signature. This is only used for typed (non-legacy) transactions. + #[serde(default, rename = "yParity", skip_serializing_if = "Option::is_none")] + pub y_parity: Option, } impl Signature { @@ -28,14 +33,24 @@ impl Signature { signature: PrimitiveSignature, chain_id: Option, ) -> Self { - Self { r: signature.r, s: signature.s, v: U256::from(signature.v(chain_id)) } + Self { + r: signature.r, + s: signature.s, + v: U256::from(signature.v(chain_id)), + y_parity: None, + } } /// Creates a new rpc signature from a non-legacy [primitive /// signature](reth_primitives::Signature). This sets the `v` value to `0` or `1` depending on /// the signature's `odd_y_parity`. pub(crate) fn from_typed_primitive_signature(signature: PrimitiveSignature) -> Self { - Self { r: signature.r, s: signature.s, v: U256::from(signature.odd_y_parity as u8) } + Self { + r: signature.r, + s: signature.s, + v: U256::from(signature.odd_y_parity as u8), + y_parity: Some(Parity(signature.odd_y_parity)), + } } /// Creates a new rpc signature from a legacy [primitive @@ -58,3 +73,168 @@ impl Signature { } } } + +/// Type that represents the signature parity byte, meant for use in RPC. +/// +/// This will be serialized as "0x0" if false, and "0x1" if true. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Parity( + #[serde(serialize_with = "serialize_parity", deserialize_with = "deserialize_parity")] pub bool, +); + +fn serialize_parity(parity: &bool, serializer: S) -> Result +where + S: serde::Serializer, +{ + serializer.serialize_str(if *parity { "0x1" } else { "0x0" }) +} + +/// This implementation disallows serialization of the y parity bit that are not `"0x0"` or `"0x1"`. +fn deserialize_parity<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + match s.as_str() { + "0x0" => Ok(false), + "0x1" => Ok(true), + _ => Err(serde::de::Error::custom(format!( + "invalid parity value, parity should be either \"0x0\" or \"0x1\": {}", + s + ))), + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + + #[test] + fn deserialize_without_parity() { + let raw_signature_without_y_parity = r#"{ + "r":"0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0", + "s":"0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05", + "v":"0x1" + }"#; + + let signature: Signature = serde_json::from_str(raw_signature_without_y_parity).unwrap(); + let expected = Signature { + r: U256::from_str("0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0") + .unwrap(), + s: U256::from_str("0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05") + .unwrap(), + v: U256::from_str("1").unwrap(), + y_parity: None, + }; + + assert_eq!(signature, expected); + } + + #[test] + fn deserialize_with_parity() { + let raw_signature_with_y_parity = r#"{ + "r":"0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0", + "s":"0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05", + "v":"0x1", + "yParity": "0x1" + }"#; + + let signature: Signature = serde_json::from_str(raw_signature_with_y_parity).unwrap(); + let expected = Signature { + r: U256::from_str("0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0") + .unwrap(), + s: U256::from_str("0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05") + .unwrap(), + v: U256::from_str("1").unwrap(), + y_parity: Some(Parity(true)), + }; + + assert_eq!(signature, expected); + } + + #[test] + fn serialize_both_parity() { + // this test should be removed if the struct moves to an enum based on tx type + let signature = Signature { + r: U256::from_str("0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0") + .unwrap(), + s: U256::from_str("0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05") + .unwrap(), + v: U256::from_str("1").unwrap(), + y_parity: Some(Parity(true)), + }; + + let serialized = serde_json::to_string(&signature).unwrap(); + assert_eq!( + serialized, + r#"{"r":"0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0","s":"0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05","v":"0x1","yParity":"0x1"}"# + ); + } + + #[test] + fn serialize_v_only() { + // this test should be removed if the struct moves to an enum based on tx type + let signature = Signature { + r: U256::from_str("0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0") + .unwrap(), + s: U256::from_str("0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05") + .unwrap(), + v: U256::from_str("1").unwrap(), + y_parity: None, + }; + + let expected = r#"{"r":"0xc569c92f176a3be1a6352dd5005bfc751dcb32f57623dd2a23693e64bf4447b0","s":"0x1a891b566d369e79b7a66eecab1e008831e22daa15f91a0a0cf4f9f28f47ee05","v":"0x1"}"#; + + let serialized = serde_json::to_string(&signature).unwrap(); + assert_eq!(serialized, expected); + } + + #[test] + fn serialize_parity() { + let parity = Parity(true); + let serialized = serde_json::to_string(&parity).unwrap(); + assert_eq!(serialized, r#""0x1""#); + + let parity = Parity(false); + let serialized = serde_json::to_string(&parity).unwrap(); + assert_eq!(serialized, r#""0x0""#); + } + + #[test] + fn deserialize_parity() { + let raw_parity = r#""0x1""#; + let parity: Parity = serde_json::from_str(raw_parity).unwrap(); + assert_eq!(parity, Parity(true)); + + let raw_parity = r#""0x0""#; + let parity: Parity = serde_json::from_str(raw_parity).unwrap(); + assert_eq!(parity, Parity(false)); + } + + #[test] + fn deserialize_parity_invalid() { + let raw_parity = r#""0x2""#; + let parity: Result = serde_json::from_str(raw_parity); + assert!(parity.is_err()); + + let raw_parity = r#""0x""#; + let parity: Result = serde_json::from_str(raw_parity); + assert!(parity.is_err()); + + // In the spec this is defined as a uint, which requires 0x + // yParity: + // + // + // uint: + // + let raw_parity = r#""1""#; + let parity: Result = serde_json::from_str(raw_parity); + assert!(parity.is_err()); + + let raw_parity = r#""0""#; + let parity: Result = serde_json::from_str(raw_parity); + assert!(parity.is_err()); + } +} From ca69f5fc3f4dc0ce05c5180cade450327e02cd42 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 17 Jul 2023 16:25:46 +0200 Subject: [PATCH 110/150] feat: add network txpool example (#3809) --- Cargo.lock | 1 + examples/Cargo.toml | 5 +++ examples/network-txpool.rs | 86 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 examples/network-txpool.rs diff --git a/Cargo.lock b/Cargo.lock index a4a7b44b51b..40fe1b49bd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2157,6 +2157,7 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" name = "examples" version = "0.0.0" dependencies = [ + "async-trait", "eyre", "futures", "reth-beacon-consensus", diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 6114a573277..fd244ad0fe6 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -25,6 +25,7 @@ reth-tasks.workspace = true eyre = "0.6.8" futures.workspace = true +async-trait.workspace = true tokio.workspace = true [[example]] @@ -38,3 +39,7 @@ path = "db-access.rs" [[example]] name = "network" path = "network.rs" + +[[example]] +name = "network-txpool" +path = "network-txpool.rs" diff --git a/examples/network-txpool.rs b/examples/network-txpool.rs new file mode 100644 index 00000000000..d85426d1fa6 --- /dev/null +++ b/examples/network-txpool.rs @@ -0,0 +1,86 @@ +//! Example of how to use the network as a standalone component together with a transaction pool and +//! a custom pool validator. +//! +//! Run with +//! +//! ```not_rust +//! cargo run --example network-txpool +//! ``` + +use reth_network::{config::rng_secret_key, NetworkConfig, NetworkManager}; +use reth_provider::test_utils::NoopProvider; +use reth_transaction_pool::{ + GasCostOrdering, PoolTransaction, PooledTransaction, TransactionOrigin, TransactionPool, + TransactionValidationOutcome, TransactionValidator, +}; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + // This block provider implementation is used for testing purposes. + // NOTE: This also means that we don't have access to the blockchain and are not able to serve + // any requests for headers or bodies which can result in dropped connections initiated by + // remote or able to validate transaction against the latest state. + let client = NoopProvider::default(); + + let pool = reth_transaction_pool::Pool::new( + OkValidator::default(), + GasCostOrdering::default(), + Default::default(), + ); + + // The key that's used for encrypting sessions and to identify our node. + let local_key = rng_secret_key(); + + // Configure the network + let config = + NetworkConfig::::builder(local_key).mainnet_boot_nodes().build(client); + + // create the network instance + let (_handle, network, txpool, _) = + NetworkManager::builder(config).await?.transactions(pool.clone()).split_with_handle(); + + // spawn the network task + tokio::task::spawn(network); + // spawn the pool task + tokio::task::spawn(txpool); + + // listen for new transactions + let mut txs = pool.pending_transactions_listener(); + + while let Some(tx) = txs.recv().await { + println!("Received new transaction: {:?}", tx); + } + + Ok(()) +} + +/// A transaction validator that determines all transactions to be valid. +/// +/// An actual validator impl like +/// [EthTransactionValidator](reth_transaction_pool::EthTransactionValidator) would require up to +/// date db access. +/// +/// CAUTION: This validator is not safe to use since it doesn't actually validate the transaction's +/// properties such as chain id, balance, nonce, etc. +#[derive(Default)] +#[non_exhaustive] +struct OkValidator; + +#[async_trait::async_trait] +impl TransactionValidator for OkValidator { + type Transaction = PooledTransaction; + + async fn validate_transaction( + &self, + _origin: TransactionOrigin, + transaction: Self::Transaction, + ) -> TransactionValidationOutcome { + // Always return valid + TransactionValidationOutcome::Valid { + balance: transaction.cost(), + state_nonce: transaction.nonce(), + transaction, + propagate: false, + } + } +} From dc2dfbbd4ca272101100b455dfe9d02c18a538e7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 17 Jul 2023 18:31:25 +0200 Subject: [PATCH 111/150] chore: use 50M default gas limit for calls in rpc (#3812) --- bin/reth/src/args/rpc_server_args.rs | 5 +---- crates/rpc/rpc-builder/src/constants.rs | 7 +++++++ crates/rpc/rpc-builder/src/eth.rs | 8 +++++--- crates/rpc/rpc/src/debug.rs | 9 ++++++++- crates/rpc/rpc/src/eth/api/mod.rs | 3 +-- crates/rpc/rpc/src/eth/api/transactions.rs | 13 +++++++++++-- crates/rpc/rpc/src/eth/revm_utils.rs | 10 +++++----- crates/rpc/rpc/src/trace.rs | 2 ++ 8 files changed, 40 insertions(+), 17 deletions(-) diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 4a1f8d2fb89..a1c954b4a8a 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -7,7 +7,6 @@ use clap::{ }; use futures::TryFutureExt; use reth_network_api::{NetworkInfo, Peers}; -use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_provider::{ BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, EvmEnvProvider, HeaderProvider, StateProviderFactory, @@ -24,6 +23,7 @@ use reth_rpc::{ use reth_rpc_builder::{ auth::{AuthServerConfig, AuthServerHandle}, constants, + constants::RPC_DEFAULT_GAS_CAP, error::RpcError, EthConfig, IpcServerBuilder, RethRpcModule, RpcModuleBuilder, RpcModuleConfig, RpcModuleSelection, RpcServerConfig, RpcServerHandle, ServerBuilder, TransportRpcModuleConfig, @@ -49,9 +49,6 @@ pub(crate) const RPC_DEFAULT_MAX_CONNECTIONS: u32 = 100; /// Default number of incoming connections. pub(crate) const RPC_DEFAULT_MAX_TRACING_REQUESTS: u32 = 25; -/// Default max gas limit for `eth_call` and call tracing RPC methods. -pub(crate) const RPC_DEFAULT_GAS_CAP: u64 = ETHEREUM_BLOCK_GAS_LIMIT; - /// Parameters for configuring the rpc more granularity via CLI #[derive(Debug, Args, PartialEq, Eq, Default)] #[command(next_help_heading = "RPC")] diff --git a/crates/rpc/rpc-builder/src/constants.rs b/crates/rpc/rpc-builder/src/constants.rs index a1b2bc36a82..6768ca62a65 100644 --- a/crates/rpc/rpc-builder/src/constants.rs +++ b/crates/rpc/rpc-builder/src/constants.rs @@ -7,6 +7,13 @@ pub const DEFAULT_WS_RPC_PORT: u16 = 8546; /// The default port for the auth server. pub const DEFAULT_AUTH_PORT: u16 = 8551; +/// The default gas limit for eth_call and adjacent calls. +/// +/// This is different from the default to regular 30M block gas limit +/// [ETHEREUM_BLOCK_GAS_LIMIT](reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT) to allow for +/// more complex calls. +pub const RPC_DEFAULT_GAS_CAP: u64 = 50_000_000; + /// The default IPC endpoint #[cfg(windows)] pub const DEFAULT_IPC_ENDPOINT: &str = r"\\.\pipe\reth.ipc"; diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs index 767660249e2..5b2d6780a53 100644 --- a/crates/rpc/rpc-builder/src/eth.rs +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -1,4 +1,4 @@ -use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; +use crate::constants::RPC_DEFAULT_GAS_CAP; use reth_rpc::{ eth::{ cache::{EthStateCache, EthStateCacheConfig}, @@ -38,7 +38,9 @@ pub struct EthConfig { pub max_tracing_requests: u32, /// Maximum number of logs that can be returned in a single response in `eth_getLogs` calls. pub max_logs_per_response: usize, - /// Maximum gas limit for `eth_call` and call tracing RPC methods. + /// Gas limit for `eth_call` and call tracing RPC methods. + /// + /// Defaults to [RPC_DEFAULT_GAS_CAP] pub rpc_gas_cap: u64, } @@ -49,7 +51,7 @@ impl Default for EthConfig { gas_oracle: GasPriceOracleConfig::default(), max_tracing_requests: DEFAULT_MAX_TRACING_REQUESTS, max_logs_per_response: DEFAULT_MAX_LOGS_PER_RESPONSE, - rpc_gas_cap: ETHEREUM_BLOCK_GAS_LIMIT, + rpc_gas_cap: RPC_DEFAULT_GAS_CAP, } } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 3f8fd11a026..d6abad21fe2 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -319,7 +319,14 @@ where let state = self.inner.eth_api.state_at(at)?; let mut db = SubState::new(State::new(state)); let has_state_overrides = overrides.has_state(); - let env = prepare_call_env(cfg, block_env, call, &mut db, overrides)?; + let env = prepare_call_env( + cfg, + block_env, + call, + self.inner.eth_api.call_gas_limit(), + &mut db, + overrides, + )?; // If the caller provided state overrides we need to clone the DB so the js // service has access these modifications diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index f291bf2566f..50e634f424a 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -160,8 +160,7 @@ where } /// Returns the configured gas limit cap for `eth_call` and tracing related calls - #[allow(unused)] - pub(crate) fn gas_cap(&self) -> u64 { + pub fn gas_cap(&self) -> u64 { self.inner.gas_cap } diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 6a5e21d5e16..80d83fdb809 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -43,6 +43,9 @@ pub(crate) type StateCacheDB<'r> = CacheDB>>; /// Commonly used transaction related functions for the [EthApi] type in the `eth_` namespace #[async_trait::async_trait] pub trait EthTransactions: Send + Sync { + /// Returns default gas limit to use for `eth_call` and tracing RPC methods. + fn call_gas_limit(&self) -> u64; + /// Returns the state at the given [BlockId] fn state_at(&self, at: BlockId) -> EthResult>; @@ -226,6 +229,10 @@ where Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, Network: NetworkInfo + Send + Sync + 'static, { + fn call_gas_limit(&self) -> u64 { + self.inner.gas_cap + } + fn state_at(&self, at: BlockId) -> EthResult> { self.state_at_block_id(at) } @@ -480,7 +487,8 @@ where let state = self.state_at(at)?; let mut db = SubState::new(State::new(state)); - let env = prepare_call_env(cfg, block_env, request, &mut db, overrides)?; + let env = + prepare_call_env(cfg, block_env, request, self.call_gas_limit(), &mut db, overrides)?; f(db, env) } @@ -520,7 +528,8 @@ where let state = self.state_at(at)?; let mut db = SubState::new(State::new(state)); - let env = prepare_call_env(cfg, block_env, request, &mut db, overrides)?; + let env = + prepare_call_env(cfg, block_env, request, self.call_gas_limit(), &mut db, overrides)?; inspect_and_return_db(db, env, inspector) } diff --git a/crates/rpc/rpc/src/eth/revm_utils.rs b/crates/rpc/rpc/src/eth/revm_utils.rs index 259c34589be..d40689abe18 100644 --- a/crates/rpc/rpc/src/eth/revm_utils.rs +++ b/crates/rpc/rpc/src/eth/revm_utils.rs @@ -2,8 +2,7 @@ use crate::eth::error::{EthApiError, EthResult, RpcInvalidTransactionError}; use reth_primitives::{ - constants::ETHEREUM_BLOCK_GAS_LIMIT, AccessList, Address, TransactionSigned, - TransactionSignedEcRecovered, TxHash, H256, U256, + AccessList, Address, TransactionSigned, TransactionSignedEcRecovered, TxHash, H256, U256, }; use reth_revm::env::{fill_tx_env, fill_tx_env_with_recovered}; use reth_rpc_types::{ @@ -203,6 +202,7 @@ pub(crate) fn prepare_call_env( mut cfg: CfgEnv, block: BlockEnv, request: CallRequest, + gas_limit: u64, db: &mut CacheDB, overrides: EvmOverrides, ) -> EthResult @@ -247,10 +247,10 @@ where // If no gas price is specified, use maximum allowed gas limit. The reason for this is // that both Erigon and Geth use pre-configured gas cap even if it's possible // to derive the gas limit from the block: - // https://github.com/ledgerwatch/erigon/blob/eae2d9a79cb70dbe30b3a6b79c436872e4605458/cmd/rpcdaemon/commands/trace_adhoc.go#L956 - // https://github.com/ledgerwatch/erigon/blob/eae2d9a79cb70dbe30b3a6b79c436872e4605458/eth/ethconfig/config.go#L94 + // trace!(target: "rpc::eth::call", ?env, "Applying gas limit cap as the maximum gas limit"); - env.tx.gas_limit = ETHEREUM_BLOCK_GAS_LIMIT; + env.tx.gas_limit = gas_limit; } } diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 9a19b9282b5..2fcf2d237b7 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -191,6 +191,7 @@ where let (cfg, block_env, at) = self.inner.eth_api.evm_env_at(at).await?; self.on_blocking_task(|this| async move { + let gas_limit = this.inner.eth_api.call_gas_limit(); // execute all transactions on top of each other and record the traces this.inner.eth_api.with_state_at_block(at, move |state| { let mut results = Vec::with_capacity(calls.len()); @@ -203,6 +204,7 @@ where cfg.clone(), block_env.clone(), call, + gas_limit, &mut db, Default::default(), )?; From fb50aa788d2eb01ea7d1b3bd109d668471782c51 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 17 Jul 2023 23:18:20 +0200 Subject: [PATCH 112/150] chore(deps): bump some deps (#3820) --- Cargo.lock | 598 +++++++++++++++-------------- Cargo.toml | 9 +- bin/reth/Cargo.toml | 2 +- crates/primitives/Cargo.toml | 2 +- crates/primitives/src/chain/mod.rs | 8 +- crates/rpc/rpc-builder/Cargo.toml | 2 +- 6 files changed, 335 insertions(+), 286 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40fe1b49bd2..d013576e75c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,10 +163,10 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759d98a5db12e9c9d98ef2b92f794ae5c7ded6ec18d21c3fa485c9c65bec237d" dependencies = [ - "itertools", + "itertools 0.10.5", "proc-macro-error", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -222,9 +222,9 @@ version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -275,8 +275,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fee3da8ef1276b0bee5dd1c7258010d8fffd31801447323115a25560e1327b89" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -388,8 +388,8 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "regex", "rustc-hash", "shlex", @@ -409,12 +409,12 @@ dependencies = [ "lazycell", "peeking_take_while", "prettyplease", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "regex", "rustc-hash", "shlex", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -440,9 +440,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" [[package]] name = "bitvec" @@ -496,23 +496,23 @@ dependencies = [ [[package]] name = "boa_ast" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.3.3", "boa_interner", "boa_macros", - "indexmap 1.9.3", + "indexmap 2.0.0", "num-bigint", "rustc-hash", ] [[package]] name = "boa_engine" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.3.3", "boa_ast", "boa_gc", "boa_icu_provider", @@ -524,8 +524,8 @@ dependencies = [ "dashmap", "fast-float", "icu_normalizer", - "indexmap 1.9.3", - "itertools", + "indexmap 2.0.0", + "itertools 0.11.0", "num-bigint", "num-integer", "num-traits", @@ -547,8 +547,8 @@ dependencies = [ [[package]] name = "boa_gc" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" dependencies = [ "boa_macros", "boa_profiler", @@ -557,8 +557,8 @@ dependencies = [ [[package]] name = "boa_icu_provider" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" dependencies = [ "icu_collections", "icu_normalizer", @@ -570,13 +570,13 @@ dependencies = [ [[package]] name = "boa_interner" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.0", - "indexmap 1.9.3", + "indexmap 2.0.0", "once_cell", "phf", "rustc-hash", @@ -585,21 +585,21 @@ dependencies = [ [[package]] name = "boa_macros" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", "synstructure 0.13.0", ] [[package]] name = "boa_parser" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.3.3", "boa_ast", "boa_icu_provider", "boa_interner", @@ -616,8 +616,8 @@ dependencies = [ [[package]] name = "boa_profiler" -version = "0.16.0" -source = "git+https://github.com/boa-dev/boa#0e1b32a232109fc0e192c1297a7274091af2ac61" +version = "0.17.0" +source = "git+https://github.com/boa-dev/boa#b51e7cfb84fd6a4dc474d103e5654efedfe94a4e" [[package]] name = "brotli" @@ -864,8 +864,8 @@ checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -890,10 +890,10 @@ version = "0.1.0-alpha.4" dependencies = [ "convert_case 0.6.0", "parity-scale-codec", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "serde", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -964,13 +964,13 @@ dependencies = [ [[package]] name = "comfy-table" -version = "6.1.4" +version = "7.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7b787b0dc42e8111badfdbe4c3059158ccb2db8780352fa1b01e8ccf45cc4d" +checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" dependencies = [ - "crossterm", - "strum", - "strum_macros", + "crossterm 0.26.1", + "strum 0.24.1", + "strum_macros 0.24.3", "unicode-width", ] @@ -1096,7 +1096,7 @@ dependencies = [ "criterion-plot", "futures", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -1117,7 +1117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -1185,11 +1185,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "crossterm" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +dependencies = [ + "bitflags 1.3.2", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.1", + "signal-hook", + "signal-hook-mio", + "winapi", +] + [[package]] name = "crossterm_winapi" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ "winapi", ] @@ -1228,7 +1244,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote 1.0.28", + "quote 1.0.31", "syn 1.0.109", ] @@ -1286,8 +1302,8 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "scratch", "syn 1.0.109", ] @@ -1304,8 +1320,8 @@ version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1347,8 +1363,8 @@ checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "strsim 0.9.3", "syn 1.0.109", ] @@ -1361,8 +1377,8 @@ checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "strsim 0.10.0", "syn 1.0.109", ] @@ -1375,10 +1391,10 @@ checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "strsim 0.10.0", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -1388,7 +1404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ "darling_core 0.10.2", - "quote 1.0.28", + "quote 1.0.31", "syn 1.0.109", ] @@ -1399,7 +1415,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" dependencies = [ "darling_core 0.14.3", - "quote 1.0.28", + "quote 1.0.31", "syn 1.0.109", ] @@ -1410,21 +1426,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core 0.20.1", - "quote 1.0.28", - "syn 2.0.18", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown 0.14.0", "lock_api", "once_cell", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.8", ] [[package]] @@ -1468,8 +1484,8 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cdeb9ec472d588e539a818b2dee436825730da08ad0017c4b1a17676bdc8b7" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1481,8 +1497,8 @@ checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0" dependencies = [ "darling 0.10.2", "derive_builder_core", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1493,8 +1509,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef" dependencies = [ "darling 0.10.2", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1505,8 +1521,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "rustc_version", "syn 1.0.109", ] @@ -1597,7 +1613,7 @@ dependencies = [ [[package]] name = "discv5" version = "0.3.0" -source = "git+https://github.com/sigp/discv5#47844ca54e8d22f4fd3db4594645e65afb288bb6" +source = "git+https://github.com/sigp/discv5#f78d538ef8f3c3b3981cfbb8ce2ba3179295eeab" dependencies = [ "aes 0.7.5", "aes-gcm", @@ -1630,9 +1646,9 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -1709,8 +1725,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0188e3c3ba8df5753894d54461f0e39bc91741dc5b22e1c46999ec2c71f4e4" dependencies = [ "enum-ordinalize", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1806,8 +1822,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "570d109b813e904becc80d8d5da38376818a143348413f7149f1340fe04754d4" dependencies = [ "heck", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1818,8 +1834,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ "heck", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1831,8 +1847,8 @@ checksum = "a62bb1df8b45ecb7ffa78dca1c17a438fb193eb083db0b1b494d2a61bcb5096a" dependencies = [ "num-bigint", "num-traits", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "rustc_version", "syn 1.0.109", ] @@ -1843,9 +1859,9 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48016319042fb7c87b78d2993084a831793a897a5cd1a2a67cab9d1eeb4b7d76" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -1958,14 +1974,15 @@ dependencies = [ [[package]] name = "ethers-contract" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e066a0d9cfc70c454672bf16bb433b0243427420076dc5b2f49c448fb5a10628" +checksum = "0d4719a44c3d37ab07c6dea99ab174068d8c35e441b60b6c20ce4e48357273e8" dependencies = [ "ethers-contract-abigen", "ethers-contract-derive", "ethers-core", "ethers-providers", + "ethers-signers", "futures-util", "hex", "once_cell", @@ -1977,9 +1994,9 @@ dependencies = [ [[package]] name = "ethers-contract-abigen" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c113e3e86b6bc16d98484b2c3bb2d01d6fed9f489fe2e592e5cc87c3024d616b" +checksum = "155ea1b84d169d231317ed86e307af6f2bed6b40dd17e5e94bc84da21cadb21c" dependencies = [ "Inflector", "dunce", @@ -1987,37 +2004,37 @@ dependencies = [ "eyre", "hex", "prettyplease", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "regex", "serde", "serde_json", - "syn 2.0.18", + "syn 2.0.26", "toml 0.7.5", "walkdir", ] [[package]] name = "ethers-contract-derive" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3fb5adee25701c79ec58fcf2c63594cd8829bc9ad6037ff862d5a111101ed2" +checksum = "8567ff196c4a37c1a8c90ec73bda0ad2062e191e4f0a6dc4d943e2ec4830fc88" dependencies = [ "Inflector", "ethers-contract-abigen", "ethers-core", "hex", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "serde_json", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] name = "ethers-core" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da5fa198af0d3be20c19192df2bd9590b92ce09a8421e793bec8851270f1b05" +checksum = "60ca2514feb98918a0a31de7e1983c29f2267ebf61b2dc5d4294f91e5b866623" dependencies = [ "arrayvec", "bytes", @@ -2035,8 +2052,8 @@ dependencies = [ "rlp", "serde", "serde_json", - "strum", - "syn 2.0.18", + "strum 0.25.0", + "syn 2.0.26", "tempfile", "thiserror", "tiny-keccak", @@ -2045,9 +2062,9 @@ dependencies = [ [[package]] name = "ethers-etherscan" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ebb401ba97c6f5af278c2c9936c4546cad75dec464b439ae6df249906f4caa" +checksum = "22b3a8269d3df0ed6364bc05b4735b95f4bf830ce3aef87d5e760fb0e93e5b91" dependencies = [ "ethers-core", "reqwest", @@ -2060,9 +2077,9 @@ dependencies = [ [[package]] name = "ethers-middleware" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740f4a773c19dd6d6a68c8c2e0996c096488d38997d524e21dc612c55da3bd24" +checksum = "e0c339aad74ae5c451d27e0e49c7a3c7d22620b119b4f9291d7aa21f72d7f366" dependencies = [ "async-trait", "auto_impl", @@ -2087,9 +2104,9 @@ dependencies = [ [[package]] name = "ethers-providers" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56b498fd2a6c019d023e43e83488cd1fb0721f299055975aa6bac8dbf1e95f2c" +checksum = "b411b119f1cf0efb69e2190883dee731251882bb21270f893ee9513b3a697c48" dependencies = [ "async-trait", "auto_impl", @@ -2124,9 +2141,9 @@ dependencies = [ [[package]] name = "ethers-signers" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c4b7e15f212fa7cc2e1251868320221d4ff77a3d48068e69f47ce1c491df2d" +checksum = "4864d387456a9c09a1157fa10e1528b29d90f1d859443acf06a1b23365fb518c" dependencies = [ "async-trait", "coins-bip32", @@ -2359,9 +2376,9 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -2957,8 +2974,8 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b728b9421e93eff1d9f8681101b78fa745e0748c95c655c83f337044a7e10" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3045,8 +3062,8 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3174,6 +3191,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -3323,8 +3349,8 @@ checksum = "c6027ac0b197ce9543097d02a290f550ce1d9432bf301524b013053c0b75cc94" dependencies = [ "heck", "proc-macro-crate", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3439,9 +3465,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libloading" @@ -3529,9 +3555,9 @@ checksum = "3a04a5b2b6f54acba899926491d0a6c59d98012938ca2ab5befb281c034e8f94" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -3539,12 +3565,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "lru" @@ -3672,8 +3695,8 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3746,14 +3769,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -3778,8 +3801,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" dependencies = [ "cfg-if", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3799,8 +3822,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3971,9 +3994,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -4027,8 +4050,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "003b2be5c6c53c1cfeb0a238b8a1c3915cd410feb684457a36c10038f764bb1c" dependencies = [ "bytes", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -4110,8 +4133,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b26a931f824dd4eca30b3e43bb4f31cd5f0d3a403c5f5ff27106b805bfde7b" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -4147,7 +4170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.8", ] [[package]] @@ -4166,15 +4189,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.45.0", + "windows-targets 0.48.1", ] [[package]] @@ -4235,9 +4258,9 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", "phf_shared", @@ -4255,22 +4278,22 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92aacdc5f16768709a569e913f7451034034178b05bdc8acda226659a3dccc66" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator", "phf_shared", - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 1.0.109", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] @@ -4290,9 +4313,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -4437,7 +4460,7 @@ checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" dependencies = [ "difflib", "float-cmp", - "itertools", + "itertools 0.10.5", "normalize-line-endings", "predicates-core", "regex", @@ -4477,8 +4500,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ - "proc-macro2 1.0.63", - "syn 2.0.18", + "proc-macro2 1.0.66", + "syn 2.0.26", ] [[package]] @@ -4512,8 +4535,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "version_check", ] @@ -4524,8 +4547,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "version_check", ] @@ -4540,9 +4563,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -4661,11 +4684,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" dependencies = [ - "proc-macro2 1.0.63", + "proc-macro2 1.0.66", ] [[package]] @@ -4925,7 +4948,7 @@ dependencies = [ "comfy-table", "confy", "const-str", - "crossterm", + "crossterm 0.25.0", "dirs-next", "eyre", "fdlimit", @@ -5201,7 +5224,7 @@ dependencies = [ "assert_matches", "futures", "futures-util", - "itertools", + "itertools 0.10.5", "pin-project", "rayon", "reth-db", @@ -5334,7 +5357,7 @@ dependencies = [ name = "reth-libmdbx" version = "0.1.0-alpha.4" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.3.3", "byteorder", "criterion", "derive_more", @@ -5375,11 +5398,11 @@ version = "0.1.0-alpha.4" dependencies = [ "metrics", "once_cell", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "regex", "serial_test 0.10.0", - "syn 2.0.18", + "syn 2.0.26", "trybuild", ] @@ -5522,7 +5545,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "strum", + "strum 0.25.0", "sucds", "test-fuzz", "thiserror", @@ -5542,7 +5565,7 @@ version = "0.1.0-alpha.4" dependencies = [ "auto_impl", "derive_more", - "itertools", + "itertools 0.10.5", "parking_lot 0.12.1", "pin-project", "reth-db", @@ -5628,9 +5651,9 @@ dependencies = [ name = "reth-rlp-derive" version = "0.1.0-alpha.4" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -5725,7 +5748,7 @@ dependencies = [ "reth-transaction-pool", "serde", "serde_json", - "strum", + "strum 0.25.0", "thiserror", "tokio", "tower", @@ -5779,7 +5802,7 @@ dependencies = [ "async-trait", "criterion", "futures-util", - "itertools", + "itertools 0.10.5", "num-traits", "paste", "pin-project", @@ -6005,8 +6028,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e33d7b2abe0c340d8797fe2907d3f20d3b5ea5908683618bfe80df7f621f672a" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -6131,9 +6154,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "rusty-fork" @@ -6196,8 +6219,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61fa974aea2d63dd18a4ec3a49d59af9f34178c73a4f56d2f18205628d00681e" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -6335,9 +6358,9 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] @@ -6353,20 +6376,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" dependencies = [ "itoa", "ryu", @@ -6417,8 +6440,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1966009f3c05f095697c537312f5415d1e3ed31ce0a56942bac4c771c5c335e" dependencies = [ "darling 0.14.3", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -6456,8 +6479,8 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b64f9e531ce97c88b4778aad0ceee079216071cffec6ac9b904277f8f92e7fe3" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -6467,9 +6490,9 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -6567,9 +6590,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +checksum = "b824b6e687aff278cdbf3b36f07aa52d4bd4099699324d5da86a2ebce3aa00b3" dependencies = [ "libc", "signal-hook-registry", @@ -6660,9 +6683,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "smol_str" @@ -6771,8 +6794,14 @@ name = "strum" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.1", ] [[package]] @@ -6782,12 +6811,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "rustversion", "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" +dependencies = [ + "heck", + "proc-macro2 1.0.66", + "quote 1.0.31", + "rustversion", + "syn 2.0.26", +] + [[package]] name = "subprocess" version = "0.2.9" @@ -6866,19 +6908,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.18" +version = "2.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "unicode-ident", ] @@ -6888,8 +6930,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -6900,9 +6942,9 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "285ba80e733fac80aa4270fbcdf83772a79b80aa35c97075320abfee4a915b06" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", "unicode-xid 0.2.4", ] @@ -6959,10 +7001,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "385624eb0031d550fe1bf99c08af79b838605fc4fcec2c4d55e229a2c342fdd0" dependencies = [ "cargo_metadata", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "serde", - "strum_macros", + "strum_macros 0.24.3", ] [[package]] @@ -6973,12 +7015,12 @@ checksum = "69247423e2d89bd51160e42200f6f45f921a23e5b44a0e5b57b888a378334037" dependencies = [ "darling 0.20.1", "if_chain", - "itertools", + "itertools 0.10.5", "lazy_static", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "subprocess", - "syn 2.0.18", + "syn 2.0.26", "test-fuzz-internal", "toolchain_find", ] @@ -7005,22 +7047,22 @@ checksum = "aac81b6fd6beb5884b0cf3321b8117e6e5d47ecb6fc89f414cfdcca8b2fe2dd8" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -7129,9 +7171,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -7333,8 +7375,8 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -7529,7 +7571,7 @@ checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1" dependencies = [ "bitflags 1.3.2", "cassowary", - "crossterm", + "crossterm 0.25.0", "unicode-segmentation", "unicode-width", ] @@ -7594,9 +7636,9 @@ checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -7784,8 +7826,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "wasm-bindgen-shared", ] @@ -7808,7 +7850,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ - "quote 1.0.28", + "quote 1.0.31", "wasm-bindgen-macro-support", ] @@ -7818,8 +7860,8 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -7899,7 +7941,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -7932,7 +7974,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -7952,9 +7994,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm 0.48.0", "windows_aarch64_msvc 0.48.0", @@ -8146,8 +8188,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af46c169923ed7516eef0aa32b56d2651b229f57458ebe46b49ddd6efef5b7a2" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "synstructure 0.12.6", ] @@ -8167,8 +8209,8 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4eae7c1f7d4b8eafce526bc0771449ddc2f250881ae31c50d22c032b5a1c499" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "synstructure 0.12.6", ] @@ -8188,9 +8230,9 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -8210,8 +8252,8 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "486558732d5dde10d0f8cb2936507c1bb21bc539d924c949baf5f36a58e51bac" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "synstructure 0.12.6", ] diff --git a/Cargo.toml b/Cargo.toml index 523be9caa9c..92ed754a35a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,10 +105,10 @@ reth-network = { path = "./crates/net/network" } reth-network-api = { path = "./crates/net/network-api" } ## eth -ethers-core = { version = "2.0.7", default-features = false } -ethers-providers = { version = "2.0.7", default-features = false } -ethers-signers = { version = "2.0.7", default-features = false } -ethers-middleware = { version = "2.0.7", default-features = false } +ethers-core = { version = "2.0.8", default-features = false } +ethers-providers = { version = "2.0.8", default-features = false } +ethers-signers = { version = "2.0.8", default-features = false } +ethers-middleware = { version = "2.0.8", default-features = false } ## misc bytes = "1.4" @@ -117,6 +117,7 @@ thiserror = "1.0.37" serde_json = "1.0.94" serde = { version = "1.0", default-features = false } rand = "0.8.5" +strum = "0.25" ### proc-macros proc-macro2 = "1.0" diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 143200ab809..5d98c73e012 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -62,7 +62,7 @@ metrics-process = "1.0.9" proptest = "1.0" # tui -comfy-table = "6.1.4" +comfy-table = "7.0" crossterm = "0.25.0" tui = "0.19.0" human_bytes = "0.4.1" diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index f75c681bc4e..f9b592dea9c 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -68,7 +68,7 @@ hash-db = "0.15" arbitrary = { version = "1.1.7", features = ["derive"], optional = true } proptest = { version = "1.0", optional = true } proptest-derive = { version = "0.3", optional = true } -strum = { version = "0.24", features = ["derive"] } +strum = { workspace = true, features = ["derive"] } [dev-dependencies] serde_json = { workspace = true } diff --git a/crates/primitives/src/chain/mod.rs b/crates/primitives/src/chain/mod.rs index e1c04290c88..9b1a9da20b0 100644 --- a/crates/primitives/src/chain/mod.rs +++ b/crates/primitives/src/chain/mod.rs @@ -263,8 +263,14 @@ mod tests { } #[test] - fn test_legacy_named_chain() { + fn test_optimism_chain() { let chain = Chain::Named(ethers_core::types::Chain::Optimism); + assert!(!chain.is_legacy()); + } + + #[test] + fn test_legacy_named_chain() { + let chain = Chain::Named(ethers_core::types::Chain::BinanceSmartChain); assert!(chain.is_legacy()); } diff --git a/crates/rpc/rpc-builder/Cargo.toml b/crates/rpc/rpc-builder/Cargo.toml index b4159bfb682..8d51e178022 100644 --- a/crates/rpc/rpc-builder/Cargo.toml +++ b/crates/rpc/rpc-builder/Cargo.toml @@ -29,7 +29,7 @@ tower = { version = "0.4", features = ["full"] } hyper = "0.14" # misc -strum = { version = "0.24", features = ["derive"] } +strum = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tracing = { workspace = true } From c182acc3f8ae1ae322bf030e574211bef3ed4b52 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 17 Jul 2023 20:40:43 -0400 Subject: [PATCH 113/150] chore: fix SessionManagerMetrics typo (#3823) --- crates/net/network/src/metrics.rs | 2 +- crates/net/network/src/session/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/net/network/src/metrics.rs b/crates/net/network/src/metrics.rs index 085b1f093b2..43c3b03f481 100644 --- a/crates/net/network/src/metrics.rs +++ b/crates/net/network/src/metrics.rs @@ -48,7 +48,7 @@ pub struct NetworkMetrics { /// Metrics for SessionManager #[derive(Metrics)] #[metrics(scope = "network")] -pub struct SesssionManagerMetrics { +pub struct SessionManagerMetrics { /// Number of dials that resulted in a peer being added to the peerset pub(crate) total_dial_successes: Counter, } diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index d31d2c635bc..df19d144e43 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -1,7 +1,7 @@ //! Support for handling peer sessions. use crate::{ message::PeerMessage, - metrics::SesssionManagerMetrics, + metrics::SessionManagerMetrics, session::{active::ActiveSession, config::SessionCounter}, }; pub use crate::{message::PeerRequestSender, session::handle::PeerInfo}; @@ -100,7 +100,7 @@ pub struct SessionManager { /// Used to measure inbound & outbound bandwidth across all managed streams bandwidth_meter: BandwidthMeter, /// Metrics for the session manager. - metrics: SesssionManagerMetrics, + metrics: SessionManagerMetrics, } // === impl SessionManager === From b44f74779b02b62ba9a65018d09f4f910c2d0fa2 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 18 Jul 2023 12:18:35 +0300 Subject: [PATCH 114/150] chore(ci): enable docker push on workflow dispatch (#3825) --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 80f02f0cf8c..06dbff26f2d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,7 @@ name: docker on: + workflow_dispatch: {} push: tags: - v* From 733a78e115f2aa6bd429a5e0e4c8bda4cbc16183 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 18 Jul 2023 14:06:27 +0200 Subject: [PATCH 115/150] chore: make clippy happy (#3827) --- bin/reth/src/init.rs | 1 + bin/reth/src/stage/dump/execution.rs | 2 +- bin/reth/src/stage/dump/mod.rs | 2 +- crates/consensus/beacon/src/engine/sync.rs | 2 +- crates/primitives/src/hex_bytes.rs | 2 +- crates/revm/revm-inspectors/src/tracing/mod.rs | 6 +++--- crates/rlp/benches/bench.rs | 2 +- crates/rpc/rpc-builder/tests/it/utils.rs | 2 +- crates/rpc/rpc-types/src/eth/fee.rs | 5 +---- crates/trie/src/hashed_cursor/post_state.rs | 3 +-- 10 files changed, 12 insertions(+), 15 deletions(-) diff --git a/bin/reth/src/init.rs b/bin/reth/src/init.rs index 37efd40817d..4c45edbe25b 100644 --- a/bin/reth/src/init.rs +++ b/bin/reth/src/init.rs @@ -190,6 +190,7 @@ mod tests { }; use std::collections::HashMap; + #[allow(clippy::type_complexity)] fn collect_table_entries( tx: &>::TX, ) -> Result, InitDatabaseError> diff --git a/bin/reth/src/stage/dump/execution.rs b/bin/reth/src/stage/dump/execution.rs index 9200065d551..b2d7bbed32f 100644 --- a/bin/reth/src/stage/dump/execution.rs +++ b/bin/reth/src/stage/dump/execution.rs @@ -35,7 +35,7 @@ pub(crate) async fn dump_execution_stage( /// Imports all the tables that can be copied over a range. fn import_tables_with_range( output_db: &DatabaseEnv, - db_tool: &mut DbTool<'_, DB>, + db_tool: &DbTool<'_, DB>, from: u64, to: u64, ) -> eyre::Result<()> { diff --git a/bin/reth/src/stage/dump/mod.rs b/bin/reth/src/stage/dump/mod.rs index 14c5fd25900..5e7206cf1a5 100644 --- a/bin/reth/src/stage/dump/mod.rs +++ b/bin/reth/src/stage/dump/mod.rs @@ -131,7 +131,7 @@ pub(crate) fn setup( from: u64, to: u64, output_db: &PathBuf, - db_tool: &mut DbTool<'_, DB>, + db_tool: &DbTool<'_, DB>, ) -> eyre::Result<(DatabaseEnv, u64)> { assert!(from < to, "FROM block should be bigger than TO block."); diff --git a/crates/consensus/beacon/src/engine/sync.rs b/crates/consensus/beacon/src/engine/sync.rs index 02da63d5aec..ba988f1be87 100644 --- a/crates/consensus/beacon/src/engine/sync.rs +++ b/crates/consensus/beacon/src/engine/sync.rs @@ -325,7 +325,7 @@ struct OrderedSealedBlock(SealedBlock); impl PartialOrd for OrderedSealedBlock { fn partial_cmp(&self, other: &Self) -> Option { - self.0.number.partial_cmp(&other.0.number) + Some(self.cmp(other)) } } diff --git a/crates/primitives/src/hex_bytes.rs b/crates/primitives/src/hex_bytes.rs index 322b9ff04e6..0e2db39e554 100644 --- a/crates/primitives/src/hex_bytes.rs +++ b/crates/primitives/src/hex_bytes.rs @@ -328,7 +328,7 @@ mod tests { assert_eq!(b, vec[..]); assert_eq!(vec[..], b); - let wrong_vec = vec![1, 3, 52, 137]; + let wrong_vec = [1, 3, 52, 137]; assert_ne!(b, wrong_vec[..]); assert_ne!(wrong_vec[..], b); } diff --git a/crates/revm/revm-inspectors/src/tracing/mod.rs b/crates/revm/revm-inspectors/src/tracing/mod.rs index 53a26f765a3..fe6c5b8609d 100644 --- a/crates/revm/revm-inspectors/src/tracing/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/mod.rs @@ -236,7 +236,7 @@ impl TracingInspector { /// /// This expects an existing [CallTrace], in other words, this panics if not within the context /// of a call. - fn start_step(&mut self, interp: &mut Interpreter, data: &mut EVMData<'_, DB>) { + fn start_step(&mut self, interp: &Interpreter, data: &EVMData<'_, DB>) { let trace_idx = self.last_trace_idx(); let trace = &mut self.traces.arena[trace_idx]; @@ -283,8 +283,8 @@ impl TracingInspector { /// Invoked on [Inspector::step_end]. fn fill_step_on_step_end( &mut self, - interp: &mut Interpreter, - data: &mut EVMData<'_, DB>, + interp: &Interpreter, + data: &EVMData<'_, DB>, status: InstructionResult, ) { let StackStep { trace_idx, step_idx } = diff --git a/crates/rlp/benches/bench.rs b/crates/rlp/benches/bench.rs index abb70c7eb09..e6969518314 100644 --- a/crates/rlp/benches/bench.rs +++ b/crates/rlp/benches/bench.rs @@ -48,7 +48,7 @@ fn bench_decode(c: &mut Criterion) { }); c.bench_function("decode_u256", |b| { b.iter(|| { - let data = vec![ + let data = [ 0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0, diff --git a/crates/rpc/rpc-builder/tests/it/utils.rs b/crates/rpc/rpc-builder/tests/it/utils.rs index 0b111b33cc8..7869f61574e 100644 --- a/crates/rpc/rpc-builder/tests/it/utils.rs +++ b/crates/rpc/rpc-builder/tests/it/utils.rs @@ -30,7 +30,7 @@ pub async fn launch_auth(secret: JwtSecret) -> AuthServerHandle { MAINNET.clone(), beacon_engine_handle, spawn_test_payload_service().into(), - Box::new(TokioTaskExecutor::default()), + Box::::default(), ); let module = AuthRpcModule::new(engine_api); module.start_server(config).await.unwrap() diff --git a/crates/rpc/rpc-types/src/eth/fee.rs b/crates/rpc/rpc-types/src/eth/fee.rs index 6463e7b433c..c7f93f6eaf6 100644 --- a/crates/rpc/rpc-types/src/eth/fee.rs +++ b/crates/rpc/rpc-types/src/eth/fee.rs @@ -12,10 +12,7 @@ pub struct TxGasAndReward { impl PartialOrd for TxGasAndReward { fn partial_cmp(&self, other: &Self) -> Option { - // compare only the reward - // see: - // - self.reward.partial_cmp(&other.reward) + Some(self.cmp(other)) } } diff --git a/crates/trie/src/hashed_cursor/post_state.rs b/crates/trie/src/hashed_cursor/post_state.rs index 662a71163ae..c5418bc6924 100644 --- a/crates/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/src/hashed_cursor/post_state.rs @@ -885,8 +885,7 @@ mod tests { let tx = db.tx().unwrap(); let factory = HashedPostStateCursorFactory::new(&tx, &hashed_post_state); let expected = - [(address, db_storage.into_iter().chain(post_state_storage.into_iter()).collect())] - .into_iter(); + [(address, db_storage.into_iter().chain(post_state_storage).collect())].into_iter(); assert_storage_cursor_order(&factory, expected); } From 10d1b61742c3a283b3f06e2f51de8b6cf430ed00 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 18 Jul 2023 09:55:08 -0400 Subject: [PATCH 116/150] chore: add traces for _new_ invalid blocks (#3821) Co-authored-by: Matthias Seitz --- crates/consensus/beacon/src/engine/mod.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index e42bc3c7170..f9fb030bf69 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -870,7 +870,7 @@ where // check if the new head was previously invalidated, if so then we deem this FCU // as invalid if let Some(invalid_ancestor) = self.check_invalid_ancestor(state.head_block_hash) { - debug!(target: "consensus::engine", head=?state.head_block_hash, "Head was previously marked as invalid"); + debug!(target: "consensus::engine", head=?state.head_block_hash, current_error=?error, "Head was previously marked as invalid"); return invalid_ancestor } @@ -1170,6 +1170,8 @@ where let (block, error) = err.split(); if error.is_invalid_block() { + warn!(target: "consensus::engine", invalid_hash=?block.hash, invalid_number=?block.number, ?error, "Invalid block error on new payload"); + // all of these occurred if the payload is invalid let parent_hash = block.parent_hash; @@ -1271,7 +1273,10 @@ where Err(err) => { warn!(target: "consensus::engine", ?err, "Failed to insert downloaded block"); if err.kind().is_invalid_block() { - self.invalid_headers.insert(err.into_block().header); + let (block, err) = err.split(); + warn!(target: "consensus::engine", invalid_number=?block.number, invalid_hash=?block.hash, ?err, "Marking block as invalid"); + + self.invalid_headers.insert(block.header); } } } @@ -1429,7 +1434,7 @@ where } if let ControlFlow::Unwind { bad_block, .. } = ctrl { - trace!(target: "consensus::engine", hash=?bad_block.hash, "Bad block detected in unwind"); + warn!(target: "consensus::engine", invalid_hash=?bad_block.hash, invalid_number=?bad_block.number, "Bad block detected in unwind"); // update the `invalid_headers` cache with the new invalid headers self.invalid_headers.insert(bad_block); From 686362b86890dd83739d92beeb8547310d737ee9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 18 Jul 2023 16:48:24 +0200 Subject: [PATCH 117/150] feat: build local pending block (#3811) --- crates/rpc/rpc/src/eth/api/block.rs | 3 +- crates/rpc/rpc/src/eth/api/mod.rs | 49 ++++-- crates/rpc/rpc/src/eth/api/pending_block.rs | 158 ++++++++++++++++++-- crates/rpc/rpc/src/eth/api/transactions.rs | 2 +- 4 files changed, 185 insertions(+), 27 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/block.rs b/crates/rpc/rpc/src/eth/api/block.rs index 5220f907a5a..1cb547e3153 100644 --- a/crates/rpc/rpc/src/eth/api/block.rs +++ b/crates/rpc/rpc/src/eth/api/block.rs @@ -7,6 +7,7 @@ use crate::{ }, EthApi, }; +use reth_network_api::NetworkInfo; use reth_primitives::{BlockId, BlockNumberOrTag, TransactionMeta}; use reth_provider::{BlockReaderIdExt, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::{Block, Index, RichBlock, TransactionReceipt}; @@ -16,7 +17,7 @@ impl EthApi where Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, Pool: TransactionPool + Clone + 'static, - Network: Send + Sync + 'static, + Network: NetworkInfo + Send + Sync + 'static, { /// Returns the uncle headers of the given block /// diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 50e634f424a..e68dd3686ef 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -4,6 +4,7 @@ //! files. use crate::eth::{ + api::pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}, cache::EthStateCache, error::{EthApiError, EthResult}, gas_oracle::GasPriceOracle, @@ -20,7 +21,11 @@ use reth_rpc_types::{SyncInfo, SyncStatus}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnv}; -use std::{future::Future, sync::Arc, time::Instant}; +use std::{ + future::Future, + sync::Arc, + time::{Duration, Instant}, +}; use tokio::sync::{oneshot, Mutex}; mod block; @@ -32,7 +37,6 @@ mod sign; mod state; mod transactions; -use crate::eth::api::pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; pub use transactions::{EthTransactions, TransactionSource}; /// `Eth` API trait. @@ -220,7 +224,7 @@ impl EthApi where Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, Pool: TransactionPool + Clone + 'static, - Network: Send + Sync + 'static, + Network: NetworkInfo + Send + Sync + 'static, { /// Configures the [CfgEnv] and [BlockEnv] for the pending block /// @@ -261,24 +265,41 @@ where // no pending block from the CL yet, so we need to build it ourselves via txpool self.on_blocking_task(|this| async move { - let PendingBlockEnv { cfg: _, block_env, origin } = pending; - let lock = this.inner.pending_block.lock().await; + let mut lock = this.inner.pending_block.lock().await; let now = Instant::now(); - // this is guaranteed to be the `latest` header - let parent_header = origin.into_header(); // check if the block is still good - if let Some(pending) = lock.as_ref() { - if block_env.number.to::() == pending.block.number && - pending.block.parent_hash == parent_header.parent_hash && - now <= pending.expires_at + if let Some(pending_block) = lock.as_ref() { + // this is guaranteed to be the `latest` header + if pending.block_env.number.to::() == pending_block.block.number && + pending.origin.header().hash == pending_block.block.parent_hash && + now <= pending_block.expires_at { - return Ok(Some(pending.block.clone())) + return Ok(Some(pending_block.block.clone())) } } - // TODO(mattsse): actually build the pending block - Ok(None) + // if we're currently syncing, we're unable to build a pending block + if this.network().is_syncing() { + return Ok(None) + } + + // we rebuild the block + let pending_block = match pending.build_block(this.provider(), this.pool()) { + Ok(block) => block, + Err(err) => { + tracing::debug!(target = "rpc", "Failed to build pending block: {:?}", err); + return Ok(None) + } + }; + + let now = Instant::now(); + *lock = Some(PendingBlock { + block: pending_block.clone(), + expires_at: now + Duration::from_secs(3), + }); + + Ok(Some(pending_block)) }) .await } diff --git a/crates/rpc/rpc/src/eth/api/pending_block.rs b/crates/rpc/rpc/src/eth/api/pending_block.rs index 8e57f893dca..a427df3279a 100644 --- a/crates/rpc/rpc/src/eth/api/pending_block.rs +++ b/crates/rpc/rpc/src/eth/api/pending_block.rs @@ -1,7 +1,18 @@ //! Support for building a pending block via local txpool. -use reth_primitives::{SealedBlock, SealedHeader}; -use revm_primitives::{BlockEnv, CfgEnv}; +use crate::eth::error::EthResult; +use reth_primitives::{ + constants::{BEACON_NONCE, EMPTY_WITHDRAWALS}, + proofs, Block, Header, IntoRecoveredTransaction, Receipt, SealedBlock, SealedHeader, + EMPTY_OMMER_ROOT, H256, U256, +}; +use reth_provider::{PostState, StateProviderFactory}; +use reth_revm::{ + database::State, env::tx_env_with_recovered, executor::commit_state_changes, into_reth_log, +}; +use reth_transaction_pool::TransactionPool; +use revm::db::CacheDB; +use revm_primitives::{BlockEnv, CfgEnv, EVMError, Env, InvalidTransaction, ResultAndState}; use std::time::Instant; /// Configured [BlockEnv] and [CfgEnv] for a pending block @@ -15,6 +26,133 @@ pub(crate) struct PendingBlockEnv { pub(crate) origin: PendingBlockEnvOrigin, } +impl PendingBlockEnv { + /// Builds a pending block from the given client and pool. + pub(crate) fn build_block( + self, + client: &Client, + pool: &Pool, + ) -> EthResult + where + Client: StateProviderFactory, + Pool: TransactionPool, + { + let Self { cfg, block_env, origin } = self; + + let parent_hash = origin.build_target_hash(); + let state = State::new(client.history_by_block_hash(parent_hash)?); + let mut db = CacheDB::new(state); + let mut post_state = PostState::default(); + + let mut cumulative_gas_used = 0; + let block_gas_limit: u64 = block_env.gas_limit.try_into().unwrap_or(u64::MAX); + let base_fee = block_env.basefee.to::(); + let block_number = block_env.number.to::(); + + let mut executed_txs = Vec::new(); + let mut best_txs = pool.best_transactions_with_base_fee(base_fee as u128); + + while let Some(pool_tx) = best_txs.next() { + // ensure we still have capacity for this transaction + if cumulative_gas_used + pool_tx.gas_limit() > block_gas_limit { + // we can't fit this transaction into the block, so we need to mark it as invalid + // which also removes all dependent transaction from the iterator before we can + // continue + best_txs.mark_invalid(&pool_tx); + continue + } + + // convert tx to a signed transaction + let tx = pool_tx.to_recovered_transaction(); + + // Configure the environment for the block. + let env = + Env { cfg: cfg.clone(), block: block_env.clone(), tx: tx_env_with_recovered(&tx) }; + + let mut evm = revm::EVM::with_env(env); + evm.database(&mut db); + + let ResultAndState { result, state } = match evm.transact() { + Ok(res) => res, + Err(err) => { + match err { + EVMError::Transaction(err) => { + if matches!(err, InvalidTransaction::NonceTooLow { .. }) { + // if the nonce is too low, we can skip this transaction + } else { + // if the transaction is invalid, we can skip it and all of its + // descendants + best_txs.mark_invalid(&pool_tx); + } + continue + } + err => { + // this is an error that we should treat as fatal for this attempt + return Err(err.into()) + } + } + } + }; + + let gas_used = result.gas_used(); + + // commit changes + commit_state_changes(&mut db, &mut post_state, block_number, state, true); + + // add gas used by the transaction to cumulative gas used, before creating the receipt + cumulative_gas_used += gas_used; + + // Push transaction changeset and calculate header bloom filter for receipt. + post_state.add_receipt( + block_number, + Receipt { + tx_type: tx.tx_type(), + success: result.is_success(), + cumulative_gas_used, + logs: result.logs().into_iter().map(into_reth_log).collect(), + }, + ); + // append transaction to the list of executed transactions + executed_txs.push(tx.into_signed()); + } + + let receipts_root = post_state.receipts_root(block_number); + let logs_bloom = post_state.logs_bloom(block_number); + + // calculate the state root + let state_root = db.db.state().state_root(post_state)?; + + // create the block header + let transactions_root = proofs::calculate_transaction_root(&executed_txs); + + let header = Header { + parent_hash, + ommers_hash: EMPTY_OMMER_ROOT, + beneficiary: block_env.coinbase, + state_root, + transactions_root, + receipts_root, + withdrawals_root: Some(EMPTY_WITHDRAWALS), + logs_bloom, + timestamp: block_env.timestamp.to::(), + mix_hash: block_env.prevrandao.unwrap_or_default(), + nonce: BEACON_NONCE, + base_fee_per_gas: Some(base_fee), + number: block_number, + gas_limit: block_gas_limit, + difficulty: U256::ZERO, + gas_used: cumulative_gas_used, + extra_data: Default::default(), + }; + + // seal the block + let block = Block { header, body: executed_txs, ommers: vec![], withdrawals: Some(vec![]) }; + let sealed_block = block.seal_slow(); + + Ok(sealed_block) + } +} + /// The origin for a configured [PendingBlockEnv] #[derive(Clone, Debug)] pub(crate) enum PendingBlockEnvOrigin { @@ -38,18 +176,18 @@ impl PendingBlockEnvOrigin { } } - /// Returns the header this pending block is based on. - pub(crate) fn header(&self) -> &SealedHeader { + /// Returns the hash of the pending block should be built on + fn build_target_hash(&self) -> H256 { match self { - PendingBlockEnvOrigin::ActualPending(block) => &block.header, - PendingBlockEnvOrigin::DerivedFromLatest(header) => header, + PendingBlockEnvOrigin::ActualPending(block) => block.parent_hash, + PendingBlockEnvOrigin::DerivedFromLatest(header) => header.hash, } } - /// Consumes the type and returns the header this pending block is based on. - pub(crate) fn into_header(self) -> SealedHeader { + /// Returns the header this pending block is based on. + pub(crate) fn header(&self) -> &SealedHeader { match self { - PendingBlockEnvOrigin::ActualPending(block) => block.header, + PendingBlockEnvOrigin::ActualPending(block) => &block.header, PendingBlockEnvOrigin::DerivedFromLatest(header) => header, } } @@ -63,5 +201,3 @@ pub(crate) struct PendingBlock { /// Timestamp when the pending block is considered outdated pub(crate) expires_at: Instant, } - -impl PendingBlock {} diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 80d83fdb809..e8dd2ed7ccb 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -663,7 +663,7 @@ impl EthApi where Pool: TransactionPool + 'static, Provider: BlockReaderIdExt + StateProviderFactory + EvmEnvProvider + 'static, - Network: Send + Sync + 'static, + Network: NetworkInfo + Send + Sync + 'static, { pub(crate) fn sign_request( &self, From fa005c4130d6ed2c00467b835fd2de3ff0cf1cc9 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 18 Jul 2023 10:52:09 -0400 Subject: [PATCH 118/150] chore: rename missing_parent to missing_ancestor (#3822) Co-authored-by: Matthias Seitz --- crates/blockchain-tree/src/blockchain_tree.rs | 10 +++++----- crates/consensus/beacon/src/engine/mod.rs | 4 +++- crates/interfaces/src/blockchain_tree/mod.rs | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 43fe1be6fa5..819d60d4c38 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -201,7 +201,7 @@ impl BlockchainTree // check if block is disconnected if let Some(block) = self.buffered_blocks.block(block) { - return Ok(Some(BlockStatus::Disconnected { missing_parent: block.parent_num_hash() })) + return Ok(Some(BlockStatus::Disconnected { missing_ancestor: block.parent_num_hash() })) } Ok(None) @@ -360,7 +360,7 @@ impl BlockchainTree ) })?; - Ok(BlockStatus::Disconnected { missing_parent: lowest_ancestor.parent_num_hash() }) + Ok(BlockStatus::Disconnected { missing_ancestor: lowest_ancestor.parent_num_hash() }) } /// This tries to append the given block to the canonical chain. @@ -1274,7 +1274,7 @@ mod tests { assert_eq!( tree.insert_block(block2.clone()).unwrap(), InsertPayloadOk::Inserted(BlockStatus::Disconnected { - missing_parent: block2.parent_num_hash() + missing_ancestor: block2.parent_num_hash() }) ); @@ -1293,7 +1293,7 @@ mod tests { assert_eq!( tree.is_block_known(block2.num_hash()).unwrap(), - Some(BlockStatus::Disconnected { missing_parent: block2.parent_num_hash() }) + Some(BlockStatus::Disconnected { missing_ancestor: block2.parent_num_hash() }) ); // check if random block is known @@ -1575,7 +1575,7 @@ mod tests { assert_eq!( tree.insert_block(block2b.clone()).unwrap(), InsertPayloadOk::Inserted(BlockStatus::Disconnected { - missing_parent: block2b.parent_num_hash() + missing_ancestor: block2b.parent_num_hash() }) ); diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index f9fb030bf69..e4f46ad9ea1 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1262,7 +1262,9 @@ where // block is connected to the canonical chain, but not the current head self.try_make_sync_target_canonical(downloaded_num_hash); } - InsertPayloadOk::Inserted(BlockStatus::Disconnected { missing_parent }) => { + InsertPayloadOk::Inserted(BlockStatus::Disconnected { + missing_ancestor: missing_parent, + }) => { // block is not connected to the canonical head, we need to download its // missing branch first self.on_disconnected_block(downloaded_num_hash, missing_parent); diff --git a/crates/interfaces/src/blockchain_tree/mod.rs b/crates/interfaces/src/blockchain_tree/mod.rs index 8025c790112..d8334df7dec 100644 --- a/crates/interfaces/src/blockchain_tree/mod.rs +++ b/crates/interfaces/src/blockchain_tree/mod.rs @@ -144,8 +144,8 @@ pub enum BlockStatus { Accepted, /// If blocks is not connected to canonical chain. Disconnected { - /// The lowest parent block that is not connected to the canonical chain. - missing_parent: BlockNumHash, + /// The lowest ancestor block that is not connected to the canonical chain. + missing_ancestor: BlockNumHash, }, } From f3b6d9e5efc5c8274835ce9f4dc99bc3872dd5ed Mon Sep 17 00:00:00 2001 From: Panagiotis Ganelis <50522617+PanGan21@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:02:13 +0300 Subject: [PATCH 119/150] feat: add timestamp to pool update (#3833) --- crates/transaction-pool/src/maintain.rs | 2 ++ crates/transaction-pool/src/pool/mod.rs | 1 + crates/transaction-pool/src/traits.rs | 2 ++ 3 files changed, 5 insertions(+) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index e8b8a8f1ed8..1b4084c0f30 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -266,6 +266,7 @@ pub async fn maintain_transaction_pool( changed_accounts, // all transactions mined in the new chain need to be removed from the pool mined_transactions: new_mined_transactions.into_iter().collect(), + timestamp: new_tip.timestamp, }; pool.on_canonical_state_change(update); @@ -331,6 +332,7 @@ pub async fn maintain_transaction_pool( pending_block_base_fee, changed_accounts, mined_transactions, + timestamp: tip.timestamp, }; pool.on_canonical_state_change(update); } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 2c055bf9f75..d3d3c5ca9aa 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -257,6 +257,7 @@ where pending_block_base_fee, changed_accounts, mined_transactions, + timestamp: _, } = update; let changed_senders = self.changed_senders(changed_accounts.into_iter()); let block_info = BlockInfo { diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 830971c9d9a..5af49270ab7 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -402,6 +402,8 @@ pub struct CanonicalStateUpdate { pub changed_accounts: Vec, /// All mined transactions in the block range. pub mined_transactions: Vec, + /// Timestamp of the latest chain update + pub timestamp: u64, } impl fmt::Display for CanonicalStateUpdate { From b7beb9defd2d5b60d577adc6229750240bbbdb73 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 18 Jul 2023 17:34:38 +0200 Subject: [PATCH 120/150] fix: map more errors to messages (#3837) --- .../revm-inspectors/src/tracing/config.rs | 19 +++++++++++++ .../revm/revm-inspectors/src/tracing/types.rs | 28 +++++++++++++++---- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/crates/revm/revm-inspectors/src/tracing/config.rs b/crates/revm/revm-inspectors/src/tracing/config.rs index f968993062d..5096d830756 100644 --- a/crates/revm/revm-inspectors/src/tracing/config.rs +++ b/crates/revm/revm-inspectors/src/tracing/config.rs @@ -1,5 +1,24 @@ use reth_rpc_types::trace::geth::GethDefaultTracingOptions; +/// What kind of tracing style this is. +/// +/// This affects things like error messages. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum TraceStyle { + /// Parity style tracer + Parity, + /// Geth style tracer + #[allow(unused)] + Geth, +} + +impl TraceStyle { + /// Returns true if this is a parity style tracer. + pub(crate) const fn is_parity(self) -> bool { + matches!(self, Self::Parity) + } +} + /// Gives guidance to the [TracingInspector](crate::tracing::TracingInspector). /// /// Use [TracingInspectorConfig::default_parity] or [TracingInspectorConfig::default_geth] to get diff --git a/crates/revm/revm-inspectors/src/tracing/types.rs b/crates/revm/revm-inspectors/src/tracing/types.rs index 548ae72b878..76ff0f51008 100644 --- a/crates/revm/revm-inspectors/src/tracing/types.rs +++ b/crates/revm/revm-inspectors/src/tracing/types.rs @@ -1,6 +1,6 @@ //! Types for representing call trace items. -use crate::tracing::utils::convert_memory; +use crate::tracing::{config::TraceStyle, utils::convert_memory}; use reth_primitives::{abi::decode_revert_reason, bytes::Bytes, Address, H256, U256}; use reth_rpc_types::trace::{ geth::{CallFrame, CallLogFrame, GethDefaultTracingOptions, StructLog}, @@ -160,9 +160,26 @@ impl CallTrace { } /// Returns the error message if it is an erroneous result. - pub(crate) fn as_error(&self) -> Option { + pub(crate) fn as_error(&self, kind: TraceStyle) -> Option { + // See also self.is_error().then(|| match self.status { - InstructionResult::Revert => "Reverted".to_string(), + InstructionResult::Revert => { + if kind.is_parity() { "Reverted" } else { "execution reverted" }.to_string() + } + InstructionResult::OutOfGas | InstructionResult::MemoryOOG => { + if kind.is_parity() { "Out of gas" } else { "out of gas" }.to_string() + } + InstructionResult::OpcodeNotFound => { + if kind.is_parity() { "Bad instruction" } else { "invalid opcode" }.to_string() + } + InstructionResult::StackOverflow => "Out of stack".to_string(), + InstructionResult::InvalidJump => { + if kind.is_parity() { "Bad jump destination" } else { "invalid jump destination" } + .to_string() + } + InstructionResult::PrecompileError => { + if kind.is_parity() { "Built-in failed" } else { "precompiled failed" }.to_string() + } status => format!("{:?}", status), }) } @@ -324,7 +341,7 @@ impl CallTraceNode { pub(crate) fn parity_transaction_trace(&self, trace_address: Vec) -> TransactionTrace { let action = self.parity_action(); let output = self.parity_trace_output(); - let error = self.trace.as_error(); + let error = self.trace.as_error(TraceStyle::Parity); TransactionTrace { action, error, @@ -402,7 +419,8 @@ impl CallTraceNode { // we need to populate error and revert reason if !self.trace.success { call_frame.revert_reason = decode_revert_reason(self.trace.output.clone()); - call_frame.error = self.trace.as_error(); + // Note: the call tracer mimics parity's trace transaction and geth maps errors to parity style error messages, + call_frame.error = self.trace.as_error(TraceStyle::Parity); } if include_logs && !self.logs.is_empty() { From ed92285d7a966b8b4cd622df83322c9b8abd0394 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 18 Jul 2023 18:55:59 +0200 Subject: [PATCH 121/150] feat: add builder type for eth tx validator (#3828) --- crates/transaction-pool/src/validate/eth.rs | 185 ++++++++++++++++---- crates/transaction-pool/src/validate/mod.rs | 2 +- 2 files changed, 154 insertions(+), 33 deletions(-) diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index b5ca93709dd..2a4bbbf8ec2 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -28,10 +28,18 @@ pub struct EthTransactionValidator { // === impl EthTransactionValidator === +impl EthTransactionValidator<(), ()> { + /// Convenience method to create a [EthTransactionValidatorBuilder] + pub fn builder(chain_spec: Arc) -> EthTransactionValidatorBuilder { + EthTransactionValidatorBuilder::new(chain_spec) + } +} + impl EthTransactionValidator { /// Creates a new instance for the given [ChainSpec] /// - /// This will spawn a single validation tasks that performs the actual validation. + /// This will spawn a single validation tasks that performs the actual validation. + /// See [EthTransactionValidator::with_additional_tasks] pub fn new(client: Client, chain_spec: Arc, tasks: T) -> Self where T: TaskSpawner, @@ -41,6 +49,11 @@ impl EthTransactionValidator { /// Creates a new instance for the given [ChainSpec] /// + /// By default this will enable support for: + /// - shanghai + /// - eip1559 + /// - eip2930 + /// /// This will always spawn a validation task that performs the actual validation. It will spawn /// `num_additional_tasks` additional tasks. pub fn with_additional_tasks( @@ -52,37 +65,9 @@ impl EthTransactionValidator { where T: TaskSpawner, { - let inner = EthTransactionValidatorInner { - chain_spec, - client, - shanghai: true, - eip2718: true, - eip1559: true, - block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT, - minimum_priority_fee: None, - _marker: Default::default(), - }; - - let (tx, task) = ValidationTask::new(); - - // Spawn validation tasks, they are blocking because they perform db lookups - for _ in 0..num_additional_tasks { - let task = task.clone(); - tasks.spawn_blocking(Box::pin(async move { - task.run().await; - })); - } - - tasks.spawn_critical_blocking( - "transaction-validation-service", - Box::pin(async move { - task.run().await; - }), - ); - - let to_validation_task = Arc::new(Mutex::new(tx)); - - Self { inner: Arc::new(inner), to_validation_task } + EthTransactionValidatorBuilder::new(chain_spec) + .with_additional_tasks(num_additional_tasks) + .build(client, tasks) } /// Returns the configured chain id @@ -134,6 +119,142 @@ where } } +/// A builder for [EthTransactionValidator] +#[derive(Debug, Clone)] +pub struct EthTransactionValidatorBuilder { + chain_spec: Arc, + /// Fork indicator whether we are in the Shanghai stage. + shanghai: bool, + /// Fork indicator whether we are using EIP-2718 type transactions. + eip2718: bool, + /// Fork indicator whether we are using EIP-1559 type transactions. + eip1559: bool, + /// The current max gas limit + block_gas_limit: u64, + /// Minimum priority fee to enforce for acceptance into the pool. + minimum_priority_fee: Option, + /// Determines how many additional tasks to spawn + /// + /// Default is 1 + additional_tasks: usize, +} + +impl EthTransactionValidatorBuilder { + /// Creates a new builder for the given [ChainSpec] + pub fn new(chain_spec: Arc) -> Self { + Self { + chain_spec, + shanghai: true, + eip2718: true, + eip1559: true, + block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT, + minimum_priority_fee: None, + additional_tasks: 1, + } + } + + /// Disables the Shanghai fork. + pub fn no_shanghai(self) -> Self { + self.set_shanghai(false) + } + + /// Set the Shanghai fork. + pub fn set_shanghai(mut self, shanghai: bool) -> Self { + self.shanghai = shanghai; + self + } + + /// Disables the eip2718 support. + pub fn no_eip2718(self) -> Self { + self.set_eip2718(false) + } + + /// Set eip2718 support. + pub fn set_eip2718(mut self, eip2718: bool) -> Self { + self.eip2718 = eip2718; + self + } + + /// Disables the eip1559 support. + pub fn no_eip1559(self) -> Self { + self.set_eip1559(false) + } + + /// Set the eip1559 support. + pub fn set_eip1559(mut self, eip1559: bool) -> Self { + self.eip1559 = eip1559; + self + } + + /// Sets a minimum priority fee that's enforced for acceptance into the pool. + pub fn with_minimum_priority_fee(mut self, minimum_priority_fee: u128) -> Self { + self.minimum_priority_fee = Some(minimum_priority_fee); + self + } + + /// Sets the number of additional tasks to spawn. + pub fn with_additional_tasks(mut self, additional_tasks: usize) -> Self { + self.additional_tasks = additional_tasks; + self + } + + /// Builds a [EthTransactionValidator] + /// + /// The validator will spawn `additional_tasks` additional tasks for validation. + /// + /// By default this will spawn 1 additional task. + pub fn build( + self, + client: Client, + tasks: T, + ) -> EthTransactionValidator + where + T: TaskSpawner, + { + let Self { + chain_spec, + shanghai, + eip2718, + eip1559, + block_gas_limit, + minimum_priority_fee, + additional_tasks, + } = self; + + let inner = EthTransactionValidatorInner { + chain_spec, + client, + shanghai, + eip2718, + eip1559, + block_gas_limit, + minimum_priority_fee, + _marker: Default::default(), + }; + + let (tx, task) = ValidationTask::new(); + + // Spawn validation tasks, they are blocking because they perform db lookups + for _ in 0..additional_tasks { + let task = task.clone(); + tasks.spawn_blocking(Box::pin(async move { + task.run().await; + })); + } + + tasks.spawn_critical_blocking( + "transaction-validation-service", + Box::pin(async move { + task.run().await; + }), + ); + + let to_validation_task = Arc::new(Mutex::new(tx)); + + EthTransactionValidator { inner: Arc::new(inner), to_validation_task } + } +} + /// A [TransactionValidator] implementation that validates ethereum transaction. #[derive(Debug, Clone)] struct EthTransactionValidatorInner { diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index a3892f52ffe..f63c08c334c 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -14,7 +14,7 @@ mod eth; mod task; /// A [TransactionValidator] implementation that validates ethereum transaction. -pub use eth::EthTransactionValidator; +pub use eth::{EthTransactionValidator, EthTransactionValidatorBuilder}; /// A spawnable task that performs transaction validation. pub use task::ValidationTask; From 12155416581781e28df610078ffec79732f09d6a Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Tue, 18 Jul 2023 10:49:23 -0700 Subject: [PATCH 122/150] feat: implement reth_getBalanceChangesInBlock (#3768) Co-authored-by: Matthias Seitz --- bin/reth/src/args/rpc_server_args.rs | 6 +- crates/rpc/rpc-api/src/lib.rs | 2 + crates/rpc/rpc-api/src/reth.rs | 15 +++ crates/rpc/rpc-builder/src/lib.rs | 35 ++++-- crates/rpc/rpc/src/lib.rs | 2 + crates/rpc/rpc/src/reth.rs | 118 ++++++++++++++++++ crates/storage/provider/src/lib.rs | 4 +- .../src/providers/database/provider.rs | 18 ++- crates/storage/provider/src/providers/mod.rs | 13 +- .../storage/provider/src/test_utils/noop.rs | 14 ++- crates/storage/provider/src/traits/account.rs | 8 ++ crates/storage/provider/src/traits/mod.rs | 2 +- 12 files changed, 219 insertions(+), 18 deletions(-) create mode 100644 crates/rpc/rpc-api/src/reth.rs create mode 100644 crates/rpc/rpc/src/reth.rs diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index a1c954b4a8a..4dbde75cbd4 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -8,8 +8,8 @@ use clap::{ use futures::TryFutureExt; use reth_network_api::{NetworkInfo, Peers}; use reth_provider::{ - BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, EvmEnvProvider, HeaderProvider, - StateProviderFactory, + BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, EvmEnvProvider, + HeaderProvider, StateProviderFactory, }; use reth_rpc::{ eth::{ @@ -249,6 +249,7 @@ impl RpcServerArgs { + StateProviderFactory + EvmEnvProvider + ChainSpecProvider + + ChangeSetReader + Clone + Unpin + 'static, @@ -310,6 +311,7 @@ impl RpcServerArgs { + StateProviderFactory + EvmEnvProvider + ChainSpecProvider + + ChangeSetReader + Clone + Unpin + 'static, diff --git a/crates/rpc/rpc-api/src/lib.rs b/crates/rpc/rpc-api/src/lib.rs index acca756711c..7fba5adba9e 100644 --- a/crates/rpc/rpc-api/src/lib.rs +++ b/crates/rpc/rpc-api/src/lib.rs @@ -26,6 +26,7 @@ mod eth; mod eth_filter; mod eth_pubsub; mod net; +mod reth; mod rpc; mod trace; mod txpool; @@ -44,6 +45,7 @@ pub mod servers { eth_filter::EthFilterApiServer, eth_pubsub::EthPubSubApiServer, net::NetApiServer, + reth::RethApiServer, rpc::RpcApiServer, trace::TraceApiServer, txpool::TxPoolApiServer, diff --git a/crates/rpc/rpc-api/src/reth.rs b/crates/rpc/rpc-api/src/reth.rs new file mode 100644 index 00000000000..1e9c4314ab1 --- /dev/null +++ b/crates/rpc/rpc-api/src/reth.rs @@ -0,0 +1,15 @@ +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use reth_primitives::{Address, BlockId, U256}; +use std::collections::HashMap; + +/// Reth API namespace for reth-specific methods +#[cfg_attr(not(feature = "client"), rpc(server, namespace = "reth"))] +#[cfg_attr(feature = "client", rpc(server, client, namespace = "reth"))] +pub trait RethApi { + /// Returns all ETH balance changes in a block + #[method(name = "getBalanceChangesInBlock")] + async fn reth_get_balance_changes_in_block( + &self, + block_id: BlockId, + ) -> RpcResult>; +} diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index d01036f8d4b..368960517b7 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -31,13 +31,13 @@ //! //! ``` //! use reth_network_api::{NetworkInfo, Peers}; -//! use reth_provider::{BlockReaderIdExt, ChainSpecProvider, CanonStateSubscriptions, StateProviderFactory, EvmEnvProvider}; +//! use reth_provider::{BlockReaderIdExt, ChainSpecProvider, CanonStateSubscriptions, StateProviderFactory, EvmEnvProvider, ChangeSetReader}; //! use reth_rpc_builder::{RethRpcModule, RpcModuleBuilder, RpcServerConfig, ServerBuilder, TransportRpcModuleConfig}; //! use reth_tasks::TokioTaskExecutor; //! use reth_transaction_pool::TransactionPool; //! pub async fn launch(provider: Provider, pool: Pool, network: Network, events: Events) //! where -//! Provider: BlockReaderIdExt + ChainSpecProvider + StateProviderFactory + EvmEnvProvider + Clone + Unpin + 'static, +//! Provider: BlockReaderIdExt + ChainSpecProvider + ChangeSetReader + StateProviderFactory + EvmEnvProvider + Clone + Unpin + 'static, //! Pool: TransactionPool + Clone + 'static, //! Network: NetworkInfo + Peers + Clone + 'static, //! Events: CanonStateSubscriptions + Clone + 'static, @@ -64,7 +64,7 @@ //! ``` //! use tokio::try_join; //! use reth_network_api::{NetworkInfo, Peers}; -//! use reth_provider::{BlockReaderIdExt, ChainSpecProvider, CanonStateSubscriptions, StateProviderFactory, EvmEnvProvider}; +//! use reth_provider::{BlockReaderIdExt, ChainSpecProvider, CanonStateSubscriptions, StateProviderFactory, EvmEnvProvider, ChangeSetReader}; //! use reth_rpc::JwtSecret; //! use reth_rpc_builder::{RethRpcModule, RpcModuleBuilder, RpcServerConfig, TransportRpcModuleConfig}; //! use reth_tasks::TokioTaskExecutor; @@ -73,7 +73,7 @@ //! use reth_rpc_builder::auth::AuthServerConfig; //! pub async fn launch(provider: Provider, pool: Pool, network: Network, events: Events, engine_api: EngineApi) //! where -//! Provider: BlockReaderIdExt + ChainSpecProvider + StateProviderFactory + EvmEnvProvider + Clone + Unpin + 'static, +//! Provider: BlockReaderIdExt + ChainSpecProvider + ChangeSetReader + StateProviderFactory + EvmEnvProvider + Clone + Unpin + 'static, //! Pool: TransactionPool + Clone + 'static, //! Network: NetworkInfo + Peers + Clone + 'static, //! Events: CanonStateSubscriptions + Clone + 'static, @@ -113,8 +113,8 @@ use jsonrpsee::{ use reth_ipc::server::IpcServer; use reth_network_api::{NetworkInfo, Peers}; use reth_provider::{ - BlockReader, BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, EvmEnvProvider, - StateProviderFactory, + BlockReader, BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, + EvmEnvProvider, StateProviderFactory, }; use reth_rpc::{ eth::{ @@ -122,7 +122,7 @@ use reth_rpc::{ gas_oracle::GasPriceOracle, }, AdminApi, DebugApi, EngineEthApi, EthApi, EthFilter, EthPubSub, EthSubscriptionIdProvider, - NetApi, RPCApi, TraceApi, TracingCallGuard, TxPoolApi, Web3Api, + NetApi, RPCApi, RethApi, TraceApi, TracingCallGuard, TxPoolApi, Web3Api, }; use reth_rpc_api::{servers::*, EngineApiServer}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; @@ -176,6 +176,7 @@ where + StateProviderFactory + EvmEnvProvider + ChainSpecProvider + + ChangeSetReader + Clone + Unpin + 'static, @@ -320,6 +321,7 @@ where + StateProviderFactory + EvmEnvProvider + ChainSpecProvider + + ChangeSetReader + Clone + Unpin + 'static, @@ -543,6 +545,7 @@ impl RpcModuleSelection { + StateProviderFactory + EvmEnvProvider + ChainSpecProvider + + ChangeSetReader + Clone + Unpin + 'static, @@ -646,6 +649,8 @@ pub enum RethRpcModule { Web3, /// `rpc_` module Rpc, + /// `reth_` module + Reth, } // === impl RethRpcModule === @@ -758,6 +763,7 @@ where + StateProviderFactory + EvmEnvProvider + ChainSpecProvider + + ChangeSetReader + Clone + Unpin + 'static, @@ -839,6 +845,15 @@ where self } + /// Register Reth namespace + pub fn register_reth(&mut self) -> &mut Self { + self.modules.insert( + RethRpcModule::Reth, + RethApi::new(self.provider.clone(), Box::new(self.executor.clone())).into_rpc().into(), + ); + self + } + /// Helper function to create a [RpcModule] if it's not `None` fn maybe_module(&mut self, config: Option<&RpcModuleSelection>) -> Option> { let config = config?; @@ -921,6 +936,11 @@ where ) .into_rpc() .into(), + RethRpcModule::Reth => { + RethApi::new(self.provider.clone(), Box::new(self.executor.clone())) + .into_rpc() + .into() + } }) .clone() }) @@ -1754,6 +1774,7 @@ mod tests { "trace" => RethRpcModule::Trace, "web3" => RethRpcModule::Web3, "rpc" => RethRpcModule::Rpc, + "reth" => RethRpcModule::Reth, ); } diff --git a/crates/rpc/rpc/src/lib.rs b/crates/rpc/rpc/src/lib.rs index 8ec0893a028..64083f746b2 100644 --- a/crates/rpc/rpc/src/lib.rs +++ b/crates/rpc/rpc/src/lib.rs @@ -37,6 +37,7 @@ mod engine; pub mod eth; mod layers; mod net; +mod reth; mod rpc; mod trace; mod txpool; @@ -49,6 +50,7 @@ pub use engine::{EngineApi, EngineEthApi}; pub use eth::{EthApi, EthApiSpec, EthFilter, EthPubSub, EthSubscriptionIdProvider}; pub use layers::{AuthLayer, AuthValidator, Claims, JwtAuthValidator, JwtError, JwtSecret}; pub use net::NetApi; +pub use reth::RethApi; pub use rpc::RPCApi; pub use trace::TraceApi; pub use txpool::TxPoolApi; diff --git a/crates/rpc/rpc/src/reth.rs b/crates/rpc/rpc/src/reth.rs new file mode 100644 index 00000000000..7fbdf25ab46 --- /dev/null +++ b/crates/rpc/rpc/src/reth.rs @@ -0,0 +1,118 @@ +use crate::eth::error::{EthApiError, EthResult}; +use async_trait::async_trait; +use jsonrpsee::core::RpcResult; +use reth_interfaces::Result; +use reth_primitives::{Address, BlockId, U256}; +use reth_provider::{BlockReaderIdExt, ChangeSetReader, StateProviderFactory}; +use reth_rpc_api::RethApiServer; +use reth_tasks::TaskSpawner; +use std::{collections::HashMap, future::Future, sync::Arc}; +use tokio::sync::oneshot; + +/// `reth` API implementation. +/// +/// This type provides the functionality for handling `reth` prototype RPC requests. +pub struct RethApi { + inner: Arc>, +} + +// === impl RethApi === + +impl RethApi { + /// The provider that can interact with the chain. + pub fn provider(&self) -> &Provider { + &self.inner.provider + } + + /// Create a new instance of the [RethApi] + pub fn new(provider: Provider, task_spawner: Box) -> Self { + let inner = Arc::new(RethApiInner { provider, task_spawner }); + Self { inner } + } +} + +impl RethApi +where + Provider: BlockReaderIdExt + ChangeSetReader + StateProviderFactory + 'static, +{ + /// Executes the future on a new blocking task. + async fn on_blocking_task(&self, c: C) -> EthResult + where + C: FnOnce(Self) -> F, + F: Future> + Send + 'static, + R: Send + 'static, + { + let (tx, rx) = oneshot::channel(); + let this = self.clone(); + let f = c(this); + self.inner.task_spawner.spawn_blocking(Box::pin(async move { + let res = f.await; + let _ = tx.send(res); + })); + rx.await.map_err(|_| EthApiError::InternalEthError)? + } + + /// Returns a map of addresses to changed account balanced for a particular block. + pub async fn balance_changes_in_block( + &self, + block_id: BlockId, + ) -> EthResult> { + self.on_blocking_task(|this| async move { this.try_balance_changes_in_block(block_id) }) + .await + } + + fn try_balance_changes_in_block(&self, block_id: BlockId) -> EthResult> { + let block_id = block_id; + let Some(block_number) = self.provider().block_number_for_id(block_id)? else { + return Err(EthApiError::UnknownBlockNumber) + }; + + let state = self.provider().state_by_block_id(block_id)?; + let accounts_before = self.provider().account_block_changeset(block_number)?; + let hash_map = accounts_before.iter().try_fold( + HashMap::new(), + |mut hash_map, account_before| -> Result<_> { + let current_balance = state.account_balance(account_before.address)?; + let prev_balance = account_before.info.map(|info| info.balance); + if current_balance != prev_balance { + hash_map.insert(account_before.address, current_balance.unwrap_or_default()); + } + Ok(hash_map) + }, + )?; + Ok(hash_map) + } +} + +#[async_trait] +impl RethApiServer for RethApi +where + Provider: BlockReaderIdExt + ChangeSetReader + StateProviderFactory + 'static, +{ + /// Handler for `reth_getBalanceChangesInBlock` + async fn reth_get_balance_changes_in_block( + &self, + block_id: BlockId, + ) -> RpcResult> { + Ok(RethApi::balance_changes_in_block(self, block_id).await?) + } +} + +impl std::fmt::Debug for RethApi { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RethApi").finish_non_exhaustive() + } +} + +impl Clone for RethApi { + fn clone(&self) -> Self { + Self { inner: Arc::clone(&self.inner) } + } +} + +struct RethApiInner { + /// The provider that can interact with the chain. + provider: Provider, + /// The type that can spawn tasks which would otherwise block. + task_spawner: Box, +} diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index a6a1f6bf298..64c9932711e 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -25,8 +25,8 @@ pub use traits::{ BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, BlockWriter, BlockchainTreePendingStateProvider, CanonChainTracker, CanonStateNotification, CanonStateNotificationSender, CanonStateNotifications, CanonStateSubscriptions, - ChainSpecProvider, EvmEnvProvider, ExecutorFactory, HashingWriter, HeaderProvider, - HistoryWriter, PostStateDataProvider, ReceiptProvider, ReceiptProviderIdExt, + ChainSpecProvider, ChangeSetReader, EvmEnvProvider, ExecutorFactory, HashingWriter, + HeaderProvider, HistoryWriter, PostStateDataProvider, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StageCheckpointWriter, StateProvider, StateProviderBox, StateProviderFactory, StateRootProvider, StorageReader, TransactionsProvider, WithdrawalsProvider, diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 16f25d9792f..7205710ab1a 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,6 +1,8 @@ use crate::{ post_state::StorageChangeset, - traits::{AccountExtReader, BlockSource, ReceiptProvider, StageCheckpointWriter}, + traits::{ + AccountExtReader, BlockSource, ChangeSetReader, ReceiptProvider, StageCheckpointWriter, + }, AccountReader, BlockExecutionWriter, BlockHashReader, BlockNumReader, BlockReader, BlockWriter, EvmEnvProvider, HashingWriter, HeaderProvider, HistoryWriter, PostState, ProviderError, StageCheckpointReader, StorageReader, TransactionsProvider, WithdrawalsProvider, @@ -726,6 +728,20 @@ impl<'this, TX: DbTx<'this>> AccountExtReader for DatabaseProvider<'this, TX> { } } +impl<'this, TX: DbTx<'this>> ChangeSetReader for DatabaseProvider<'this, TX> { + fn account_block_changeset(&self, block_number: BlockNumber) -> Result> { + let range = block_number..=block_number; + self.tx + .cursor_read::()? + .walk_range(range)? + .map(|result| -> Result<_> { + let (_, account_before) = result?; + Ok(account_before) + }) + .collect() + } +} + impl<'this, TX: DbTx<'this>> HeaderProvider for DatabaseProvider<'this, TX> { fn header(&self, block_hash: &BlockHash) -> Result> { if let Some(num) = self.block_number(*block_hash)? { diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 083f15f9887..27e4903bb1d 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -1,7 +1,7 @@ use crate::{ BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockchainTreePendingStateProvider, CanonChainTracker, CanonStateNotifications, - CanonStateSubscriptions, ChainSpecProvider, EvmEnvProvider, HeaderProvider, + CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, EvmEnvProvider, HeaderProvider, PostStateDataProvider, ProviderError, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProviderBox, StateProviderFactory, TransactionsProvider, WithdrawalsProvider, @@ -39,6 +39,7 @@ mod state; use crate::{providers::chain_info::ChainInfoTracker, traits::BlockSource}; pub use database::*; pub use post_state_provider::PostStateProvider; +use reth_db::models::AccountBeforeTx; use reth_interfaces::blockchain_tree::{ error::InsertBlockError, CanonicalOutcome, InsertPayloadOk, }; @@ -813,3 +814,13 @@ where self.tree.subscribe_to_canonical_state() } } + +impl ChangeSetReader for BlockchainProvider +where + DB: Database, + Tree: Sync + Send, +{ + fn account_block_changeset(&self, block_number: BlockNumber) -> Result> { + self.database.provider()?.account_block_changeset(block_number) + } +} diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index 8fa349b11b5..def01741e68 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -1,11 +1,11 @@ use crate::{ traits::{BlockSource, ReceiptProvider}, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, - ChainSpecProvider, EvmEnvProvider, HeaderProvider, PostState, ReceiptProviderIdExt, - StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, - StateRootProvider, TransactionsProvider, WithdrawalsProvider, + ChainSpecProvider, ChangeSetReader, EvmEnvProvider, HeaderProvider, PostState, + ReceiptProviderIdExt, StageCheckpointReader, StateProvider, StateProviderBox, + StateProviderFactory, StateRootProvider, TransactionsProvider, WithdrawalsProvider, }; -use reth_db::models::StoredBlockBodyIndices; +use reth_db::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_interfaces::Result; use reth_primitives::{ stage::{StageCheckpoint, StageId}, @@ -235,6 +235,12 @@ impl AccountReader for NoopProvider { } } +impl ChangeSetReader for NoopProvider { + fn account_block_changeset(&self, _block_number: BlockNumber) -> Result> { + Ok(Vec::default()) + } +} + impl StateRootProvider for NoopProvider { fn state_root(&self, _post_state: PostState) -> Result { todo!() diff --git a/crates/storage/provider/src/traits/account.rs b/crates/storage/provider/src/traits/account.rs index d08d15a1218..5ae4fe60a77 100644 --- a/crates/storage/provider/src/traits/account.rs +++ b/crates/storage/provider/src/traits/account.rs @@ -1,4 +1,5 @@ use auto_impl::auto_impl; +use reth_db::models::AccountBeforeTx; use reth_interfaces::Result; use reth_primitives::{Account, Address, BlockNumber}; use std::{ @@ -42,3 +43,10 @@ pub trait AccountExtReader: Send + Sync { range: RangeInclusive, ) -> Result>>; } + +/// AccountChange reader +#[auto_impl(&, Arc, Box)] +pub trait ChangeSetReader: Send + Sync { + /// Iterate over account changesets and return the account state from before this block. + fn account_block_changeset(&self, block_number: BlockNumber) -> Result>; +} diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index 0411c995fce..5343185bdf1 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -1,7 +1,7 @@ //! Collection of common provider traits. mod account; -pub use account::{AccountExtReader, AccountReader}; +pub use account::{AccountExtReader, AccountReader, ChangeSetReader}; mod storage; pub use storage::StorageReader; From ac0acc293de30c9fdaad01f5a99113ba2cfe36f3 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 18 Jul 2023 14:16:17 -0400 Subject: [PATCH 123/150] feat: let consensus tests configure pipeline, executor, and client (#3839) --- Cargo.lock | 2 + crates/consensus/beacon/Cargo.toml | 2 + crates/consensus/beacon/src/engine/mod.rs | 398 ++++++++++++++++------ 3 files changed, 289 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d013576e75c..b5cfbf016df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5050,12 +5050,14 @@ dependencies = [ "reth-blockchain-tree", "reth-consensus-common", "reth-db", + "reth-downloaders", "reth-interfaces", "reth-metrics", "reth-payload-builder", "reth-primitives", "reth-provider", "reth-prune", + "reth-revm", "reth-rpc-types", "reth-stages", "reth-tasks", diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index 239afd997b3..47967ea76f4 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -40,6 +40,8 @@ reth-blockchain-tree = { path = "../../blockchain-tree", features = ["test-utils reth-db = { path = "../../storage/db", features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-tracing = { path = "../../tracing" } +reth-revm = { path = "../../revm" } +reth-downloaders = { path = "../../net/downloaders" } assert_matches = "1.5" diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index e4f46ad9ea1..4e1e6668261 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1716,20 +1716,30 @@ mod tests { BlockchainTree, ShareableBlockchainTree, }; use reth_db::{test_utils::create_test_rw_db, DatabaseEnv}; + use reth_downloaders::{ + bodies::bodies::BodiesDownloaderBuilder, + headers::reverse_headers::ReverseHeadersDownloaderBuilder, + }; use reth_interfaces::{ + consensus::Consensus, + p2p::either::EitherDownloader, sync::NoopSyncStateUpdater, test_utils::{NoopFullBlockClient, TestConsensus}, }; use reth_payload_builder::test_utils::spawn_test_payload_service; use reth_primitives::{stage::StageCheckpoint, ChainSpec, ChainSpecBuilder, H256, MAINNET}; use reth_provider::{ - providers::BlockchainProvider, test_utils::TestExecutorFactory, BlockWriter, - ProviderFactory, + providers::BlockchainProvider, test_utils::TestExecutorFactory, BlockExecutor, BlockWriter, + ExecutorFactory, ProviderFactory, StateProvider, }; + use reth_revm::Factory; use reth_rpc_types::engine::{ ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, }; - use reth_stages::{test_utils::TestStages, ExecOutput, PipelineError, StageError}; + use reth_stages::{ + sets::DefaultStages, stages::HeaderSyncMode, test_utils::TestStages, ExecOutput, + PipelineError, StageError, + }; use reth_tasks::TokioTaskExecutor; use std::{collections::VecDeque, sync::Arc, time::Duration}; use tokio::sync::{ @@ -1737,13 +1747,17 @@ mod tests { watch, }; - type TestBeaconConsensusEngine = BeaconConsensusEngine< + type TestBeaconConsensusEngine = BeaconConsensusEngine< Arc, BlockchainProvider< Arc, - ShareableBlockchainTree, TestConsensus, TestExecutorFactory>, + ShareableBlockchainTree< + Arc, + Arc, + EitherExecutorFactory, + >, >, - NoopFullBlockClient, + Arc>, >; struct TestEnv { @@ -1806,22 +1820,124 @@ mod tests { } } - struct TestConsensusEngineBuilder { + /// Represents either test pipeline outputs, or real pipeline configuration. + #[derive(Default)] + enum TestPipelineConfig { + /// Test pipeline outputs. + Test(VecDeque>), + /// Real pipeline configuration. + #[default] + Real, + } + + /// Represents either test executor results, or real executor configuration. + #[derive(Default)] + enum TestExecutorConfig { + /// Test executor results. + Test(Vec), + /// Real executor configuration. + #[default] + Real, + } + + /// A type that represents one of two possible executor factories. + #[derive(Debug, Clone)] + enum EitherExecutorFactory { + /// The first factory variant + Left(A), + /// The second factory variant + Right(B), + } + + // A type that represents one of two possible BlockExecutor types. + #[derive(Debug)] + enum EitherBlockExecutor { + /// The first executor variant + Left(A), + /// The second executor variant + Right(B), + } + + impl BlockExecutor for EitherBlockExecutor + where + A: BlockExecutor, + B: BlockExecutor, + SP: StateProvider, + { + fn execute( + &mut self, + block: &reth_primitives::Block, + total_difficulty: U256, + senders: Option>, + ) -> Result { + match self { + EitherBlockExecutor::Left(a) => a.execute(block, total_difficulty, senders), + EitherBlockExecutor::Right(b) => b.execute(block, total_difficulty, senders), + } + } + + fn execute_and_verify_receipt( + &mut self, + block: &reth_primitives::Block, + total_difficulty: U256, + senders: Option>, + ) -> Result { + match self { + EitherBlockExecutor::Left(a) => { + a.execute_and_verify_receipt(block, total_difficulty, senders) + } + EitherBlockExecutor::Right(b) => { + b.execute_and_verify_receipt(block, total_difficulty, senders) + } + } + } + } + + impl ExecutorFactory for EitherExecutorFactory + where + A: ExecutorFactory, + B: ExecutorFactory, + { + type Executor = EitherBlockExecutor, B::Executor>; + + fn chain_spec(&self) -> &ChainSpec { + match self { + EitherExecutorFactory::Left(a) => a.chain_spec(), + EitherExecutorFactory::Right(b) => b.chain_spec(), + } + } + + fn with_sp(&self, sp: SP) -> Self::Executor { + match self { + EitherExecutorFactory::Left(a) => EitherBlockExecutor::Left(a.with_sp(sp)), + EitherExecutorFactory::Right(b) => EitherBlockExecutor::Right(b.with_sp(sp)), + } + } + } + + /// A builder for `TestConsensusEngine`, allows configuration of mocked pipeline outputs and + /// mocked executor results. + struct TestConsensusEngineBuilder { chain_spec: Arc, - pipeline_exec_outputs: VecDeque>, - executor_results: Vec, + pipeline_config: TestPipelineConfig, + executor_config: TestExecutorConfig, pipeline_run_threshold: Option, max_block: Option, + client: Option, } - impl TestConsensusEngineBuilder { + impl TestConsensusEngineBuilder + where + Client: HeadersClient + BodiesClient + 'static, + { /// Create a new `TestConsensusEngineBuilder` with the given `ChainSpec`. fn new(chain_spec: Arc) -> Self { Self { chain_spec, - pipeline_exec_outputs: VecDeque::new(), - executor_results: Vec::new(), + pipeline_config: Default::default(), + executor_config: Default::default(), pipeline_run_threshold: None, + client: None, max_block: None, } } @@ -1831,13 +1947,13 @@ mod tests { mut self, pipeline_exec_outputs: VecDeque>, ) -> Self { - self.pipeline_exec_outputs = pipeline_exec_outputs; + self.pipeline_config = TestPipelineConfig::Test(pipeline_exec_outputs); self } /// Set the executor results to use for the test consensus engine. fn with_executor_results(mut self, executor_results: Vec) -> Self { - self.executor_results = executor_results; + self.executor_config = TestExecutorConfig::Test(executor_results); self } @@ -1847,6 +1963,13 @@ mod tests { self } + /// Sets the client to use for network operations. + #[allow(dead_code)] + fn with_client(mut self, client: Client) -> Self { + self.client = Some(client); + self + } + /// Disables blockchain tree driven sync. This is the same as setting the pipeline run /// threshold to 0. fn disable_blockchain_tree_sync(mut self) -> Self { @@ -1855,20 +1978,55 @@ mod tests { } /// Builds the test consensus engine into a `TestConsensusEngine` and `TestEnv`. - fn build(self) -> (TestBeaconConsensusEngine, TestEnv>) { + fn build(self) -> (TestBeaconConsensusEngine, TestEnv>) { reth_tracing::init_test_tracing(); let db = create_test_rw_db(); - let consensus = TestConsensus::default(); + let consensus = Arc::new(TestConsensus::default()); let payload_builder = spawn_test_payload_service(); - let executor_factory = TestExecutorFactory::new(self.chain_spec.clone()); - executor_factory.extend(self.executor_results); + // use either noop client or a user provided client (for example TestFullBlockClient) + let client = Arc::new( + self.client + .map(EitherDownloader::Left) + .unwrap_or_else(|| EitherDownloader::Right(NoopFullBlockClient::default())), + ); + + // use either test executor or real executor + let executor_factory = match self.executor_config { + TestExecutorConfig::Test(results) => { + let executor_factory = TestExecutorFactory::new(self.chain_spec.clone()); + executor_factory.extend(results); + EitherExecutorFactory::Left(executor_factory) + } + TestExecutorConfig::Real => { + EitherExecutorFactory::Right(Factory::new(self.chain_spec.clone())) + } + }; // Setup pipeline let (tip_tx, tip_rx) = watch::channel(H256::default()); - let mut pipeline = Pipeline::builder() - .add_stages(TestStages::new(self.pipeline_exec_outputs, Default::default())) - .with_tip_sender(tip_tx); + let mut pipeline = match self.pipeline_config { + TestPipelineConfig::Test(outputs) => Pipeline::builder() + .add_stages(TestStages::new(outputs, Default::default())) + .with_tip_sender(tip_tx), + TestPipelineConfig::Real => { + let header_downloader = ReverseHeadersDownloaderBuilder::default() + .build(client.clone(), consensus.clone()) + .into_task(); + + let body_downloader = BodiesDownloaderBuilder::default() + .build(client.clone(), consensus.clone(), db.clone()) + .into_task(); + + Pipeline::builder().add_stages(DefaultStages::new( + HeaderSyncMode::Tip(tip_rx.clone()), + Arc::clone(&consensus) as Arc, + header_downloader, + body_downloader, + executor_factory.clone(), + )) + } + }; if let Some(max_block) = self.max_block { pipeline = pipeline.with_max_block(max_block); @@ -1896,7 +2054,7 @@ mod tests { let pruner = Pruner::new(5, 0); let (mut engine, handle) = BeaconConsensusEngine::new( - NoopFullBlockClient::default(), + client, pipeline, blockchain_provider, Box::::default(), @@ -1918,8 +2076,8 @@ mod tests { } } - fn spawn_consensus_engine( - engine: TestBeaconConsensusEngine, + fn spawn_consensus_engine( + engine: TestBeaconConsensusEngine, ) -> oneshot::Receiver> { let (tx, rx) = oneshot::channel(); tokio::spawn(async move { @@ -1940,11 +2098,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) - .disable_blockchain_tree_sync() - .with_max_block(1) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) + .disable_blockchain_tree_sync() + .with_max_block(1) + .build(); let res = spawn_consensus_engine(consensus_engine); @@ -1971,11 +2130,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) - .disable_blockchain_tree_sync() - .with_max_block(1) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) + .disable_blockchain_tree_sync() + .with_max_block(1) + .build(); let mut rx = spawn_consensus_engine(consensus_engine); @@ -2033,14 +2193,15 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(1), done: true }), - Err(StageError::ChannelClosed), - ])) - .disable_blockchain_tree_sync() - .with_max_block(2) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(1), done: true }), + Err(StageError::ChannelClosed), + ])) + .disable_blockchain_tree_sync() + .with_max_block(2) + .build(); let rx = spawn_consensus_engine(consensus_engine); @@ -2068,14 +2229,15 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(max_block), - done: true, - })])) - .with_max_block(max_block) - .disable_blockchain_tree_sync() - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(max_block), + done: true, + })])) + .with_max_block(max_block) + .disable_blockchain_tree_sync() + .build(); let rx = spawn_consensus_engine(consensus_engine); @@ -2117,12 +2279,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let mut engine_rx = spawn_consensus_engine(consensus_engine); @@ -2148,12 +2311,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2197,13 +2361,14 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - ])) - .disable_blockchain_tree_sync() - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + ])) + .disable_blockchain_tree_sync() + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2247,13 +2412,14 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .disable_blockchain_tree_sync() - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .disable_blockchain_tree_sync() + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2285,12 +2451,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - ])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + ])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let mut block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2338,12 +2505,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - ])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + ])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2385,12 +2553,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let mut engine_rx = spawn_consensus_engine(consensus_engine); @@ -2420,12 +2589,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2470,12 +2640,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); @@ -2526,13 +2697,14 @@ mod tests { .build(), ); - let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .with_executor_results(Vec::from([exec_result2])) - .build(); + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .with_executor_results(Vec::from([exec_result2])) + .build(); insert_blocks( env.db.as_ref(), From fdc119b6e9af8c7e085fd6ffd9d05cdfb9295435 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 18 Jul 2023 21:28:29 +0200 Subject: [PATCH 124/150] feat(book): more precise disk requirements (#3838) --- book/installation/installation.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/book/installation/installation.md b/book/installation/installation.md index d3ad78fc913..a7fd324bf13 100644 --- a/book/installation/installation.md +++ b/book/installation/installation.md @@ -14,12 +14,12 @@ The hardware requirements for running Reth depend on the node configuration and The most important requirement is by far the disk, whereas CPU and RAM requirements are relatively flexible. -| | Archive Node | Full Node | -|-----------|----------------------------------------|-------------------------------------| -| Disk | At least 2TB (TLC NVMe recommended) | TBD | -| Memory | 8GB+ | 8GB+ | -| CPU | Higher clock speed over core count | Higher clock speeds over core count | -| Bandwidth | Stable 24Mbps+ | Stable 24Mbps+ | +| | Archive Node | Full Node | +|-----------|---------------------------------------|-------------------------------------| +| Disk | At least 2.1TB (TLC NVMe recommended) | TBD | +| Memory | 8GB+ | 8GB+ | +| CPU | Higher clock speed over core count | Higher clock speeds over core count | +| Bandwidth | Stable 24Mbps+ | Stable 24Mbps+ | #### QLC and TLC @@ -35,12 +35,12 @@ Prior to purchasing an NVMe drive, it is advisable to research and determine whe There are multiple types of disks to sync Reth, with varying size requirements, depending on the syncing mode: -* Archive Node: At least 2TB is required to store +* Archive Node: At least 2.1TB is required (as of July 2023, at block number 17.7M) * Full Node: TBD NVMe drives are recommended for the best performance, with SSDs being a cheaper alternative. HDDs are the cheapest option, but they will take the longest to sync, and are not recommended. -At the time of writing, syncing an Ethereum mainnet node to block 17.4M on NVMe drives takes about 50 hours, while on a GCP "Persistent SSD" it takes around 5 days. +As of July 2023, syncing an Ethereum mainnet node to block 17.7M on NVMe drives takes about 50 hours, while on a GCP "Persistent SSD" it takes around 5 days. > **Note** > From 22c2ce3af717da0bd5414cb917ac50f9d7bb350c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 18 Jul 2023 21:28:47 +0200 Subject: [PATCH 125/150] chore: set trace result to null if non revert or selfdestruct (#3840) --- .../revm/revm-inspectors/src/tracing/types.rs | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/revm/revm-inspectors/src/tracing/types.rs b/crates/revm/revm-inspectors/src/tracing/types.rs index 76ff0f51008..22bebdbb247 100644 --- a/crates/revm/revm-inspectors/src/tracing/types.rs +++ b/crates/revm/revm-inspectors/src/tracing/types.rs @@ -159,6 +159,11 @@ impl CallTrace { self.status as u8 >= InstructionResult::Revert as u8 } + // Returns true if the status code is a revert + pub(crate) fn is_revert(&self) -> bool { + self.status == InstructionResult::Revert + } + /// Returns the error message if it is an erroneous result. pub(crate) fn as_error(&self, kind: TraceStyle) -> Option { // See also @@ -340,15 +345,16 @@ impl CallTraceNode { /// Converts this node into a parity `TransactionTrace` pub(crate) fn parity_transaction_trace(&self, trace_address: Vec) -> TransactionTrace { let action = self.parity_action(); - let output = self.parity_trace_output(); + let result = if action.is_selfdestruct() || + (self.trace.is_error() && !self.trace.is_revert()) + { + // if the trace is a selfdestruct or an error that is not a revert, the result is None + None + } else { + Some(self.parity_trace_output()) + }; let error = self.trace.as_error(TraceStyle::Parity); - TransactionTrace { - action, - error, - result: Some(output), - trace_address, - subtraces: self.children.len(), - } + TransactionTrace { action, error, result, trace_address, subtraces: self.children.len() } } /// Returns the `Output` for a parity trace From bdb649d953abf14b80cc53dcd53db484c3a9aea2 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 18 Jul 2023 16:52:25 -0400 Subject: [PATCH 126/150] feat: add key gen util and simple fcu validation test with custom alloc (#3842) --- crates/consensus/beacon/src/engine/mod.rs | 57 ++++++++++++++++++- .../interfaces/src/test_utils/generators.rs | 14 +++-- crates/primitives/src/lib.rs | 2 +- crates/primitives/src/transaction/util.rs | 17 ++++-- 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 4e1e6668261..8be81f2bbdc 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -2538,8 +2538,11 @@ mod tests { mod new_payload { use super::*; - use reth_interfaces::test_utils::{generators, generators::random_block}; - use reth_primitives::{Hardfork, U256}; + use reth_interfaces::test_utils::{ + generators, + generators::{generate_keys, random_block}, + }; + use reth_primitives::{public_key_to_address, Genesis, GenesisAccount, Hardfork, U256}; use reth_provider::test_utils::blocks::BlockChainTestData; #[tokio::test] @@ -2629,6 +2632,56 @@ mod tests { assert_matches!(engine_rx.try_recv(), Err(TryRecvError::Empty)); } + #[tokio::test] + async fn simple_validate_block() { + let mut rng = generators::rng(); + let genesis_keys = generate_keys(&mut rng, 16); + let amount = 1000000000000000000u64; + let alloc = genesis_keys.iter().map(|pair| { + ( + public_key_to_address(pair.public_key()), + GenesisAccount::default().with_balance(U256::from(amount)), + ) + }); + + let genesis = Genesis::default().extend_accounts(alloc); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(genesis) + .shanghai_activated() + .build(), + ); + + let (consensus_engine, env) = + TestConsensusEngineBuilder::::new(chain_spec.clone()).build(); + + let genesis = + SealedBlock { header: chain_spec.sealed_genesis_header(), ..Default::default() }; + let block1 = random_block(&mut rng, 1, Some(chain_spec.genesis_hash()), None, Some(0)); + + // TODO: add transactions that transfer from the alloc accounts, generating the new + // block tx and state root + + insert_blocks(env.db.as_ref(), chain_spec.clone(), [&genesis, &block1].into_iter()); + + let mut engine_rx = spawn_consensus_engine(consensus_engine); + + // Send forkchoice + let res = env + .send_forkchoice_updated(ForkchoiceState { + head_block_hash: block1.hash, + finalized_block_hash: block1.hash, + ..Default::default() + }) + .await; + let expected_result = PayloadStatus::from_status(PayloadStatusEnum::Valid) + .with_latest_valid_hash(block1.hash); + assert_matches!(res, Ok(ForkchoiceUpdated { payload_status, .. }) => assert_eq!(payload_status, expected_result)); + assert_matches!(engine_rx.try_recv(), Err(TryRecvError::Empty)); + } + #[tokio::test] async fn payload_parent_unknown() { let mut rng = generators::rng(); diff --git a/crates/interfaces/src/test_utils/generators.rs b/crates/interfaces/src/test_utils/generators.rs index c5d7132a563..d9399f8f814 100644 --- a/crates/interfaces/src/test_utils/generators.rs +++ b/crates/interfaces/src/test_utils/generators.rs @@ -104,6 +104,12 @@ pub fn sign_tx_with_key_pair(key_pair: KeyPair, tx: Transaction) -> TransactionS TransactionSigned::from_transaction_and_signature(tx, signature) } +/// Generates a set of [KeyPair]s based on the desired count. +pub fn generate_keys(rng: &mut R, count: usize) -> Vec { + let secp = Secp256k1::new(); + (0..count).map(|_| KeyPair::new(&secp, rng)).collect() +} + /// Generate a random block filled with signed transactions (generated using /// [random_signed_tx]). If no transaction count is provided, the number of transactions /// will be random, otherwise the provided count will be used. @@ -365,7 +371,9 @@ mod test { use super::*; use hex_literal::hex; - use reth_primitives::{keccak256, AccessList, Address, TransactionKind, TxEip1559}; + use reth_primitives::{ + keccak256, public_key_to_address, AccessList, Address, TransactionKind, TxEip1559, + }; use secp256k1::KeyPair; #[test] @@ -395,9 +403,7 @@ mod test { let signed = TransactionSigned::from_transaction_and_signature(tx.clone(), signature); let recovered = signed.recover_signer().unwrap(); - let public_key_hash = keccak256(&key_pair.public_key().serialize_uncompressed()[1..]); - let expected = Address::from_slice(&public_key_hash[12..]); - + let expected = public_key_to_address(key_pair.public_key()); assert_eq!(recovered, expected); } } diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 541289b3d30..44f5ecc30d1 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -86,7 +86,7 @@ pub use revm_primitives::JumpMap; pub use serde_helper::JsonU256; pub use storage::StorageEntry; pub use transaction::{ - util::secp256k1::{recover_signer, sign_message}, + util::secp256k1::{public_key_to_address, recover_signer, sign_message}, AccessList, AccessListItem, AccessListWithGasUsed, FromRecoveredTransaction, IntoRecoveredTransaction, InvalidTransactionError, Signature, Transaction, TransactionKind, TransactionMeta, TransactionSigned, TransactionSignedEcRecovered, TransactionSignedNoHash, diff --git a/crates/primitives/src/transaction/util.rs b/crates/primitives/src/transaction/util.rs index f67e152b582..da3fb63494c 100644 --- a/crates/primitives/src/transaction/util.rs +++ b/crates/primitives/src/transaction/util.rs @@ -6,7 +6,7 @@ pub(crate) mod secp256k1 { pub(crate) use ::secp256k1::Error; use ::secp256k1::{ ecdsa::{RecoverableSignature, RecoveryId}, - Message, SecretKey, SECP256K1, + Message, PublicKey, SecretKey, SECP256K1, }; use revm_primitives::{B256, U256}; @@ -18,11 +18,7 @@ pub(crate) mod secp256k1 { RecoverableSignature::from_compact(&sig[0..64], RecoveryId::from_i32(sig[64] as i32)?)?; let public = SECP256K1.recover_ecdsa(&Message::from_slice(&msg[..32])?, &sig)?; - - // strip out the first byte because that should be the SECP256K1_TAG_PUBKEY_UNCOMPRESSED - // tag returned by libsecp's uncompressed pubkey serialization - let hash = keccak256(&public.serialize_uncompressed()[1..]); - Ok(Address::from_slice(&hash[12..])) + Ok(public_key_to_address(public)) } /// Signs message with the given secret key. @@ -39,6 +35,15 @@ pub(crate) mod secp256k1 { }; Ok(signature) } + + /// Converts a public key into an ethereum address by hashing the encoded public key with + /// keccak256. + pub fn public_key_to_address(public: PublicKey) -> Address { + // strip out the first byte because that should be the SECP256K1_TAG_PUBKEY_UNCOMPRESSED + // tag returned by libsecp's uncompressed pubkey serialization + let hash = keccak256(&public.serialize_uncompressed()[1..]); + Address::from_slice(&hash[12..]) + } } #[cfg(test)] mod tests { From 3a5ef842824a32e03babe5e46dbae62c70b9e29b Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 26 Feb 2023 11:55:26 +0200 Subject: [PATCH 127/150] feat: deposit tx --- crates/primitives/src/transaction/mod.rs | 44 +++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 4e02a42499e..202de416adb 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1161,7 +1161,49 @@ impl TransactionSigned { /// Inner encoding function that is used for both rlp [`Encodable`] trait and for calculating /// hash that for eip2718 does not require rlp header pub(crate) fn encode_inner(&self, out: &mut dyn bytes::BufMut, with_header: bool) { - self.transaction.encode_with_signature(&self.signature, out, with_header); + match self.transaction { + Transaction::Legacy(TxLegacy { chain_id, .. }) => { + // do nothing w/ with_header + let payload_length = self.transaction.fields_len() + + self.signature.payload_len_with_eip155_chain_id(chain_id); + let header = Header { list: true, payload_length }; + header.encode(out); + self.transaction.encode_fields(out); + self.signature.encode_with_eip155_chain_id(out, chain_id); + } + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => { + let payload_length = self.transaction.fields_len() + self.signature.payload_len(); + if with_header { + Header { + list: false, + payload_length: 1 + 1 + length_of_length(payload_length) + payload_length, + } + .encode(out); + } + out.put_u8(self.transaction.tx_type() as u8); + out.put_u8(DEPOSIT_VERSION); + let header = Header { list: true, payload_length }; + header.encode(out); + self.transaction.encode_fields(out); + self.signature.encode(out); + } + _ => { + let payload_length = self.transaction.fields_len() + self.signature.payload_len(); + if with_header { + Header { + list: false, + payload_length: 1 + length_of_length(payload_length) + payload_length, + } + .encode(out); + } + out.put_u8(self.transaction.tx_type() as u8); + let header = Header { list: true, payload_length }; + header.encode(out); + self.transaction.encode_fields(out); + self.signature.encode(out); + } + } } /// Output the length of the encode_inner(out, true). Note to assume that `with_header` is only From 4f135141e33520b18ae0bab40b8194fe50cbb4e8 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 26 Feb 2023 14:41:49 +0200 Subject: [PATCH 128/150] feat: payload attributes op fields --- crates/rpc/rpc-types/Cargo.toml | 3 +- crates/rpc/rpc-types/src/eth/engine.rs | 246 +++++++++++++++++++++++++ 2 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 crates/rpc/rpc-types/src/eth/engine.rs diff --git a/crates/rpc/rpc-types/Cargo.toml b/crates/rpc/rpc-types/Cargo.toml index c7037a9ebf3..06866c2ce09 100644 --- a/crates/rpc/rpc-types/Cargo.toml +++ b/crates/rpc/rpc-types/Cargo.toml @@ -24,8 +24,7 @@ serde_json = { workspace = true } jsonrpsee-types = { version = "0.18" } [dev-dependencies] -# reth -reth-interfaces = { workspace = true, features = ["test-utils"] } +reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } # misc rand = { workspace = true } diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs new file mode 100644 index 00000000000..22c736dc1ab --- /dev/null +++ b/crates/rpc/rpc-types/src/eth/engine.rs @@ -0,0 +1,246 @@ +//! Engine API types: and following the execution specs + +#![allow(missing_docs)] + +use reth_primitives::{ + Address, Block, Bloom, Bytes, SealedBlock, Withdrawal, H256, H64, U256, U64, +}; +use reth_rlp::Encodable; +use serde::{Deserialize, Serialize}; + +/// The list of supported Engine capabilities +pub const CAPABILITIES: [&str; 9] = [ + "engine_forkchoiceUpdatedV1", + "engine_forkchoiceUpdatedV2", + "engine_exchangeTransitionConfigurationV1", + "engine_getPayloadV1", + "engine_getPayloadV2", + "engine_newPayloadV1", + "engine_newPayloadV2", + "engine_getPayloadBodiesByHashV1", + "engine_getPayloadBodiesByRangeV1", +]; + +/// This structure maps on the ExecutionPayload structure of the beacon chain spec. +/// +/// See also: +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecutionPayload { + pub parent_hash: H256, + pub fee_recipient: Address, + pub state_root: H256, + pub receipts_root: H256, + pub logs_bloom: Bloom, + pub prev_randao: H256, + pub block_number: U64, + pub gas_limit: U64, + pub gas_used: U64, + pub timestamp: U64, + pub extra_data: Bytes, + pub base_fee_per_gas: U256, + pub block_hash: H256, + pub transactions: Vec, + /// Array of [`Withdrawal`] enabled with V2 + /// See + #[serde(default, skip_serializing_if = "Option::is_none")] + pub withdrawals: Option>, +} + +impl From for ExecutionPayload { + fn from(value: SealedBlock) -> Self { + let transactions = value + .body + .iter() + .map(|tx| { + let mut encoded = Vec::new(); + tx.encode(&mut encoded); + encoded.into() + }) + .collect(); + ExecutionPayload { + parent_hash: value.parent_hash, + fee_recipient: value.beneficiary, + state_root: value.state_root, + receipts_root: value.receipts_root, + logs_bloom: value.logs_bloom, + prev_randao: value.mix_hash, + block_number: value.number.into(), + gas_limit: value.gas_limit.into(), + gas_used: value.gas_used.into(), + timestamp: value.timestamp.into(), + extra_data: value.extra_data.clone(), + base_fee_per_gas: U256::from(value.base_fee_per_gas.unwrap_or_default()), + block_hash: value.hash(), + transactions, + withdrawals: value.withdrawals, + } + } +} + +/// This structure contains a body of an execution payload. +/// +/// See also: +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExecutionPayloadBody { + pub transactions: Vec, + pub withdrawals: Vec, +} + +impl From for ExecutionPayloadBody { + fn from(value: Block) -> Self { + let transactions = value.body.into_iter().map(|tx| { + let mut out = Vec::new(); + tx.encode(&mut out); + out.into() + }); + ExecutionPayloadBody { + transactions: transactions.collect(), + withdrawals: value.withdrawals.unwrap_or_default(), + } + } +} + +/// The execution payload body response that allows for `null` values. +pub type ExecutionPayloadBodies = Vec>; + +/// This structure encapsulates the fork choice state +#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ForkchoiceState { + pub head_block_hash: H256, + pub safe_block_hash: H256, + pub finalized_block_hash: H256, +} + +/// This structure contains the attributes required to initiate a payload build process in the +/// context of an `engine_forkchoiceUpdated` call. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PayloadAttributes { + pub timestamp: U64, + pub prev_randao: H256, + pub suggested_fee_recipient: Address, + /// Array of [`Withdrawal`] enabled with V2 + /// See + #[serde(default, skip_serializing_if = "Option::is_none")] + pub withdrawals: Option>, + + #[cfg(feature = "optimism")] + /// Transactions is a field for rollups: the transactions list is forced into the block + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transactions: Option>, + + #[cfg(feature = "optimism")] + /// If true, the no transactions are taken out of the tx-pool, only transactions from the above + /// Transactions list will be included. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub no_tx_pool: Option, + + #[cfg(feature = "optimism")] + /// If set, this sets the exact gas limit the block produced with. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gas_limit: Option, +} + +/// This structure contains the result of processing a payload +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PayloadStatus { + #[serde(flatten)] + pub status: PayloadStatusEnum, + /// Hash of the most recent valid block in the branch defined by payload and its ancestors + pub latest_valid_hash: Option, +} + +impl PayloadStatus { + pub fn new(status: PayloadStatusEnum, latest_valid_hash: H256) -> Self { + Self { status, latest_valid_hash: Some(latest_valid_hash) } + } + + pub fn from_status(status: PayloadStatusEnum) -> Self { + Self { status, latest_valid_hash: None } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum PayloadStatusEnum { + Valid, + Invalid { + #[serde(rename = "validationError")] + validation_error: String, + }, + Syncing, + Accepted, + InvalidBlockHash { + #[serde(rename = "validationError")] + validation_error: String, + }, +} + +/// This structure contains configurable settings of the transition process. +#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransitionConfiguration { + /// Maps on the TERMINAL_TOTAL_DIFFICULTY parameter of EIP-3675 + pub terminal_total_difficulty: U256, + /// Maps on TERMINAL_BLOCK_HASH parameter of EIP-3675 + pub terminal_block_hash: H256, + /// Maps on TERMINAL_BLOCK_NUMBER parameter of EIP-3675 + pub terminal_block_number: U64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ForkchoiceUpdated { + pub payload_status: PayloadStatus, + pub payload_id: Option, +} + +impl ForkchoiceUpdated { + pub fn new(payload_status: PayloadStatus) -> Self { + Self { payload_status, payload_id: None } + } + + pub fn from_status(status: PayloadStatusEnum) -> Self { + Self { payload_status: PayloadStatus::from_status(status), payload_id: None } + } + + pub fn with_latest_valid_hash(mut self, hash: H256) -> Self { + self.payload_status.latest_valid_hash = Some(hash); + self + } + + pub fn with_payload_id(mut self, id: H64) -> Self { + self.payload_id = Some(id); + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_interfaces::test_utils::generators::random_block_range; + use reth_primitives::{TransactionSigned, H256}; + use reth_rlp::Decodable; + + #[test] + fn payload_body_roundtrip() { + for block in random_block_range(0..100, H256::default(), 0..2) { + let unsealed = block.clone().unseal(); + let payload_body: ExecutionPayloadBody = unsealed.into(); + + assert_eq!( + Ok(block.body), + payload_body + .transactions + .iter() + .map(|x| TransactionSigned::decode(&mut &x[..])) + .collect::, _>>(), + ); + + assert_eq!(block.withdrawals.unwrap_or_default(), payload_body.withdrawals); + } + } +} From 46f28618ab7ac9e3a258ab24e3ea001ade745d95 Mon Sep 17 00:00:00 2001 From: Andreas Bigger Date: Sun, 26 Feb 2023 18:39:58 -0700 Subject: [PATCH 129/150] small fixes --- crates/primitives/src/transaction/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 202de416adb..62f6ebd4ce5 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -721,6 +721,17 @@ impl Transaction { /// Encodes only the transaction's fields into the desired buffer, without a RLP header. pub(crate) fn encode_fields(&self, out: &mut dyn bytes::BufMut) { match self { + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { + tx_env.gas_limit = *gas_limit; + tx_env.gas_price = U256::from(*mint); + tx_env.gas_priority_fee = None; + tx_env.transact_to = TransactTo::Call(*to); + tx_env.value = U256::from(*value); + tx_env.data = input.0.clone(); + tx_env.chain_id = None; + tx_env.nonce = None; + } Transaction::Legacy(TxLegacy { chain_id: _, nonce, From 0c69e20d2b3ee20fde4b04cb3666147991e6b87d Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 26 Feb 2023 20:36:42 -0500 Subject: [PATCH 130/150] Small changes to the deposit tx primitive --- crates/primitives/src/transaction/mod.rs | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 62f6ebd4ce5..4a23417cf71 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -13,7 +13,7 @@ use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, }; -use serde::{Deserialize, Serialize}; +use revm_primitives::U256; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; @@ -721,17 +721,6 @@ impl Transaction { /// Encodes only the transaction's fields into the desired buffer, without a RLP header. pub(crate) fn encode_fields(&self, out: &mut dyn bytes::BufMut) { match self { - #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { - tx_env.gas_limit = *gas_limit; - tx_env.gas_price = U256::from(*mint); - tx_env.gas_priority_fee = None; - tx_env.transact_to = TransactTo::Call(*to); - tx_env.value = U256::from(*value); - tx_env.data = input.0.clone(); - tx_env.chain_id = None; - tx_env.nonce = None; - } Transaction::Legacy(TxLegacy { chain_id: _, nonce, @@ -1197,7 +1186,16 @@ impl TransactionSigned { let header = Header { list: true, payload_length }; header.encode(out); self.transaction.encode_fields(out); - self.signature.encode(out); + // Deposit transactions do not have a signature. If the signature's values are not + // zero, then the transaction is invalid. + if self.signature().v(self.chain_id()) != 0 || + self.signature().r != U256::ZERO || + self.signature().s != U256::ZERO + { + // TODO: Ensure that this transaction may never have a non-zero signature + // higher up - we shouldn't be panicking here. + panic!("Deposit transactions must have a zero signature"); + } } _ => { let payload_length = self.transaction.fields_len() + self.signature.payload_len(); @@ -1309,6 +1307,8 @@ impl TransactionSigned { // If the transaction is a deposit, we need to first ensure that the version // byte is correct. #[cfg(feature = "optimism")] + // If the transaction is a deposit, we need to first ensure that the version + // byte is correct. if tx_type == DEPOSIT_TX_TYPE { let version = *data.first().ok_or(DecodeError::InputTooShort)?; if version != DEPOSIT_VERSION { @@ -1328,6 +1328,8 @@ impl TransactionSigned { // If the transaction is a deposit, we need to add one to the length to account for the // version byte. #[cfg(feature = "optimism")] + // If the transaction is a deposit, we need to add one to the length to account for the + // version byte. let tx_length = if tx_type == DEPOSIT_TX_TYPE { tx_length + 1 } else { tx_length }; // decode common fields From 4b99875b66dbdc7956ae2a2bf4ce64463f24cea6 Mon Sep 17 00:00:00 2001 From: Andreas Bigger Date: Sun, 26 Feb 2023 19:39:42 -0700 Subject: [PATCH 131/150] more small fixes --- crates/primitives/src/transaction/mod.rs | 1 + crates/revm/revm-primitives/src/env.rs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 4a23417cf71..6ac7f716ca4 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -13,6 +13,7 @@ use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, }; +#[cfg(feature = "optimism")] use revm_primitives::U256; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index b1d15f2529e..63fea99038e 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -122,6 +122,17 @@ where { tx_env.caller = sender; match transaction.as_ref() { + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { + tx_env.gas_limit = *gas_limit; + tx_env.gas_price = U256::from(*mint); + tx_env.gas_priority_fee = None; + tx_env.transact_to = TransactTo::Call(*to); + tx_env.value = U256::from(*value); + tx_env.data = input.0.clone(); + tx_env.chain_id = None; + tx_env.nonce = None; + } Transaction::Legacy(TxLegacy { nonce, chain_id, From 53f850a855c9e12b5686a6c47c9a77c32e698ed8 Mon Sep 17 00:00:00 2001 From: Andreas Bigger Date: Sun, 26 Feb 2023 22:26:58 -0700 Subject: [PATCH 132/150] compiles --- crates/consensus/Cargo.toml | 24 +++++++++++++++++++ crates/consensus/common/src/validation.rs | 10 ++++++++ crates/revm/revm-primitives/src/env.rs | 11 +++++++-- crates/rpc/rpc-types/src/eth/engine.rs | 2 +- .../rpc/rpc-types/src/eth/transaction/mod.rs | 5 ++++ crates/transaction-pool/src/traits.rs | 2 ++ 6 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 crates/consensus/Cargo.toml diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml new file mode 100644 index 00000000000..bd742d0bed0 --- /dev/null +++ b/crates/consensus/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "reth-consensus" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/paradigmxyz/reth" +readme = "README.md" + +[dependencies] +# reth +reth-primitives = { path = "../primitives" } +reth-interfaces = { path = "../interfaces" } +reth-provider = { path = "../storage/provider" } + +# async +tokio = { version = "1", features = ["sync"] } + +[dev-dependencies] +reth-interfaces = { path = "../interfaces", features = ["test-utils"] } +reth-provider = { path = "../storage/provider", features = ["test-utils"] } +assert_matches = "1.5.0" + +[features] +optimism = [] \ No newline at end of file diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index a1907dbe9e1..e2007ff962f 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -10,6 +10,11 @@ use std::{ time::SystemTime, }; +<<<<<<< HEAD +======= +use reth_primitives::constants; + +>>>>>>> 1c17ad88 (compiles) #[cfg(feature = "optimism")] use reth_primitives::TxDeposit; @@ -67,6 +72,11 @@ pub fn validate_transaction_regarding_header( base_fee: Option, ) -> Result<(), ConsensusError> { let chain_id = match transaction { + #[cfg(feature = "optimism")] + Transaction::Deposit(TxDeposit { .. }) => { + // TODO: get the chain id + None + } Transaction::Legacy(TxLegacy { chain_id, .. }) => { // EIP-155: Simple replay attack protection: https://eips.ethereum.org/EIPS/eip-155 if chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(at_block_number) && diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index 63fea99038e..cf5d5e1f0b4 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -125,9 +125,16 @@ where #[cfg(feature = "optimism")] Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { tx_env.gas_limit = *gas_limit; - tx_env.gas_price = U256::from(*mint); + if let Some(m) = mint { + tx_env.gas_price = U256::from(*m); + } else { + tx_env.gas_price = U256::ZERO; + } tx_env.gas_priority_fee = None; - tx_env.transact_to = TransactTo::Call(*to); + match to { + TransactionKind::Call(to) => tx_env.transact_to = TransactTo::Call(*to), + TransactionKind::Create => tx_env.transact_to = TransactTo::create(), + } tx_env.value = U256::from(*value); tx_env.data = input.0.clone(); tx_env.chain_id = None; diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs index 22c736dc1ab..e8be77f3adc 100644 --- a/crates/rpc/rpc-types/src/eth/engine.rs +++ b/crates/rpc/rpc-types/src/eth/engine.rs @@ -140,7 +140,7 @@ pub struct PayloadAttributes { #[cfg(feature = "optimism")] /// If set, this sets the exact gas limit the block produced with. #[serde(default, skip_serializing_if = "Option::is_none")] - pub gas_limit: Option, + pub gas_limit: Option, } /// This structure contains the result of processing a payload diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index 109257909a1..5371ba9fea4 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -111,6 +111,7 @@ impl Transaction { let (gas_price, max_fee_per_gas) = match signed_tx.tx_type() { TxType::Legacy => (Some(U128::from(signed_tx.max_fee_per_gas())), None), +<<<<<<< HEAD TxType::EIP2930 => (Some(U128::from(signed_tx.max_fee_per_gas())), None), TxType::EIP1559 => { // the gas price field for EIP1559 is set to `min(tip, gasFeeCap - baseFee) + @@ -123,6 +124,10 @@ impl Transaction { (Some(U128::from(gas_price)), Some(U128::from(signed_tx.max_fee_per_gas()))) } +======= + TxType::EIP2930 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), + TxType::EIP1559 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), +>>>>>>> 1c17ad88 (compiles) #[cfg(feature = "optimism")] TxType::DEPOSIT => (None, None), }; diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 5af49270ab7..b5219573530 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -595,6 +595,8 @@ impl PoolTransaction for PooledTransaction { /// This will return `None` for non-EIP1559 transactions fn max_priority_fee_per_gas(&self) -> Option { match &self.transaction.transaction { + #[cfg(feature = "optimism")] + Transaction::Deposit(_) => None, Transaction::Legacy(_) => None, Transaction::Eip2930(_) => None, Transaction::Eip1559(tx) => Some(tx.max_priority_fee_per_gas), From 0a2301f7051866403ceed38029d67100a3cf11dd Mon Sep 17 00:00:00 2001 From: clabby Date: Mon, 27 Feb 2023 09:05:01 -0500 Subject: [PATCH 133/150] Move feature flags below comments --- crates/primitives/src/transaction/mod.rs | 5 +++++ crates/rpc/rpc-types/src/eth/engine.rs | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 6ac7f716ca4..bde3c6beff9 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1305,11 +1305,15 @@ impl TransactionSigned { let tx_type = *data.first().ok_or(DecodeError::InputTooShort)?; data.advance(1); +<<<<<<< HEAD // If the transaction is a deposit, we need to first ensure that the version // byte is correct. #[cfg(feature = "optimism")] +======= +>>>>>>> 74cee5d0 (Move feature flags below comments) // If the transaction is a deposit, we need to first ensure that the version // byte is correct. + #[cfg(feature = "optimism")] if tx_type == DEPOSIT_TX_TYPE { let version = *data.first().ok_or(DecodeError::InputTooShort)?; if version != DEPOSIT_VERSION { @@ -1331,6 +1335,7 @@ impl TransactionSigned { #[cfg(feature = "optimism")] // If the transaction is a deposit, we need to add one to the length to account for the // version byte. + #[cfg(feature = "optimism")] let tx_length = if tx_type == DEPOSIT_TX_TYPE { tx_length + 1 } else { tx_length }; // decode common fields diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs index e8be77f3adc..548ed16645b 100644 --- a/crates/rpc/rpc-types/src/eth/engine.rs +++ b/crates/rpc/rpc-types/src/eth/engine.rs @@ -126,19 +126,19 @@ pub struct PayloadAttributes { #[serde(default, skip_serializing_if = "Option::is_none")] pub withdrawals: Option>, - #[cfg(feature = "optimism")] /// Transactions is a field for rollups: the transactions list is forced into the block + #[cfg(feature = "optimism")] #[serde(default, skip_serializing_if = "Option::is_none")] pub transactions: Option>, - #[cfg(feature = "optimism")] /// If true, the no transactions are taken out of the tx-pool, only transactions from the above /// Transactions list will be included. + #[cfg(feature = "optimism")] #[serde(default, skip_serializing_if = "Option::is_none")] pub no_tx_pool: Option, - #[cfg(feature = "optimism")] /// If set, this sets the exact gas limit the block produced with. + #[cfg(feature = "optimism")] #[serde(default, skip_serializing_if = "Option::is_none")] pub gas_limit: Option, } From a4e54c60a6abdc7798ce8dff8d8de44b7826dd64 Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 5 Mar 2023 13:08:35 -0700 Subject: [PATCH 134/150] WIP: OP Goerli genesis --- crates/rpc/rpc-engine-api/src/engine_api.rs | 113 ++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 2a0581595f4..a1435b28519 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -242,6 +242,119 @@ where Ok(result) } + /// When the Consensus layer receives a new block via the consensus gossip protocol, + /// the transactions in the block are sent to the execution layer in the form of a + /// `ExecutionPayload`. The Execution layer executes the transactions and validates the + /// state in the block header, then passes validation data back to Consensus layer, that + /// adds the block to the head of its own blockchain and attests to it. The block is then + /// broadcasted over the consensus p2p network in the form of a "Beacon block". + pub fn new_payload(&mut self, payload: ExecutionPayload) -> EngineApiResult { + let block = match self.try_construct_block(payload) { + Ok(b) => b, + Err(err) => { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::InvalidBlockHash { + validation_error: err.to_string(), + })) + } + }; + let block_hash = block.header.hash(); + let parent_hash = block.parent_hash; + + // The block already exists in our database + if self.client.is_known(&block_hash)? { + return Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)) + } + + let Some(parent) = self.client.block_by_hash(parent_hash)? else { + // TODO: cache block for storing later + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Syncing)) + }; + + let parent_td = if let Some(parent_td) = self.client.header_td(&block.parent_hash)? { + parent_td + } else { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::PayloadPreMerge.to_string(), + })) + }; + + // Short circuit the check by passing parent total difficulty. + if !self.chain_spec.fork(Hardfork::Paris).active_at_ttd(parent_td, U256::ZERO) { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::PayloadPreMerge.to_string(), + })) + } + + if block.timestamp <= parent.timestamp { + return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::PayloadTimestamp { + invalid: block.timestamp, + latest: parent.timestamp, + } + .to_string(), + })) + } + + let state_provider = self.client.latest()?; + let total_difficulty = parent_td + block.header.difficulty; + match executor::execute_and_verify_receipt( + &block.unseal(), + total_difficulty, + None, + &self.chain_spec, + &mut SubState::new(State::new(&state_provider)), + ) { + Ok(_) => Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)), + Err(err) => Ok(PayloadStatus::new( + PayloadStatusEnum::Invalid { validation_error: err.to_string() }, + parent_hash, // The parent hash is already in our database hence it is valid + )), + } + } + + /// Called to resolve chain forks and ensure that the Execution layer is working with the latest + /// valid chain. + pub fn fork_choice_updated( + &self, + fork_choice_state: ForkchoiceState, + payload_attributes: Option, + ) -> EngineApiResult { + let ForkchoiceState { head_block_hash, finalized_block_hash, .. } = fork_choice_state; + + if head_block_hash.is_zero() { + return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Invalid { + validation_error: EngineApiError::ForkchoiceEmptyHead.to_string(), + })) + } + + // Block is not known, nothing to do. + if !self.client.is_known(&head_block_hash)? { + return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) + } + + // The finalized block hash is not known, we are still syncing + if !finalized_block_hash.is_zero() && !self.client.is_known(&finalized_block_hash)? { + return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) + } + + if let Err(error) = self.forkchoice_state_tx.send(fork_choice_state) { + tracing::error!(target: "rpc::engine_api", ?error, "Failed to update forkchoice state"); + } + + if let Some(attr) = payload_attributes { + #[cfg(feature = "optimism")] + if self.chain_spec.optimism.is_some() && attr.gas_limit.is_none() { + return Err(EngineApiError::MissingGasLimitInPayloadAttributes) + } + + // TODO: optionally build the block + } + + let chain_info = self.client.chain_info()?; + Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Valid) + .with_latest_valid_hash(chain_info.best_hash)) + } + /// Called to verify network configuration parameters and ensure that Consensus and Execution /// layers are using the latest configuration. pub async fn exchange_transition_configuration( From 9840c126449b20520ed03de26c607ba0df565e6a Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 19 Mar 2023 16:34:05 -0400 Subject: [PATCH 135/150] TEMP: Expose some CLI items --- bin/reth/src/cli.rs | 6 +----- bin/reth/src/dirs.rs | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index c32b1ae17ed..06f4d3a6a2f 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -91,10 +91,6 @@ struct Cli { #[derive(Debug, Args)] #[command(next_help_heading = "Logging")] pub struct Logs { - /// The flag to enable persistent logs. - #[arg(long = "log.persistent", global = true, conflicts_with = "journald")] - persistent: bool, - /// The path to put log files in. #[arg( long = "log.directory", @@ -116,7 +112,7 @@ pub struct Logs { impl Logs { /// Builds a tracing layer from the current log options. - pub fn layer(&self) -> eyre::Result, Option)>> + pub fn layer(&self) -> (BoxedLayer, Option) where S: Subscriber, for<'a> S: LookupSpan<'a>, diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index 796377b6a16..ccdd4125750 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -86,7 +86,6 @@ impl XdgPath for LogsDir { /// A small helper trait for unit structs that represent a standard path following the XDG /// path specification. pub trait XdgPath { - /// Resolve the standard path. fn resolve() -> Option; } From 168846b625e8962a1e029b19921500c049661406 Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 19 Mar 2023 16:53:10 -0400 Subject: [PATCH 136/150] Resolve conflicts Remove `consensus` crate `Cargo.toml` Reduce diff --- bin/reth/src/dirs.rs | 1 + crates/consensus/Cargo.toml | 24 --------------------- crates/consensus/common/Cargo.toml | 1 - crates/consensus/common/src/validation.rs | 10 --------- crates/revm/Cargo.toml | 3 +-- crates/revm/revm-primitives/src/env.rs | 18 ---------------- crates/rpc/rpc-engine-api/src/engine_api.rs | 4 ++-- crates/transaction-pool/src/traits.rs | 10 +++++++-- 8 files changed, 12 insertions(+), 59 deletions(-) delete mode 100644 crates/consensus/Cargo.toml diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index ccdd4125750..796377b6a16 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -86,6 +86,7 @@ impl XdgPath for LogsDir { /// A small helper trait for unit structs that represent a standard path following the XDG /// path specification. pub trait XdgPath { + /// Resolve the standard path. fn resolve() -> Option; } diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml deleted file mode 100644 index bd742d0bed0..00000000000 --- a/crates/consensus/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "reth-consensus" -version = "0.1.0" -edition = "2021" -license = "MIT OR Apache-2.0" -repository = "https://github.com/paradigmxyz/reth" -readme = "README.md" - -[dependencies] -# reth -reth-primitives = { path = "../primitives" } -reth-interfaces = { path = "../interfaces" } -reth-provider = { path = "../storage/provider" } - -# async -tokio = { version = "1", features = ["sync"] } - -[dev-dependencies] -reth-interfaces = { path = "../interfaces", features = ["test-utils"] } -reth-provider = { path = "../storage/provider", features = ["test-utils"] } -assert_matches = "1.5.0" - -[features] -optimism = [] \ No newline at end of file diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml index 4090fc07050..112b8d89171 100644 --- a/crates/consensus/common/Cargo.toml +++ b/crates/consensus/common/Cargo.toml @@ -25,4 +25,3 @@ optimism = [ "reth-interfaces/optimism", "reth-provider/optimism" ] - diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index e2007ff962f..a1907dbe9e1 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -10,11 +10,6 @@ use std::{ time::SystemTime, }; -<<<<<<< HEAD -======= -use reth_primitives::constants; - ->>>>>>> 1c17ad88 (compiles) #[cfg(feature = "optimism")] use reth_primitives::TxDeposit; @@ -72,11 +67,6 @@ pub fn validate_transaction_regarding_header( base_fee: Option, ) -> Result<(), ConsensusError> { let chain_id = match transaction { - #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { .. }) => { - // TODO: get the chain id - None - } Transaction::Legacy(TxLegacy { chain_id, .. }) => { // EIP-155: Simple replay attack protection: https://eips.ethereum.org/EIPS/eip-155 if chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(at_block_number) && diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index 31622ab3a1f..8a17c83aea9 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -17,8 +17,7 @@ reth-revm-primitives = { path = "./revm-primitives" } reth-revm-inspectors = { path = "./revm-inspectors" } reth-consensus-common = { path = "../consensus/common" } -# revm -revm = { workspace = true } +revm = { version = "3.0.0" } # common tracing = { workspace = true } diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index cf5d5e1f0b4..b1d15f2529e 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -122,24 +122,6 @@ where { tx_env.caller = sender; match transaction.as_ref() { - #[cfg(feature = "optimism")] - Transaction::Deposit(TxDeposit { to, mint, value, gas_limit, input, .. }) => { - tx_env.gas_limit = *gas_limit; - if let Some(m) = mint { - tx_env.gas_price = U256::from(*m); - } else { - tx_env.gas_price = U256::ZERO; - } - tx_env.gas_priority_fee = None; - match to { - TransactionKind::Call(to) => tx_env.transact_to = TransactTo::Call(*to), - TransactionKind::Create => tx_env.transact_to = TransactTo::create(), - } - tx_env.value = U256::from(*value); - tx_env.data = input.0.clone(); - tx_env.chain_id = None; - tx_env.nonce = None; - } Transaction::Legacy(TxLegacy { nonce, chain_id, diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index a1435b28519..c4ec8e0f38f 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -341,9 +341,9 @@ where tracing::error!(target: "rpc::engine_api", ?error, "Failed to update forkchoice state"); } - if let Some(attr) = payload_attributes { + if let Some(_attr) = payload_attributes { #[cfg(feature = "optimism")] - if self.chain_spec.optimism.is_some() && attr.gas_limit.is_none() { + if self.chain_spec.optimism.is_some() && _attr.gas_limit.is_none() { return Err(EngineApiError::MissingGasLimitInPayloadAttributes) } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index b5219573530..dbf28f2d406 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -595,8 +595,6 @@ impl PoolTransaction for PooledTransaction { /// This will return `None` for non-EIP1559 transactions fn max_priority_fee_per_gas(&self) -> Option { match &self.transaction.transaction { - #[cfg(feature = "optimism")] - Transaction::Deposit(_) => None, Transaction::Legacy(_) => None, Transaction::Eip2930(_) => None, Transaction::Eip1559(tx) => Some(tx.max_priority_fee_per_gas), @@ -644,6 +642,14 @@ impl FromRecoveredTransaction for PooledTransaction { // because they already pay for their gas on L1. U256::ZERO } + #[cfg(feature = "optimism")] + Transaction::Deposit(t) => { + // TODO: fix this gas price estimate + let gas_price = U256::from(0); + let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); + let effective_gas_price = 0u128; + (cost, effective_gas_price) + } }; let cost = gas_cost + U256::from(tx.value()); From 70bbdb11e12b4e754a8d689888f006e0dbbc66eb Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 11 Jun 2023 14:46:56 +0200 Subject: [PATCH 137/150] feat: executor deposits --- crates/executor/Cargo.toml | 58 +++++++++++++++++++++++++++++++ crates/interfaces/src/executor.rs | 4 +++ crates/revm/src/executor.rs | 2 +- 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 crates/executor/Cargo.toml diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml new file mode 100644 index 00000000000..53acae3de9f --- /dev/null +++ b/crates/executor/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "reth-executor" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/paradigmxyz/reth" +readme = "README.md" + +[package.metadata.cargo-udeps.ignore] +normal = [ + # Used for diagrams in docs + "aquamarine", +] + +[dependencies] +# reth +reth-primitives = { path = "../primitives" } +reth-interfaces = { path = "../interfaces" } +reth-revm = { path = "../revm" } +reth-revm-inspectors = { path = "../revm/revm-inspectors" } +reth-rlp = { path = "../rlp" } +reth-db = { path = "../storage/db" } +reth-provider = { path = "../storage/provider" } + +# revm +revm = { version = "3.0.0" } + +# common +async-trait = "0.1.57" +thiserror = "1.0.37" +auto_impl = "1.0" +tracing = "0.1.37" +tokio = { version = "1.21.2", features = ["sync"] } + +# mics +aquamarine = "0.3.0" +parking_lot = { version = "0.12", optional = true } + +triehash = "0.8" +# See to replace hashers to simplify libraries +plain_hasher = "0.2" +hash-db = "0.15" +# todo replace with faster rlp impl +rlp = { version = "0.5", default-features = false } +# replace with tiny-keccak (it is faster hasher) +sha3 = { version = "0.10", default-features = false } + + +[dev-dependencies] +reth-db = { path = "../storage/db", features = ["test-utils"] } +reth-interfaces = { path = "../interfaces", features = ["test-utils"] } +reth-primitives = { path = "../primitives", features = ["test-utils"] } +reth-provider = { path = "../storage/provider", features = ["test-utils"] } +parking_lot = "0.12" + +[features] +test-utils = ["parking_lot"] +optimism = [] diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 7f315688a35..ca82163ae5d 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -7,6 +7,10 @@ use thiserror::Error; pub enum BlockValidationError { #[error("EVM reported invalid transaction ({hash:?}): {message}")] EVM { hash: H256, message: String }, + #[error("Verification failed")] + VerificationFailed, + #[error("Fatal internal error")] + ExecutionFatalError, #[error("Failed to recover sender for transaction")] SenderRecoveryError, #[error("Receipt root {got:?} is different than expected {expected:?}.")] diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 9d95d7e6123..9a9f84c99ac 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -17,7 +17,7 @@ use revm::{ db::{AccountState, CacheDB, DatabaseRef}, primitives::{ hash_map::{self, Entry}, - Account as RevmAccount, AccountInfo, ResultAndState, + Account as RevmAccount, AccountInfo, ExecutionResult, ResultAndState, }, EVM, }; From fc97260e69102d1d19d3d6e1148574b9a08e2d74 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 11 Jun 2023 21:43:32 +0200 Subject: [PATCH 138/150] feat: receipts, l1 cost wip --- crates/executor/src/lib.rs | 30 +++++ crates/interfaces/src/executor.rs | 7 ++ crates/net/eth-wire/src/types/receipts.rs | 22 ++++ crates/primitives/src/receipt.rs | 144 ++++++++++++++++------ crates/transaction-pool/src/traits.rs | 4 +- 5 files changed, 171 insertions(+), 36 deletions(-) create mode 100644 crates/executor/src/lib.rs diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs new file mode 100644 index 00000000000..27be2283d23 --- /dev/null +++ b/crates/executor/src/lib.rs @@ -0,0 +1,30 @@ +#![warn(missing_docs, unreachable_pub)] +#![deny(unused_must_use, rust_2018_idioms)] +#![doc(test( + no_crate_inject, + attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) +))] + +//! Reth executor executes transaction in block of data. + +pub mod eth_dao_fork; +pub mod substate; + +/// Execution result types. +pub use reth_provider::post_state; + +pub mod blockchain_tree; + +/// Executor +pub mod executor; + +/// ExecutorFactory impl +pub mod factory; +pub use factory::Factory; + +#[cfg(any(test, feature = "test-utils"))] +/// Common test helpers for mocking out executor and executor factory +pub mod test_utils; + +#[cfg(feature = "optimism")] +pub mod optimism; diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index ca82163ae5d..2946b454104 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -46,6 +46,7 @@ pub enum BlockExecutionError { CanonicalRevert { inner: String }, #[error("Transaction error on commit: {inner:?}")] CanonicalCommit { inner: String }, +<<<<<<< HEAD // === tree errors === // TODO(mattsse): move this to tree error #[error("Block hash {block_hash} not found in blockchain tree chain")] @@ -74,4 +75,10 @@ impl BlockExecutionError { pub fn is_fatal(&self) -> bool { matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. }) } +======= + #[error("Transaction error on pipeline status update: {inner:?}")] + PipelineStatusUpdate { inner: String }, + #[error("DB Error during transaction execution: {inner:?}")] + DBError { inner: String }, +>>>>>>> 6266b9f3 (feat: receipts, l1 cost wip) } diff --git a/crates/net/eth-wire/src/types/receipts.rs b/crates/net/eth-wire/src/types/receipts.rs index af40108aa99..146c9163567 100644 --- a/crates/net/eth-wire/src/types/receipts.rs +++ b/crates/net/eth-wire/src/types/receipts.rs @@ -53,6 +53,9 @@ mod test { deposit_nonce: None, }, bloom: Default::default(), + logs: vec![], + #[cfg(feature = "optimism")] + deposit_nonce: None, }]]); let mut out = vec![]; @@ -123,9 +126,14 @@ mod test { #[cfg(feature = "optimism")] deposit_nonce: None, }, +<<<<<<< HEAD bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), }, ]]), +======= + ], + ]), +>>>>>>> 6266b9f3 (feat: receipts, l1 cost wip) }; request.encode(&mut data); assert_eq!(data, expected); @@ -161,6 +169,20 @@ mod test { deposit_nonce: None, }, bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), + cumulative_gas_used: 0x1u64, + logs: vec![ + Log { + address: hex!("0000000000000000000000000000000000000011").into(), + topics: vec![ + hex!("000000000000000000000000000000000000000000000000000000000000dead").into(), + hex!("000000000000000000000000000000000000000000000000000000000000beef").into(), + ], + data: hex!("0100ff")[..].into(), + }, + ], + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, ], ]), diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 85b286c1462..bd4ac14df5b 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -34,40 +34,36 @@ pub struct Receipt { } impl Receipt { - /// Calculates [`Log`]'s bloom filter. this is slow operation and [ReceiptWithBloom] can - /// be used to cache this value. - pub fn bloom_slow(&self) -> Bloom { - logs_bloom(self.logs.iter()) - } + /// Returns the rlp header for the receipt payload. + fn receipt_rlp_header(&self) -> reth_rlp::Header { + let mut rlp_head = reth_rlp::Header { list: true, payload_length: 0 }; - /// Calculates the bloom filter for the receipt and returns the [ReceiptWithBloom] container - /// type. - pub fn with_bloom(self) -> ReceiptWithBloom { - self.into() - } -} + rlp_head.payload_length += self.success.length(); + rlp_head.payload_length += self.cumulative_gas_used.length(); + rlp_head.payload_length += self.bloom.length(); + rlp_head.payload_length += self.logs.length(); + #[cfg(feature = "optimism")] + if self.tx_type == TxType::DEPOSIT { + // Transactions pre-Regolith don't have a deposit nonce + if let Some(nonce) = self.deposit_nonce { + rlp_head.payload_length += nonce.length(); + } + } -impl From for ReceiptWithBloom { - fn from(receipt: Receipt) -> Self { - let bloom = receipt.bloom_slow(); - ReceiptWithBloom { receipt, bloom } + rlp_head } -} - -/// [`Receipt`] with calculated bloom filter. -#[main_codec] -#[derive(Clone, Debug, PartialEq, Eq, Default)] -pub struct ReceiptWithBloom { - /// Bloom filter build from logs. - pub bloom: Bloom, - /// Main receipt body - pub receipt: Receipt, -} -impl ReceiptWithBloom { - /// Create new [ReceiptWithBloom] - pub fn new(receipt: Receipt, bloom: Bloom) -> Self { - Self { receipt, bloom } + /// Encodes the receipt data. + fn encode_fields(&self, out: &mut dyn BufMut) { + self.receipt_rlp_header().encode(out); + self.success.encode(out); + self.cumulative_gas_used.encode(out); + self.bloom.encode(out); + self.logs.encode(out); + #[cfg(feature = "optimism")] + if let Some(nonce) = self.deposit_nonce { + nonce.encode(out); + } } /// Consume the structure, returning only the receipt @@ -89,7 +85,33 @@ impl ReceiptWithBloom { impl ReceiptWithBloom { /// Encode receipt with or without the header data. pub fn encode_inner(&self, out: &mut dyn BufMut, with_header: bool) { - self.as_encoder().encode_inner(out, with_header) + if matches!(self.tx_type, TxType::Legacy) { + self.encode_fields(out); + return + } + + let mut payload = BytesMut::new(); + self.encode_fields(&mut payload); + + if with_header { + let payload_length = payload.len() + 1; + let header = reth_rlp::Header { list: false, payload_length }; + header.encode(out); + } + + match self.tx_type { + TxType::EIP2930 => out.put_u8(0x01), + TxType::EIP1559 => out.put_u8(0x02), + TxType::DEPOSIT => out.put_u8(0x7E), + TxType::Legacy => unreachable!("legacy handled; qed."), + } + out.put_slice(payload.as_ref()); + } + + /// Returns the length of the receipt data. + fn receipt_length(&self) -> usize { + let rlp_head = self.receipt_rlp_header(); + length_of_length(rlp_head.payload_length) + rlp_head.payload_length } /// Decodes the receipt payload @@ -101,10 +123,26 @@ impl ReceiptWithBloom { } let started_len = b.len(); - let success = reth_rlp::Decodable::decode(b)?; - let cumulative_gas_used = reth_rlp::Decodable::decode(b)?; - let bloom = Decodable::decode(b)?; - let logs = reth_rlp::Decodable::decode(b)?; + let this = match tx_type { + #[cfg(feature = "optimism")] + TxType::DEPOSIT => Self { + tx_type, + success: reth_rlp::Decodable::decode(b)?, + cumulative_gas_used: reth_rlp::Decodable::decode(b)?, + bloom: reth_rlp::Decodable::decode(b)?, + logs: reth_rlp::Decodable::decode(b)?, + deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), + }, + _ => Self { + tx_type, + success: reth_rlp::Decodable::decode(b)?, + cumulative_gas_used: reth_rlp::Decodable::decode(b)?, + bloom: reth_rlp::Decodable::decode(b)?, + logs: reth_rlp::Decodable::decode(b)?, + #[cfg(feature = "optimism")] + deposit_nonce: None, + }, + }; let receipt = match tx_type { #[cfg(feature = "optimism")] @@ -349,6 +387,24 @@ mod tests { deposit_nonce: None, }, bloom: [0; 256].into(), + cumulative_gas_used: 0x1u64, + logs: vec![Log { + address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), + topics: vec![ + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000dead", + ) + .unwrap(), + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000beef", + ) + .unwrap(), + ], + data: Bytes::from_str("0100ff").unwrap().0.into(), + }], + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; receipt.encode(&mut data); @@ -387,6 +443,24 @@ mod tests { deposit_nonce: None, }, bloom: [0; 256].into(), + cumulative_gas_used: 0x1u64, + logs: vec![Log { + address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), + topics: vec![ + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000dead", + ) + .unwrap(), + H256::from_str( + "000000000000000000000000000000000000000000000000000000000000beef", + ) + .unwrap(), + ], + data: Bytes::from_str("0100ff").unwrap().0.into(), + }], + success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index dbf28f2d406..1b09ec44447 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -644,7 +644,9 @@ impl FromRecoveredTransaction for PooledTransaction { } #[cfg(feature = "optimism")] Transaction::Deposit(t) => { - // TODO: fix this gas price estimate + // Gas price is always set to 0 for deposits in order to zero out ETH refunds. + // The actual L1 cost varies depending on the gas oracle feed, and is calculated + // during tx execution. let gas_price = U256::from(0); let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); let effective_gas_price = 0u128; From 2544de42e6cef63d5a4fb0d8520b5a12193e65fc Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 11 Jun 2023 22:02:13 +0200 Subject: [PATCH 139/150] chore: small fixes --- crates/executor/src/lib.rs | 1 + crates/primitives/src/proofs.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 27be2283d23..c4e4b13664e 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -27,4 +27,5 @@ pub use factory::Factory; pub mod test_utils; #[cfg(feature = "optimism")] +/// Optimism-specific utilities for the executor pub mod optimism; diff --git a/crates/primitives/src/proofs.rs b/crates/primitives/src/proofs.rs index 40740d119af..49fbbb5a7bd 100644 --- a/crates/primitives/src/proofs.rs +++ b/crates/primitives/src/proofs.rs @@ -175,6 +175,9 @@ mod tests { deposit_nonce: None, }, bloom, + logs, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; let receipt = vec![receipt]; let root = calculate_receipt_root(&receipt); From 1cc1e7155de0b66ac2752a339be66edbe4a6f038 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:48:03 +0200 Subject: [PATCH 140/150] wip: transact() optimism changes --- crates/interfaces/src/executor.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 2946b454104..ca82163ae5d 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -46,7 +46,6 @@ pub enum BlockExecutionError { CanonicalRevert { inner: String }, #[error("Transaction error on commit: {inner:?}")] CanonicalCommit { inner: String }, -<<<<<<< HEAD // === tree errors === // TODO(mattsse): move this to tree error #[error("Block hash {block_hash} not found in blockchain tree chain")] @@ -75,10 +74,4 @@ impl BlockExecutionError { pub fn is_fatal(&self) -> bool { matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. }) } -======= - #[error("Transaction error on pipeline status update: {inner:?}")] - PipelineStatusUpdate { inner: String }, - #[error("DB Error during transaction execution: {inner:?}")] - DBError { inner: String }, ->>>>>>> 6266b9f3 (feat: receipts, l1 cost wip) } From 651731e71de9bbfe481ee3f490875124d93e9031 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sat, 17 Jun 2023 22:25:28 +0200 Subject: [PATCH 141/150] wip: removed receipts changes + improved gas handling --- crates/interfaces/src/executor.rs | 3 +++ crates/primitives/src/receipt.rs | 18 ------------------ 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index ca82163ae5d..009b5fb1320 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -67,6 +67,7 @@ pub enum BlockExecutionError { #[cfg(feature = "optimism")] #[error("Insufficient funds to cover transaction L1 cost: want {want}, have {have}")] InsufficientFundsForL1Cost { want: u64, have: u64 }, +<<<<<<< HEAD } impl BlockExecutionError { @@ -74,4 +75,6 @@ impl BlockExecutionError { pub fn is_fatal(&self) -> bool { matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. }) } +======= +>>>>>>> 891b841f (wip: removed receipts changes + improved gas handling) } diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index bd4ac14df5b..3e4c6d85da4 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -42,13 +42,6 @@ impl Receipt { rlp_head.payload_length += self.cumulative_gas_used.length(); rlp_head.payload_length += self.bloom.length(); rlp_head.payload_length += self.logs.length(); - #[cfg(feature = "optimism")] - if self.tx_type == TxType::DEPOSIT { - // Transactions pre-Regolith don't have a deposit nonce - if let Some(nonce) = self.deposit_nonce { - rlp_head.payload_length += nonce.length(); - } - } rlp_head } @@ -60,10 +53,6 @@ impl Receipt { self.cumulative_gas_used.encode(out); self.bloom.encode(out); self.logs.encode(out); - #[cfg(feature = "optimism")] - if let Some(nonce) = self.deposit_nonce { - nonce.encode(out); - } } /// Consume the structure, returning only the receipt @@ -131,7 +120,6 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, - deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), }, _ => Self { tx_type, @@ -139,8 +127,6 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, - #[cfg(feature = "optimism")] - deposit_nonce: None, }, }; @@ -403,8 +389,6 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; receipt.encode(&mut data); @@ -459,8 +443,6 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); From a7aef80bf3e3de3f9ce7dbe4258d5f865c322856 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Sun, 18 Jun 2023 09:36:26 +0200 Subject: [PATCH 142/150] chore: cleanup --- crates/primitives/src/receipt.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 3e4c6d85da4..ea38cc575af 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -42,6 +42,13 @@ impl Receipt { rlp_head.payload_length += self.cumulative_gas_used.length(); rlp_head.payload_length += self.bloom.length(); rlp_head.payload_length += self.logs.length(); + #[cfg(feature = "optimism")] + if self.tx_type == TxType::DEPOSIT { + // Transactions pre-Regolith don't have a deposit nonce + if let Some(nonce) = self.deposit_nonce { + rlp_head.payload_length += nonce.length(); + } + } rlp_head } @@ -53,6 +60,10 @@ impl Receipt { self.cumulative_gas_used.encode(out); self.bloom.encode(out); self.logs.encode(out); + #[cfg(feature = "optimism")] + if let Some(nonce) = self.deposit_nonce { + nonce.encode(out); + } } /// Consume the structure, returning only the receipt @@ -91,8 +102,10 @@ impl ReceiptWithBloom { match self.tx_type { TxType::EIP2930 => out.put_u8(0x01), TxType::EIP1559 => out.put_u8(0x02), - TxType::DEPOSIT => out.put_u8(0x7E), TxType::Legacy => unreachable!("legacy handled; qed."), + + #[cfg(feature = "optimism")] + TxType::DEPOSIT => out.put_u8(0x7E), } out.put_slice(payload.as_ref()); } @@ -120,6 +133,7 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, + deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), }, _ => Self { tx_type, @@ -127,6 +141,8 @@ impl ReceiptWithBloom { cumulative_gas_used: reth_rlp::Decodable::decode(b)?, bloom: reth_rlp::Decodable::decode(b)?, logs: reth_rlp::Decodable::decode(b)?, + #[cfg(feature = "optimism")] + deposit_nonce: None, }, }; @@ -389,6 +405,8 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; receipt.encode(&mut data); @@ -443,6 +461,8 @@ mod tests { data: Bytes::from_str("0100ff").unwrap().0.into(), }], success: false, + #[cfg(feature = "optimism")] + deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); From 3c5ee55ef2cb64c28a8e72461f5db36a25d0ce22 Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Mon, 19 Jun 2023 19:50:17 +0200 Subject: [PATCH 143/150] chore: system_tx and is_success method --- crates/interfaces/src/executor.rs | 3 --- crates/revm/src/executor.rs | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 009b5fb1320..ca82163ae5d 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -67,7 +67,6 @@ pub enum BlockExecutionError { #[cfg(feature = "optimism")] #[error("Insufficient funds to cover transaction L1 cost: want {want}, have {have}")] InsufficientFundsForL1Cost { want: u64, have: u64 }, -<<<<<<< HEAD } impl BlockExecutionError { @@ -75,6 +74,4 @@ impl BlockExecutionError { pub fn is_fatal(&self) -> bool { matches!(self, Self::CanonicalCommit { .. } | Self::CanonicalRevert { .. }) } -======= ->>>>>>> 891b841f (wip: removed receipts changes + improved gas handling) } diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 9a9f84c99ac..9d95d7e6123 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -17,7 +17,7 @@ use revm::{ db::{AccountState, CacheDB, DatabaseRef}, primitives::{ hash_map::{self, Entry}, - Account as RevmAccount, AccountInfo, ExecutionResult, ResultAndState, + Account as RevmAccount, AccountInfo, ResultAndState, }, EVM, }; From a1fe33c349fd4f1e81faf0d0b35c22c0d3709f1d Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Wed, 21 Jun 2023 09:21:36 +0200 Subject: [PATCH 144/150] chore: minor fixes --- crates/transaction-pool/src/traits.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 1b09ec44447..6be533b91c2 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -644,9 +644,8 @@ impl FromRecoveredTransaction for PooledTransaction { } #[cfg(feature = "optimism")] Transaction::Deposit(t) => { - // Gas price is always set to 0 for deposits in order to zero out ETH refunds. - // The actual L1 cost varies depending on the gas oracle feed, and is calculated - // during tx execution. + // Gas price is always set to 0 for deposits in order to zero out ETH refunds, + // because they already pay for their gas on L1. let gas_price = U256::from(0); let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); let effective_gas_price = 0u128; From 7212a789ef8fea38f728e52e94e7b0005d44972f Mon Sep 17 00:00:00 2001 From: clabby Date: Sat, 8 Jul 2023 18:47:49 -0400 Subject: [PATCH 145/150] Start resolving conflicts --- bin/reth/src/cli.rs | 6 +- crates/executor/Cargo.toml | 58 ----- crates/executor/src/lib.rs | 31 --- crates/interfaces/src/executor.rs | 4 - crates/net/eth-wire/src/types/receipts.rs | 22 -- crates/primitives/src/proofs.rs | 3 - crates/primitives/src/receipt.rs | 146 +++-------- crates/primitives/src/transaction/mod.rs | 60 +---- crates/revm/Cargo.toml | 3 +- crates/rpc/rpc-engine-api/src/engine_api.rs | 113 -------- crates/rpc/rpc-types/Cargo.toml | 3 +- crates/rpc/rpc-types/src/eth/engine.rs | 246 ------------------ .../rpc/rpc-types/src/eth/transaction/mod.rs | 5 - crates/transaction-pool/src/traits.rs | 9 - 14 files changed, 46 insertions(+), 663 deletions(-) delete mode 100644 crates/executor/Cargo.toml delete mode 100644 crates/executor/src/lib.rs delete mode 100644 crates/rpc/rpc-types/src/eth/engine.rs diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index 06f4d3a6a2f..c32b1ae17ed 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -91,6 +91,10 @@ struct Cli { #[derive(Debug, Args)] #[command(next_help_heading = "Logging")] pub struct Logs { + /// The flag to enable persistent logs. + #[arg(long = "log.persistent", global = true, conflicts_with = "journald")] + persistent: bool, + /// The path to put log files in. #[arg( long = "log.directory", @@ -112,7 +116,7 @@ pub struct Logs { impl Logs { /// Builds a tracing layer from the current log options. - pub fn layer(&self) -> (BoxedLayer, Option) + pub fn layer(&self) -> eyre::Result, Option)>> where S: Subscriber, for<'a> S: LookupSpan<'a>, diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml deleted file mode 100644 index 53acae3de9f..00000000000 --- a/crates/executor/Cargo.toml +++ /dev/null @@ -1,58 +0,0 @@ -[package] -name = "reth-executor" -version = "0.1.0" -edition = "2021" -license = "MIT OR Apache-2.0" -repository = "https://github.com/paradigmxyz/reth" -readme = "README.md" - -[package.metadata.cargo-udeps.ignore] -normal = [ - # Used for diagrams in docs - "aquamarine", -] - -[dependencies] -# reth -reth-primitives = { path = "../primitives" } -reth-interfaces = { path = "../interfaces" } -reth-revm = { path = "../revm" } -reth-revm-inspectors = { path = "../revm/revm-inspectors" } -reth-rlp = { path = "../rlp" } -reth-db = { path = "../storage/db" } -reth-provider = { path = "../storage/provider" } - -# revm -revm = { version = "3.0.0" } - -# common -async-trait = "0.1.57" -thiserror = "1.0.37" -auto_impl = "1.0" -tracing = "0.1.37" -tokio = { version = "1.21.2", features = ["sync"] } - -# mics -aquamarine = "0.3.0" -parking_lot = { version = "0.12", optional = true } - -triehash = "0.8" -# See to replace hashers to simplify libraries -plain_hasher = "0.2" -hash-db = "0.15" -# todo replace with faster rlp impl -rlp = { version = "0.5", default-features = false } -# replace with tiny-keccak (it is faster hasher) -sha3 = { version = "0.10", default-features = false } - - -[dev-dependencies] -reth-db = { path = "../storage/db", features = ["test-utils"] } -reth-interfaces = { path = "../interfaces", features = ["test-utils"] } -reth-primitives = { path = "../primitives", features = ["test-utils"] } -reth-provider = { path = "../storage/provider", features = ["test-utils"] } -parking_lot = "0.12" - -[features] -test-utils = ["parking_lot"] -optimism = [] diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs deleted file mode 100644 index c4e4b13664e..00000000000 --- a/crates/executor/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -#![warn(missing_docs, unreachable_pub)] -#![deny(unused_must_use, rust_2018_idioms)] -#![doc(test( - no_crate_inject, - attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) -))] - -//! Reth executor executes transaction in block of data. - -pub mod eth_dao_fork; -pub mod substate; - -/// Execution result types. -pub use reth_provider::post_state; - -pub mod blockchain_tree; - -/// Executor -pub mod executor; - -/// ExecutorFactory impl -pub mod factory; -pub use factory::Factory; - -#[cfg(any(test, feature = "test-utils"))] -/// Common test helpers for mocking out executor and executor factory -pub mod test_utils; - -#[cfg(feature = "optimism")] -/// Optimism-specific utilities for the executor -pub mod optimism; diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index ca82163ae5d..7f315688a35 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -7,10 +7,6 @@ use thiserror::Error; pub enum BlockValidationError { #[error("EVM reported invalid transaction ({hash:?}): {message}")] EVM { hash: H256, message: String }, - #[error("Verification failed")] - VerificationFailed, - #[error("Fatal internal error")] - ExecutionFatalError, #[error("Failed to recover sender for transaction")] SenderRecoveryError, #[error("Receipt root {got:?} is different than expected {expected:?}.")] diff --git a/crates/net/eth-wire/src/types/receipts.rs b/crates/net/eth-wire/src/types/receipts.rs index 146c9163567..af40108aa99 100644 --- a/crates/net/eth-wire/src/types/receipts.rs +++ b/crates/net/eth-wire/src/types/receipts.rs @@ -53,9 +53,6 @@ mod test { deposit_nonce: None, }, bloom: Default::default(), - logs: vec![], - #[cfg(feature = "optimism")] - deposit_nonce: None, }]]); let mut out = vec![]; @@ -126,14 +123,9 @@ mod test { #[cfg(feature = "optimism")] deposit_nonce: None, }, -<<<<<<< HEAD bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), }, ]]), -======= - ], - ]), ->>>>>>> 6266b9f3 (feat: receipts, l1 cost wip) }; request.encode(&mut data); assert_eq!(data, expected); @@ -169,20 +161,6 @@ mod test { deposit_nonce: None, }, bloom: hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").into(), - cumulative_gas_used: 0x1u64, - logs: vec![ - Log { - address: hex!("0000000000000000000000000000000000000011").into(), - topics: vec![ - hex!("000000000000000000000000000000000000000000000000000000000000dead").into(), - hex!("000000000000000000000000000000000000000000000000000000000000beef").into(), - ], - data: hex!("0100ff")[..].into(), - }, - ], - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }, ], ]), diff --git a/crates/primitives/src/proofs.rs b/crates/primitives/src/proofs.rs index 49fbbb5a7bd..40740d119af 100644 --- a/crates/primitives/src/proofs.rs +++ b/crates/primitives/src/proofs.rs @@ -175,9 +175,6 @@ mod tests { deposit_nonce: None, }, bloom, - logs, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; let receipt = vec![receipt]; let root = calculate_receipt_root(&receipt); diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index ea38cc575af..85b286c1462 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -34,36 +34,40 @@ pub struct Receipt { } impl Receipt { - /// Returns the rlp header for the receipt payload. - fn receipt_rlp_header(&self) -> reth_rlp::Header { - let mut rlp_head = reth_rlp::Header { list: true, payload_length: 0 }; + /// Calculates [`Log`]'s bloom filter. this is slow operation and [ReceiptWithBloom] can + /// be used to cache this value. + pub fn bloom_slow(&self) -> Bloom { + logs_bloom(self.logs.iter()) + } - rlp_head.payload_length += self.success.length(); - rlp_head.payload_length += self.cumulative_gas_used.length(); - rlp_head.payload_length += self.bloom.length(); - rlp_head.payload_length += self.logs.length(); - #[cfg(feature = "optimism")] - if self.tx_type == TxType::DEPOSIT { - // Transactions pre-Regolith don't have a deposit nonce - if let Some(nonce) = self.deposit_nonce { - rlp_head.payload_length += nonce.length(); - } - } + /// Calculates the bloom filter for the receipt and returns the [ReceiptWithBloom] container + /// type. + pub fn with_bloom(self) -> ReceiptWithBloom { + self.into() + } +} - rlp_head +impl From for ReceiptWithBloom { + fn from(receipt: Receipt) -> Self { + let bloom = receipt.bloom_slow(); + ReceiptWithBloom { receipt, bloom } } +} - /// Encodes the receipt data. - fn encode_fields(&self, out: &mut dyn BufMut) { - self.receipt_rlp_header().encode(out); - self.success.encode(out); - self.cumulative_gas_used.encode(out); - self.bloom.encode(out); - self.logs.encode(out); - #[cfg(feature = "optimism")] - if let Some(nonce) = self.deposit_nonce { - nonce.encode(out); - } +/// [`Receipt`] with calculated bloom filter. +#[main_codec] +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct ReceiptWithBloom { + /// Bloom filter build from logs. + pub bloom: Bloom, + /// Main receipt body + pub receipt: Receipt, +} + +impl ReceiptWithBloom { + /// Create new [ReceiptWithBloom] + pub fn new(receipt: Receipt, bloom: Bloom) -> Self { + Self { receipt, bloom } } /// Consume the structure, returning only the receipt @@ -85,35 +89,7 @@ impl Receipt { impl ReceiptWithBloom { /// Encode receipt with or without the header data. pub fn encode_inner(&self, out: &mut dyn BufMut, with_header: bool) { - if matches!(self.tx_type, TxType::Legacy) { - self.encode_fields(out); - return - } - - let mut payload = BytesMut::new(); - self.encode_fields(&mut payload); - - if with_header { - let payload_length = payload.len() + 1; - let header = reth_rlp::Header { list: false, payload_length }; - header.encode(out); - } - - match self.tx_type { - TxType::EIP2930 => out.put_u8(0x01), - TxType::EIP1559 => out.put_u8(0x02), - TxType::Legacy => unreachable!("legacy handled; qed."), - - #[cfg(feature = "optimism")] - TxType::DEPOSIT => out.put_u8(0x7E), - } - out.put_slice(payload.as_ref()); - } - - /// Returns the length of the receipt data. - fn receipt_length(&self) -> usize { - let rlp_head = self.receipt_rlp_header(); - length_of_length(rlp_head.payload_length) + rlp_head.payload_length + self.as_encoder().encode_inner(out, with_header) } /// Decodes the receipt payload @@ -125,26 +101,10 @@ impl ReceiptWithBloom { } let started_len = b.len(); - let this = match tx_type { - #[cfg(feature = "optimism")] - TxType::DEPOSIT => Self { - tx_type, - success: reth_rlp::Decodable::decode(b)?, - cumulative_gas_used: reth_rlp::Decodable::decode(b)?, - bloom: reth_rlp::Decodable::decode(b)?, - logs: reth_rlp::Decodable::decode(b)?, - deposit_nonce: Some(reth_rlp::Decodable::decode(b)?), - }, - _ => Self { - tx_type, - success: reth_rlp::Decodable::decode(b)?, - cumulative_gas_used: reth_rlp::Decodable::decode(b)?, - bloom: reth_rlp::Decodable::decode(b)?, - logs: reth_rlp::Decodable::decode(b)?, - #[cfg(feature = "optimism")] - deposit_nonce: None, - }, - }; + let success = reth_rlp::Decodable::decode(b)?; + let cumulative_gas_used = reth_rlp::Decodable::decode(b)?; + let bloom = Decodable::decode(b)?; + let logs = reth_rlp::Decodable::decode(b)?; let receipt = match tx_type { #[cfg(feature = "optimism")] @@ -389,24 +349,6 @@ mod tests { deposit_nonce: None, }, bloom: [0; 256].into(), - cumulative_gas_used: 0x1u64, - logs: vec![Log { - address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), - topics: vec![ - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000dead", - ) - .unwrap(), - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000beef", - ) - .unwrap(), - ], - data: Bytes::from_str("0100ff").unwrap().0.into(), - }], - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; receipt.encode(&mut data); @@ -445,24 +387,6 @@ mod tests { deposit_nonce: None, }, bloom: [0; 256].into(), - cumulative_gas_used: 0x1u64, - logs: vec![Log { - address: Address::from_str("0000000000000000000000000000000000000011").unwrap(), - topics: vec![ - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000dead", - ) - .unwrap(), - H256::from_str( - "000000000000000000000000000000000000000000000000000000000000beef", - ) - .unwrap(), - ], - data: Bytes::from_str("0100ff").unwrap().0.into(), - }], - success: false, - #[cfg(feature = "optimism")] - deposit_nonce: None, }; let receipt = ReceiptWithBloom::decode(&mut &data[..]).unwrap(); diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index bde3c6beff9..ccb59db342c 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -15,6 +15,7 @@ use reth_rlp::{ }; #[cfg(feature = "optimism")] use revm_primitives::U256; +use serde::{Deserialize, Serialize}; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; @@ -1162,58 +1163,7 @@ impl TransactionSigned { /// Inner encoding function that is used for both rlp [`Encodable`] trait and for calculating /// hash that for eip2718 does not require rlp header pub(crate) fn encode_inner(&self, out: &mut dyn bytes::BufMut, with_header: bool) { - match self.transaction { - Transaction::Legacy(TxLegacy { chain_id, .. }) => { - // do nothing w/ with_header - let payload_length = self.transaction.fields_len() + - self.signature.payload_len_with_eip155_chain_id(chain_id); - let header = Header { list: true, payload_length }; - header.encode(out); - self.transaction.encode_fields(out); - self.signature.encode_with_eip155_chain_id(out, chain_id); - } - #[cfg(feature = "optimism")] - Transaction::Deposit(_) => { - let payload_length = self.transaction.fields_len() + self.signature.payload_len(); - if with_header { - Header { - list: false, - payload_length: 1 + 1 + length_of_length(payload_length) + payload_length, - } - .encode(out); - } - out.put_u8(self.transaction.tx_type() as u8); - out.put_u8(DEPOSIT_VERSION); - let header = Header { list: true, payload_length }; - header.encode(out); - self.transaction.encode_fields(out); - // Deposit transactions do not have a signature. If the signature's values are not - // zero, then the transaction is invalid. - if self.signature().v(self.chain_id()) != 0 || - self.signature().r != U256::ZERO || - self.signature().s != U256::ZERO - { - // TODO: Ensure that this transaction may never have a non-zero signature - // higher up - we shouldn't be panicking here. - panic!("Deposit transactions must have a zero signature"); - } - } - _ => { - let payload_length = self.transaction.fields_len() + self.signature.payload_len(); - if with_header { - Header { - list: false, - payload_length: 1 + length_of_length(payload_length) + payload_length, - } - .encode(out); - } - out.put_u8(self.transaction.tx_type() as u8); - let header = Header { list: true, payload_length }; - header.encode(out); - self.transaction.encode_fields(out); - self.signature.encode(out); - } - } + self.transaction.encode_with_signature(&self.signature, out, with_header); } /// Output the length of the encode_inner(out, true). Note to assume that `with_header` is only @@ -1305,12 +1255,6 @@ impl TransactionSigned { let tx_type = *data.first().ok_or(DecodeError::InputTooShort)?; data.advance(1); -<<<<<<< HEAD - // If the transaction is a deposit, we need to first ensure that the version - // byte is correct. - #[cfg(feature = "optimism")] -======= ->>>>>>> 74cee5d0 (Move feature flags below comments) // If the transaction is a deposit, we need to first ensure that the version // byte is correct. #[cfg(feature = "optimism")] diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index 8a17c83aea9..31622ab3a1f 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -17,7 +17,8 @@ reth-revm-primitives = { path = "./revm-primitives" } reth-revm-inspectors = { path = "./revm-inspectors" } reth-consensus-common = { path = "../consensus/common" } -revm = { version = "3.0.0" } +# revm +revm = { workspace = true } # common tracing = { workspace = true } diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index c4ec8e0f38f..2a0581595f4 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -242,119 +242,6 @@ where Ok(result) } - /// When the Consensus layer receives a new block via the consensus gossip protocol, - /// the transactions in the block are sent to the execution layer in the form of a - /// `ExecutionPayload`. The Execution layer executes the transactions and validates the - /// state in the block header, then passes validation data back to Consensus layer, that - /// adds the block to the head of its own blockchain and attests to it. The block is then - /// broadcasted over the consensus p2p network in the form of a "Beacon block". - pub fn new_payload(&mut self, payload: ExecutionPayload) -> EngineApiResult { - let block = match self.try_construct_block(payload) { - Ok(b) => b, - Err(err) => { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::InvalidBlockHash { - validation_error: err.to_string(), - })) - } - }; - let block_hash = block.header.hash(); - let parent_hash = block.parent_hash; - - // The block already exists in our database - if self.client.is_known(&block_hash)? { - return Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)) - } - - let Some(parent) = self.client.block_by_hash(parent_hash)? else { - // TODO: cache block for storing later - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Syncing)) - }; - - let parent_td = if let Some(parent_td) = self.client.header_td(&block.parent_hash)? { - parent_td - } else { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::PayloadPreMerge.to_string(), - })) - }; - - // Short circuit the check by passing parent total difficulty. - if !self.chain_spec.fork(Hardfork::Paris).active_at_ttd(parent_td, U256::ZERO) { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::PayloadPreMerge.to_string(), - })) - } - - if block.timestamp <= parent.timestamp { - return Ok(PayloadStatus::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::PayloadTimestamp { - invalid: block.timestamp, - latest: parent.timestamp, - } - .to_string(), - })) - } - - let state_provider = self.client.latest()?; - let total_difficulty = parent_td + block.header.difficulty; - match executor::execute_and_verify_receipt( - &block.unseal(), - total_difficulty, - None, - &self.chain_spec, - &mut SubState::new(State::new(&state_provider)), - ) { - Ok(_) => Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)), - Err(err) => Ok(PayloadStatus::new( - PayloadStatusEnum::Invalid { validation_error: err.to_string() }, - parent_hash, // The parent hash is already in our database hence it is valid - )), - } - } - - /// Called to resolve chain forks and ensure that the Execution layer is working with the latest - /// valid chain. - pub fn fork_choice_updated( - &self, - fork_choice_state: ForkchoiceState, - payload_attributes: Option, - ) -> EngineApiResult { - let ForkchoiceState { head_block_hash, finalized_block_hash, .. } = fork_choice_state; - - if head_block_hash.is_zero() { - return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Invalid { - validation_error: EngineApiError::ForkchoiceEmptyHead.to_string(), - })) - } - - // Block is not known, nothing to do. - if !self.client.is_known(&head_block_hash)? { - return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) - } - - // The finalized block hash is not known, we are still syncing - if !finalized_block_hash.is_zero() && !self.client.is_known(&finalized_block_hash)? { - return Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Syncing)) - } - - if let Err(error) = self.forkchoice_state_tx.send(fork_choice_state) { - tracing::error!(target: "rpc::engine_api", ?error, "Failed to update forkchoice state"); - } - - if let Some(_attr) = payload_attributes { - #[cfg(feature = "optimism")] - if self.chain_spec.optimism.is_some() && _attr.gas_limit.is_none() { - return Err(EngineApiError::MissingGasLimitInPayloadAttributes) - } - - // TODO: optionally build the block - } - - let chain_info = self.client.chain_info()?; - Ok(ForkchoiceUpdated::from_status(PayloadStatusEnum::Valid) - .with_latest_valid_hash(chain_info.best_hash)) - } - /// Called to verify network configuration parameters and ensure that Consensus and Execution /// layers are using the latest configuration. pub async fn exchange_transition_configuration( diff --git a/crates/rpc/rpc-types/Cargo.toml b/crates/rpc/rpc-types/Cargo.toml index 06866c2ce09..c7037a9ebf3 100644 --- a/crates/rpc/rpc-types/Cargo.toml +++ b/crates/rpc/rpc-types/Cargo.toml @@ -24,7 +24,8 @@ serde_json = { workspace = true } jsonrpsee-types = { version = "0.18" } [dev-dependencies] -reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } +# reth +reth-interfaces = { workspace = true, features = ["test-utils"] } # misc rand = { workspace = true } diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs deleted file mode 100644 index 548ed16645b..00000000000 --- a/crates/rpc/rpc-types/src/eth/engine.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! Engine API types: and following the execution specs - -#![allow(missing_docs)] - -use reth_primitives::{ - Address, Block, Bloom, Bytes, SealedBlock, Withdrawal, H256, H64, U256, U64, -}; -use reth_rlp::Encodable; -use serde::{Deserialize, Serialize}; - -/// The list of supported Engine capabilities -pub const CAPABILITIES: [&str; 9] = [ - "engine_forkchoiceUpdatedV1", - "engine_forkchoiceUpdatedV2", - "engine_exchangeTransitionConfigurationV1", - "engine_getPayloadV1", - "engine_getPayloadV2", - "engine_newPayloadV1", - "engine_newPayloadV2", - "engine_getPayloadBodiesByHashV1", - "engine_getPayloadBodiesByRangeV1", -]; - -/// This structure maps on the ExecutionPayload structure of the beacon chain spec. -/// -/// See also: -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ExecutionPayload { - pub parent_hash: H256, - pub fee_recipient: Address, - pub state_root: H256, - pub receipts_root: H256, - pub logs_bloom: Bloom, - pub prev_randao: H256, - pub block_number: U64, - pub gas_limit: U64, - pub gas_used: U64, - pub timestamp: U64, - pub extra_data: Bytes, - pub base_fee_per_gas: U256, - pub block_hash: H256, - pub transactions: Vec, - /// Array of [`Withdrawal`] enabled with V2 - /// See - #[serde(default, skip_serializing_if = "Option::is_none")] - pub withdrawals: Option>, -} - -impl From for ExecutionPayload { - fn from(value: SealedBlock) -> Self { - let transactions = value - .body - .iter() - .map(|tx| { - let mut encoded = Vec::new(); - tx.encode(&mut encoded); - encoded.into() - }) - .collect(); - ExecutionPayload { - parent_hash: value.parent_hash, - fee_recipient: value.beneficiary, - state_root: value.state_root, - receipts_root: value.receipts_root, - logs_bloom: value.logs_bloom, - prev_randao: value.mix_hash, - block_number: value.number.into(), - gas_limit: value.gas_limit.into(), - gas_used: value.gas_used.into(), - timestamp: value.timestamp.into(), - extra_data: value.extra_data.clone(), - base_fee_per_gas: U256::from(value.base_fee_per_gas.unwrap_or_default()), - block_hash: value.hash(), - transactions, - withdrawals: value.withdrawals, - } - } -} - -/// This structure contains a body of an execution payload. -/// -/// See also: -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ExecutionPayloadBody { - pub transactions: Vec, - pub withdrawals: Vec, -} - -impl From for ExecutionPayloadBody { - fn from(value: Block) -> Self { - let transactions = value.body.into_iter().map(|tx| { - let mut out = Vec::new(); - tx.encode(&mut out); - out.into() - }); - ExecutionPayloadBody { - transactions: transactions.collect(), - withdrawals: value.withdrawals.unwrap_or_default(), - } - } -} - -/// The execution payload body response that allows for `null` values. -pub type ExecutionPayloadBodies = Vec>; - -/// This structure encapsulates the fork choice state -#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ForkchoiceState { - pub head_block_hash: H256, - pub safe_block_hash: H256, - pub finalized_block_hash: H256, -} - -/// This structure contains the attributes required to initiate a payload build process in the -/// context of an `engine_forkchoiceUpdated` call. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PayloadAttributes { - pub timestamp: U64, - pub prev_randao: H256, - pub suggested_fee_recipient: Address, - /// Array of [`Withdrawal`] enabled with V2 - /// See - #[serde(default, skip_serializing_if = "Option::is_none")] - pub withdrawals: Option>, - - /// Transactions is a field for rollups: the transactions list is forced into the block - #[cfg(feature = "optimism")] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub transactions: Option>, - - /// If true, the no transactions are taken out of the tx-pool, only transactions from the above - /// Transactions list will be included. - #[cfg(feature = "optimism")] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub no_tx_pool: Option, - - /// If set, this sets the exact gas limit the block produced with. - #[cfg(feature = "optimism")] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gas_limit: Option, -} - -/// This structure contains the result of processing a payload -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PayloadStatus { - #[serde(flatten)] - pub status: PayloadStatusEnum, - /// Hash of the most recent valid block in the branch defined by payload and its ancestors - pub latest_valid_hash: Option, -} - -impl PayloadStatus { - pub fn new(status: PayloadStatusEnum, latest_valid_hash: H256) -> Self { - Self { status, latest_valid_hash: Some(latest_valid_hash) } - } - - pub fn from_status(status: PayloadStatusEnum) -> Self { - Self { status, latest_valid_hash: None } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "status", rename_all = "SCREAMING_SNAKE_CASE")] -pub enum PayloadStatusEnum { - Valid, - Invalid { - #[serde(rename = "validationError")] - validation_error: String, - }, - Syncing, - Accepted, - InvalidBlockHash { - #[serde(rename = "validationError")] - validation_error: String, - }, -} - -/// This structure contains configurable settings of the transition process. -#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TransitionConfiguration { - /// Maps on the TERMINAL_TOTAL_DIFFICULTY parameter of EIP-3675 - pub terminal_total_difficulty: U256, - /// Maps on TERMINAL_BLOCK_HASH parameter of EIP-3675 - pub terminal_block_hash: H256, - /// Maps on TERMINAL_BLOCK_NUMBER parameter of EIP-3675 - pub terminal_block_number: U64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ForkchoiceUpdated { - pub payload_status: PayloadStatus, - pub payload_id: Option, -} - -impl ForkchoiceUpdated { - pub fn new(payload_status: PayloadStatus) -> Self { - Self { payload_status, payload_id: None } - } - - pub fn from_status(status: PayloadStatusEnum) -> Self { - Self { payload_status: PayloadStatus::from_status(status), payload_id: None } - } - - pub fn with_latest_valid_hash(mut self, hash: H256) -> Self { - self.payload_status.latest_valid_hash = Some(hash); - self - } - - pub fn with_payload_id(mut self, id: H64) -> Self { - self.payload_id = Some(id); - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - use reth_interfaces::test_utils::generators::random_block_range; - use reth_primitives::{TransactionSigned, H256}; - use reth_rlp::Decodable; - - #[test] - fn payload_body_roundtrip() { - for block in random_block_range(0..100, H256::default(), 0..2) { - let unsealed = block.clone().unseal(); - let payload_body: ExecutionPayloadBody = unsealed.into(); - - assert_eq!( - Ok(block.body), - payload_body - .transactions - .iter() - .map(|x| TransactionSigned::decode(&mut &x[..])) - .collect::, _>>(), - ); - - assert_eq!(block.withdrawals.unwrap_or_default(), payload_body.withdrawals); - } - } -} diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index 5371ba9fea4..109257909a1 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -111,7 +111,6 @@ impl Transaction { let (gas_price, max_fee_per_gas) = match signed_tx.tx_type() { TxType::Legacy => (Some(U128::from(signed_tx.max_fee_per_gas())), None), -<<<<<<< HEAD TxType::EIP2930 => (Some(U128::from(signed_tx.max_fee_per_gas())), None), TxType::EIP1559 => { // the gas price field for EIP1559 is set to `min(tip, gasFeeCap - baseFee) + @@ -124,10 +123,6 @@ impl Transaction { (Some(U128::from(gas_price)), Some(U128::from(signed_tx.max_fee_per_gas()))) } -======= - TxType::EIP2930 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), - TxType::EIP1559 => (None, Some(U128::from(signed_tx.max_fee_per_gas()))), ->>>>>>> 1c17ad88 (compiles) #[cfg(feature = "optimism")] TxType::DEPOSIT => (None, None), }; diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 6be533b91c2..5af49270ab7 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -642,15 +642,6 @@ impl FromRecoveredTransaction for PooledTransaction { // because they already pay for their gas on L1. U256::ZERO } - #[cfg(feature = "optimism")] - Transaction::Deposit(t) => { - // Gas price is always set to 0 for deposits in order to zero out ETH refunds, - // because they already pay for their gas on L1. - let gas_price = U256::from(0); - let cost = U256::from(gas_price) * U256::from(t.gas_limit) + U256::from(t.value); - let effective_gas_price = 0u128; - (cost, effective_gas_price) - } }; let cost = gas_cost + U256::from(tx.value()); From 57824e2275177aaea927cbb61603865a8be45132 Mon Sep 17 00:00:00 2001 From: clabby Date: Sat, 8 Jul 2023 20:53:30 -0400 Subject: [PATCH 146/150] More conflict resolution / updates x --- crates/primitives/src/transaction/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index ccb59db342c..72f39afabcb 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -13,8 +13,6 @@ use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, }; -#[cfg(feature = "optimism")] -use revm_primitives::U256; use serde::{Deserialize, Serialize}; pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; From 17f771e7bf11208eb20fd23d697d4859bb364f18 Mon Sep 17 00:00:00 2001 From: Roberto Bayardo Date: Tue, 18 Jul 2023 20:22:29 -0400 Subject: [PATCH 147/150] fix build failures with optimism feature --- crates/primitives/src/transaction/mod.rs | 2 ++ crates/primitives/src/transaction/optimism.rs | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 72f39afabcb..d6c34bf3e91 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -320,6 +320,8 @@ impl Transaction { Transaction::Legacy(tx) => tx.size(), Transaction::Eip2930(tx) => tx.size(), Transaction::Eip1559(tx) => tx.size(), + #[cfg(feature = "optimism")] + Transaction::Deposit(tx) => tx.size(), } } } diff --git a/crates/primitives/src/transaction/optimism.rs b/crates/primitives/src/transaction/optimism.rs index faad6908d48..546a6300b6b 100644 --- a/crates/primitives/src/transaction/optimism.rs +++ b/crates/primitives/src/transaction/optimism.rs @@ -1,6 +1,7 @@ use crate::{Address, Bytes, TransactionKind, H256}; use reth_codecs::{main_codec, Compact}; use reth_rlp::{Encodable, EMPTY_STRING_CODE}; +use std::mem; /// EIP-2718 transaction type selector. pub const DEPOSIT_TX_TYPE: u8 = 126; @@ -36,6 +37,19 @@ pub struct TxDeposit { } impl TxDeposit { + /// Calculates a heuristic for the in-memory size of the [TxDeposit] transaction. + #[inline] + pub fn size(&self) -> usize { + mem::size_of::() + // source_hash + mem::size_of::
() + // from + self.to.size() + // to + mem::size_of::>() + // mint + mem::size_of::() + // value + mem::size_of::() + // gas_limit + mem::size_of::() + // is_system_transaction + self.input.len() // input + } + /// Outputs the length of the transaction's fields, without a RLP header or length of the /// eip155 fields. pub(crate) fn fields_len(&self) -> usize { From 9669a1c2e1947997b8697fc8a1d8af5df31357ea Mon Sep 17 00:00:00 2001 From: clabby Date: Tue, 18 Jul 2023 20:31:41 -0400 Subject: [PATCH 148/150] Fix feature flag --- crates/rpc/rpc/Cargo.toml | 3 +++ crates/rpc/rpc/src/eth/api/pending_block.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 11111cafbc6..8c804ce56f7 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -66,3 +66,6 @@ jsonrpsee = { version = "0.18", features = ["client"] } assert_matches = "1.5.0" tempfile = "3.5.0" reth-interfaces = { workspace = true, features = ["test-utils"] } + +[features] +optimism = ["reth-primitives/optimism"] diff --git a/crates/rpc/rpc/src/eth/api/pending_block.rs b/crates/rpc/rpc/src/eth/api/pending_block.rs index a427df3279a..d5fc4ce6295 100644 --- a/crates/rpc/rpc/src/eth/api/pending_block.rs +++ b/crates/rpc/rpc/src/eth/api/pending_block.rs @@ -110,6 +110,8 @@ impl PendingBlockEnv { success: result.is_success(), cumulative_gas_used, logs: result.logs().into_iter().map(into_reth_log).collect(), + #[cfg(feature = "optimism")] + deposit_nonce: None, }, ); // append transaction to the list of executed transactions From 632d1bfb6a930f162e0ba391bff6da7929d7bc4f Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Mon, 17 Jul 2023 16:26:14 -0700 Subject: [PATCH 149/150] Use TTD(0) fork condition for OP_GOERLI Paris hardfork Signed-off-by: Brian Bland --- crates/primitives/src/chain/spec.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 732fb13abac..06e40cd6765 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -147,7 +147,7 @@ pub static OP_GOERLI: Lazy> = Lazy::new(|| { "c1fc15cd51159b1f1e5cbc4b82e85c1447ddfa33c52cf1d98d14fba0d6354be1" ))), fork_timestamps: ForkTimestamps::default(), // TODO(clabby): update this - paris_block_and_final_difficulty: None, // TODO(clabby): update this + paris_block_and_final_difficulty: Some((0, U256::from(0))), hardforks: BTreeMap::from([ (Hardfork::Byzantium, ForkCondition::Block(0)), (Hardfork::Constantinople, ForkCondition::Block(0)), @@ -156,7 +156,10 @@ pub static OP_GOERLI: Lazy> = Lazy::new(|| { (Hardfork::MuirGlacier, ForkCondition::Block(0)), (Hardfork::Berlin, ForkCondition::Block(0)), (Hardfork::London, ForkCondition::Block(0)), - (Hardfork::Paris, ForkCondition::Block(0)), + ( + Hardfork::Paris, + ForkCondition::TTD { fork_block: Some(0), total_difficulty: U256::from(0) }, + ), (Hardfork::Regolith, ForkCondition::Timestamp(1679079600)), ]), optimism: Some(OptimismConfig { eip_1559_elasticity: 10, eip_1559_denominator: 50 }), From a9a84c56bcf28851a9505650af8239c7bf2583ba Mon Sep 17 00:00:00 2001 From: nicolas <48695862+merklefruit@users.noreply.github.com> Date: Thu, 27 Jul 2023 10:36:24 +0200 Subject: [PATCH 150/150] chore: fix merge conflicts --- .gitignore | 7 - bin/reth/Cargo.toml | 1 + bin/reth/src/args/rpc_server_args.rs | 2 +- crates/consensus/beacon/src/engine/mod.rs | 192 +++++++++---------- crates/consensus/common/src/validation.rs | 3 - crates/primitives/src/transaction/mod.rs | 5 - crates/primitives/src/transaction/tx_type.rs | 6 +- crates/revm/src/executor.rs | 11 ++ crates/rpc/rpc-builder/src/constants.rs | 7 - crates/rpc/rpc-builder/src/eth.rs | 2 +- crates/rpc/rpc-types/src/eth/fee.rs | 5 +- etc/grafana/dashboards/overview.json | 66 ++----- 12 files changed, 122 insertions(+), 185 deletions(-) diff --git a/.gitignore b/.gitignore index 9e35cf5cd7c..bf70ac93f2b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,13 +9,6 @@ target/ # MSVC Windows builds of rustc generate these, which store debugging information *.pdb -<<<<<<< HEAD -# Generated by VSCode -.vscode - - -======= ->>>>>>> 49420f5bb3dc9e2a92f841d0e9c93c1cc45ad4e3 # Generated by Intellij-based IDEs. .idea diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 19aa4fc7f6e..4f2d7e5bd70 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -104,6 +104,7 @@ optimism = [ "reth-revm/optimism", "reth-interfaces/optimism", "reth-rpc-engine-api/optimism", + "reth-rpc/optimism", "reth-transaction-pool/optimism", "reth-provider/optimism", "reth-beacon-consensus/optimism", diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 66b6f9f78fd..d05ce42b431 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -17,13 +17,13 @@ use reth_rpc::{ DEFAULT_BLOCK_CACHE_MAX_LEN, DEFAULT_ENV_CACHE_MAX_LEN, DEFAULT_RECEIPT_CACHE_MAX_LEN, }, gas_oracle::GasPriceOracleConfig, + RPC_DEFAULT_GAS_CAP, }, JwtError, JwtSecret, }; use reth_rpc_builder::{ auth::{AuthServerConfig, AuthServerHandle}, constants, - constants::RPC_DEFAULT_GAS_CAP, error::RpcError, EthConfig, IpcServerBuilder, RethRpcModule, RpcModuleBuilder, RpcModuleConfig, RpcModuleSelection, RpcServerConfig, RpcServerHandle, ServerBuilder, TransportRpcModuleConfig, diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 9fb636c418b..7c74bc39575 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1761,12 +1761,11 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) - .disable_blockchain_tree_sync() - .with_max_block(1) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) + .disable_blockchain_tree_sync() + .with_max_block(1) + .build(); let res = spawn_consensus_engine(consensus_engine); @@ -1793,12 +1792,11 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) - .disable_blockchain_tree_sync() - .with_max_block(1) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Err(StageError::ChannelClosed)])) + .disable_blockchain_tree_sync() + .with_max_block(1) + .build(); let mut rx = spawn_consensus_engine(consensus_engine); @@ -1856,15 +1854,14 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(1), done: true }), - Err(StageError::ChannelClosed), - ])) - .disable_blockchain_tree_sync() - .with_max_block(2) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(1), done: true }), + Err(StageError::ChannelClosed), + ])) + .disable_blockchain_tree_sync() + .with_max_block(2) + .build(); let rx = spawn_consensus_engine(consensus_engine); @@ -1892,15 +1889,14 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(max_block), - done: true, - })])) - .with_max_block(max_block) - .disable_blockchain_tree_sync() - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(max_block), + done: true, + })])) + .with_max_block(max_block) + .disable_blockchain_tree_sync() + .build(); let rx = spawn_consensus_engine(consensus_engine); @@ -1942,13 +1938,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let mut engine_rx = spawn_consensus_engine(consensus_engine); @@ -1974,13 +1969,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2024,14 +2018,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - ])) - .disable_blockchain_tree_sync() - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + ])) + .disable_blockchain_tree_sync() + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2075,14 +2068,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .disable_blockchain_tree_sync() - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .disable_blockchain_tree_sync() + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2114,13 +2106,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - ])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + ])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let mut block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2168,13 +2159,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([ - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), - ])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([ + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + Ok(ExecOutput { checkpoint: StageCheckpoint::new(0), done: true }), + ])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2219,13 +2209,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let mut engine_rx = spawn_consensus_engine(consensus_engine); @@ -2255,13 +2244,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); let block1 = random_block(&mut rng, 1, Some(genesis.hash), None, Some(0)); @@ -2359,13 +2347,12 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .build(); let genesis = random_block(&mut rng, 0, None, None, Some(0)); @@ -2416,14 +2403,13 @@ mod tests { .build(), ); - let (consensus_engine, env) = - TestConsensusEngineBuilder::::new(chain_spec.clone()) - .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { - checkpoint: StageCheckpoint::new(0), - done: true, - })])) - .with_executor_results(Vec::from([exec_result2])) - .build(); + let (consensus_engine, env) = TestConsensusEngineBuilder::new(chain_spec.clone()) + .with_pipeline_exec_outputs(VecDeque::from([Ok(ExecOutput { + checkpoint: StageCheckpoint::new(0), + done: true, + })])) + .with_executor_results(Vec::from([exec_result2])) + .build(); insert_blocks( env.db.as_ref(), diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index 575da865c4b..5fffb7f2a2e 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -10,9 +10,6 @@ use std::collections::{hash_map::Entry, HashMap}; #[cfg(feature = "optimism")] use reth_primitives::TxDeposit; -#[cfg(feature = "optimism")] -use reth_primitives::TxDeposit; - /// Validate header standalone pub fn validate_header_standalone( header: &SealedHeader, diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index be90ee01448..7c56a807752 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -467,11 +467,6 @@ impl Compact for Transaction { Transaction::Deposit(tx) => { tx.to_compact(buf); } - #[cfg(feature = "optimism")] - Transaction::Deposit(tx) => { - tx.to_compact(buf); - 126 - } } identifier } diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index 0f487c6dbeb..5b256c2d9c8 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -63,8 +63,10 @@ impl Compact for TxType { TxType::Legacy => LEGACY_TX_TYPE_ID as usize, TxType::EIP2930 => EIP2930_TX_TYPE_ID as usize, TxType::EIP1559 => EIP1559_TX_TYPE_ID as usize, - #[cfg(feature = "optimism")] - TxType::DEPOSIT => DEPOSIT_TX_TYPE as usize, + _ => { + buf.put_u8(self as u8); + 3 + } } } diff --git a/crates/revm/src/executor.rs b/crates/revm/src/executor.rs index 9d95d7e6123..d76eb644295 100644 --- a/crates/revm/src/executor.rs +++ b/crates/revm/src/executor.rs @@ -236,6 +236,17 @@ where let mut cumulative_gas_used = 0; let mut post_state = PostState::with_tx_capacity(block.number, block.body.len()); for (transaction, sender) in block.body.iter().zip(senders) { + // The sum of the transaction’s gas limit, Tg, and the gas utilised in this block prior, + // must be no greater than the block’s gasLimit. + let block_available_gas = block.header.gas_limit - cumulative_gas_used; + if transaction.gas_limit() > block_available_gas { + return Err(BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { + transaction_gas_limit: transaction.gas_limit(), + block_available_gas, + } + .into()) + } + #[cfg(feature = "optimism")] { let db = self.db(); diff --git a/crates/rpc/rpc-builder/src/constants.rs b/crates/rpc/rpc-builder/src/constants.rs index 6768ca62a65..a1b2bc36a82 100644 --- a/crates/rpc/rpc-builder/src/constants.rs +++ b/crates/rpc/rpc-builder/src/constants.rs @@ -7,13 +7,6 @@ pub const DEFAULT_WS_RPC_PORT: u16 = 8546; /// The default port for the auth server. pub const DEFAULT_AUTH_PORT: u16 = 8551; -/// The default gas limit for eth_call and adjacent calls. -/// -/// This is different from the default to regular 30M block gas limit -/// [ETHEREUM_BLOCK_GAS_LIMIT](reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT) to allow for -/// more complex calls. -pub const RPC_DEFAULT_GAS_CAP: u64 = 50_000_000; - /// The default IPC endpoint #[cfg(windows)] pub const DEFAULT_IPC_ENDPOINT: &str = r"\\.\pipe\reth.ipc"; diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs index eac5219fc30..34ec4989c62 100644 --- a/crates/rpc/rpc-builder/src/eth.rs +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -1,8 +1,8 @@ -use crate::constants::RPC_DEFAULT_GAS_CAP; use reth_rpc::{ eth::{ cache::{EthStateCache, EthStateCacheConfig}, gas_oracle::GasPriceOracleConfig, + RPC_DEFAULT_GAS_CAP, }, EthApi, EthFilter, EthPubSub, }; diff --git a/crates/rpc/rpc-types/src/eth/fee.rs b/crates/rpc/rpc-types/src/eth/fee.rs index c7f93f6eaf6..6463e7b433c 100644 --- a/crates/rpc/rpc-types/src/eth/fee.rs +++ b/crates/rpc/rpc-types/src/eth/fee.rs @@ -12,7 +12,10 @@ pub struct TxGasAndReward { impl PartialOrd for TxGasAndReward { fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) + // compare only the reward + // see: + // + self.reward.partial_cmp(&other.reward) } } diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 103b8c6653d..7a5cd77ff50 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -27,11 +27,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", -<<<<<<< HEAD - "version": "9.5.3" -======= "version": "10.0.2" ->>>>>>> 49420f5bb3dc9e2a92f841d0e9c93c1cc45ad4e3 }, { "type": "panel", @@ -154,20 +150,14 @@ "options": { "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, -<<<<<<< HEAD - "pluginVersion": "9.5.3", -======= "pluginVersion": "10.0.2", ->>>>>>> 49420f5bb3dc9e2a92f841d0e9c93c1cc45ad4e3 "targets": [ { "datasource": { @@ -225,20 +215,14 @@ "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, "showUnfilled": true, "valueMode": "color" }, -<<<<<<< HEAD - "pluginVersion": "9.5.3", -======= "pluginVersion": "10.0.2", ->>>>>>> 49420f5bb3dc9e2a92f841d0e9c93c1cc45ad4e3 "targets": [ { "datasource": { @@ -621,11 +605,7 @@ "unit": "percentunit" } }, -<<<<<<< HEAD - "pluginVersion": "9.5.3", -======= "pluginVersion": "10.0.2", ->>>>>>> 49420f5bb3dc9e2a92f841d0e9c93c1cc45ad4e3 "targets": [ { "datasource": { @@ -676,22 +656,16 @@ }, "id": 48, "options": { - "displayLabels": [ - "name" - ], + "displayLabels": ["name"], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, - "values": [ - "value" - ] + "values": ["value"] }, "pieType": "pie", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -847,15 +821,11 @@ "displayMode": "table", "placement": "right", "showLegend": true, - "values": [ - "value" - ] + "values": ["value"] }, "pieType": "pie", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1015,18 +985,12 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true }, -<<<<<<< HEAD - "pluginVersion": "9.5.3", -======= "pluginVersion": "10.0.2", ->>>>>>> 49420f5bb3dc9e2a92f841d0e9c93c1cc45ad4e3 "targets": [ { "datasource": { @@ -1485,9 +1449,7 @@ "displayMode": "table", "placement": "right", "showLegend": true, - "values": [ - "value" - ] + "values": ["value"] }, "tooltip": { "mode": "multi", @@ -1566,9 +1528,7 @@ }, "pieType": "pie", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -4923,10 +4883,6 @@ "timezone": "", "title": "reth", "uid": "2k8BXz24x", -<<<<<<< HEAD - "version": 2, -======= "version": 3, ->>>>>>> 49420f5bb3dc9e2a92f841d0e9c93c1cc45ad4e3 "weekStart": "" -} \ No newline at end of file +}