diff --git a/Cargo.lock b/Cargo.lock index 556f8da69d7..d06b220e05f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6772,6 +6772,7 @@ dependencies = [ "reth-primitives", "reth-primitives-traits", "reth-provider", + "reth-revm", "reth-stages-types", "reth-trie", "serde", diff --git a/bin/reth/src/commands/stage/drop.rs b/bin/reth/src/commands/stage/drop.rs index ec32af330e9..eadcc45807b 100644 --- a/bin/reth/src/commands/stage/drop.rs +++ b/bin/reth/src/commands/stage/drop.rs @@ -105,7 +105,7 @@ impl Command { Default::default(), )?; let alloc = &self.env.chain.genesis().alloc; - insert_genesis_state::(tx, alloc.len(), alloc.iter())?; + insert_genesis_state::(tx, alloc.iter())?; } StageEnum::AccountHashing => { tx.clear::()?; diff --git a/crates/evm/execution-types/src/execution_outcome.rs b/crates/evm/execution-types/src/execution_outcome.rs index 322e397a6be..c391e347722 100644 --- a/crates/evm/execution-types/src/execution_outcome.rs +++ b/crates/evm/execution-types/src/execution_outcome.rs @@ -1,13 +1,12 @@ use reth_primitives::{ logs_bloom, Account, Address, BlockNumber, Bloom, Bytecode, Log, Receipt, Receipts, Requests, - StorageEntry, B256, U256, + B256, U256, }; use reth_trie::HashedPostState; use revm::{ db::{states::BundleState, BundleAccount}, primitives::AccountInfo, }; -use std::collections::HashMap; /// Represents the outcome of block execution, including post-execution changes and reverts. /// @@ -35,16 +34,6 @@ pub struct ExecutionOutcome { pub requests: Vec, } -/// Type used to initialize revms bundle state. -pub type BundleStateInit = - HashMap, Option, HashMap)>; - -/// Types used inside `RevertsInit` to initialize revms reverts. -pub type AccountRevertInit = (Option>, Vec); - -/// Type used to initialize revms reverts. -pub type RevertsInit = HashMap>; - impl ExecutionOutcome { /// Creates a new `ExecutionOutcome`. /// @@ -59,48 +48,6 @@ impl ExecutionOutcome { Self { bundle, receipts, first_block, requests } } - /// Creates a new `ExecutionOutcome` from initialization parameters. - /// - /// This constructor initializes a new `ExecutionOutcome` instance using detailed - /// initialization parameters. - pub fn new_init( - state_init: BundleStateInit, - revert_init: RevertsInit, - contracts_init: Vec<(B256, Bytecode)>, - receipts: Receipts, - first_block: BlockNumber, - requests: Vec, - ) -> Self { - // sort reverts by block number - let mut reverts = revert_init.into_iter().collect::>(); - reverts.sort_unstable_by_key(|a| a.0); - - // initialize revm bundle - let bundle = BundleState::new( - state_init.into_iter().map(|(address, (original, present, storage))| { - ( - address, - original.map(Into::into), - present.map(Into::into), - storage.into_iter().map(|(k, s)| (k.into(), s)).collect(), - ) - }), - reverts.into_iter().map(|(_, reverts)| { - // does not needs to be sorted, it is done when taking reverts. - reverts.into_iter().map(|(address, (original, storage))| { - ( - address, - original.map(|i| i.map(Into::into)), - storage.into_iter().map(|entry| (entry.key.into(), entry.value)), - ) - }) - }), - contracts_init.into_iter().map(|(code_hash, bytecode)| (code_hash, bytecode.0)), - ); - - Self { bundle, receipts, first_block, requests } - } - /// Return revm bundle state. pub const fn state(&self) -> &BundleState { &self.bundle @@ -394,37 +341,7 @@ mod tests { }; // Assert that creating a new ExecutionOutcome using the constructor matches exec_res - assert_eq!( - ExecutionOutcome::new(bundle, receipts.clone(), first_block, requests.clone()), - exec_res - ); - - // Create a BundleStateInit object and insert initial data - let mut state_init: BundleStateInit = HashMap::new(); - state_init - .insert(Address::new([2; 20]), (None, Some(Account::default()), HashMap::default())); - - // Create a HashMap for account reverts and insert initial data - let mut revert_inner: HashMap = HashMap::new(); - revert_inner.insert(Address::new([2; 20]), (None, vec![])); - - // Create a RevertsInit object and insert the revert_inner data - let mut revert_init: RevertsInit = HashMap::new(); - revert_init.insert(123, revert_inner); - - // Assert that creating a new ExecutionOutcome using the new_init method matches - // exec_res - assert_eq!( - ExecutionOutcome::new_init( - state_init, - revert_init, - vec![], - receipts, - first_block, - requests, - ), - exec_res - ); + assert_eq!(ExecutionOutcome::new(bundle, receipts, first_block, requests), exec_res); } #[test] diff --git a/crates/primitives/src/account.rs b/crates/primitives/src/account.rs new file mode 100644 index 00000000000..49d7d6d88c2 --- /dev/null +++ b/crates/primitives/src/account.rs @@ -0,0 +1,160 @@ +use crate::revm_primitives::{Bytecode as RevmBytecode, Bytes}; +use byteorder::{BigEndian, ReadBytesExt}; +use bytes::Buf; +use derive_more::Deref; +use reth_codecs::Compact; +use revm_primitives::JumpTable; +use serde::{Deserialize, Serialize}; + +pub use reth_primitives_traits::Account; + +/// Bytecode for an account. +/// +/// A wrapper around [`revm::primitives::Bytecode`][RevmBytecode] with encoding/decoding support. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Deref)] +pub struct Bytecode(pub RevmBytecode); + +impl Bytecode { + /// Create new bytecode from raw bytes. + /// + /// No analysis will be performed. + pub fn new_raw(bytes: Bytes) -> Self { + Self(RevmBytecode::new_raw(bytes)) + } +} + +impl From for RevmBytecode { + fn from(value: Bytecode) -> Self { + value.0 + } +} + +impl Compact for Bytecode { + fn to_compact(self, buf: &mut B) -> usize + where + B: bytes::BufMut + AsMut<[u8]>, + { + let bytecode = &self.0.bytecode()[..]; + buf.put_u32(bytecode.len() as u32); + buf.put_slice(bytecode); + let len = match &self.0 { + RevmBytecode::LegacyRaw(_) => { + buf.put_u8(0); + 1 + } + // `1` has been removed. + RevmBytecode::LegacyAnalyzed(analyzed) => { + buf.put_u8(2); + buf.put_u64(analyzed.original_len() as u64); + let map = analyzed.jump_table().as_slice(); + buf.put_slice(map); + 1 + 8 + map.len() + } + RevmBytecode::Eof(_) => { + // buf.put_u8(3); + // TODO(EOF) + todo!("EOF") + } + }; + len + bytecode.len() + 4 + } + + // # Panics + // + // A panic will be triggered if a bytecode variant of 1 or greater than 2 is passed from the + // database. + fn from_compact(mut buf: &[u8], _: usize) -> (Self, &[u8]) { + let len = buf.read_u32::().expect("could not read bytecode length"); + let bytes = Bytes::from(buf.copy_to_bytes(len as usize)); + let variant = buf.read_u8().expect("could not read bytecode variant"); + let decoded = match variant { + 0 => Self(RevmBytecode::new_raw(bytes)), + 1 => unreachable!("Junk data in database: checked Bytecode variant was removed"), + 2 => Self(unsafe { + RevmBytecode::new_analyzed( + bytes, + buf.read_u64::().unwrap() as usize, + JumpTable::from_slice(buf), + ) + }), + // TODO(EOF) + 3 => todo!("EOF"), + _ => unreachable!("Junk data in database: unknown Bytecode variant"), + }; + (decoded, &[]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{hex_literal::hex, B256, KECCAK_EMPTY, U256}; + use revm_primitives::LegacyAnalyzedBytecode; + + #[test] + fn test_account() { + let mut buf = vec![]; + let mut acc = Account::default(); + let len = acc.to_compact(&mut buf); + assert_eq!(len, 2); + + acc.balance = U256::from(2); + let len = acc.to_compact(&mut buf); + assert_eq!(len, 3); + + acc.nonce = 2; + let len = acc.to_compact(&mut buf); + assert_eq!(len, 4); + } + + #[test] + fn test_empty_account() { + let mut acc = Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None }; + // Nonce 0, balance 0, and bytecode hash set to None is considered empty. + assert!(acc.is_empty()); + + acc.bytecode_hash = Some(KECCAK_EMPTY); + // Nonce 0, balance 0, and bytecode hash set to KECCAK_EMPTY is considered empty. + assert!(acc.is_empty()); + + acc.balance = U256::from(2); + // Non-zero balance makes it non-empty. + assert!(!acc.is_empty()); + + acc.balance = U256::ZERO; + acc.nonce = 10; + // Non-zero nonce makes it non-empty. + assert!(!acc.is_empty()); + + acc.nonce = 0; + acc.bytecode_hash = Some(B256::from(U256::ZERO)); + // Non-empty bytecode hash makes it non-empty. + assert!(!acc.is_empty()); + } + + #[test] + fn test_bytecode() { + let mut buf = vec![]; + let bytecode = Bytecode::new_raw(Bytes::default()); + let len = bytecode.to_compact(&mut buf); + assert_eq!(len, 5); + + let mut buf = vec![]; + let bytecode = Bytecode::new_raw(Bytes::from(&hex!("ffff"))); + let len = bytecode.to_compact(&mut buf); + assert_eq!(len, 7); + + let mut buf = vec![]; + let bytecode = Bytecode(RevmBytecode::LegacyAnalyzed(LegacyAnalyzedBytecode::new( + Bytes::from(&hex!("ffff")), + 2, + JumpTable::from_slice(&[0]), + ))); + let len = bytecode.clone().to_compact(&mut buf); + assert_eq!(len, 16); + + let (decoded, remainder) = Bytecode::from_compact(&buf, len); + assert_eq!(decoded, bytecode); + assert!(remainder.is_empty()); + } +} diff --git a/crates/storage/db-common/Cargo.toml b/crates/storage/db-common/Cargo.toml index d80236defd3..2b9ccba1b82 100644 --- a/crates/storage/db-common/Cargo.toml +++ b/crates/storage/db-common/Cargo.toml @@ -19,6 +19,7 @@ reth-trie.workspace = true reth-etl.workspace = true reth-codecs.workspace = true reth-stages-types.workspace = true +reth-revm.workspace = true reth-fs-util.workspace = true # eth diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index 76314f0c8b7..b9e5572dc16 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -8,16 +8,17 @@ use reth_db::tables; use reth_db_api::{database::Database, transaction::DbTxMut, DatabaseError}; use reth_etl::Collector; use reth_primitives::{ - Account, Address, Bytecode, Receipts, StaticFileSegment, StorageEntry, B256, U256, + revm::compat::into_revm_acc, Account, Address, Bytecode, ChainSpec, GenesisAccount, Receipts, + StaticFileSegment, StorageEntry, B256, U256, }; use reth_provider::{ - bundle_state::{BundleStateInit, RevertsInit}, errors::provider::ProviderResult, providers::{StaticFileProvider, StaticFileWriter}, BlockHashReader, BlockNumReader, ChainSpecProvider, DatabaseProviderRW, ExecutionOutcome, HashingWriter, HistoryWriter, OriginalValuesKnown, ProviderError, ProviderFactory, StageCheckpointWriter, StateWriter, StaticFileProviderFactory, }; +use reth_revm::db::states::BundleBuilder; use reth_stages_types::{StageCheckpoint, StageId}; use reth_trie::{IntermediateStateRootState, StateRoot as StateRootComputer, StateRootProgress}; use serde::{Deserialize, Serialize}; @@ -117,7 +118,7 @@ pub fn init_genesis(factory: ProviderFactory) -> Result(tx, &static_file_provider, chain.clone())?; - insert_genesis_state::(tx, alloc.len(), alloc.iter())?; + insert_genesis_state::(tx, alloc.iter())?; // insert sync stage for stage in StageId::ALL { @@ -133,28 +134,24 @@ pub fn init_genesis(factory: ProviderFactory) -> Result( tx: &::TXMut, - capacity: usize, alloc: impl Iterator, ) -> ProviderResult<()> { - insert_state::(tx, capacity, alloc, 0) + insert_state::(tx, alloc, 0) } /// Inserts state at given block into database. pub fn insert_state<'a, 'b, DB: Database>( tx: &::TXMut, - capacity: usize, alloc: impl Iterator, block: u64, ) -> ProviderResult<()> { - let mut state_init: BundleStateInit = HashMap::with_capacity(capacity); - let mut reverts_init = HashMap::with_capacity(capacity); - let mut contracts: HashMap = HashMap::with_capacity(capacity); + let mut bundle_builder = BundleBuilder::default(); for (address, account) in alloc { let bytecode_hash = if let Some(code) = &account.code { let bytecode = Bytecode::new_raw(code.clone()); let hash = bytecode.hash_slow(); - contracts.insert(hash, bytecode); + bundle_builder = bundle_builder.contract(hash, bytecode.into()); Some(hash) } else { None @@ -168,40 +165,31 @@ pub fn insert_state<'a, 'b, DB: Database>( m.iter() .map(|(key, value)| { let value = U256::from_be_bytes(value.0); - (*key, (U256::ZERO, value)) + ((*key).into(), (U256::ZERO, value)) }) .collect::>() }) .unwrap_or_default(); - reverts_init.insert( - *address, - (Some(None), storage.keys().map(|k| StorageEntry::new(*k, U256::ZERO)).collect()), - ); - - state_init.insert( - *address, - ( - None, - Some(Account { + bundle_builder = bundle_builder + .revert_storage( + block, + *address, + storage.keys().map(|k| (*k, U256::ZERO)).collect::>(), + ) + .state_present_account_info( + *address, + into_revm_acc(Account { nonce: account.nonce.unwrap_or_default(), balance: account.balance, bytecode_hash, }), - storage, - ), - ); + ) + .state_storage(*address, storage); } - let all_reverts_init: RevertsInit = HashMap::from([(block, reverts_init)]); - let execution_outcome = ExecutionOutcome::new_init( - state_init, - all_reverts_init, - contracts.into_iter().collect(), - Receipts::default(), - block, - Vec::new(), - ); + let execution_outcome = + ExecutionOutcome::new(bundle_builder.build(), Receipts::default(), block, Vec::new()); execution_outcome.write_to_storage(tx, None, OriginalValuesKnown::Yes)?; @@ -438,7 +426,6 @@ fn dump_state( let tx = provider_rw.deref_mut().tx_mut(); insert_state::( tx, - accounts.len(), accounts.iter().map(|(address, account)| (address, account)), block, )?; diff --git a/crates/storage/provider/src/bundle_state/mod.rs b/crates/storage/provider/src/bundle_state/mod.rs index eaf3dab43ee..08aa1338e54 100644 --- a/crates/storage/provider/src/bundle_state/mod.rs +++ b/crates/storage/provider/src/bundle_state/mod.rs @@ -6,7 +6,7 @@ mod hashed_state_changes; mod state_changes; mod state_reverts; -pub use execution_outcome::{AccountRevertInit, BundleStateInit, OriginalValuesKnown, RevertsInit}; +pub use execution_outcome::{ExecutionOutcome, OriginalValuesKnown}; pub use hashed_state_changes::HashedStateChanges; pub use state_changes::StateChanges; pub use state_reverts::{StateReverts, StorageRevertsIter}; diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 32b666c5018..ec8f1aa995c 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,5 +1,5 @@ use crate::{ - bundle_state::{BundleStateInit, HashedStateChanges, RevertsInit}, + bundle_state::{ExecutionOutcome, HashedStateChanges}, providers::{database::metrics, static_file::StaticFileWriter, StaticFileProvider}, to_range, traits::{ @@ -29,7 +29,7 @@ use reth_db_api::{ DatabaseError, }; use reth_evm::ConfigureEvmEnv; -use reth_execution_types::{Chain, ExecutionOutcome}; +use reth_execution_types::Chain; use reth_network_p2p::headers::downloader::SyncTarget; use reth_primitives::{ keccak256, Account, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, @@ -46,7 +46,7 @@ use reth_trie::{ updates::TrieUpdates, HashedPostState, Nibbles, StateRoot, }; -use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; +use revm::{db::states::BundleBuilder, primitives::{BlockEnv, CfgEnvWithHandlerCfg}}; use std::{ cmp::Ordering, collections::{hash_map, BTreeMap, BTreeSet, HashMap, HashSet}, @@ -628,13 +628,14 @@ impl DatabaseProvider { let storage_changeset = self.get_or_take::(storage_range)?; - let account_changeset = self.get_or_take::(range)?; + let account_changeset = + self.get_or_take::(range.clone())?; // iterate previous value and get plain state value to create changeset // Double option around Account represent if Account state is know (first option) and // account is removed (Second Option) - let mut state: BundleStateInit = HashMap::new(); + let mut bundle_builder = BundleBuilder::new(range); // This is not working for blocks that are not at tip. as plain state is not the last // state of end range. We should rename the functions or add support to access @@ -643,39 +644,49 @@ impl DatabaseProvider { let mut plain_accounts_cursor = self.tx.cursor_write::()?; let mut plain_storage_cursor = self.tx.cursor_dup_write::()?; - let mut reverts: RevertsInit = HashMap::new(); - // add account changeset changes for (block_number, account_before) in account_changeset.into_iter().rev() { let AccountBeforeTx { info: old_info, address } = account_before; - match state.entry(address) { - hash_map::Entry::Vacant(entry) => { - let new_info = plain_accounts_cursor.seek_exact(address)?.map(|kv| kv.1); - entry.insert((old_info, new_info, HashMap::new())); - } - hash_map::Entry::Occupied(mut entry) => { - // overwrite old account state. - entry.get_mut().0 = old_info; + if !bundle_builder.get_states().contains(&address) { + let new_info = plain_accounts_cursor.seek_exact(address)?.map(|kv| kv.1); + if let Some(new_info) = new_info { + bundle_builder = + bundle_builder.state_present_account_info(address, into_revm_acc(new_info)); } } + if let Some(old_info) = old_info { + bundle_builder = + bundle_builder.state_original_account_info(address, into_revm_acc(old_info)); + } + // insert old info into reverts. - reverts.entry(block_number).or_default().entry(address).or_default().0 = Some(old_info); + bundle_builder = bundle_builder.revert_account_info( + block_number, + address, + Some(old_info.map(into_revm_acc)), + ); } // add storage changeset changes for (block_and_address, old_storage) in storage_changeset.into_iter().rev() { let BlockNumberAddress((block_number, address)) = block_and_address; // get account state or insert from plain state. - let account_state = match state.entry(address) { - hash_map::Entry::Vacant(entry) => { - let present_info = plain_accounts_cursor.seek_exact(address)?.map(|kv| kv.1); - entry.insert((present_info, present_info, HashMap::new())) + if !bundle_builder.get_states().contains(&address) { + let present_info = plain_accounts_cursor.seek_exact(address)?.map(|kv| kv.1); + if let Some(present_info) = present_info { + bundle_builder = bundle_builder + .state_original_account_info(address, into_revm_acc(present_info)) + .state_present_account_info(address, into_revm_acc(present_info)); } + } + + let account_state = match bundle_builder.get_state_storage_mut().entry(address) { hash_map::Entry::Occupied(entry) => entry.into_mut(), + hash_map::Entry::Vacant(entry) => entry.insert(HashMap::new()), }; // match storage. - match account_state.2.entry(old_storage.key) { + match account_state.entry(old_storage.key.into()) { hash_map::Entry::Vacant(entry) => { let new_storage = plain_storage_cursor .seek_by_key_subkey(address, old_storage.key)? @@ -688,44 +699,46 @@ impl DatabaseProvider { } }; - reverts - .entry(block_number) - .or_default() - .entry(address) + bundle_builder + .get_revert_storage_mut() + .entry((block_number, address)) .or_default() - .1 - .push(old_storage); + .push((old_storage.key.into(), old_storage.value)); } + let bundle_state = bundle_builder.build(); + if TAKE { // iterate over local plain state remove all account and all storages. - for (address, (old_account, new_account, storage)) in &state { + + for (address, bundle_account) in &bundle_state.state { // revert account if needed. - if old_account != new_account { + if bundle_account.is_info_changed() { let existing_entry = plain_accounts_cursor.seek_exact(*address)?; - if let Some(account) = old_account { - plain_accounts_cursor.upsert(*address, *account)?; + if let Some(account) = &bundle_account.original_info { + plain_accounts_cursor.upsert(*address, into_reth_acc(account.clone()))?; } else if existing_entry.is_some() { plain_accounts_cursor.delete_current()?; } } // revert storages - for (storage_key, (old_storage_value, _new_storage_value)) in storage { + for (storage_key, storage_slot) in &bundle_account.storage { + let storage_key: B256 = (*storage_key).into(); let storage_entry = - StorageEntry { key: *storage_key, value: *old_storage_value }; + StorageEntry { key: storage_key, value: storage_slot.original_value() }; // delete previous value // TODO: This does not use dupsort features if plain_storage_cursor - .seek_by_key_subkey(*address, *storage_key)? - .filter(|s| s.key == *storage_key) + .seek_by_key_subkey(*address, storage_key)? + .filter(|s| s.key == storage_key) .is_some() { plain_storage_cursor.delete_current()? } // insert value if needed - if *old_storage_value != U256::ZERO { + if storage_slot.original_value() != U256::ZERO { plain_storage_cursor.upsert(*address, storage_entry)?; } } @@ -749,11 +762,9 @@ impl DatabaseProvider { receipts.push(block_receipts); } - Ok(ExecutionOutcome::new_init( - state, - reverts, - Vec::new(), - receipts.into(), + Ok(ExecutionOutcome::new( + bundle_state, + reth_primitives::Receipts::from(receipts), start_block_number, Vec::new(), ))