From c11348fb8c3d7191b960b656d342941b28b8d6af Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:13:46 +0000 Subject: [PATCH 01/20] fix(provider): require changeset cache for historical state providers Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db016-ea05-74e4-a085-27f4173a1f9f Co-authored-by: Amp --- crates/engine/tree/src/tree/mod.rs | 133 +++++++++- .../tree/src/tree/payload_processor/mod.rs | 60 ++++- .../src/tree/payload_processor/prewarm.rs | 28 ++- .../engine/tree/src/tree/payload_validator.rs | 27 +- crates/ethereum/node/src/node.rs | 56 ++++- crates/node/builder/src/rpc.rs | 16 ++ .../src/providers/database/provider.rs | 11 +- .../src/providers/state/historical.rs | 231 ++++++++++++------ .../storage/provider/src/test_utils/mock.rs | 21 +- 9 files changed, 475 insertions(+), 108 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index d0bd2141986..49cb7df6237 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -29,11 +29,13 @@ use reth_primitives_traits::{ FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, }; use reth_provider::{ - BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader, - DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader, - StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, - StorageSettingsCache, TransactionVariant, + BlockExecutionOutput, BlockExecutionResult, BlockHashReader, BlockNumReader, BlockReader, + ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, HistoricalStateProvider, + LatestStateProvider, NodePrimitivesProvider, ProviderError, PruneCheckpointReader, + RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateProviderFactory, + StateReader, StorageChangeSetReader, StorageSettingsCache, TransactionVariant, }; +use reth_prune::PruneSegment; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; use reth_tasks::{spawn_os_thread, utils::increase_thread_priority}; @@ -103,6 +105,8 @@ pub struct StateProviderBuilder { historical: B256, /// The blocks that form the chain from historical to target and are in memory. overlay: Option>>, + /// Changeset cache handle for building historical providers. + changeset_cache: ChangesetCache, } impl StateProviderBuilder { @@ -112,18 +116,67 @@ impl StateProviderBuilder { provider_factory: P, historical: B256, overlay: Option>>, + changeset_cache: ChangesetCache, ) -> Self { - Self { provider_factory, historical, overlay } + Self { provider_factory, historical, overlay, changeset_cache } } } impl StateProviderBuilder where - P: BlockReader + StateProviderFactory + StateReader + Clone, + P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + 'static + + NodePrimitivesProvider, { + fn build_database_state_provider(&self) -> ProviderResult { + let provider = self.provider_factory.database_provider_ro()?; + let block_number = provider + .block_number(self.historical)? + .ok_or(ProviderError::BlockHashNotFound(self.historical))?; + + if block_number == provider.best_block_number().unwrap_or_default() && + block_number == provider.last_block_number().unwrap_or_default() + { + return Ok(Box::new(LatestStateProvider::new(provider))) + } + + let account_history_prune_checkpoint = + provider.get_prune_checkpoint(PruneSegment::AccountHistory)?; + let storage_history_prune_checkpoint = + provider.get_prune_checkpoint(PruneSegment::StorageHistory)?; + + let mut state_provider = + HistoricalStateProvider::new(provider, block_number + 1, self.changeset_cache.clone()); + + if let Some(prune_checkpoint_block_number) = + account_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) + { + state_provider = state_provider.with_lowest_available_account_history_block_number( + prune_checkpoint_block_number + 1, + ); + } + if let Some(prune_checkpoint_block_number) = + storage_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) + { + state_provider = state_provider.with_lowest_available_storage_history_block_number( + prune_checkpoint_block_number + 1, + ); + } + + Ok(Box::new(state_provider)) + } + /// Creates a new state provider from this builder. pub fn build(&self) -> ProviderResult { - let mut provider = self.provider_factory.state_by_block_hash(self.historical)?; + let mut provider = self.build_database_state_provider()?; if let Some(overlay) = self.overlay.clone() { provider = Box::new(MemoryOverlayStateProvider::new(provider, overlay)) } @@ -351,11 +404,16 @@ where + HashedPostStateProvider + Clone + 'static, - P::Provider: BlockReader + P::Provider: BlockHashReader + + BlockNumReader + + BlockReader + StageCheckpointReader + + PruneCheckpointReader + ChangeSetReader + + RocksDBProviderFactory + StorageChangeSetReader - + StorageSettingsCache, + + StorageSettingsCache + + NodePrimitivesProvider, C: ConfigureEvm + 'static, T: PayloadTypes>, V: EngineValidator + WaitForCaches, @@ -2830,7 +2888,19 @@ where fn insert_payload( &mut self, payload: T::ExecutionData, - ) -> Result> { + ) -> Result> + where + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider + + 'static, + { self.insert_block_or_payload( payload.block_with_parent(), payload, @@ -2842,7 +2912,19 @@ where fn insert_block( &mut self, block: SealedBlock, - ) -> Result> { + ) -> Result> + where + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider + + 'static, + { self.insert_block_or_payload( block.block_with_parent(), block, @@ -2882,6 +2964,16 @@ where ) -> Result where Err: From>, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider + + 'static, { let block_insert_start = Instant::now(); let block_num_hash = block_id.block; @@ -3271,7 +3363,16 @@ where hash: B256, ) -> ProviderResult>> where - P: BlockReader + StateProviderFactory + StateReader + Clone, + P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider, { if let Some((historical, blocks)) = self.state.tree_state.blocks_by_hash(hash) { debug!(target: "engine::tree", %hash, %historical, "found canonical state for block in memory, creating provider builder"); @@ -3280,6 +3381,7 @@ where self.provider.clone(), historical, Some(blocks), + self.changeset_cache.clone(), ))) } @@ -3288,7 +3390,12 @@ where debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); // For persisted blocks, we create a builder that will fetch state directly from the // database - return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None))) + return Ok(Some(StateProviderBuilder::new( + self.provider.clone(), + hash, + None, + self.changeset_cache.clone(), + ))) } debug!(target: "engine::tree", %hash, "no canonical state found for block"); diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index bc274525a81..e7cacf31c43 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -22,7 +22,10 @@ use reth_evm::{ }; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ - BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProviderFactory, StateReader, + BlockExecutionOutput, BlockHashReader, BlockNumReader, BlockReader, ChangeSetReader, + DatabaseProviderFactory, DatabaseProviderROFactory, NodePrimitivesProvider, + PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, + StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::BundleState; use reth_tasks::{utils::increase_thread_priority, ForEachOrdered, Runtime}; @@ -244,7 +247,21 @@ where bal: Option>, ) -> IteratorPayloadHandle where - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + + BlockReader + + StateProviderFactory + + StateReader + + Clone + + 'static, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider, F: DatabaseProviderROFactory + Clone + Send @@ -294,7 +311,21 @@ where bal: Option>, ) -> IteratorPayloadHandle where - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + + BlockReader + + StateProviderFactory + + StateReader + + Clone + + 'static, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider, { let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions, env.transaction_count); @@ -467,7 +498,21 @@ where bal: Option>, ) -> CacheTaskHandle where - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + + BlockReader + + StateProviderFactory + + StateReader + + Clone + + 'static, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider, { let skip_prewarm = self.disable_transaction_prewarming || env.transaction_count < SMALL_BLOCK_TX_THRESHOLD; @@ -1236,7 +1281,12 @@ mod tests { Vec::, core::convert::Infallible>>::new(), std::convert::identity, ), - StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None), + StateProviderBuilder::new( + provider_factory.clone(), + genesis_hash, + None, + ChangesetCache::new(), + ), OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()), &TreeConfig::default(), None, // No BAL for test diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 1f9a88c2cde..62e79d1ddf5 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -28,8 +28,10 @@ use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, use reth_metrics::Metrics; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ - AccountReader, BlockExecutionOutput, BlockReader, StateProvider, StateProviderFactory, - StateReader, + AccountReader, BlockExecutionOutput, BlockHashReader, BlockNumReader, BlockReader, + ChangeSetReader, DatabaseProviderFactory, NodePrimitivesProvider, PruneCheckpointReader, + RocksDBProviderFactory, StageCheckpointReader, StateProvider, StateProviderFactory, + StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::{database::StateProviderDatabase, state::EvmState}; use reth_tasks::{pool::WorkerPool, Runtime}; @@ -81,7 +83,16 @@ where impl PrewarmCacheTask where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider, Evm: ConfigureEvm + 'static, { /// Initializes the task with the given transactions pending execution @@ -546,7 +557,16 @@ type PrewarmEvmState = impl PrewarmContext where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P::Provider: BlockHashReader + + BlockNumReader + + ChangeSetReader + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache + + NodePrimitivesProvider, Evm: ConfigureEvm + 'static, { /// Creates a per-thread EVM for prewarming. diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 9ba884c8be7..a0c81c1fc10 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -77,10 +77,11 @@ use reth_primitives_traits::{ RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, }; use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, - ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider, - StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, + providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockHashReader, BlockNumReader, + BlockReader, ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, + HashedPostStateProvider, NodePrimitivesProvider, ProviderError, PruneCheckpointReader, + RocksDBProviderFactory, StageCheckpointReader, StateProvider, StateProviderFactory, + StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot}; @@ -203,12 +204,15 @@ where N: NodePrimitives, P: DatabaseProviderFactory< Provider: BlockReader + + BlockHashReader + StageCheckpointReader + PruneCheckpointReader + ChangeSetReader + + RocksDBProviderFactory + StorageChangeSetReader + BlockNumReader - + StorageSettingsCache, + + StorageSettingsCache + + NodePrimitivesProvider, > + BlockReader
+ ChangeSetReader + BlockNumReader @@ -1506,6 +1510,7 @@ where self.provider.clone(), historical, Some(blocks), + self.changeset_cache.clone(), ))) } @@ -1514,7 +1519,12 @@ where debug!(target: "engine::tree::payload_validator", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); // For persisted blocks, we create a builder that will fetch state directly from the // database - return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None))) + return Ok(Some(StateProviderBuilder::new( + self.provider.clone(), + hash, + None, + self.changeset_cache.clone(), + ))) } debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); @@ -1957,12 +1967,15 @@ impl EngineValidator for BasicEngineValidator + BlockReader
+ StateProviderFactory + StateReader diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index eca9d36fbdc..66590588595 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -34,7 +34,12 @@ use reth_node_builder::{ BuilderContext, DebugNode, Node, NodeAdapter, }; use reth_payload_primitives::PayloadTypes; -use reth_provider::{providers::ProviderFactoryBuilder, EthStorage}; +use reth_provider::{ + providers::ProviderFactoryBuilder, BlockHashReader, BlockNumReader, BlockReader, + ChangeSetReader, DatabaseProviderFactory, EthStorage, NodePrimitivesProvider, + PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StorageChangeSetReader, + StorageSettingsCache, +}; use reth_rpc::{ eth::core::{EthApiFor, EthRpcConverterFor}, TestingApi, ValidationApi, @@ -311,6 +316,17 @@ where PVB: Send, EB: EngineApiBuilder, EVB: EngineValidatorBuilder, + N::Provider: DatabaseProviderFactory, + ::Provider: BlockHashReader + + BlockNumReader + + BlockReader + + ChangeSetReader + + NodePrimitivesProvider + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, RpcMiddleware: RethRpcMiddleware, @@ -385,6 +401,17 @@ where PVB: PayloadValidatorBuilder, EB: EngineApiBuilder, EVB: EngineValidatorBuilder, + N::Provider: DatabaseProviderFactory, + ::Provider: BlockHashReader + + BlockNumReader + + BlockReader + + ChangeSetReader + + NodePrimitivesProvider + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, RpcMiddleware: RethRpcMiddleware, @@ -427,6 +454,17 @@ where impl Node for EthereumNode where N: FullNodeTypes, + N::Provider: DatabaseProviderFactory, + ::Provider: BlockHashReader + + BlockNumReader + + BlockReader + + ChangeSetReader + + NodePrimitivesProvider + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache, { type ComponentsBuilder = ComponentsBuilder< N, @@ -449,7 +487,21 @@ where } } -impl> DebugNode for EthereumNode { +impl DebugNode for EthereumNode +where + N: FullNodeComponents, + N::Provider: DatabaseProviderFactory, + ::Provider: BlockHashReader + + BlockNumReader + + BlockReader + + ChangeSetReader + + NodePrimitivesProvider + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache, +{ type RpcBlock = alloy_rpc_types_eth::Block; fn rpc_to_primitive_block(rpc_block: Self::RpcBlock) -> reth_ethereum_primitives::Block { diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index a10c40841f7..31a10ece291 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -32,6 +32,11 @@ use reth_node_core::{ version::{version_metadata, CLIENT_CODE}, }; use reth_payload_builder::{PayloadBuilderHandle, PayloadStore}; +use reth_provider::{ + BlockHashReader, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, + NodePrimitivesProvider, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, + StorageChangeSetReader, StorageSettingsCache, +}; use reth_rpc::{ eth::{core::EthRpcConverterFor, DevSigner, EthApiTypes, FullEthApiServer}, AdminApi, @@ -1452,6 +1457,17 @@ where ::Payload, Block = BlockTy, > + Clone, + Node::Provider: DatabaseProviderFactory, + ::Provider: BlockHashReader + + BlockNumReader + + BlockReader + + ChangeSetReader + + NodePrimitivesProvider + + PruneCheckpointReader + + RocksDBProviderFactory + + StageCheckpointReader + + StorageChangeSetReader + + StorageSettingsCache, { type EngineValidator = BasicEngineValidator; diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 5991beaf73f..bb65a369f16 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -316,8 +316,8 @@ impl DatabaseProvider { let storage_history_prune_checkpoint = self.get_prune_checkpoint(PruneSegment::StorageHistory)?; - let mut state_provider = HistoricalStateProviderRef::new(self, block_number); - + let mut state_provider = + HistoricalStateProviderRef::new(self, block_number, self.changeset_cache.clone()); // If we pruned account or storage history, we can't return state on every historical block. // Instead, we should cap it at the latest prune checkpoint for corresponding prune segment. if let Some(prune_checkpoint_block_number) = @@ -933,8 +933,9 @@ impl TryIntoHistoricalStateProvider for Databa self.get_prune_checkpoint(PruneSegment::AccountHistory)?; let storage_history_prune_checkpoint = self.get_prune_checkpoint(PruneSegment::StorageHistory)?; + let changeset_cache = self.changeset_cache.clone(); - let mut state_provider = HistoricalStateProvider::new(self, block_number); + let mut state_provider = HistoricalStateProvider::new(self, block_number, changeset_cache); // If we pruned account or storage history, we can't return state on every historical block. // Instead, we should cap it at the latest prune checkpoint for corresponding prune segment. @@ -4970,7 +4971,9 @@ mod tests { assert_eq!(account_cs[0].address, address); let historical_value = - HistoricalStateProviderRef::new(&*provider_rw, 0).storage(address, slot_key).unwrap(); + HistoricalStateProviderRef::new(&*provider_rw, 0, ChangesetCache::new()) + .storage(address, slot_key) + .unwrap(); assert_eq!(historical_value, None); } diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index fddd4ac3cc9..e878a861657 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -12,9 +12,10 @@ use reth_db_api::{ BlockNumberList, }; use reth_primitives_traits::{Account, Bytecode}; +use reth_stages_types::StageId; use reth_storage_api::{ - BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StateProofProvider, - StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, + BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StageCheckpointReader, + StateProofProvider, StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ @@ -28,7 +29,7 @@ use reth_trie::{ TrieInput, TrieInputSorted, }; use reth_trie_db::{ - hashed_storage_from_reverts_with_provider, DatabaseProof, DatabaseStateRoot, + hashed_storage_from_reverts_with_provider, ChangesetCache, DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, }; @@ -123,6 +124,8 @@ impl HistoryInfo { pub struct HistoricalStateProviderRef<'b, Provider> { /// Database provider provider: &'b Provider, + /// Changeset cache handle for retrieving trie changesets. + changeset_cache: ChangesetCache, /// Block number is main index for the history state of accounts and storages. block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. @@ -133,8 +136,17 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block HistoricalStateProviderRef<'b, Provider> { /// Create new `StateProvider` for historical block number - pub fn new(provider: &'b Provider, block_number: BlockNumber) -> Self { - Self { provider, block_number, lowest_available_blocks: Default::default() } + pub fn new( + provider: &'b Provider, + block_number: BlockNumber, + changeset_cache: ChangesetCache, + ) -> Self { + Self { + provider, + changeset_cache, + block_number, + lowest_available_blocks: Default::default(), + } } /// Create new `StateProvider` for historical block number and lowest block numbers at which @@ -143,8 +155,9 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block provider: &'b Provider, block_number: BlockNumber, lowest_available_blocks: LowestAvailableBlocks, + changeset_cache: ChangesetCache, ) -> Self { - Self { provider, block_number, lowest_available_blocks } + Self { provider, changeset_cache, block_number, lowest_available_blocks } } /// Lookup an account in the `AccountsHistory` table using `EitherReader`. @@ -275,6 +288,37 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block reth_trie_db::from_reverts_auto(self.provider, self.block_number..) } + /// Retrieve cached trie reverts and hashed state reverts for this history provider. + fn revert_input(&self) -> ProviderResult + where + Provider: StageCheckpointReader + StorageSettingsCache, + { + let revert_state = self.revert_state()?; + let db_tip_block = self + .provider + .get_stage_checkpoint(StageId::Finish)? + .as_ref() + .map(|chk| chk.block_number) + .ok_or_else(|| ProviderError::InsufficientChangesets { + requested: self.block_number, + available: 0..=0, + })?; + + if self.block_number > db_tip_block { + return Ok(TrieInput::from_state(revert_state.into())) + } + + let trie_reverts = self + .changeset_cache + .get_or_compute_range(self.provider, self.block_number..=db_tip_block)?; + + if trie_reverts.is_empty() { + Ok(TrieInput::from_state(revert_state.into())) + } else { + Ok(TrieInput::from_blocks_sorted([(&revert_state, &trie_reverts)])) + } + } + /// Retrieve revert hashed storage for this history provider and target address. fn revert_storage(&self, address: Address) -> ProviderResult where @@ -378,22 +422,25 @@ impl< + ChangeSetReader + StorageChangeSetReader + BlockNumReader + + StageCheckpointReader + StorageSettingsCache, > StateRootProvider for HistoricalStateProviderRef<'_, Provider> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut revert_state = self.revert_state()?; - let hashed_state_sorted = hashed_state.into_sorted(); - revert_state.extend_ref_and_sort(&hashed_state_sorted); - Ok(>::overlay_root(self.tx(), &revert_state)?) + let mut input = self.revert_input()?; + input.append(hashed_state); + Ok(>::overlay_root_from_nodes( + self.tx(), + TrieInputSorted::from_unsorted(input), + )?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend(self.revert_state()?.into()); + input.prepend_self(self.revert_input()?); Ok(>::overlay_root_from_nodes( self.tx(), TrieInputSorted::from_unsorted(input), @@ -406,10 +453,12 @@ impl< hashed_state: HashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider, |A| { - let mut revert_state = self.revert_state()?; - let hashed_state_sorted = hashed_state.into_sorted(); - revert_state.extend_ref_and_sort(&hashed_state_sorted); - Ok(>::overlay_root_with_updates(self.tx(), &revert_state)?) + let mut input = self.revert_input()?; + input.append(hashed_state); + Ok(>::overlay_root_from_nodes_with_updates( + self.tx(), + TrieInputSorted::from_unsorted(input), + )?) }) } @@ -419,7 +468,7 @@ impl< ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend(self.revert_state()?.into()); + input.prepend_self(self.revert_input()?); Ok(>::overlay_root_from_nodes_with_updates( self.tx(), TrieInputSorted::from_unsorted(input), @@ -493,6 +542,7 @@ impl< + ChangeSetReader + StorageChangeSetReader + BlockNumReader + + StageCheckpointReader + StorageSettingsCache, > StateProofProvider for HistoricalStateProviderRef<'_, Provider> { @@ -505,7 +555,7 @@ impl< ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend(self.revert_state()?.into()); + input.prepend_self(self.revert_input()?); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) }) @@ -518,7 +568,7 @@ impl< ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend(self.revert_state()?.into()); + input.prepend_self(self.revert_input()?); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_multiproof(input, targets).map_err(ProviderError::from) }) @@ -532,7 +582,7 @@ impl< ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend(self.revert_state()?.into()); + input.prepend_self(self.revert_input()?); let nodes_sorted = input.nodes.into_sorted(); let state_sorted = input.state.into_sorted(); let witness = TrieWitness::new( @@ -572,6 +622,7 @@ impl< + BlockHashReader + ChangeSetReader + StorageChangeSetReader + + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider, @@ -602,6 +653,8 @@ impl BytecodeReader pub struct HistoricalStateProvider { /// Database provider. provider: Provider, + /// Changeset cache handle for retrieving trie changesets. + changeset_cache: ChangesetCache, /// State at the block number is the main indexer of the state. block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. @@ -612,8 +665,17 @@ impl { /// Create new `StateProvider` for historical block number - pub fn new(provider: Provider, block_number: BlockNumber) -> Self { - Self { provider, block_number, lowest_available_blocks: Default::default() } + pub fn new( + provider: Provider, + block_number: BlockNumber, + changeset_cache: ChangesetCache, + ) -> Self { + Self { + provider, + changeset_cache, + block_number, + lowest_available_blocks: Default::default(), + } } /// Set the lowest block number at which the account history is available. @@ -636,17 +698,18 @@ impl HistoricalStateProviderRef<'_, Provider> { + fn as_ref(&self) -> HistoricalStateProviderRef<'_, Provider> { HistoricalStateProviderRef::new_with_lowest_available_blocks( &self.provider, self.block_number, self.lowest_available_blocks, + self.changeset_cache.clone(), ) } } // Delegates all provider impls to [HistoricalStateProviderRef] -reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]); +reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]); /// Lowest blocks at which different parts of the state are available. /// They may be [Some] if pruning is enabled. @@ -779,9 +842,11 @@ mod tests { use reth_primitives_traits::{Account, StorageEntry}; use reth_storage_api::{ BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, - NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache, + NodePrimitivesProvider, StageCheckpointReader, StorageChangeSetReader, + StorageSettingsCache, }; use reth_storage_errors::provider::ProviderError; + use reth_trie_db::ChangesetCache; const ADDRESS: Address = address!("0x0000000000000000000000000000000000000001"); const HIGHER_ADDRESS: Address = address!("0x0000000000000000000000000000000000000005"); @@ -795,6 +860,7 @@ mod tests { + BlockNumReader + BlockHashReader + ChangeSetReader + + StageCheckpointReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory @@ -803,6 +869,32 @@ mod tests { assert_state_provider::>(); } + fn historical_state_provider_ref( + provider: &Provider, + block_number: u64, + ) -> HistoricalStateProviderRef<'_, Provider> + where + Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader, + { + HistoricalStateProviderRef::new(provider, block_number, ChangesetCache::new()) + } + + fn historical_state_provider_ref_with_lowest_available_blocks( + provider: &Provider, + block_number: u64, + lowest_available_blocks: LowestAvailableBlocks, + ) -> HistoricalStateProviderRef<'_, Provider> + where + Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader, + { + HistoricalStateProviderRef::new_with_lowest_available_blocks( + provider, + block_number, + lowest_available_blocks, + ChangesetCache::new(), + ) + } + #[test] fn history_provider_get_account() { let factory = create_test_provider_factory(); @@ -869,49 +961,46 @@ mod tests { let db = factory.provider().unwrap(); // run + assert!(matches!(historical_state_provider_ref(&db, 1).basic_account(&ADDRESS), Ok(None))); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1).basic_account(&ADDRESS), - Ok(None) - )); - assert!(matches!( - HistoricalStateProviderRef::new(&db, 2).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 2).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at3 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 3).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at3 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 4).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at7 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 7).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at7 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 9).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at10 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 10).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at10 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 11).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at15 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16).basic_account(&ADDRESS), + historical_state_provider_ref(&db, 16).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_plain )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1).basic_account(&HIGHER_ADDRESS), + historical_state_provider_ref(&db, 1).basic_account(&HIGHER_ADDRESS), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000).basic_account(&HIGHER_ADDRESS), + historical_state_provider_ref(&db, 1000).basic_account(&HIGHER_ADDRESS), Ok(Some(acc)) if acc == higher_acc_plain )); } @@ -970,43 +1059,43 @@ mod tests { // run assert!(matches!( - HistoricalStateProviderRef::new(&db, 0).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 0).storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 3).storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 4).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 7).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 9).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 10).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 11).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at15.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 16).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_plain.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1).storage(HIGHER_ADDRESS, STORAGE), + historical_state_provider_ref(&db, 1).storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), + historical_state_provider_ref(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == higher_entry_plain.value )); } @@ -1018,7 +1107,7 @@ mod tests { // provider block_number < lowest available block number, // i.e. state at provider block is pruned - let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( + let provider = historical_state_provider_ref_with_lowest_available_blocks( &db, 2, LowestAvailableBlocks { @@ -1037,7 +1126,7 @@ mod tests { // provider block_number == lowest available block number, // i.e. state at provider block is available - let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( + let provider = historical_state_provider_ref_with_lowest_available_blocks( &db, 2, LowestAvailableBlocks { @@ -1056,7 +1145,7 @@ mod tests { // provider block_number == lowest available block number, // i.e. state at provider block is available - let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( + let provider = historical_state_provider_ref_with_lowest_available_blocks( &db, 2, LowestAvailableBlocks { @@ -1143,43 +1232,43 @@ mod tests { let db = factory.provider().unwrap(); assert!(matches!( - HistoricalStateProviderRef::new(&db, 0).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 0).storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 3).storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 4).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 7).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 9).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 10).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 11).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at15.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 16).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_plain.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1).storage(HIGHER_ADDRESS, STORAGE), + historical_state_provider_ref(&db, 1).storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), + historical_state_provider_ref(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == higher_entry_plain.value )); } @@ -1283,43 +1372,43 @@ mod tests { let db = factory.provider().unwrap(); assert!(matches!( - HistoricalStateProviderRef::new(&db, 0).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 0).storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 3).storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 4).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(7) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 7).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(7) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 9).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(10) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 10).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(10) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 11).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(15) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16).storage(ADDRESS, STORAGE), + historical_state_provider_ref(&db, 16).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(100) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1).storage(HIGHER_ADDRESS, STORAGE), + historical_state_provider_ref(&db, 1).storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), + historical_state_provider_ref(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(1000) )); } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 72fa695a0a2..63314c9c4fd 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -1,9 +1,11 @@ use crate::{ + providers::RocksDBProvider, traits::{BlockSource, ReceiptProvider}, AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, HeaderProvider, - PruneCheckpointReader, ReceiptProviderIdExt, StateProvider, StateProviderBox, - StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, TransactionsProvider, + PruneCheckpointReader, ReceiptProviderIdExt, RocksDBProviderFactory, StateProvider, + StateProviderBox, StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, + TransactionsProvider, }; use alloy_consensus::{ constants::EMPTY_ROOT_HASH, @@ -929,6 +931,21 @@ impl StorageSettingsCache fn set_storage_settings_cache(&self, _settings: StorageSettings) {} } +impl RocksDBProviderFactory + for MockEthProvider +{ + fn rocksdb_provider(&self) -> RocksDBProvider { + RocksDBProvider::new(std::env::temp_dir().join(format!("reth-mock-rocksdb-{:p}", self))) + .expect("failed to create mock RocksDB provider") + } + + fn set_pending_rocksdb_batch(&self, _batch: rocksdb::WriteBatchWithTransaction) {} + + fn commit_pending_rocksdb_batches(&self) -> ProviderResult<()> { + Ok(()) + } +} + impl StateProviderFactory for MockEthProvider { From 31c0f96feb886dfb307861f2db5519065f741f26 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:39:01 +0000 Subject: [PATCH 02/20] test(node): tighten ethereum e2e helper provider bounds Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db037-4b3b-77c1-9832-42d24c58560e Co-authored-by: Amp --- crates/ethereum/node/tests/e2e/utils.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 6f413468dc6..1e8f9cfddbf 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -14,7 +14,9 @@ use reth_e2e_test_utils::{wallet::Wallet, NodeHelperType, TmpDB}; use reth_ethereum_primitives::TxType; use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; -use reth_provider::FullProvider; +use reth_provider::{ + DatabaseProviderFactory, FullProvider, NodePrimitivesProvider, RocksDBProviderFactory, +}; /// Helper function to create a new eth payload attributes pub(crate) const fn eth_payload_attributes(timestamp: u64) -> PayloadAttributes { @@ -50,6 +52,8 @@ pub(crate) async fn advance_with_random_transactions( ) -> eyre::Result<()> where Provider: FullProvider>, + ::Provider: + NodePrimitivesProvider + RocksDBProviderFactory, { let provider = ProviderBuilder::new().connect_http(node.rpc_url()); let signers = Wallet::new(1).with_chain_id(provider.get_chain_id().await?).wallet_gen(); From 58d2a3e815a0d4383c1b0dd478a26b95b78e4ef4 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:00:29 +0000 Subject: [PATCH 03/20] refactor(engine): simplify state provider builder cache wiring Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db037-4b3b-77c1-9832-42d24c58560e Co-authored-by: Amp --- crates/engine/tree/src/tree/mod.rs | 74 ++----------------- .../tree/src/tree/payload_processor/mod.rs | 7 +- .../engine/tree/src/tree/payload_validator.rs | 8 +- crates/node/builder/src/launch/engine.rs | 8 +- 4 files changed, 15 insertions(+), 82 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 49cb7df6237..b5f83cbcb94 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -30,12 +30,11 @@ use reth_primitives_traits::{ }; use reth_provider::{ BlockExecutionOutput, BlockExecutionResult, BlockHashReader, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, HistoricalStateProvider, - LatestStateProvider, NodePrimitivesProvider, ProviderError, PruneCheckpointReader, - RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateProviderFactory, - StateReader, StorageChangeSetReader, StorageSettingsCache, TransactionVariant, + ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, NodePrimitivesProvider, + ProviderError, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, + StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, + StorageSettingsCache, TransactionVariant, }; -use reth_prune::PruneSegment; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; use reth_tasks::{spawn_os_thread, utils::increase_thread_priority}; @@ -105,8 +104,6 @@ pub struct StateProviderBuilder { historical: B256, /// The blocks that form the chain from historical to target and are in memory. overlay: Option>>, - /// Changeset cache handle for building historical providers. - changeset_cache: ChangesetCache, } impl StateProviderBuilder { @@ -116,67 +113,18 @@ impl StateProviderBuilder { provider_factory: P, historical: B256, overlay: Option>>, - changeset_cache: ChangesetCache, ) -> Self { - Self { provider_factory, historical, overlay, changeset_cache } + Self { provider_factory, historical, overlay } } } impl StateProviderBuilder where - P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + 'static - + NodePrimitivesProvider, + P: BlockReader + StateProviderFactory + StateReader + Clone, { - fn build_database_state_provider(&self) -> ProviderResult { - let provider = self.provider_factory.database_provider_ro()?; - let block_number = provider - .block_number(self.historical)? - .ok_or(ProviderError::BlockHashNotFound(self.historical))?; - - if block_number == provider.best_block_number().unwrap_or_default() && - block_number == provider.last_block_number().unwrap_or_default() - { - return Ok(Box::new(LatestStateProvider::new(provider))) - } - - let account_history_prune_checkpoint = - provider.get_prune_checkpoint(PruneSegment::AccountHistory)?; - let storage_history_prune_checkpoint = - provider.get_prune_checkpoint(PruneSegment::StorageHistory)?; - - let mut state_provider = - HistoricalStateProvider::new(provider, block_number + 1, self.changeset_cache.clone()); - - if let Some(prune_checkpoint_block_number) = - account_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) - { - state_provider = state_provider.with_lowest_available_account_history_block_number( - prune_checkpoint_block_number + 1, - ); - } - if let Some(prune_checkpoint_block_number) = - storage_history_prune_checkpoint.and_then(|checkpoint| checkpoint.block_number) - { - state_provider = state_provider.with_lowest_available_storage_history_block_number( - prune_checkpoint_block_number + 1, - ); - } - - Ok(Box::new(state_provider)) - } - /// Creates a new state provider from this builder. pub fn build(&self) -> ProviderResult { - let mut provider = self.build_database_state_provider()?; + let mut provider = self.provider_factory.state_by_block_hash(self.historical)?; if let Some(overlay) = self.overlay.clone() { provider = Box::new(MemoryOverlayStateProvider::new(provider, overlay)) } @@ -3381,7 +3329,6 @@ where self.provider.clone(), historical, Some(blocks), - self.changeset_cache.clone(), ))) } @@ -3390,12 +3337,7 @@ where debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); // For persisted blocks, we create a builder that will fetch state directly from the // database - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - hash, - None, - self.changeset_cache.clone(), - ))) + return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None))) } debug!(target: "engine::tree", %hash, "no canonical state found for block"); diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index e7cacf31c43..1e9918d04ec 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -1281,12 +1281,7 @@ mod tests { Vec::, core::convert::Infallible>>::new(), std::convert::identity, ), - StateProviderBuilder::new( - provider_factory.clone(), - genesis_hash, - None, - ChangesetCache::new(), - ), + StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None), OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()), &TreeConfig::default(), None, // No BAL for test diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index a0c81c1fc10..e4f85dc6373 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -1510,7 +1510,6 @@ where self.provider.clone(), historical, Some(blocks), - self.changeset_cache.clone(), ))) } @@ -1519,12 +1518,7 @@ where debug!(target: "engine::tree::payload_validator", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); // For persisted blocks, we create a builder that will fetch state directly from the // database - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - hash, - None, - self.changeset_cache.clone(), - ))) + return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None))) } debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 6a33f20abfe..0b568564930 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -205,10 +205,12 @@ impl EngineNodeLauncher { ctx.blockchain_db().clone(), ctx.components().evm_config().clone(), || async { - // Create a separate cache for reorg validator (not shared with main engine) - let reorg_cache = ChangesetCache::new(); validator_builder - .build_tree_validator(&add_ons_ctx, engine_tree_config.clone(), reorg_cache) + .build_tree_validator( + &add_ons_ctx, + engine_tree_config.clone(), + changeset_cache.clone(), + ) .await }, node_config.debug.reorg_frequency, From 5bd1a9932f4cc8d3023d947d903f4d0ba90e28b0 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:22:06 +0000 Subject: [PATCH 04/20] fix(provider): restore minimal historical cache plumbing Restore the historical provider behavior to match origin/main while keeping the ChangesetCache constructor plumbing and shared reorg-validator cache wiring. Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db05b-f81c-754e-9229-81b57d6d32f5 Co-authored-by: Amp --- crates/engine/tree/src/tree/mod.rs | 63 +----- .../tree/src/tree/payload_processor/mod.rs | 53 +---- .../src/tree/payload_processor/prewarm.rs | 28 +-- .../engine/tree/src/tree/payload_validator.rs | 19 +- crates/ethereum/node/src/node.rs | 56 +---- crates/ethereum/node/tests/e2e/utils.rs | 6 +- crates/node/builder/src/rpc.rs | 16 -- .../src/providers/state/historical.rs | 211 +++++++----------- .../storage/provider/src/test_utils/mock.rs | 21 +- 9 files changed, 106 insertions(+), 367 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index b5f83cbcb94..d0bd2141986 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -29,9 +29,8 @@ use reth_primitives_traits::{ FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, }; use reth_provider::{ - BlockExecutionOutput, BlockExecutionResult, BlockHashReader, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, NodePrimitivesProvider, - ProviderError, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, + BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader, + DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, TransactionVariant, }; @@ -352,16 +351,11 @@ where + HashedPostStateProvider + Clone + 'static, - P::Provider: BlockHashReader - + BlockNumReader - + BlockReader + P::Provider: BlockReader + StageCheckpointReader - + PruneCheckpointReader + ChangeSetReader - + RocksDBProviderFactory + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider, + + StorageSettingsCache, C: ConfigureEvm + 'static, T: PayloadTypes>, V: EngineValidator + WaitForCaches, @@ -2836,19 +2830,7 @@ where fn insert_payload( &mut self, payload: T::ExecutionData, - ) -> Result> - where - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider - + 'static, - { + ) -> Result> { self.insert_block_or_payload( payload.block_with_parent(), payload, @@ -2860,19 +2842,7 @@ where fn insert_block( &mut self, block: SealedBlock, - ) -> Result> - where - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider - + 'static, - { + ) -> Result> { self.insert_block_or_payload( block.block_with_parent(), block, @@ -2912,16 +2882,6 @@ where ) -> Result where Err: From>, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider - + 'static, { let block_insert_start = Instant::now(); let block_num_hash = block_id.block; @@ -3311,16 +3271,7 @@ where hash: B256, ) -> ProviderResult>> where - P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider, + P: BlockReader + StateProviderFactory + StateReader + Clone, { if let Some((historical, blocks)) = self.state.tree_state.blocks_by_hash(hash) { debug!(target: "engine::tree", %hash, %historical, "found canonical state for block in memory, creating provider builder"); diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 1e9918d04ec..bc274525a81 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -22,10 +22,7 @@ use reth_evm::{ }; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ - BlockExecutionOutput, BlockHashReader, BlockNumReader, BlockReader, ChangeSetReader, - DatabaseProviderFactory, DatabaseProviderROFactory, NodePrimitivesProvider, - PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, - StateReader, StorageChangeSetReader, StorageSettingsCache, + BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProviderFactory, StateReader, }; use reth_revm::db::BundleState; use reth_tasks::{utils::increase_thread_priority, ForEachOrdered, Runtime}; @@ -247,21 +244,7 @@ where bal: Option>, ) -> IteratorPayloadHandle where - P: DatabaseProviderFactory - + BlockReader - + StateProviderFactory - + StateReader - + Clone - + 'static, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, F: DatabaseProviderROFactory + Clone + Send @@ -311,21 +294,7 @@ where bal: Option>, ) -> IteratorPayloadHandle where - P: DatabaseProviderFactory - + BlockReader - + StateProviderFactory - + StateReader - + Clone - + 'static, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, { let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions, env.transaction_count); @@ -498,21 +467,7 @@ where bal: Option>, ) -> CacheTaskHandle where - P: DatabaseProviderFactory - + BlockReader - + StateProviderFactory - + StateReader - + Clone - + 'static, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, { let skip_prewarm = self.disable_transaction_prewarming || env.transaction_count < SMALL_BLOCK_TX_THRESHOLD; diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 62e79d1ddf5..1f9a88c2cde 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -28,10 +28,8 @@ use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, use reth_metrics::Metrics; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ - AccountReader, BlockExecutionOutput, BlockHashReader, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, NodePrimitivesProvider, PruneCheckpointReader, - RocksDBProviderFactory, StageCheckpointReader, StateProvider, StateProviderFactory, - StateReader, StorageChangeSetReader, StorageSettingsCache, + AccountReader, BlockExecutionOutput, BlockReader, StateProvider, StateProviderFactory, + StateReader, }; use reth_revm::{database::StateProviderDatabase, state::EvmState}; use reth_tasks::{pool::WorkerPool, Runtime}; @@ -83,16 +81,7 @@ where impl PrewarmCacheTask where N: NodePrimitives, - P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone + 'static, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, Evm: ConfigureEvm + 'static, { /// Initializes the task with the given transactions pending execution @@ -557,16 +546,7 @@ type PrewarmEvmState = impl PrewarmContext where N: NodePrimitives, - P: DatabaseProviderFactory + BlockReader + StateProviderFactory + StateReader + Clone + 'static, - P::Provider: BlockHashReader - + BlockNumReader - + ChangeSetReader - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache - + NodePrimitivesProvider, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, Evm: ConfigureEvm + 'static, { /// Creates a per-thread EVM for prewarming. diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index e4f85dc6373..9ba884c8be7 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -77,11 +77,10 @@ use reth_primitives_traits::{ RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, }; use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockHashReader, BlockNumReader, - BlockReader, ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, - HashedPostStateProvider, NodePrimitivesProvider, ProviderError, PruneCheckpointReader, - RocksDBProviderFactory, StageCheckpointReader, StateProvider, StateProviderFactory, - StateReader, StorageChangeSetReader, StorageSettingsCache, + providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader, + ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, + ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider, + StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot}; @@ -204,15 +203,12 @@ where N: NodePrimitives, P: DatabaseProviderFactory< Provider: BlockReader - + BlockHashReader + StageCheckpointReader + PruneCheckpointReader + ChangeSetReader - + RocksDBProviderFactory + StorageChangeSetReader + BlockNumReader - + StorageSettingsCache - + NodePrimitivesProvider, + + StorageSettingsCache, > + BlockReader
+ ChangeSetReader + BlockNumReader @@ -1961,15 +1957,12 @@ impl EngineValidator for BasicEngineValidator + BlockReader
+ StateProviderFactory + StateReader diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 66590588595..eca9d36fbdc 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -34,12 +34,7 @@ use reth_node_builder::{ BuilderContext, DebugNode, Node, NodeAdapter, }; use reth_payload_primitives::PayloadTypes; -use reth_provider::{ - providers::ProviderFactoryBuilder, BlockHashReader, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, EthStorage, NodePrimitivesProvider, - PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StorageChangeSetReader, - StorageSettingsCache, -}; +use reth_provider::{providers::ProviderFactoryBuilder, EthStorage}; use reth_rpc::{ eth::core::{EthApiFor, EthRpcConverterFor}, TestingApi, ValidationApi, @@ -316,17 +311,6 @@ where PVB: Send, EB: EngineApiBuilder, EVB: EngineValidatorBuilder, - N::Provider: DatabaseProviderFactory, - ::Provider: BlockHashReader - + BlockNumReader - + BlockReader - + ChangeSetReader - + NodePrimitivesProvider - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, RpcMiddleware: RethRpcMiddleware, @@ -401,17 +385,6 @@ where PVB: PayloadValidatorBuilder, EB: EngineApiBuilder, EVB: EngineValidatorBuilder, - N::Provider: DatabaseProviderFactory, - ::Provider: BlockHashReader - + BlockNumReader - + BlockReader - + ChangeSetReader - + NodePrimitivesProvider - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, RpcMiddleware: RethRpcMiddleware, @@ -454,17 +427,6 @@ where impl Node for EthereumNode where N: FullNodeTypes, - N::Provider: DatabaseProviderFactory, - ::Provider: BlockHashReader - + BlockNumReader - + BlockReader - + ChangeSetReader - + NodePrimitivesProvider - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache, { type ComponentsBuilder = ComponentsBuilder< N, @@ -487,21 +449,7 @@ where } } -impl DebugNode for EthereumNode -where - N: FullNodeComponents, - N::Provider: DatabaseProviderFactory, - ::Provider: BlockHashReader - + BlockNumReader - + BlockReader - + ChangeSetReader - + NodePrimitivesProvider - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache, -{ +impl> DebugNode for EthereumNode { type RpcBlock = alloy_rpc_types_eth::Block; fn rpc_to_primitive_block(rpc_block: Self::RpcBlock) -> reth_ethereum_primitives::Block { diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 1e8f9cfddbf..6f413468dc6 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -14,9 +14,7 @@ use reth_e2e_test_utils::{wallet::Wallet, NodeHelperType, TmpDB}; use reth_ethereum_primitives::TxType; use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; -use reth_provider::{ - DatabaseProviderFactory, FullProvider, NodePrimitivesProvider, RocksDBProviderFactory, -}; +use reth_provider::FullProvider; /// Helper function to create a new eth payload attributes pub(crate) const fn eth_payload_attributes(timestamp: u64) -> PayloadAttributes { @@ -52,8 +50,6 @@ pub(crate) async fn advance_with_random_transactions( ) -> eyre::Result<()> where Provider: FullProvider>, - ::Provider: - NodePrimitivesProvider + RocksDBProviderFactory, { let provider = ProviderBuilder::new().connect_http(node.rpc_url()); let signers = Wallet::new(1).with_chain_id(provider.get_chain_id().await?).wallet_gen(); diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 31a10ece291..a10c40841f7 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -32,11 +32,6 @@ use reth_node_core::{ version::{version_metadata, CLIENT_CODE}, }; use reth_payload_builder::{PayloadBuilderHandle, PayloadStore}; -use reth_provider::{ - BlockHashReader, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, - NodePrimitivesProvider, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, - StorageChangeSetReader, StorageSettingsCache, -}; use reth_rpc::{ eth::{core::EthRpcConverterFor, DevSigner, EthApiTypes, FullEthApiServer}, AdminApi, @@ -1457,17 +1452,6 @@ where ::Payload, Block = BlockTy, > + Clone, - Node::Provider: DatabaseProviderFactory, - ::Provider: BlockHashReader - + BlockNumReader - + BlockReader - + ChangeSetReader - + NodePrimitivesProvider - + PruneCheckpointReader - + RocksDBProviderFactory - + StageCheckpointReader - + StorageChangeSetReader - + StorageSettingsCache, { type EngineValidator = BasicEngineValidator; diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index e878a861657..f3bd49fd72d 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -12,10 +12,9 @@ use reth_db_api::{ BlockNumberList, }; use reth_primitives_traits::{Account, Bytecode}; -use reth_stages_types::StageId; use reth_storage_api::{ - BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StageCheckpointReader, - StateProofProvider, StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, + BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StateProofProvider, + StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ @@ -125,7 +124,7 @@ pub struct HistoricalStateProviderRef<'b, Provider> { /// Database provider provider: &'b Provider, /// Changeset cache handle for retrieving trie changesets. - changeset_cache: ChangesetCache, + _changeset_cache: ChangesetCache, /// Block number is main index for the history state of accounts and storages. block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. @@ -143,7 +142,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block ) -> Self { Self { provider, - changeset_cache, + _changeset_cache: changeset_cache, block_number, lowest_available_blocks: Default::default(), } @@ -157,7 +156,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block lowest_available_blocks: LowestAvailableBlocks, changeset_cache: ChangesetCache, ) -> Self { - Self { provider, changeset_cache, block_number, lowest_available_blocks } + Self { provider, _changeset_cache: changeset_cache, block_number, lowest_available_blocks } } /// Lookup an account in the `AccountsHistory` table using `EitherReader`. @@ -288,37 +287,6 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block reth_trie_db::from_reverts_auto(self.provider, self.block_number..) } - /// Retrieve cached trie reverts and hashed state reverts for this history provider. - fn revert_input(&self) -> ProviderResult - where - Provider: StageCheckpointReader + StorageSettingsCache, - { - let revert_state = self.revert_state()?; - let db_tip_block = self - .provider - .get_stage_checkpoint(StageId::Finish)? - .as_ref() - .map(|chk| chk.block_number) - .ok_or_else(|| ProviderError::InsufficientChangesets { - requested: self.block_number, - available: 0..=0, - })?; - - if self.block_number > db_tip_block { - return Ok(TrieInput::from_state(revert_state.into())) - } - - let trie_reverts = self - .changeset_cache - .get_or_compute_range(self.provider, self.block_number..=db_tip_block)?; - - if trie_reverts.is_empty() { - Ok(TrieInput::from_state(revert_state.into())) - } else { - Ok(TrieInput::from_blocks_sorted([(&revert_state, &trie_reverts)])) - } - } - /// Retrieve revert hashed storage for this history provider and target address. fn revert_storage(&self, address: Address) -> ProviderResult where @@ -422,25 +390,22 @@ impl< + ChangeSetReader + StorageChangeSetReader + BlockNumReader - + StageCheckpointReader + StorageSettingsCache, > StateRootProvider for HistoricalStateProviderRef<'_, Provider> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut input = self.revert_input()?; - input.append(hashed_state); - Ok(>::overlay_root_from_nodes( - self.tx(), - TrieInputSorted::from_unsorted(input), - )?) + let mut revert_state = self.revert_state()?; + let hashed_state_sorted = hashed_state.into_sorted(); + revert_state.extend_ref_and_sort(&hashed_state_sorted); + Ok(>::overlay_root(self.tx(), &revert_state)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend_self(self.revert_input()?); + input.prepend(self.revert_state()?.into()); Ok(>::overlay_root_from_nodes( self.tx(), TrieInputSorted::from_unsorted(input), @@ -453,12 +418,10 @@ impl< hashed_state: HashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider, |A| { - let mut input = self.revert_input()?; - input.append(hashed_state); - Ok(>::overlay_root_from_nodes_with_updates( - self.tx(), - TrieInputSorted::from_unsorted(input), - )?) + let mut revert_state = self.revert_state()?; + let hashed_state_sorted = hashed_state.into_sorted(); + revert_state.extend_ref_and_sort(&hashed_state_sorted); + Ok(>::overlay_root_with_updates(self.tx(), &revert_state)?) }) } @@ -468,7 +431,7 @@ impl< ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend_self(self.revert_input()?); + input.prepend(self.revert_state()?.into()); Ok(>::overlay_root_from_nodes_with_updates( self.tx(), TrieInputSorted::from_unsorted(input), @@ -542,7 +505,6 @@ impl< + ChangeSetReader + StorageChangeSetReader + BlockNumReader - + StageCheckpointReader + StorageSettingsCache, > StateProofProvider for HistoricalStateProviderRef<'_, Provider> { @@ -555,7 +517,7 @@ impl< ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend_self(self.revert_input()?); + input.prepend(self.revert_state()?.into()); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) }) @@ -568,7 +530,7 @@ impl< ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend_self(self.revert_input()?); + input.prepend(self.revert_state()?.into()); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_multiproof(input, targets).map_err(ProviderError::from) }) @@ -582,7 +544,7 @@ impl< ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider, |A| { let mut input = input; - input.prepend_self(self.revert_input()?); + input.prepend(self.revert_state()?.into()); let nodes_sorted = input.nodes.into_sorted(); let state_sorted = input.state.into_sorted(); let witness = TrieWitness::new( @@ -622,7 +584,6 @@ impl< + BlockHashReader + ChangeSetReader + StorageChangeSetReader - + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider, @@ -709,7 +670,7 @@ impl where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]); +reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]); /// Lowest blocks at which different parts of the state are available. /// They may be [Some] if pruning is enabled. @@ -842,8 +803,7 @@ mod tests { use reth_primitives_traits::{Account, StorageEntry}; use reth_storage_api::{ BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, - NodePrimitivesProvider, StageCheckpointReader, StorageChangeSetReader, - StorageSettingsCache, + NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderError; use reth_trie_db::ChangesetCache; @@ -860,7 +820,6 @@ mod tests { + BlockNumReader + BlockHashReader + ChangeSetReader - + StageCheckpointReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory @@ -869,32 +828,6 @@ mod tests { assert_state_provider::>(); } - fn historical_state_provider_ref( - provider: &Provider, - block_number: u64, - ) -> HistoricalStateProviderRef<'_, Provider> - where - Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader, - { - HistoricalStateProviderRef::new(provider, block_number, ChangesetCache::new()) - } - - fn historical_state_provider_ref_with_lowest_available_blocks( - provider: &Provider, - block_number: u64, - lowest_available_blocks: LowestAvailableBlocks, - ) -> HistoricalStateProviderRef<'_, Provider> - where - Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader, - { - HistoricalStateProviderRef::new_with_lowest_available_blocks( - provider, - block_number, - lowest_available_blocks, - ChangesetCache::new(), - ) - } - #[test] fn history_provider_get_account() { let factory = create_test_provider_factory(); @@ -961,46 +894,50 @@ mod tests { let db = factory.provider().unwrap(); // run - assert!(matches!(historical_state_provider_ref(&db, 1).basic_account(&ADDRESS), Ok(None))); assert!(matches!( - historical_state_provider_ref(&db, 2).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()).basic_account(&ADDRESS), + Ok(None) + )); + assert!(matches!( + HistoricalStateProviderRef::new(&db, 2, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at3 )); assert!(matches!( - historical_state_provider_ref(&db, 3).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at3 )); assert!(matches!( - historical_state_provider_ref(&db, 4).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at7 )); assert!(matches!( - historical_state_provider_ref(&db, 7).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at7 )); assert!(matches!( - historical_state_provider_ref(&db, 9).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at10 )); assert!(matches!( - historical_state_provider_ref(&db, 10).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at10 )); assert!(matches!( - historical_state_provider_ref(&db, 11).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at15 )); assert!(matches!( - historical_state_provider_ref(&db, 16).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_plain )); assert!(matches!( - historical_state_provider_ref(&db, 1).basic_account(&HIGHER_ADDRESS), + HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + .basic_account(&HIGHER_ADDRESS), Ok(None) )); assert!(matches!( - historical_state_provider_ref(&db, 1000).basic_account(&HIGHER_ADDRESS), + HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).basic_account(&HIGHER_ADDRESS), Ok(Some(acc)) if acc == higher_acc_plain )); } @@ -1059,43 +996,46 @@ mod tests { // run assert!(matches!( - historical_state_provider_ref(&db, 0).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new()) + .storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - historical_state_provider_ref(&db, 3).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()) + .storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - historical_state_provider_ref(&db, 4).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - historical_state_provider_ref(&db, 7).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - historical_state_provider_ref(&db, 9).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - historical_state_provider_ref(&db, 10).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - historical_state_provider_ref(&db, 11).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at15.value )); assert!(matches!( - historical_state_provider_ref(&db, 16).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_plain.value )); assert!(matches!( - historical_state_provider_ref(&db, 1).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + .storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - historical_state_provider_ref(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == higher_entry_plain.value )); } @@ -1107,13 +1047,14 @@ mod tests { // provider block_number < lowest available block number, // i.e. state at provider block is pruned - let provider = historical_state_provider_ref_with_lowest_available_blocks( + let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( &db, 2, LowestAvailableBlocks { account_history_block_number: Some(3), storage_history_block_number: Some(3), }, + ChangesetCache::new(), ); assert!(matches!( provider.account_history_lookup(ADDRESS), @@ -1126,13 +1067,14 @@ mod tests { // provider block_number == lowest available block number, // i.e. state at provider block is available - let provider = historical_state_provider_ref_with_lowest_available_blocks( + let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( &db, 2, LowestAvailableBlocks { account_history_block_number: Some(2), storage_history_block_number: Some(2), }, + ChangesetCache::new(), ); assert!(matches!( provider.account_history_lookup(ADDRESS), @@ -1145,13 +1087,14 @@ mod tests { // provider block_number == lowest available block number, // i.e. state at provider block is available - let provider = historical_state_provider_ref_with_lowest_available_blocks( + let provider = HistoricalStateProviderRef::new_with_lowest_available_blocks( &db, 2, LowestAvailableBlocks { account_history_block_number: Some(1), storage_history_block_number: Some(1), }, + ChangesetCache::new(), ); assert!(matches!( provider.account_history_lookup(ADDRESS), @@ -1232,43 +1175,46 @@ mod tests { let db = factory.provider().unwrap(); assert!(matches!( - historical_state_provider_ref(&db, 0).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new()) + .storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - historical_state_provider_ref(&db, 3).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()) + .storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - historical_state_provider_ref(&db, 4).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - historical_state_provider_ref(&db, 7).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - historical_state_provider_ref(&db, 9).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - historical_state_provider_ref(&db, 10).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - historical_state_provider_ref(&db, 11).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at15.value )); assert!(matches!( - historical_state_provider_ref(&db, 16).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_plain.value )); assert!(matches!( - historical_state_provider_ref(&db, 1).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + .storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - historical_state_provider_ref(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == higher_entry_plain.value )); } @@ -1372,43 +1318,46 @@ mod tests { let db = factory.provider().unwrap(); assert!(matches!( - historical_state_provider_ref(&db, 0).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new()) + .storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - historical_state_provider_ref(&db, 3).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()) + .storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - historical_state_provider_ref(&db, 4).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(7) )); assert!(matches!( - historical_state_provider_ref(&db, 7).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(7) )); assert!(matches!( - historical_state_provider_ref(&db, 9).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(10) )); assert!(matches!( - historical_state_provider_ref(&db, 10).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(10) )); assert!(matches!( - historical_state_provider_ref(&db, 11).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(15) )); assert!(matches!( - historical_state_provider_ref(&db, 16).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(100) )); assert!(matches!( - historical_state_provider_ref(&db, 1).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + .storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - historical_state_provider_ref(&db, 1000).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(1000) )); } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 63314c9c4fd..72fa695a0a2 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -1,11 +1,9 @@ use crate::{ - providers::RocksDBProvider, traits::{BlockSource, ReceiptProvider}, AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, HeaderProvider, - PruneCheckpointReader, ReceiptProviderIdExt, RocksDBProviderFactory, StateProvider, - StateProviderBox, StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, - TransactionsProvider, + PruneCheckpointReader, ReceiptProviderIdExt, StateProvider, StateProviderBox, + StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, TransactionsProvider, }; use alloy_consensus::{ constants::EMPTY_ROOT_HASH, @@ -931,21 +929,6 @@ impl StorageSettingsCache fn set_storage_settings_cache(&self, _settings: StorageSettings) {} } -impl RocksDBProviderFactory - for MockEthProvider -{ - fn rocksdb_provider(&self) -> RocksDBProvider { - RocksDBProvider::new(std::env::temp_dir().join(format!("reth-mock-rocksdb-{:p}", self))) - .expect("failed to create mock RocksDB provider") - } - - fn set_pending_rocksdb_batch(&self, _batch: rocksdb::WriteBatchWithTransaction) {} - - fn commit_pending_rocksdb_batches(&self) -> ProviderResult<()> { - Ok(()) - } -} - impl StateProviderFactory for MockEthProvider { From f652994b35c67a8dc9faf2050ac91e1d35c19929 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:19:18 +0000 Subject: [PATCH 05/20] refactor(provider): extract overlay builder Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db074-fac6-76d2-8950-3e84b3c895a4 Co-authored-by: Amp --- .../tree/src/tree/payload_processor/mod.rs | 7 +- .../src/tree/payload_processor/sparse_trie.rs | 9 +- .../engine/tree/src/tree/payload_validator.rs | 39 +++-- crates/storage/provider/src/providers/mod.rs | 2 +- .../provider/src/providers/state/overlay.rs | 161 ++++++++++-------- crates/trie/parallel/src/proof_task.rs | 2 +- crates/trie/parallel/src/root.rs | 9 +- 7 files changed, 130 insertions(+), 99 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index bc274525a81..cd93746ff0d 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -962,7 +962,7 @@ mod tests { use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{Account, Recovered, StorageEntry}; use reth_provider::{ - providers::{BlockchainProvider, OverlayStateProviderFactory}, + providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory}, test_utils::create_test_provider_factory_with_chain_spec, ChainSpecProvider, HashingWriter, }; @@ -1237,7 +1237,10 @@ mod tests { std::convert::identity, ), StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None), - OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()), + OverlayStateProviderFactory::new( + provider_factory, + OverlayBuilder::new(ChangesetCache::new()), + ), &TreeConfig::default(), None, // No BAL for test ); diff --git a/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs b/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs index 668002a0824..0fcf7e22714 100644 --- a/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs +++ b/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs @@ -894,7 +894,8 @@ mod tests { use super::*; use alloy_primitives::{keccak256, Address, B256, U256}; use reth_provider::{ - providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory, + providers::{OverlayBuilder, OverlayStateProviderFactory}, + test_utils::create_test_provider_factory, }; use reth_trie_db::ChangesetCache; use reth_trie_parallel::proof_task::ProofTaskCtx; @@ -983,8 +984,10 @@ mod tests { fn run_returns_parent_root_without_revealing_blind_trie_when_no_state_updates() { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); - let overlay_factory = - OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()); + let overlay_factory = OverlayStateProviderFactory::new( + provider_factory, + OverlayBuilder::new(ChangesetCache::new()), + ); let proof_worker_handle = ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 9ba884c8be7..cb8156a2ed1 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -77,10 +77,11 @@ use reth_primitives_traits::{ RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, }; use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, - ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider, - StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, + providers::{OverlayBuilder, OverlayStateProviderFactory}, + BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, + DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, + StageCheckpointReader, StateProvider, StateProviderFactory, StateReader, + StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot}; @@ -523,10 +524,12 @@ where // Create overlay factory for payload processor (StateRootTask path needs it for // multiproofs) - let overlay_factory = - OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone()) + let overlay_factory = OverlayStateProviderFactory::new( + self.provider.clone(), + OverlayBuilder::new(self.changeset_cache.clone()) .with_block_hash(Some(anchor_hash)) - .with_lazy_overlay(lazy_overlay); + .with_lazy_overlay(lazy_overlay), + ); // Spawn the appropriate processor based on strategy let mut handle = ensure_ok!(self.spawn_payload_processor( @@ -1095,8 +1098,11 @@ where // need to use the prefix sets which were generated from it to indicate to the // ParallelStateRoot which parts of the trie need to be recomputed. let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let overlay_factory = - overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()); + let (factory, overlay_builder) = overlay_factory.into_parts(); + let overlay_factory = OverlayStateProviderFactory::new( + factory, + overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()), + ); ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone()) .incremental_root_with_updates() } @@ -1115,8 +1121,11 @@ where // need to use the prefix sets which were generated from it to indicate to the // StateRoot which parts of the trie need to be recomputed. let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let overlay_factory = - overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()); + let (factory, overlay_builder) = overlay_factory.into_parts(); + let overlay_factory = OverlayStateProviderFactory::new( + factory, + overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()), + ); let provider = overlay_factory.database_provider_ro()?; @@ -2026,10 +2035,12 @@ where state: &EngineApiTreeState, ) -> Option { let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, state); - let overlay_factory = - OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone()) + let overlay_factory = OverlayStateProviderFactory::new( + self.provider.clone(), + OverlayBuilder::new(self.changeset_cache.clone()) .with_block_hash(Some(anchor_hash)) - .with_lazy_overlay(lazy_overlay); + .with_lazy_overlay(lazy_overlay), + ); Some(self.payload_processor.spawn_state_root( overlay_factory, diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index d46a8902f71..d4469b945ff 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -20,7 +20,7 @@ pub use state::{ HistoricalStateProviderRef, HistoryInfo, LowestAvailableBlocks, }, latest::{LatestStateProvider, LatestStateProviderRef}, - overlay::{OverlayStateProvider, OverlayStateProviderFactory}, + overlay::{OverlayBuilder, OverlayStateProvider, OverlayStateProviderFactory}, }; mod consistent_view; diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index f52498c71d8..c760aa41426 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -85,37 +85,30 @@ impl OverlaySource { } } -/// Factory for creating overlay state providers with optional reverts and overlays. +/// Builder for calculating trie and hashed-state overlays. /// -/// This factory allows building an `OverlayStateProvider` whose DB state has been reverted to a -/// particular block, and/or with additional overlay information added on top. +/// This stores the overlay configuration and the logic for resolving immediate/lazy overlays and +/// collecting reverts. It is intentionally independent from any provider factory or overlay cache. #[derive(Debug, Clone)] -pub struct OverlayStateProviderFactory { - /// The underlying database provider factory - factory: F, - /// Optional block hash for collecting reverts +pub struct OverlayBuilder { + /// Optional block hash for collecting reverts. block_hash: Option, /// Optional overlay source (lazy or immediate). overlay_source: Option, - /// Changeset cache handle for retrieving trie changesets + /// Changeset cache handle for retrieving trie changesets. changeset_cache: ChangesetCache, - /// Metrics for tracking provider operations + /// Metrics for tracking provider operations. metrics: OverlayStateProviderMetrics, - /// A cache which maps `db_tip -> Overlay`. If the db tip changes during usage of the factory - /// then a new entry will get added to this, but in most cases only one entry is present. - overlay_cache: Arc>, } -impl OverlayStateProviderFactory { - /// Create a new overlay state provider factory - pub fn new(factory: F, changeset_cache: ChangesetCache) -> Self { +impl OverlayBuilder { + /// Create a new overlay builder. + pub fn new(changeset_cache: ChangesetCache) -> Self { Self { - factory, block_hash: None, overlay_source: None, changeset_cache, metrics: OverlayStateProviderMetrics::default(), - overlay_cache: Default::default(), } } @@ -131,8 +124,6 @@ impl OverlayStateProviderFactory { /// This overlay will be applied on top of any reverts applied via `with_block_hash`. pub fn with_overlay_source(mut self, source: Option) -> Self { self.overlay_source = source; - // Clear the overlay cache since we've updated the source. - self.overlay_cache = Default::default(); self } @@ -141,8 +132,6 @@ impl OverlayStateProviderFactory { /// Convenience method that wraps the lazy overlay in `OverlaySource::Lazy`. pub fn with_lazy_overlay(mut self, lazy_overlay: Option) -> Self { self.overlay_source = lazy_overlay.map(OverlaySource::Lazy); - // Clear the overlay cache since we've updated the source. - self.overlay_cache = Default::default(); self } @@ -158,8 +147,6 @@ impl OverlayStateProviderFactory { trie: Arc::new(TrieUpdatesSorted::default()), state, }); - // Clear the overlay cache since we've updated the source. - self.overlay_cache = Default::default(); } self } @@ -174,7 +161,6 @@ impl OverlayStateProviderFactory { Arc::make_mut(state).extend_ref_and_sort(&other); } Some(OverlaySource::Lazy(lazy)) => { - // Resolve lazy overlay and convert to immediate with extension let (trie, mut state) = lazy.as_overlay(); Arc::make_mut(&mut state).extend_ref_and_sort(&other); self.overlay_source = Some(OverlaySource::Immediate { trie, state }); @@ -186,23 +172,9 @@ impl OverlayStateProviderFactory { }); } } - // Clear the overlay cache since we've updated the source. - self.overlay_cache = Default::default(); self } -} -impl OverlayStateProviderFactory -where - F: DatabaseProviderFactory, - F::Provider: StageCheckpointReader - + PruneCheckpointReader - + ChangeSetReader - + StorageChangeSetReader - + DBProvider - + BlockNumReader - + StorageSettingsCache, -{ /// Resolves the effective overlay (trie updates, hashed state). /// /// If an overlay source is set, it is resolved (blocking if lazy). @@ -217,10 +189,13 @@ where } /// Returns the block number for [`Self`]'s `block_hash` field, if any. - fn get_requested_block_number( + fn get_requested_block_number( &self, - provider: &F::Provider, - ) -> ProviderResult> { + provider: &Provider, + ) -> ProviderResult> + where + Provider: BlockNumReader, + { if let Some(block_hash) = self.block_hash { Ok(Some( provider @@ -234,7 +209,10 @@ where /// Returns the block which is at the tip of the DB, i.e. the block which the state tables of /// the DB are currently synced to. - fn get_db_tip_block_number(&self, provider: &F::Provider) -> ProviderResult { + fn get_db_tip_block_number(&self, provider: &Provider) -> ProviderResult + where + Provider: StageCheckpointReader, + { provider .get_stage_checkpoint(StageId::Finish)? .as_ref() @@ -247,21 +225,19 @@ where /// /// Takes into account both the stage checkpoint and the prune checkpoint to determine the /// available data range. - fn reverts_required( + fn reverts_required( &self, - provider: &F::Provider, + provider: &Provider, db_tip_block: BlockNumber, requested_block: BlockNumber, - ) -> ProviderResult { - // If the requested block is the DB tip then there won't be any reverts necessary, and we - // can simply return Ok. + ) -> ProviderResult + where + Provider: PruneCheckpointReader, + { if db_tip_block == requested_block { return Ok(false) } - // Check account history prune checkpoint to determine the lower bound of available data. - // The prune checkpoint's block_number is the highest pruned block, so data is available - // starting from the next block. let prune_checkpoint = provider.get_prune_checkpoint(PruneSegment::AccountHistory)?; let lower_bound = prune_checkpoint .and_then(|chk| chk.block_number) @@ -270,7 +246,6 @@ where let available_range = lower_bound..=db_tip_block; - // Check if the requested block is within the available range if !available_range.contains(&requested_block) { return Err(ProviderError::InsufficientChangesets { requested: requested_block, @@ -288,20 +263,25 @@ where skip_all, fields(%db_tip_block) )] - fn calculate_overlay( + fn calculate_overlay( &self, - provider: &F::Provider, + provider: &Provider, db_tip_block: BlockNumber, - ) -> ProviderResult { - // - // Set up variables we'll use for recording metrics. There's two different code-paths here, - // and we want to make sure both record metrics, so we do metrics recording after. + ) -> ProviderResult + where + Provider: ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader + + StageCheckpointReader + + PruneCheckpointReader + + StorageSettingsCache, + { let retrieve_trie_reverts_duration; let retrieve_hashed_state_reverts_duration; let trie_updates_total_len; let hashed_state_updates_total_len; - // If block_hash is provided, collect reverts let (trie_updates, hashed_post_state) = if let Some(from_block) = self.get_requested_block_number(provider)? && self.reverts_required(provider, db_tip_block, from_block)? @@ -316,16 +296,12 @@ where "Collecting trie reverts for overlay state provider" ); - // Collect trie reverts using changeset cache let mut trie_reverts = { let _guard = debug_span!(target: "providers::state::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); - - // Use changeset cache to retrieve and accumulate reverts to restore state after - // from_block let accumulated_reverts = self .changeset_cache .get_or_compute_range(provider, (from_block + 1)..=db_tip_block)?; @@ -334,7 +310,6 @@ where accumulated_reverts }; - // Collect state reverts let mut hashed_state_reverts = { let _guard = debug_span!(target: "providers::state::overlay", "retrieving_hashed_state_reverts").entered(); @@ -344,8 +319,6 @@ where res }; - // Resolve overlays (lazy or immediate) and extend reverts with them. - // If reverts are empty, use overlays directly to avoid cloning. let (overlay_trie, overlay_state) = self.resolve_overlays(); let trie_updates = if trie_reverts.is_empty() { @@ -380,7 +353,6 @@ where (trie_updates, hashed_state_updates) } else { - // If no block_hash, use overlays directly (resolving lazy if set) let (trie_updates, hashed_state) = self.resolve_overlays(); retrieve_trie_reverts_duration = Duration::ZERO; @@ -391,7 +363,6 @@ where (trie_updates, hashed_state) }; - // Record metrics self.metrics .retrieve_trie_reverts_duration .record(retrieve_trie_reverts_duration.as_secs_f64()); @@ -404,11 +375,23 @@ where Ok(Overlay { trie_updates, hashed_post_state }) } - /// Fetches an [`Overlay`] from the cache based on the current db tip block. If there is no - /// cached value then this calculates the [`Overlay`] and populates the cache. + /// Builds the effective overlay for the given provider, using the supplied cache for + /// db-tip-scoped memoization. #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] - fn get_overlay(&self, provider: &F::Provider) -> ProviderResult { - // No anchor block — just resolve the in-memory overlay directly. + fn build_overlay( + &self, + provider: &Provider, + overlay_cache: &DashMap, + ) -> ProviderResult + where + Provider: StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader + + StorageSettingsCache, + { if self.block_hash.is_none() { let (trie_updates, hashed_post_state) = self.resolve_overlays(); return Ok(Overlay { trie_updates, hashed_post_state }) @@ -416,7 +399,7 @@ where let db_tip_block = self.get_db_tip_block_number(provider)?; - let overlay = match self.overlay_cache.entry(db_tip_block) { + let overlay = match overlay_cache.entry(db_tip_block) { dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { self.metrics.overlay_cache_misses.increment(1); @@ -430,6 +413,33 @@ where } } +/// Factory for creating overlay state providers with optional reverts and overlays. +/// +/// This factory allows building an `OverlayStateProvider` whose DB state has been reverted to a +/// particular block, and/or with additional overlay information added on top. +#[derive(Debug, Clone)] +pub struct OverlayStateProviderFactory { + /// The underlying database provider factory + factory: F, + /// Overlay builder containing the configuration and overlay calculation logic. + overlay_builder: OverlayBuilder, + /// A cache which maps `db_tip -> Overlay`. If the db tip changes during usage of the factory + /// then a new entry will get added to this, but in most cases only one entry is present. + overlay_cache: Arc>, +} + +impl OverlayStateProviderFactory { + /// Create a new overlay state provider factory + pub fn new(factory: F, overlay_builder: OverlayBuilder) -> Self { + Self { factory, overlay_builder, overlay_cache: Default::default() } + } + + /// Deconstruct the factory into its underlying provider factory and overlay builder. + pub fn into_parts(self) -> (F, OverlayBuilder) { + (self.factory, self.overlay_builder) + } +} + impl DatabaseProviderROFactory for OverlayStateProviderFactory where F: DatabaseProviderFactory, @@ -451,14 +461,15 @@ where let provider = { let start = Instant::now(); let res = self.factory.database_provider_ro()?; - self.metrics.create_provider_duration.record(start.elapsed()); + self.overlay_builder.metrics.create_provider_duration.record(start.elapsed()); res }; - let Overlay { trie_updates, hashed_post_state } = self.get_overlay(&provider)?; + let Overlay { trie_updates, hashed_post_state } = + self.overlay_builder.build_overlay(&provider, &self.overlay_cache)?; let is_v2 = provider.cached_storage_settings().is_v2(); - self.metrics.database_provider_ro_duration.record(overall_start.elapsed()); + self.overlay_builder.metrics.database_provider_ro_duration.record(overall_start.elapsed()); Ok(OverlayStateProvider::new(provider, trie_updates, hashed_post_state, is_v2)) } } diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 2e48bbbaff1..03dfc55cd55 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -1164,7 +1164,7 @@ mod tests { let changeset_cache = reth_trie_db::ChangesetCache::new(); let factory = reth_provider::providers::OverlayStateProviderFactory::new( provider_factory, - changeset_cache, + reth_provider::providers::OverlayBuilder::new(changeset_cache), ); let ctx = test_ctx(factory); diff --git a/crates/trie/parallel/src/root.rs b/crates/trie/parallel/src/root.rs index edd453cca1f..9a209294ab2 100644 --- a/crates/trie/parallel/src/root.rs +++ b/crates/trie/parallel/src/root.rs @@ -285,7 +285,7 @@ mod tests { let changeset_cache = reth_trie_db::ChangesetCache::new(); let mut overlay_factory = reth_provider::providers::OverlayStateProviderFactory::new( factory.clone(), - changeset_cache, + reth_provider::providers::OverlayBuilder::new(changeset_cache), ); let mut rng = rand::rng(); @@ -362,8 +362,11 @@ mod tests { } let prefix_sets = hashed_state.construct_prefix_sets(); - overlay_factory = - overlay_factory.with_hashed_state_overlay(Some(Arc::new(hashed_state.into_sorted()))); + let (factory, overlay_builder) = overlay_factory.into_parts(); + overlay_factory = reth_provider::providers::OverlayStateProviderFactory::new( + factory, + overlay_builder.with_hashed_state_overlay(Some(Arc::new(hashed_state.into_sorted()))), + ); assert_eq!( ParallelStateRoot::new(overlay_factory, prefix_sets.freeze(), runtime) From 52eec466d7353b34a5f5bea4500ea3da18b85940 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:27:43 +0000 Subject: [PATCH 06/20] refactor(provider): keep overlay caching in factory Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db074-fac6-76d2-8950-3e84b3c895a4 Co-authored-by: Amp --- .../engine/tree/src/tree/payload_validator.rs | 65 +++++++++++++------ .../provider/src/providers/state/overlay.rs | 54 ++++++++------- crates/trie/parallel/src/root.rs | 6 +- 3 files changed, 77 insertions(+), 48 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index cb8156a2ed1..b2b0c1ddb8c 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -524,12 +524,12 @@ where // Create overlay factory for payload processor (StateRootTask path needs it for // multiproofs) - let overlay_factory = OverlayStateProviderFactory::new( - self.provider.clone(), - OverlayBuilder::new(self.changeset_cache.clone()) - .with_block_hash(Some(anchor_hash)) - .with_lazy_overlay(lazy_overlay), - ); + let provider_factory = self.provider.clone(); + let overlay_builder = OverlayBuilder::new(self.changeset_cache.clone()) + .with_block_hash(Some(anchor_hash)) + .with_lazy_overlay(lazy_overlay); + let overlay_factory = + OverlayStateProviderFactory::new(provider_factory.clone(), overlay_builder.clone()); // Spawn the appropriate processor based on strategy let mut handle = ensure_ok!(self.spawn_payload_processor( @@ -667,7 +667,8 @@ where let task_result = ensure_ok_post_block!( self.await_state_root_with_timeout( &mut handle, - overlay_factory.clone(), + provider_factory.clone(), + overlay_builder.clone(), &hashed_state, ), block @@ -691,7 +692,8 @@ where // Compare trie updates with serial computation if configured if self.config.always_compare_trie_updates() { let _has_diff = self.compare_trie_updates_with_serial( - overlay_factory.clone(), + provider_factory.clone(), + overlay_builder.clone(), &hashed_state, trie_updates.as_ref().clone(), ); @@ -730,7 +732,11 @@ where } StateRootStrategy::Parallel => { debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm"); - match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) { + match self.compute_state_root_parallel( + provider_factory.clone(), + overlay_builder.clone(), + &hashed_state, + ) { Ok(result) => { let elapsed = root_time.elapsed(); info!( @@ -766,7 +772,11 @@ where } let (root, updates) = ensure_ok_post_block!( - Self::compute_state_root_serial(overlay_factory.clone(), &hashed_state), + Self::compute_state_root_serial( + provider_factory.clone(), + overlay_builder.clone(), + &hashed_state, + ), block ); @@ -1090,7 +1100,8 @@ where #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] fn compute_state_root_parallel( &self, - overlay_factory: OverlayStateProviderFactory

, + provider_factory: P, + overlay_builder: OverlayBuilder, hashed_state: &LazyHashedPostState, ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { let hashed_state = hashed_state.get(); @@ -1098,9 +1109,8 @@ where // need to use the prefix sets which were generated from it to indicate to the // ParallelStateRoot which parts of the trie need to be recomputed. let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let (factory, overlay_builder) = overlay_factory.into_parts(); let overlay_factory = OverlayStateProviderFactory::new( - factory, + provider_factory, overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()), ); ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone()) @@ -1113,7 +1123,8 @@ where /// [`HashedPostState`] containing the changes of this block, to compute the state root and /// trie updates for this block. fn compute_state_root_serial( - overlay_factory: OverlayStateProviderFactory

, + provider_factory: P, + overlay_builder: OverlayBuilder, hashed_state: &LazyHashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { let hashed_state = hashed_state.get(); @@ -1121,9 +1132,8 @@ where // need to use the prefix sets which were generated from it to indicate to the // StateRoot which parts of the trie need to be recomputed. let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let (factory, overlay_builder) = overlay_factory.into_parts(); let overlay_factory = OverlayStateProviderFactory::new( - factory, + provider_factory, overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()), ); @@ -1156,7 +1166,8 @@ where fn await_state_root_with_timeout( &self, handle: &mut PayloadHandle, - overlay_factory: OverlayStateProviderFactory

, + provider_factory: P, + overlay_builder: OverlayBuilder, hashed_state: &LazyHashedPostState, ) -> ProviderResult> { let Some(timeout) = self.config.state_root_task_timeout() else { @@ -1181,10 +1192,15 @@ where let (seq_tx, seq_rx) = std::sync::mpsc::channel::>(); - let seq_overlay = overlay_factory; + let seq_provider_factory = provider_factory; + let seq_overlay_builder = overlay_builder; let seq_hashed_state = hashed_state.clone(); self.payload_processor.executor().spawn_blocking_named("serial-root", move || { - let result = Self::compute_state_root_serial(seq_overlay, &seq_hashed_state); + let result = Self::compute_state_root_serial( + seq_provider_factory, + seq_overlay_builder, + &seq_hashed_state, + ); let _ = seq_tx.send(result); }); @@ -1248,13 +1264,18 @@ where /// updates. fn compare_trie_updates_with_serial( &self, - overlay_factory: OverlayStateProviderFactory

, + provider_factory: P, + overlay_builder: OverlayBuilder, hashed_state: &LazyHashedPostState, task_trie_updates: TrieUpdates, ) -> bool { debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation"); - match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) { + match Self::compute_state_root_serial( + provider_factory.clone(), + overlay_builder.clone(), + hashed_state, + ) { Ok((serial_root, serial_trie_updates)) => { debug!( target: "engine::tree::payload_validator", @@ -1263,6 +1284,8 @@ where ); // Get a database provider to use as trie cursor factory + let overlay_factory = + OverlayStateProviderFactory::new(provider_factory, overlay_builder); match overlay_factory.database_provider_ro() { Ok(provider) => { match super::trie_updates::compare_trie_updates( diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index c760aa41426..f27e22ac57f 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -375,14 +375,9 @@ impl OverlayBuilder { Ok(Overlay { trie_updates, hashed_post_state }) } - /// Builds the effective overlay for the given provider, using the supplied cache for - /// db-tip-scoped memoization. + /// Builds the effective overlay for the given provider. #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] - fn build_overlay( - &self, - provider: &Provider, - overlay_cache: &DashMap, - ) -> ProviderResult + fn build_overlay(&self, provider: &Provider) -> ProviderResult where Provider: StageCheckpointReader + PruneCheckpointReader @@ -398,18 +393,7 @@ impl OverlayBuilder { } let db_tip_block = self.get_db_tip_block_number(provider)?; - - let overlay = match overlay_cache.entry(db_tip_block) { - dashmap::Entry::Occupied(entry) => entry.get().clone(), - dashmap::Entry::Vacant(entry) => { - self.metrics.overlay_cache_misses.increment(1); - let overlay = self.calculate_overlay(provider, db_tip_block)?; - entry.insert(overlay.clone()); - overlay - } - }; - - Ok(overlay) + self.calculate_overlay(provider, db_tip_block) } } @@ -434,9 +418,32 @@ impl OverlayStateProviderFactory { Self { factory, overlay_builder, overlay_cache: Default::default() } } - /// Deconstruct the factory into its underlying provider factory and overlay builder. - pub fn into_parts(self) -> (F, OverlayBuilder) { - (self.factory, self.overlay_builder) + /// Fetches an [`Overlay`] from the cache based on the current db tip block. If there is no + /// cached value then this calculates the [`Overlay`] and populates the cache. + #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] + fn get_overlay(&self, provider: &Provider) -> ProviderResult + where + Provider: StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader + + StorageSettingsCache, + { + let db_tip_block = self.overlay_builder.get_db_tip_block_number(provider)?; + + let overlay = match self.overlay_cache.entry(db_tip_block) { + dashmap::Entry::Occupied(entry) => entry.get().clone(), + dashmap::Entry::Vacant(entry) => { + self.overlay_builder.metrics.overlay_cache_misses.increment(1); + let overlay = self.overlay_builder.build_overlay(provider)?; + entry.insert(overlay.clone()); + overlay + } + }; + + Ok(overlay) } } @@ -465,8 +472,7 @@ where res }; - let Overlay { trie_updates, hashed_post_state } = - self.overlay_builder.build_overlay(&provider, &self.overlay_cache)?; + let Overlay { trie_updates, hashed_post_state } = self.get_overlay(&provider)?; let is_v2 = provider.cached_storage_settings().is_v2(); self.overlay_builder.metrics.database_provider_ro_duration.record(overall_start.elapsed()); diff --git a/crates/trie/parallel/src/root.rs b/crates/trie/parallel/src/root.rs index 9a209294ab2..bf504a12f7a 100644 --- a/crates/trie/parallel/src/root.rs +++ b/crates/trie/parallel/src/root.rs @@ -283,9 +283,10 @@ mod tests { async fn random_parallel_root() { let factory = create_test_provider_factory(); let changeset_cache = reth_trie_db::ChangesetCache::new(); + let overlay_builder = reth_provider::providers::OverlayBuilder::new(changeset_cache); let mut overlay_factory = reth_provider::providers::OverlayStateProviderFactory::new( factory.clone(), - reth_provider::providers::OverlayBuilder::new(changeset_cache), + overlay_builder.clone(), ); let mut rng = rand::rng(); @@ -362,9 +363,8 @@ mod tests { } let prefix_sets = hashed_state.construct_prefix_sets(); - let (factory, overlay_builder) = overlay_factory.into_parts(); overlay_factory = reth_provider::providers::OverlayStateProviderFactory::new( - factory, + factory.clone(), overlay_builder.with_hashed_state_overlay(Some(Arc::new(hashed_state.into_sorted()))), ); From 73f8609233963c73668dd68590295014b93789b6 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:41:07 +0000 Subject: [PATCH 07/20] refactor(provider): add historical overlay helper Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db0ad-77b5-76ba-8b0b-6683a56f63c5 Co-authored-by: Amp --- .../src/providers/state/historical.rs | 31 +++++++++++++++++-- .../provider/src/providers/state/overlay.rs | 8 ++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index f3bd49fd72d..ca75758ab5d 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -1,3 +1,4 @@ +use super::overlay::{Overlay, OverlayBuilder, OverlaySource}; use crate::{ AccountReader, BlockHashReader, ChangeSetReader, EitherReader, HashedPostStateProvider, ProviderError, RocksDBProviderFactory, StateProvider, StateRootProvider, @@ -13,8 +14,9 @@ use reth_db_api::{ }; use reth_primitives_traits::{Account, Bytecode}; use reth_storage_api::{ - BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StateProofProvider, - StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, + BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, PruneCheckpointReader, + StageCheckpointReader, StateProofProvider, StorageChangeSetReader, StorageRootProvider, + StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ @@ -307,6 +309,31 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block hashed_storage_from_reverts_with_provider(self.provider, address, self.block_number) } + #[allow(dead_code)] + fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult + where + Provider: + BlockHashReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache, + { + // Historical providers expose state at the start of `self.block_number`, so the overlay + // builder needs the previous canonical block hash to preserve those semantics. + let target_block = self.block_number.saturating_sub(1); + let block_hash = self + .provider + .block_hash(target_block)? + .ok_or_else(|| ProviderError::HeaderNotFound(target_block.into()))?; + + let TrieInputSorted { nodes, state, mut prefix_sets } = input; + let overlay_builder = OverlayBuilder::new(self._changeset_cache.clone()) + .with_block_hash(Some(block_hash)) + .with_overlay_source(Some(OverlaySource::Immediate { trie: nodes, state })); + let Overlay { trie_updates, hashed_post_state } = + overlay_builder.build_overlay(self.provider)?; + + prefix_sets.extend(hashed_post_state.construct_prefix_sets()); + Ok(TrieInputSorted::new(trie_updates, hashed_post_state, prefix_sets)) + } + /// Set the lowest block number at which the account history is available. pub const fn with_lowest_available_account_history_block_number( mut self, diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index f27e22ac57f..7b4af22fd45 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -51,9 +51,9 @@ pub(crate) struct OverlayStateProviderMetrics { /// Contains all fields required to initialize an [`OverlayStateProvider`]. #[derive(Debug, Clone)] -struct Overlay { - trie_updates: Arc, - hashed_post_state: Arc, +pub(super) struct Overlay { + pub(super) trie_updates: Arc, + pub(super) hashed_post_state: Arc, } /// Source of overlay data for [`OverlayStateProviderFactory`]. @@ -377,7 +377,7 @@ impl OverlayBuilder { /// Builds the effective overlay for the given provider. #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] - fn build_overlay(&self, provider: &Provider) -> ProviderResult + pub(super) fn build_overlay(&self, provider: &Provider) -> ProviderResult where Provider: StageCheckpointReader + PruneCheckpointReader From 2fde20fe2a3f1ed2baab419375ae530198b9c13f Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:44:44 +0000 Subject: [PATCH 08/20] refactor(provider): preserve historical overlay prefix sets Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db0ad-77b5-76ba-8b0b-6683a56f63c5 Co-authored-by: Amp --- crates/storage/provider/src/providers/state/historical.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index ca75758ab5d..895b78d6b62 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -323,14 +323,13 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block .block_hash(target_block)? .ok_or_else(|| ProviderError::HeaderNotFound(target_block.into()))?; - let TrieInputSorted { nodes, state, mut prefix_sets } = input; + let TrieInputSorted { nodes, state, prefix_sets } = input; let overlay_builder = OverlayBuilder::new(self._changeset_cache.clone()) .with_block_hash(Some(block_hash)) .with_overlay_source(Some(OverlaySource::Immediate { trie: nodes, state })); let Overlay { trie_updates, hashed_post_state } = overlay_builder.build_overlay(self.provider)?; - prefix_sets.extend(hashed_post_state.construct_prefix_sets()); Ok(TrieInputSorted::new(trie_updates, hashed_post_state, prefix_sets)) } From 56112da7a7c6a23c31ae8725d62930e319c72c61 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:46:51 +0000 Subject: [PATCH 09/20] refactor(provider): warn on old historical overlay reverts Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db0ad-77b5-76ba-8b0b-6683a56f63c5 Co-authored-by: Amp --- crates/storage/provider/src/providers/state/historical.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 895b78d6b62..6da98eab774 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -315,6 +315,14 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block Provider: BlockHashReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache, { + if self.check_distance_against_limit(EPOCH_SLOTS)? { + tracing::warn!( + target: "providers::historical_sp", + target = self.block_number, + "Attempt to calculate state root for an old block might result in OOM" + ); + } + // Historical providers expose state at the start of `self.block_number`, so the overlay // builder needs the previous canonical block hash to preserve those semantics. let target_block = self.block_number.saturating_sub(1); From 93b05687297926fffb8966f3875d19def684e796 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:56:01 +0000 Subject: [PATCH 10/20] refactor(provider): use build_overlay in historical trie paths Co-Authored-By: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- .../src/providers/state/historical.rs | 181 +++++++++--------- 1 file changed, 91 insertions(+), 90 deletions(-) diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 6da98eab774..10eda6d9f51 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -25,13 +25,12 @@ use reth_trie::{ trie_cursor::InMemoryTrieCursorFactory, updates::TrieUpdates, witness::TrieWitness, - AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, - KeccakKeyHasher, MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageRoot, - TrieInput, TrieInputSorted, + AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, KeccakKeyHasher, + MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageRoot, TrieInput, + TrieInputSorted, }; use reth_trie_db::{ - hashed_storage_from_reverts_with_provider, ChangesetCache, DatabaseProof, DatabaseStateRoot, - DatabaseStorageProof, DatabaseStorageRoot, + ChangesetCache, DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, }; use std::fmt::Debug; @@ -267,49 +266,6 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block Ok(tip.saturating_sub(self.block_number) > limit) } - /// Retrieve revert hashed state for this history provider. - fn revert_state(&self) -> ProviderResult - where - Provider: StorageSettingsCache, - { - if !self.lowest_available_blocks.is_account_history_available(self.block_number) || - !self.lowest_available_blocks.is_storage_history_available(self.block_number) - { - return Err(ProviderError::StateAtBlockPruned(self.block_number)) - } - - if self.check_distance_against_limit(EPOCH_SLOTS)? { - tracing::warn!( - target: "providers::historical_sp", - target = self.block_number, - "Attempt to calculate state root for an old block might result in OOM" - ); - } - - reth_trie_db::from_reverts_auto(self.provider, self.block_number..) - } - - /// Retrieve revert hashed storage for this history provider and target address. - fn revert_storage(&self, address: Address) -> ProviderResult - where - Provider: StorageSettingsCache, - { - if !self.lowest_available_blocks.is_storage_history_available(self.block_number) { - return Err(ProviderError::StateAtBlockPruned(self.block_number)) - } - - if self.check_distance_against_limit(EPOCH_SLOTS * 10)? { - tracing::warn!( - target: "providers::historical_sp", - target = self.block_number, - "Attempt to calculate storage root for an old block might result in OOM" - ); - } - - hashed_storage_from_reverts_with_provider(self.provider, address, self.block_number) - } - - #[allow(dead_code)] fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult where Provider: @@ -424,26 +380,25 @@ impl< + ChangeSetReader + StorageChangeSetReader + BlockNumReader + + BlockHashReader + + PruneCheckpointReader + + StageCheckpointReader + StorageSettingsCache, > StateRootProvider for HistoricalStateProviderRef<'_, Provider> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut revert_state = self.revert_state()?; - let hashed_state_sorted = hashed_state.into_sorted(); - revert_state.extend_ref_and_sort(&hashed_state_sorted); - Ok(>::overlay_root(self.tx(), &revert_state)?) + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(hashed_state), + ))?; + Ok(>::overlay_root_from_nodes(self.tx(), input)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut input = input; - input.prepend(self.revert_state()?.into()); - Ok(>::overlay_root_from_nodes( - self.tx(), - TrieInputSorted::from_unsorted(input), - )?) + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + Ok(>::overlay_root_from_nodes(self.tx(), input)?) }) } @@ -452,10 +407,10 @@ impl< hashed_state: HashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider, |A| { - let mut revert_state = self.revert_state()?; - let hashed_state_sorted = hashed_state.into_sorted(); - revert_state.extend_ref_and_sort(&hashed_state_sorted); - Ok(>::overlay_root_with_updates(self.tx(), &revert_state)?) + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(hashed_state), + ))?; + Ok(>::overlay_root_from_nodes_with_updates(self.tx(), input)?) }) } @@ -464,12 +419,8 @@ impl< input: TrieInput, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider, |A| { - let mut input = input; - input.prepend(self.revert_state()?.into()); - Ok(>::overlay_root_from_nodes_with_updates( - self.tx(), - TrieInputSorted::from_unsorted(input), - )?) + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + Ok(>::overlay_root_from_nodes_with_updates(self.tx(), input)?) }) } } @@ -479,6 +430,9 @@ impl< + ChangeSetReader + StorageChangeSetReader + BlockNumReader + + BlockHashReader + + PruneCheckpointReader + + StageCheckpointReader + StorageSettingsCache, > StorageRootProvider for HistoricalStateProviderRef<'_, Provider> { @@ -488,9 +442,20 @@ impl< hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut revert_storage = self.revert_storage(address)?; - revert_storage.extend(&hashed_storage); - >::overlay_root(self.tx(), address, revert_storage) + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + )), + ))?; + let hashed_storage = input + .state + .account_storages() + .get(&alloy_primitives::keccak256(address)) + .cloned() + .unwrap_or_default() + .into(); + >::overlay_root(self.tx(), address, hashed_storage) .map_err(|err| ProviderError::Database(err.into())) }) } @@ -502,13 +467,24 @@ impl< hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut revert_storage = self.revert_storage(address)?; - revert_storage.extend(&hashed_storage); + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + )), + ))?; + let hashed_storage = input + .state + .account_storages() + .get(&alloy_primitives::keccak256(address)) + .cloned() + .unwrap_or_default() + .into(); >::overlay_storage_proof( self.tx(), address, slot, - revert_storage, + hashed_storage, ) .map_err(ProviderError::from) }) @@ -521,13 +497,24 @@ impl< hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut revert_storage = self.revert_storage(address)?; - revert_storage.extend(&hashed_storage); + let input = self.build_overlay(TrieInputSorted::from_unsorted( + TrieInput::from_state(HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + )), + ))?; + let hashed_storage = input + .state + .account_storages() + .get(&alloy_primitives::keccak256(address)) + .cloned() + .unwrap_or_default() + .into(); >::overlay_storage_multiproof( self.tx(), address, slots, - revert_storage, + hashed_storage, ) .map_err(ProviderError::from) }) @@ -539,6 +526,9 @@ impl< + ChangeSetReader + StorageChangeSetReader + BlockNumReader + + BlockHashReader + + PruneCheckpointReader + + StageCheckpointReader + StorageSettingsCache, > StateProofProvider for HistoricalStateProviderRef<'_, Provider> { @@ -550,8 +540,12 @@ impl< slots: &[B256], ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut input = input; - input.prepend(self.revert_state()?.into()); + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = TrieInput::new( + (*input.nodes).clone().into(), + (*input.state).clone().into(), + input.prefix_sets, + ); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) }) @@ -563,8 +557,12 @@ impl< targets: MultiProofTargets, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let mut input = input; - input.prepend(self.revert_state()?.into()); + let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = TrieInput::new( + (*input.nodes).clone().into(), + (*input.state).clone().into(), + input.prefix_sets, + ); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_multiproof(input, targets).map_err(ProviderError::from) }) @@ -577,21 +575,19 @@ impl< mode: ExecutionWitnessMode, ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider, |A| { - let mut input = input; - input.prepend(self.revert_state()?.into()); - let nodes_sorted = input.nodes.into_sorted(); - let state_sorted = input.state.into_sorted(); + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; let witness = TrieWitness::new( InMemoryTrieCursorFactory::new( reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()), - &nodes_sorted, + nodes.as_ref(), ), HashedPostStateCursorFactory::new( reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()), - &state_sorted, + state.as_ref(), ), ) - .with_prefix_sets_mut(input.prefix_sets) + .with_prefix_sets_mut(prefix_sets) .with_execution_witness_mode(mode); let witness = if mode.is_canonical() { witness } else { witness.always_include_root_node() }; @@ -618,6 +614,8 @@ impl< + BlockHashReader + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider, @@ -704,7 +702,7 @@ impl where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]); +reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]); /// Lowest blocks at which different parts of the state are available. /// They may be [Some] if pruning is enabled. @@ -837,7 +835,8 @@ mod tests { use reth_primitives_traits::{Account, StorageEntry}; use reth_storage_api::{ BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, - NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache, + NodePrimitivesProvider, PruneCheckpointReader, StageCheckpointReader, + StorageChangeSetReader, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderError; use reth_trie_db::ChangesetCache; @@ -855,6 +854,8 @@ mod tests { + BlockHashReader + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider, From 79fc5c5d7c6be6b29fea091fdb65f12c5a117fbb Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:11:16 +0000 Subject: [PATCH 11/20] fix(engine): remove redundant overlay-builder clones Co-Authored-By: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- crates/engine/tree/src/tree/payload_validator.rs | 6 +----- crates/trie/parallel/src/root.rs | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index b2b0c1ddb8c..8294cfcfb91 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -772,11 +772,7 @@ where } let (root, updates) = ensure_ok_post_block!( - Self::compute_state_root_serial( - provider_factory.clone(), - overlay_builder.clone(), - &hashed_state, - ), + Self::compute_state_root_serial(provider_factory, overlay_builder, &hashed_state), block ); diff --git a/crates/trie/parallel/src/root.rs b/crates/trie/parallel/src/root.rs index bf504a12f7a..ba586d4b99c 100644 --- a/crates/trie/parallel/src/root.rs +++ b/crates/trie/parallel/src/root.rs @@ -364,7 +364,7 @@ mod tests { let prefix_sets = hashed_state.construct_prefix_sets(); overlay_factory = reth_provider::providers::OverlayStateProviderFactory::new( - factory.clone(), + factory, overlay_builder.with_hashed_state_overlay(Some(Arc::new(hashed_state.into_sorted()))), ); From 5561eb0c17be2286407de6e24fe2281e5802f067 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:22:34 +0000 Subject: [PATCH 12/20] fix(provider): avoid db-tip lookup for unanchored overlays Co-Authored-By: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- crates/storage/provider/src/providers/state/overlay.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index 7b4af22fd45..baeac82f3d2 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -431,6 +431,10 @@ impl OverlayStateProviderFactory { + BlockNumReader + StorageSettingsCache, { + if self.overlay_builder.block_hash.is_none() { + return self.overlay_builder.build_overlay(provider) + } + let db_tip_block = self.overlay_builder.get_db_tip_block_number(provider)?; let overlay = match self.overlay_cache.entry(db_tip_block) { From 5357dc9fea6e066883cd4d0eadff8e2ddd54b0fe Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Wed, 22 Apr 2026 11:31:54 +0000 Subject: [PATCH 13/20] fix(engine): use state provider for serial state root fallback Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db4f1-b5cb-75ee-b759-0c6badfe0ff6 Co-authored-by: Amp --- .../engine/tree/src/tree/payload_validator.rs | 61 +++++++------------ 1 file changed, 22 insertions(+), 39 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 8294cfcfb91..78aeecd5724 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -80,11 +80,11 @@ use reth_provider::{ providers::{OverlayBuilder, OverlayStateProviderFactory}, BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, - StageCheckpointReader, StateProvider, StateProviderFactory, StateReader, + StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; -use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot}; +use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState}; use reth_trie_db::ChangesetCache; use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; use revm_primitives::{Address, KECCAK_EMPTY}; @@ -470,6 +470,7 @@ where ) .into()) }; + let serial_state_provider_builder = provider_builder.clone(); let mut state_provider = ensure_ok!(provider_builder.build()); drop(_enter); @@ -667,8 +668,7 @@ where let task_result = ensure_ok_post_block!( self.await_state_root_with_timeout( &mut handle, - provider_factory.clone(), - overlay_builder.clone(), + serial_state_provider_builder.clone(), &hashed_state, ), block @@ -692,6 +692,7 @@ where // Compare trie updates with serial computation if configured if self.config.always_compare_trie_updates() { let _has_diff = self.compare_trie_updates_with_serial( + serial_state_provider_builder.clone(), provider_factory.clone(), overlay_builder.clone(), &hashed_state, @@ -772,7 +773,9 @@ where } let (root, updates) = ensure_ok_post_block!( - Self::compute_state_root_serial(provider_factory, overlay_builder, &hashed_state), + serial_state_provider_builder + .build() + .and_then(|provider| Self::compute_state_root_serial(provider, &hashed_state)), block ); @@ -1115,29 +1118,13 @@ where /// Compute state root for the given hashed post state in serial. /// - /// Uses an overlay factory which provides the state of the parent block, along with the - /// [`HashedPostState`] containing the changes of this block, to compute the state root and - /// trie updates for this block. + /// Uses the same provider construction path as main execution and computes the state root and + /// trie updates for this block directly via [`StateProvider::state_root_with_updates`]. fn compute_state_root_serial( - provider_factory: P, - overlay_builder: OverlayBuilder, + state_provider: StateProviderBox, hashed_state: &LazyHashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { - let hashed_state = hashed_state.get(); - // The `hashed_state` argument will be taken into account as part of the overlay, but we - // need to use the prefix sets which were generated from it to indicate to the - // StateRoot which parts of the trie need to be recomputed. - let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let overlay_factory = OverlayStateProviderFactory::new( - provider_factory, - overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()), - ); - - let provider = overlay_factory.database_provider_ro()?; - - Ok(StateRoot::new(&provider, &provider) - .with_prefix_sets(prefix_sets) - .root_with_updates()?) + state_provider.state_root_with_updates(hashed_state.get().clone()) } /// Awaits the state root from the background task, with an optional timeout fallback. @@ -1162,8 +1149,7 @@ where fn await_state_root_with_timeout( &self, handle: &mut PayloadHandle, - provider_factory: P, - overlay_builder: OverlayBuilder, + state_provider_builder: StateProviderBuilder, hashed_state: &LazyHashedPostState, ) -> ProviderResult> { let Some(timeout) = self.config.state_root_task_timeout() else { @@ -1188,15 +1174,12 @@ where let (seq_tx, seq_rx) = std::sync::mpsc::channel::>(); - let seq_provider_factory = provider_factory; - let seq_overlay_builder = overlay_builder; + let seq_state_provider_builder = state_provider_builder; let seq_hashed_state = hashed_state.clone(); self.payload_processor.executor().spawn_blocking_named("serial-root", move || { - let result = Self::compute_state_root_serial( - seq_provider_factory, - seq_overlay_builder, - &seq_hashed_state, - ); + let result = seq_state_provider_builder.build().and_then(|provider| { + Self::compute_state_root_serial(provider, &seq_hashed_state) + }); let _ = seq_tx.send(result); }); @@ -1260,6 +1243,7 @@ where /// updates. fn compare_trie_updates_with_serial( &self, + state_provider_builder: StateProviderBuilder, provider_factory: P, overlay_builder: OverlayBuilder, hashed_state: &LazyHashedPostState, @@ -1267,11 +1251,10 @@ where ) -> bool { debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation"); - match Self::compute_state_root_serial( - provider_factory.clone(), - overlay_builder.clone(), - hashed_state, - ) { + match state_provider_builder + .build() + .and_then(|provider| Self::compute_state_root_serial(provider, hashed_state)) + { Ok((serial_root, serial_trie_updates)) => { debug!( target: "engine::tree::payload_validator", From 93cb59ef7a3f4e6c9f81654ebca0afdc36611249 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Wed, 22 Apr 2026 11:38:08 +0000 Subject: [PATCH 14/20] refactor(engine): inline serial provider builder reuse Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db4f1-b5cb-75ee-b759-0c6badfe0ff6 Co-authored-by: Amp --- crates/engine/tree/src/tree/payload_validator.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 78aeecd5724..888788590b4 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -470,7 +470,6 @@ where ) .into()) }; - let serial_state_provider_builder = provider_builder.clone(); let mut state_provider = ensure_ok!(provider_builder.build()); drop(_enter); @@ -536,7 +535,7 @@ where let mut handle = ensure_ok!(self.spawn_payload_processor( env.clone(), txs, - provider_builder, + provider_builder.clone(), overlay_factory.clone(), strategy, block_access_list, @@ -668,7 +667,7 @@ where let task_result = ensure_ok_post_block!( self.await_state_root_with_timeout( &mut handle, - serial_state_provider_builder.clone(), + provider_builder.clone(), &hashed_state, ), block @@ -692,7 +691,7 @@ where // Compare trie updates with serial computation if configured if self.config.always_compare_trie_updates() { let _has_diff = self.compare_trie_updates_with_serial( - serial_state_provider_builder.clone(), + provider_builder.clone(), provider_factory.clone(), overlay_builder.clone(), &hashed_state, @@ -773,7 +772,7 @@ where } let (root, updates) = ensure_ok_post_block!( - serial_state_provider_builder + provider_builder .build() .and_then(|provider| Self::compute_state_root_serial(provider, &hashed_state)), block @@ -1174,10 +1173,9 @@ where let (seq_tx, seq_rx) = std::sync::mpsc::channel::>(); - let seq_state_provider_builder = state_provider_builder; let seq_hashed_state = hashed_state.clone(); self.payload_processor.executor().spawn_blocking_named("serial-root", move || { - let result = seq_state_provider_builder.build().and_then(|provider| { + let result = state_provider_builder.build().and_then(|provider| { Self::compute_state_root_serial(provider, &seq_hashed_state) }); let _ = seq_tx.send(result); From 4b9af81f3fee22ec5d8d5860359a0e413989a6e8 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:04:29 +0000 Subject: [PATCH 15/20] refactor(provider): restore overlay comments Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db4f1-b5cb-75ee-b759-0c6badfe0ff6 Co-authored-by: Amp --- .../provider/src/providers/state/overlay.rs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index baeac82f3d2..cdec14e20ee 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -91,13 +91,13 @@ impl OverlaySource { /// collecting reverts. It is intentionally independent from any provider factory or overlay cache. #[derive(Debug, Clone)] pub struct OverlayBuilder { - /// Optional block hash for collecting reverts. + /// Optional block hash for collecting reverts block_hash: Option, /// Optional overlay source (lazy or immediate). overlay_source: Option, - /// Changeset cache handle for retrieving trie changesets. + /// Changeset cache handle for retrieving trie changesets changeset_cache: ChangesetCache, - /// Metrics for tracking provider operations. + /// Metrics for tracking provider operations metrics: OverlayStateProviderMetrics, } @@ -161,6 +161,7 @@ impl OverlayBuilder { Arc::make_mut(state).extend_ref_and_sort(&other); } Some(OverlaySource::Lazy(lazy)) => { + // Resolve lazy overlay and convert to immediate with extension let (trie, mut state) = lazy.as_overlay(); Arc::make_mut(&mut state).extend_ref_and_sort(&other); self.overlay_source = Some(OverlaySource::Immediate { trie, state }); @@ -234,10 +235,15 @@ impl OverlayBuilder { where Provider: PruneCheckpointReader, { + // If the requested block is the DB tip then there won't be any reverts necessary, and we + // can simply return Ok. if db_tip_block == requested_block { return Ok(false) } + // Check account history prune checkpoint to determine the lower bound of available data. + // The prune checkpoint's block_number is the highest pruned block, so data is available + // starting from the next block. let prune_checkpoint = provider.get_prune_checkpoint(PruneSegment::AccountHistory)?; let lower_bound = prune_checkpoint .and_then(|chk| chk.block_number) @@ -246,6 +252,7 @@ impl OverlayBuilder { let available_range = lower_bound..=db_tip_block; + // Check if the requested block is within the available range if !available_range.contains(&requested_block) { return Err(ProviderError::InsufficientChangesets { requested: requested_block, @@ -277,11 +284,15 @@ impl OverlayBuilder { + PruneCheckpointReader + StorageSettingsCache, { + // + // Set up variables we'll use for recording metrics. There's two different code-paths here, + // and we want to make sure both record metrics, so we do metrics recording after. let retrieve_trie_reverts_duration; let retrieve_hashed_state_reverts_duration; let trie_updates_total_len; let hashed_state_updates_total_len; + // If block_hash is provided, collect reverts let (trie_updates, hashed_post_state) = if let Some(from_block) = self.get_requested_block_number(provider)? && self.reverts_required(provider, db_tip_block, from_block)? @@ -296,12 +307,16 @@ impl OverlayBuilder { "Collecting trie reverts for overlay state provider" ); + // Collect trie reverts using changeset cache let mut trie_reverts = { let _guard = debug_span!(target: "providers::state::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); + + // Use changeset cache to retrieve and accumulate reverts to restore state after + // from_block let accumulated_reverts = self .changeset_cache .get_or_compute_range(provider, (from_block + 1)..=db_tip_block)?; @@ -310,6 +325,7 @@ impl OverlayBuilder { accumulated_reverts }; + // Collect state reverts let mut hashed_state_reverts = { let _guard = debug_span!(target: "providers::state::overlay", "retrieving_hashed_state_reverts").entered(); @@ -319,6 +335,8 @@ impl OverlayBuilder { res }; + // Resolve overlays (lazy or immediate) and extend reverts with them. + // If reverts are empty, use overlays directly to avoid cloning. let (overlay_trie, overlay_state) = self.resolve_overlays(); let trie_updates = if trie_reverts.is_empty() { @@ -353,6 +371,7 @@ impl OverlayBuilder { (trie_updates, hashed_state_updates) } else { + // If no block_hash, use overlays directly (resolving lazy if set) let (trie_updates, hashed_state) = self.resolve_overlays(); retrieve_trie_reverts_duration = Duration::ZERO; @@ -363,6 +382,7 @@ impl OverlayBuilder { (trie_updates, hashed_state) }; + // Record metrics self.metrics .retrieve_trie_reverts_duration .record(retrieve_trie_reverts_duration.as_secs_f64()); @@ -431,6 +451,7 @@ impl OverlayStateProviderFactory { + BlockNumReader + StorageSettingsCache, { + // No anchor block — just resolve the in-memory overlay directly. if self.overlay_builder.block_hash.is_none() { return self.overlay_builder.build_overlay(provider) } From 7fd39e6ac27df4b06174871b7149eee0795a40a3 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:06:59 +0000 Subject: [PATCH 16/20] refactor(provider): rename historical changeset cache field Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db4f1-b5cb-75ee-b759-0c6badfe0ff6 Co-authored-by: Amp --- crates/storage/provider/src/providers/state/historical.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 10eda6d9f51..c7ab98d5fa8 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -125,7 +125,7 @@ pub struct HistoricalStateProviderRef<'b, Provider> { /// Database provider provider: &'b Provider, /// Changeset cache handle for retrieving trie changesets. - _changeset_cache: ChangesetCache, + changeset_cache: ChangesetCache, /// Block number is main index for the history state of accounts and storages. block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. @@ -143,7 +143,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block ) -> Self { Self { provider, - _changeset_cache: changeset_cache, + changeset_cache, block_number, lowest_available_blocks: Default::default(), } @@ -157,7 +157,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block lowest_available_blocks: LowestAvailableBlocks, changeset_cache: ChangesetCache, ) -> Self { - Self { provider, _changeset_cache: changeset_cache, block_number, lowest_available_blocks } + Self { provider, changeset_cache, block_number, lowest_available_blocks } } /// Lookup an account in the `AccountsHistory` table using `EitherReader`. @@ -288,7 +288,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block .ok_or_else(|| ProviderError::HeaderNotFound(target_block.into()))?; let TrieInputSorted { nodes, state, prefix_sets } = input; - let overlay_builder = OverlayBuilder::new(self._changeset_cache.clone()) + let overlay_builder = OverlayBuilder::new(self.changeset_cache.clone()) .with_block_hash(Some(block_hash)) .with_overlay_source(Some(OverlaySource::Immediate { trie: nodes, state })); let Overlay { trie_updates, hashed_post_state } = From ef63a7a09d6bbde1e808562d4679394bd9ee7c49 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:09:38 +0000 Subject: [PATCH 17/20] fix(provider): avoid extra clones in historical proofs Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db4f1-b5cb-75ee-b759-0c6badfe0ff6 Co-authored-by: Amp --- .../src/providers/state/historical.rs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index c7ab98d5fa8..d515e490660 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -33,7 +33,7 @@ use reth_trie_db::{ ChangesetCache, DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, }; -use std::fmt::Debug; +use std::{fmt::Debug, sync::Arc}; type DbStateRoot<'a, TX, A> = StateRoot< reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, @@ -540,11 +540,12 @@ impl< slots: &[B256], ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; let input = TrieInput::new( - (*input.nodes).clone().into(), - (*input.state).clone().into(), - input.prefix_sets, + Arc::unwrap_or_clone(nodes).into(), + Arc::unwrap_or_clone(state).into(), + prefix_sets, ); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) @@ -557,11 +558,12 @@ impl< targets: MultiProofTargets, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider, |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let TrieInputSorted { nodes, state, prefix_sets } = + self.build_overlay(TrieInputSorted::from_unsorted(input))?; let input = TrieInput::new( - (*input.nodes).clone().into(), - (*input.state).clone().into(), - input.prefix_sets, + Arc::unwrap_or_clone(nodes).into(), + Arc::unwrap_or_clone(state).into(), + prefix_sets, ); let proof = as DatabaseProof>::from_tx(self.tx()); proof.overlay_multiproof(input, targets).map_err(ProviderError::from) From 2bb7e7ff15cf9e2fad218a9edb187b62b15ae19f Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:39:22 +0000 Subject: [PATCH 18/20] fix(engine): address payload validator CI failures Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db5d3-fb9b-7309-b8e9-dfc2ca6290a7 Co-authored-by: Amp --- crates/engine/tree/src/tree/payload_validator.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 888788590b4..d08a8aedeb7 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -692,8 +692,8 @@ where if self.config.always_compare_trie_updates() { let _has_diff = self.compare_trie_updates_with_serial( provider_builder.clone(), - provider_factory.clone(), - overlay_builder.clone(), + provider_factory, + overlay_builder, &hashed_state, trie_updates.as_ref().clone(), ); @@ -733,8 +733,8 @@ where StateRootStrategy::Parallel => { debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm"); match self.compute_state_root_parallel( - provider_factory.clone(), - overlay_builder.clone(), + provider_factory, + overlay_builder, &hashed_state, ) { Ok(result) => { @@ -1118,7 +1118,8 @@ where /// Compute state root for the given hashed post state in serial. /// /// Uses the same provider construction path as main execution and computes the state root and - /// trie updates for this block directly via [`StateProvider::state_root_with_updates`]. + /// trie updates for this block directly via + /// [`reth_provider::StateRootProvider::state_root_with_updates`]. fn compute_state_root_serial( state_provider: StateProviderBox, hashed_state: &LazyHashedPostState, From 17d58a73d5b9e8d4b45541f5c4e67e61c0952ffa Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Thu, 23 Apr 2026 12:26:28 +0000 Subject: [PATCH 19/20] fix(provider): wait for readers before unwind static-file commit Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db9e5-51dc-763e-b1ec-4e09fbc9e8e8 Co-authored-by: Amp --- .../provider/src/providers/database/provider.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index bb65a369f16..8ef4a4f0ae5 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -264,8 +264,8 @@ impl DatabaseProvider { /// This keeps MDBX as the first durable step so an interrupted unwind can be recovered by /// truncating static files from checkpoints on the next startup. /// - /// For `storage_v2`, this waits after the MDBX commit so readers holding older MDBX-visible - /// views cannot overlap the `RocksDB` unwind. + /// This waits after the MDBX commit so readers holding older MDBX-visible views cannot overlap + /// later cross-store unwind steps. /// /// Historical `storage_v2` reads ignore `RocksDB` history entries above their MDBX-visible tip, /// so no additional post-`RocksDB` wait is needed before static-file commit. @@ -274,11 +274,11 @@ impl DatabaseProvider { let reader_txn_tracker = self.reader_txn_tracker.clone(); self.tx.commit()?; - if storage_v2 { - if let Some(reader_txn_tracker) = reader_txn_tracker.as_ref() { - reader_txn_tracker.wait_for_pre_commit_readers(); - } + if let Some(reader_txn_tracker) = reader_txn_tracker.as_ref() { + reader_txn_tracker.wait_for_pre_commit_readers(); + } + if storage_v2 { let batches = std::mem::take(&mut *self.pending_rocksdb_batches.lock()); for batch in batches { self.rocksdb_provider.commit_batch(batch)?; @@ -3961,7 +3961,6 @@ mod tests { #[test] fn unwind_commit_waits_for_pre_commit_readers() { let factory = create_test_provider_factory(); - factory.set_storage_settings_cache(StorageSettings::v2()); let reader = factory.provider().unwrap(); let provider_rw = factory.unwind_provider_rw().unwrap(); From 63f3b1dc978e1d9833094c2031d474d963b16473 Mon Sep 17 00:00:00 2001 From: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Date: Thu, 23 Apr 2026 12:50:04 +0000 Subject: [PATCH 20/20] test(stages): drop reader before execution unwind Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Amp-Thread-ID: https://ampcode.com/threads/T-019db9e5-51dc-763e-b1ec-4e09fbc9e8e8 Co-authored-by: Amp --- .../stages/stages/src/stages/execution/mod.rs | 75 ++++++++++--------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index 2a05915391d..bfd3b87104a 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -1020,41 +1020,46 @@ mod tests { done: true } if processed == total && total == block.gas_used); - let provider = factory.provider().unwrap(); - - // check post state - let account1 = address!("0x1000000000000000000000000000000000000000"); - let account1_info = - Account { balance: U256::ZERO, nonce: 0x00, bytecode_hash: Some(code_hash) }; - let account2 = address!("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba"); - let account2_info = Account { - balance: U256::from(0x1bc16d674ece94bau128), - nonce: 0x00, - bytecode_hash: None, - }; - let account3 = address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"); - let account3_info = Account { - balance: U256::from(0x3635c9adc5de996b46u128), - nonce: 0x01, - bytecode_hash: None, - }; - - // assert accounts - assert!( - matches!(provider.basic_account(&account1), Ok(Some(acc)) if acc == account1_info) - ); - assert!( - matches!(provider.basic_account(&account2), Ok(Some(acc)) if acc == account2_info) - ); - assert!( - matches!(provider.basic_account(&account3), Ok(Some(acc)) if acc == account3_info) - ); - // assert storage - // Get on dupsort would return only first value. This is good enough for this test. - assert!(matches!( - provider.tx_ref().get::(account1), - Ok(Some(entry)) if entry.key == B256::with_last_byte(1) && entry.value == U256::from(2) - )); + { + let provider = factory.provider().unwrap(); + + // check post state + let account1 = address!("0x1000000000000000000000000000000000000000"); + let account1_info = + Account { balance: U256::ZERO, nonce: 0x00, bytecode_hash: Some(code_hash) }; + let account2 = address!("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba"); + let account2_info = Account { + balance: U256::from(0x1bc16d674ece94bau128), + nonce: 0x00, + bytecode_hash: None, + }; + let account3 = address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"); + let account3_info = Account { + balance: U256::from(0x3635c9adc5de996b46u128), + nonce: 0x01, + bytecode_hash: None, + }; + + // assert accounts + assert!(matches!( + provider.basic_account(&account1), + Ok(Some(acc)) if acc == account1_info + )); + assert!(matches!( + provider.basic_account(&account2), + Ok(Some(acc)) if acc == account2_info + )); + assert!(matches!( + provider.basic_account(&account3), + Ok(Some(acc)) if acc == account3_info + )); + // assert storage + // Get on dupsort would return only first value. This is good enough for this test. + assert!(matches!( + provider.tx_ref().get::(account1), + Ok(Some(entry)) if entry.key == B256::with_last_byte(1) && entry.value == U256::from(2) + )); + } let mut provider = factory.database_provider_rw().unwrap(); let mut stage = stage();