From c6e86c8f74e9f85a4e78c332ca9f681f5cdcae77 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Tue, 30 Sep 2025 12:16:03 +0100 Subject: [PATCH 01/13] feat(storage): read headers and transactions only from static files --- crates/engine/tree/src/tree/mod.rs | 4 +- .../engine/tree/src/tree/payload_validator.rs | 2 +- crates/exex/exex/src/manager.rs | 2 +- crates/exex/exex/src/notifications.rs | 2 +- crates/rpc/rpc-eth-types/src/cache/mod.rs | 2 +- crates/rpc/rpc/src/debug.rs | 2 +- .../src/providers/blockchain_provider.rs | 24 +- .../provider/src/providers/consistent.rs | 20 +- .../provider/src/providers/database/mod.rs | 77 ++---- .../src/providers/database/provider.rs | 243 +++--------------- .../provider/src/providers/static_file/jar.rs | 12 +- .../src/providers/static_file/manager.rs | 33 ++- .../provider/src/providers/static_file/mod.rs | 4 +- .../storage/provider/src/test_utils/mock.rs | 12 +- crates/storage/rpc-provider/src/lib.rs | 18 +- crates/storage/storage-api/src/block.rs | 11 +- crates/storage/storage-api/src/header.rs | 10 +- crates/storage/storage-api/src/noop.rs | 8 +- examples/db-access/src/main.rs | 2 +- 19 files changed, 170 insertions(+), 318 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 498907bb991..24bdc069f09 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2659,7 +2659,7 @@ where let mut canonical = self.canonical_in_memory_state.header_by_hash(hash); if canonical.is_none() { - canonical = self.provider.header(&hash)?.map(|header| SealedHeader::new(header, hash)); + canonical = self.provider.header(hash)?.map(|header| SealedHeader::new(header, hash)); } Ok(canonical) @@ -2883,7 +2883,7 @@ where } // Check if the block is persisted - if let Some(header) = self.provider.header(&hash)? { + if let Some(header) = self.provider.header(hash)? { debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); // For persisted blocks, we create a builder that will fetch state directly from the // database diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index b0ef9d2cec8..9ebd88fc88d 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -890,7 +890,7 @@ where } // Check if the block is persisted - if let Some(header) = self.provider.header(&hash)? { + if let Some(header) = self.provider.header(hash)? { debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); // For persisted blocks, we create a builder that will fetch state directly from the // database diff --git a/crates/exex/exex/src/manager.rs b/crates/exex/exex/src/manager.rs index 9ebf69457b6..99694f0a51b 100644 --- a/crates/exex/exex/src/manager.rs +++ b/crates/exex/exex/src/manager.rs @@ -370,7 +370,7 @@ where .map(|(exex_id, num_hash)| { num_hash.map_or(Ok((exex_id, num_hash, false)), |num_hash| { self.provider - .is_known(&num_hash.hash) + .is_known(num_hash.hash) // Save the ExEx ID, finished height, and whether the hash is canonical .map(|is_canonical| (exex_id, Some(num_hash), is_canonical)) }) diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index 0def7a510ed..c6a54e647cf 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -308,7 +308,7 @@ where /// we're not on the canonical chain and we need to revert the notification with the ExEx /// head block. fn check_canonical(&mut self) -> eyre::Result>> { - if self.provider.is_known(&self.initial_exex_head.block.hash)? && + if self.provider.is_known(self.initial_exex_head.block.hash)? && self.initial_exex_head.block.number <= self.initial_local_head.number { // we have the targeted block and that block is below the current head diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index 6df612261d9..e0bea2cf463 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -539,7 +539,7 @@ where this.action_task_spawner.spawn_blocking(Box::pin(async move { // Acquire permit let _permit = rate_limiter.acquire().await; - let header = provider.header(&block_hash).and_then(|header| { + let header = provider.header(block_hash).and_then(|header| { header.ok_or_else(|| { ProviderError::HeaderNotFound(block_hash.into()) }) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index e0f5b4eabce..b3715c0e8e0 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -917,7 +917,7 @@ where /// Handler for `debug_getRawHeader` async fn raw_header(&self, block_id: BlockId) -> RpcResult { let header = match block_id { - BlockId::Hash(hash) => self.provider().header(&hash.into()).to_rpc_result()?, + BlockId::Hash(hash) => self.provider().header(hash.into()).to_rpc_result()?, BlockId::Number(number_or_tag) => { let number = self .provider() diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index c5ba6890ee8..890b98124a5 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -182,7 +182,7 @@ impl StaticFileProviderFactory for BlockchainProvider { impl HeaderProvider for BlockchainProvider { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.consistent_provider()?.header(block_hash) } @@ -190,7 +190,7 @@ impl HeaderProvider for BlockchainProvider { self.consistent_provider()?.header_by_number(num) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { + fn header_td(&self, hash: BlockHash) -> ProviderResult> { self.consistent_provider()?.header_td(hash) } @@ -342,6 +342,10 @@ impl BlockReader for BlockchainProvider { ) -> ProviderResult>> { self.consistent_provider()?.recovered_block_range(range) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + self.consistent_provider()?.block_by_transaction_id(id) + } } impl TransactionsProvider for BlockchainProvider { @@ -2290,13 +2294,15 @@ mod tests { let test_tx_index = 0; test_non_range!([ - // TODO: header should use B256 like others instead of &B256 - // ( - // ONE, - // header, - // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), - // (&B256::random()) - // ), + ( + ONE, + header, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.hash(), + Some(block.header().clone()) + ), + B256::random() + ), ( ONE, header_by_number, diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index cd74ab36965..03615d5357b 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -646,9 +646,9 @@ impl StaticFileProviderFactory for ConsistentProvider { impl HeaderProvider for ConsistentProvider { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.get_in_memory_or_storage_by_block( - (*block_hash).into(), + block_hash.into(), |db_provider| db_provider.header(block_hash), |block_state| Ok(Some(block_state.block_ref().recovered_block().clone_header())), ) @@ -662,8 +662,8 @@ impl HeaderProvider for ConsistentProvider { ) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(*hash)? { + fn header_td(&self, hash: BlockHash) -> ProviderResult> { + if let Some(num) = self.block_number(hash)? { self.header_td_by_number(num) } else { Ok(None) @@ -917,6 +917,14 @@ impl BlockReader for ConsistentProvider { |_| true, ) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + id.into(), + |db_provider| db_provider.block_by_transaction_id(id), + |_, _, block_state| Ok(Some(block_state.number())), + ) + } } impl TransactionsProvider for ConsistentProvider { @@ -1305,14 +1313,14 @@ impl BlockReaderIdExt for ConsistentProvider { ) -> ProviderResult>>> { Ok(match id { BlockId::Number(num) => self.sealed_header_by_number_or_tag(num)?, - BlockId::Hash(hash) => self.header(&hash.block_hash)?.map(SealedHeader::seal_slow), + BlockId::Hash(hash) => self.header(hash.block_hash)?.map(SealedHeader::seal_slow), }) } fn header_by_id(&self, id: BlockId) -> ProviderResult>> { Ok(match id { BlockId::Number(num) => self.header_by_number_or_tag(num)?, - BlockId::Hash(hash) => self.header(&hash.block_hash)?, + BlockId::Hash(hash) => self.header(hash.block_hash)?, }) } } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index c9a19936af8..54642a94757 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -234,57 +234,41 @@ impl HeaderSyncGapProvider for ProviderFactory { impl HeaderProvider for ProviderFactory { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.provider()?.header(block_hash) } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - num, - |static_file| static_file.header_by_number(num), - || self.provider()?.header_by_number(num), - ) + self.static_file_provider.header_by_number(num) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { + fn header_td(&self, hash: BlockHash) -> ProviderResult> { self.provider()?.header_td(hash) } fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - self.provider()?.header_td_by_number(number) + self.static_file_provider.header_td_by_number(number) } fn headers_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, _| static_file.headers_range(range), - |range, _| self.provider()?.headers_range(range), - |_| true, - ) + self.static_file_provider.headers_range(range) } fn sealed_header( &self, number: BlockNumber, ) -> ProviderResult>> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.sealed_header(number), - || self.provider()?.sealed_header(number), - ) + self.static_file_provider.sealed_header(number) } fn sealed_headers_range( &self, range: impl RangeBounds, ) -> ProviderResult>> { - self.sealed_headers_while(range, |_| true) + self.static_file_provider.sealed_headers_range(range) } fn sealed_headers_while( @@ -292,24 +276,13 @@ impl HeaderProvider for ProviderFactory { range: impl RangeBounds, predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult>> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, predicate| static_file.sealed_headers_while(range, predicate), - |range, predicate| self.provider()?.sealed_headers_while(range, predicate), - predicate, - ) + self.static_file_provider.sealed_headers_while(range, predicate) } } impl BlockHashReader for ProviderFactory { fn block_hash(&self, number: u64) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.block_hash(number), - || self.provider()?.block_hash(number), - ) + self.static_file_provider.block_hash(number) } fn canonical_hashes_range( @@ -317,13 +290,7 @@ impl BlockHashReader for ProviderFactory { start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - start..end, - |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), - |range, _| self.provider()?.canonical_hashes_range(range.start, range.end), - |_| true, - ) + self.static_file_provider.canonical_hashes_range(start, end) } } @@ -337,7 +304,7 @@ impl BlockNumReader for ProviderFactory { } fn last_block_number(&self) -> ProviderResult { - self.provider()?.last_block_number() + self.static_file_provider.last_block_number() } fn earliest_block_number(&self) -> ProviderResult { @@ -409,6 +376,10 @@ impl BlockReader for ProviderFactory { ) -> ProviderResult>> { self.provider()?.recovered_block_range(range) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + self.provider()?.block_by_transaction_id(id) + } } impl TransactionsProvider for ProviderFactory { @@ -419,24 +390,14 @@ impl TransactionsProvider for ProviderFactory { } fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id(id), - || self.provider()?.transaction_by_id(id), - ) + self.static_file_provider.transaction_by_id(id) } fn transaction_by_id_unhashed( &self, id: TxNumber, ) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id_unhashed(id), - || self.provider()?.transaction_by_id_unhashed(id), - ) + self.static_file_provider.transaction_by_id_unhashed(id) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { @@ -472,7 +433,7 @@ impl TransactionsProvider for ProviderFactory { &self, range: impl RangeBounds, ) -> ProviderResult> { - self.provider()?.transactions_by_tx_range(range) + self.static_file_provider.transactions_by_tx_range(range) } fn senders_by_tx_range( @@ -489,6 +450,7 @@ impl TransactionsProvider for ProviderFactory { impl ReceiptProvider for ProviderFactory { type Receipt = ReceiptTy; + fn receipt(&self, id: TxNumber) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( StaticFileSegment::Receipts, @@ -674,7 +636,6 @@ mod tests { StaticFileProvider::read_write(static_dir_path).unwrap(), ) .unwrap(); - let provider = factory.provider().unwrap(); provider.block_hash(0).unwrap(); let provider_rw = factory.provider_rw().unwrap(); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index bdde71b2ce1..01868376ddd 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -22,7 +22,7 @@ use alloy_consensus::{ transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, BlockHeader, TxReceipt, }; -use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber}; +use alloy_eips::BlockHashOrNumber; use alloy_primitives::{ keccak256, map::{hash_map, B256Map, HashMap, HashSet}, @@ -42,7 +42,7 @@ use reth_db_api::{ table::Table, tables, transaction::{DbTx, DbTxMut}, - BlockNumberList, DatabaseError, PlainAccountState, PlainStorageState, + BlockNumberList, PlainAccountState, PlainStorageState, }; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_node_types::{BlockTy, BodyTy, HeaderTy, NodeTypes, ReceiptTy, TxTy}; @@ -74,7 +74,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, ops::{Deref, DerefMut, Not, Range, RangeBounds, RangeInclusive}, - sync::{mpsc, Arc}, + sync::Arc, }; use tracing::{debug, trace}; @@ -563,23 +563,6 @@ impl DatabaseProvider { } impl DatabaseProvider { - fn transactions_by_tx_range_with_cursor( - &self, - range: impl RangeBounds, - cursor: &mut C, - ) -> ProviderResult>> - where - C: DbCursorRO>>, - { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Transactions, - to_range(range), - |static_file, range, _| static_file.transactions_by_tx_range(range), - |range, _| self.cursor_collect(cursor, range), - |_| true, - ) - } - fn recovered_block( &self, id: BlockHashOrNumber, @@ -649,7 +632,6 @@ impl DatabaseProvider { let mut blocks = Vec::with_capacity(len); let headers = headers_range(range.clone())?; - let mut tx_cursor = self.tx.cursor_read::>>()?; // If the body indices are not found, this means that the transactions either do // not exist in the database yet, or they do exit but are @@ -668,7 +650,7 @@ impl DatabaseProvider { let transactions = if tx_range.is_empty() { Vec::new() } else { - self.transactions_by_tx_range_with_cursor(tx_range.clone(), &mut tx_cursor)? + self.transactions_by_tx_range(tx_range.clone())? }; inputs.push((header.as_ref(), transactions)); @@ -1007,8 +989,8 @@ impl HeaderSyncGapProvider impl HeaderProvider for DatabaseProvider { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(*block_hash)? { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { + if let Some(num) = self.block_number(block_hash)? { Ok(self.header_by_number(num)?) } else { Ok(None) @@ -1016,16 +998,11 @@ impl HeaderProvider for DatabasePro } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - num, - |static_file| static_file.header_by_number(num), - || Ok(self.tx.get::>(num)?), - ) + self.static_file_provider.header_by_number(num) } - fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(*block_hash)? { + fn header_td(&self, block_hash: BlockHash) -> ProviderResult> { + if let Some(num) = self.block_number(block_hash)? { self.header_td_by_number(num) } else { Ok(None) @@ -1041,46 +1018,21 @@ impl HeaderProvider for DatabasePro return Ok(Some(td)) } - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.header_td_by_number(number), - || Ok(self.tx.get::(number)?.map(|td| td.0)), - ) + self.static_file_provider.header_td_by_number(number) } fn headers_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, _| static_file.headers_range(range), - |range, _| self.cursor_read_collect::>(range), - |_| true, - ) + self.static_file_provider.headers_range(range) } fn sealed_header( &self, number: BlockNumber, ) -> ProviderResult>> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.sealed_header(number), - || { - if let Some(header) = self.header_by_number(number)? { - let hash = self - .block_hash(number)? - .ok_or_else(|| ProviderError::HeaderNotFound(number.into()))?; - Ok(Some(SealedHeader::new(header, hash))) - } else { - Ok(None) - } - }, - ) + self.static_file_provider.sealed_header(number) } fn sealed_headers_while( @@ -1088,40 +1040,13 @@ impl HeaderProvider for DatabasePro range: impl RangeBounds, predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult>> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, predicate| static_file.sealed_headers_while(range, predicate), - |range, mut predicate| { - let mut headers = vec![]; - for entry in - self.tx.cursor_read::>()?.walk_range(range)? - { - let (number, header) = entry?; - let hash = self - .block_hash(number)? - .ok_or_else(|| ProviderError::HeaderNotFound(number.into()))?; - let sealed = SealedHeader::new(header, hash); - if !predicate(&sealed) { - break - } - headers.push(sealed); - } - Ok(headers) - }, - predicate, - ) + self.static_file_provider.sealed_headers_while(range, predicate) } } impl BlockHashReader for DatabaseProvider { fn block_hash(&self, number: u64) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.block_hash(number), - || Ok(self.tx.get::(number)?), - ) + self.static_file_provider.block_hash(number) } fn canonical_hashes_range( @@ -1129,13 +1054,7 @@ impl BlockHashReader for DatabaseProvider ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - start..end, - |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), - |range, _| self.cursor_read_collect::(range), - |_| true, - ) + self.static_file_provider.canonical_hashes_range(start, end) } } @@ -1156,15 +1075,7 @@ impl BlockNumReader for DatabaseProvider ProviderResult { - Ok(self - .tx - .cursor_read::()? - .last()? - .map(|(num, _)| num) - .max( - self.static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers), - ) - .unwrap_or_default()) + self.static_file_provider.last_block_number() } fn block_number(&self, hash: B256) -> ProviderResult> { @@ -1216,6 +1127,7 @@ impl BlockReader for DatabaseProvid Ok(None) } + fn pending_block(&self) -> ProviderResult>> { Ok(None) } @@ -1313,6 +1225,14 @@ impl BlockReader for DatabaseProvid }, ) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + Ok(self + .tx + .cursor_read::()? + .seek(id) + .map(|b| b.map(|(_, bn)| bn))?) + } } impl TransactionsProviderExt @@ -1324,66 +1244,7 @@ impl TransactionsProviderExt &self, tx_range: Range, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Transactions, - tx_range, - |static_file, range, _| static_file.transaction_hashes_by_range(range), - |tx_range, _| { - let mut tx_cursor = self.tx.cursor_read::>>()?; - let tx_range_size = tx_range.clone().count(); - let tx_walker = tx_cursor.walk_range(tx_range)?; - - let chunk_size = (tx_range_size / rayon::current_num_threads()).max(1); - let mut channels = Vec::with_capacity(chunk_size); - let mut transaction_count = 0; - - #[inline] - fn calculate_hash( - entry: Result<(TxNumber, T), DatabaseError>, - rlp_buf: &mut Vec, - ) -> Result<(B256, TxNumber), Box> - where - T: Encodable2718, - { - let (tx_id, tx) = entry.map_err(|e| Box::new(e.into()))?; - tx.encode_2718(rlp_buf); - Ok((keccak256(rlp_buf), tx_id)) - } - - for chunk in &tx_walker.chunks(chunk_size) { - let (tx, rx) = mpsc::channel(); - channels.push(rx); - - // Note: Unfortunate side-effect of how chunk is designed in itertools (it is - // not Send) - let chunk: Vec<_> = chunk.collect(); - transaction_count += chunk.len(); - - // Spawn the task onto the global rayon pool - // This task will send the results through the channel after it has calculated - // the hash. - rayon::spawn(move || { - let mut rlp_buf = Vec::with_capacity(128); - for entry in chunk { - rlp_buf.clear(); - let _ = tx.send(calculate_hash(entry, &mut rlp_buf)); - } - }); - } - let mut tx_list = Vec::with_capacity(transaction_count); - - // Iterate over channels and append the tx hashes unsorted - for channel in channels { - while let Ok(tx) = channel.recv() { - let (tx_hash, tx_id) = tx.map_err(|boxed| *boxed)?; - tx_list.push((tx_hash, tx_id)); - } - } - - Ok(tx_list) - }, - |_| true, - ) + self.static_file_provider.transaction_hashes_by_range(tx_range) } } @@ -1396,24 +1257,14 @@ impl TransactionsProvider for Datab } fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id(id), - || Ok(self.tx.get::>(id)?), - ) + self.static_file_provider.transaction_by_id(id) } fn transaction_by_id_unhashed( &self, id: TxNumber, ) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id_unhashed(id), - || Ok(self.tx.get::>(id)?), - ) + self.static_file_provider.transaction_by_id_unhashed(id) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { @@ -1428,11 +1279,9 @@ impl TransactionsProvider for Datab &self, tx_hash: TxHash, ) -> ProviderResult> { - let mut transaction_cursor = self.tx.cursor_read::()?; if let Some(transaction_id) = self.transaction_id(tx_hash)? && let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? && - let Some(block_number) = - transaction_cursor.seek(transaction_id).map(|b| b.map(|(_, bn)| bn))? && + let Some(block_number) = self.block_by_transaction_id(transaction_id)? && let Some(sealed_header) = self.sealed_header(block_number)? { let (header, block_hash) = sealed_header.split(); @@ -1469,8 +1318,6 @@ impl TransactionsProvider for Datab &self, id: BlockHashOrNumber, ) -> ProviderResult>> { - let mut tx_cursor = self.tx.cursor_read::>()?; - if let Some(block_number) = self.convert_hash_or_number(id)? && let Some(body) = self.block_body_indices(block_number)? { @@ -1478,7 +1325,7 @@ impl TransactionsProvider for Datab return if tx_range.is_empty() { Ok(Some(Vec::new())) } else { - Ok(Some(self.transactions_by_tx_range_with_cursor(tx_range, &mut tx_cursor)?)) + self.transactions_by_tx_range(tx_range).map(Some) } } Ok(None) @@ -1489,7 +1336,6 @@ impl TransactionsProvider for Datab range: impl RangeBounds, ) -> ProviderResult>> { let range = to_range(range); - let mut tx_cursor = self.tx.cursor_read::>()?; self.block_body_indices_range(range.start..=range.end.saturating_sub(1))? .into_iter() @@ -1498,10 +1344,7 @@ impl TransactionsProvider for Datab if tx_num_range.is_empty() { Ok(Vec::new()) } else { - Ok(self - .transactions_by_tx_range_with_cursor(tx_num_range, &mut tx_cursor)? - .into_iter() - .collect()) + self.transactions_by_tx_range(tx_num_range) } }) .collect() @@ -1511,10 +1354,7 @@ impl TransactionsProvider for Datab &self, range: impl RangeBounds, ) -> ProviderResult> { - self.transactions_by_tx_range_with_cursor( - range, - &mut self.tx.cursor_read::>()?, - ) + self.static_file_provider.transactions_by_tx_range(range) } fn senders_by_tx_range( @@ -2698,16 +2538,17 @@ impl BlockWrite type Block = BlockTy; type Receipt = ReceiptTy; - /// Inserts the block into the database, always modifying the following tables: - /// * [`CanonicalHeaders`](tables::CanonicalHeaders) - /// * [`Headers`](tables::Headers) - /// * [`HeaderNumbers`](tables::HeaderNumbers) - /// * [`HeaderTerminalDifficulties`](tables::HeaderTerminalDifficulties) - /// * [`BlockBodyIndices`](tables::BlockBodyIndices) + /// Inserts the block into the database, always modifying the following static file segments and + /// tables: + /// * [`StaticFileSegment::Headers`] + /// * [`tables::HeaderNumbers`] + /// * [`tables::HeaderTerminalDifficulties`] + /// * [`tables::BlockBodyIndices`] /// - /// If there are transactions in the block, the following tables will be modified: - /// * [`Transactions`](tables::Transactions) - /// * [`TransactionBlocks`](tables::TransactionBlocks) + /// If there are transactions in the block, the following static file segments and tables will + /// be modified: + /// * [`StaticFileSegment::Transactions`] + /// * [`tables::TransactionBlocks`] /// /// If ommers are not empty, this will modify [`BlockOmmers`](tables::BlockOmmers). /// If withdrawals are not empty, this will modify diff --git a/crates/storage/provider/src/providers/static_file/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs index ab19fbf732c..9906583f900 100644 --- a/crates/storage/provider/src/providers/static_file/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -89,11 +89,11 @@ impl<'a, N: NodePrimitives> StaticFileJarProvider<'a, N> { impl> HeaderProvider for StaticFileJarProvider<'_, N> { type Header = N::BlockHeader; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { Ok(self .cursor()? - .get_two::>(block_hash.into())? - .filter(|(_, hash)| hash == block_hash) + .get_two::>((&block_hash).into())? + .filter(|(_, hash)| hash == &block_hash) .map(|(header, _)| header)) } @@ -101,11 +101,11 @@ impl> HeaderProvider for StaticFileJarProv self.cursor()?.get_one::>(num.into()) } - fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header_td(&self, block_hash: BlockHash) -> ProviderResult> { Ok(self .cursor()? - .get_two::(block_hash.into())? - .filter(|(_, hash)| hash == block_hash) + .get_two::((&block_hash).into())? + .filter(|(_, hash)| hash == &block_hash) .map(|(td, _)| td.into())) } diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 6af6313f6a2..f2703c7553b 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1374,13 +1374,13 @@ impl StaticFileWriter for StaticFileProvider { impl> HeaderProvider for StaticFileProvider { type Header = N::BlockHeader; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.find_static_file(StaticFileSegment::Headers, |jar_provider| { Ok(jar_provider .cursor()? - .get_two::>(block_hash.into())? + .get_two::>((&block_hash).into())? .and_then(|(header, hash)| { - if &hash == block_hash { + if hash == block_hash { return Some(header) } None @@ -1400,12 +1400,12 @@ impl> HeaderProvider for StaticFileProvide }) } - fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header_td(&self, block_hash: BlockHash) -> ProviderResult> { self.find_static_file(StaticFileSegment::Headers, |jar_provider| { Ok(jar_provider .cursor()? - .get_two::(block_hash.into())? - .and_then(|(td, hash)| (&hash == block_hash).then_some(td.0))) + .get_two::((&block_hash).into())? + .and_then(|(td, hash)| (hash == block_hash).then_some(td.0))) }) } @@ -1468,7 +1468,15 @@ impl> HeaderProvider for StaticFileProvide impl BlockHashReader for StaticFileProvider { fn block_hash(&self, num: u64) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)?.block_hash(num) + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None) + .and_then(|provider| provider.block_hash(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileBlock(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn canonical_hashes_range( @@ -1712,8 +1720,6 @@ impl> TransactionsPr } } -/* Cannot be successfully implemented but must exist for trait requirements */ - impl BlockNumReader for StaticFileProvider { fn chain_info(&self) -> ProviderResult { // Required data not present in static_files @@ -1726,8 +1732,7 @@ impl BlockNumReader for StaticFileProvider { } fn last_block_number(&self) -> ProviderResult { - // Required data not present in static_files - Err(ProviderError::UnsupportedProvider) + Ok(self.get_highest_static_file_block(StaticFileSegment::Headers).unwrap_or_default()) } fn block_number(&self, _hash: B256) -> ProviderResult> { @@ -1736,6 +1741,8 @@ impl BlockNumReader for StaticFileProvider { } } +/* Cannot be successfully implemented but must exist for trait requirements */ + impl> BlockReader for StaticFileProvider { @@ -1803,6 +1810,10 @@ impl> ) -> ProviderResult>> { Err(ProviderError::UnsupportedProvider) } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } } impl BlockBodyIndicesProvider for StaticFileProvider { diff --git a/crates/storage/provider/src/providers/static_file/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs index 97a8ea95433..1c3bfd58a79 100644 --- a/crates/storage/provider/src/providers/static_file/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -146,12 +146,12 @@ mod tests { let header = header.unseal(); // Compare Header - assert_eq!(header, db_provider.header(&header_hash).unwrap().unwrap()); + assert_eq!(header, db_provider.header(header_hash).unwrap().unwrap()); assert_eq!(header, jar_provider.header_by_number(header.number).unwrap().unwrap()); // Compare HeaderTerminalDifficulties assert_eq!( - db_provider.header_td(&header_hash).unwrap().unwrap(), + db_provider.header_td(header_hash).unwrap().unwrap(), jar_provider.header_td_by_number(header.number).unwrap().unwrap() ); } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 220cff07ce1..d5e3fe4da7b 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -281,9 +281,9 @@ impl HeaderP { type Header = ::Header; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { let lock = self.headers.lock(); - Ok(lock.get(block_hash).cloned()) + Ok(lock.get(&block_hash).cloned()) } fn header_by_number(&self, num: u64) -> ProviderResult> { @@ -291,9 +291,9 @@ impl HeaderP Ok(lock.values().find(|h| h.number() == num).cloned()) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { + fn header_td(&self, hash: BlockHash) -> ProviderResult> { let lock = self.headers.lock(); - Ok(lock.get(hash).map(|target| { + Ok(lock.get(&hash).map(|target| { lock.values() .filter(|h| h.number() < target.number()) .fold(target.difficulty(), |td, h| td + h.difficulty()) @@ -718,6 +718,10 @@ impl BlockRe ) -> ProviderResult>> { Ok(vec![]) } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Ok(None) + } } impl BlockReaderIdExt for MockEthProvider diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs index 00b19df49dc..76e511d52d4 100644 --- a/crates/storage/rpc-provider/src/lib.rs +++ b/crates/storage/rpc-provider/src/lib.rs @@ -338,9 +338,9 @@ where { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { let block_response = self.block_on_async(async { - self.provider.get_block_by_hash(*block_hash).await.map_err(ProviderError::other) + self.provider.get_block_by_hash(block_hash).await.map_err(ProviderError::other) })?; let Some(block_response) = block_response else { @@ -364,7 +364,7 @@ where Ok(Some(sealed_header.into_header())) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { + fn header_td(&self, hash: BlockHash) -> ProviderResult> { let header = self.header(hash).map_err(ProviderError::other)?; Ok(header.map(|b| b.difficulty())) @@ -510,6 +510,10 @@ where ) -> ProviderResult>> { Err(ProviderError::UnsupportedProvider) } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } } impl BlockReaderIdExt for RpcBlockchainProvider @@ -1539,6 +1543,10 @@ where ) -> Result>, ProviderError> { Err(ProviderError::UnsupportedProvider) } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } } impl TransactionsProvider for RpcBlockchainStateProvider @@ -1658,7 +1666,7 @@ where { type Header = HeaderTy; - fn header(&self, _block_hash: &BlockHash) -> Result, ProviderError> { + fn header(&self, _block_hash: BlockHash) -> Result, ProviderError> { Err(ProviderError::UnsupportedProvider) } @@ -1666,7 +1674,7 @@ where Err(ProviderError::UnsupportedProvider) } - fn header_td(&self, _hash: &BlockHash) -> Result, ProviderError> { + fn header_td(&self, _hash: BlockHash) -> Result, ProviderError> { Err(ProviderError::UnsupportedProvider) } diff --git a/crates/storage/storage-api/src/block.rs b/crates/storage/storage-api/src/block.rs index 40a009935ca..b9ab206a6b8 100644 --- a/crates/storage/storage-api/src/block.rs +++ b/crates/storage/storage-api/src/block.rs @@ -4,7 +4,7 @@ use crate::{ }; use alloc::{sync::Arc, vec::Vec}; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; -use alloy_primitives::{BlockNumber, B256}; +use alloy_primitives::{BlockNumber, TxNumber, B256}; use core::ops::RangeInclusive; use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_storage_errors::provider::ProviderResult; @@ -144,6 +144,9 @@ pub trait BlockReader: &self, range: RangeInclusive, ) -> ProviderResult>>; + + /// Returns the block number that contains the given transaction. + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult>; } impl BlockReader for Arc { @@ -202,6 +205,9 @@ impl BlockReader for Arc { ) -> ProviderResult>> { T::recovered_block_range(self, range) } + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + T::block_by_transaction_id(self, id) + } } impl BlockReader for &T { @@ -260,6 +266,9 @@ impl BlockReader for &T { ) -> ProviderResult>> { T::recovered_block_range(self, range) } + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + T::block_by_transaction_id(self, id) + } } /// Trait extension for `BlockReader`, for types that implement `BlockId` conversion. diff --git a/crates/storage/storage-api/src/header.rs b/crates/storage/storage-api/src/header.rs index a4c9b215f82..7e3133ec712 100644 --- a/crates/storage/storage-api/src/header.rs +++ b/crates/storage/storage-api/src/header.rs @@ -15,19 +15,19 @@ pub trait HeaderProvider: Send + Sync { type Header: BlockHeader; /// Check if block is known - fn is_known(&self, block_hash: &BlockHash) -> ProviderResult { + fn is_known(&self, block_hash: BlockHash) -> ProviderResult { self.header(block_hash).map(|header| header.is_some()) } /// Get header by block hash - fn header(&self, block_hash: &BlockHash) -> ProviderResult>; + fn header(&self, block_hash: BlockHash) -> ProviderResult>; /// Retrieves the header sealed by the given block hash. fn sealed_header_by_hash( &self, block_hash: BlockHash, ) -> ProviderResult>> { - Ok(self.header(&block_hash)?.map(|header| SealedHeader::new(header, block_hash))) + Ok(self.header(block_hash)?.map(|header| SealedHeader::new(header, block_hash))) } /// Get header by block number @@ -39,13 +39,13 @@ pub trait HeaderProvider: Send + Sync { hash_or_num: BlockHashOrNumber, ) -> ProviderResult> { match hash_or_num { - BlockHashOrNumber::Hash(hash) => self.header(&hash), + BlockHashOrNumber::Hash(hash) => self.header(hash), BlockHashOrNumber::Number(num) => self.header_by_number(num), } } /// Get total difficulty by block hash. - fn header_td(&self, hash: &BlockHash) -> ProviderResult>; + fn header_td(&self, hash: BlockHash) -> ProviderResult>; /// Get total difficulty by block number. fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult>; diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 4c0117fe54f..44e499ae006 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -237,6 +237,10 @@ impl BlockReader for NoopProvider { ) -> ProviderResult>> { Ok(Vec::new()) } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Ok(None) + } } impl TransactionsProvider for NoopProvider { @@ -343,7 +347,7 @@ impl ReceiptProviderIdExt for NoopProvider HeaderProvider for NoopProvider { type Header = N::BlockHeader; - fn header(&self, _block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, _block_hash: BlockHash) -> ProviderResult> { Ok(None) } @@ -351,7 +355,7 @@ impl HeaderProvider for NoopProvider { Ok(None) } - fn header_td(&self, _hash: &BlockHash) -> ProviderResult> { + fn header_td(&self, _hash: BlockHash) -> ProviderResult> { Ok(None) } diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index 4027beb70cb..93896accbbc 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -63,7 +63,7 @@ fn header_provider_example(provider: T, number: u64) -> eyre: // Can also query the header by hash! let header_by_hash = - provider.header(&sealed_header.hash())?.ok_or(eyre::eyre!("header by hash not found"))?; + provider.header(sealed_header.hash())?.ok_or(eyre::eyre!("header by hash not found"))?; assert_eq!(sealed_header.header(), &header_by_hash); // The header's total difficulty is stored in a separate table, so we have a separate call for From ba3f65da7136f34ec49ed295bbc8ce4cdfe3848e Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Tue, 30 Sep 2025 13:12:32 +0100 Subject: [PATCH 02/13] test: downloaders --- Cargo.lock | 2 - crates/net/downloaders/Cargo.toml | 9 +-- crates/net/downloaders/src/bodies/bodies.rs | 72 +++++-------------- crates/net/downloaders/src/bodies/task.rs | 2 +- .../net/downloaders/src/bodies/test_utils.rs | 39 +++++++--- crates/net/downloaders/src/file_client.rs | 4 +- 6 files changed, 52 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d6fceddd11..6b07bac7023 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7845,8 +7845,6 @@ dependencies = [ "reth-chainspec", "reth-config", "reth-consensus", - "reth-db", - "reth-db-api", "reth-ethereum-primitives", "reth-metrics", "reth-network-p2p", diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml index 1dd5ec0947b..8a73f30b123 100644 --- a/crates/net/downloaders/Cargo.toml +++ b/crates/net/downloaders/Cargo.toml @@ -22,9 +22,8 @@ reth-storage-api.workspace = true reth-tasks.workspace = true # optional deps for the test-utils feature -reth-db = { workspace = true, optional = true } -reth-db-api = { workspace = true, optional = true } reth-ethereum-primitives = { workspace = true, optional = true } +reth-provider = { workspace = true, optional = true } reth-testing-utils = { workspace = true, optional = true } # ethereum @@ -58,11 +57,8 @@ itertools.workspace = true async-compression = { workspace = true, features = ["gzip", "tokio"] } reth-ethereum-primitives.workspace = true reth-chainspec.workspace = true -reth-db = { workspace = true, features = ["test-utils"] } -reth-db-api.workspace = true reth-consensus = { workspace = true, features = ["test-utils"] } reth-network-p2p = { workspace = true, features = ["test-utils"] } -reth-provider = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true reth-tracing.workspace = true @@ -76,13 +72,10 @@ default = [] file-client = ["dep:async-compression", "dep:alloy-rlp"] test-utils = [ "tempfile", - "reth-db-api", - "reth-db/test-utils", "reth-consensus/test-utils", "reth-network-p2p/test-utils", "reth-testing-utils", "reth-chainspec/test-utils", - "reth-db-api?/test-utils", "reth-provider/test-utils", "reth-primitives-traits/test-utils", "dep:reth-ethereum-primitives", diff --git a/crates/net/downloaders/src/bodies/bodies.rs b/crates/net/downloaders/src/bodies/bodies.rs index 0c7b1e62012..09eb22854d4 100644 --- a/crates/net/downloaders/src/bodies/bodies.rs +++ b/crates/net/downloaders/src/bodies/bodies.rs @@ -618,12 +618,8 @@ mod tests { }; use alloy_primitives::B256; use assert_matches::assert_matches; - use reth_chainspec::MAINNET; use reth_consensus::test_utils::TestConsensus; - use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; - use reth_provider::{ - providers::StaticFileProvider, test_utils::MockNodeTypesWithDB, ProviderFactory, - }; + use reth_provider::test_utils::create_test_provider_factory; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::collections::HashMap; @@ -632,25 +628,20 @@ mod tests { #[tokio::test] async fn streams_bodies_in_order() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=19); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true), ); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); downloader.set_download_range(0..=19).expect("failed to set download range"); @@ -666,7 +657,7 @@ mod tests { #[tokio::test] async fn requests_correct_number_of_times() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let mut rng = generators::rng(); let blocks = random_block_range( &mut rng, @@ -680,22 +671,17 @@ mod tests { .map(|block| (block.hash(), block.into_body())) .collect::>(); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let request_limit = 10; let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone())); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_request_limit(request_limit) .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); downloader.set_download_range(0..=199).expect("failed to set download range"); @@ -708,28 +694,23 @@ mod tests { #[tokio::test] async fn streams_bodies_in_order_after_range_reset() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=99); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let stream_batch_size = 20; let request_limit = 10; let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true), ); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_stream_batch_size(stream_batch_size) .with_request_limit(request_limit) .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); let mut range_start = 0; @@ -750,24 +731,19 @@ mod tests { #[tokio::test] async fn can_download_new_range_after_termination() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=199); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone())); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_stream_batch_size(100) .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); // Set and download the first range @@ -792,14 +768,13 @@ mod tests { #[tokio::test] async fn can_download_after_exceeding_limit() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=199); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone())); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); // Set the max buffered block size to 1 byte, to make sure that every response exceeds the // limit let mut downloader = BodiesDownloaderBuilder::default() @@ -809,11 +784,7 @@ mod tests { .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); // Set and download the entire range @@ -829,16 +800,15 @@ mod tests { #[tokio::test] async fn can_tolerate_empty_responses() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=99); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); // respond with empty bodies for every other request. let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_empty_responses(2), ); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_request_limit(3) @@ -846,11 +816,7 @@ mod tests { .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); // Download the requested range diff --git a/crates/net/downloaders/src/bodies/task.rs b/crates/net/downloaders/src/bodies/task.rs index df1d5540db3..4da5946fffb 100644 --- a/crates/net/downloaders/src/bodies/task.rs +++ b/crates/net/downloaders/src/bodies/task.rs @@ -190,7 +190,7 @@ mod tests { let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=19); - insert_headers(factory.db_ref().db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true), diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index aeb4488eb0d..39061a25eee 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -4,11 +4,13 @@ use alloy_consensus::BlockHeader; use alloy_primitives::B256; -use reth_db::DatabaseEnv; -use reth_db_api::{database::Database, tables, transaction::DbTxMut}; use reth_ethereum_primitives::BlockBody; use reth_network_p2p::bodies::response::BlockResponse; use reth_primitives_traits::{Block, SealedBlock, SealedHeader}; +use reth_provider::{ + test_utils::MockNodeTypesWithDB, HeaderProvider, ProviderFactory, StaticFileProviderFactory, + StaticFileSegment, StaticFileWriter, +}; use std::collections::HashMap; pub(crate) fn zip_blocks<'a, B: Block>( @@ -42,12 +44,29 @@ pub(crate) fn create_raw_bodies( } #[inline] -pub(crate) fn insert_headers(db: &DatabaseEnv, headers: &[SealedHeader]) { - db.update(|tx| { - for header in headers { - tx.put::(header.number, header.hash()).unwrap(); - tx.put::(header.number, header.clone_header()).unwrap(); - } - }) - .expect("failed to commit") +pub(crate) fn insert_headers( + factory: &ProviderFactory, + headers: &[SealedHeader], +) { + let provider_rw = factory.provider_rw().expect("failed to create provider"); + let static_file_provider = provider_rw.static_file_provider(); + let mut writer = static_file_provider + .latest_writer(StaticFileSegment::Headers) + .expect("failed to create writer"); + + for header in headers { + let ttd = if header.number() == 0 { + header.difficulty() + } else { + let parent_block_number = header.number() - 1; + let parent_ttd = + provider_rw.header_td_by_number(parent_block_number).unwrap().unwrap_or_default(); + parent_ttd + header.difficulty() + }; + + writer + .append_header(header.header(), ttd, &header.hash()) + .expect("failed to append header"); + } + provider_rw.commit().expect("failed to commit"); } diff --git a/crates/net/downloaders/src/file_client.rs b/crates/net/downloaders/src/file_client.rs index 3f6233615c3..34c2f56b75c 100644 --- a/crates/net/downloaders/src/file_client.rs +++ b/crates/net/downloaders/src/file_client.rs @@ -675,7 +675,7 @@ mod tests { let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=19); - insert_headers(factory.db_ref().db(), &headers); + insert_headers(&factory, &headers); // create an empty file let file = tempfile::tempfile().unwrap(); @@ -770,7 +770,7 @@ mod tests { Arc::new(FileClient::from_file(file, NoopConsensus::arc()).await.unwrap()); // insert headers in db for the bodies downloader - insert_headers(factory.db_ref().db(), &headers); + insert_headers(&factory, &headers); let mut downloader = BodiesDownloaderBuilder::default().build::( client.clone(), From 02e53fd8888d6a9cb30b38eb24482379678a78e4 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:45:18 +0100 Subject: [PATCH 03/13] include reth-provider dep --- crates/net/downloaders/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml index 8a73f30b123..57094813eee 100644 --- a/crates/net/downloaders/Cargo.toml +++ b/crates/net/downloaders/Cargo.toml @@ -59,6 +59,7 @@ reth-ethereum-primitives.workspace = true reth-chainspec.workspace = true reth-consensus = { workspace = true, features = ["test-utils"] } reth-network-p2p = { workspace = true, features = ["test-utils"] } +reth-provider = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true reth-tracing.workspace = true From 1e1c05ead0c2cb33edc79ec4dacc64e1de3a4e74 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 15:56:36 +0100 Subject: [PATCH 04/13] fix(provider): detect static file not found in range iter --- .../provider/src/providers/static_file/manager.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index f2703c7553b..ab98debd0f4 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1118,7 +1118,11 @@ impl StaticFileProvider { }; let mut result = Vec::with_capacity((range.end - range.start).min(100) as usize); - let mut provider = get_provider(range.start)?; + let mut provider = match get_provider(range.start) { + Ok(provider) => provider, + Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(vec![]), + Err(err) => return Err(err), + }; let mut cursor = provider.cursor()?; // advances number in range @@ -1160,7 +1164,11 @@ impl StaticFileProvider { // before requesting the next one. drop(cursor); drop(provider); - provider = get_provider(number)?; + provider = match get_provider(number) { + Ok(provider) => provider, + Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(result), + Err(err) => return Err(err), + }; cursor = provider.cursor()?; retrying = true; } From 77075060eaddab08cf6e4cf889d6c64bacbaa878 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 16:16:36 +0100 Subject: [PATCH 05/13] test: drop writer in insert_headers --- crates/net/downloaders/src/bodies/test_utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index 39061a25eee..9c459571d13 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -68,5 +68,6 @@ pub(crate) fn insert_headers( .append_header(header.header(), ttd, &header.hash()) .expect("failed to append header"); } + drop(writer); provider_rw.commit().expect("failed to commit"); } From 191665bb0ff1e66e22d08c09219a8ee9524e1863 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 17:06:17 +0100 Subject: [PATCH 06/13] test: fix stage tests --- crates/prune/prune/src/segments/receipts.rs | 10 +- .../prune/src/segments/static_file/headers.rs | 40 ++++---- .../src/segments/static_file/transactions.rs | 13 ++- .../src/segments/user/account_history.rs | 4 +- .../src/segments/user/receipts_by_logs.rs | 11 ++- .../src/segments/user/sender_recovery.rs | 10 +- .../src/segments/user/storage_history.rs | 4 +- .../src/segments/user/transaction_lookup.rs | 10 +- crates/stages/stages/benches/setup/mod.rs | 4 +- crates/stages/stages/src/stages/merkle.rs | 8 +- crates/stages/stages/src/stages/mod.rs | 2 +- crates/stages/stages/src/stages/prune.rs | 6 +- .../stages/src/stages/sender_recovery.rs | 18 ++-- crates/stages/stages/src/stages/tx_lookup.rs | 18 ++-- .../stages/stages/src/test_utils/test_db.rs | 97 ++++++------------- .../static-file/src/static_file_producer.rs | 14 +-- 16 files changed, 106 insertions(+), 163 deletions(-) diff --git a/crates/prune/prune/src/segments/receipts.rs b/crates/prune/prune/src/segments/receipts.rs index 12ad6e2c203..d114744184e 100644 --- a/crates/prune/prune/src/segments/receipts.rs +++ b/crates/prune/prune/src/segments/receipts.rs @@ -89,11 +89,11 @@ mod tests { Itertools, }; use reth_db_api::tables; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader}; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_testing_utils::generators::{ self, random_block_range, random_receipt, BlockRangeParams, }; @@ -109,7 +109,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let mut receipts = Vec::new(); for block in &blocks { @@ -125,11 +125,11 @@ mod tests { db.insert_receipts(receipts).expect("insert receipts"); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), db.table::().unwrap().len() ); diff --git a/crates/prune/prune/src/segments/static_file/headers.rs b/crates/prune/prune/src/segments/static_file/headers.rs index 9f3c291bf44..aa913433ce7 100644 --- a/crates/prune/prune/src/segments/static_file/headers.rs +++ b/crates/prune/prune/src/segments/static_file/headers.rs @@ -216,19 +216,19 @@ mod tests { static_file::headers::HEADER_TABLES_TO_PRUNE, PruneInput, PruneLimiter, Segment, SegmentOutput, }; - use alloy_primitives::{BlockNumber, B256, U256}; + use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; - use reth_db_api::{tables, transaction::DbTx}; + use reth_db_api::tables; use reth_provider::{ DBProvider, DatabaseProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, - StaticFileProviderFactory, + StaticFileProviderFactory, StatsReader, }; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, SegmentOutputCheckpoint, }; use reth_stages::test_utils::TestStageDB; - use reth_testing_utils::{generators, generators::random_header_range}; + use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use tracing::trace; #[test] @@ -238,16 +238,17 @@ mod tests { let db = TestStageDB::default(); let mut rng = generators::rng(); - let headers = random_header_range(&mut rng, 0..100, B256::ZERO); - let tx = db.factory.provider_rw().unwrap().into_tx(); - for header in &headers { - TestStageDB::insert_header(None, &tx, header, U256::ZERO).unwrap(); - } - tx.commit().unwrap(); + let blocks = random_block_range( + &mut rng, + 0..=99, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..0, ..Default::default() }, + ); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); - assert_eq!(db.table::().unwrap().len(), headers.len()); - assert_eq!(db.table::().unwrap().len(), headers.len()); - assert_eq!(db.table::().unwrap().len(), headers.len()); + assert_eq!( + db.factory.provider().unwrap().count_entries::().unwrap(), + blocks.len() + ); let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { let segment = super::Headers::new(db.factory.static_file_provider()); @@ -304,17 +305,10 @@ mod tests { ); assert_eq!( - db.table::().unwrap().len(), - headers.len() - (last_pruned_block_number + 1) as usize - ); - assert_eq!( - db.table::().unwrap().len(), - headers.len() - (last_pruned_block_number + 1) as usize - ); - assert_eq!( - db.table::().unwrap().len(), - headers.len() - (last_pruned_block_number + 1) as usize + db.factory.provider().unwrap().count_entries::().unwrap(), + blocks.len() - (last_pruned_block_number + 1) as usize ); + assert_eq!( db.factory.provider().unwrap().get_prune_checkpoint(PruneSegment::Headers).unwrap(), Some(PruneCheckpoint { diff --git a/crates/prune/prune/src/segments/static_file/transactions.rs b/crates/prune/prune/src/segments/static_file/transactions.rs index 115ee2ca39a..2e236622a96 100644 --- a/crates/prune/prune/src/segments/static_file/transactions.rs +++ b/crates/prune/prune/src/segments/static_file/transactions.rs @@ -102,13 +102,13 @@ mod tests { use reth_db_api::tables; use reth_provider::{ DBProvider, DatabaseProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, - StaticFileProviderFactory, + StaticFileProviderFactory, StatsReader, }; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, SegmentOutput, }; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; @@ -122,12 +122,15 @@ mod tests { 1..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let transactions = blocks.iter().flat_map(|block| &block.body().transactions).collect::>(); - assert_eq!(db.table::().unwrap().len(), transactions.len()); + assert_eq!( + db.factory.provider().unwrap().count_entries::().unwrap(), + transactions.len() + ); let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { let segment = super::Transactions::new(db.factory.static_file_provider()); @@ -199,7 +202,7 @@ mod tests { .checked_sub(if result.progress.is_finished() { 0 } else { 1 }); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), transactions.len() - (last_pruned_tx_number + 1) ); assert_eq!( diff --git a/crates/prune/prune/src/segments/user/account_history.rs b/crates/prune/prune/src/segments/user/account_history.rs index 3c18cd1befc..3965da41c9d 100644 --- a/crates/prune/prune/src/segments/user/account_history.rs +++ b/crates/prune/prune/src/segments/user/account_history.rs @@ -137,7 +137,7 @@ mod tests { use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, }; @@ -153,7 +153,7 @@ mod tests { 1..=5000, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::>(); diff --git a/crates/prune/prune/src/segments/user/receipts_by_logs.rs b/crates/prune/prune/src/segments/user/receipts_by_logs.rs index 0849db52518..3208f9ec041 100644 --- a/crates/prune/prune/src/segments/user/receipts_by_logs.rs +++ b/crates/prune/prune/src/segments/user/receipts_by_logs.rs @@ -233,10 +233,11 @@ mod tests { use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; use reth_primitives_traits::InMemorySize; use reth_provider::{ - DBProvider, DatabaseProviderFactory, PruneCheckpointReader, TransactionsProvider, + DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader, + TransactionsProvider, }; use reth_prune_types::{PruneMode, PruneSegment, ReceiptsLogPruneConfig}; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_testing_utils::generators::{ self, random_block_range, random_eoa_account, random_log, random_receipt, BlockRangeParams, }; @@ -268,7 +269,7 @@ mod tests { ), ] .concat(); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let mut receipts = Vec::new(); @@ -288,11 +289,11 @@ mod tests { db.insert_receipts(receipts).expect("insert receipts"); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), db.table::().unwrap().len() ); diff --git a/crates/prune/prune/src/segments/user/sender_recovery.rs b/crates/prune/prune/src/segments/user/sender_recovery.rs index 35ee487203a..62d035d066a 100644 --- a/crates/prune/prune/src/segments/user/sender_recovery.rs +++ b/crates/prune/prune/src/segments/user/sender_recovery.rs @@ -91,9 +91,9 @@ mod tests { }; use reth_db_api::tables; use reth_primitives_traits::SignerRecoverable; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment}; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; @@ -107,7 +107,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let mut transaction_senders = Vec::new(); for block in &blocks { @@ -123,11 +123,11 @@ mod tests { db.insert_transaction_senders(transaction_senders).expect("insert transaction senders"); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), db.table::().unwrap().len() ); diff --git a/crates/prune/prune/src/segments/user/storage_history.rs b/crates/prune/prune/src/segments/user/storage_history.rs index ee7447c37da..cedd1b1c174 100644 --- a/crates/prune/prune/src/segments/user/storage_history.rs +++ b/crates/prune/prune/src/segments/user/storage_history.rs @@ -142,7 +142,7 @@ mod tests { use reth_db_api::{tables, BlockNumberList}; use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment}; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, }; @@ -158,7 +158,7 @@ mod tests { 0..=5000, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::>(); diff --git a/crates/prune/prune/src/segments/user/transaction_lookup.rs b/crates/prune/prune/src/segments/user/transaction_lookup.rs index 2ed08f7d1a7..7fbd4424a3c 100644 --- a/crates/prune/prune/src/segments/user/transaction_lookup.rs +++ b/crates/prune/prune/src/segments/user/transaction_lookup.rs @@ -139,11 +139,11 @@ mod tests { Itertools, }; use reth_db_api::tables; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader}; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; @@ -157,7 +157,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let mut tx_hash_numbers = Vec::new(); for block in &blocks { @@ -170,11 +170,11 @@ mod tests { db.insert_tx_hash_numbers(tx_hash_numbers).expect("insert tx hash numbers"); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.table::().unwrap().len(), + db.factory.provider().unwrap().count_entries::().unwrap(), db.table::().unwrap().len() ); diff --git a/crates/stages/stages/benches/setup/mod.rs b/crates/stages/stages/benches/setup/mod.rs index bd1fb59ebe9..f11b247784c 100644 --- a/crates/stages/stages/benches/setup/mod.rs +++ b/crates/stages/stages/benches/setup/mod.rs @@ -14,7 +14,7 @@ use reth_provider::{ }; use reth_stages::{ stages::{AccountHashingStage, StorageHashingStage}, - test_utils::{StorageKind, TestStageDB}, + test_utils::TestStageDB, }; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_contract_account_range, @@ -197,7 +197,7 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> TestStageDB { cloned_last.into_body(), ); - db.insert_blocks(blocks.iter(), StorageKind::Static).unwrap(); + db.insert_blocks(blocks.iter(), 0).unwrap(); // initialize TD db.commit(|tx| { diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index c5115316243..fe8dc22b3e0 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -442,8 +442,8 @@ fn validate_state_root( mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, - TestRunnerError, TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, + TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::{keccak256, U256}; use assert_matches::assert_matches; @@ -653,7 +653,7 @@ mod tests { ..Default::default() }, )); - self.db.insert_blocks(preblocks.iter(), StorageKind::Static)?; + self.db.insert_blocks(preblocks.iter(), 0)?; } let num_of_accounts = 31; @@ -692,7 +692,7 @@ mod tests { BlockRangeParams { parent: Some(head_hash), tx_count: 0..3, ..Default::default() }, )); let last_block = blocks.last().cloned().unwrap(); - self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; + self.db.insert_blocks(blocks.iter(), 0)?; let (transitions, final_state) = random_changeset_range( &mut rng, diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index f9b2312f5ab..0e31e35aa86 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -270,7 +270,7 @@ mod tests { 0..=tip, BlockRangeParams { parent: Some(genesis_hash), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Static)?; + db.insert_blocks(blocks.iter(), 0)?; let mut receipts = Vec::with_capacity(blocks.len()); let mut tx_num = 0u64; diff --git a/crates/stages/stages/src/stages/prune.rs b/crates/stages/stages/src/stages/prune.rs index f62259dcfdd..3284dae955e 100644 --- a/crates/stages/stages/src/stages/prune.rs +++ b/crates/stages/stages/src/stages/prune.rs @@ -171,8 +171,8 @@ where mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, - TestRunnerError, TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, + TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::B256; use reth_ethereum_primitives::Block; @@ -218,7 +218,7 @@ mod tests { input.checkpoint().block_number..=input.target(), BlockRangeParams { parent: Some(B256::ZERO), tx_count: 1..3, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; + self.db.insert_blocks(blocks.iter(), 0)?; self.db.insert_transaction_senders( blocks.iter().flat_map(|block| block.body().transactions.iter()).enumerate().map( |(i, tx)| (i as u64, tx.recover_signer().expect("failed to recover signer")), diff --git a/crates/stages/stages/src/stages/sender_recovery.rs b/crates/stages/stages/src/stages/sender_recovery.rs index 2a2870f07ca..73c5f8b75d2 100644 --- a/crates/stages/stages/src/stages/sender_recovery.rs +++ b/crates/stages/stages/src/stages/sender_recovery.rs @@ -369,8 +369,8 @@ struct FailedSenderRecoveryError { mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, - TestRunnerError, TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, + TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; @@ -416,10 +416,7 @@ mod tests { ) }) .collect::>(); - runner - .db - .insert_blocks(blocks.iter(), StorageKind::Static) - .expect("failed to insert blocks"); + runner.db.insert_blocks(blocks.iter(), 0).expect("failed to insert blocks"); let rx = runner.execute(input); @@ -456,10 +453,7 @@ mod tests { stage_progress + 1..=previous_stage, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..4, ..Default::default() }, ); // set tx count range high enough to hit the threshold - runner - .db - .insert_blocks(seed.iter(), StorageKind::Static) - .expect("failed to seed execution"); + runner.db.insert_blocks(seed.iter(), 0).expect("failed to seed execution"); let total_transactions = runner .db @@ -530,7 +524,7 @@ mod tests { 0..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..10, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let max_pruned_block = 30; let max_processed_block = 70; @@ -647,7 +641,7 @@ mod tests { stage_progress..=end, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; + self.db.insert_blocks(blocks.iter(), 0)?; Ok(blocks) } diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index 20a0770d8c8..53cc48aa639 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -254,8 +254,8 @@ where mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, - TestRunnerError, TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, + TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; @@ -301,10 +301,7 @@ mod tests { ) }) .collect::>(); - runner - .db - .insert_blocks(blocks.iter(), StorageKind::Static) - .expect("failed to insert blocks"); + runner.db.insert_blocks(blocks.iter(), 0).expect("failed to insert blocks"); let rx = runner.execute(input); @@ -345,10 +342,7 @@ mod tests { stage_progress + 1..=previous_stage, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - runner - .db - .insert_blocks(seed.iter(), StorageKind::Static) - .expect("failed to seed execution"); + runner.db.insert_blocks(seed.iter(), 0).expect("failed to seed execution"); runner.set_prune_mode(PruneMode::Before(prune_target)); @@ -383,7 +377,7 @@ mod tests { 0..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..10, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let max_pruned_block = 30; let max_processed_block = 70; @@ -513,7 +507,7 @@ mod tests { stage_progress + 1..=end, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; + self.db.insert_blocks(blocks.iter(), 0)?; Ok(blocks) } diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index f3e29c1fa66..1b1b84d8206 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -142,30 +142,25 @@ impl TestStageDB { /// Insert header to static file if `writer` exists, otherwise to DB. pub fn insert_header( - writer: Option<&mut StaticFileProviderRWRefMut<'_, EthPrimitives>>, + &self, + writer: &mut StaticFileProviderRWRefMut<'_, EthPrimitives>, tx: &TX, header: &SealedHeader, td: U256, ) -> ProviderResult<()> { - if let Some(writer) = writer { - // Backfill: some tests start at a forward block number, but static files require no - // gaps. - let segment_header = writer.user_header(); - if segment_header.block_end().is_none() && segment_header.expected_block_start() == 0 { - for block_number in 0..header.number { - let mut prev = header.clone_header(); - prev.number = block_number; - writer.append_header(&prev, U256::ZERO, &B256::ZERO)?; - } + // Backfill: some tests start at a forward block number, but static files require no + // gaps. + let segment_header = writer.user_header(); + if segment_header.block_end().is_none() && segment_header.expected_block_start() == 0 { + for block_number in 0..header.number { + let mut prev = header.clone_header(); + prev.number = block_number; + writer.append_header(&prev, U256::ZERO, &B256::ZERO)?; } - - writer.append_header(header.header(), td, &header.hash())?; - } else { - tx.put::(header.number, header.hash())?; - tx.put::(header.number, td.into())?; - tx.put::(header.number, header.header().clone())?; } + writer.append_header(header.header(), td, &header.hash())?; + tx.put::(header.hash(), header.number)?; Ok(()) } @@ -183,7 +178,7 @@ impl TestStageDB { if TD { td += header.difficulty; } - Self::insert_header(Some(&mut writer), &tx, header, td)?; + self.insert_header(&mut writer, &tx, header, td)?; } writer.commit()?; @@ -218,35 +213,29 @@ impl TestStageDB { /// database. /// /// Assumes that there's a single transition for each transaction (i.e. no block rewards). - pub fn insert_blocks<'a, I>(&self, blocks: I, storage_kind: StorageKind) -> ProviderResult<()> + pub fn insert_blocks<'a, I>(&self, blocks: I, tx_offset: u64) -> ProviderResult<()> where I: IntoIterator>, { let provider = self.factory.static_file_provider(); let tx = self.factory.provider_rw().unwrap().into_tx(); - let mut next_tx_num = storage_kind.tx_offset(); + let mut next_tx_num = tx_offset; let blocks = blocks.into_iter().collect::>(); { - let mut headers_writer = storage_kind - .is_static() - .then(|| provider.latest_writer(StaticFileSegment::Headers).unwrap()); + let mut headers_writer = provider.latest_writer(StaticFileSegment::Headers).unwrap(); blocks.iter().try_for_each(|block| { - Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header(), U256::ZERO) + self.insert_header(&mut headers_writer, &tx, block.sealed_header(), U256::ZERO) })?; - if let Some(mut writer) = headers_writer { - writer.commit()?; - } + headers_writer.commit()?; } { - let mut txs_writer = storage_kind - .is_static() - .then(|| provider.latest_writer(StaticFileSegment::Transactions).unwrap()); + let mut txs_writer = provider.latest_writer(StaticFileSegment::Transactions).unwrap(); blocks.into_iter().try_for_each(|block| { // Insert into body tables. @@ -264,34 +253,26 @@ impl TestStageDB { tx.put::(block.number, block_body_indices)?; let res = block.body().transactions.iter().try_for_each(|body_tx| { - if let Some(txs_writer) = &mut txs_writer { - txs_writer.append_transaction(next_tx_num, body_tx)?; - } else { - tx.put::(next_tx_num, body_tx.clone())? - } + txs_writer.append_transaction(next_tx_num, body_tx)?; next_tx_num += 1; Ok::<(), ProviderError>(()) }); - if let Some(txs_writer) = &mut txs_writer { - // Backfill: some tests start at a forward block number, but static files - // require no gaps. - let segment_header = txs_writer.user_header(); - if segment_header.block_end().is_none() && - segment_header.expected_block_start() == 0 - { - for block in 0..block.number { - txs_writer.increment_block(block)?; - } + // Backfill: some tests start at a forward block number, but static files + // require no gaps. + let segment_header = txs_writer.user_header(); + if segment_header.block_end().is_none() && + segment_header.expected_block_start() == 0 + { + for block in 0..block.number { + txs_writer.increment_block(block)?; } - txs_writer.increment_block(block.number)?; } + txs_writer.increment_block(block.number)?; res })?; - if let Some(txs_writer) = &mut txs_writer { - txs_writer.commit()?; - } + txs_writer.commit()?; } tx.commit()?; @@ -476,21 +457,3 @@ pub enum StorageKind { Database(Option), Static, } - -impl StorageKind { - #[expect(dead_code)] - const fn is_database(&self) -> bool { - matches!(self, Self::Database(_)) - } - - const fn is_static(&self) -> bool { - matches!(self, Self::Static) - } - - fn tx_offset(&self) -> u64 { - if let Self::Database(offset) = self { - return offset.unwrap_or_default(); - } - 0 - } -} diff --git a/crates/static-file/static-file/src/static_file_producer.rs b/crates/static-file/static-file/src/static_file_producer.rs index 9b75e5683d9..7ba3c81b025 100644 --- a/crates/static-file/static-file/src/static_file_producer.rs +++ b/crates/static-file/static-file/src/static_file_producer.rs @@ -255,15 +255,14 @@ mod tests { use crate::static_file_producer::{ StaticFileProducer, StaticFileProducerInner, StaticFileTargets, }; - use alloy_primitives::{B256, U256}; + use alloy_primitives::B256; use assert_matches::assert_matches; - use reth_db_api::{database::Database, transaction::DbTx}; use reth_provider::{ providers::StaticFileWriter, test_utils::MockNodeTypesWithDB, ProviderError, ProviderFactory, StaticFileProviderFactory, }; use reth_prune_types::PruneModes; - use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_stages::test_utils::TestStageDB; use reth_static_file_types::{HighestStaticFiles, StaticFileSegment}; use reth_testing_utils::generators::{ self, random_block_range, random_receipt, BlockRangeParams, @@ -280,7 +279,7 @@ mod tests { 0..=3, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); // Unwind headers from static_files and manually insert them into the database, so we're // able to check that static_file_producer works let static_file_provider = db.factory.static_file_provider(); @@ -291,12 +290,7 @@ mod tests { static_file_writer.commit().expect("prune headers"); drop(static_file_writer); - let tx = db.factory.db_ref().tx_mut().expect("init tx"); - for block in &blocks { - TestStageDB::insert_header(None, &tx, block.sealed_header(), U256::ZERO) - .expect("insert block header"); - } - tx.commit().expect("commit tx"); + db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); let mut receipts = Vec::new(); for block in &blocks { From 4d084f28dc77344447411c83e143f0489298d5e0 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 17:16:52 +0100 Subject: [PATCH 07/13] test: not empty range oops --- crates/prune/prune/src/segments/static_file/headers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/prune/prune/src/segments/static_file/headers.rs b/crates/prune/prune/src/segments/static_file/headers.rs index aa913433ce7..ae97672b304 100644 --- a/crates/prune/prune/src/segments/static_file/headers.rs +++ b/crates/prune/prune/src/segments/static_file/headers.rs @@ -241,7 +241,7 @@ mod tests { let blocks = random_block_range( &mut rng, 0..=99, - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..0, ..Default::default() }, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, ); db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); From 2e509c9bc6431666fd85dae7515c92bc009b00f7 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 17:22:53 +0100 Subject: [PATCH 08/13] test: insert into database for prune tests --- crates/prune/prune/src/segments/receipts.rs | 10 +- .../prune/src/segments/static_file/headers.rs | 40 ++++--- .../src/segments/static_file/transactions.rs | 13 +-- .../src/segments/user/account_history.rs | 4 +- .../src/segments/user/receipts_by_logs.rs | 11 +- .../src/segments/user/sender_recovery.rs | 10 +- .../src/segments/user/storage_history.rs | 4 +- .../src/segments/user/transaction_lookup.rs | 4 +- crates/stages/stages/benches/setup/mod.rs | 4 +- crates/stages/stages/src/stages/merkle.rs | 8 +- crates/stages/stages/src/stages/mod.rs | 4 +- crates/stages/stages/src/stages/prune.rs | 6 +- .../stages/src/stages/sender_recovery.rs | 18 ++-- crates/stages/stages/src/stages/tx_lookup.rs | 18 ++-- .../stages/stages/src/test_utils/test_db.rs | 101 ++++++++++++------ .../static-file/src/static_file_producer.rs | 6 +- 16 files changed, 156 insertions(+), 105 deletions(-) diff --git a/crates/prune/prune/src/segments/receipts.rs b/crates/prune/prune/src/segments/receipts.rs index d114744184e..12ad6e2c203 100644 --- a/crates/prune/prune/src/segments/receipts.rs +++ b/crates/prune/prune/src/segments/receipts.rs @@ -89,11 +89,11 @@ mod tests { Itertools, }; use reth_db_api::tables; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{ self, random_block_range, random_receipt, BlockRangeParams, }; @@ -109,7 +109,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let mut receipts = Vec::new(); for block in &blocks { @@ -125,11 +125,11 @@ mod tests { db.insert_receipts(receipts).expect("insert receipts"); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.table::().unwrap().len(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.table::().unwrap().len(), db.table::().unwrap().len() ); diff --git a/crates/prune/prune/src/segments/static_file/headers.rs b/crates/prune/prune/src/segments/static_file/headers.rs index ae97672b304..9f3c291bf44 100644 --- a/crates/prune/prune/src/segments/static_file/headers.rs +++ b/crates/prune/prune/src/segments/static_file/headers.rs @@ -216,19 +216,19 @@ mod tests { static_file::headers::HEADER_TABLES_TO_PRUNE, PruneInput, PruneLimiter, Segment, SegmentOutput, }; - use alloy_primitives::{BlockNumber, B256}; + use alloy_primitives::{BlockNumber, B256, U256}; use assert_matches::assert_matches; - use reth_db_api::tables; + use reth_db_api::{tables, transaction::DbTx}; use reth_provider::{ DBProvider, DatabaseProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, - StaticFileProviderFactory, StatsReader, + StaticFileProviderFactory, }; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, SegmentOutputCheckpoint, }; use reth_stages::test_utils::TestStageDB; - use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; + use reth_testing_utils::{generators, generators::random_header_range}; use tracing::trace; #[test] @@ -238,17 +238,16 @@ mod tests { let db = TestStageDB::default(); let mut rng = generators::rng(); - let blocks = random_block_range( - &mut rng, - 0..=99, - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, - ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + let headers = random_header_range(&mut rng, 0..100, B256::ZERO); + let tx = db.factory.provider_rw().unwrap().into_tx(); + for header in &headers { + TestStageDB::insert_header(None, &tx, header, U256::ZERO).unwrap(); + } + tx.commit().unwrap(); - assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), - blocks.len() - ); + assert_eq!(db.table::().unwrap().len(), headers.len()); + assert_eq!(db.table::().unwrap().len(), headers.len()); + assert_eq!(db.table::().unwrap().len(), headers.len()); let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { let segment = super::Headers::new(db.factory.static_file_provider()); @@ -305,10 +304,17 @@ mod tests { ); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), - blocks.len() - (last_pruned_block_number + 1) as usize + db.table::().unwrap().len(), + headers.len() - (last_pruned_block_number + 1) as usize + ); + assert_eq!( + db.table::().unwrap().len(), + headers.len() - (last_pruned_block_number + 1) as usize + ); + assert_eq!( + db.table::().unwrap().len(), + headers.len() - (last_pruned_block_number + 1) as usize ); - assert_eq!( db.factory.provider().unwrap().get_prune_checkpoint(PruneSegment::Headers).unwrap(), Some(PruneCheckpoint { diff --git a/crates/prune/prune/src/segments/static_file/transactions.rs b/crates/prune/prune/src/segments/static_file/transactions.rs index 2e236622a96..115ee2ca39a 100644 --- a/crates/prune/prune/src/segments/static_file/transactions.rs +++ b/crates/prune/prune/src/segments/static_file/transactions.rs @@ -102,13 +102,13 @@ mod tests { use reth_db_api::tables; use reth_provider::{ DBProvider, DatabaseProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, - StaticFileProviderFactory, StatsReader, + StaticFileProviderFactory, }; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, SegmentOutput, }; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; @@ -122,15 +122,12 @@ mod tests { 1..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let transactions = blocks.iter().flat_map(|block| &block.body().transactions).collect::>(); - assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), - transactions.len() - ); + assert_eq!(db.table::().unwrap().len(), transactions.len()); let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { let segment = super::Transactions::new(db.factory.static_file_provider()); @@ -202,7 +199,7 @@ mod tests { .checked_sub(if result.progress.is_finished() { 0 } else { 1 }); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.table::().unwrap().len(), transactions.len() - (last_pruned_tx_number + 1) ); assert_eq!( diff --git a/crates/prune/prune/src/segments/user/account_history.rs b/crates/prune/prune/src/segments/user/account_history.rs index 3965da41c9d..3c18cd1befc 100644 --- a/crates/prune/prune/src/segments/user/account_history.rs +++ b/crates/prune/prune/src/segments/user/account_history.rs @@ -137,7 +137,7 @@ mod tests { use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, }; @@ -153,7 +153,7 @@ mod tests { 1..=5000, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::>(); diff --git a/crates/prune/prune/src/segments/user/receipts_by_logs.rs b/crates/prune/prune/src/segments/user/receipts_by_logs.rs index 3208f9ec041..0849db52518 100644 --- a/crates/prune/prune/src/segments/user/receipts_by_logs.rs +++ b/crates/prune/prune/src/segments/user/receipts_by_logs.rs @@ -233,11 +233,10 @@ mod tests { use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; use reth_primitives_traits::InMemorySize; use reth_provider::{ - DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader, - TransactionsProvider, + DBProvider, DatabaseProviderFactory, PruneCheckpointReader, TransactionsProvider, }; use reth_prune_types::{PruneMode, PruneSegment, ReceiptsLogPruneConfig}; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{ self, random_block_range, random_eoa_account, random_log, random_receipt, BlockRangeParams, }; @@ -269,7 +268,7 @@ mod tests { ), ] .concat(); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let mut receipts = Vec::new(); @@ -289,11 +288,11 @@ mod tests { db.insert_receipts(receipts).expect("insert receipts"); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.table::().unwrap().len(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.table::().unwrap().len(), db.table::().unwrap().len() ); diff --git a/crates/prune/prune/src/segments/user/sender_recovery.rs b/crates/prune/prune/src/segments/user/sender_recovery.rs index 62d035d066a..35ee487203a 100644 --- a/crates/prune/prune/src/segments/user/sender_recovery.rs +++ b/crates/prune/prune/src/segments/user/sender_recovery.rs @@ -91,9 +91,9 @@ mod tests { }; use reth_db_api::tables; use reth_primitives_traits::SignerRecoverable; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment}; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; @@ -107,7 +107,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let mut transaction_senders = Vec::new(); for block in &blocks { @@ -123,11 +123,11 @@ mod tests { db.insert_transaction_senders(transaction_senders).expect("insert transaction senders"); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.table::().unwrap().len(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.table::().unwrap().len(), db.table::().unwrap().len() ); diff --git a/crates/prune/prune/src/segments/user/storage_history.rs b/crates/prune/prune/src/segments/user/storage_history.rs index cedd1b1c174..ee7447c37da 100644 --- a/crates/prune/prune/src/segments/user/storage_history.rs +++ b/crates/prune/prune/src/segments/user/storage_history.rs @@ -142,7 +142,7 @@ mod tests { use reth_db_api::{tables, BlockNumberList}; use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment}; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, }; @@ -158,7 +158,7 @@ mod tests { 0..=5000, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::>(); diff --git a/crates/prune/prune/src/segments/user/transaction_lookup.rs b/crates/prune/prune/src/segments/user/transaction_lookup.rs index 7fbd4424a3c..0a47d04327e 100644 --- a/crates/prune/prune/src/segments/user/transaction_lookup.rs +++ b/crates/prune/prune/src/segments/user/transaction_lookup.rs @@ -143,7 +143,7 @@ mod tests { use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; @@ -157,7 +157,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::StaticFile).expect("insert blocks"); let mut tx_hash_numbers = Vec::new(); for block in &blocks { diff --git a/crates/stages/stages/benches/setup/mod.rs b/crates/stages/stages/benches/setup/mod.rs index f11b247784c..f06db6d7b80 100644 --- a/crates/stages/stages/benches/setup/mod.rs +++ b/crates/stages/stages/benches/setup/mod.rs @@ -14,7 +14,7 @@ use reth_provider::{ }; use reth_stages::{ stages::{AccountHashingStage, StorageHashingStage}, - test_utils::TestStageDB, + test_utils::{StorageKind, TestStageDB}, }; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_contract_account_range, @@ -197,7 +197,7 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> TestStageDB { cloned_last.into_body(), ); - db.insert_blocks(blocks.iter(), 0).unwrap(); + db.insert_blocks(blocks.iter(), StorageKind::StaticFile).unwrap(); // initialize TD db.commit(|tx| { diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index fe8dc22b3e0..67f74a1b520 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -442,8 +442,8 @@ fn validate_state_root( mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, - TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, + TestRunnerError, TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::{keccak256, U256}; use assert_matches::assert_matches; @@ -653,7 +653,7 @@ mod tests { ..Default::default() }, )); - self.db.insert_blocks(preblocks.iter(), 0)?; + self.db.insert_blocks(preblocks.iter(), StorageKind::StaticFile)?; } let num_of_accounts = 31; @@ -692,7 +692,7 @@ mod tests { BlockRangeParams { parent: Some(head_hash), tx_count: 0..3, ..Default::default() }, )); let last_block = blocks.last().cloned().unwrap(); - self.db.insert_blocks(blocks.iter(), 0)?; + self.db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; let (transitions, final_state) = random_changeset_range( &mut rng, diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index 0e31e35aa86..dce49f0f753 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -270,7 +270,7 @@ mod tests { 0..=tip, BlockRangeParams { parent: Some(genesis_hash), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0)?; + db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; let mut receipts = Vec::with_capacity(blocks.len()); let mut tx_num = 0u64; @@ -282,7 +282,7 @@ mod tests { } receipts.push((block.number, block_receipts)); } - db.insert_receipts_by_block(receipts, StorageKind::Static)?; + db.insert_receipts_by_block(receipts, StorageKind::StaticFile)?; // simulate pipeline by setting all checkpoints to inserted height. let provider_rw = db.factory.provider_rw()?; diff --git a/crates/stages/stages/src/stages/prune.rs b/crates/stages/stages/src/stages/prune.rs index 3284dae955e..675391060c6 100644 --- a/crates/stages/stages/src/stages/prune.rs +++ b/crates/stages/stages/src/stages/prune.rs @@ -171,8 +171,8 @@ where mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, - TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, + TestRunnerError, TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::B256; use reth_ethereum_primitives::Block; @@ -218,7 +218,7 @@ mod tests { input.checkpoint().block_number..=input.target(), BlockRangeParams { parent: Some(B256::ZERO), tx_count: 1..3, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), 0)?; + self.db.insert_blocks(blocks.iter(), StorageKind::Database(None))?; self.db.insert_transaction_senders( blocks.iter().flat_map(|block| block.body().transactions.iter()).enumerate().map( |(i, tx)| (i as u64, tx.recover_signer().expect("failed to recover signer")), diff --git a/crates/stages/stages/src/stages/sender_recovery.rs b/crates/stages/stages/src/stages/sender_recovery.rs index 73c5f8b75d2..02bbbeba1ba 100644 --- a/crates/stages/stages/src/stages/sender_recovery.rs +++ b/crates/stages/stages/src/stages/sender_recovery.rs @@ -369,8 +369,8 @@ struct FailedSenderRecoveryError { mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, - TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, + TestRunnerError, TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; @@ -416,7 +416,10 @@ mod tests { ) }) .collect::>(); - runner.db.insert_blocks(blocks.iter(), 0).expect("failed to insert blocks"); + runner + .db + .insert_blocks(blocks.iter(), StorageKind::StaticFile) + .expect("failed to insert blocks"); let rx = runner.execute(input); @@ -453,7 +456,10 @@ mod tests { stage_progress + 1..=previous_stage, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..4, ..Default::default() }, ); // set tx count range high enough to hit the threshold - runner.db.insert_blocks(seed.iter(), 0).expect("failed to seed execution"); + runner + .db + .insert_blocks(seed.iter(), StorageKind::StaticFile) + .expect("failed to seed execution"); let total_transactions = runner .db @@ -524,7 +530,7 @@ mod tests { 0..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..10, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::StaticFile).expect("insert blocks"); let max_pruned_block = 30; let max_processed_block = 70; @@ -641,7 +647,7 @@ mod tests { stage_progress..=end, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), 0)?; + self.db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; Ok(blocks) } diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index 53cc48aa639..b52b713c33f 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -254,8 +254,8 @@ where mod tests { use super::*; use crate::test_utils::{ - stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, - TestStageDB, UnwindStageTestRunner, + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, StorageKind, + TestRunnerError, TestStageDB, UnwindStageTestRunner, }; use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; @@ -301,7 +301,10 @@ mod tests { ) }) .collect::>(); - runner.db.insert_blocks(blocks.iter(), 0).expect("failed to insert blocks"); + runner + .db + .insert_blocks(blocks.iter(), StorageKind::StaticFile) + .expect("failed to insert blocks"); let rx = runner.execute(input); @@ -342,7 +345,10 @@ mod tests { stage_progress + 1..=previous_stage, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - runner.db.insert_blocks(seed.iter(), 0).expect("failed to seed execution"); + runner + .db + .insert_blocks(seed.iter(), StorageKind::StaticFile) + .expect("failed to seed execution"); runner.set_prune_mode(PruneMode::Before(prune_target)); @@ -377,7 +383,7 @@ mod tests { 0..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..10, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::StaticFile).expect("insert blocks"); let max_pruned_block = 30; let max_processed_block = 70; @@ -507,7 +513,7 @@ mod tests { stage_progress + 1..=end, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), 0)?; + self.db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; Ok(blocks) } diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index 1b1b84d8206..0940cb43c58 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -142,24 +142,29 @@ impl TestStageDB { /// Insert header to static file if `writer` exists, otherwise to DB. pub fn insert_header( - &self, - writer: &mut StaticFileProviderRWRefMut<'_, EthPrimitives>, + writer: Option<&mut StaticFileProviderRWRefMut<'_, EthPrimitives>>, tx: &TX, header: &SealedHeader, td: U256, ) -> ProviderResult<()> { - // Backfill: some tests start at a forward block number, but static files require no - // gaps. - let segment_header = writer.user_header(); - if segment_header.block_end().is_none() && segment_header.expected_block_start() == 0 { - for block_number in 0..header.number { - let mut prev = header.clone_header(); - prev.number = block_number; - writer.append_header(&prev, U256::ZERO, &B256::ZERO)?; + if let Some(writer) = writer { + // Backfill: some tests start at a forward block number, but static files require no + // gaps. + let segment_header = writer.user_header(); + if segment_header.block_end().is_none() && segment_header.expected_block_start() == 0 { + for block_number in 0..header.number { + let mut prev = header.clone_header(); + prev.number = block_number; + writer.append_header(&prev, U256::ZERO, &B256::ZERO)?; + } } - } - writer.append_header(header.header(), td, &header.hash())?; + writer.append_header(header.header(), td, &header.hash())?; + } else { + tx.put::(header.number, header.hash())?; + tx.put::(header.number, td.into())?; + tx.put::(header.number, header.header().clone())?; + } tx.put::(header.hash(), header.number)?; Ok(()) @@ -178,7 +183,7 @@ impl TestStageDB { if TD { td += header.difficulty; } - self.insert_header(&mut writer, &tx, header, td)?; + Self::insert_header(Some(&mut writer), &tx, header, td)?; } writer.commit()?; @@ -213,29 +218,35 @@ impl TestStageDB { /// database. /// /// Assumes that there's a single transition for each transaction (i.e. no block rewards). - pub fn insert_blocks<'a, I>(&self, blocks: I, tx_offset: u64) -> ProviderResult<()> + pub fn insert_blocks<'a, I>(&self, blocks: I, storage_kind: StorageKind) -> ProviderResult<()> where I: IntoIterator>, { let provider = self.factory.static_file_provider(); let tx = self.factory.provider_rw().unwrap().into_tx(); - let mut next_tx_num = tx_offset; + let mut next_tx_num = storage_kind.tx_offset(); let blocks = blocks.into_iter().collect::>(); { - let mut headers_writer = provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut headers_writer = storage_kind + .is_static() + .then(|| provider.latest_writer(StaticFileSegment::Headers).unwrap()); blocks.iter().try_for_each(|block| { - self.insert_header(&mut headers_writer, &tx, block.sealed_header(), U256::ZERO) + Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header(), U256::ZERO) })?; - headers_writer.commit()?; + if let Some(mut writer) = headers_writer { + writer.commit()?; + } } { - let mut txs_writer = provider.latest_writer(StaticFileSegment::Transactions).unwrap(); + let mut txs_writer = storage_kind + .is_static() + .then(|| provider.latest_writer(StaticFileSegment::Transactions).unwrap()); blocks.into_iter().try_for_each(|block| { // Insert into body tables. @@ -253,26 +264,34 @@ impl TestStageDB { tx.put::(block.number, block_body_indices)?; let res = block.body().transactions.iter().try_for_each(|body_tx| { - txs_writer.append_transaction(next_tx_num, body_tx)?; + if let Some(txs_writer) = &mut txs_writer { + txs_writer.append_transaction(next_tx_num, body_tx)?; + } else { + tx.put::(next_tx_num, body_tx.clone())? + } next_tx_num += 1; Ok::<(), ProviderError>(()) }); - // Backfill: some tests start at a forward block number, but static files - // require no gaps. - let segment_header = txs_writer.user_header(); - if segment_header.block_end().is_none() && - segment_header.expected_block_start() == 0 - { - for block in 0..block.number { - txs_writer.increment_block(block)?; + if let Some(txs_writer) = &mut txs_writer { + // Backfill: some tests start at a forward block number, but static files + // require no gaps. + let segment_header = txs_writer.user_header(); + if segment_header.block_end().is_none() && + segment_header.expected_block_start() == 0 + { + for block in 0..block.number { + txs_writer.increment_block(block)?; + } } + txs_writer.increment_block(block.number)?; } - txs_writer.increment_block(block.number)?; res })?; - txs_writer.commit()?; + if let Some(txs_writer) = &mut txs_writer { + txs_writer.commit()?; + } } tx.commit()?; @@ -325,7 +344,7 @@ impl TestStageDB { Ok(()) }) }), - StorageKind::Static => { + StorageKind::StaticFile => { let provider = self.factory.static_file_provider(); let mut writer = provider.latest_writer(StaticFileSegment::Receipts)?; let res = receipts.into_iter().try_for_each(|(block_num, receipts)| { @@ -455,5 +474,23 @@ impl TestStageDB { #[derive(Debug)] pub enum StorageKind { Database(Option), - Static, + StaticFile, +} + +impl StorageKind { + #[expect(dead_code)] + const fn is_database(&self) -> bool { + matches!(self, Self::Database(_)) + } + + const fn is_static(&self) -> bool { + matches!(self, Self::StaticFile) + } + + fn tx_offset(&self) -> u64 { + if let Self::Database(offset) = self { + return offset.unwrap_or_default(); + } + 0 + } } diff --git a/crates/static-file/static-file/src/static_file_producer.rs b/crates/static-file/static-file/src/static_file_producer.rs index 7ba3c81b025..73c726c6b14 100644 --- a/crates/static-file/static-file/src/static_file_producer.rs +++ b/crates/static-file/static-file/src/static_file_producer.rs @@ -262,7 +262,7 @@ mod tests { ProviderFactory, StaticFileProviderFactory, }; use reth_prune_types::PruneModes; - use reth_stages::test_utils::TestStageDB; + use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_static_file_types::{HighestStaticFiles, StaticFileSegment}; use reth_testing_utils::generators::{ self, random_block_range, random_receipt, BlockRangeParams, @@ -279,7 +279,7 @@ mod tests { 0..=3, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); // Unwind headers from static_files and manually insert them into the database, so we're // able to check that static_file_producer works let static_file_provider = db.factory.static_file_provider(); @@ -290,7 +290,7 @@ mod tests { static_file_writer.commit().expect("prune headers"); drop(static_file_writer); - db.insert_blocks(blocks.iter(), 0).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let mut receipts = Vec::new(); for block in &blocks { From 20d0d63f4f9bb8c1214f91263fd68cf4ad213ea9 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:06:05 +0100 Subject: [PATCH 09/13] test: insert dummy TD in bodies downloader --- crates/net/downloaders/src/bodies/test_utils.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index 9c459571d13..ff5180291f0 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -55,17 +55,8 @@ pub(crate) fn insert_headers( .expect("failed to create writer"); for header in headers { - let ttd = if header.number() == 0 { - header.difficulty() - } else { - let parent_block_number = header.number() - 1; - let parent_ttd = - provider_rw.header_td_by_number(parent_block_number).unwrap().unwrap_or_default(); - parent_ttd + header.difficulty() - }; - writer - .append_header(header.header(), ttd, &header.hash()) + .append_header(header.header(), U256::ZERO, &header.hash()) .expect("failed to append header"); } drop(writer); From dca1454fb6fadf98709734af6df3e4ab895663dd Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:07:09 +0100 Subject: [PATCH 10/13] chore: rename StorageKind::StaticFile back to StorageKind::Static --- crates/stages/stages/benches/setup/mod.rs | 2 +- crates/stages/stages/src/stages/merkle.rs | 4 ++-- crates/stages/stages/src/stages/mod.rs | 4 ++-- crates/stages/stages/src/stages/sender_recovery.rs | 8 ++++---- crates/stages/stages/src/stages/tx_lookup.rs | 8 ++++---- crates/stages/stages/src/test_utils/test_db.rs | 6 +++--- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/stages/stages/benches/setup/mod.rs b/crates/stages/stages/benches/setup/mod.rs index f06db6d7b80..bd1fb59ebe9 100644 --- a/crates/stages/stages/benches/setup/mod.rs +++ b/crates/stages/stages/benches/setup/mod.rs @@ -197,7 +197,7 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> TestStageDB { cloned_last.into_body(), ); - db.insert_blocks(blocks.iter(), StorageKind::StaticFile).unwrap(); + db.insert_blocks(blocks.iter(), StorageKind::Static).unwrap(); // initialize TD db.commit(|tx| { diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index 67f74a1b520..c5115316243 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -653,7 +653,7 @@ mod tests { ..Default::default() }, )); - self.db.insert_blocks(preblocks.iter(), StorageKind::StaticFile)?; + self.db.insert_blocks(preblocks.iter(), StorageKind::Static)?; } let num_of_accounts = 31; @@ -692,7 +692,7 @@ mod tests { BlockRangeParams { parent: Some(head_hash), tx_count: 0..3, ..Default::default() }, )); let last_block = blocks.last().cloned().unwrap(); - self.db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; + self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; let (transitions, final_state) = random_changeset_range( &mut rng, diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index dce49f0f753..f9b2312f5ab 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -270,7 +270,7 @@ mod tests { 0..=tip, BlockRangeParams { parent: Some(genesis_hash), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; + db.insert_blocks(blocks.iter(), StorageKind::Static)?; let mut receipts = Vec::with_capacity(blocks.len()); let mut tx_num = 0u64; @@ -282,7 +282,7 @@ mod tests { } receipts.push((block.number, block_receipts)); } - db.insert_receipts_by_block(receipts, StorageKind::StaticFile)?; + db.insert_receipts_by_block(receipts, StorageKind::Static)?; // simulate pipeline by setting all checkpoints to inserted height. let provider_rw = db.factory.provider_rw()?; diff --git a/crates/stages/stages/src/stages/sender_recovery.rs b/crates/stages/stages/src/stages/sender_recovery.rs index 02bbbeba1ba..2a2870f07ca 100644 --- a/crates/stages/stages/src/stages/sender_recovery.rs +++ b/crates/stages/stages/src/stages/sender_recovery.rs @@ -418,7 +418,7 @@ mod tests { .collect::>(); runner .db - .insert_blocks(blocks.iter(), StorageKind::StaticFile) + .insert_blocks(blocks.iter(), StorageKind::Static) .expect("failed to insert blocks"); let rx = runner.execute(input); @@ -458,7 +458,7 @@ mod tests { ); // set tx count range high enough to hit the threshold runner .db - .insert_blocks(seed.iter(), StorageKind::StaticFile) + .insert_blocks(seed.iter(), StorageKind::Static) .expect("failed to seed execution"); let total_transactions = runner @@ -530,7 +530,7 @@ mod tests { 0..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..10, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::StaticFile).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks"); let max_pruned_block = 30; let max_processed_block = 70; @@ -647,7 +647,7 @@ mod tests { stage_progress..=end, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; + self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; Ok(blocks) } diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index b52b713c33f..20a0770d8c8 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -303,7 +303,7 @@ mod tests { .collect::>(); runner .db - .insert_blocks(blocks.iter(), StorageKind::StaticFile) + .insert_blocks(blocks.iter(), StorageKind::Static) .expect("failed to insert blocks"); let rx = runner.execute(input); @@ -347,7 +347,7 @@ mod tests { ); runner .db - .insert_blocks(seed.iter(), StorageKind::StaticFile) + .insert_blocks(seed.iter(), StorageKind::Static) .expect("failed to seed execution"); runner.set_prune_mode(PruneMode::Before(prune_target)); @@ -383,7 +383,7 @@ mod tests { 0..=100, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..10, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::StaticFile).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks"); let max_pruned_block = 30; let max_processed_block = 70; @@ -513,7 +513,7 @@ mod tests { stage_progress + 1..=end, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..2, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), StorageKind::StaticFile)?; + self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; Ok(blocks) } diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index 0940cb43c58..f3e29c1fa66 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -344,7 +344,7 @@ impl TestStageDB { Ok(()) }) }), - StorageKind::StaticFile => { + StorageKind::Static => { let provider = self.factory.static_file_provider(); let mut writer = provider.latest_writer(StaticFileSegment::Receipts)?; let res = receipts.into_iter().try_for_each(|(block_num, receipts)| { @@ -474,7 +474,7 @@ impl TestStageDB { #[derive(Debug)] pub enum StorageKind { Database(Option), - StaticFile, + Static, } impl StorageKind { @@ -484,7 +484,7 @@ impl StorageKind { } const fn is_static(&self) -> bool { - matches!(self, Self::StaticFile) + matches!(self, Self::Static) } fn tx_offset(&self) -> u64 { From 35c498477db7f2a21dac7555fff5b5f0e76120d3 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:11:55 +0100 Subject: [PATCH 11/13] refactor: introduce TestStageDB::count_entries --- crates/net/downloaders/src/bodies/test_utils.rs | 6 +++--- .../prune/src/segments/user/transaction_lookup.rs | 8 ++++---- crates/stages/stages/src/stages/hashing_account.rs | 2 +- crates/stages/stages/src/stages/hashing_storage.rs | 4 ++-- crates/stages/stages/src/stages/merkle.rs | 12 ++++++------ crates/stages/stages/src/stages/prune.rs | 2 +- crates/stages/stages/src/stages/sender_recovery.rs | 2 +- crates/stages/stages/src/stages/tx_lookup.rs | 5 ++--- crates/stages/stages/src/test_utils/test_db.rs | 7 ++++++- 9 files changed, 26 insertions(+), 22 deletions(-) diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index ff5180291f0..a7172ec1a00 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -3,13 +3,13 @@ #![allow(dead_code)] use alloy_consensus::BlockHeader; -use alloy_primitives::B256; +use alloy_primitives::{B256, U256}; use reth_ethereum_primitives::BlockBody; use reth_network_p2p::bodies::response::BlockResponse; use reth_primitives_traits::{Block, SealedBlock, SealedHeader}; use reth_provider::{ - test_utils::MockNodeTypesWithDB, HeaderProvider, ProviderFactory, StaticFileProviderFactory, - StaticFileSegment, StaticFileWriter, + test_utils::MockNodeTypesWithDB, ProviderFactory, StaticFileProviderFactory, StaticFileSegment, + StaticFileWriter, }; use std::collections::HashMap; diff --git a/crates/prune/prune/src/segments/user/transaction_lookup.rs b/crates/prune/prune/src/segments/user/transaction_lookup.rs index 0a47d04327e..e218f623ed5 100644 --- a/crates/prune/prune/src/segments/user/transaction_lookup.rs +++ b/crates/prune/prune/src/segments/user/transaction_lookup.rs @@ -139,7 +139,7 @@ mod tests { Itertools, }; use reth_db_api::tables; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StatsReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; @@ -157,7 +157,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::StaticFile).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks"); let mut tx_hash_numbers = Vec::new(); for block in &blocks { @@ -170,11 +170,11 @@ mod tests { db.insert_tx_hash_numbers(tx_hash_numbers).expect("insert tx hash numbers"); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.count_entries::().unwrap(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.factory.provider().unwrap().count_entries::().unwrap(), + db.count_entries::().unwrap(), db.table::().unwrap().len() ); diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index c48381d4fe9..cc86db14d38 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -344,7 +344,7 @@ mod tests { done: true, }) if block_number == previous_stage && processed == total && - total == runner.db.table::().unwrap().len() as u64 + total == runner.db.count_entries::().unwrap() as u64 ); // Validate the stage execution diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index e0eb9716537..c52f800a018 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -266,7 +266,7 @@ mod tests { }, .. }) if processed == previous_checkpoint.progress.processed + 1 && - total == runner.db.table::().unwrap().len() as u64); + total == runner.db.count_entries::().unwrap() as u64); // Continue from checkpoint input.checkpoint = Some(checkpoint); @@ -280,7 +280,7 @@ mod tests { }, .. }) if processed == total && - total == runner.db.table::().unwrap().len() as u64); + total == runner.db.count_entries::().unwrap() as u64); // Validate the stage execution assert!( diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index c5115316243..6cbed3ab20e 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -493,8 +493,8 @@ mod tests { done: true }) if block_number == previous_stage && processed == total && total == ( - runner.db.table::().unwrap().len() + - runner.db.table::().unwrap().len() + runner.db.count_entries::().unwrap() + + runner.db.count_entries::().unwrap() ) as u64 ); @@ -533,8 +533,8 @@ mod tests { done: true }) if block_number == previous_stage && processed == total && total == ( - runner.db.table::().unwrap().len() + - runner.db.table::().unwrap().len() + runner.db.count_entries::().unwrap() + + runner.db.count_entries::().unwrap() ) as u64 ); @@ -575,8 +575,8 @@ mod tests { done: true }) if block_number == previous_stage && processed == total && total == ( - runner.db.table::().unwrap().len() + - runner.db.table::().unwrap().len() + runner.db.count_entries::().unwrap() + + runner.db.count_entries::().unwrap() ) as u64 ); diff --git a/crates/stages/stages/src/stages/prune.rs b/crates/stages/stages/src/stages/prune.rs index 675391060c6..f62259dcfdd 100644 --- a/crates/stages/stages/src/stages/prune.rs +++ b/crates/stages/stages/src/stages/prune.rs @@ -218,7 +218,7 @@ mod tests { input.checkpoint().block_number..=input.target(), BlockRangeParams { parent: Some(B256::ZERO), tx_count: 1..3, ..Default::default() }, ); - self.db.insert_blocks(blocks.iter(), StorageKind::Database(None))?; + self.db.insert_blocks(blocks.iter(), StorageKind::Static)?; self.db.insert_transaction_senders( blocks.iter().flat_map(|block| block.body().transactions.iter()).enumerate().map( |(i, tx)| (i as u64, tx.recover_signer().expect("failed to recover signer")), diff --git a/crates/stages/stages/src/stages/sender_recovery.rs b/crates/stages/stages/src/stages/sender_recovery.rs index 2a2870f07ca..947f0620954 100644 --- a/crates/stages/stages/src/stages/sender_recovery.rs +++ b/crates/stages/stages/src/stages/sender_recovery.rs @@ -490,7 +490,7 @@ mod tests { ExecOutput { checkpoint: StageCheckpoint::new(expected_progress).with_entities_stage_checkpoint( EntitiesCheckpoint { - processed: runner.db.table::().unwrap().len() + processed: runner.db.count_entries::().unwrap() as u64, total: total_transactions } diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index 20a0770d8c8..8b1c531736b 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -264,7 +264,6 @@ mod tests { use reth_primitives_traits::SealedBlock; use reth_provider::{ providers::StaticFileWriter, BlockBodyIndicesProvider, DatabaseProviderFactory, - StaticFileProviderFactory, }; use reth_stages_api::StageUnitCheckpoint; use reth_testing_utils::generators::{ @@ -320,7 +319,7 @@ mod tests { total })) }, done: true }) if block_number == previous_stage && processed == total && - total == runner.db.factory.static_file_provider().count_entries::().unwrap() as u64 + total == runner.db.count_entries::().unwrap() as u64 ); // Validate the stage execution @@ -366,7 +365,7 @@ mod tests { total })) }, done: true }) if block_number == previous_stage && processed == total && - total == runner.db.factory.static_file_provider().count_entries::().unwrap() as u64 + total == runner.db.count_entries::().unwrap() as u64 ); // Validate the stage execution diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index f3e29c1fa66..f38f77b2247 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -19,7 +19,7 @@ use reth_primitives_traits::{Account, SealedBlock, SealedHeader, StorageEntry}; use reth_provider::{ providers::{StaticFileProvider, StaticFileProviderRWRefMut, StaticFileWriter}, test_utils::MockNodeTypesWithDB, - HistoryWriter, ProviderError, ProviderFactory, StaticFileProviderFactory, + HistoryWriter, ProviderError, ProviderFactory, StaticFileProviderFactory, StatsReader, }; use reth_static_file_types::StaticFileSegment; use reth_storage_errors::provider::ProviderResult; @@ -103,6 +103,11 @@ impl TestStageDB { }) } + /// Return the number of entries in the table or static file segment + pub fn count_entries(&self) -> ProviderResult { + self.factory.provider()?.count_entries::() + } + /// Check that there is no table entry above a given /// number by [`Table::Key`] pub fn ensure_no_entry_above(&self, num: u64, mut selector: F) -> ProviderResult<()> From 39623deaa127f7868376fe00ae80f2aca478ac24 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Fri, 3 Oct 2025 14:43:24 +0100 Subject: [PATCH 12/13] fix: handle MissingStaticFileTx error --- .../src/providers/static_file/manager.rs | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index ab98debd0f4..0438036608d 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1109,20 +1109,31 @@ impl StaticFileProvider { F: FnMut(&mut StaticFileCursor<'_>, u64) -> ProviderResult>, P: FnMut(&T) -> bool, { - let get_provider = |start: u64| { - if segment.is_block_based() { - self.get_segment_provider_from_block(segment, start, None) - } else { - self.get_segment_provider_from_transaction(segment, start, None) - } - }; - let mut result = Vec::with_capacity((range.end - range.start).min(100) as usize); - let mut provider = match get_provider(range.start) { - Ok(provider) => provider, - Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(vec![]), - Err(err) => return Err(err), - }; + + /// Resolves to the provider for the given block or transaction number. + /// + /// If the static file is missing, the `result` is returned. + macro_rules! get_provider { + ($number:expr) => {{ + let provider = if segment.is_block_based() { + self.get_segment_provider_from_block(segment, $number, None) + } else { + self.get_segment_provider_from_transaction(segment, $number, None) + }; + + match provider { + Ok(provider) => provider, + Err( + ProviderError::MissingStaticFileBlock(_, _) | + ProviderError::MissingStaticFileTx(_, _), + ) => return Ok(result), + Err(err) => return Err(err), + } + }}; + } + + let mut provider = get_provider!(range.start); let mut cursor = provider.cursor()?; // advances number in range @@ -1164,11 +1175,7 @@ impl StaticFileProvider { // before requesting the next one. drop(cursor); drop(provider); - provider = match get_provider(number) { - Ok(provider) => provider, - Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(result), - Err(err) => return Err(err), - }; + provider = get_provider!(number); cursor = provider.cursor()?; retrying = true; } From 3338eb772f7c75c6a9902e07d8908e4ddd5633e0 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Fri, 3 Oct 2025 14:49:22 +0100 Subject: [PATCH 13/13] fix: return result after first retry --- .../provider/src/providers/static_file/manager.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 0438036608d..434d3836fb2 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1155,19 +1155,7 @@ impl StaticFileProvider { } None => { if retrying { - warn!( - target: "provider::static_file", - ?segment, - ?number, - "Could not find block or tx number on a range request" - ); - - let err = if segment.is_block_based() { - ProviderError::MissingStaticFileBlock(segment, number) - } else { - ProviderError::MissingStaticFileTx(segment, number) - }; - return Err(err) + return Ok(result) } // There is a very small chance of hitting a deadlock if two consecutive // static files share the same bucket in the