diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 06c90465d41..bfdd6a711cb 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -51,7 +51,6 @@ use reth_provider::{ providers::BlockchainProvider, BlockHashReader, BlockReader, CanonStateSubscriptions, HeaderProvider, ProviderFactory, StageCheckpointReader, }; -use reth_prune::BatchSizes; use reth_revm::Factory; use reth_revm_inspectors::stack::Hook; use reth_rpc_engine_api::EngineApi; @@ -429,7 +428,7 @@ impl NodeCommand { self.chain.clone(), prune_config.block_interval, prune_config.parts, - BatchSizes::default(), + self.chain.prune_batch_sizes, ) }); diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index cc54312553e..15dcb42f1ad 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -283,14 +283,14 @@ impl Default for IndexHistoryConfig { #[serde(default)] pub struct PruneConfig { /// Minimum pruning interval measured in blocks. - pub block_interval: u64, + pub block_interval: usize, /// Pruning configuration for every part of the data that can be pruned. pub parts: PruneModes, } impl Default for PruneConfig { fn default() -> Self { - Self { block_interval: 10, parts: PruneModes::default() } + Self { block_interval: 5, parts: PruneModes::default() } } } diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 78b3825e6b3..2aafd028da3 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -20,12 +20,12 @@ use reth_interfaces::{ test_utils::{NoopFullBlockClient, TestConsensus}, }; use reth_payload_builder::test_utils::spawn_test_payload_service; -use reth_primitives::{BlockNumber, ChainSpec, PruneModes, H256, U256}; +use reth_primitives::{BlockNumber, ChainSpec, PruneBatchSizes, PruneModes, H256, U256}; use reth_provider::{ providers::BlockchainProvider, test_utils::TestExecutorFactory, BlockExecutor, ExecutorFactory, ProviderFactory, StateProvider, }; -use reth_prune::{BatchSizes, Pruner}; +use reth_prune::Pruner; use reth_revm::Factory; use reth_rpc_types::engine::{ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus}; use reth_stages::{ @@ -470,7 +470,7 @@ where self.base_config.chain_spec.clone(), 5, PruneModes::default(), - BatchSizes::default(), + PruneBatchSizes::default(), ); let (mut engine, handle) = BeaconConsensusEngine::new( diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index f82d93052c2..6b4bfffa0d2 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -7,7 +7,7 @@ use crate::{ header::Head, proofs::genesis_state_root, Address, BlockNumber, Chain, ForkFilter, ForkHash, ForkId, Genesis, Hardfork, Header, - SealedHeader, H160, H256, U256, + PruneBatchSizes, SealedHeader, H160, H256, U256, }; use hex_literal::hex; use once_cell::sync::Lazy; @@ -63,7 +63,8 @@ pub static MAINNET: Lazy> = Lazy::new(|| { 11052984, H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")), )), - ..Default::default() + base_fee_params: BaseFeeParams::ethereum(), + prune_batch_sizes: PruneBatchSizes::mainnet(), } .into() }); @@ -104,7 +105,8 @@ pub static GOERLI: Lazy> = Lazy::new(|| { 4367322, H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")), )), - ..Default::default() + base_fee_params: BaseFeeParams::ethereum(), + prune_batch_sizes: PruneBatchSizes::testnet(), } .into() }); @@ -149,7 +151,8 @@ pub static SEPOLIA: Lazy> = Lazy::new(|| { 1273020, H256(hex!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5")), )), - ..Default::default() + base_fee_params: BaseFeeParams::ethereum(), + prune_batch_sizes: PruneBatchSizes::testnet(), } .into() }); @@ -203,7 +206,7 @@ pub struct BaseFeeParams { } impl BaseFeeParams { - /// Get the base fee parameters for ethereum mainnet + /// Get the base fee parameters for Ethereum mainnet pub const fn ethereum() -> BaseFeeParams { BaseFeeParams { max_change_denominator: EIP1559_DEFAULT_BASE_FEE_MAX_CHANGE_DENOMINATOR, @@ -247,12 +250,18 @@ pub struct ChainSpec { /// The active hard forks and their activation conditions pub hardforks: BTreeMap, - /// The deposit contract deployed for PoS. + /// The deposit contract deployed for PoS #[serde(skip, default)] pub deposit_contract: Option, /// The parameters that configure how a block's base fee is computed pub base_fee_params: BaseFeeParams, + + /// The batch sizes for pruner, per block. In the actual pruner run it will be multiplied by + /// the amount of blocks between pruner runs to account for the difference in amount of new + /// data coming in. + #[serde(default)] + pub prune_batch_sizes: PruneBatchSizes, } impl Default for ChainSpec { @@ -266,6 +275,7 @@ impl Default for ChainSpec { hardforks: Default::default(), deposit_contract: Default::default(), base_fee_params: BaseFeeParams::ethereum(), + prune_batch_sizes: Default::default(), } } } diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 898ccc40af8..22435f436b3 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -81,8 +81,8 @@ pub use net::{ }; pub use peer::{PeerId, WithPeerId}; pub use prune::{ - PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, ReceiptsLogPruneConfig, - MINIMUM_PRUNING_DISTANCE, + PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, + ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE, }; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef}; pub use revm_primitives::JumpMap; diff --git a/crates/primitives/src/prune/batch_sizes.rs b/crates/primitives/src/prune/batch_sizes.rs new file mode 100644 index 00000000000..9498ea627b4 --- /dev/null +++ b/crates/primitives/src/prune/batch_sizes.rs @@ -0,0 +1,83 @@ +use paste::paste; +use serde::{Deserialize, Serialize}; + +/// Batch sizes for configuring the pruner. +/// The batch size for each prune part should be both large enough to prune the data which was +/// generated with each new block, and small enough to not generate an excessive load on the +/// database due to deletion of too many rows at once. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PruneBatchSizes { + /// Maximum number of receipts to prune, per block. + receipts: usize, + /// Maximum number of transaction lookup entries to prune, per block. + transaction_lookup: usize, + /// Maximum number of transaction senders to prune, per block. + transaction_senders: usize, + /// Maximum number of account history entries to prune, per block. + /// Measured in the number of `AccountChangeSet` table rows. + account_history: usize, + /// Maximum number of storage history entries to prune, per block. + /// Measured in the number of `StorageChangeSet` table rows. + storage_history: usize, +} + +macro_rules! impl_prune_batch_size_methods { + ($(($human_name:expr, $name:ident)),+) => { + paste! { + impl PruneBatchSizes { + $( + #[doc = concat!("Maximum number of ", $human_name, " to prune, accounting for the block interval.")] + pub fn $name(&self, block_interval: usize) -> usize { + self.$name * block_interval + } + + #[doc = concat!("Set the maximum number of ", $human_name, " to prune per block.")] + pub fn [](mut self, batch_size: usize) -> Self { + self.$name = batch_size; + self + } + )+ + } + } + }; +} + +impl_prune_batch_size_methods!( + ("receipts", receipts), + ("transaction lookup entries", transaction_lookup), + ("transaction senders", transaction_senders), + ("account history entries", account_history), + ("storage history entries", storage_history) +); + +impl PruneBatchSizes { + /// Default prune batch sizes for Ethereum mainnet. + /// These settings are sufficient to prune more data than generated with each new block. + pub const fn mainnet() -> Self { + Self { + receipts: 250, + transaction_lookup: 250, + transaction_senders: 250, + account_history: 1000, + storage_history: 1000, + } + } + + /// Default prune batch sizes for Ethereum testnets. + /// These settings are sufficient to prune more data than generated with each new block. + pub const fn testnet() -> Self { + Self { + receipts: 100, + transaction_lookup: 100, + transaction_senders: 100, + account_history: 500, + storage_history: 500, + } + } +} + +impl Default for PruneBatchSizes { + fn default() -> Self { + Self::mainnet() + } +} diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index 48bdacdb9e8..a2249f1c5b2 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -1,9 +1,11 @@ +mod batch_sizes; mod checkpoint; mod mode; mod part; mod target; use crate::{Address, BlockNumber}; +pub use batch_sizes::PruneBatchSizes; pub use checkpoint::PruneCheckpoint; pub use mode::PruneMode; pub use part::{PrunePart, PrunePartError}; diff --git a/crates/prune/src/lib.rs b/crates/prune/src/lib.rs index 56999c50c13..9c08cfff461 100644 --- a/crates/prune/src/lib.rs +++ b/crates/prune/src/lib.rs @@ -4,4 +4,4 @@ mod pruner; use crate::metrics::Metrics; pub use error::PrunerError; -pub use pruner::{BatchSizes, Pruner, PrunerResult, PrunerWithResult}; +pub use pruner::{Pruner, PrunerResult, PrunerWithResult}; diff --git a/crates/prune/src/pruner.rs b/crates/prune/src/pruner.rs index 4332534d39f..df2e8ce633d 100644 --- a/crates/prune/src/pruner.rs +++ b/crates/prune/src/pruner.rs @@ -12,8 +12,8 @@ use reth_db::{ BlockNumberList, }; use reth_primitives::{ - BlockNumber, ChainSpec, PruneCheckpoint, PruneMode, PruneModes, PrunePart, TxNumber, - MINIMUM_PRUNING_DISTANCE, + BlockNumber, ChainSpec, PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart, + TxNumber, MINIMUM_PRUNING_DISTANCE, }; use reth_provider::{ BlockReader, DatabaseProviderRW, ProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, @@ -31,46 +31,19 @@ pub type PrunerResult = Result; /// The pruner type itself with the result of [Pruner::run] pub type PrunerWithResult = (Pruner, PrunerResult); -pub struct BatchSizes { - /// Maximum number of receipts to prune in one run. - receipts: usize, - /// Maximum number of transaction lookup entries to prune in one run. - transaction_lookup: usize, - /// Maximum number of transaction senders to prune in one run. - transaction_senders: usize, - /// Maximum number of account history entries to prune in one run. - /// Measured in the number of [tables::AccountChangeSet] rows. - account_history: usize, - /// Maximum number of storage history entries to prune in one run. - /// Measured in the number of [tables::StorageChangeSet] rows. - storage_history: usize, -} - -impl Default for BatchSizes { - fn default() -> Self { - Self { - receipts: 1000, - transaction_lookup: 1000, - transaction_senders: 1000, - account_history: 1000, - storage_history: 1000, - } - } -} - /// Pruning routine. Main pruning logic happens in [Pruner::run]. pub struct Pruner { metrics: Metrics, provider_factory: ProviderFactory, /// Minimum pruning interval measured in blocks. All prune parts are checked and, if needed, /// pruned, when the chain advances by the specified number of blocks. - min_block_interval: u64, + min_block_interval: usize, /// Last pruned block number. Used in conjunction with `min_block_interval` to determine /// when the pruning needs to be initiated. last_pruned_block_number: Option, modes: PruneModes, - /// Maximum entries to prune per one run, per prune part. - batch_sizes: BatchSizes, + /// Maximum entries to prune per block, per prune part. + batch_sizes: PruneBatchSizes, } impl Pruner { @@ -78,9 +51,9 @@ impl Pruner { pub fn new( db: DB, chain_spec: Arc, - min_block_interval: u64, + min_block_interval: usize, modes: PruneModes, - batch_sizes: BatchSizes, + batch_sizes: PruneBatchSizes, ) -> Self { Self { metrics: Metrics::default(), @@ -263,7 +236,8 @@ impl Pruner { // Saturating subtraction is needed for the case when the chain was reverted, meaning // current block number might be less than the previously pruned block number. If // that's the case, no pruning is needed as outdated data is also reverted. - tip_block_number.saturating_sub(last_pruned_block_number) >= self.min_block_interval + tip_block_number.saturating_sub(last_pruned_block_number) >= + self.min_block_interval as u64 }) { debug!( target: "pruner", @@ -372,7 +346,7 @@ impl Pruner { let mut last_pruned_transaction = tx_range_end; let (deleted, done) = provider.prune_table_with_range::( tx_range, - self.batch_sizes.receipts, + self.batch_sizes.receipts(self.min_block_interval), |_| false, |row| last_pruned_transaction = row.0, )?; @@ -490,7 +464,7 @@ impl Pruner { "Calculated block ranges and filtered addresses", ); - let mut limit = self.batch_sizes.receipts; + let mut limit = self.batch_sizes.receipts(self.min_block_interval); let mut done = true; let mut last_pruned_transaction = None; for (start_block, end_block, num_addresses) in block_ranges { @@ -603,7 +577,10 @@ impl Pruner { } } .into_inner(); - let tx_range = start..=(end.min(start + self.batch_sizes.transaction_lookup as u64 - 1)); + let tx_range = start..= + (end.min( + start + self.batch_sizes.transaction_lookup(self.min_block_interval) as u64 - 1, + )); let tx_range_end = *tx_range.end(); // Retrieve transactions in the range and calculate their hashes in parallel @@ -624,7 +601,7 @@ impl Pruner { let mut last_pruned_transaction = tx_range_end; let (deleted, done) = provider.prune_table_with_iterator::( hashes, - self.batch_sizes.transaction_lookup, + self.batch_sizes.transaction_lookup(self.min_block_interval), |row| last_pruned_transaction = row.1, )?; trace!(target: "pruner", %deleted, %done, "Pruned transaction lookup"); @@ -673,7 +650,7 @@ impl Pruner { let mut last_pruned_transaction = tx_range_end; let (deleted, done) = provider.prune_table_with_range::( tx_range, - self.batch_sizes.transaction_senders, + self.batch_sizes.transaction_senders(self.min_block_interval), |_| false, |row| last_pruned_transaction = row.0, )?; @@ -722,7 +699,7 @@ impl Pruner { let mut last_changeset_pruned_block = None; let (rows, done) = provider.prune_table_with_range::( range, - self.batch_sizes.account_history, + self.batch_sizes.account_history(self.min_block_interval), |_| false, |row| last_changeset_pruned_block = Some(row.0), )?; @@ -778,7 +755,7 @@ impl Pruner { let mut last_changeset_pruned_block = None; let (rows, done) = provider.prune_table_with_range::( BlockNumberAddress::range(range), - self.batch_sizes.storage_history, + self.batch_sizes.storage_history(self.min_block_interval), |_| false, |row| last_changeset_pruned_block = Some(row.0.block_number()), )?; @@ -916,7 +893,7 @@ impl Pruner { #[cfg(test)] mod tests { - use crate::{pruner::BatchSizes, Pruner}; + use crate::Pruner; use assert_matches::assert_matches; use itertools::{ FoldWhile::{Continue, Done}, @@ -934,8 +911,8 @@ mod tests { }, }; use reth_primitives::{ - BlockNumber, PruneCheckpoint, PruneMode, PruneModes, PrunePart, ReceiptsLogPruneConfig, - TxNumber, H256, MAINNET, + BlockNumber, PruneBatchSizes, PruneCheckpoint, PruneMode, PruneModes, PrunePart, + ReceiptsLogPruneConfig, TxNumber, H256, MAINNET, }; use reth_provider::{PruneCheckpointReader, TransactionsProvider}; use reth_stages::test_utils::TestTransaction; @@ -945,14 +922,14 @@ mod tests { fn is_pruning_needed() { let db = create_test_rw_db(); let pruner = - Pruner::new(db, MAINNET.clone(), 5, PruneModes::default(), BatchSizes::default()); + Pruner::new(db, MAINNET.clone(), 5, PruneModes::default(), PruneBatchSizes::default()); // No last pruned block number was set before let first_block_number = 1; assert!(pruner.is_pruning_needed(first_block_number)); // Delta is not less than min block interval - let second_block_number = first_block_number + pruner.min_block_interval; + let second_block_number = first_block_number + pruner.min_block_interval as u64; assert!(pruner.is_pruning_needed(second_block_number)); // Delta is less than min block interval @@ -991,13 +968,10 @@ mod tests { let pruner = Pruner::new( tx.inner_raw(), MAINNET.clone(), - 5, + 1, PruneModes { receipts: Some(prune_mode), ..Default::default() }, - BatchSizes { - // Less than total amount of blocks to prune to test the batching logic - receipts: 10, - ..Default::default() - }, + // Less than total amount of blocks to prune to test the batching logic + PruneBatchSizes::default().with_receipts(10), ); let next_tx_number_to_prune = tx @@ -1008,11 +982,12 @@ mod tests { .map(|tx_number| tx_number + 1) .unwrap_or_default(); - let last_pruned_tx_number = blocks - .iter() - .map(|block| block.body.len()) - .sum::() - .min(next_tx_number_to_prune as usize + pruner.batch_sizes.receipts - 1); + let last_pruned_tx_number = + blocks.iter().map(|block| block.body.len()).sum::().min( + next_tx_number_to_prune as usize + + pruner.batch_sizes.receipts(pruner.min_block_interval) - + 1, + ); let last_pruned_block_number = blocks .iter() @@ -1087,13 +1062,10 @@ mod tests { let pruner = Pruner::new( tx.inner_raw(), MAINNET.clone(), - 5, + 1, PruneModes { transaction_lookup: Some(prune_mode), ..Default::default() }, - BatchSizes { - // Less than total amount of blocks to prune to test the batching logic - transaction_lookup: 10, - ..Default::default() - }, + // Less than total amount of blocks to prune to test the batching logic + PruneBatchSizes::default().with_transaction_lookup(10), ); let next_tx_number_to_prune = tx @@ -1106,7 +1078,9 @@ mod tests { let last_pruned_tx_number = blocks.iter().map(|block| block.body.len()).sum::().min( - next_tx_number_to_prune as usize + pruner.batch_sizes.transaction_lookup - 1, + next_tx_number_to_prune as usize + + pruner.batch_sizes.transaction_lookup(pruner.min_block_interval) - + 1, ); let last_pruned_block_number = blocks @@ -1185,13 +1159,10 @@ mod tests { let pruner = Pruner::new( tx.inner_raw(), MAINNET.clone(), - 5, + 1, PruneModes { sender_recovery: Some(prune_mode), ..Default::default() }, - BatchSizes { - // Less than total amount of blocks to prune to test the batching logic - transaction_senders: 10, - ..Default::default() - }, + // Less than total amount of blocks to prune to test the batching logic + PruneBatchSizes::default().with_transaction_senders(10), ); let next_tx_number_to_prune = tx @@ -1204,7 +1175,9 @@ mod tests { let last_pruned_tx_number = blocks.iter().map(|block| block.body.len()).sum::().min( - next_tx_number_to_prune as usize + pruner.batch_sizes.transaction_senders - 1, + next_tx_number_to_prune as usize + + pruner.batch_sizes.transaction_senders(pruner.min_block_interval) - + 1, ); let last_pruned_block_number = blocks @@ -1292,13 +1265,10 @@ mod tests { let pruner = Pruner::new( tx.inner_raw(), MAINNET.clone(), - 5, + 1, PruneModes { account_history: Some(prune_mode), ..Default::default() }, - BatchSizes { - // Less than total amount of blocks to prune to test the batching logic - account_history: 2000, - ..Default::default() - }, + // Less than total amount of blocks to prune to test the batching logic + PruneBatchSizes::default().with_account_history(2000), ); let provider = tx.inner_rw(); @@ -1320,7 +1290,7 @@ mod tests { .iter() .enumerate() .skip_while(|(i, (block_number, _))| { - *i < pruner.batch_sizes.account_history * run && + *i < pruner.batch_sizes.account_history(pruner.min_block_interval) * run && *block_number <= to_block as usize }) .next() @@ -1422,13 +1392,10 @@ mod tests { let pruner = Pruner::new( tx.inner_raw(), MAINNET.clone(), - 5, + 1, PruneModes { storage_history: Some(prune_mode), ..Default::default() }, - BatchSizes { - // Less than total amount of blocks to prune to test the batching logic - storage_history: 2000, - ..Default::default() - }, + // Less than total amount of blocks to prune to test the batching logic + PruneBatchSizes::default().with_storage_history(2000), ); let provider = tx.inner_rw(); @@ -1452,7 +1419,7 @@ mod tests { .iter() .enumerate() .skip_while(|(i, (block_number, _, _))| { - *i < pruner.batch_sizes.storage_history * run && + *i < pruner.batch_sizes.storage_history(pruner.min_block_interval) * run && *block_number <= to_block as usize }) .next() @@ -1564,11 +1531,8 @@ mod tests { receipts_log_filter: receipts_log_filter.clone(), ..Default::default() }, - BatchSizes { - // Less than total amount of blocks to prune to test the batching logic - receipts: 10, - ..Default::default() - }, + // Less than total amount of blocks to prune to test the batching logic + PruneBatchSizes::default().with_storage_history(10), ); let result = pruner.prune_receipts_by_logs(&provider, tip);