From 7dc7b776a6f833bc0d4aafcdb7b8415927f433ed Mon Sep 17 00:00:00 2001 From: sadiq1971 Date: Tue, 16 Dec 2025 01:11:20 +0600 Subject: [PATCH 1/4] pruning progress and metrics improvements --- crates/optimism/trie/src/api.rs | 5 +- crates/optimism/trie/src/db/store.rs | 44 +++++++++++++---- crates/optimism/trie/src/in_memory.rs | 22 ++++++++- crates/optimism/trie/src/metrics.rs | 2 +- crates/optimism/trie/src/prune/error.rs | 17 ++++--- crates/optimism/trie/src/prune/metrics.rs | 15 ++++++ crates/optimism/trie/src/prune/pruner.rs | 4 +- etc/grafana/dashboards/op-proof-history.json | 52 ++++++++++++++++++++ 8 files changed, 139 insertions(+), 22 deletions(-) diff --git a/crates/optimism/trie/src/api.rs b/crates/optimism/trie/src/api.rs index 9581f8e3adf..66c27a4efb2 100644 --- a/crates/optimism/trie/src/api.rs +++ b/crates/optimism/trie/src/api.rs @@ -4,6 +4,7 @@ use crate::OpProofsStorageResult; use alloy_eips::eip1898::BlockWithParent; use alloy_primitives::{map::HashMap, B256, U256}; use auto_impl::auto_impl; +use derive_more::{AddAssign, Constructor}; use reth_primitives_traits::Account; use reth_trie::{ hashed_cursor::{HashedCursor, HashedStorageCursor}, @@ -31,7 +32,7 @@ impl BlockStateDiff { } /// Counts of trie updates written to storage. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, AddAssign, Constructor, Eq, PartialEq)] pub struct WriteCounts { /// Number of account trie updates written pub account_trie_updates_written_total: u64, @@ -170,7 +171,7 @@ pub trait OpProofsStore: Send + Sync + Debug { &self, new_earliest_block_ref: BlockWithParent, diff: BlockStateDiff, - ) -> impl Future> + Send; + ) -> impl Future> + Send; /// Remove account, storage and trie updates from historical storage for all blocks till /// the specified block (inclusive). diff --git a/crates/optimism/trie/src/db/store.rs b/crates/optimism/trie/src/db/store.rs index 5db93c30000..b7565e41d78 100644 --- a/crates/optimism/trie/src/db/store.rs +++ b/crates/optimism/trie/src/db/store.rs @@ -31,6 +31,7 @@ use reth_trie::{ BranchNodeCompact, HashedPostState, Nibbles, }; use std::{cmp::max, ops::RangeBounds, path::Path}; +use tracing::info; /// MDBX implementation of [`OpProofsStore`]. #[derive(Debug)] @@ -235,10 +236,19 @@ impl MdbxProofsStorage { &self, tx: &(impl DbTxMut + DbTx), block_range: impl RangeBounds, - ) -> OpProofsStorageResult<()> { + ) -> OpProofsStorageResult { + let mut write_count = WriteCounts::default(); let mut change_set_cursor = tx.cursor_write::()?; let mut walker = change_set_cursor.walk_range(block_range)?; + let mut blocks_deleted = 0; while let Some(Ok((block_number, change_set))) = walker.next() { + write_count += WriteCounts::new( + change_set.account_trie_keys.len() as u64, + change_set.storage_trie_keys.len() as u64, + change_set.hashed_account_keys.len() as u64, + change_set.hashed_storage_keys.len() as u64, + ); + self.delete_dup_sorted::( tx, block_number, @@ -260,9 +270,15 @@ impl MdbxProofsStorage { change_set.hashed_storage_keys, )?; - walker.delete_current()? + walker.delete_current()?; + + blocks_deleted += 1; + // Progress log: only every 20 blocks, only if total >= 20 + if blocks_deleted >= 20 && blocks_deleted % 20 == 0 { + info!(target: "optimism.trie", %blocks_deleted, "Deleting Proofs History"); + } } - Ok(()) + Ok(write_count) } /// Write trie/state history for `block_number` from `block_state_diff`. @@ -675,32 +691,42 @@ impl OpProofsStore for MdbxProofsStorage { &self, new_earliest_block_ref: BlockWithParent, diff: BlockStateDiff, - ) -> OpProofsStorageResult<()> { + ) -> OpProofsStorageResult { + let mut write_counts = WriteCounts::default(); + let new_earliest_block_number = new_earliest_block_ref.block.number; let Some((old_earliest_block_number, _)) = self.get_earliest_block_number().await? else { - return Ok(()); // Nothing to prune + return Ok(write_counts); // Nothing to prune }; if old_earliest_block_number >= new_earliest_block_number { - return Ok(()); // Nothing to prune + return Ok(write_counts); // Nothing to prune } self.env.update(|tx| { // Update the initial state (block zero) - self.store_trie_updates_for_block(tx, 0, diff, false)?; + let change_set = self.store_trie_updates_for_block(tx, 0, diff, false)?; + write_counts += WriteCounts::new( + change_set.account_trie_keys.len() as u64, + change_set.storage_trie_keys.len() as u64, + change_set.hashed_account_keys.len() as u64, + change_set.hashed_storage_keys.len() as u64, + ); // Delete the old entries for the block range excluding block 0 - self.delete_history_ranged( + let delete_counts = self.delete_history_ranged( tx, max(old_earliest_block_number, 1)..=new_earliest_block_number, )?; + write_counts += delete_counts; // Set the earliest block number to the new value Self::inner_set_earliest_block_number( tx, new_earliest_block_number, new_earliest_block_ref.block.hash, - ) + )?; + Ok(write_counts) })? } diff --git a/crates/optimism/trie/src/in_memory.rs b/crates/optimism/trie/src/in_memory.rs index ac868f79745..579f90276ec 100644 --- a/crates/optimism/trie/src/in_memory.rs +++ b/crates/optimism/trie/src/in_memory.rs @@ -610,7 +610,8 @@ impl OpProofsStore for InMemoryProofsStorage { &self, new_earliest_block_ref: BlockWithParent, diff: BlockStateDiff, - ) -> OpProofsStorageResult<()> { + ) -> OpProofsStorageResult { + let mut write_counts = WriteCounts::default(); let mut inner = self.inner.write().await; let branches_diff = diff.sorted_trie_updates; @@ -618,6 +619,7 @@ impl OpProofsStore for InMemoryProofsStorage { // Apply branch updates to the earliest state (block 0) for (path, branch) in &branches_diff.account_nodes { + write_counts.account_trie_updates_written_total += 1; match branch { Some(br) => _ = inner.account_branches.insert((0, *path), Some(br.clone())), None => _ = inner.account_branches.remove(&(0, *path)), @@ -635,17 +637,20 @@ impl OpProofsStore for InMemoryProofsStorage { } None => _ = inner.storage_branches.remove(&(0, *hashed_address, *path)), } + write_counts.storage_trie_updates_written_total += 1; } } // Apply account updates for (hashed_address, account) in &leaves_diff.accounts { + write_counts.hashed_accounts_written_total += 1; inner.hashed_accounts.insert((0, *hashed_address), *account); } // Apply storage updates for (hashed_address, storage) in &leaves_diff.storages { for (slot, value) in storage.storage_slots_ref() { + write_counts.hashed_storages_written_total += 1; inner.hashed_storages.insert((0, *hashed_address, *slot), *value); } } @@ -657,22 +662,35 @@ impl OpProofsStore for InMemoryProofsStorage { } // Remove all data for blocks before new_earliest_block_number (except block 0) + let length_before_prune = inner.account_branches.len(); inner .account_branches .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number); + write_counts.account_trie_updates_written_total += + (length_before_prune - inner.account_branches.len()) as u64; + inner .storage_branches .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number); + write_counts.storage_trie_updates_written_total += + (length_before_prune - inner.storage_branches.len()) as u64; + inner .hashed_accounts .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number); + write_counts.hashed_accounts_written_total += + (length_before_prune - inner.hashed_accounts.len()) as u64; + inner .hashed_storages .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number); + write_counts.hashed_storages_written_total += + (length_before_prune - inner.hashed_storages.len()) as u64; + inner.trie_updates.retain(|block, _| *block >= new_earliest_block_number); inner.post_states.retain(|block, _| *block >= new_earliest_block_number); - Ok(()) + Ok(write_counts) } async fn unwind_history( diff --git a/crates/optimism/trie/src/metrics.rs b/crates/optimism/trie/src/metrics.rs index 051a548093c..b416f0365be 100644 --- a/crates/optimism/trie/src/metrics.rs +++ b/crates/optimism/trie/src/metrics.rs @@ -539,7 +539,7 @@ where &self, new_earliest_block_ref: BlockWithParent, diff: BlockStateDiff, - ) -> OpProofsStorageResult<()> { + ) -> OpProofsStorageResult { self.storage.prune_earliest_state(new_earliest_block_ref, diff).await } diff --git a/crates/optimism/trie/src/prune/error.rs b/crates/optimism/trie/src/prune/error.rs index 8f46f29d20d..e2d5b32a33c 100644 --- a/crates/optimism/trie/src/prune/error.rs +++ b/crates/optimism/trie/src/prune/error.rs @@ -1,4 +1,4 @@ -use crate::OpProofsStorageError; +use crate::{api::WriteCounts, OpProofsStorageError}; use reth_provider::ProviderError; use std::{ fmt, @@ -24,20 +24,24 @@ pub struct PrunerOutput { pub start_block: u64, /// New earliest block at the end of the run. pub end_block: u64, - /// Total number of entries removed across tables. - pub total_entries_pruned: u64, + /// Number of entries updated/removed per table. + pub write_counts: WriteCounts, } impl Display for PrunerOutput { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { let blocks = self.end_block.saturating_sub(self.start_block); + let total_entries = self.write_counts.hashed_accounts_written_total + + self.write_counts.hashed_storages_written_total + + self.write_counts.account_trie_updates_written_total + + self.write_counts.storage_trie_updates_written_total; write!( f, "Pruned {}→{} ({} blocks), entries={}, elapsed={:.3}s", self.start_block, self.end_block, blocks, - self.total_entries_pruned, + total_entries, self.duration.as_secs_f64(), ) } @@ -62,6 +66,7 @@ pub enum PrunerError { #[cfg(test)] mod tests { use super::PrunerOutput; + use crate::api::WriteCounts; use std::time::Duration; #[test] @@ -72,10 +77,10 @@ mod tests { prune_duration: Duration::from_secs(5), start_block: 1, end_block: 2, - total_entries_pruned: 3, + write_counts: WriteCounts::new(1, 2, 3, 4), }; let formatted_pruner_output = format!("{}", pruner_output); - assert_eq!(formatted_pruner_output, "Pruned 1→2 (1 blocks), entries=3, elapsed=10.000s"); + assert_eq!(formatted_pruner_output, "Pruned 1→2 (1 blocks), entries=10, elapsed=10.000s"); } } diff --git a/crates/optimism/trie/src/prune/metrics.rs b/crates/optimism/trie/src/prune/metrics.rs index 62c51fd82b0..dba8962a0d2 100644 --- a/crates/optimism/trie/src/prune/metrics.rs +++ b/crates/optimism/trie/src/prune/metrics.rs @@ -15,6 +15,14 @@ pub(crate) struct Metrics { pub(crate) prune_duration_seconds: Histogram, /// Number of pruned blocks pub(crate) pruned_blocks: Gauge, + /// Number of account trie updates written in the prune run + pub(crate) account_trie_updates_written: Gauge, + /// Number of storage trie updates written in the prune run + pub(crate) storage_trie_updates_written: Gauge, + /// Number of hashed accounts written in the prune run + pub(crate) hashed_accounts_written: Gauge, + /// Number of hashed storages written in the prune run + pub(crate) hashed_storages_written: Gauge, } impl Metrics { @@ -25,6 +33,13 @@ impl Metrics { self.state_diff_fetch_duration_seconds.record(result.fetch_duration.as_secs_f64()); self.prune_duration_seconds.record(result.prune_duration.as_secs_f64()); self.pruned_blocks.set(blocks_pruned as f64); + + // Consume write counts + let wc = &result.write_counts; + self.account_trie_updates_written.set(wc.account_trie_updates_written_total as f64); + self.storage_trie_updates_written.set(wc.storage_trie_updates_written_total as f64); + self.hashed_accounts_written.set(wc.hashed_accounts_written_total as f64); + self.hashed_storages_written.set(wc.hashed_storages_written_total as f64); } } } diff --git a/crates/optimism/trie/src/prune/pruner.rs b/crates/optimism/trie/src/prune/pruner.rs index 0834f6fdc9d..2825035121e 100644 --- a/crates/optimism/trie/src/prune/pruner.rs +++ b/crates/optimism/trie/src/prune/pruner.rs @@ -126,7 +126,7 @@ where block: BlockNumHash { number: new_earliest_block, hash: new_earliest_block_hash }, }; - self.provider.prune_earliest_state(block_with_parent, final_diff).await?; + let write_count = self.provider.prune_earliest_state(block_with_parent, final_diff).await?; let total_duration = t.elapsed(); let prune_output = PrunerOutput { @@ -135,7 +135,7 @@ where prune_duration: total_duration.saturating_sub(stat_diff_fetch_duration), start_block: earliest_block, end_block: new_earliest_block, - total_entries_pruned: 0, // TODO: get it from the prune_earliest_state + write_counts: write_count, }; #[cfg(feature = "metrics")] self.metrics.record_prune_result(prune_output.clone()); diff --git a/etc/grafana/dashboards/op-proof-history.json b/etc/grafana/dashboards/op-proof-history.json index 718734f935c..8db17b3cf4c 100644 --- a/etc/grafana/dashboards/op-proof-history.json +++ b/etc/grafana/dashboards/op-proof-history.json @@ -1953,6 +1953,58 @@ "legendFormat": "Pruned Blocks", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_optimism_trie_pruner_account_trie_updates_written", + "hide": false, + "instant": false, + "legendFormat": "Account Trie Nodes", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_optimism_trie_pruner_storage_trie_updates_written", + "hide": false, + "instant": false, + "legendFormat": "Storage Trie Nodes", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_optimism_trie_pruner_hashed_accounts_written", + "hide": false, + "instant": false, + "legendFormat": "Hashed Accounts", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_optimism_trie_pruner_hashed_storages_written", + "hide": false, + "instant": false, + "legendFormat": "Hashed Storage", + "range": true, + "refId": "E" } ], "title": "Prune State Over Time", From 605a31ce6ecf58c288b518935a21e3504033d8a7 Mon Sep 17 00:00:00 2001 From: sadiq1971 Date: Tue, 16 Dec 2025 01:15:13 +0600 Subject: [PATCH 2/4] fix inmem --- crates/optimism/trie/src/in_memory.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/optimism/trie/src/in_memory.rs b/crates/optimism/trie/src/in_memory.rs index 579f90276ec..cd57d6998fd 100644 --- a/crates/optimism/trie/src/in_memory.rs +++ b/crates/optimism/trie/src/in_memory.rs @@ -662,25 +662,28 @@ impl OpProofsStore for InMemoryProofsStorage { } // Remove all data for blocks before new_earliest_block_number (except block 0) - let length_before_prune = inner.account_branches.len(); + let mut length_before_prune = inner.account_branches.len(); inner .account_branches .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number); write_counts.account_trie_updates_written_total += (length_before_prune - inner.account_branches.len()) as u64; - + + length_before_prune = inner.storage_branches.len(); inner .storage_branches .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number); write_counts.storage_trie_updates_written_total += (length_before_prune - inner.storage_branches.len()) as u64; + length_before_prune = inner.hashed_accounts.len(); inner .hashed_accounts .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number); write_counts.hashed_accounts_written_total += (length_before_prune - inner.hashed_accounts.len()) as u64; + length_before_prune = inner.hashed_storages.len(); inner .hashed_storages .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number); From 5a910a1996e8e7c3b5c4c3f8e2a610d1c8823960 Mon Sep 17 00:00:00 2001 From: sadiq1971 Date: Wed, 17 Dec 2025 20:44:46 +0600 Subject: [PATCH 3/4] logging frequency decresed --- crates/optimism/trie/src/db/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/optimism/trie/src/db/store.rs b/crates/optimism/trie/src/db/store.rs index b7565e41d78..2476a320731 100644 --- a/crates/optimism/trie/src/db/store.rs +++ b/crates/optimism/trie/src/db/store.rs @@ -274,7 +274,7 @@ impl MdbxProofsStorage { blocks_deleted += 1; // Progress log: only every 20 blocks, only if total >= 20 - if blocks_deleted >= 20 && blocks_deleted % 20 == 0 { + if blocks_deleted >= 1000 && blocks_deleted % 1000 == 0 { info!(target: "optimism.trie", %blocks_deleted, "Deleting Proofs History"); } } From e31ceca6de908917d4d53894def6ba3a277cf007 Mon Sep 17 00:00:00 2001 From: sadiq1971 Date: Wed, 17 Dec 2025 21:39:04 +0600 Subject: [PATCH 4/4] lint fix --- crates/optimism/trie/src/in_memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/optimism/trie/src/in_memory.rs b/crates/optimism/trie/src/in_memory.rs index cd57d6998fd..f040b4279f5 100644 --- a/crates/optimism/trie/src/in_memory.rs +++ b/crates/optimism/trie/src/in_memory.rs @@ -668,7 +668,7 @@ impl OpProofsStore for InMemoryProofsStorage { .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number); write_counts.account_trie_updates_written_total += (length_before_prune - inner.account_branches.len()) as u64; - + length_before_prune = inner.storage_branches.len(); inner .storage_branches