feat: pruning progress and metrics improvements

crates/optimism/trie/src/api.rs

@@ -4,6 +4,7 @@ use crate::OpProofsStorageResult;
 use alloy_eips::eip1898::BlockWithParent;
 use alloy_primitives::{map::HashMap, B256, U256};
 use auto_impl::auto_impl;
+use derive_more::{AddAssign, Constructor};
 use reth_primitives_traits::Account;
 use reth_trie::{
     hashed_cursor::{HashedCursor, HashedStorageCursor},
@@ -31,7 +32,7 @@ impl BlockStateDiff {
 }
 
 /// Counts of trie updates written to storage.
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, AddAssign, Constructor, Eq, PartialEq)]
 pub struct WriteCounts {
     /// Number of account trie updates written
     pub account_trie_updates_written_total: u64,
@@ -170,7 +171,7 @@ pub trait OpProofsStore: Send + Sync + Debug {
         &self,
         new_earliest_block_ref: BlockWithParent,
         diff: BlockStateDiff,
-    ) -> impl Future<Output = OpProofsStorageResult<()>> + Send;
+    ) -> impl Future<Output = OpProofsStorageResult<WriteCounts>> + Send;
 
     /// Remove account, storage and trie updates from historical storage for all blocks till
     /// the specified block (inclusive).

crates/optimism/trie/src/db/store.rs

@@ -31,6 +31,7 @@ use reth_trie::{
     BranchNodeCompact, HashedPostState, Nibbles,
 };
 use std::{cmp::max, ops::RangeBounds, path::Path};
+use tracing::info;
 
 /// MDBX implementation of [`OpProofsStore`].
 #[derive(Debug)]
@@ -235,10 +236,19 @@ impl MdbxProofsStorage {
         &self,
         tx: &(impl DbTxMut + DbTx),
         block_range: impl RangeBounds<u64>,
-    ) -> OpProofsStorageResult<()> {
+    ) -> OpProofsStorageResult<WriteCounts> {
+        let mut write_count = WriteCounts::default();
         let mut change_set_cursor = tx.cursor_write::<BlockChangeSet>()?;
         let mut walker = change_set_cursor.walk_range(block_range)?;
+        let mut blocks_deleted = 0;
         while let Some(Ok((block_number, change_set))) = walker.next() {
+            write_count += WriteCounts::new(
+                change_set.account_trie_keys.len() as u64,
+                change_set.storage_trie_keys.len() as u64,
+                change_set.hashed_account_keys.len() as u64,
+                change_set.hashed_storage_keys.len() as u64,
+            );
+
             self.delete_dup_sorted::<AccountTrieHistory, _, _>(
                 tx,
                 block_number,
@@ -260,9 +270,15 @@ impl MdbxProofsStorage {
                 change_set.hashed_storage_keys,
             )?;
 
-            walker.delete_current()?
+            walker.delete_current()?;
+
+            blocks_deleted += 1;
+            // Progress log: only every 20 blocks, only if total >= 20
+            if blocks_deleted >= 1000 && blocks_deleted % 1000 == 0 {
+                info!(target: "optimism.trie",  %blocks_deleted, "Deleting Proofs History");
+            }
         }
-        Ok(())
+        Ok(write_count)
     }
 
     /// Write trie/state history for `block_number` from `block_state_diff`.
@@ -675,32 +691,42 @@ impl OpProofsStore for MdbxProofsStorage {
         &self,
         new_earliest_block_ref: BlockWithParent,
         diff: BlockStateDiff,
-    ) -> OpProofsStorageResult<()> {
+    ) -> OpProofsStorageResult<WriteCounts> {
+        let mut write_counts = WriteCounts::default();
+
         let new_earliest_block_number = new_earliest_block_ref.block.number;
         let Some((old_earliest_block_number, _)) = self.get_earliest_block_number().await? else {
-            return Ok(()); // Nothing to prune
+            return Ok(write_counts); // Nothing to prune
         };
 
         if old_earliest_block_number >= new_earliest_block_number {
-            return Ok(()); // Nothing to prune
+            return Ok(write_counts); // Nothing to prune
         }
 
         self.env.update(|tx| {
             // Update the initial state (block zero)
-            self.store_trie_updates_for_block(tx, 0, diff, false)?;
+            let change_set = self.store_trie_updates_for_block(tx, 0, diff, false)?;
+            write_counts += WriteCounts::new(
+                change_set.account_trie_keys.len() as u64,
+                change_set.storage_trie_keys.len() as u64,
+                change_set.hashed_account_keys.len() as u64,
+                change_set.hashed_storage_keys.len() as u64,
+            );
 
             // Delete the old entries for the block range excluding block 0
-            self.delete_history_ranged(
+            let delete_counts = self.delete_history_ranged(
                 tx,
                 max(old_earliest_block_number, 1)..=new_earliest_block_number,
             )?;
+            write_counts += delete_counts;
 
             // Set the earliest block number to the new value
             Self::inner_set_earliest_block_number(
                 tx,
                 new_earliest_block_number,
                 new_earliest_block_ref.block.hash,
-            )
+            )?;
+            Ok(write_counts)
         })?
     }
 

crates/optimism/trie/src/in_memory.rs

@@ -610,14 +610,16 @@ impl OpProofsStore for InMemoryProofsStorage {
         &self,
         new_earliest_block_ref: BlockWithParent,
         diff: BlockStateDiff,
-    ) -> OpProofsStorageResult<()> {
+    ) -> OpProofsStorageResult<WriteCounts> {
+        let mut write_counts = WriteCounts::default();
         let mut inner = self.inner.write().await;
 
         let branches_diff = diff.sorted_trie_updates;
         let leaves_diff = diff.sorted_post_state;
 
         // Apply branch updates to the earliest state (block 0)
         for (path, branch) in &branches_diff.account_nodes {
+            write_counts.account_trie_updates_written_total += 1;
             match branch {
                 Some(br) => _ = inner.account_branches.insert((0, *path), Some(br.clone())),
                 None => _ = inner.account_branches.remove(&(0, *path)),
@@ -635,17 +637,20 @@ impl OpProofsStore for InMemoryProofsStorage {
                     }
                     None => _ = inner.storage_branches.remove(&(0, *hashed_address, *path)),
                 }
+                write_counts.storage_trie_updates_written_total += 1;
             }
         }
 
         // Apply account updates
         for (hashed_address, account) in &leaves_diff.accounts {
+            write_counts.hashed_accounts_written_total += 1;
             inner.hashed_accounts.insert((0, *hashed_address), *account);
         }
 
         // Apply storage updates
         for (hashed_address, storage) in &leaves_diff.storages {
             for (slot, value) in storage.storage_slots_ref() {
+                write_counts.hashed_storages_written_total += 1;
                 inner.hashed_storages.insert((0, *hashed_address, *slot), *value);
             }
         }
@@ -657,22 +662,38 @@ impl OpProofsStore for InMemoryProofsStorage {
         }
 
         // Remove all data for blocks before new_earliest_block_number (except block 0)
+        let mut length_before_prune = inner.account_branches.len();
         inner
             .account_branches
             .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number);
+        write_counts.account_trie_updates_written_total +=
+            (length_before_prune - inner.account_branches.len()) as u64;
+
+        length_before_prune = inner.storage_branches.len();
         inner
             .storage_branches
             .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number);
+        write_counts.storage_trie_updates_written_total +=
+            (length_before_prune - inner.storage_branches.len()) as u64;
+
+        length_before_prune = inner.hashed_accounts.len();
         inner
             .hashed_accounts
             .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number);
+        write_counts.hashed_accounts_written_total +=
+            (length_before_prune - inner.hashed_accounts.len()) as u64;
+
+        length_before_prune = inner.hashed_storages.len();
         inner
             .hashed_storages
             .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number);
+        write_counts.hashed_storages_written_total +=
+            (length_before_prune - inner.hashed_storages.len()) as u64;
+
         inner.trie_updates.retain(|block, _| *block >= new_earliest_block_number);
         inner.post_states.retain(|block, _| *block >= new_earliest_block_number);
 
-        Ok(())
+        Ok(write_counts)
     }
 
     async fn unwind_history(

crates/optimism/trie/src/metrics.rs

@@ -539,7 +539,7 @@ where
         &self,
         new_earliest_block_ref: BlockWithParent,
         diff: BlockStateDiff,
-    ) -> OpProofsStorageResult<()> {
+    ) -> OpProofsStorageResult<WriteCounts> {
         self.metrics.block_metrics.earliest_number.set(new_earliest_block_ref.block.number as f64);
         self.storage.prune_earliest_state(new_earliest_block_ref, diff).await
     }

crates/optimism/trie/src/prune/error.rs

@@ -1,4 +1,4 @@
-use crate::OpProofsStorageError;
+use crate::{api::WriteCounts, OpProofsStorageError};
 use reth_provider::ProviderError;
 use std::{
     fmt,
@@ -24,20 +24,24 @@ pub struct PrunerOutput {
     pub start_block: u64,
     /// New earliest block at the end of the run.
     pub end_block: u64,
-    /// Total number of entries removed across tables.
-    pub total_entries_pruned: u64,
+    /// Number of entries updated/removed per table.
+    pub write_counts: WriteCounts,
 }
 
 impl Display for PrunerOutput {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         let blocks = self.end_block.saturating_sub(self.start_block);
+        let total_entries = self.write_counts.hashed_accounts_written_total +
+            self.write_counts.hashed_storages_written_total +
+            self.write_counts.account_trie_updates_written_total +
+            self.write_counts.storage_trie_updates_written_total;
         write!(
             f,
             "Pruned {}→{} ({} blocks), entries={}, elapsed={:.3}s",
             self.start_block,
             self.end_block,
             blocks,
-            self.total_entries_pruned,
+            total_entries,
             self.duration.as_secs_f64(),
         )
     }
@@ -62,6 +66,7 @@ pub enum PrunerError {
 #[cfg(test)]
 mod tests {
     use super::PrunerOutput;
+    use crate::api::WriteCounts;
     use std::time::Duration;
 
     #[test]
@@ -72,10 +77,10 @@ mod tests {
             prune_duration: Duration::from_secs(5),
             start_block: 1,
             end_block: 2,
-            total_entries_pruned: 3,
+            write_counts: WriteCounts::new(1, 2, 3, 4),
         };
         let formatted_pruner_output = format!("{}", pruner_output);
 
-        assert_eq!(formatted_pruner_output, "Pruned 1→2 (1 blocks), entries=3, elapsed=10.000s");
+        assert_eq!(formatted_pruner_output, "Pruned 1→2 (1 blocks), entries=10, elapsed=10.000s");
     }
 }

crates/optimism/trie/src/prune/metrics.rs

@@ -15,6 +15,14 @@ pub(crate) struct Metrics {
     pub(crate) prune_duration_seconds: Histogram,
     /// Number of pruned blocks
     pub(crate) pruned_blocks: Gauge,
+    /// Number of account trie updates written in the prune run
+    pub(crate) account_trie_updates_written: Gauge,
+    /// Number of storage trie updates written in the prune run
+    pub(crate) storage_trie_updates_written: Gauge,
+    /// Number of hashed accounts written in the prune run
+    pub(crate) hashed_accounts_written: Gauge,
+    /// Number of hashed storages written in the prune run
+    pub(crate) hashed_storages_written: Gauge,
 }
 
 impl Metrics {
@@ -25,6 +33,13 @@ impl Metrics {
             self.state_diff_fetch_duration_seconds.record(result.fetch_duration.as_secs_f64());
             self.prune_duration_seconds.record(result.prune_duration.as_secs_f64());
             self.pruned_blocks.set(blocks_pruned as f64);
+
+            // Consume write counts
+            let wc = &result.write_counts;
+            self.account_trie_updates_written.set(wc.account_trie_updates_written_total as f64);
+            self.storage_trie_updates_written.set(wc.storage_trie_updates_written_total as f64);
+            self.hashed_accounts_written.set(wc.hashed_accounts_written_total as f64);
+            self.hashed_storages_written.set(wc.hashed_storages_written_total as f64);
         }
     }
 }

crates/optimism/trie/src/prune/pruner.rs

@@ -130,7 +130,7 @@ where
             block: BlockNumHash { number: new_earliest_block, hash: new_earliest_block_hash },
         };
 
-        self.provider.prune_earliest_state(block_with_parent, final_diff).await?;
+        let write_count = self.provider.prune_earliest_state(block_with_parent, final_diff).await?;
 
         let total_duration = t.elapsed();
         let prune_output = PrunerOutput {
@@ -139,7 +139,7 @@ where
             prune_duration: total_duration.saturating_sub(stat_diff_fetch_duration),
             start_block: earliest_block,
             end_block: new_earliest_block,
-            total_entries_pruned: 0, // TODO: get it from the prune_earliest_state
+            write_counts: write_count,
         };
         #[cfg(feature = "metrics")]
         self.metrics.record_prune_result(prune_output.clone());

etc/grafana/dashboards/op-proof-history.json

@@ -1953,6 +1953,58 @@
           "legendFormat": "Pruned Blocks",
           "range": true,
           "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "reth_optimism_trie_pruner_account_trie_updates_written",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Account Trie Nodes",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "reth_optimism_trie_pruner_storage_trie_updates_written",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Storage Trie Nodes",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "reth_optimism_trie_pruner_hashed_accounts_written",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Hashed Accounts",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "reth_optimism_trie_pruner_hashed_storages_written",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Hashed Storage",
+          "range": true,
+          "refId": "E"
         }
       ],
       "title": "Prune State Over Time",