fix(engine): configure invalid header cache hit eviction

crates/engine/primitives/src/config.rs

@@ -15,6 +15,9 @@ pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0;
 /// The size of proof targets chunk to spawn in one multiproof calculation.
 pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 5;
 
+/// Default number of cache hits before an invalid header entry is evicted and reprocessed.
+pub const DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
+
 /// Gas threshold below which the small block chunk size is used.
 pub const SMALL_BLOCK_GAS_THRESHOLD: u64 = 20_000_000;
 
@@ -102,6 +105,11 @@ pub struct TreeConfig {
     block_buffer_limit: u32,
     /// Number of invalid headers to keep in cache.
     max_invalid_header_cache_length: u32,
+    /// Number of cache hits before an invalid header entry is evicted and reprocessed.
+    ///
+    /// Setting this to `0` effectively disables the cache because entries are evicted on the
+    /// first lookup.
+    invalid_header_hit_eviction_threshold: u8,
     /// Maximum number of blocks to execute sequentially in a batch.
     ///
     /// This is used as a cutoff to prevent long-running sequential block execution when we receive
@@ -195,6 +203,7 @@ impl Default for TreeConfig {
             persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
             block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT,
             max_invalid_header_cache_length: DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH,
+            invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD,
             max_execute_block_batch_size: DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE,
             legacy_state_root: false,
             always_compare_trie_updates: false,
@@ -234,6 +243,7 @@ impl TreeConfig {
         persistence_backpressure_threshold: u64,
         block_buffer_limit: u32,
         max_invalid_header_cache_length: u32,
+        invalid_header_hit_eviction_threshold: u8,
         max_execute_block_batch_size: usize,
         legacy_state_root: bool,
         always_compare_trie_updates: bool,
@@ -267,6 +277,7 @@ impl TreeConfig {
             persistence_backpressure_threshold,
             block_buffer_limit,
             max_invalid_header_cache_length,
+            invalid_header_hit_eviction_threshold,
             max_execute_block_batch_size,
             legacy_state_root,
             always_compare_trie_updates,
@@ -321,6 +332,14 @@ impl TreeConfig {
         self.max_invalid_header_cache_length
     }
 
+    /// Return the invalid header cache hit eviction threshold.
+    ///
+    /// Setting this to `0` effectively disables the cache because entries are evicted on the
+    /// first lookup.
+    pub const fn invalid_header_hit_eviction_threshold(&self) -> u8 {
+        self.invalid_header_hit_eviction_threshold
+    }
+
     /// Return the maximum execute block batch size.
     pub const fn max_execute_block_batch_size(&self) -> usize {
         self.max_execute_block_batch_size
@@ -451,6 +470,15 @@ impl TreeConfig {
         self
     }
 
+    /// Setter for the invalid header cache hit eviction threshold.
+    pub const fn with_invalid_header_hit_eviction_threshold(
+        mut self,
+        invalid_header_hit_eviction_threshold: u8,
+    ) -> Self {
+        self.invalid_header_hit_eviction_threshold = invalid_header_hit_eviction_threshold;
+        self
+    }
+
     /// Setter for maximum execute block batch size.
     pub const fn with_max_execute_block_batch_size(
         mut self,

crates/engine/tree/src/tree/invalid_headers.rs

@@ -8,25 +8,28 @@ use schnellru::{ByLength, LruMap};
 use std::fmt::Debug;
 use tracing::warn;
 
-/// The max hit counter for invalid headers in the cache before it is forcefully evicted.
-///
-/// In other words, if a header is referenced more than this number of times, it will be evicted to
-/// allow for reprocessing.
-const INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
-
 /// Keeps track of invalid headers.
 #[derive(Debug)]
 pub struct InvalidHeaderCache {
     /// This maps a header hash to a reference to its invalid ancestor.
     headers: LruMap<B256, HeaderEntry>,
+    /// Number of cache hits before an invalid header entry is evicted and reprocessed.
+    hit_eviction_threshold: u8,
     /// Metrics for the cache.
     metrics: InvalidHeaderCacheMetrics,
 }
 
 impl InvalidHeaderCache {
     /// Invalid header cache constructor.
-    pub fn new(max_length: u32) -> Self {
-        Self { headers: LruMap::new(ByLength::new(max_length)), metrics: Default::default() }
+    ///
+    /// Setting `hit_eviction_threshold` to `0` effectively disables the cache because entries are
+    /// evicted on the first lookup.
+    pub fn new(max_length: u32, hit_eviction_threshold: u8) -> Self {
+        Self {
+            headers: LruMap::new(ByLength::new(max_length)),
+            hit_eviction_threshold,
+            metrics: Default::default(),
+        }
     }
 
     fn insert_entry(&mut self, hash: B256, header: BlockWithParent) {
@@ -41,7 +44,7 @@ impl InvalidHeaderCache {
         {
             let entry = self.headers.get(hash)?;
             entry.hit_count += 1;
-            if entry.hit_count < INVALID_HEADER_HIT_EVICTION_THRESHOLD {
+            if entry.hit_count < self.hit_eviction_threshold {
                 return Some(entry.header)
             }
         }
@@ -110,17 +113,28 @@ mod tests {
 
     #[test]
     fn test_hit_eviction() {
-        let mut cache = InvalidHeaderCache::new(10);
+        let hit_eviction_threshold = 3;
+        let mut cache = InvalidHeaderCache::new(10, hit_eviction_threshold);
         let header = Header::default();
         let header = SealedHeader::seal_slow(header);
         cache.insert(header.block_with_parent());
         assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, 0);
 
-        for hit in 1..INVALID_HEADER_HIT_EVICTION_THRESHOLD {
+        for hit in 1..hit_eviction_threshold {
             assert!(cache.get(&header.hash()).is_some());
             assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, hit);
         }
 
         assert!(cache.get(&header.hash()).is_none());
     }
+
+    #[test]
+    fn test_zero_hit_eviction_threshold_effectively_disables_cache() {
+        let mut cache = InvalidHeaderCache::new(10, 0);
+        let header = SealedHeader::seal_slow(Header::default());
+        cache.insert(header.block_with_parent());
+
+        assert!(cache.get(&header.hash()).is_none());
+        assert_eq!(cache.headers.len(), 0);
+    }
 }

crates/engine/tree/src/tree/mod.rs

@@ -151,11 +151,15 @@ impl<N: NodePrimitives> EngineApiTreeState<N> {
     fn new(
         block_buffer_limit: u32,
         max_invalid_header_cache_length: u32,
+        invalid_header_hit_eviction_threshold: u8,
         canonical_block: BlockNumHash,
         engine_kind: EngineApiKind,
     ) -> Self {
         Self {
-            invalid_headers: InvalidHeaderCache::new(max_invalid_header_cache_length),
+            invalid_headers: InvalidHeaderCache::new(
+                max_invalid_header_cache_length,
+                invalid_header_hit_eviction_threshold,
+            ),
             buffer: BlockBuffer::new(block_buffer_limit),
             tree_state: TreeState::new(canonical_block, engine_kind),
             forkchoice_state_tracker: ForkchoiceStateTracker::default(),
@@ -436,6 +440,7 @@ where
         let state = EngineApiTreeState::new(
             config.block_buffer_limit(),
             config.max_invalid_header_cache_length(),
+            config.invalid_header_hit_eviction_threshold(),
             header.num_hash(),
             kind,
         );

crates/engine/tree/src/tree/tests.rs

@@ -184,11 +184,18 @@ impl TestHarness {
         let payload_validator = MockEngineValidator;
 
         let (from_tree_tx, from_tree_rx) = unbounded_channel();
+        let tree_config =
+            TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true);
 
         let header = chain_spec.genesis_header().clone();
         let header = SealedHeader::seal_slow(header);
-        let engine_api_tree_state =
-            EngineApiTreeState::new(10, 10, header.num_hash(), EngineApiKind::Ethereum);
+        let engine_api_tree_state = EngineApiTreeState::new(
+            10,
+            10,
+            tree_config.invalid_header_hit_eviction_threshold(),
+            header.num_hash(),
+            EngineApiKind::Ethereum,
+        );
         let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None);
 
         let (to_payload_service, _payload_command_rx) = unbounded_channel();
@@ -217,8 +224,7 @@ impl TestHarness {
             persistence_handle,
             PersistenceState { last_persisted_block: BlockNumHash::default(), rx: None },
             payload_builder,
-            // always assume enough parallelism for tests
-            TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
+            tree_config,
             EngineApiKind::Ethereum,
             evm_config,
             changeset_cache,

crates/node/core/src/args/engine.rs

@@ -4,8 +4,9 @@ use clap::{builder::Resettable, Args};
 use eyre::ensure;
 use reth_cli_util::{parse_duration_from_secs_or_ms, parsers::format_duration_as_secs_or_ms};
 use reth_engine_primitives::{
-    TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
-    DEFAULT_SPARSE_TRIE_MAX_HOT_ACCOUNTS, DEFAULT_SPARSE_TRIE_MAX_HOT_SLOTS,
+    TreeConfig, DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE,
+    DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, DEFAULT_SPARSE_TRIE_MAX_HOT_ACCOUNTS,
+    DEFAULT_SPARSE_TRIE_MAX_HOT_SLOTS,
 };
 use std::{sync::OnceLock, time::Duration};
 
@@ -25,6 +26,7 @@ pub struct DefaultEngineValues {
     persistence_threshold: u64,
     persistence_backpressure_threshold: u64,
     memory_block_buffer_target: u64,
+    invalid_header_hit_eviction_threshold: u8,
     legacy_state_root_task_enabled: bool,
     state_cache_disabled: bool,
     prewarming_disabled: bool,
@@ -81,6 +83,12 @@ impl DefaultEngineValues {
         self
     }
 
+    /// Set the invalid header cache hit eviction threshold
+    pub const fn with_invalid_header_hit_eviction_threshold(mut self, v: u8) -> Self {
+        self.invalid_header_hit_eviction_threshold = v;
+        self
+    }
+
     /// Set whether to enable legacy state root task by default
     pub const fn with_legacy_state_root_task_enabled(mut self, v: bool) -> Self {
         self.legacy_state_root_task_enabled = v;
@@ -241,6 +249,7 @@ impl Default for DefaultEngineValues {
             persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD,
             persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
             memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
+            invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD,
             legacy_state_root_task_enabled: false,
             state_cache_disabled: false,
             prewarming_disabled: false,
@@ -293,6 +302,14 @@ pub struct EngineArgs {
     #[arg(long = "engine.memory-block-buffer-target", default_value_t = DefaultEngineValues::get_global().memory_block_buffer_target)]
     pub memory_block_buffer_target: u64,
 
+    /// Configure how many cache hits an invalid header can accumulate before it is evicted and
+    /// reprocessed.
+    ///
+    /// Set to `0` to effectively disable the cache because entries are evicted on the first
+    /// lookup.
+    #[arg(long = "engine.invalid-header-cache-hit-eviction-threshold", default_value_t = DefaultEngineValues::get_global().invalid_header_hit_eviction_threshold)]
+    pub invalid_header_hit_eviction_threshold: u8,
+
     /// Enable legacy state root
     #[arg(long = "engine.legacy-state-root", default_value_t = DefaultEngineValues::get_global().legacy_state_root_task_enabled)]
     pub legacy_state_root_task_enabled: bool,
@@ -499,6 +516,7 @@ impl Default for EngineArgs {
             persistence_threshold,
             persistence_backpressure_threshold,
             memory_block_buffer_target,
+            invalid_header_hit_eviction_threshold,
             legacy_state_root_task_enabled,
             state_cache_disabled,
             prewarming_disabled,
@@ -529,6 +547,7 @@ impl Default for EngineArgs {
             persistence_threshold,
             persistence_backpressure_threshold,
             memory_block_buffer_target,
+            invalid_header_hit_eviction_threshold,
             legacy_state_root_task_enabled,
             state_root_task_compare_updates,
             caching_and_prewarming_enabled: true,
@@ -584,6 +603,7 @@ impl EngineArgs {
             .with_persistence_threshold(self.persistence_threshold)
             .with_persistence_backpressure_threshold(self.persistence_backpressure_threshold)
             .with_memory_block_buffer_target(self.memory_block_buffer_target)
+            .with_invalid_header_hit_eviction_threshold(self.invalid_header_hit_eviction_threshold)
             .with_legacy_state_root(self.legacy_state_root_task_enabled)
             .without_state_cache(self.state_cache_disabled)
             .without_prewarming(self.prewarming_disabled)
@@ -643,6 +663,7 @@ mod tests {
             persistence_threshold: 100,
             persistence_backpressure_threshold: 101,
             memory_block_buffer_target: 50,
+            invalid_header_hit_eviction_threshold: 7,
             legacy_state_root_task_enabled: true,
             caching_and_prewarming_enabled: true,
             state_cache_disabled: true,
@@ -684,6 +705,8 @@ mod tests {
             "101",
             "--engine.memory-block-buffer-target",
             "50",
+            "--engine.invalid-header-cache-hit-eviction-threshold",
+            "7",
             "--engine.legacy-state-root",
             "--engine.disable-state-cache",
             "--engine.disable-prewarming",
@@ -763,6 +786,28 @@ mod tests {
         assert_eq!(args.slow_block_threshold, Some(Duration::from_millis(500)));
     }
 
+    #[test]
+    fn test_parse_invalid_header_hit_eviction_threshold() {
+        let args = CommandParser::<EngineArgs>::parse_from(["reth"]).args;
+        assert_eq!(
+            args.invalid_header_hit_eviction_threshold,
+            DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD
+        );
+        assert_eq!(
+            args.tree_config().invalid_header_hit_eviction_threshold(),
+            DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD
+        );
+
+        let args = CommandParser::<EngineArgs>::parse_from([
+            "reth",
+            "--engine.invalid-header-cache-hit-eviction-threshold",
+            "0",
+        ])
+        .args;
+        assert_eq!(args.invalid_header_hit_eviction_threshold, 0);
+        assert_eq!(args.tree_config().invalid_header_hit_eviction_threshold(), 0);
+    }
+
     #[test]
     fn test_parse_share_sparse_trie_flag() {
         let args = CommandParser::<EngineArgs>::parse_from(["reth"]).args;

docs/vocs/docs/pages/cli/reth/node.mdx

@@ -950,6 +950,13 @@ Engine:
 
           [default: 0]
 
+      --engine.invalid-header-cache-hit-eviction-threshold <INVALID_HEADER_HIT_EVICTION_THRESHOLD>
+          Configure how many cache hits an invalid header can accumulate before it is evicted and reprocessed.
+
+          Set to `0` to effectively disable the cache because entries are evicted on the first lookup.
+
+          [default: 128]
+
       --engine.legacy-state-root
           Enable legacy state root