diff --git a/crates/node/p2p/src/gossip/block_validity.rs b/crates/node/p2p/src/gossip/block_validity.rs
index 8735f45a52..bdc43ff020 100644
--- a/crates/node/p2p/src/gossip/block_validity.rs
+++ b/crates/node/p2p/src/gossip/block_validity.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashSet, time::SystemTime};
+use std::time::SystemTime;
 
 use alloy_consensus::Block;
 use alloy_eips::eip7685::EMPTY_REQUESTS_HASH;
@@ -160,13 +160,6 @@ impl BlockHandler {
                     block_hash: envelope.payload.block_hash(),
                 });
             }
-
-            seen_hashes_at_height.insert(envelope.payload.block_hash());
-        } else {
-            self.seen_hashes.insert(
-                envelope.payload.block_number(),
-                HashSet::from([envelope.payload.block_hash()]),
-            );
         }
 
         // CHECK: The signature is valid.
@@ -183,6 +176,11 @@ impl BlockHandler {
             return Err(BlockInvalidError::Signer { expected: msg_signer, received: block_signer });
         }
 
+        self.seen_hashes
+            .entry(envelope.payload.block_number())
+            .or_default()
+            .insert(envelope.payload.block_hash());
+
         // Mark the block as seen.
         if self.seen_hashes.len() >= Self::SEEN_HASH_CACHE_SIZE {
             self.seen_hashes.pop_first();
@@ -576,6 +574,7 @@ pub(crate) mod tests {
         signature_bytes[0] = !signature_bytes[0];
         envelope.signature = Signature::from_raw_array(&signature_bytes).unwrap();
 
+        assert!(handler.seen_hashes.is_empty());
         assert!(matches!(handler.block_valid(&envelope), Err(BlockInvalidError::Signature)));
     }