fix(flat): periodically clear ReadOnlySnapshotBundle cache

src/Nethermind/Nethermind.State.Flat.Test/FlatDbManagerTests.cs

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Config;
@@ -140,6 +141,37 @@ public async Task AddSnapshot_ValidSnapshot_AddsToRepository()
         _snapshotRepository.Received(1).TryAddSnapshot(snapshot);
     }
 
+    [Test]
+    public async Task GatherReadOnlySnapshotBundle_CacheClearedPeriodically()
+    {
+        StateId stateId = CreateStateId(10);
+
+        IPersistence.IPersistenceReader mockReader = Substitute.For<IPersistence.IPersistenceReader>();
+        mockReader.CurrentState.Returns(stateId);
+
+        _persistenceManager.LeaseReader().Returns(mockReader);
+        _snapshotRepository.AssembleSnapshots(stateId, stateId, Arg.Any<int>())
+            .Returns(new SnapshotPooledList(0));
+
+        await using FlatDbManager manager = CreateManager();
+
+        // First call populates the cache
+        using (ReadOnlySnapshotBundle bundle1 = manager.GatherReadOnlySnapshotBundle(stateId)) { }
+
+        // Second call should hit cache (no new LeaseReader call)
+        _persistenceManager.ClearReceivedCalls();
+        using (ReadOnlySnapshotBundle bundle2 = manager.GatherReadOnlySnapshotBundle(stateId)) { }
+        _persistenceManager.DidNotReceive().LeaseReader();
+
+        // Wait for periodic clear (15s + margin)
+        await Task.Delay(TimeSpan.FromSeconds(17));
+
+        // After cache clear, next call needs a new reader
+        _persistenceManager.ClearReceivedCalls();
+        using (ReadOnlySnapshotBundle bundle3 = manager.GatherReadOnlySnapshotBundle(stateId)) { }
+        _persistenceManager.Received(1).LeaseReader();
+    }
+
     [Test]
     public async Task AddSnapshot_DuplicateSnapshot_DisposesSnapshotAndReturnsResource()
     {

src/Nethermind/Nethermind.State.Flat/FlatDbManager.cs

@@ -5,7 +5,6 @@
 using System.Diagnostics;
 using System.Threading.Channels;
 using Nethermind.Config;
-using Nethermind.Core.Collections;
 using Nethermind.Core.Crypto;
 using Nethermind.Db;
 using Nethermind.Logging;
@@ -45,6 +44,9 @@ public class FlatDbManager : IFlatDbManager, IAsyncDisposable
     private readonly Task _persistenceTask;
     private readonly Channel<StateId> _persistenceJobs;
 
+    // Periodically clear the ReadOnlySnapshotBundle cache to prevent stale entries
+    private readonly Task _clearBundleCacheTask;
+
     private readonly int _compactSize;
 
     // For debugging. Do the compaction synchronously
@@ -86,6 +88,7 @@ public FlatDbManager(
         _compactorTask = RunCompactor(_cancelTokenSource.Token);
         _populateTrieNodeCacheTask = RunTrieCachePopulator(_cancelTokenSource.Token);
         _persistenceTask = RunPersistence(_cancelTokenSource.Token);
+        _clearBundleCacheTask = RunClearBundleCache(_cancelTokenSource.Token);
     }
 
     private async Task RunCompactor(CancellationToken cancellationToken)
@@ -351,14 +354,26 @@ public void AddSnapshot(Snapshot snapshot, TransientResource transientResource)
         }
     }
 
-    private void ClearReadOnlyBundleCache()
+    private async Task RunClearBundleCache(CancellationToken cancellationToken)
     {
-        using ArrayPoolListRef<StateId> statesToRemove = new();
-        statesToRemove.AddRange(_readonlySnapshotBundleCache.Keys);
+        using PeriodicTimer timer = new(TimeSpan.FromSeconds(15));
+        try
+        {
+            while (await timer.WaitForNextTickAsync(cancellationToken))
+            {
+                ClearReadOnlyBundleCache();
+            }
+        }
+        catch (OperationCanceledException)
+        {
+        }
+    }
 
-        foreach (StateId stateId in statesToRemove)
+    private void ClearReadOnlyBundleCache()
+    {
+        foreach (KeyValuePair<StateId, ReadOnlySnapshotBundle> entry in _readonlySnapshotBundleCache)
         {
-            if (_readonlySnapshotBundleCache.TryRemove(stateId, out ReadOnlySnapshotBundle? bundle))
+            if (_readonlySnapshotBundleCache.TryRemove(entry.Key, out ReadOnlySnapshotBundle? bundle))
             {
                 bundle.Dispose();
             }
@@ -403,6 +418,7 @@ public async ValueTask DisposeAsync()
         await _compactorTask;
         await _populateTrieNodeCacheTask;
         await _persistenceTask;
+        await _clearBundleCacheTask;
 
         _cancelTokenSource.Dispose();
     }

src/Nethermind/Nethermind.State.Flat/Persistence/RefCountingPersistenceReader.cs

@@ -16,7 +16,6 @@ public class RefCountingPersistenceReader : RefCountingDisposable, IPersistence.
     private const int NoAccessors = 0; // Same as parent's constant
     private const int Disposing = -1; // Same as parent's constant
     private readonly IPersistence.IPersistenceReader _innerReader;
-
     public RefCountingPersistenceReader(IPersistence.IPersistenceReader innerReader, ILogger logger)
     {
         _innerReader = innerReader;