Reduce storage tracking allocations

src/Nethermind/Nethermind.Core/Caching/StaticPool.cs

@@ -0,0 +1,91 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Collections.Concurrent;
+using System.Threading;
+using Nethermind.Core.Resettables;
+
+namespace Nethermind.Core.Caching;
+
+/// <summary>
+/// High performance static pool for reference types that support reset semantics.
+/// </summary>
+/// <typeparam name="T">
+/// The pooled type. Must be a reference type that implements <see cref="IResettable"/> and
+/// has a public parameterless constructor.
+/// </typeparam>
+public static class StaticPool<T> where T : class, IResettable, new()
+{
+    /// <summary>
+    /// Hard cap for the total number of items that can be stored in the shared pool.
+    /// Prevents unbounded growth under bursty workloads while still allowing reuse.
+    /// </summary>
+    private const int MaxPooledCount = 4096;
+
+    /// <summary>
+    /// Global pool shared between threads.
+    /// </summary>
+    private static readonly ConcurrentQueue<T> _pool = [];
+
+    /// <summary>
+    /// Manual count of items in the queue.
+    /// We maintain this separately because ConcurrentQueue.Count
+    /// is an O(n) traversal — it walks the internal segment chain.
+    /// Keeping our own count avoids that cost and keeps the hot path O(1).
+    /// </summary>
+    private static int _poolCount;
+
+    /// <summary>
+    /// Rents an instance of <typeparamref name="T"/> from the pool.
+    /// </summary>
+    /// <remarks>
+    /// The method first attempts to dequeue an existing instance from the shared pool.
+    /// If the pool is empty, a new instance is created using the parameterless constructor.
+    /// </remarks>
+    /// <returns>
+    /// A reusable instance of <typeparamref name="T"/>. The returned instance is not guaranteed
+    /// to be zeroed or reset beyond the guarantees provided by <see cref="IResettable"/> and
+    /// the constructor. Callers should treat it as a freshly created instance.
+    /// </returns>
+    public static T Rent()
+    {
+        // Try to pop from the global pool — this is only hit when a thread
+        // has exhausted its own fast slot or is cross-thread renting.
+        if (Volatile.Read(ref _poolCount) > 0 && _pool.TryDequeue(out T? item))
+        {
+            // We track count manually with Interlocked ops instead of using queue.Count.
+            Interlocked.Decrement(ref _poolCount);
+            return item;
+        }
+
+        // Nothing available, allocate new instance
+        return new();
+    }
+
+    /// <summary>
+    /// Returns an instance of <typeparamref name="T"/> to the pool for reuse.
+    /// </summary>
+    /// <remarks>
+    /// The instance is reset via <see cref="IResettable.Reset"/> before being enqueued.
+    /// If adding the instance would exceed <see cref="MaxPooledCount"/>, the instance is
+    /// discarded and not pooled.
+    /// </remarks>
+    /// <param name="item">
+    /// The instance to return to the pool. Must not be <see langword="null"/>.
+    /// After returning, the caller must not use the instance again.
+    /// </param>
+    public static void Return(T item)
+    {
+        // We use Interlocked.Increment to reserve a slot up front.
+        // This guarantees a bounded queue length without relying on slow Count().
+        if (Interlocked.Increment(ref _poolCount) > MaxPooledCount)
+        {
+            // Roll back reservation if we'd exceed the cap.
+            Interlocked.Decrement(ref _poolCount);
+            return;
+        }
+
+        item.Reset();
+        _pool.Enqueue(item);
+    }
+}

src/Nethermind/Nethermind.Core/Collections/StackList.cs

@@ -4,10 +4,12 @@
 using System;
 using System.Collections.Generic;
 using System.Runtime.InteropServices;
+using Nethermind.Core.Caching;
+using Nethermind.Core.Resettables;
 
 namespace Nethermind.Core.Collections
 {
-    public sealed class StackList<T> : List<T>
+    public sealed class StackList<T> : List<T>, IResettable, IReturnable
         where T : struct, IComparable<T>
     {
         public T Peek() => this[^1];
@@ -47,10 +49,7 @@ public bool TryPop(out T item)
             }
         }
 
-        public void Push(T item)
-        {
-            Add(item);
-        }
+        public void Push(T item) => Add(item);
 
         public bool TryGetSearchedItem(T activation, out T item)
         {
@@ -79,5 +78,21 @@ public bool TryGetSearchedItem(T activation, out T item)
 
             return result;
         }
+
+        internal static StackList<T> Rent()
+            => StaticPool<StackList<T>>.Rent();
+
+        public void Return() => Return(this);
+        public void Reset() => Clear();
+
+        private static void Return(StackList<T> value)
+        {
+            const int MaxPooledCapacity = 128;
+
+            if (value.Capacity > MaxPooledCapacity)
+                return;
+
+            StaticPool<StackList<T>>.Return(value);
+        }
     }
 }

src/Nethermind/Nethermind.Core/Extensions/DictionaryExtensions.cs

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Collections.Generic;
+using Nethermind.Core.Resettables;
+
+namespace Nethermind.Core.Extensions;
+
+public static class DictionaryExtensions
+{
+    /// <summary>
+    /// Returns all values in the dictionary to their pool by calling <see cref="IReturnable.Return"/> on each value,
+    /// then clears the dictionary.
+    /// </summary>
+    /// <typeparam name="TKey">The type of the keys in the dictionary.</typeparam>
+    /// <typeparam name="TValue">The type of the values in the dictionary, which must implement <see cref="IReturnable"/>.</typeparam>
+    /// <param name="dictionary">The dictionary whose values will be returned and cleared.</param>
+    /// <remarks>
+    /// Use this method when you need to both return pooled objects and clear the dictionary in one operation.
+    /// </remarks>
+    public static void ResetAndClear<TKey, TValue>(this IDictionary<TKey, TValue> dictionary)
+        where TValue : class, IReturnable
+    {
+        foreach (TValue value in dictionary.Values)
+        {
+            value.Return();
+        }
+        dictionary.Clear();
+    }
+}

src/Nethermind/Nethermind.Core/Resettables/IResettable.cs

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Core.Resettables;
+
+/// <summary>
+/// Defines a contract for objects that can be reset to their initial state.
+/// </summary>
+public interface IResettable
+{
+    /// <summary>
+    /// Resets the object to its initial state.
+    /// </summary>
+    void Reset();
+}

src/Nethermind/Nethermind.Core/Resettables/IReturnable.cs

@@ -0,0 +1,17 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Core.Resettables;
+
+/// <summary>
+/// Defines a contract for objects that can be returned to a pool or resettable resource manager.
+/// Implementations should ensure that <see cref="Return"/> releases or resets the object for reuse.
+/// </summary>
+public interface IReturnable
+{
+    /// <summary>
+    /// Returns the object to its pool or resource manager, making it available for reuse.
+    /// Implementations should ensure the object is properly reset or cleaned up.
+    /// </summary>
+    void Return();
+}

src/Nethermind/Nethermind.Evm/StackPool.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Runtime.Intrinsics;
+using System.Threading;
 
 using static Nethermind.Evm.EvmState;
 
@@ -29,24 +30,35 @@ private readonly struct StackItem(byte[] dataStack, ReturnState[] returnStack)
     /// <param name="returnStack"></param>
     public void ReturnStacks(byte[] dataStack, ReturnState[] returnStack)
     {
-        if (_stackPool.Count <= MaxStacksPooled)
+        // Reserve a slot first - O(1) bound without touching ConcurrentQueue.Count.
+        if (Interlocked.Increment(ref _poolCount) > MaxStacksPooled)
         {
-            _stackPool.Enqueue(new(dataStack, returnStack));
+            // Cap hit - roll back the reservation and drop the item.
+            Interlocked.Decrement(ref _poolCount);
+            return;
         }
+
+        _stackPool.Enqueue(new StackItem(dataStack, returnStack));
     }
 
+    // Manual reservation count - upper bound on items actually in the queue.
+    private int _poolCount;
+
     public const int StackLength = (EvmStack.MaxStackSize + EvmStack.RegisterLength) * 32;
 
     public (byte[], ReturnState[]) RentStacks()
     {
-        if (_stackPool.TryDequeue(out StackItem result))
+        if (Volatile.Read(ref _poolCount) > 0 && _stackPool.TryDequeue(out StackItem result))
         {
+            Interlocked.Decrement(ref _poolCount);
             return (result.DataStack, result.ReturnStack);
         }
 
+        // Count was positive but we lost the race or the enqueuer has not published yet.
+        // Include extra Vector256<byte>.Count and pin so we can align to 32 bytes.
+        // This ensures the stack is properly aligned for SIMD operations.
         return
         (
-            // Include extra Vector256<byte>.Count and pin so we can align to 32 bytes
             GC.AllocateUninitializedArray<byte>(StackLength + Vector256<byte>.Count, pinned: true),
             new ReturnState[EvmStack.ReturnStackSize]
         );

src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
@@ -7,6 +7,7 @@
 using Nethermind.Core;
 using Nethermind.Core.Collections;
 using Nethermind.Core.Resettables;
+using Nethermind.Core.Extensions;
 using Nethermind.Evm.Tracing.State;
 using Nethermind.Logging;
 
@@ -148,29 +149,6 @@ public void Commit(bool commitRoots = true)
             Commit(NullStateTracer.Instance, commitRoots);
         }
 
-        protected struct ChangeTrace
-        {
-            public static readonly ChangeTrace _zeroBytes = new(StorageTree.ZeroBytes, StorageTree.ZeroBytes);
-            public static ref readonly ChangeTrace ZeroBytes => ref _zeroBytes;
-
-            public ChangeTrace(byte[]? before, byte[]? after)
-            {
-                After = after ?? StorageTree.ZeroBytes;
-                Before = before ?? StorageTree.ZeroBytes;
-            }
-
-            public ChangeTrace(byte[]? after)
-            {
-                After = after ?? StorageTree.ZeroBytes;
-                Before = StorageTree.ZeroBytes;
-                IsInitialValue = true;
-            }
-
-            public byte[] Before;
-            public byte[] After;
-            public bool IsInitialValue;
-        }
-
         /// <summary>
         /// Commit persistent storage
         /// </summary>
@@ -202,22 +180,19 @@ protected virtual void CommitStorageRoots()
         /// Used for storage-specific logic
         /// </summary>
         /// <param name="tracer">Storage tracer</param>
-        protected virtual void CommitCore(IStorageTracer tracer)
-        {
-            _changes.Clear();
-            _intraBlockCache.Clear();
-            _transactionChangesSnapshots.Clear();
-        }
+        protected virtual void CommitCore(IStorageTracer tracer) => Reset();
 
         /// <summary>
         /// Reset the storage state
         /// </summary>
-        public virtual void Reset(bool resetBlockChanges = true)
+        public virtual void Reset(bool resetBlockChanges = true) => Reset();
+
+        private void Reset()
         {
             if (_logger.IsTrace) _logger.Trace("Resetting storage");
 
             _changes.Clear();
-            _intraBlockCache.Clear();
+            _intraBlockCache.ResetAndClear();
             _transactionChangesSnapshots.Clear();
         }
 
@@ -270,7 +245,7 @@ protected StackList<int> SetupRegistry(in StorageCell cell)
             ref StackList<int>? value = ref CollectionsMarshal.GetValueRefOrAddDefault(_intraBlockCache, cell, out bool exists);
             if (!exists)
             {
-                value = new StackList<int>();
+                value = StackList<int>.Rent();
             }
 
             return value;