diff --git a/src/Nethermind/Nethermind.Core/Caching/StaticPool.cs b/src/Nethermind/Nethermind.Core/Caching/StaticPool.cs new file mode 100644 index 00000000000..520010fb55a --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Caching/StaticPool.cs @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using System.Threading; +using Nethermind.Core.Resettables; + +namespace Nethermind.Core.Caching; + +/// +/// High performance static pool for reference types that support reset semantics. +/// +/// +/// The pooled type. Must be a reference type that implements and +/// has a public parameterless constructor. +/// +public static class StaticPool where T : class, IResettable, new() +{ + /// + /// Hard cap for the total number of items that can be stored in the shared pool. + /// Prevents unbounded growth under bursty workloads while still allowing reuse. + /// + private const int MaxPooledCount = 4096; + + /// + /// Global pool shared between threads. + /// + private static readonly ConcurrentQueue _pool = []; + + /// + /// Manual count of items in the queue. + /// We maintain this separately because ConcurrentQueue.Count + /// is an O(n) traversal — it walks the internal segment chain. + /// Keeping our own count avoids that cost and keeps the hot path O(1). + /// + private static int _poolCount; + + /// + /// Rents an instance of from the pool. + /// + /// + /// The method first attempts to dequeue an existing instance from the shared pool. + /// If the pool is empty, a new instance is created using the parameterless constructor. + /// + /// + /// A reusable instance of . The returned instance is not guaranteed + /// to be zeroed or reset beyond the guarantees provided by and + /// the constructor. Callers should treat it as a freshly created instance. + /// + public static T Rent() + { + // Try to pop from the global pool — this is only hit when a thread + // has exhausted its own fast slot or is cross-thread renting. + if (Volatile.Read(ref _poolCount) > 0 && _pool.TryDequeue(out T? item)) + { + // We track count manually with Interlocked ops instead of using queue.Count. + Interlocked.Decrement(ref _poolCount); + return item; + } + + // Nothing available, allocate new instance + return new(); + } + + /// + /// Returns an instance of to the pool for reuse. + /// + /// + /// The instance is reset via before being enqueued. + /// If adding the instance would exceed , the instance is + /// discarded and not pooled. + /// + /// + /// The instance to return to the pool. Must not be . + /// After returning, the caller must not use the instance again. + /// + public static void Return(T item) + { + // We use Interlocked.Increment to reserve a slot up front. + // This guarantees a bounded queue length without relying on slow Count(). + if (Interlocked.Increment(ref _poolCount) > MaxPooledCount) + { + // Roll back reservation if we'd exceed the cap. + Interlocked.Decrement(ref _poolCount); + return; + } + + item.Reset(); + _pool.Enqueue(item); + } +} diff --git a/src/Nethermind/Nethermind.Core/Collections/StackList.cs b/src/Nethermind/Nethermind.Core/Collections/StackList.cs index 2170d94de1b..1537fc69d04 100644 --- a/src/Nethermind/Nethermind.Core/Collections/StackList.cs +++ b/src/Nethermind/Nethermind.Core/Collections/StackList.cs @@ -4,10 +4,12 @@ using System; using System.Collections.Generic; using System.Runtime.InteropServices; +using Nethermind.Core.Caching; +using Nethermind.Core.Resettables; namespace Nethermind.Core.Collections { - public sealed class StackList : List + public sealed class StackList : List, IResettable, IReturnable where T : struct, IComparable { public T Peek() => this[^1]; @@ -47,10 +49,7 @@ public bool TryPop(out T item) } } - public void Push(T item) - { - Add(item); - } + public void Push(T item) => Add(item); public bool TryGetSearchedItem(T activation, out T item) { @@ -79,5 +78,21 @@ public bool TryGetSearchedItem(T activation, out T item) return result; } + + internal static StackList Rent() + => StaticPool>.Rent(); + + public void Return() => Return(this); + public void Reset() => Clear(); + + private static void Return(StackList value) + { + const int MaxPooledCapacity = 128; + + if (value.Capacity > MaxPooledCapacity) + return; + + StaticPool>.Return(value); + } } } diff --git a/src/Nethermind/Nethermind.Core/Extensions/DictionaryExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/DictionaryExtensions.cs new file mode 100644 index 00000000000..6168e96121f --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Extensions/DictionaryExtensions.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core.Resettables; + +namespace Nethermind.Core.Extensions; + +public static class DictionaryExtensions +{ + /// + /// Returns all values in the dictionary to their pool by calling on each value, + /// then clears the dictionary. + /// + /// The type of the keys in the dictionary. + /// The type of the values in the dictionary, which must implement . + /// The dictionary whose values will be returned and cleared. + /// + /// Use this method when you need to both return pooled objects and clear the dictionary in one operation. + /// + public static void ResetAndClear(this IDictionary dictionary) + where TValue : class, IReturnable + { + foreach (TValue value in dictionary.Values) + { + value.Return(); + } + dictionary.Clear(); + } +} diff --git a/src/Nethermind/Nethermind.Core/Resettables/IResettable.cs b/src/Nethermind/Nethermind.Core/Resettables/IResettable.cs new file mode 100644 index 00000000000..3680f13bbf2 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Resettables/IResettable.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Core.Resettables; + +/// +/// Defines a contract for objects that can be reset to their initial state. +/// +public interface IResettable +{ + /// + /// Resets the object to its initial state. + /// + void Reset(); +} diff --git a/src/Nethermind/Nethermind.Core/Resettables/IReturnable.cs b/src/Nethermind/Nethermind.Core/Resettables/IReturnable.cs new file mode 100644 index 00000000000..228bb5ebb86 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Resettables/IReturnable.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Core.Resettables; + +/// +/// Defines a contract for objects that can be returned to a pool or resettable resource manager. +/// Implementations should ensure that releases or resets the object for reuse. +/// +public interface IReturnable +{ + /// + /// Returns the object to its pool or resource manager, making it available for reuse. + /// Implementations should ensure the object is properly reset or cleaned up. + /// + void Return(); +} diff --git a/src/Nethermind/Nethermind.Evm/StackPool.cs b/src/Nethermind/Nethermind.Evm/StackPool.cs index a51014738ee..2be787ee0b2 100644 --- a/src/Nethermind/Nethermind.Evm/StackPool.cs +++ b/src/Nethermind/Nethermind.Evm/StackPool.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Concurrent; using System.Runtime.Intrinsics; +using System.Threading; using static Nethermind.Evm.EvmState; @@ -29,24 +30,35 @@ private readonly struct StackItem(byte[] dataStack, ReturnState[] returnStack) /// public void ReturnStacks(byte[] dataStack, ReturnState[] returnStack) { - if (_stackPool.Count <= MaxStacksPooled) + // Reserve a slot first - O(1) bound without touching ConcurrentQueue.Count. + if (Interlocked.Increment(ref _poolCount) > MaxStacksPooled) { - _stackPool.Enqueue(new(dataStack, returnStack)); + // Cap hit - roll back the reservation and drop the item. + Interlocked.Decrement(ref _poolCount); + return; } + + _stackPool.Enqueue(new StackItem(dataStack, returnStack)); } + // Manual reservation count - upper bound on items actually in the queue. + private int _poolCount; + public const int StackLength = (EvmStack.MaxStackSize + EvmStack.RegisterLength) * 32; public (byte[], ReturnState[]) RentStacks() { - if (_stackPool.TryDequeue(out StackItem result)) + if (Volatile.Read(ref _poolCount) > 0 && _stackPool.TryDequeue(out StackItem result)) { + Interlocked.Decrement(ref _poolCount); return (result.DataStack, result.ReturnStack); } + // Count was positive but we lost the race or the enqueuer has not published yet. + // Include extra Vector256.Count and pin so we can align to 32 bytes. + // This ensures the stack is properly aligned for SIMD operations. return ( - // Include extra Vector256.Count and pin so we can align to 32 bytes GC.AllocateUninitializedArray(StackLength + Vector256.Count, pinned: true), new ReturnState[EvmStack.ReturnStackSize] ); diff --git a/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs b/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs index 57c3253791e..d1b125d01e9 100644 --- a/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs +++ b/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; @@ -7,6 +7,7 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Resettables; +using Nethermind.Core.Extensions; using Nethermind.Evm.Tracing.State; using Nethermind.Logging; @@ -148,29 +149,6 @@ public void Commit(bool commitRoots = true) Commit(NullStateTracer.Instance, commitRoots); } - protected struct ChangeTrace - { - public static readonly ChangeTrace _zeroBytes = new(StorageTree.ZeroBytes, StorageTree.ZeroBytes); - public static ref readonly ChangeTrace ZeroBytes => ref _zeroBytes; - - public ChangeTrace(byte[]? before, byte[]? after) - { - After = after ?? StorageTree.ZeroBytes; - Before = before ?? StorageTree.ZeroBytes; - } - - public ChangeTrace(byte[]? after) - { - After = after ?? StorageTree.ZeroBytes; - Before = StorageTree.ZeroBytes; - IsInitialValue = true; - } - - public byte[] Before; - public byte[] After; - public bool IsInitialValue; - } - /// /// Commit persistent storage /// @@ -202,22 +180,19 @@ protected virtual void CommitStorageRoots() /// Used for storage-specific logic /// /// Storage tracer - protected virtual void CommitCore(IStorageTracer tracer) - { - _changes.Clear(); - _intraBlockCache.Clear(); - _transactionChangesSnapshots.Clear(); - } + protected virtual void CommitCore(IStorageTracer tracer) => Reset(); /// /// Reset the storage state /// - public virtual void Reset(bool resetBlockChanges = true) + public virtual void Reset(bool resetBlockChanges = true) => Reset(); + + private void Reset() { if (_logger.IsTrace) _logger.Trace("Resetting storage"); _changes.Clear(); - _intraBlockCache.Clear(); + _intraBlockCache.ResetAndClear(); _transactionChangesSnapshots.Clear(); } @@ -270,7 +245,7 @@ protected StackList SetupRegistry(in StorageCell cell) ref StackList? value = ref CollectionsMarshal.GetValueRefOrAddDefault(_intraBlockCache, cell, out bool exists); if (!exists) { - value = new StackList(); + value = StackList.Rent(); } return value; diff --git a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs index 6e01ca39f2f..a62d6d194c5 100644 --- a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; @@ -8,18 +8,20 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Threading; using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; +using Nethermind.Core.Resettables; using Nethermind.Core.Threading; using Nethermind.Evm.Tracing.State; -using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Serialization.Rlp; -using Nethermind.Trie; using Nethermind.Trie.Pruning; +using Nethermind.Int256; +using Nethermind.Trie; namespace Nethermind.State; @@ -34,7 +36,7 @@ internal sealed class PersistentStorageProvider : PartialStorageProviderBase private readonly ITrieStore _trieStore; private readonly StateProvider _stateProvider; private readonly ILogManager? _logManager; - internal readonly IStorageTreeFactory _storageTreeFactory; + private readonly IStorageTreeFactory _storageTreeFactory; private readonly Dictionary _storages = new(4_096); private readonly Dictionary _toUpdateRoots = new(); @@ -66,7 +68,7 @@ public PersistentStorageProvider(ITrieStore trieStore, } public Hash256 StateRoot { get; set; } = null!; - private readonly bool _populatePreBlockCache; + internal readonly bool _populatePreBlockCache; /// /// Reset the storage state @@ -78,7 +80,7 @@ public override void Reset(bool resetBlockChanges = true) _committedThisRound.Clear(); if (resetBlockChanges) { - _storages.Clear(); + _storages.ResetAndClear(); _toUpdateRoots.Clear(); } } @@ -140,10 +142,10 @@ protected override void CommitCore(IStorageTracer tracer) HashSet toUpdateRoots = (_tempToUpdateRoots ??= new()); bool isTracing = tracer.IsTracingStorage; - Dictionary? trace = null; + Dictionary? trace = null; if (isTracing) { - trace = new Dictionary(); + trace = []; } for (int i = 0; i <= currentPosition; i++) @@ -158,7 +160,7 @@ protected override void CommitCore(IStorageTracer tracer) { if (isTracing && change.ChangeType == ChangeType.JustCache) { - trace![change.StorageCell] = new ChangeTrace(change.Value, trace[change.StorageCell].After); + trace![change.StorageCell] = new StorageChangeTrace(change.Value, trace[change.StorageCell].After); } continue; @@ -204,7 +206,7 @@ protected override void CommitCore(IStorageTracer tracer) if (isTracing) { - trace![change.StorageCell] = new ChangeTrace(change.Value); + trace![change.StorageCell] = new StorageChangeTrace(change.Value); } } } @@ -353,21 +355,23 @@ public void CommitTrees(IBlockCommitter blockCommitter) // may make it worse. Always check on mainnet. using ArrayPoolListRef commitTask = new(_storages.Count); - foreach (KeyValuePair storage in _storages) + foreach (PerContractState storage in _storages.Values) { - storage.Value.EnsureStorageTree(); // Cannot be called concurrently + storage.EnsureStorageTree(); // Cannot be called concurrently if (blockCommitter.TryRequestConcurrencyQuota()) { commitTask.Add(Task.Factory.StartNew((ctx) => { PerContractState st = (PerContractState)ctx; st.Commit(); + st.Return(); blockCommitter.ReturnConcurrencyQuota(); - }, storage.Value)); + }, storage)); } else { - storage.Value.Commit(); + storage.Commit(); + storage.Return(); } } @@ -379,7 +383,7 @@ public void CommitTrees(IBlockCommitter blockCommitter) private PerContractState GetOrCreateStorage(Address address) { ref PerContractState? value = ref CollectionsMarshal.GetValueRefOrAddDefault(_storages, address, out bool exists); - if (!exists) value = new PerContractState(address, this); + if (!exists) value = PerContractState.Rent(address, this); return value; } @@ -411,9 +415,9 @@ private void PushToRegistryOnly(in StorageCell cell, byte[] value) _changes.Add(new Change(in cell, value, ChangeType.JustCache)); } - private static void ReportChanges(IStorageTracer tracer, Dictionary trace) + private static void ReportChanges(IStorageTracer tracer, Dictionary trace) { - foreach ((StorageCell address, ChangeTrace change) in trace) + foreach ((StorageCell address, StorageChangeTrace change) in trace) { byte[] before = change.Before; byte[] after = change.After; @@ -452,38 +456,44 @@ public StorageTree Create(Address address, IScopedTrieStore trieStore, Hash256 s private sealed class DefaultableDictionary() { private bool _missingAreDefault; - private readonly Dictionary _dictionary = new(Comparer.Instance); + private readonly Dictionary _dictionary = new(Comparer.Instance); public int EstimatedSize => _dictionary.Count; + public int Capacity => _dictionary.Capacity; + public void Reset() + { + _missingAreDefault = false; + _dictionary.Clear(); + } public void ClearAndSetMissingAsDefault() { _missingAreDefault = true; _dictionary.Clear(); } - public ref ChangeTrace GetValueRefOrAddDefault(UInt256 storageCellIndex, out bool exists) + public ref StorageChangeTrace GetValueRefOrAddDefault(UInt256 storageCellIndex, out bool exists) { - ref ChangeTrace value = ref CollectionsMarshal.GetValueRefOrAddDefault(_dictionary, storageCellIndex, out exists); + ref StorageChangeTrace value = ref CollectionsMarshal.GetValueRefOrAddDefault(_dictionary, storageCellIndex, out exists); if (!exists && _missingAreDefault) { // Where we know the rest of the tree is empty // we can say the value was found but is default // rather than having to check the database - value = ChangeTrace.ZeroBytes; + value = StorageChangeTrace.ZeroBytes; exists = true; } return ref value; } - public ref ChangeTrace GetValueRefOrNullRef(UInt256 storageCellIndex) + public ref StorageChangeTrace GetValueRefOrNullRef(UInt256 storageCellIndex) => ref CollectionsMarshal.GetValueRefOrNullRef(_dictionary, storageCellIndex); - public ChangeTrace this[UInt256 key] + public StorageChangeTrace this[UInt256 key] { set => _dictionary[key] = value; } - public Dictionary.Enumerator GetEnumerator() => _dictionary.GetEnumerator(); + public Dictionary.Enumerator GetEnumerator() => _dictionary.GetEnumerator(); private sealed class Comparer : IEqualityComparer { @@ -499,27 +509,35 @@ public int GetHashCode([DisallowNull] UInt256 obj) } } - private sealed class PerContractState + private sealed class PerContractState : IReturnable { + private static readonly Func _loadFromTreeStorageFunc = LoadFromTreeStorage; + + private readonly DefaultableDictionary BlockChange = new(); + private PersistentStorageProvider _provider; + private Address _address; private StorageTree? StorageTree; - private DefaultableDictionary BlockChange = new DefaultableDictionary(); private bool _wasWritten = false; - private readonly Func _loadFromTreeStorageFunc; - private readonly Address _address; - private readonly PersistentStorageProvider _provider; - public PerContractState(Address address, - PersistentStorageProvider provider) + private PerContractState(Address address, PersistentStorageProvider provider) => Initialize(address, provider); + + private void Initialize(Address address, PersistentStorageProvider provider) { _address = address; _provider = provider; - _loadFromTreeStorageFunc = LoadFromTreeStorage; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void EnsureStorageTree() { if (StorageTree is not null) return; + CreateStorageTree(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void CreateStorageTree() + { // Note: GetStorageRoot is not concurrent safe! And so do this whole method! Account? acc = _provider._stateProvider.GetAccount(_address); Hash256 storageRoot = acc?.StorageRoot ?? Keccak.EmptyTreeHash; @@ -558,23 +576,32 @@ public void Clear() BlockChange.ClearAndSetMissingAsDefault(); } + public void Return() + { + _address = null; + _provider = null; + StorageTree = null; + _wasWritten = false; + Pool.Return(this); + } + public void SaveChange(StorageCell storageCell, byte[] value) { _wasWritten = true; - ref ChangeTrace valueChanges = ref BlockChange.GetValueRefOrAddDefault(storageCell.Index, out bool exists); + ref StorageChangeTrace valueChanges = ref BlockChange.GetValueRefOrAddDefault(storageCell.Index, out bool exists); if (!exists) { - valueChanges = new ChangeTrace(value); + valueChanges = new StorageChangeTrace(value); } else { - valueChanges.After = value; + valueChanges = new StorageChangeTrace(valueChanges.Before, value); } } public ReadOnlySpan LoadFromTree(in StorageCell storageCell) { - ref ChangeTrace valueChange = ref BlockChange.GetValueRefOrAddDefault(storageCell.Index, out bool exists); + ref StorageChangeTrace valueChange = ref BlockChange.GetValueRefOrAddDefault(storageCell.Index, out bool exists); if (!exists) { byte[] value = !_provider._populatePreBlockCache ? @@ -610,7 +637,7 @@ private byte[] LoadFromTreePopulatePrewarmCache(in StorageCell storageCell) long priorReads = Db.Metrics.ThreadLocalStorageTreeReads; byte[] value = _provider._preBlockCache is not null - ? _provider._preBlockCache.GetOrAdd(storageCell, _loadFromTreeStorageFunc) + ? _provider._preBlockCache.GetOrAdd(storageCell, _loadFromTreeStorageFunc, this) : LoadFromTreeStorage(storageCell); if (Db.Metrics.ThreadLocalStorageTreeReads == priorReads) @@ -621,7 +648,7 @@ private byte[] LoadFromTreePopulatePrewarmCache(in StorageCell storageCell) return value; } - private byte[] LoadFromTreeStorage(StorageCell storageCell) + private byte[] LoadFromTreeStorage(in StorageCell storageCell) { Db.Metrics.IncrementStorageTreeReads(); @@ -631,6 +658,9 @@ private byte[] LoadFromTreeStorage(StorageCell storageCell) : StorageTree.GetArray(storageCell.Hash.Bytes); } + private static byte[] LoadFromTreeStorage(StorageCell storageCell, PerContractState @this) + => @this.LoadFromTreeStorage(storageCell); + public (int writes, int skipped) ProcessStorageChanges() { EnsureStorageTree(); @@ -659,7 +689,7 @@ private byte[] LoadFromTreeStorage(StorageCell storageCell) using ArrayPoolListRef bulkWrite = new(BlockChange.EstimatedSize); Span keyBuf = stackalloc byte[32]; - foreach (KeyValuePair kvp in BlockChange) + foreach (KeyValuePair kvp in BlockChange) { byte[] after = kvp.Value.After; if (!Bytes.AreEqual(kvp.Value.Before, after) || kvp.Value.IsInitialValue) @@ -692,5 +722,68 @@ public void RemoveStorageTree() { StorageTree = null; } + + internal static PerContractState Rent(Address address, PersistentStorageProvider persistentStorageProvider) + => Pool.Rent(address, persistentStorageProvider); + + private static class Pool + { + private static readonly ConcurrentQueue _pool = []; + private static int _poolCount; + + public static PerContractState Rent(Address address, PersistentStorageProvider provider) + { + if (Volatile.Read(ref _poolCount) > 0 && _pool.TryDequeue(out PerContractState item)) + { + Interlocked.Decrement(ref _poolCount); + item.Initialize(address, provider); + return item; + } + + return new PerContractState(address, provider); + } + + public static void Return(PerContractState item) + { + const int MaxItemSize = 512; + const int MaxPooledCount = 2048; + + if (item.BlockChange.Capacity > MaxItemSize) + return; + + // shared pool fallback + if (Interlocked.Increment(ref _poolCount) > MaxPooledCount) + { + Interlocked.Decrement(ref _poolCount); + return; + } + + item.BlockChange.Reset(); + _pool.Enqueue(item); + } + } + } + + private readonly struct StorageChangeTrace + { + public static readonly StorageChangeTrace _zeroBytes = new(StorageTree.ZeroBytes, StorageTree.ZeroBytes); + public static ref readonly StorageChangeTrace ZeroBytes => ref _zeroBytes; + + public StorageChangeTrace(byte[]? before, byte[]? after) + { + After = after ?? StorageTree.ZeroBytes; + Before = before ?? StorageTree.ZeroBytes; + } + + public StorageChangeTrace(byte[]? after) + { + After = after ?? StorageTree.ZeroBytes; + Before = StorageTree.ZeroBytes; + IsInitialValue = true; + } + + public readonly byte[] Before; + public readonly byte[] After; + public readonly bool IsInitialValue; } } diff --git a/src/Nethermind/Nethermind.State/StateProvider.cs b/src/Nethermind/Nethermind.State/StateProvider.cs index 5abb93111b2..7587ebd28e7 100644 --- a/src/Nethermind/Nethermind.State/StateProvider.cs +++ b/src/Nethermind/Nethermind.State/StateProvider.cs @@ -33,7 +33,7 @@ internal class StateProvider { private static readonly UInt256 _zero = UInt256.Zero; - private readonly Dictionary> _intraTxCache = new(); + private readonly Dictionary> _intraTxCache = new(); private readonly HashSet _committedThisRound = new(); private readonly HashSet _nullAccountReads = new(); // Only guarding against hot duplicates so filter doesn't need to be too big @@ -111,7 +111,7 @@ public bool IsContract(Address address) } public bool AccountExists(Address address) => - _intraTxCache.TryGetValue(address, out Stack value) + _intraTxCache.TryGetValue(address, out StackList value) ? _changes[value.Peek()]!.ChangeType != ChangeType.Delete : GetAndAddToCache(address) is not null; @@ -426,7 +426,7 @@ public void Restore(int snapshot) { int nextPosition = lastIndex - i; ref readonly Change change = ref changes[nextPosition]; - Stack stack = _intraTxCache[change!.Address]; + StackList stack = _intraTxCache[change!.Address]; int actualPosition = stack.Pop(); if (actualPosition != nextPosition) ThrowUnexpectedPosition(lastIndex, i, actualPosition); @@ -441,7 +441,10 @@ public void Restore(int snapshot) else { // Remove address entry entirely if no more changes - _intraTxCache.Remove(change.Address); + if (_intraTxCache.Remove(change.Address, out StackList? removed)) + { + removed.Return(); + } } } } @@ -487,7 +490,7 @@ void Trace(Address address, in UInt256 balance, in UInt256 nonce) public void CreateEmptyAccountIfDeletedOrNew(Address address) { - if (_intraTxCache.TryGetValue(address, out Stack value)) + if (_intraTxCache.TryGetValue(address, out StackList value)) { //we only want to persist empty accounts if they were deleted or created as empty //we don't want to do it for account empty due to a change (e.g. changed balance to zero) @@ -587,7 +590,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool continue; } - Stack stack = _intraTxCache[change.Address]; + StackList stack = _intraTxCache[change.Address]; int forAssertion = stack.Pop(); if (forAssertion != stepsBack - i) { @@ -670,7 +673,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool _changes.Clear(); _committedThisRound.Clear(); _nullAccountReads.Clear(); - _intraTxCache.Clear(); + _intraTxCache.ResetAndClear(); if (commitRoots) { @@ -697,9 +700,9 @@ Task CommitCodeAsync() } // Mark all inserted codes as persisted - foreach (var kvp in dict) + foreach (Hash256AsKey kvp in dict.Keys) { - _persistedCodeInsertFilter.Set(kvp.Key.Value.ValueHash256); + _persistedCodeInsertFilter.Set(kvp.Value.ValueHash256); } // Reuse Dictionary if not already re-initialized @@ -759,7 +762,7 @@ private void FlushToTree() KeccakCache.ComputeTo(key.Value.Bytes, out ValueHash256 keccak); - var account = change.After; + Account account = change.After; Rlp accountRlp = account is null ? null : account.IsTotallyEmpty ? StateTree.EmptyAccountRlp : Rlp.Encode(account); bulkWrite.Add(new PatriciaTree.BulkSetEntry(keccak, accountRlp?.Bytes)); @@ -855,7 +858,7 @@ private void SetState(Address address, Account? account) private Account? GetThroughCache(Address address) { - if (_intraTxCache.TryGetValue(address, out Stack value)) + if (_intraTxCache.TryGetValue(address, out StackList value)) { return _changes[value.Peek()].Account; } @@ -881,7 +884,7 @@ private void PushDelete(Address address) private void Push(Address address, Account? touchedAccount, ChangeType changeType) { - Stack stack = SetupCache(address); + StackList stack = SetupCache(address); if (changeType == ChangeType.Touch && _changes[stack.Peek()]!.ChangeType == ChangeType.Touch) { @@ -894,23 +897,23 @@ private void Push(Address address, Account? touchedAccount, ChangeType changeTyp private void PushNew(Address address, Account account) { - Stack stack = SetupCache(address); + StackList stack = SetupCache(address); stack.Push(_changes.Count); _changes.Add(new Change(address, account, ChangeType.New)); } - private void PushRecreateEmpty(Address address, Account account, Stack stack) + private void PushRecreateEmpty(Address address, Account account, StackList stack) { stack.Push(_changes.Count); _changes.Add(new Change(address, account, ChangeType.RecreateEmpty)); } - private Stack SetupCache(Address address) + private StackList SetupCache(Address address) { - ref Stack? value = ref CollectionsMarshal.GetValueRefOrAddDefault(_intraTxCache, address, out bool exists); + ref StackList? value = ref CollectionsMarshal.GetValueRefOrAddDefault(_intraTxCache, address, out bool exists); if (!exists) { - value = new Stack(); + value = StackList.Rent(); } return value; @@ -943,7 +946,7 @@ public void Reset(bool resetBlockChanges = true) _blockChanges.Clear(); _codeBatch?.Clear(); } - _intraTxCache.Clear(); + _intraTxCache.ResetAndClear(); _committedThisRound.Clear(); _nullAccountReads.Clear(); _changes.Clear(); diff --git a/src/Nethermind/Nethermind.State/WorldState.cs b/src/Nethermind/Nethermind.State/WorldState.cs index c9767295a04..91d32300615 100644 --- a/src/Nethermind/Nethermind.State/WorldState.cs +++ b/src/Nethermind/Nethermind.State/WorldState.cs @@ -86,12 +86,11 @@ private void GuardInScope() if (!_isInScope) ThrowOutOfScope(); } + [Conditional("DEBUG")] [MethodImpl(MethodImplOptions.AggressiveInlining)] private void DebugGuardInScope() { -#if DEBUG if (!_isInScope) ThrowOutOfScope(); -#endif } [StackTraceHidden, DoesNotReturn]