diff --git a/cspell.json b/cspell.json index e2fcac415fcd..554a8612d55c 100644 --- a/cspell.json +++ b/cspell.json @@ -790,6 +790,7 @@ "unreferred", "unrequested", "unresolve", + "unshifted", "unsub", "unsubscription", "unsynchronized", diff --git a/src/Nethermind/Nethermind.Benchmark/Core/SeqlockCacheBenchmarks.cs b/src/Nethermind/Nethermind.Benchmark/Core/SeqlockCacheBenchmarks.cs new file mode 100644 index 000000000000..9ba71e47e39e --- /dev/null +++ b/src/Nethermind/Nethermind.Benchmark/Core/SeqlockCacheBenchmarks.cs @@ -0,0 +1,345 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +#nullable enable + +using System; +using System.Collections.Concurrent; +using BenchmarkDotNet.Attributes; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Int256; + +namespace Nethermind.Benchmarks.Core; + +[MemoryDiagnoser] +[DisassemblyDiagnoser(maxDepth: 3)] +public class SeqlockCacheBenchmarks +{ + private SeqlockCache _seqlockCache = null!; + private ConcurrentDictionary _concurrentDict = null!; + + private StorageCell[] _keys = null!; + private byte[][] _values = null!; + private StorageCell _missKey; + + [Params(1000)] + public int KeyCount { get; set; } + + [GlobalSetup] + public void Setup() + { + _seqlockCache = new SeqlockCache(); + _concurrentDict = new ConcurrentDictionary(); + + _keys = new StorageCell[KeyCount]; + _values = new byte[KeyCount][]; + + var random = new Random(42); + for (int i = 0; i < KeyCount; i++) + { + var addressBytes = new byte[20]; + random.NextBytes(addressBytes); + var address = new Address(addressBytes); + var index = new UInt256((ulong)i); + + _keys[i] = new StorageCell(address, index); + _values[i] = new byte[32]; + random.NextBytes(_values[i]); + + // Pre-populate both caches + _seqlockCache.Set(in _keys[i], _values[i]); + _concurrentDict[_keys[i]] = _values[i]; + } + + // Create a key that won't be in the cache + var missAddressBytes = new byte[20]; + random.NextBytes(missAddressBytes); + _missKey = new StorageCell(new Address(missAddressBytes), UInt256.MaxValue); + } + + // ==================== TryGetValue (Hit) ==================== + + [Benchmark(Baseline = true)] + public bool SeqlockCache_TryGetValue_Hit() + { + return _seqlockCache.TryGetValue(in _keys[500], out _); + } + + [Benchmark] + public bool ConcurrentDict_TryGetValue_Hit() + { + return _concurrentDict.TryGetValue(_keys[500], out _); + } + + // ==================== TryGetValue (Miss) ==================== + + [Benchmark] + public bool SeqlockCache_TryGetValue_Miss() + { + return _seqlockCache.TryGetValue(in _missKey, out _); + } + + [Benchmark] + public bool ConcurrentDict_TryGetValue_Miss() + { + return _concurrentDict.TryGetValue(_missKey, out _); + } + + // ==================== Set (Existing Key) ==================== + + [Benchmark] + public void SeqlockCache_Set_Existing() + { + _seqlockCache.Set(in _keys[500], _values[500]); + } + + [Benchmark] + public void ConcurrentDict_Set_Existing() + { + _concurrentDict[_keys[500]] = _values[500]; + } + + // ==================== GetOrAdd (Hit) ==================== + + [Benchmark] + public byte[]? SeqlockCache_GetOrAdd_Hit() + { + return _seqlockCache.GetOrAdd(in _keys[500], static (in StorageCell _) => new byte[32]); + } + + [Benchmark] + public byte[] ConcurrentDict_GetOrAdd_Hit() + { + return _concurrentDict.GetOrAdd(_keys[500], static _ => new byte[32]); + } + + // ==================== GetOrAdd (Miss - measures factory overhead) ==================== + + private int _missCounter; + + [Benchmark] + public byte[]? SeqlockCache_GetOrAdd_Miss() + { + // Use incrementing key to always miss + var key = new StorageCell(_keys[0].Address, new UInt256((ulong)(KeyCount + _missCounter++))); + return _seqlockCache.GetOrAdd(in key, static (in StorageCell _) => new byte[32]); + } + + [Benchmark] + public byte[] ConcurrentDict_GetOrAdd_Miss() + { + var key = new StorageCell(_keys[0].Address, new UInt256((ulong)(KeyCount + _missCounter++))); + return _concurrentDict.GetOrAdd(key, static _ => new byte[32]); + } +} + +/// +/// Benchmark comparing read-heavy workloads (90% reads, 10% writes) +/// +[MemoryDiagnoser] +public class SeqlockCacheMixedWorkloadBenchmarks +{ + private SeqlockCache _seqlockCache = null!; + private ConcurrentDictionary _concurrentDict = null!; + + private StorageCell[] _keys = null!; + private byte[][] _values = null!; + + private const int KeyCount = 10000; + private const int OperationsPerInvoke = 1000; + + [GlobalSetup] + public void Setup() + { + _seqlockCache = new SeqlockCache(); + _concurrentDict = new ConcurrentDictionary(); + + _keys = new StorageCell[KeyCount]; + _values = new byte[KeyCount][]; + + var random = new Random(42); + for (int i = 0; i < KeyCount; i++) + { + var addressBytes = new byte[20]; + random.NextBytes(addressBytes); + var address = new Address(addressBytes); + var index = new UInt256((ulong)i); + + _keys[i] = new StorageCell(address, index); + _values[i] = new byte[32]; + random.NextBytes(_values[i]); + + // Pre-populate both caches + _seqlockCache.Set(in _keys[i], _values[i]); + _concurrentDict[_keys[i]] = _values[i]; + } + } + + [Benchmark(Baseline = true, OperationsPerInvoke = OperationsPerInvoke)] + public int SeqlockCache_MixedWorkload_90Read_10Write() + { + int hits = 0; + for (int i = 0; i < OperationsPerInvoke; i++) + { + int keyIndex = i % KeyCount; + if (i % 10 == 0) + { + // 10% writes + _seqlockCache.Set(in _keys[keyIndex], _values[keyIndex]); + } + else + { + // 90% reads + if (_seqlockCache.TryGetValue(in _keys[keyIndex], out _)) + hits++; + } + } + return hits; + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public int ConcurrentDict_MixedWorkload_90Read_10Write() + { + int hits = 0; + for (int i = 0; i < OperationsPerInvoke; i++) + { + int keyIndex = i % KeyCount; + if (i % 10 == 0) + { + // 10% writes + _concurrentDict[_keys[keyIndex]] = _values[keyIndex]; + } + else + { + // 90% reads + if (_concurrentDict.TryGetValue(_keys[keyIndex], out _)) + hits++; + } + } + return hits; + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public int SeqlockCache_ReadOnly() + { + int hits = 0; + for (int i = 0; i < OperationsPerInvoke; i++) + { + int keyIndex = i % KeyCount; + if (_seqlockCache.TryGetValue(in _keys[keyIndex], out _)) + hits++; + } + return hits; + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public int ConcurrentDict_ReadOnly() + { + int hits = 0; + for (int i = 0; i < OperationsPerInvoke; i++) + { + int keyIndex = i % KeyCount; + if (_concurrentDict.TryGetValue(_keys[keyIndex], out _)) + hits++; + } + return hits; + } +} + +/// +/// Benchmark measuring effective hit rate after populating with N keys. +/// This directly measures the impact of collision rate. +/// +public class SeqlockCacheHitRateBenchmarks +{ + private SeqlockCache _seqlockCache = null!; + private StorageCell[] _keys = null!; + private byte[][] _values = null!; + + [Params(1000, 5000, 10000, 20000)] + public int KeyCount { get; set; } + + [GlobalSetup] + public void Setup() + { + _seqlockCache = new SeqlockCache(); + _keys = new StorageCell[KeyCount]; + _values = new byte[KeyCount][]; + + var random = new Random(42); + for (int i = 0; i < KeyCount; i++) + { + var addressBytes = new byte[20]; + random.NextBytes(addressBytes); + _keys[i] = new StorageCell(new Address(addressBytes), new UInt256((ulong)i)); + _values[i] = new byte[32]; + random.NextBytes(_values[i]); + _seqlockCache.Set(in _keys[i], _values[i]); + } + } + + [Benchmark] + public double MeasureHitRate() + { + int hits = 0; + for (int i = 0; i < KeyCount; i++) + { + if (_seqlockCache.TryGetValue(in _keys[i], out byte[]? val) && ReferenceEquals(val, _values[i])) + hits++; + } + return (double)hits / KeyCount * 100; + } +} + +[MemoryDiagnoser] +public class SeqlockCacheCallSiteBenchmarks +{ + private SeqlockCache _cache = null!; + private SeqlockCache.ValueFactory _cachedFactory = null!; + private StorageCell _key; + private byte[] _value = null!; + + [GlobalSetup] + public void Setup() + { + _cache = new SeqlockCache(); + + byte[] addressBytes = new byte[20]; + new Random(123).NextBytes(addressBytes); + _key = new StorageCell(new Address(addressBytes), UInt256.One); + _value = new byte[32]; + + _cache.Set(in _key, _value); + _cachedFactory = LoadFromBackingStore; + } + + [Benchmark(Baseline = true)] + public byte[]? GetOrAdd_Hit_PerCallMethodGroup() + { + return _cache.GetOrAdd(in _key, LoadFromBackingStore); + } + + [Benchmark] + public byte[]? GetOrAdd_Hit_CachedDelegate() + { + return _cache.GetOrAdd(in _key, _cachedFactory); + } + + [Benchmark] + public bool TryGetValue_WithIn() + { + return _cache.TryGetValue(in _key, out _); + } + + [Benchmark] + public bool TryGetValue_WithoutIn() + { + return _cache.TryGetValue(_key, out _); + } + + private byte[] LoadFromBackingStore(in StorageCell _) + { + return _value; + } +} diff --git a/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs b/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs index ea638750cea7..200341f05cdb 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs @@ -25,6 +25,7 @@ public class BlockhashProvider( private readonly IBlockhashStore _blockhashStore = new BlockhashStore(worldState); private readonly ILogger _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); private Hash256[]? _hashes; + private long _prefetchVersion; public Hash256? GetBlockhash(BlockHeader currentBlock, long number, IReleaseSpec spec) { @@ -39,7 +40,7 @@ public class BlockhashProvider( } long depth = currentBlock.Number - number; - Hash256[]? hashes = _hashes; + Hash256[]? hashes = Volatile.Read(ref _hashes); return depth switch { @@ -60,7 +61,8 @@ public class BlockhashProvider( public async Task Prefetch(BlockHeader currentBlock, CancellationToken token) { - _hashes = null; + long prefetchVersion = Interlocked.Increment(ref _prefetchVersion); + Volatile.Write(ref _hashes, null); Hash256[]? hashes = await blockhashCache.Prefetch(currentBlock, token); // This leverages that branch processing is single threaded @@ -69,9 +71,9 @@ public async Task Prefetch(BlockHeader currentBlock, CancellationToken token) // This allows us to avoid await on Prefetch in BranchProcessor lock (_blockhashStore) { - if (!token.IsCancellationRequested) + if (!token.IsCancellationRequested && prefetchVersion == Interlocked.Read(ref _prefetchVersion)) { - _hashes = hashes; + Volatile.Write(ref _hashes, hashes); } } } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs index 80340ae16a8b..fb7ab548d42f 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs @@ -57,7 +57,7 @@ public Task PreWarmCaches(Block suggestedBlock, BlockHeader? parent, IReleaseSpe if (preBlockCaches is not null) { CacheType result = preBlockCaches.ClearCaches(); - result |= nodeStorageCache.ClearCaches() ? CacheType.Rlp : CacheType.None; + nodeStorageCache.ClearCaches(); nodeStorageCache.Enabled = true; if (result != default) { diff --git a/src/Nethermind/Nethermind.Core.Test/Collections/SeqlockCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/Collections/SeqlockCacheTests.cs new file mode 100644 index 000000000000..ee076970d44b --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Collections/SeqlockCacheTests.cs @@ -0,0 +1,428 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Nethermind.Core.Collections; +using Nethermind.Int256; +using NUnit.Framework; + +namespace Nethermind.Core.Test.Collections; + +public class SeqlockCacheTests +{ + private static StorageCell CreateKey(int seed) + { + byte[] addressBytes = new byte[20]; + new Random(seed).NextBytes(addressBytes); + return new StorageCell(new Address(addressBytes), new UInt256((ulong)seed)); + } + + private static byte[] CreateValue(int seed) + { + byte[] value = new byte[32]; + new Random(seed).NextBytes(value); + return value; + } + + [Test] + public void New_cache_returns_miss() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + + bool found = cache.TryGetValue(in key, out byte[]? value); + + found.Should().BeFalse(); + value.Should().BeNull(); + } + + [Test] + public void Set_then_get_returns_value() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] expected = CreateValue(1); + + cache.Set(in key, expected); + bool found = cache.TryGetValue(in key, out byte[]? value); + + found.Should().BeTrue(); + value.Should().BeSameAs(expected); + } + + [Test] + public void Set_overwrites_existing_value() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] first = CreateValue(1); + byte[] second = CreateValue(2); + + cache.Set(in key, first); + cache.Set(in key, second); + bool found = cache.TryGetValue(in key, out byte[]? value); + + found.Should().BeTrue(); + value.Should().BeSameAs(second); + } + + [Test] + public void Set_with_same_value_is_noop() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] expected = CreateValue(1); + + cache.Set(in key, expected); + cache.Set(in key, expected); // Same reference - should be fast-path no-op + bool found = cache.TryGetValue(in key, out byte[]? value); + + found.Should().BeTrue(); + value.Should().BeSameAs(expected); + } + + [Test] + public void Null_value_can_be_stored_and_retrieved() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + + cache.Set(in key, null); + bool found = cache.TryGetValue(in key, out byte[]? value); + + found.Should().BeTrue(); + value.Should().BeNull(); + } + + [Test] + public void GetOrAdd_returns_existing_value() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] expected = CreateValue(1); + + cache.Set(in key, expected); + byte[]? result = cache.GetOrAdd(in key, static (in StorageCell _) => new byte[32]); + + result.Should().BeSameAs(expected); + } + + [Test] + public void GetOrAdd_calls_factory_on_miss() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] factoryResult = CreateValue(1); + + byte[]? result = cache.GetOrAdd(in key, (in StorageCell _) => factoryResult); + + result.Should().BeSameAs(factoryResult); + + // Value should now be cached + bool found = cache.TryGetValue(in key, out byte[]? cached); + found.Should().BeTrue(); + cached.Should().BeSameAs(factoryResult); + } + + [Test] + public void GetOrAdd_with_func_returns_existing_value() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] expected = CreateValue(1); + + cache.Set(in key, expected); + byte[]? result = cache.GetOrAdd(in key, static (in _) => new byte[32]); + + result.Should().BeSameAs(expected); + } + + [Test] + public void GetOrAdd_with_func_calls_factory_on_miss() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] factoryResult = CreateValue(1); + + byte[]? result = cache.GetOrAdd(in key, (in _) => factoryResult); + + result.Should().BeSameAs(factoryResult); + } + + [Test] + public void Clear_invalidates_all_entries() + { + SeqlockCache cache = new(); + StorageCell key1 = CreateKey(1); + StorageCell key2 = CreateKey(2); + + cache.Set(in key1, CreateValue(1)); + cache.Set(in key2, CreateValue(2)); + + cache.Clear(); + + cache.TryGetValue(in key1, out _).Should().BeFalse(); + cache.TryGetValue(in key2, out _).Should().BeFalse(); + } + + [Test] + public void Clear_allows_new_entries() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] beforeClear = CreateValue(1); + byte[] afterClear = CreateValue(2); + + cache.Set(in key, beforeClear); + cache.Clear(); + cache.Set(in key, afterClear); + + bool found = cache.TryGetValue(in key, out byte[]? value); + found.Should().BeTrue(); + value.Should().BeSameAs(afterClear); + } + + [Test] + public void Multiple_clears_work() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + + for (int i = 0; i < 100; i++) + { + byte[] value = CreateValue(i); + cache.Set(in key, value); + cache.TryGetValue(in key, out byte[]? retrieved).Should().BeTrue(); + retrieved.Should().BeSameAs(value); + cache.Clear(); + cache.TryGetValue(in key, out _).Should().BeFalse(); + } + } + + [Test] + public void Different_keys_can_be_stored() + { + SeqlockCache cache = new(); + const int count = 100; + + StorageCell[] keys = new StorageCell[count]; + byte[][] values = new byte[count][]; + + for (int i = 0; i < count; i++) + { + keys[i] = CreateKey(i); + values[i] = CreateValue(i); + cache.Set(in keys[i], values[i]); + } + + // Note: This is a direct-mapped cache, so some entries may be evicted + // due to hash collisions. We just verify that at least some survive. + int hits = 0; + for (int i = 0; i < count; i++) + { + if (cache.TryGetValue(in keys[i], out byte[]? value) && ReferenceEquals(value, values[i])) + { + hits++; + } + } + + hits.Should().BeGreaterThan(0, "at least some entries should survive"); + } + + [Test] + public void Concurrent_reads_are_safe() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] expected = CreateValue(1); + cache.Set(in key, expected); + + const int threadCount = 8; + const int iterations = 10000; + int successCount = 0; + + Parallel.For(0, threadCount, _ => + { + for (int i = 0; i < iterations; i++) + { + if (cache.TryGetValue(in key, out byte[]? value) && ReferenceEquals(value, expected)) + { + Interlocked.Increment(ref successCount); + } + } + }); + + successCount.Should().Be(threadCount * iterations); + } + + [Test] + public void Concurrent_writes_do_not_corrupt() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + + const int threadCount = 8; + const int iterations = 1000; + byte[][] values = new byte[threadCount][]; + for (int i = 0; i < threadCount; i++) + { + values[i] = CreateValue(i); + } + + Parallel.For(0, threadCount, t => + { + for (int i = 0; i < iterations; i++) + { + cache.Set(in key, values[t]); + } + }); + + // After concurrent writes, the cache should contain one of the values + bool found = cache.TryGetValue(in key, out byte[]? result); + if (found) + { + // Value should be one of the values we wrote + bool isValid = false; + for (int i = 0; i < threadCount; i++) + { + if (ReferenceEquals(result, values[i])) + { + isValid = true; + break; + } + } + isValid.Should().BeTrue("cached value should be one of the written values"); + } + } + + [Test] + public void Concurrent_read_write_is_safe() + { + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] value1 = CreateValue(1); + byte[] value2 = CreateValue(2); + + const int iterations = 10000; + bool stop = false; + + // Writer thread + Task writer = Task.Run(() => + { + for (int i = 0; i < iterations && !stop; i++) + { + cache.Set(in key, i % 2 == 0 ? value1 : value2); + } + }); + + // Reader thread + int validReads = 0; + int misses = 0; + Task reader = Task.Run(() => + { + for (int i = 0; i < iterations; i++) + { + if (cache.TryGetValue(in key, out byte[]? value)) + { + // Value should be either value1 or value2 + if (ReferenceEquals(value, value1) || ReferenceEquals(value, value2)) + { + Interlocked.Increment(ref validReads); + } + } + else + { + Interlocked.Increment(ref misses); + } + } + }); + + Task.WaitAll(writer, reader); + stop = true; + + // All reads should have returned valid values (or miss due to concurrent write) + (validReads + misses).Should().Be(iterations); + } + + [Test] + public void AddressAsKey_works_with_cache() + { + SeqlockCache cache = new(); + Address address = new Address("0x1234567890123456789012345678901234567890"); + AddressAsKey key = address; + Account account = new Account(100, 1); + + cache.Set(in key, account); + bool found = cache.TryGetValue(in key, out Account? result); + + found.Should().BeTrue(); + result.Should().BeSameAs(account); + } + + [Test] + public void Concurrent_set_same_value_fast_path_is_safe() + { + // Tests the fast-path optimization where Set skips write if value matches. + // This exercises the seqlock protocol in the fast-path to avoid torn reads. + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[] value = CreateValue(1); + + cache.Set(in key, value); + + const int threadCount = 8; + const int iterations = 10000; + + // Multiple threads all trying to set the same key to the same value + // This hammers the fast-path check + Parallel.For(0, threadCount, _ => + { + for (int i = 0; i < iterations; i++) + { + cache.Set(in key, value); + } + }); + + // Value should still be retrievable and correct + bool found = cache.TryGetValue(in key, out byte[]? result); + found.Should().BeTrue(); + result.Should().BeSameAs(value); + } + + [Test] + public void Concurrent_set_alternating_values_is_safe() + { + // Tests concurrent writes with different values interleaved with same-value writes. + // Exercises both the fast-path (same value) and slow-path (different value). + SeqlockCache cache = new(); + StorageCell key = CreateKey(1); + byte[][] values = new byte[4][]; + for (int i = 0; i < values.Length; i++) + { + values[i] = CreateValue(i); + } + + const int threadCount = 8; + const int iterations = 5000; + + Parallel.For(0, threadCount, t => + { + for (int i = 0; i < iterations; i++) + { + // Each thread cycles through values, creating both fast-path and slow-path scenarios + cache.Set(in key, values[(t + i) % values.Length]); + } + }); + + // After all writes, cache should contain one of the valid values + bool found = cache.TryGetValue(in key, out byte[]? result); + if (found) + { + bool isValid = Array.Exists(values, v => ReferenceEquals(v, result)); + isValid.Should().BeTrue("cached value should be one of the written values"); + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index 0bad6dd83609..80516f59c95c 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -9,7 +9,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Text.Json.Serialization; - +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Int256; @@ -20,7 +20,7 @@ namespace Nethermind.Core [JsonConverter(typeof(AddressConverter))] [TypeConverter(typeof(AddressTypeConverter))] [DebuggerDisplay("{ToString()}")] - public class Address : IEquatable
, IComparable
+ public sealed class Address : IEquatable
, IComparable
{ public const int Size = 20; private const int HexCharsCount = 2 * Size; // 5a4eab120fb44eb6684e5e32785702ff45ea344d @@ -273,9 +273,11 @@ public ValueHash256 ToHash() return result; } + + internal long GetHashCode64() => SpanExtensions.FastHash64For20Bytes(ref MemoryMarshal.GetArrayDataReference(Bytes)); } - public readonly struct AddressAsKey(Address key) : IEquatable + public readonly struct AddressAsKey(Address key) : IEquatable, IHash64bit { private readonly Address _key = key; public Address Value => _key; @@ -289,6 +291,10 @@ public override string ToString() { return _key?.ToString() ?? ""; } + + public long GetHashCode64() => _key is not null ? _key.GetHashCode64() : 0; + + public bool Equals(in AddressAsKey other) => _key == other._key; } public ref struct AddressStructRef diff --git a/src/Nethermind/Nethermind.Core/Collections/IHash64bit.cs b/src/Nethermind/Nethermind.Core/Collections/IHash64bit.cs new file mode 100644 index 000000000000..746146576cf4 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Collections/IHash64bit.cs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Core.Collections; + +/// +/// Provides a 64-bit hash code for high-performance caching with reduced collision probability. +/// +/// +/// Types implementing this interface can be used with caches that require extended hash bits +/// for collision resistance (for example, Seqlock-based caches that use additional bits beyond +/// the standard hash code for bucket indexing and collision detection). +/// The 64-bit hash should have good distribution across all bits. +/// +public interface IHash64bit +{ + /// + /// Returns a 64-bit hash code for the current instance. + /// + /// A 64-bit hash code with good distribution across all bits. + long GetHashCode64(); + bool Equals(in TKey other); +} diff --git a/src/Nethermind/Nethermind.Core/Collections/SeqlockCache.cs b/src/Nethermind/Nethermind.Core/Collections/SeqlockCache.cs new file mode 100644 index 000000000000..6ce764fa46b6 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Collections/SeqlockCache.cs @@ -0,0 +1,400 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; +using System.Threading; + +namespace Nethermind.Core.Collections; + +/// +/// A high-performance 2-way skew-associative cache using a seqlock-style header per entry. +/// +/// Design goals: +/// - Lock-free reads (seqlock pattern) - readers never take locks. +/// - Best-effort writes - writers skip on contention. +/// - O(1) logical Clear() via a global epoch (no per-entry zeroing). +/// - 2-way skew-associative: each way uses independent hash bits for set indexing, +/// breaking correlation between ways ("power of two choices"). Keys that collide +/// in way 0 scatter to different sets in way 1, virtually eliminating conflict misses. +/// +/// Hash bit partitioning (64-bit hash): +/// Bits 0-13: way 0 set index (14 bits) +/// Bits 14-41: hash signature stored in header (28 bits) +/// Bits 42-55: way 1 set index (14 bits, independent from way 0) +/// +/// Header layout (64-bit): +/// [Lock:1][Epoch:26][Hash:28][Seq:8][Occ:1] +/// - Lock (bit 63): set during writes - readers retry/miss +/// - Epoch (bits 37-62): global epoch tag - changes on Clear() +/// - Hash (bits 9-36): per-bucket hash signature (28 bits) +/// - Seq (bits 1- 8): per-entry sequence counter (8 bits) - increments on every successful write +/// - Occ (bit 0): occupied flag - set when slot contains valid data (value may still be null) +/// +/// Array layout: [way0_set0..way0_set16383, way1_set0..way1_set16383] (split, not interleaved). +/// +/// The key type (struct implementing IHash64bit) +/// The value type (reference type, nullable allowed) +public sealed class SeqlockCache + where TKey : struct, IHash64bit + where TValue : class? +{ + /// + /// Number of sets. Must be a power of 2 for mask operations. + /// 16384 sets × 2 ways = 32768 total entries. + /// + private const int Sets = 1 << 14; // 16384 + private const int SetMask = Sets - 1; + + // Header bit layout: + // [Lock:1][Epoch:26][Hash:28][Seq:8][Occ:1] + + private const long LockMarker = unchecked((long)0x8000_0000_0000_0000); // bit 63 + + private const int EpochShift = 37; + private const long EpochMask = 0x7FFF_FFE0_0000_0000; // bits 37-62 (26 bits) + + private const long HashMask = 0x0000_0001_FFFF_FE00; // bits 9-36 (28 bits) + + private const long SeqMask = 0x0000_0000_0000_01FE; // bits 1-8 (8 bits) + private const long SeqInc = 0x0000_0000_0000_0002; // +1 in seq field + + private const long OccupiedBit = 1L; // bit 0 + + // Mask of all "identity" bits for an entry, excluding Lock and Seq. + private const long TagMask = EpochMask | HashMask | OccupiedBit; + + // Mask for checking if an entry is live in the current epoch. + private const long EpochOccMask = EpochMask | OccupiedBit; + + // With 14-bit set index (bits 0-13) for way 0, hash signature needs bits 14+. + // HashShift=5 maps header bits 9-36 to original bits 14-41, avoiding overlap with both ways. + private const int HashShift = 5; + + // Way 1 uses bits 42-55 of the original hash (completely independent from way 0's bits 0-13). + private const int Way1Shift = 42; + + /// + /// Array of entries: [way0_set0..way0_setN, way1_set0..way1_setN]. + /// Split layout ensures each way is a contiguous block for better prefetch behavior. + /// + private readonly Entry[] _entries; + + /// + /// Current epoch counter (unshifted, informational / debugging). + /// + private long _epoch; + + /// + /// Pre-shifted epoch tag: (_epoch << EpochShift) & EpochMask. + /// Readers use this directly to avoid shift/mask in the hot path. + /// + private long _shiftedEpoch; + + public SeqlockCache() + { + _entries = new Entry[Sets << 1]; // Sets * 2 + _epoch = 0; + _shiftedEpoch = 0; + } + + /// + /// Tries to get a value from the cache using a seqlock pattern (lock-free reads). + /// Checks both ways of the target set for the key. + /// + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe bool TryGetValue(in TKey key, out TValue? value) + { + long hashCode = key.GetHashCode64(); + int idx0 = (int)hashCode & SetMask; + int idx1 = Sets + ((int)(hashCode >> Way1Shift) & SetMask); + + long epochTag = Volatile.Read(ref _shiftedEpoch); + long hashPart = (hashCode >> HashShift) & HashMask; + long expectedTag = epochTag | hashPart | OccupiedBit; + + ref Entry entries = ref MemoryMarshal.GetArrayDataReference(_entries); + + // Prefetch way 1 while we check way 0 — hides L2/L3 latency for skew layout. + if (Sse.IsSupported) + { + Sse.PrefetchNonTemporal(Unsafe.AsPointer(ref Unsafe.Add(ref entries, idx1))); + } + + // === Way 0 === + ref Entry e0 = ref Unsafe.Add(ref entries, idx0); + long h1 = Volatile.Read(ref e0.HashEpochSeqLock); + + if ((h1 & (TagMask | LockMarker)) == expectedTag) + { + ref readonly TKey storedKey = ref e0.Key; + TValue? storedValue = e0.Value; + + long h2 = Volatile.Read(ref e0.HashEpochSeqLock); + if (h1 == h2 && storedKey.Equals(in key)) + { + value = storedValue; + return true; + } + } + + // === Way 1 === + ref Entry e1 = ref Unsafe.Add(ref entries, idx1); + long w1 = Volatile.Read(ref e1.HashEpochSeqLock); + + if ((w1 & (TagMask | LockMarker)) == expectedTag) + { + ref readonly TKey storedKey = ref e1.Key; + TValue? storedValue = e1.Value; + + long w2 = Volatile.Read(ref e1.HashEpochSeqLock); + if (w1 == w2 && storedKey.Equals(in key)) + { + value = storedValue; + return true; + } + } + + value = default; + return false; + } + + /// + /// Delegate-based factory that avoids copying large keys (passes by in). + /// Prefer this over Func<TKey, TValue?> when TKey is big (eg 48 bytes). + /// + public delegate TValue? ValueFactory(in TKey key); + + /// + /// Gets a value from the cache, or adds it using the factory if not present. + /// + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public TValue? GetOrAdd(in TKey key, ValueFactory valueFactory) + { + long hashCode = key.GetHashCode64(); + int idx0 = (int)hashCode & SetMask; + int idx1 = Sets + ((int)(hashCode >> Way1Shift) & SetMask); + long hashPart = (hashCode >> HashShift) & HashMask; + + if (TryGetValueCore(in key, idx0, idx1, hashPart, out TValue? value)) + { + return value; + } + + return GetOrAddMiss(in key, valueFactory, idx0, idx1, hashPart); + } + + /// + /// Cold path for GetOrAdd: invokes factory and stores the result. + /// Kept out-of-line so the hot path (cache hit) compiles to a lean method body + /// with minimal register saves and stack frame. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private TValue? GetOrAddMiss(in TKey key, ValueFactory valueFactory, int idx0, int idx1, long hashPart) + { + TValue? value = valueFactory(in key); + SetCore(in key, value, idx0, idx1, hashPart); + return value; + } + + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe bool TryGetValueCore(in TKey key, int idx0, int idx1, long hashPart, out TValue? value) + { + long epochTag = Volatile.Read(ref _shiftedEpoch); + long expectedTag = epochTag | hashPart | OccupiedBit; + + ref Entry entries = ref MemoryMarshal.GetArrayDataReference(_entries); + + if (Sse.IsSupported) + { + Sse.PrefetchNonTemporal(Unsafe.AsPointer(ref Unsafe.Add(ref entries, idx1))); + } + + // Way 0 + ref Entry e0 = ref Unsafe.Add(ref entries, idx0); + long h1 = Volatile.Read(ref e0.HashEpochSeqLock); + + if ((h1 & (TagMask | LockMarker)) == expectedTag) + { + ref readonly TKey storedKey = ref e0.Key; + TValue? storedValue = e0.Value; + + long h2 = Volatile.Read(ref e0.HashEpochSeqLock); + if (h1 == h2 && storedKey.Equals(in key)) + { + value = storedValue; + return true; + } + } + + // Way 1 + ref Entry e1 = ref Unsafe.Add(ref entries, idx1); + long w1 = Volatile.Read(ref e1.HashEpochSeqLock); + + if ((w1 & (TagMask | LockMarker)) == expectedTag) + { + ref readonly TKey storedKey = ref e1.Key; + TValue? storedValue = e1.Value; + + long w2 = Volatile.Read(ref e1.HashEpochSeqLock); + if (w1 == w2 && storedKey.Equals(in key)) + { + value = storedValue; + return true; + } + } + + value = default; + return false; + } + + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetCore(in TKey key, TValue? value, int idx0, int idx1, long hashPart) + { + long epochTag = Volatile.Read(ref _shiftedEpoch); + long tagToStore = epochTag | hashPart | OccupiedBit; + long epochOccTag = epochTag | OccupiedBit; + + ref Entry entries = ref MemoryMarshal.GetArrayDataReference(_entries); + ref Entry e0 = ref Unsafe.Add(ref entries, idx0); + + long h0 = Volatile.Read(ref e0.HashEpochSeqLock); + + // === Way 0: check for matching key === + if (h0 >= 0 && (h0 & TagMask) == tagToStore) + { + ref readonly TKey k0 = ref e0.Key; + TValue? v0 = e0.Value; + + long h0_2 = Volatile.Read(ref e0.HashEpochSeqLock); + if (h0 == h0_2 && k0.Equals(in key)) + { + if (ReferenceEquals(v0, value)) return; // fast-path: same key+value, no-op + WriteEntry(ref e0, h0_2, in key, value, tagToStore); + return; + } + h0 = h0_2; + } + + // === Way 1: check for matching key === + ref Entry e1 = ref Unsafe.Add(ref entries, idx1); + long h1 = Volatile.Read(ref e1.HashEpochSeqLock); + + if (h1 >= 0 && (h1 & TagMask) == tagToStore) + { + ref readonly TKey k1 = ref e1.Key; + TValue? v1 = e1.Value; + + long h1_2 = Volatile.Read(ref e1.HashEpochSeqLock); + if (h1 == h1_2 && k1.Equals(in key)) + { + if (ReferenceEquals(v1, value)) return; // fast-path: same key+value, no-op + WriteEntry(ref e1, h1_2, in key, value, tagToStore); + return; + } + h1 = h1_2; + } + + // === Key not in either way. Evict into an available slot. === + // Priority: stale/empty unlocked > live (alternating by hash bit) > any unlocked > skip. + // The decision tree selects which way to evict into, then issues a single WriteEntry call. + bool h0Live = h0 >= 0 && (h0 & EpochOccMask) == epochOccTag; + bool h1Live = h1 >= 0 && (h1 & EpochOccMask) == epochOccTag; + + bool pick0; + if (!h0Live && h0 >= 0) pick0 = true; + else if (!h1Live && h1 >= 0) pick0 = false; + else if (h0Live && h1Live) pick0 = (hashPart & (1L << 9)) != 0; + else if (h0 >= 0) pick0 = true; + else if (h1 >= 0) pick0 = false; + else return; // both locked, skip + + WriteEntry( + ref pick0 ? ref e0 : ref e1, + pick0 ? h0 : h1, + in key, value, tagToStore); + } + + /// + /// Sets a key-value pair in the cache. + /// Checks both ways of the target set for an existing key match before evicting. + /// + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Set(in TKey key, TValue? value) + { + long hashCode = key.GetHashCode64(); + int idx0 = (int)hashCode & SetMask; + int idx1 = Sets + ((int)(hashCode >> Way1Shift) & SetMask); + long hashPart = (hashCode >> HashShift) & HashMask; + + SetCore(in key, value, idx0, idx1, hashPart); + } + + /// + /// Attempts a CAS-guarded write to a single entry. + /// Kept out-of-line: the CAS atomic dominates latency, so call overhead is invisible, + /// while de-duplication reclaims ~350 bytes of inlined copies across SetCore call sites. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private static void WriteEntry(ref Entry entry, long existing, in TKey key, TValue? value, long tagToStore) + { + if (existing < 0) return; // locked + + long newSeq = ((existing & SeqMask) + SeqInc) & SeqMask; + long lockedHeader = tagToStore | newSeq | LockMarker; + + if (Interlocked.CompareExchange(ref entry.HashEpochSeqLock, lockedHeader, existing) != existing) + { + return; + } + + entry.Key = key; + entry.Value = value; + + Volatile.Write(ref entry.HashEpochSeqLock, tagToStore | newSeq); + } + + /// + /// Clears all cached entries by incrementing the global epoch tag (O(1)). + /// Entries with stale epochs are treated as empty on subsequent lookups. + /// + public void Clear() + { + long oldShifted = Volatile.Read(ref _shiftedEpoch); + + while (true) + { + long oldEpoch = (oldShifted & EpochMask) >> EpochShift; + long newEpoch = oldEpoch + 1; + long newShifted = (newEpoch << EpochShift) & EpochMask; + + long prev = Interlocked.CompareExchange(ref _shiftedEpoch, newShifted, oldShifted); + if (prev == oldShifted) + { + Volatile.Write(ref _epoch, newEpoch); + return; + } + + oldShifted = prev; + } + } + + /// + /// Cache entry struct. + /// Header is a single 64-bit field to keep the seqlock control word in one atomic unit. + /// + [StructLayout(LayoutKind.Sequential)] + private struct Entry + { + public long HashEpochSeqLock; // [Lock|Epoch|Hash|Seq|Occ] + public TKey Key; + public TValue? Value; + } +} diff --git a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs index 5118ea293bb2..e4ef7139ddb8 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs @@ -7,6 +7,9 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using Arm = System.Runtime.Intrinsics.Arm; +using x64 = System.Runtime.Intrinsics.X86; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; @@ -397,5 +400,69 @@ static uint FinalMix(uint x) return x; } } + + /// + /// Computes a very fast, non-cryptographic 64-bit hash of exactly 32 bytes. + /// + /// Reference to the first byte of the 32-byte input. + /// A 64-bit hash value with good distribution across all bits. + /// + /// Uses AES hardware acceleration when available, falls back to CRC32C otherwise. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long FastHash64For32Bytes(ref byte start) + { + uint seed = s_instanceRandom + 32; + + if (x64.Aes.IsSupported || Arm.Aes.IsSupported) + { + Vector128 key = Unsafe.As>(ref start); + Vector128 data = Unsafe.As>(ref Unsafe.Add(ref start, 16)); + key ^= Vector128.CreateScalar(seed).AsByte(); + Vector128 mixed = x64.Aes.IsSupported + ? x64.Aes.Encrypt(data, key) + : Arm.Aes.MixColumns(Arm.Aes.Encrypt(data, key)); + return (long)(mixed.AsUInt64().GetElement(0) ^ mixed.AsUInt64().GetElement(1)); + } + + // Fallback: CRC32C-based 64-bit hash + ulong h0 = BitOperations.Crc32C(seed, Unsafe.ReadUnaligned(ref start)); + ulong h1 = BitOperations.Crc32C(seed ^ 0x9E3779B9u, Unsafe.ReadUnaligned(ref Unsafe.Add(ref start, 8))); + ulong h2 = BitOperations.Crc32C(seed ^ 0x85EBCA6Bu, Unsafe.ReadUnaligned(ref Unsafe.Add(ref start, 16))); + ulong h3 = BitOperations.Crc32C(seed ^ 0xC2B2AE35u, Unsafe.ReadUnaligned(ref Unsafe.Add(ref start, 24))); + return (long)((h0 | (h1 << 32)) ^ (h2 | (h3 << 32))); + } + + /// + /// Computes a very fast, non-cryptographic 64-bit hash of exactly 20 bytes (Address size). + /// + /// Reference to the first byte of the 20-byte input. + /// A 64-bit hash value with good distribution across all bits. + /// + /// Uses AES hardware acceleration when available, falls back to CRC32C otherwise. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long FastHash64For20Bytes(ref byte start) + { + uint seed = s_instanceRandom + 20; + + if (x64.Aes.IsSupported || Arm.Aes.IsSupported) + { + Vector128 key = Unsafe.As>(ref start); + uint last4 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref start, 16)); + Vector128 data = Vector128.CreateScalar(last4).AsByte(); + key ^= Vector128.CreateScalar(seed).AsByte(); + Vector128 mixed = x64.Aes.IsSupported + ? x64.Aes.Encrypt(data, key) + : Arm.Aes.MixColumns(Arm.Aes.Encrypt(data, key)); + return (long)(mixed.AsUInt64().GetElement(0) ^ mixed.AsUInt64().GetElement(1)); + } + + // Fallback: CRC32C-based 64-bit hash + ulong h0 = BitOperations.Crc32C(seed, Unsafe.ReadUnaligned(ref start)); + ulong h1 = BitOperations.Crc32C(seed ^ 0x9E3779B9u, Unsafe.ReadUnaligned(ref Unsafe.Add(ref start, 8))); + uint h2 = BitOperations.Crc32C(seed ^ 0x85EBCA6Bu, Unsafe.ReadUnaligned(ref Unsafe.Add(ref start, 16))); + return (long)((h0 | (h1 << 32)) ^ ((ulong)h2 * 0x9E3779B97F4A7C15)); + } } } diff --git a/src/Nethermind/Nethermind.Core/StorageCell.cs b/src/Nethermind/Nethermind.Core/StorageCell.cs index a35bc2eda97c..d28c5648f364 100644 --- a/src/Nethermind/Nethermind.Core/StorageCell.cs +++ b/src/Nethermind/Nethermind.Core/StorageCell.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Int256; @@ -13,12 +14,13 @@ namespace Nethermind.Core { [DebuggerDisplay("{Address}->{Index}")] - public readonly struct StorageCell : IEquatable + public readonly struct StorageCell : IEquatable, IHash64bit { + private readonly AddressAsKey _address; private readonly UInt256 _index; private readonly bool _isHash; - public Address Address { get; } + public Address Address => _address.Value; public bool IsHash => _isHash; public UInt256 Index => _index; @@ -33,21 +35,45 @@ private ValueHash256 GetHash() public StorageCell(Address address, in UInt256 index) { - Address = address; + _address = address; _index = index; } public StorageCell(Address address, ValueHash256 hash) { - Address = address; + _address = address; _index = Unsafe.As(ref hash); _isHash = true; } - public bool Equals(StorageCell other) => - _isHash == other._isHash && - Unsafe.As>(ref Unsafe.AsRef(in _index)) == Unsafe.As>(ref Unsafe.AsRef(in other._index)) && - Address.Equals(other.Address); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Equals(in StorageCell other) + { + if (_isHash != other._isHash) + return false; + + if (Unsafe.As>(ref Unsafe.AsRef(in _index)) != + Unsafe.As>(ref Unsafe.AsRef(in other._index))) + return false; + + // Inline 20-byte Address comparison: avoids the Address.Equals call + // that the JIT refuses to inline when called from deep inline chains + // (e.g. SeqlockCache.TryGetValue). Address.Bytes is always exactly 20 bytes. + Address a = _address.Value; + Address b = other._address.Value; + if (ReferenceEquals(a, b)) + return true; + + ref byte ab = ref MemoryMarshal.GetArrayDataReference(a.Bytes); + ref byte bb = ref MemoryMarshal.GetArrayDataReference(b.Bytes); + return Unsafe.As>(ref ab) == Unsafe.As>(ref bb) + && Unsafe.As(ref Unsafe.Add(ref ab, 16)) == Unsafe.As(ref Unsafe.Add(ref bb, 16)); + } + + public bool Equals(StorageCell other) => Equals(in other); + + public long GetHashCode64() + => SpanExtensions.FastHash64For32Bytes(ref Unsafe.As(ref Unsafe.AsRef(in _index))) ^ _address.Value.GetHashCode64(); public override bool Equals(object? obj) { @@ -62,12 +88,12 @@ public override bool Equals(object? obj) public override int GetHashCode() { int hash = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in _index), 1)).FastHash(); - return hash ^ Address.GetHashCode(); + return hash ^ _address.Value.GetHashCode(); } public override string ToString() { - return $"{Address}.{Index}"; + return $"{_address.Value}.{Index}"; } } } diff --git a/src/Nethermind/Nethermind.State.Test/StorageProviderTests.cs b/src/Nethermind/Nethermind.State.Test/StorageProviderTests.cs index d80eebe4be39..34728afad4b8 100644 --- a/src/Nethermind/Nethermind.State.Test/StorageProviderTests.cs +++ b/src/Nethermind/Nethermind.State.Test/StorageProviderTests.cs @@ -433,7 +433,7 @@ public void Selfdestruct_clears_cache() WorldState provider = BuildStorageProvider(ctx); StorageCell accessedStorageCell = new StorageCell(TestItem.AddressA, 1); StorageCell nonAccessedStorageCell = new StorageCell(TestItem.AddressA, 2); - preBlockCaches.StorageCache[accessedStorageCell] = [1, 2, 3]; + preBlockCaches.StorageCache.Set(accessedStorageCell, [1, 2, 3]); provider.Get(accessedStorageCell); provider.Commit(Paris.Instance); provider.ClearStorage(TestItem.AddressA); @@ -602,7 +602,7 @@ public void Selfdestruct_persist_between_commit() PreBlockCaches preBlockCaches = new PreBlockCaches(); Context ctx = new(preBlockCaches); StorageCell accessedStorageCell = new StorageCell(TestItem.AddressA, 1); - preBlockCaches.StorageCache[accessedStorageCell] = [1, 2, 3]; + preBlockCaches.StorageCache.Set(accessedStorageCell, [1, 2, 3]); WorldState provider = BuildStorageProvider(ctx); provider.Get(accessedStorageCell).ToArray().Should().BeEquivalentTo([1, 2, 3]); diff --git a/src/Nethermind/Nethermind.State/PreBlockCaches.cs b/src/Nethermind/Nethermind.State/PreBlockCaches.cs index 3fd8cf5e8cd7..e7f8bf290735 100644 --- a/src/Nethermind/Nethermind.State/PreBlockCaches.cs +++ b/src/Nethermind/Nethermind.State/PreBlockCaches.cs @@ -19,24 +19,24 @@ public class PreBlockCaches private readonly Func[] _clearCaches; - private readonly ConcurrentDictionary _storageCache = new(LockPartitions, InitialCapacity); - private readonly ConcurrentDictionary _stateCache = new(LockPartitions, InitialCapacity); - private readonly ConcurrentDictionary _rlpCache = new(LockPartitions, InitialCapacity); + private readonly SeqlockCache _storageCache = new(); + private readonly SeqlockCache _stateCache = new(); + private readonly SeqlockCache _rlpCache = new(); private readonly ConcurrentDictionary> _precompileCache = new(LockPartitions, InitialCapacity); public PreBlockCaches() { _clearCaches = [ - () => _storageCache.NoResizeClear() ? CacheType.Storage : CacheType.None, - () => _stateCache.NoResizeClear() ? CacheType.State : CacheType.None, - () => _precompileCache.NoResizeClear() ? CacheType.Precompile : CacheType.None + () => { _storageCache.Clear(); return CacheType.None; }, + () => { _stateCache.Clear(); return CacheType.None; }, + () => { _precompileCache.NoResizeClear(); return CacheType.None; } ]; } - public ConcurrentDictionary StorageCache => _storageCache; - public ConcurrentDictionary StateCache => _stateCache; - public ConcurrentDictionary RlpCache => _rlpCache; + public SeqlockCache StorageCache => _storageCache; + public SeqlockCache StateCache => _stateCache; + public SeqlockCache RlpCache => _rlpCache; public ConcurrentDictionary> PrecompileCache => _precompileCache; public CacheType ClearCaches() diff --git a/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs b/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs index d4260a7f5278..af258806395f 100644 --- a/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs +++ b/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Concurrent; using System.Diagnostics; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Metric; using Nethermind.Db; @@ -40,16 +40,26 @@ public class PrewarmerScopeProvider( public PreBlockCaches? Caches => preBlockCaches; public bool IsWarmWorldState => !populatePreBlockCache; - private sealed class ScopeWrapper( - IWorldStateScopeProvider.IScope baseScope, - PreBlockCaches preBlockCaches, - bool populatePreBlockCache) - : IWorldStateScopeProvider.IScope + private sealed class ScopeWrapper : IWorldStateScopeProvider.IScope { - ConcurrentDictionary preBlockCache = preBlockCaches.StateCache; + private readonly IWorldStateScopeProvider.IScope baseScope; + private readonly SeqlockCache preBlockCache; + private readonly SeqlockCache storageCache; + private readonly bool populatePreBlockCache; + private readonly SeqlockCache.ValueFactory _getFromBaseTree; private readonly IMetricObserver _metricObserver = Metrics.PrewarmerGetTime; private readonly bool _measureMetric = Metrics.DetailedMetricsEnabled; - private readonly PrewarmerGetTimeLabels _labels = populatePreBlockCache ? PrewarmerGetTimeLabels.Prewarmer : PrewarmerGetTimeLabels.NonPrewarmer; + private readonly PrewarmerGetTimeLabels _labels; + + public ScopeWrapper(IWorldStateScopeProvider.IScope baseScope, PreBlockCaches preBlockCaches, bool populatePreBlockCache) + { + this.baseScope = baseScope; + preBlockCache = preBlockCaches.StateCache; + storageCache = preBlockCaches.StorageCache; + this.populatePreBlockCache = populatePreBlockCache; + _labels = populatePreBlockCache ? PrewarmerGetTimeLabels.Prewarmer : PrewarmerGetTimeLabels.NonPrewarmer; + _getFromBaseTree = GetFromBaseTree; + } public void Dispose() => baseScope.Dispose(); @@ -59,7 +69,7 @@ public IWorldStateScopeProvider.IStorageTree CreateStorageTree(Address address) { return new StorageTreeWrapper( baseScope.CreateStorageTree(address), - preBlockCaches.StorageCache, + storageCache, address, populatePreBlockCache); } @@ -114,7 +124,7 @@ public void UpdateRootHash() if (populatePreBlockCache) { long priorReads = Metrics.ThreadLocalStateTreeReads; - Account? account = preBlockCache.GetOrAdd(address, GetFromBaseTree); + Account? account = preBlockCache.GetOrAdd(in addressAsKey, _getFromBaseTree); if (Metrics.ThreadLocalStateTreeReads == priorReads) { @@ -129,7 +139,7 @@ public void UpdateRootHash() } else { - if (preBlockCache?.TryGetValue(addressAsKey, out Account? account) ?? false) + if (preBlockCache.TryGetValue(in addressAsKey, out Account? account)) { if (_measureMetric) _metricObserver.Observe(Stopwatch.GetTimestamp() - sw, _labels.AddressHit); baseScope.HintGet(address, account); @@ -137,7 +147,7 @@ public void UpdateRootHash() } else { - account = GetFromBaseTree(addressAsKey); + account = GetFromBaseTree(in addressAsKey); if (_measureMetric) _metricObserver.Observe(Stopwatch.GetTimestamp() - sw, _labels.AddressMiss); } return account; @@ -146,22 +156,36 @@ public void UpdateRootHash() public void HintGet(Address address, Account? account) => baseScope.HintGet(address, account); - private Account? GetFromBaseTree(AddressAsKey address) + private Account? GetFromBaseTree(in AddressAsKey address) { return baseScope.Get(address); } } - private sealed class StorageTreeWrapper( - IWorldStateScopeProvider.IStorageTree baseStorageTree, - ConcurrentDictionary preBlockCache, - Address address, - bool populatePreBlockCache - ) : IWorldStateScopeProvider.IStorageTree + private sealed class StorageTreeWrapper : IWorldStateScopeProvider.IStorageTree { + private readonly IWorldStateScopeProvider.IStorageTree baseStorageTree; + private readonly SeqlockCache preBlockCache; + private readonly Address address; + private readonly bool populatePreBlockCache; + private readonly SeqlockCache.ValueFactory _loadFromTreeStorage; private readonly IMetricObserver _metricObserver = Db.Metrics.PrewarmerGetTime; private readonly bool _measureMetric = Db.Metrics.DetailedMetricsEnabled; - private readonly PrewarmerGetTimeLabels _labels = populatePreBlockCache ? PrewarmerGetTimeLabels.Prewarmer : PrewarmerGetTimeLabels.NonPrewarmer; + private readonly PrewarmerGetTimeLabels _labels; + + public StorageTreeWrapper( + IWorldStateScopeProvider.IStorageTree baseStorageTree, + SeqlockCache preBlockCache, + Address address, + bool populatePreBlockCache) + { + this.baseStorageTree = baseStorageTree; + this.preBlockCache = preBlockCache; + this.address = address; + this.populatePreBlockCache = populatePreBlockCache; + _labels = populatePreBlockCache ? PrewarmerGetTimeLabels.Prewarmer : PrewarmerGetTimeLabels.NonPrewarmer; + _loadFromTreeStorage = LoadFromTreeStorage; + } public Hash256 RootHash => baseStorageTree.RootHash; @@ -173,7 +197,7 @@ public byte[] Get(in UInt256 index) { long priorReads = Db.Metrics.ThreadLocalStorageTreeReads; - byte[] value = preBlockCache.GetOrAdd(storageCell, LoadFromTreeStorage); + byte[] value = preBlockCache.GetOrAdd(in storageCell, _loadFromTreeStorage); if (Db.Metrics.ThreadLocalStorageTreeReads == priorReads) { @@ -189,15 +213,15 @@ public byte[] Get(in UInt256 index) } else { - if (preBlockCache?.TryGetValue(storageCell, out byte[] value) ?? false) + if (preBlockCache.TryGetValue(in storageCell, out byte[] value)) { if (_measureMetric) _metricObserver.Observe(Stopwatch.GetTimestamp() - sw, _labels.SlotGetHit); - baseStorageTree.HintGet(index, value); + baseStorageTree.HintGet(in index, value); Db.Metrics.IncrementStorageTreeCache(); } else { - value = LoadFromTreeStorage(storageCell); + value = LoadFromTreeStorage(in storageCell); if (_measureMetric) _metricObserver.Observe(Stopwatch.GetTimestamp() - sw, _labels.SlotGetMiss); } return value; @@ -206,7 +230,7 @@ public byte[] Get(in UInt256 index) public void HintGet(in UInt256 index, byte[]? value) => baseStorageTree.HintGet(in index, value); - private byte[] LoadFromTreeStorage(StorageCell storageCell) + private byte[] LoadFromTreeStorage(in StorageCell storageCell) { Db.Metrics.IncrementStorageTreeReads(); diff --git a/src/Nethermind/Nethermind.Trie/NodeStorageCache.cs b/src/Nethermind/Nethermind.Trie/NodeStorageCache.cs index 75236286b82d..28146edf7611 100644 --- a/src/Nethermind/Nethermind.Trie/NodeStorageCache.cs +++ b/src/Nethermind/Nethermind.Trie/NodeStorageCache.cs @@ -1,15 +1,13 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; -using System.Collections.Concurrent; using Nethermind.Core.Collections; namespace Nethermind.Trie; public sealed class NodeStorageCache { - private ConcurrentDictionary _cache = new(); + private readonly SeqlockCache _cache = new(); private volatile bool _enabled = false; @@ -19,17 +17,18 @@ public bool Enabled set => _enabled = value; } - public byte[]? GetOrAdd(NodeKey nodeKey, Func tryLoadRlp) + public byte[]? GetOrAdd(in NodeKey nodeKey, SeqlockCache.ValueFactory tryLoadRlp) { if (!_enabled) { - return tryLoadRlp(nodeKey); + return tryLoadRlp(in nodeKey); } - return _cache.GetOrAdd(nodeKey, tryLoadRlp); + return _cache.GetOrAdd(in nodeKey, tryLoadRlp); } public bool ClearCaches() { - return _cache.NoResizeClear(); + _cache.Clear(); + return true; } } diff --git a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs index dc6d9b6d1fa0..3a0d52617152 100644 --- a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs @@ -3,8 +3,11 @@ using System; using System.Numerics; +using System.Runtime.CompilerServices; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Trie.Pruning; namespace Nethermind.Trie; @@ -13,8 +16,8 @@ public sealed class PreCachedTrieStore : ITrieStore { private readonly ITrieStore _inner; private readonly NodeStorageCache _preBlockCache; - private readonly Func _loadRlp; - private readonly Func _tryLoadRlp; + private readonly SeqlockCache.ValueFactory _loadRlp; + private readonly SeqlockCache.ValueFactory _tryLoadRlp; public PreCachedTrieStore(ITrieStore inner, NodeStorageCache cache) { @@ -22,8 +25,8 @@ public PreCachedTrieStore(ITrieStore inner, NodeStorageCache cache) _preBlockCache = cache; // Capture the delegate once for default path to avoid the allocation of the lambda per call - _loadRlp = (NodeKey key) => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); - _tryLoadRlp = (NodeKey key) => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); + _loadRlp = (in NodeKey key) => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); + _tryLoadRlp = (in NodeKey key) => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); } public void Dispose() @@ -43,8 +46,7 @@ public IBlockCommitter BeginBlockCommit(long blockNumber) public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) { - byte[]? rlp = _preBlockCache.GetOrAdd(new(address, in path, in keccak), - key => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash)); + byte[]? rlp = _preBlockCache.GetOrAdd(new(address, in path, in keccak), _tryLoadRlp); return rlp is not null; } @@ -60,17 +62,17 @@ public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 kecc public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => _preBlockCache.GetOrAdd(new(address, in path, hash), flags == ReadFlags.None ? _loadRlp : - key => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags)); + (in NodeKey key) => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags)); public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => _preBlockCache.GetOrAdd(new(address, in path, hash), flags == ReadFlags.None ? _tryLoadRlp : - key => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags)); + (in NodeKey key) => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags)); public INodeStorage.KeyScheme Scheme => _inner.Scheme; } -public readonly struct NodeKey : IEquatable +public readonly struct NodeKey : IEquatable, IHash64bit { public readonly Hash256? Address; public readonly TreePath Path; @@ -90,15 +92,28 @@ public NodeKey(Hash256? address, in TreePath path, Hash256 hash) Hash = hash; } - public bool Equals(NodeKey other) => - Address == other.Address && Path.Equals(in other.Path) && Hash.Equals(other.Hash); + public bool Equals(NodeKey other) => Equals(in other); public override bool Equals(object? obj) => obj is NodeKey key && Equals(key); + public bool Equals(in NodeKey other) => + Address == other.Address && Path.Equals(in other.Path) && Hash.Equals(other.Hash); + public override int GetHashCode() { uint hashCode0 = (uint)Hash.GetHashCode(); ulong hashCode1 = ((ulong)(uint)Path.GetHashCode() << 32) | (uint)(Address?.GetHashCode() ?? 1); return (int)BitOperations.Crc32C(hashCode0, hashCode1); } + + public long GetHashCode64() + { + long hashCode0 = Address is null ? 1L : SpanExtensions.FastHash64For32Bytes(ref Unsafe.As(ref Unsafe.AsRef(in Address.ValueHash256))); + long hashCode1 = SpanExtensions.FastHash64For32Bytes(ref Unsafe.As(ref Unsafe.AsRef(in Hash.ValueHash256))); + long hashCode2 = SpanExtensions.FastHash64For32Bytes(ref Unsafe.As(ref Unsafe.AsRef(in Path.Path))); + + // Rotations spaced by 64/3 ensure way 0 (bits 0-13) and way 1 (bits 42-55) + // sample non-overlapping 14-bit windows from each input + return hashCode1 + (long)BitOperations.RotateLeft((ulong)hashCode0, 21) + (long)BitOperations.RotateLeft((ulong)hashCode2, 42); + } }