From 932a5148964e57d8285f43455ea84605731b27cb Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Thu, 16 Oct 2025 00:36:06 +0300 Subject: [PATCH 01/18] Preparation for log-index --- .../FullPruning/FullPrunerTests.cs | 5 + .../Receipts/IReceiptStorage.cs | 7 +- .../Receipts/InMemoryReceiptStorage.cs | 3 + .../Receipts/NullReceiptStorage.cs | 1 + .../Receipts/PersistentReceiptStorage.cs | 3 + .../Builders/RandomExtensions.cs | 15 ++ .../Json/ByteArrayConverterTests.cs | 18 +++ .../Json/ConverterTestBase.cs | 15 +- .../Collections/CollectionExtensions.cs | 15 ++ .../Nethermind.Core/Extensions/Bytes.cs | 11 +- .../Extensions/SizeExtensions.cs | 4 +- .../Extensions/WaitHandleExtensions.cs | 8 +- .../Nethermind.Core/FakeWriteBatch.cs | 10 ++ .../Nethermind.Core/IKeyValueStore.cs | 1 + src/Nethermind/Nethermind.Core/IWriteBatch.cs | 5 +- .../JsonConverters/ByteArrayConverter.cs | 12 +- .../Nethermind.Core/ProgressLogger.cs | 9 +- .../Utils/ConcurrentWriteBatcher.cs | 18 +++ .../Nethermind.Db.Rocks/ColumnDb.cs | 27 +++- .../Nethermind.Db.Rocks/ColumnsDb.cs | 14 +- .../Config/PerTableDbConfig.cs | 16 +- .../Nethermind.Db.Rocks/DbOnTheRocks.cs | 138 +++++++++++++++++- .../MergeOperatorAdapter.cs | 80 ++++++++++ .../RocksDbIteratorWrapper.cs | 21 +++ src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs | 15 ++ src/Nethermind/Nethermind.Db/CompressingDb.cs | 22 +++ .../FullPruning/FullPruningDb.cs | 43 ++++++ src/Nethermind/Nethermind.Db/IDb.cs | 15 +- src/Nethermind/Nethermind.Db/IIterator.cs | 18 +++ .../Nethermind.Db/IMergeOperator.cs | 14 ++ .../Nethermind.Db/InMemoryWriteBatch.cs | 10 ++ src/Nethermind/Nethermind.Db/MemDb.cs | 15 ++ src/Nethermind/Nethermind.Db/NullDb.cs | 15 ++ src/Nethermind/Nethermind.Db/ReadOnlyDb.cs | 15 ++ .../Nethermind.Db/RocksDbMergeEnumerator.cs | 51 +++++++ .../Nethermind.Db/RocksDbSettings.cs | 6 + .../Nethermind.Db/SimpleFilePublicKeyDb.cs | 15 ++ .../Steps/Migrations/ReceiptMigrationTests.cs | 1 + .../Pruning/TreeStoreTests.cs | 15 ++ .../Nethermind.Trie/NullKeyValueStore.cs | 5 + .../Nethermind.TxPool.Test/TxPoolTests.cs | 2 +- 41 files changed, 700 insertions(+), 33 deletions(-) create mode 100644 src/Nethermind/Nethermind.Core.Test/Builders/RandomExtensions.cs create mode 100644 src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs create mode 100644 src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs create mode 100644 src/Nethermind/Nethermind.Db/IIterator.cs create mode 100644 src/Nethermind/Nethermind.Db/IMergeOperator.cs create mode 100644 src/Nethermind/Nethermind.Db/RocksDbMergeEnumerator.cs diff --git a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs index c13e1e2482cc..04f7bb54956f 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs @@ -411,6 +411,11 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF _context.Set(key, value, flags); } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _context.Merge(key, value, flags); + } + public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { return _context.Get(key, flags); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs index 65633a1fa7ad..daa79cf05188 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs @@ -19,8 +19,13 @@ public interface IReceiptStorage : IReceiptFinder void RemoveReceipts(Block block); /// - /// Receipts for a block are inserted + /// Receipts for a new main block are inserted /// event EventHandler ReceiptsInserted; + + /// + /// Any receipts are inserted + /// + event EventHandler AnyReceiptsInserted; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs index 1e3eec074a49..ffdf56d1b8b0 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs @@ -19,6 +19,7 @@ public class InMemoryReceiptStorage : IReceiptStorage #pragma warning disable CS0067 public event EventHandler ReceiptsInserted; + public event EventHandler? AnyReceiptsInserted; #pragma warning restore CS0067 public InMemoryReceiptStorage(bool allowReceiptIterator = true, IBlockTree? blockTree = null) @@ -73,6 +74,8 @@ public void Insert(Block block, TxReceipt[] txReceipts, IReleaseSpec spec, bool { EnsureCanonical(block); } + + AnyReceiptsInserted?.Invoke(this, new(block.Header, txReceipts)); } public bool HasBlock(long blockNumber, Hash256 hash) diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs index 1c06f8b3d21e..fd73788fcdbb 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs @@ -14,6 +14,7 @@ public class NullReceiptStorage : IReceiptStorage #pragma warning disable CS0067 public event EventHandler ReceiptsInserted; + public event EventHandler? AnyReceiptsInserted; #pragma warning restore CS0067 public Hash256? FindBlockHash(Hash256 hash) => null; diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 5d5715278943..9b9dafe6c2af 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -39,6 +39,7 @@ public class PersistentReceiptStorage : IReceiptStorage private readonly LruCache _receiptsCache = new(CacheSize, CacheSize, "receipts"); public event EventHandler ReceiptsInserted; + public event EventHandler? AnyReceiptsInserted; public PersistentReceiptStorage( IColumnsDb receiptsDb, @@ -288,6 +289,8 @@ public void Insert(Block block, TxReceipt[]? txReceipts, IReleaseSpec spec, bool { EnsureCanonical(block, lastBlockNumber); } + + AnyReceiptsInserted?.Invoke(this, new(block.Header, txReceipts)); } public long MigratedBlockNumber diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/RandomExtensions.cs b/src/Nethermind/Nethermind.Core.Test/Builders/RandomExtensions.cs new file mode 100644 index 000000000000..17135ff14fa3 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Builders/RandomExtensions.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using Nethermind.Crypto; + +namespace Nethermind.Core.Test.Builders; + +public static class RandomExtensions +{ + public static T NextFrom(this Random random, IReadOnlyList values) => values[random.Next(values.Count)]; + + public static T NextFrom(this ICryptoRandom random, IReadOnlyList values) => values[random.NextInt(values.Count)]; +} diff --git a/src/Nethermind/Nethermind.Core.Test/Json/ByteArrayConverterTests.cs b/src/Nethermind/Nethermind.Core.Test/Json/ByteArrayConverterTests.cs index d1413676fa33..5888c30c65e1 100644 --- a/src/Nethermind/Nethermind.Core.Test/Json/ByteArrayConverterTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Json/ByteArrayConverterTests.cs @@ -8,6 +8,8 @@ using System.Text.Json; using FluentAssertions; using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; using Nethermind.Serialization.Json; using NUnit.Framework; @@ -294,6 +296,22 @@ public void Fuzz_RandomHex_SegmentationInvariant() } } + [Test] + public void Test_DictionaryKey() + { + var random = new CryptoRandom(); + var dictionary = new Dictionary + { + { Bytes.FromHexString("0x0"), null }, + { Bytes.FromHexString("0x1"), random.NextInt(int.MaxValue) }, + { Build.An.Address.TestObject.Bytes, random.NextInt(int.MaxValue) }, + { random.GenerateRandomBytes(10), random.NextInt(int.MaxValue) }, + { random.GenerateRandomBytes(32), random.NextInt(int.MaxValue) }, + }; + + TestConverter(dictionary, new ByteArrayConverter()); + } + private static ReadOnlySequence JsonForLiteral(string literal) => MakeSequence(Encoding.UTF8.GetBytes(literal)); diff --git a/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs b/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs index 7f38126cda19..6425a9321510 100644 --- a/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs +++ b/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections; using System.Text.Json; using System.Text.Json.Serialization; using NUnit.Framework; @@ -10,7 +11,10 @@ namespace Nethermind.Core.Test.Json; public class ConverterTestBase { - protected void TestConverter(T? item, Func equalityComparer, JsonConverter converter) + protected void TestConverter(T? item, Func equalityComparer, JsonConverter converter) => + TestConverter(item, converter, equalityComparer); + + protected void TestConverter(R? item, JsonConverter converter, Func? equalityComparer = null) { var options = new JsonSerializerOptions { @@ -22,10 +26,15 @@ protected void TestConverter(T? item, Func equalityComparer, JsonCon string result = JsonSerializer.Serialize(item, options); - T? deserialized = JsonSerializer.Deserialize(result, options); + R? deserialized = JsonSerializer.Deserialize(result, options); #pragma warning disable CS8604 - Assert.That(equalityComparer(item, deserialized), Is.True); + if (equalityComparer != null) + Assert.That(equalityComparer(item, deserialized), Is.True); + else if (item is IEnumerable itemE && deserialized is IEnumerable deserializedE) + Assert.That(deserializedE, Is.EquivalentTo(itemE)); + else + Assert.That(deserialized, Is.EqualTo(item)); #pragma warning restore CS8604 } } diff --git a/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs b/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs index 4a5f5f476a39..6408673b22dd 100644 --- a/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs @@ -87,6 +87,21 @@ public static bool NoResizeClear(this ConcurrentDictionary(this Dictionary dictionary, TKey key, Func factory) + where TKey : notnull + { + if (dictionary.TryGetValue(key, out TValue? value)) + { + return value; + } + else + { + value = factory(key); + dictionary.Add(key, value); + return value; + } + } + private static class ClearCache where TKey : notnull { public static readonly Action> Clear = CreateNoResizeClearExpression(); diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 128268d8243b..549a200b6889 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -262,7 +262,7 @@ public static byte[] PadLeft(this ReadOnlySpan bytes, int length, byte pad return result; } - public static byte[] PadRight(this byte[] bytes, int length) + public static byte[] PadRight(this byte[] bytes, int length, byte padding = 0) { if (bytes.Length == length) { @@ -276,6 +276,15 @@ public static byte[] PadRight(this byte[] bytes, int length) byte[] result = new byte[length]; Buffer.BlockCopy(bytes, 0, result, 0, bytes.Length); + + if (padding != 0) + { + for (int i = bytes.Length; i < length; i++) + { + result[i] = padding; + } + } + return result; } diff --git a/src/Nethermind/Nethermind.Core/Extensions/SizeExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SizeExtensions.cs index ba773225fb46..2ae4af43e7e8 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/SizeExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/SizeExtensions.cs @@ -67,7 +67,7 @@ public static long KiB(this int @this) return ((long)@this).KiB(); } - public static string SizeToString(this long @this, bool useSi = false, int precision = 1) + public static string SizeToString(this long @this, bool useSi = false, bool addSpace = false, int precision = 1) { string[] suf = useSi ? ["B", "KB", "MB", "GB", "TB"] : ["B", "KiB", "MiB", "GiB", "TiB"]; if (@this == 0) @@ -77,7 +77,7 @@ public static string SizeToString(this long @this, bool useSi = false, int preci long bytes = Math.Abs(@this); int place = Math.Min(suf.Length - 1, Convert.ToInt32(Math.Floor(Math.Log(bytes, useSi ? 1000 : 1024)))); double num = Math.Round(bytes / Math.Pow(useSi ? 1000 : 1024, place), precision); - return (Math.Sign(@this) * num).ToString() + suf[place]; + return string.Concat(Math.Sign(@this) * num, addSpace ? " " : "", suf[place]); } } } diff --git a/src/Nethermind/Nethermind.Core/Extensions/WaitHandleExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/WaitHandleExtensions.cs index fd0616dca645..5090c78856f1 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/WaitHandleExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/WaitHandleExtensions.cs @@ -23,8 +23,12 @@ public static async Task WaitOneAsync(this WaitHandle handle, int millisec millisecondsTimeout, true); tokenRegistration = cancellationToken.Register( - static state => ((TaskCompletionSource)state!).TrySetCanceled(), - tcs); + static state => + { + var (tcs, ct) = ((TaskCompletionSource tcs, CancellationToken ct))state!; + tcs.TrySetCanceled(ct); + }, + (tcs, cancellationToken)); return await tcs.Task; } diff --git a/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs b/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs index ce27831eb399..a4edf7862f91 100644 --- a/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs +++ b/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs @@ -31,5 +31,15 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF { _storePretendingToSupportBatches.Set(key, value, flags); } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException("Merging is not supported by this implementation."); + } + + public void Clear() + { + throw new NotSupportedException("Clearing is not supported by this implementation."); + } } } diff --git a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs index e613308267ed..76ce78f66ef0 100644 --- a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs @@ -59,6 +59,7 @@ public interface IWriteOnlyKeyValueStore } void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None); + void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None); /// /// Some store keep the input array directly. (eg: CachingStore), and therefore passing the value by array diff --git a/src/Nethermind/Nethermind.Core/IWriteBatch.cs b/src/Nethermind/Nethermind.Core/IWriteBatch.cs index 909f184541c0..5ba4d5d6427e 100644 --- a/src/Nethermind/Nethermind.Core/IWriteBatch.cs +++ b/src/Nethermind/Nethermind.Core/IWriteBatch.cs @@ -5,5 +5,8 @@ namespace Nethermind.Core { - public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore; + public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore + { + void Clear(); + } } diff --git a/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs b/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs index e67479f53636..a40cb9f01823 100644 --- a/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs +++ b/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs @@ -173,7 +173,7 @@ public static void Convert(ref Utf8JsonReader reader, scoped Span span) private static void ThrowFormatException() => throw new FormatException(); [DoesNotReturn, StackTraceHidden] - private static void ThrowInvalidOperationException() => throw new InvalidOperationException(); + private static Exception ThrowInvalidOperationException() => throw new InvalidOperationException(); public override void Write( Utf8JsonWriter writer, @@ -246,4 +246,14 @@ public static void Convert( if (array is not null) ArrayPool.Shared.Return(array); } + + public override byte[] ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return Convert(ref reader) ?? throw ThrowInvalidOperationException(); + } + + public override void WriteAsPropertyName(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options) + { + Convert(writer, value, static (w, h) => w.WritePropertyName(h), skipLeadingZeros: false, addQuotations: false, addHexPrefix: true); + } } diff --git a/src/Nethermind/Nethermind.Core/ProgressLogger.cs b/src/Nethermind/Nethermind.Core/ProgressLogger.cs index f674106dab0d..0129ac64d514 100644 --- a/src/Nethermind/Nethermind.Core/ProgressLogger.cs +++ b/src/Nethermind/Nethermind.Core/ProgressLogger.cs @@ -47,9 +47,9 @@ public void IncrementSkipped(int skipped = 1) Interlocked.Add(ref _skipped, skipped); } - public void SetMeasuringPoint() + public void SetMeasuringPoint(bool resetCompletion = true) { - UtcEndTime = null; + if (resetCompletion) UtcEndTime = null; if (UtcStartTime is not null) { @@ -160,7 +160,8 @@ public void SetFormat(Func formatter) _formatter = formatter; } - public void LogProgress() + // TODO: should it actually ever be true? + public void LogProgress(bool resetCompletion = true) { (long, long, long, long) reportState = (CurrentValue, TargetValue, CurrentQueued, _skipped); if (reportState != _lastReportState) @@ -169,7 +170,7 @@ public void LogProgress() _lastReportState = reportState; _logger.Info(reportString); } - SetMeasuringPoint(); + SetMeasuringPoint(resetCompletion); } private string DefaultFormatter() diff --git a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs index 9ee5829dc20f..4d01fd2f4466 100644 --- a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs +++ b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs @@ -40,6 +40,24 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags ReturnWriteBatch(currentBatch); } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + IWriteBatch currentBatch = RentWriteBatch(); + currentBatch.Merge(key, value, flags); + ReturnWriteBatch(currentBatch); + } + + public void Clear() + { + if (_disposing) + throw new InvalidOperationException("Batch is already disposed."); + + foreach (IWriteBatch batch in _batches) + { + batch.Clear(); + } + } + public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) { IWriteBatch currentBatch = RentWriteBatch(); diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs index ece6bb682979..6c55bea1dd46 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs @@ -57,6 +57,11 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags _mainDb.SetWithColumnFamily(key, _columnFamily, value, writeFlags); } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags writeFlags = WriteFlags.None) + { + _mainDb.MergeWithColumnFamily(key, _columnFamily, value, writeFlags); + } + public KeyValuePair[] this[byte[][] keys] => _rocksDb.MultiGet(keys, keys.Select(k => _columnFamily).ToArray()); @@ -99,6 +104,11 @@ public void Dispose() _underlyingWriteBatch.Dispose(); } + public void Clear() + { + _underlyingWriteBatch.Clear(); + } + public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) { if (value is null) @@ -115,6 +125,11 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags { _underlyingWriteBatch.Set(key, value, _columnDb._columnFamily, flags); } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _underlyingWriteBatch.Merge(key, value, _columnDb._columnFamily, flags); + } } public void Remove(ReadOnlySpan key) @@ -130,7 +145,7 @@ public bool KeyExists(ReadOnlySpan key) public void Flush(bool onlyWal) { - _mainDb.Flush(onlyWal); + _mainDb.FlushWithColumnFamily(_columnFamily); } public void Compact() @@ -178,4 +193,14 @@ public ISortedView GetViewBetween(ReadOnlySpan firstKey, ReadOnlySpan GetEnumKeys(IReadOnlyList keys) return keys; } - protected override void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache) + protected override void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache, IMergeOperator? mergeOperator) { - base.BuildOptions(dbConfig, options, sharedCache); + base.BuildOptions(dbConfig, options, sharedCache, mergeOperator); options.SetCreateMissingColumnFamilies(); } @@ -130,9 +130,19 @@ public void Dispose() _writeBatch.Dispose(); } + public void Clear() + { + _writeBatch._writeBatch.Clear(); + } + public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) { _writeBatch._writeBatch.Set(key, value, _column._columnFamily, flags); } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _writeBatch._writeBatch.Merge(key, value, flags); + } } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index 6524901e3578..e6e7b9501831 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -38,7 +38,7 @@ private void EnsureConfigIsAvailable(string propertyName) foreach (var prefix in _prefixes) { string prefixed = string.Concat(prefix, propertyName); - if (type.GetProperty(prefixed, BindingFlags.Public | BindingFlags.Instance) is null) + if (GetProperty(type, prefixed, caseSensitive: true) is null) { throw new InvalidConfigurationException($"Configuration {propertyName} not available with prefix {prefix}. Add {prefix}{propertyName} to {nameof(IDbConfig)}.", -1); } @@ -90,13 +90,13 @@ private static string ReadRocksdbOptions(IDbConfig dbConfig, string propertyName Type type = dbConfig.GetType(); PropertyInfo? propertyInfo; - string val = (string)type.GetProperty(propertyName, BindingFlags.Public | BindingFlags.Instance)!.GetValue(dbConfig)!; + string val = (string)GetProperty(type, propertyName, caseSensitive: true)!.GetValue(dbConfig)!; foreach (var prefix in prefixes) { string prefixed = string.Concat(prefix, propertyName); - propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + propertyInfo = GetProperty(type, prefixed); if (propertyInfo is not null) { string? valObj = (string?)propertyInfo.GetValue(dbConfig); @@ -122,7 +122,7 @@ private static string ReadRocksdbOptions(IDbConfig dbConfig, string propertyName { string prefixed = string.Concat(prefix, propertyName); - propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + propertyInfo = GetProperty(type, prefixed); if (propertyInfo is not null) { if (propertyInfo.PropertyType.CanBeAssignedNull()) @@ -147,7 +147,7 @@ private static string ReadRocksdbOptions(IDbConfig dbConfig, string propertyName } // Use generic one even if its available - propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + propertyInfo = GetProperty(type, propertyName); return (T?)propertyInfo?.GetValue(dbConfig); } catch (Exception e) @@ -156,4 +156,10 @@ private static string ReadRocksdbOptions(IDbConfig dbConfig, string propertyName } } + private static PropertyInfo? GetProperty(Type type, string name, bool caseSensitive = false) + { + BindingFlags flags = BindingFlags.Public | BindingFlags.Instance; + if (!caseSensitive) flags |= BindingFlags.IgnoreCase; + return type.GetProperty(name, flags); + } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index 474ae0d50da1..df768ed6ba1a 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -66,6 +66,11 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor private readonly DbSettings _settings; + // ReSharper disable once CollectionNeverQueried.Local + // Need to keep options from GC in case of merge operator applied, as they are used in callback + // TODO: find better way? + private readonly ConcurrentBag _doNotGcOptions = []; + private readonly IRocksDbConfig _perTableDbConfig; private ulong _maxBytesForLevelBase; private ulong _targetFileSizeBase; @@ -143,7 +148,7 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan // ReSharper disable once VirtualMemberCallInConstructor if (_logger.IsDebug) _logger.Debug($"Building options for {Name} DB"); DbOptions = new DbOptions(); - BuildOptions(_perTableDbConfig, DbOptions, sharedCache); + BuildOptions(_perTableDbConfig, DbOptions, sharedCache, _settings.MergeOperator); ColumnFamilies? columnFamilies = null; if (columnNames is not null) @@ -155,7 +160,8 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan ColumnFamilyOptions options = new(); IRocksDbConfig columnConfig = _rocksDbConfigFactory.GetForDatabase(Name, columnFamily); - BuildOptions(columnConfig, options, sharedCache); + IMergeOperator? mergeOperator = _settings.MergeOperatorByColumnFamily?.GetValueOrDefault(enumColumnName); + BuildOptions(columnConfig, options, sharedCache, mergeOperator); // "default" is a special column name with rocksdb, which is what previously not specifying column goes to if (columnFamily == "Default") columnFamily = "default"; @@ -447,7 +453,7 @@ public static IDictionary ExtractOptions(string dbOptions) return asDict; } - protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache) where T : Options + protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache, IMergeOperator? mergeOperator) where T : Options { // This section is about the table factory.. and block cache apparently. // This effect the format of the SST files and usually require resync to take effect. @@ -577,6 +583,12 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio } } + if (mergeOperator is not null) + { + options.SetMergeOperator(new MergeOperatorAdapter(mergeOperator)); + _doNotGcOptions.Add(options); + } + #endregion #region read-write options @@ -893,6 +905,40 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags SetWithColumnFamily(key, null, value, writeFlags); } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + ObjectDisposedException.ThrowIf(_isDisposing, this); + + UpdateWriteMetrics(); + + try + { + _db.Merge(key, value, null, WriteFlagsToWriteOptions(flags)); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + + internal void MergeWithColumnFamily(ReadOnlySpan key, ColumnFamilyHandle? cf, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + ObjectDisposedException.ThrowIf(_isDisposing, this); + + UpdateWriteMetrics(); + + try + { + _db.Merge(key, value, cf, WriteFlagsToWriteOptions(flags)); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + public void DangerousReleaseMemory(in ReadOnlySpan span) { if (!span.IsNullOrEmpty()) @@ -982,10 +1028,22 @@ public void Remove(ReadOnlySpan key) return GetAllCore(iterator); } - protected internal Iterator CreateIterator(bool ordered = false, ColumnFamilyHandle? ch = null) + protected internal Iterator CreateIterator(bool ordered, ColumnFamilyHandle? ch = null) + { + return CreateIterator(new IteratorOptions { Ordered = ordered }, ch); + } + + protected internal Iterator CreateIterator(IteratorOptions options, ColumnFamilyHandle? ch = null) { ReadOptions readOptions = new(); - readOptions.SetTailing(!ordered); + readOptions.SetTailing(!options.Ordered); + + if (options.LowerBound is { } lowerBound) + readOptions.SetIterateLowerBound(lowerBound); + + if (options.UpperBound is { } upperBound) + readOptions.SetIterateUpperBound(upperBound); + return CreateIterator(readOptions, ch); } @@ -1222,6 +1280,13 @@ private static void ReturnWriteBatch(WriteBatch batch) _reusableWriteBatch = batch; } + public void Clear() + { + ObjectDisposedException.ThrowIf(_dbOnTheRocks._isDisposed, _dbOnTheRocks); + + _rocksBatch.Clear(); + } + public void Dispose() { ObjectDisposedException.ThrowIf(_dbOnTheRocks._isDisposed, _dbOnTheRocks); @@ -1280,6 +1345,21 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags Set(key, value, null, flags); } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + Merge(key, value, null, flags); + } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, ColumnFamilyHandle? cf = null, WriteFlags flags = WriteFlags.None) + { + ObjectDisposedException.ThrowIf(_isDisposed, this); + + _rocksBatch.Merge(key, value, cf); + _writeFlags = flags; + + if ((flags & WriteFlags.DisableWAL) != 0) FlushOnTooManyWrites(); + } + private void FlushOnTooManyWrites() { if (Interlocked.Increment(ref _writeCount) % MaxWritesOnNoWal != 0) return; @@ -1306,6 +1386,13 @@ public void Flush(bool onlyWal = false) InnerFlush(onlyWal); } + public void FlushWithColumnFamily(ColumnFamilyHandle familyHandle) + { + ObjectDisposedException.ThrowIf(_isDisposing, this); + + InnerFlush(familyHandle); + } + public virtual void Compact() { _db.CompactRange(Keccak.Zero.BytesToArray(), Keccak.MaxValue.BytesToArray()); @@ -1328,6 +1415,18 @@ private void InnerFlush(bool onlyWal) } } + private void InnerFlush(ColumnFamilyHandle columnFamilyHandle) + { + try + { + _rocksDbNative.rocksdb_flush_cf(_db.Handle, FlushOptions.DefaultFlushOptions.Handle, columnFamilyHandle.Handle); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + } + } + public void Clear() { Dispose(); @@ -1659,6 +1758,29 @@ private static IDictionary GetBlobFilesOptions() }; } + public IIterator GetIterator(bool ordered = false) + { + var iterator = CreateIterator(ordered); + return new RocksDbIteratorWrapper(iterator); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + return GetIterator(ref options, null); + } + + public IIterator GetIterator(bool ordered, ColumnFamilyHandle familyHandle) + { + var options = new IteratorOptions { Ordered = ordered }; + return GetIterator(ref options, familyHandle); + } + + public IIterator GetIterator(ref IteratorOptions options, ColumnFamilyHandle? familyHandle) + { + var iterator = CreateIterator(options, familyHandle); + return new RocksDbIteratorWrapper(iterator); + } + /// /// Iterators should not be kept for long as it will pin some memory block and sst file. This would show up as /// temporary higher disk usage or memory usage. @@ -1776,10 +1898,10 @@ private sealed class ManagedIterators : ThreadLocal public void ClearIterators() { if (_disposed) return; - if (Values is null) return; - foreach (IteratorHolder iterator in Values) + if (Values is not { } values) return; + foreach (IteratorHolder iterator in values) { - iterator.Dispose(); + iterator?.Dispose(); } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs b/src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs new file mode 100644 index 000000000000..14bf9efc98f8 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Nethermind.Core.Collections; +using RocksDbSharp; + +namespace Nethermind.Db.Rocks; + +// Also see RocksDbSharp.MergeOperatorImpl +internal class MergeOperatorAdapter(IMergeOperator inner) : MergeOperator +{ + public string Name => inner.Name; + + // TODO: fix and return array ptr instead of copying to unmanaged memory? + private static unsafe IntPtr GetResult(ArrayPoolList? data, out IntPtr resultLength, out IntPtr success) + { + if (data is null) + { + success = Convert.ToInt32(false); + resultLength = IntPtr.Zero; + return IntPtr.Zero; + } + + using (data) + { + void* resultPtr = NativeMemory.Alloc((uint)data.Count); + var result = new Span(resultPtr, data.Count); + data.AsSpan().CopyTo(result); + + resultLength = result.Length; + + // Fixing RocksDbSharp invalid callback signature, TODO: submit an issue/PR + Unsafe.SkipInit(out success); + Unsafe.As(ref success) = 1; + + return (IntPtr)resultPtr; + } + } + + unsafe IntPtr MergeOperator.PartialMerge( + IntPtr keyPtr, + UIntPtr keyLength, + IntPtr operandsList, + IntPtr operandsListLength, + int numOperands, + out IntPtr successPtr, + out IntPtr resultLength) + { + var key = new Span((void*)keyPtr, (int)keyLength); + var enumerator = new RocksDbMergeEnumerator(new((void*)operandsList, numOperands), new((void*)operandsListLength, numOperands)); + + ArrayPoolList? result = inner.PartialMerge(key, enumerator); + return GetResult(result, out resultLength, out successPtr); + } + + unsafe IntPtr MergeOperator.FullMerge( + IntPtr keyPtr, + UIntPtr keyLength, + IntPtr existingValuePtr, + UIntPtr existingValueLength, + IntPtr operandsList, + IntPtr operandsListLength, + int numOperands, + out IntPtr successPtr, + out IntPtr resultLength) + { + var key = new ReadOnlySpan((void*)keyPtr, (int)keyLength); + bool hasExistingValue = existingValuePtr != IntPtr.Zero; + Span existingValue = hasExistingValue ? new((void*)existingValuePtr, (int)existingValueLength) : Span.Empty; + var enumerator = new RocksDbMergeEnumerator(existingValue, hasExistingValue, new((void*)operandsList, numOperands), new((void*)operandsListLength, numOperands)); + + ArrayPoolList? result = inner.FullMerge(key, enumerator); + return GetResult(result, out resultLength, out successPtr); + } + + unsafe void MergeOperator.DeleteValue(IntPtr value, UIntPtr valueLength) => NativeMemory.Free((void*)value); +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs b/src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs new file mode 100644 index 000000000000..9909a4283720 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using RocksDbSharp; +using System; + +namespace Nethermind.Db.Rocks; + +public sealed class RocksDbIteratorWrapper(Iterator iterator) : IIterator +{ + public void SeekToFirst() => iterator.SeekToFirst(); + public void Seek(ReadOnlySpan key) => iterator.Seek(key); + public void SeekForPrev(ReadOnlySpan key) => iterator.SeekForPrev(key); + public void Next() => iterator.Next(); + public void Prev() => iterator.Prev(); + public bool Valid() => iterator.Valid(); + public ReadOnlySpan Key() => iterator.GetKeySpan(); + public ReadOnlySpan Value() => iterator.GetValueSpan(); + public void Dispose() => iterator.Dispose(); +} + diff --git a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs index 98279f33c4ad..12612cfc1419 100644 --- a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs +++ b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs @@ -61,6 +61,11 @@ public byte[] Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) public KeyValuePair[] this[byte[][] keys] => keys.Select(k => new KeyValuePair(k, GetThroughRpc(k))).ToArray(); + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new InvalidOperationException("RPC DB does not support writes"); + } + public void Remove(ReadOnlySpan key) { throw new InvalidOperationException("RPC DB does not support writes"); @@ -119,5 +124,15 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags public void DangerousReleaseMemory(in ReadOnlySpan span) { } + + public IIterator GetIterator(bool ordered = false) + { + throw new InvalidOperationException("RPC DB does not support iteration"); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + throw new InvalidOperationException("RPC DB does not support iteration"); + } } } diff --git a/src/Nethermind/Nethermind.Db/CompressingDb.cs b/src/Nethermind/Nethermind.Db/CompressingDb.cs index 616298983d6d..30491cadfc20 100644 --- a/src/Nethermind/Nethermind.Db/CompressingDb.cs +++ b/src/Nethermind/Nethermind.Db/CompressingDb.cs @@ -44,6 +44,8 @@ private class WriteBatch : IWriteBatch public void Dispose() => _wrapped.Dispose(); + public void Clear() => _wrapped.Clear(); + public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) => _wrapped.Set(key, Compress(value), flags); @@ -52,6 +54,11 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags _wrapped.PutSpan(key, Compress(value, stackalloc byte[value.Length]), flags); } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new InvalidOperationException("EOA compressing DB does not support merging"); + } + public bool PreferWriteByArray => _wrapped.PreferWriteByArray; public byte[]? this[ReadOnlySpan key] @@ -133,6 +140,11 @@ public IEnumerable GetAllKeys(bool ordered = false) => public IEnumerable GetAllValues(bool ordered = false) => _wrapped.GetAllValues(ordered).Select(Decompress); + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _wrapped.Merge(key, value, flags); + } + public void Remove(ReadOnlySpan key) => _wrapped.Remove(key); public bool KeyExists(ReadOnlySpan key) => _wrapped.KeyExists(key); @@ -169,6 +181,16 @@ public void Tune(ITunableDb.TuneType type) if (_wrapped is ITunableDb tunable) tunable.Tune(type); } + + public IIterator GetIterator(bool ordered = false) + { + return _wrapped.GetIterator(ordered); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + return _wrapped.GetIterator(ref options); + } } } } diff --git a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs index fb2c98942716..b2804999c4c1 100755 --- a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs +++ b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs @@ -91,6 +91,16 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags } } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _currentDb.Merge(key, value, flags); // we are writing to the main DB + IDb? cloningDb = _pruningContext?.CloningDb; + if (cloningDb is not null) // if pruning is in progress we are also writing to the secondary, copied DB + { + DuplicateMerge(cloningDb, key, value, flags); + } + } + private void Duplicate(IWriteOnlyKeyValueStore db, ReadOnlySpan key, byte[]? value, WriteFlags flags) { db.Set(key, value, flags); @@ -103,6 +113,12 @@ private void Duplicate(IWriteOnlyKeyValueStore db, ReadOnlySpan key, ReadO _updateDuplicateWriteMetrics?.Invoke(); } + private void DuplicateMerge(IWriteOnlyKeyValueStore db, ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags) + { + db.Merge(key, value, flags); + _updateDuplicateWriteMetrics?.Invoke(); + } + // we also need to duplicate writes that are in batches public IWriteBatch StartWriteBatch() => _pruningContext is null @@ -248,6 +264,11 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF _db.Duplicate(CloningDb, key, value, flags); } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _db.Merge(key, value, flags); + } + public IWriteBatch StartWriteBatch() { return CloningDb.StartWriteBatch(); @@ -317,11 +338,23 @@ public void Dispose() _clonedWriteBatch.Dispose(); } + public void Clear() + { + _writeBatch.Clear(); + _clonedWriteBatch.Clear(); + } + public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) { _writeBatch.Set(key, value, flags); _db.Duplicate(_clonedWriteBatch, key, value, flags); } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _writeBatch.Merge(key, value, flags); + _db.DuplicateMerge(_clonedWriteBatch, key, value, flags); + } } public void Tune(ITunableDb.TuneType type) @@ -331,5 +364,15 @@ public void Tune(ITunableDb.TuneType type) tunableDb.Tune(type); } } + + public IIterator GetIterator(bool ordered = false) + { + return _currentDb.GetIterator(ordered); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + return _currentDb.GetIterator(ref options); + } } } diff --git a/src/Nethermind/Nethermind.Db/IDb.cs b/src/Nethermind/Nethermind.Db/IDb.cs index 73f9e4ba1718..bf7ca37f51f3 100644 --- a/src/Nethermind/Nethermind.Db/IDb.cs +++ b/src/Nethermind/Nethermind.Db/IDb.cs @@ -14,10 +14,23 @@ public interface IDb : IKeyValueStoreWithBatching, IDbMeta, IDisposable IEnumerable> GetAll(bool ordered = false); IEnumerable GetAllKeys(bool ordered = false); IEnumerable GetAllValues(bool ordered = false); - + IIterator GetIterator(bool ordered = false); + IIterator GetIterator(ref IteratorOptions options); public IReadOnlyDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyDb(this, createInMemWriteStore); } + public ref struct IteratorOptions + { + public byte[]? LowerBound { get; init; } + public byte[]? UpperBound { get; init; } + + /// + /// Whether to create a tailing operator. + /// + /// https://github.com/facebook/rocksdb/wiki/Tailing-Iterator + public bool Ordered { get; init; } + } + // Some metadata options public interface IDbMeta { diff --git a/src/Nethermind/Nethermind.Db/IIterator.cs b/src/Nethermind/Nethermind.Db/IIterator.cs new file mode 100644 index 000000000000..fcde9e0dfebe --- /dev/null +++ b/src/Nethermind/Nethermind.Db/IIterator.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Db; + +public interface IIterator : IDisposable +{ + void SeekToFirst(); + void Seek(ReadOnlySpan key); + void SeekForPrev(ReadOnlySpan key); + void Next(); + void Prev(); + bool Valid(); + ReadOnlySpan Key(); + ReadOnlySpan Value(); +} diff --git a/src/Nethermind/Nethermind.Db/IMergeOperator.cs b/src/Nethermind/Nethermind.Db/IMergeOperator.cs new file mode 100644 index 000000000000..7df2946bd71f --- /dev/null +++ b/src/Nethermind/Nethermind.Db/IMergeOperator.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Collections; + +namespace Nethermind.Db; + +public interface IMergeOperator +{ + string Name { get; } + ArrayPoolList? FullMerge(ReadOnlySpan key, RocksDbMergeEnumerator enumerator); + ArrayPoolList? PartialMerge(ReadOnlySpan key, RocksDbMergeEnumerator enumerator); +} diff --git a/src/Nethermind/Nethermind.Db/InMemoryWriteBatch.cs b/src/Nethermind/Nethermind.Db/InMemoryWriteBatch.cs index 7c43ec39039c..c8054ac8b6d2 100644 --- a/src/Nethermind/Nethermind.Db/InMemoryWriteBatch.cs +++ b/src/Nethermind/Nethermind.Db/InMemoryWriteBatch.cs @@ -29,10 +29,20 @@ public void Dispose() GC.SuppressFinalize(this); } + public void Clear() + { + _currentItems.Clear(); + } + public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) { _currentItems[key.ToArray()] = value; _writeFlags = flags; } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException("Merging is not supported by this implementation."); + } } } diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index af6f4313e070..731a38eb077d 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -168,6 +168,21 @@ public IDbMeta.DbMetric GatherMetric(bool includeSharedCache = false) }; } + public IIterator GetIterator(bool ordered = false) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException("Merging is not supported by this implementation."); + } + private IEnumerable> OrderedDb => _db.OrderBy(kvp => kvp.Key, Bytes.Comparer); } } diff --git a/src/Nethermind/Nethermind.Db/NullDb.cs b/src/Nethermind/Nethermind.Db/NullDb.cs index 1bb3948fda07..bfebd0ccdd9b 100644 --- a/src/Nethermind/Nethermind.Db/NullDb.cs +++ b/src/Nethermind/Nethermind.Db/NullDb.cs @@ -60,5 +60,20 @@ public IWriteBatch StartWriteBatch() public void Dispose() { } + + public IIterator GetIterator(bool ordered = false) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException("Merging is not supported by this implementation."); + } } } diff --git a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs index ff73f260a3a5..93b5a58c27e3 100644 --- a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs +++ b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs @@ -87,5 +87,20 @@ public void PutSpan(ReadOnlySpan keyBytes, ReadOnlySpan value, Write public void DangerousReleaseMemory(in ReadOnlySpan span) { } public bool PreferWriteByArray => true; // Because of memdb buffer + + public IIterator GetIterator(bool ordered = false) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + _memDb.Merge(key, value, flags); + } } } diff --git a/src/Nethermind/Nethermind.Db/RocksDbMergeEnumerator.cs b/src/Nethermind/Nethermind.Db/RocksDbMergeEnumerator.cs new file mode 100644 index 000000000000..fa2da2060b30 --- /dev/null +++ b/src/Nethermind/Nethermind.Db/RocksDbMergeEnumerator.cs @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Db; + +/// +/// RocksDB enumerator for values of merge operation. +/// +// Interface was not used because of ref struct limitations. +public readonly ref struct RocksDbMergeEnumerator(ReadOnlySpan operandsList, ReadOnlySpan operandsListLength) +{ + private readonly ReadOnlySpan _operandsList = operandsList; + private readonly ReadOnlySpan _operandsListLength = operandsListLength; + + public Span ExistingValue { get; } + public bool HasExistingValue { get; } + public int OperandsCount => _operandsList.Length; + public int TotalCount => OperandsCount + (HasExistingValue ? 1 : 0); + + public RocksDbMergeEnumerator( + Span existingValue, bool hasExistingValue, + ReadOnlySpan operandsList, ReadOnlySpan operandsListLength + ) : this(operandsList, operandsListLength) + { + ExistingValue = existingValue; + HasExistingValue = hasExistingValue; + } + + public Span GetExistingValue() + { + return HasExistingValue ? ExistingValue : default; + } + + public unsafe Span GetOperand(int index) + { + return new((void*)_operandsList[index], (int)_operandsListLength[index]); + } + + public Span Get(int index) + { + if (index == 0 && HasExistingValue) + return ExistingValue; + + if (HasExistingValue) + index -= 1; + + return GetOperand(index); + } +} diff --git a/src/Nethermind/Nethermind.Db/RocksDbSettings.cs b/src/Nethermind/Nethermind.Db/RocksDbSettings.cs index f3d9eebd4e67..020e2c89902d 100644 --- a/src/Nethermind/Nethermind.Db/RocksDbSettings.cs +++ b/src/Nethermind/Nethermind.Db/RocksDbSettings.cs @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; +using System.Collections.Generic; + namespace Nethermind.Db { public class DbSettings @@ -17,6 +20,9 @@ public DbSettings(string name, string path) public bool DeleteOnStart { get; set; } public bool CanDeleteFolder { get; set; } = true; + public IMergeOperator? MergeOperator { get; set; } + public Dictionary? MergeOperatorByColumnFamily { get; set; } + public DbSettings Clone(string name, string path) { DbSettings settings = (DbSettings)MemberwiseClone(); diff --git a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs index 979ff10526d7..df5d1d844065 100644 --- a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs +++ b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs @@ -325,5 +325,20 @@ private byte[] Add(byte[] value) public void Dispose() { } + + public IIterator GetIterator(bool ordered = false) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public IIterator GetIterator(ref IteratorOptions options) + { + throw new NotSupportedException("Iteration is not supported by this implementation."); + } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException("Merging is not supported by this implementation."); + } } } diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs index 02a695382dfd..52a7aff42ec5 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs @@ -161,6 +161,7 @@ public void RemoveReceipts(Block block) #pragma warning disable CS0067 public event EventHandler ReceiptsInserted; + public event EventHandler AnyReceiptsInserted; #pragma warning restore CS0067 } } diff --git a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs index f5267e2b0d22..04046647777a 100644 --- a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs @@ -574,6 +574,11 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF _db[key.ToArray()] = value; } + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException("Merging is not supported by this implementation."); + } + public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { return _db[key.ToArray()]; @@ -601,6 +606,16 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF { _inBatched[key.ToArray()] = value; } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException("Merging is not supported by this implementation."); + } + + public void Clear() + { + throw new NotSupportedException("Clearing is not supported by this implementation."); + } } } diff --git a/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs b/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs index ee9828828594..7340fa96fabf 100644 --- a/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs @@ -25,5 +25,10 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF { throw new NotSupportedException(); } + + public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + { + throw new NotSupportedException(); + } } } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 823d402adea2..1e75ef768163 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -2415,7 +2415,7 @@ private async Task RaiseBlockAddedToMainAndWaitForNewHead(Block block, Block pre : new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject, previousBlock); Task waitTask = Wait.ForEventCondition( - default, + CancellationToken.None, e => _txPool.TxPoolHeadChanged += e, e => _txPool.TxPoolHeadChanged -= e, e => e.Number == block.Number From b8737e692e10ec63a9a375de5c7f5ed460710d88 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Thu, 16 Oct 2025 00:40:01 +0300 Subject: [PATCH 02/18] PR cleanup --- src/Nethermind/Nethermind.Core/ProgressLogger.cs | 5 ++--- src/Nethermind/Nethermind.Db/IDb.cs | 3 ++- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/ProgressLogger.cs b/src/Nethermind/Nethermind.Core/ProgressLogger.cs index 0129ac64d514..589bb60f90c4 100644 --- a/src/Nethermind/Nethermind.Core/ProgressLogger.cs +++ b/src/Nethermind/Nethermind.Core/ProgressLogger.cs @@ -160,8 +160,7 @@ public void SetFormat(Func formatter) _formatter = formatter; } - // TODO: should it actually ever be true? - public void LogProgress(bool resetCompletion = true) + public void LogProgress() { (long, long, long, long) reportState = (CurrentValue, TargetValue, CurrentQueued, _skipped); if (reportState != _lastReportState) @@ -170,7 +169,7 @@ public void LogProgress(bool resetCompletion = true) _lastReportState = reportState; _logger.Info(reportString); } - SetMeasuringPoint(resetCompletion); + SetMeasuringPoint(resetCompletion: false); } private string DefaultFormatter() diff --git a/src/Nethermind/Nethermind.Db/IDb.cs b/src/Nethermind/Nethermind.Db/IDb.cs index bf7ca37f51f3..8c0db4b03e55 100644 --- a/src/Nethermind/Nethermind.Db/IDb.cs +++ b/src/Nethermind/Nethermind.Db/IDb.cs @@ -25,7 +25,8 @@ public ref struct IteratorOptions public byte[]? UpperBound { get; init; } /// - /// Whether to create a tailing operator. + /// If false will create a tailing operator, + /// otherwise will use a regular one. /// /// https://github.com/facebook/rocksdb/wiki/Tailing-Iterator public bool Ordered { get; init; } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 1e75ef768163..823d402adea2 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -2415,7 +2415,7 @@ private async Task RaiseBlockAddedToMainAndWaitForNewHead(Block block, Block pre : new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject, previousBlock); Task waitTask = Wait.ForEventCondition( - CancellationToken.None, + default, e => _txPool.TxPoolHeadChanged += e, e => _txPool.TxPoolHeadChanged -= e, e => e.Number == block.Number From 9db5e50c9ef8d8a2da9aa1d7e525a66320e598a6 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Thu, 16 Oct 2025 12:17:07 +0300 Subject: [PATCH 03/18] Move Merge to separate interface --- .../FullPruning/FullPrunerTests.cs | 5 ----- .../Nethermind.Core/IKeyValueStore.cs | 6 +++++- src/Nethermind/Nethermind.Core/IWriteBatch.cs | 2 +- src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs | 2 +- .../Nethermind.Db.Rocks/DbOnTheRocks.cs | 2 +- src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs | 5 ----- src/Nethermind/Nethermind.Db/CompressingDb.cs | 5 ----- .../Nethermind.Db/FullPruning/FullPruningDb.cs | 17 +---------------- src/Nethermind/Nethermind.Db/MemDb.cs | 5 ----- src/Nethermind/Nethermind.Db/ReadOnlyDb.cs | 5 ----- .../Pruning/TreeStoreTests.cs | 5 ----- 11 files changed, 9 insertions(+), 50 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs index 04f7bb54956f..c13e1e2482cc 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs @@ -411,11 +411,6 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF _context.Set(key, value, flags); } - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - _context.Merge(key, value, flags); - } - public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { return _context.Get(key, flags); diff --git a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs index 76ce78f66ef0..97ac8c9bef99 100644 --- a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs @@ -59,7 +59,6 @@ public interface IWriteOnlyKeyValueStore } void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None); - void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None); /// /// Some store keep the input array directly. (eg: CachingStore), and therefore passing the value by array @@ -70,6 +69,11 @@ public interface IWriteOnlyKeyValueStore void Remove(ReadOnlySpan key) => Set(key, null); } + public interface IMergeableKeyValueStore + { + void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None); + } + public interface ISortedKeyValueStore : IKeyValueStore { byte[]? FirstKey { get; } diff --git a/src/Nethermind/Nethermind.Core/IWriteBatch.cs b/src/Nethermind/Nethermind.Core/IWriteBatch.cs index 5ba4d5d6427e..852dc132caa9 100644 --- a/src/Nethermind/Nethermind.Core/IWriteBatch.cs +++ b/src/Nethermind/Nethermind.Core/IWriteBatch.cs @@ -5,7 +5,7 @@ namespace Nethermind.Core { - public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore + public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore, IMergeableKeyValueStore { void Clear(); } diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs index 6c55bea1dd46..60697a8d8f33 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs @@ -11,7 +11,7 @@ namespace Nethermind.Db.Rocks; -public class ColumnDb : IDb, ISortedKeyValueStore +public class ColumnDb : IDb, ISortedKeyValueStore, IMergeableKeyValueStore { private readonly RocksDb _rocksDb; internal readonly DbOnTheRocks _mainDb; diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index df768ed6ba1a..8461b824c5a0 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -29,7 +29,7 @@ namespace Nethermind.Db.Rocks; -public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStore, ISortedKeyValueStore +public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStore, ISortedKeyValueStore, IMergeableKeyValueStore { protected ILogger _logger; diff --git a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs index 12612cfc1419..4ed4478be963 100644 --- a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs +++ b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs @@ -61,11 +61,6 @@ public byte[] Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) public KeyValuePair[] this[byte[][] keys] => keys.Select(k => new KeyValuePair(k, GetThroughRpc(k))).ToArray(); - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - throw new InvalidOperationException("RPC DB does not support writes"); - } - public void Remove(ReadOnlySpan key) { throw new InvalidOperationException("RPC DB does not support writes"); diff --git a/src/Nethermind/Nethermind.Db/CompressingDb.cs b/src/Nethermind/Nethermind.Db/CompressingDb.cs index 30491cadfc20..f3456ff6aaa5 100644 --- a/src/Nethermind/Nethermind.Db/CompressingDb.cs +++ b/src/Nethermind/Nethermind.Db/CompressingDb.cs @@ -140,11 +140,6 @@ public IEnumerable GetAllKeys(bool ordered = false) => public IEnumerable GetAllValues(bool ordered = false) => _wrapped.GetAllValues(ordered).Select(Decompress); - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - _wrapped.Merge(key, value, flags); - } - public void Remove(ReadOnlySpan key) => _wrapped.Remove(key); public bool KeyExists(ReadOnlySpan key) => _wrapped.KeyExists(key); diff --git a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs index b2804999c4c1..28b51264190b 100755 --- a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs +++ b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs @@ -91,16 +91,6 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags } } - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - _currentDb.Merge(key, value, flags); // we are writing to the main DB - IDb? cloningDb = _pruningContext?.CloningDb; - if (cloningDb is not null) // if pruning is in progress we are also writing to the secondary, copied DB - { - DuplicateMerge(cloningDb, key, value, flags); - } - } - private void Duplicate(IWriteOnlyKeyValueStore db, ReadOnlySpan key, byte[]? value, WriteFlags flags) { db.Set(key, value, flags); @@ -113,7 +103,7 @@ private void Duplicate(IWriteOnlyKeyValueStore db, ReadOnlySpan key, ReadO _updateDuplicateWriteMetrics?.Invoke(); } - private void DuplicateMerge(IWriteOnlyKeyValueStore db, ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags) + private void DuplicateMerge(IMergeableKeyValueStore db, ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags) { db.Merge(key, value, flags); _updateDuplicateWriteMetrics?.Invoke(); @@ -264,11 +254,6 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF _db.Duplicate(CloningDb, key, value, flags); } - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - _db.Merge(key, value, flags); - } - public IWriteBatch StartWriteBatch() { return CloningDb.StartWriteBatch(); diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index 731a38eb077d..7c783ddfceeb 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -178,11 +178,6 @@ public IIterator GetIterator(ref IteratorOptions options) throw new NotSupportedException("Iteration is not supported by this implementation."); } - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - throw new NotSupportedException("Merging is not supported by this implementation."); - } - private IEnumerable> OrderedDb => _db.OrderBy(kvp => kvp.Key, Bytes.Comparer); } } diff --git a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs index 93b5a58c27e3..bfe495b2c805 100644 --- a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs +++ b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs @@ -97,10 +97,5 @@ public IIterator GetIterator(ref IteratorOptions options) { throw new NotSupportedException("Iteration is not supported by this implementation."); } - - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - _memDb.Merge(key, value, flags); - } } } diff --git a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs index 04046647777a..ae8f2ac02ec4 100644 --- a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs @@ -574,11 +574,6 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF _db[key.ToArray()] = value; } - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - throw new NotSupportedException("Merging is not supported by this implementation."); - } - public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { return _db[key.ToArray()]; From ba8a0157fe61223f73c322fc8dd01a92d2fd7d3c Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Thu, 16 Oct 2025 16:05:14 +0300 Subject: [PATCH 04/18] PR feedback # Conflicts: # src/Nethermind/Nethermind.Db/LogIndex/LogIndexStorage.cs --- .../Json/ConverterTestBase.cs | 6 +++--- .../Collections/CollectionExtensions.cs | 15 --------------- .../Collections/DictionaryExtensions.cs | 12 ++++++++++++ .../Nethermind.Core/Extensions/Bytes.cs | 5 +---- .../Nethermind.Db.Rocks/DbOnTheRocks.cs | 11 ++++++----- src/Nethermind/Nethermind.Db/RocksDbSettings.cs | 2 +- 6 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs b/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs index 6425a9321510..8725b3750dba 100644 --- a/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs +++ b/src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs @@ -14,7 +14,7 @@ public class ConverterTestBase protected void TestConverter(T? item, Func equalityComparer, JsonConverter converter) => TestConverter(item, converter, equalityComparer); - protected void TestConverter(R? item, JsonConverter converter, Func? equalityComparer = null) + protected void TestConverter(TItem? item, JsonConverter converter, Func? equalityComparer = null) { var options = new JsonSerializerOptions { @@ -26,10 +26,10 @@ protected void TestConverter(R? item, JsonConverter converter, Func(result, options); + TItem? deserialized = JsonSerializer.Deserialize(result, options); #pragma warning disable CS8604 - if (equalityComparer != null) + if (equalityComparer is not null) Assert.That(equalityComparer(item, deserialized), Is.True); else if (item is IEnumerable itemE && deserialized is IEnumerable deserializedE) Assert.That(deserializedE, Is.EquivalentTo(itemE)); diff --git a/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs b/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs index 6408673b22dd..4a5f5f476a39 100644 --- a/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs @@ -87,21 +87,6 @@ public static bool NoResizeClear(this ConcurrentDictionary(this Dictionary dictionary, TKey key, Func factory) - where TKey : notnull - { - if (dictionary.TryGetValue(key, out TValue? value)) - { - return value; - } - else - { - value = factory(key); - dictionary.Add(key, value); - return value; - } - } - private static class ClearCache where TKey : notnull { public static readonly Action> Clear = CreateNoResizeClearExpression(); diff --git a/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs b/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs index e036bc568d54..1172180fc1b6 100644 --- a/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; using System.Runtime.InteropServices; @@ -13,4 +14,15 @@ public static void Increment(this Dictionary dictionary, TKey k ref int res = ref CollectionsMarshal.GetValueRefOrAddDefault(dictionary, key, out bool _); res++; } + + public static TValue GetOrAdd(this Dictionary dictionary, TKey key, Func factory) + where TKey : notnull + { + ref TValue? existing = ref CollectionsMarshal.GetValueRefOrAddDefault(dictionary, key, out bool exists); + + if (!exists) + existing = factory(key); + + return existing!; + } } diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 549a200b6889..70a069aab399 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -279,10 +279,7 @@ public static byte[] PadRight(this byte[] bytes, int length, byte padding = 0) if (padding != 0) { - for (int i = bytes.Length; i < length; i++) - { - result[i] = padding; - } + result.AsSpan(bytes.Length, length - bytes.Length).Fill(padding); } return result; diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index 8461b824c5a0..73a71d371cc0 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -66,10 +66,10 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor private readonly DbSettings _settings; - // ReSharper disable once CollectionNeverQueried.Local // Need to keep options from GC in case of merge operator applied, as they are used in callback - // TODO: find better way? - private readonly ConcurrentBag _doNotGcOptions = []; + // ReSharper disable once CollectionNeverQueried.Local + private readonly List _doNotGcOptions = []; + private readonly Lock _doNotGcOptionsLocker = new(); private readonly IRocksDbConfig _perTableDbConfig; private ulong _maxBytesForLevelBase; @@ -160,7 +160,7 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan ColumnFamilyOptions options = new(); IRocksDbConfig columnConfig = _rocksDbConfigFactory.GetForDatabase(Name, columnFamily); - IMergeOperator? mergeOperator = _settings.MergeOperatorByColumnFamily?.GetValueOrDefault(enumColumnName); + IMergeOperator? mergeOperator = _settings.ColumnsMergeOperators?.GetValueOrDefault(enumColumnName); BuildOptions(columnConfig, options, sharedCache, mergeOperator); // "default" is a special column name with rocksdb, which is what previously not specifying column goes to @@ -586,7 +586,8 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio if (mergeOperator is not null) { options.SetMergeOperator(new MergeOperatorAdapter(mergeOperator)); - _doNotGcOptions.Add(options); + lock(_doNotGcOptionsLocker) + _doNotGcOptions.Add(options); } #endregion diff --git a/src/Nethermind/Nethermind.Db/RocksDbSettings.cs b/src/Nethermind/Nethermind.Db/RocksDbSettings.cs index 020e2c89902d..d3ee2b073f39 100644 --- a/src/Nethermind/Nethermind.Db/RocksDbSettings.cs +++ b/src/Nethermind/Nethermind.Db/RocksDbSettings.cs @@ -21,7 +21,7 @@ public DbSettings(string name, string path) public bool CanDeleteFolder { get; set; } = true; public IMergeOperator? MergeOperator { get; set; } - public Dictionary? MergeOperatorByColumnFamily { get; set; } + public Dictionary? ColumnsMergeOperators { get; set; } public DbSettings Clone(string name, string path) { From 89a8a0fbbca5692c284d1a0cf1d200077db11ec5 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Fri, 17 Oct 2025 15:27:39 +0300 Subject: [PATCH 05/18] Receipts events renaming # Conflicts: # src/Nethermind/Nethermind.Facade/Find/LogIndexBuilder.cs --- .../ReceiptCanonicalityMonitor.cs | 4 ++-- .../Nethermind.Blockchain/Receipts/IReceiptStorage.cs | 4 ++-- .../Receipts/InMemoryReceiptStorage.cs | 8 ++++---- .../Receipts/NullReceiptStorage.cs | 4 ++-- .../Receipts/PersistentReceiptStorage.cs | 8 ++++---- .../Modules/SubscribeModuleTests.cs | 10 +++++----- .../Ethereum/Steps/Migrations/ReceiptMigrationTests.cs | 4 ++-- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs b/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs index 4d6b55283d95..eea17931dabe 100644 --- a/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs +++ b/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs @@ -25,7 +25,7 @@ public ReceiptCanonicalityMonitor(IReceiptStorage? receiptStorage, ILogManager? { _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _receiptStorage.ReceiptsInserted += OnBlockAddedToMain; + _receiptStorage.NewCanonicalReceipts += OnBlockAddedToMain; } private void OnBlockAddedToMain(object sender, BlockReplacementEventArgs e) @@ -55,7 +55,7 @@ private void TriggerReceiptInsertedEvent(Block newBlock, Block? previousBlock) public void Dispose() { - _receiptStorage.ReceiptsInserted -= OnBlockAddedToMain; + _receiptStorage.NewCanonicalReceipts -= OnBlockAddedToMain; } } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs index daa79cf05188..92823392c424 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs @@ -21,11 +21,11 @@ public interface IReceiptStorage : IReceiptFinder /// /// Receipts for a new main block are inserted /// - event EventHandler ReceiptsInserted; + event EventHandler NewCanonicalReceipts; /// /// Any receipts are inserted /// - event EventHandler AnyReceiptsInserted; + event EventHandler ReceiptsInserted; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs index ffdf56d1b8b0..9aacb6cf9436 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs @@ -18,8 +18,8 @@ public class InMemoryReceiptStorage : IReceiptStorage private readonly ConcurrentDictionary _transactions = new(); #pragma warning disable CS0067 - public event EventHandler ReceiptsInserted; - public event EventHandler? AnyReceiptsInserted; + public event EventHandler NewCanonicalReceipts; + public event EventHandler? ReceiptsInserted; #pragma warning restore CS0067 public InMemoryReceiptStorage(bool allowReceiptIterator = true, IBlockTree? blockTree = null) @@ -33,7 +33,7 @@ public InMemoryReceiptStorage(bool allowReceiptIterator = true, IBlockTree? bloc private void BlockTree_BlockAddedToMain(object? sender, BlockReplacementEventArgs e) { EnsureCanonical(e.Block); - ReceiptsInserted?.Invoke(this, e); + NewCanonicalReceipts?.Invoke(this, e); } public Hash256 FindBlockHash(Hash256 txHash) @@ -75,7 +75,7 @@ public void Insert(Block block, TxReceipt[] txReceipts, IReleaseSpec spec, bool EnsureCanonical(block); } - AnyReceiptsInserted?.Invoke(this, new(block.Header, txReceipts)); + ReceiptsInserted?.Invoke(this, new(block.Header, txReceipts)); } public bool HasBlock(long blockNumber, Hash256 hash) diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs index fd73788fcdbb..0a36ecc2fc1c 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs @@ -13,8 +13,8 @@ public class NullReceiptStorage : IReceiptStorage public static NullReceiptStorage Instance { get; } = new(); #pragma warning disable CS0067 - public event EventHandler ReceiptsInserted; - public event EventHandler? AnyReceiptsInserted; + public event EventHandler NewCanonicalReceipts; + public event EventHandler? ReceiptsInserted; #pragma warning restore CS0067 public Hash256? FindBlockHash(Hash256 hash) => null; diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 9b9dafe6c2af..5a64d4a41320 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -38,8 +38,8 @@ public class PersistentReceiptStorage : IReceiptStorage private const int CacheSize = 64; private readonly LruCache _receiptsCache = new(CacheSize, CacheSize, "receipts"); - public event EventHandler ReceiptsInserted; - public event EventHandler? AnyReceiptsInserted; + public event EventHandler NewCanonicalReceipts; + public event EventHandler? ReceiptsInserted; public PersistentReceiptStorage( IColumnsDb receiptsDb, @@ -74,7 +74,7 @@ public PersistentReceiptStorage( private void BlockTreeOnBlockAddedToMain(object? sender, BlockReplacementEventArgs e) { EnsureCanonical(e.Block); - ReceiptsInserted?.Invoke(this, e); + NewCanonicalReceipts?.Invoke(this, e); // Dont block main loop Task.Run(() => @@ -290,7 +290,7 @@ public void Insert(Block block, TxReceipt[]? txReceipts, IReleaseSpec spec, bool EnsureCanonical(block, lastBlockNumber); } - AnyReceiptsInserted?.Invoke(this, new(block.Header, txReceipts)); + ReceiptsInserted?.Invoke(this, new(block.Header, txReceipts)); } public long MigratedBlockNumber diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs index e05324b2d805..64c5f12b8676 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs @@ -128,7 +128,7 @@ private JsonRpcResult GetBlockAddedToMainResult(BlockReplacementEventArgs blockR })); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockReplacementEventArgs); - _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockReplacementEventArgs); + _receiptStorage.NewCanonicalReceipts += Raise.EventWith(new object(), blockReplacementEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(1000)).Should().Be(shouldReceiveResult); subscriptionId = newHeadSubscription.Id; @@ -148,7 +148,7 @@ private List GetLogsSubscriptionResult(Filter filter, BlockReplac })); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockEventArgs); - _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockEventArgs); + _receiptStorage.NewCanonicalReceipts += Raise.EventWith(new object(), blockEventArgs); semaphoreSlim.Wait(TimeSpan.FromMilliseconds(500)); subscriptionId = logsSubscription.Id; @@ -769,7 +769,7 @@ public void LogsSubscription_should_not_send_logs_of_new_txs_on_ReceiptsInserted BlockReplacementEventArgs blockEventArgs = new(block); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockEventArgs); - _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockEventArgs); + _receiptStorage.NewCanonicalReceipts += Raise.EventWith(new object(), blockEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(200)); jsonRpcResults.Count.Should().Be(1); @@ -949,7 +949,7 @@ public async Task MultipleSubscriptions_concurrent_fast_messages(int messages) { BlockReplacementEventArgs eventArgs = new(Build.A.Block.TestObject); blockTree.BlockAddedToMain += Raise.EventWith(eventArgs); - _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), eventArgs); + _receiptStorage.NewCanonicalReceipts += Raise.EventWith(new object(), eventArgs); } }); @@ -1204,7 +1204,7 @@ public void LogsSubscription_can_send_logs_with_removed_txs_when_inserted() })); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockEventArgs); - _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockEventArgs); + _receiptStorage.NewCanonicalReceipts += Raise.EventWith(new object(), blockEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(2000)); jsonRpcResults.Count.Should().Be(1); diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs index 52a7aff42ec5..6fb33121c458 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs @@ -160,8 +160,8 @@ public void RemoveReceipts(Block block) } #pragma warning disable CS0067 - public event EventHandler ReceiptsInserted; - public event EventHandler AnyReceiptsInserted; + public event EventHandler NewCanonicalReceipts; + public event EventHandler ReceiptsInserted; #pragma warning restore CS0067 } } From 603fe1074d8abd5bb564fcdd6a2eef18b339dbb3 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Fri, 17 Oct 2025 15:49:09 +0300 Subject: [PATCH 06/18] PR feedback --- src/Nethermind/Nethermind.Core/FakeWriteBatch.cs | 5 +---- .../Nethermind.Core/Utils/ConcurrentWriteBatcher.cs | 5 +++-- src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs b/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs index a4edf7862f91..1fcf153998a1 100644 --- a/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs +++ b/src/Nethermind/Nethermind.Core/FakeWriteBatch.cs @@ -37,9 +37,6 @@ public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags f throw new NotSupportedException("Merging is not supported by this implementation."); } - public void Clear() - { - throw new NotSupportedException("Clearing is not supported by this implementation."); - } + public void Clear() { } } } diff --git a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs index 4d01fd2f4466..3041deb7eeab 100644 --- a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs +++ b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs @@ -49,8 +49,7 @@ public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags f public void Clear() { - if (_disposing) - throw new InvalidOperationException("Batch is already disposed."); + ThrowIfDisposed(); foreach (IWriteBatch batch in _batches) { @@ -88,4 +87,6 @@ private IWriteBatch RentWriteBatch() return currentBatch; } + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposing, this); } diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs index 4338bc6d397b..88d204be059f 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs @@ -61,7 +61,7 @@ private static IReadOnlyList GetEnumKeys(IReadOnlyList keys) return keys; } - protected override void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache, IMergeOperator? mergeOperator) + protected override void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache, IMergeOperator? mergeOperator) { base.BuildOptions(dbConfig, options, sharedCache, mergeOperator); options.SetCreateMissingColumnFamilies(); From c943cc8346c416bbca8f5f5832f706b7706aaff8 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sat, 18 Oct 2025 16:01:57 +0300 Subject: [PATCH 07/18] Formatting --- src/Nethermind/Nethermind.Core/Extensions/Bytes.cs | 2 +- src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 70a069aab399..e6a5f23545bf 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -279,7 +279,7 @@ public static byte[] PadRight(this byte[] bytes, int length, byte padding = 0) if (padding != 0) { - result.AsSpan(bytes.Length, length - bytes.Length).Fill(padding); + result.AsSpan(bytes.Length, length - bytes.Length).Fill(padding); } return result; diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index 73a71d371cc0..a9fc1fc800b5 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -586,7 +586,7 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio if (mergeOperator is not null) { options.SetMergeOperator(new MergeOperatorAdapter(mergeOperator)); - lock(_doNotGcOptionsLocker) + lock (_doNotGcOptionsLocker) _doNotGcOptions.Add(options); } From a878d04a634d787212c8df8171bc044b14dc50ca Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sat, 18 Oct 2025 16:06:12 +0300 Subject: [PATCH 08/18] Code cleanup --- src/Nethermind/Nethermind.Db/NullDb.cs | 5 ----- src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs | 5 ----- src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs | 5 ----- 3 files changed, 15 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/NullDb.cs b/src/Nethermind/Nethermind.Db/NullDb.cs index bfebd0ccdd9b..8af73f9a3189 100644 --- a/src/Nethermind/Nethermind.Db/NullDb.cs +++ b/src/Nethermind/Nethermind.Db/NullDb.cs @@ -70,10 +70,5 @@ public IIterator GetIterator(ref IteratorOptions options) { throw new NotSupportedException("Iteration is not supported by this implementation."); } - - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - throw new NotSupportedException("Merging is not supported by this implementation."); - } } } diff --git a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs index df5d1d844065..7a176cd678e5 100644 --- a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs +++ b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs @@ -335,10 +335,5 @@ public IIterator GetIterator(ref IteratorOptions options) { throw new NotSupportedException("Iteration is not supported by this implementation."); } - - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - throw new NotSupportedException("Merging is not supported by this implementation."); - } } } diff --git a/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs b/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs index 7340fa96fabf..ee9828828594 100644 --- a/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Trie/NullKeyValueStore.cs @@ -25,10 +25,5 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF { throw new NotSupportedException(); } - - public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) - { - throw new NotSupportedException(); - } } } From b747de83ba023d4aac3dc62da1e7741f13d9ba93 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sat, 18 Oct 2025 16:10:59 +0300 Subject: [PATCH 09/18] Code cleanup --- src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs index 3041deb7eeab..5d582ca4b319 100644 --- a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs +++ b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Concurrent; +using System.Diagnostics; using System.Threading; namespace Nethermind.Core.Utils; @@ -88,5 +89,6 @@ private IWriteBatch RentWriteBatch() return currentBatch; } + [StackTraceHidden] private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposing, this); } From f9e1b73d3f03de08a8008f7cc8f5447e60af307b Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sun, 19 Oct 2025 01:18:15 +0300 Subject: [PATCH 10/18] Fix DB config validation --- src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index e6e7b9501831..548bde045abe 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -38,7 +38,7 @@ private void EnsureConfigIsAvailable(string propertyName) foreach (var prefix in _prefixes) { string prefixed = string.Concat(prefix, propertyName); - if (GetProperty(type, prefixed, caseSensitive: true) is null) + if (GetProperty(type, prefixed) is null) { throw new InvalidConfigurationException($"Configuration {propertyName} not available with prefix {prefix}. Add {prefix}{propertyName} to {nameof(IDbConfig)}.", -1); } From 32b5e9899a3c1e67bfeee218cc9b8a44249baeb8 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sun, 19 Oct 2025 01:07:26 +0300 Subject: [PATCH 11/18] Use sorted view instead of iterator # Conflicts: # src/Nethermind/Nethermind.Db/LogIndex/LogIndexStorage.cs --- src/Nethermind/Nethermind.Core/IKeyValueStore.cs | 1 + src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs index 97ac8c9bef99..d60f58189b23 100644 --- a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs @@ -87,6 +87,7 @@ public interface ISortedKeyValueStore : IKeyValueStore /// public interface ISortedView : IDisposable { + public bool StartBefore(ReadOnlySpan value); public bool MoveNext(); public ReadOnlySpan CurrentKey { get; } public ReadOnlySpan CurrentValue { get; } diff --git a/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs b/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs index e617000d5320..a64fe85d5b49 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs @@ -22,6 +22,15 @@ public void Dispose() _iterator.Dispose(); } + public bool StartBefore(ReadOnlySpan value) + { + if (_started) + throw new InvalidOperationException($"{nameof(StartBefore)} can only be called before starting iteration."); + + _iterator.SeekForPrev(value); + return _started = _iterator.Valid(); + } + public bool MoveNext() { if (!_started) From 2275555d7373629238158f9ef0eb9cadc6b746c9 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sun, 19 Oct 2025 01:15:14 +0300 Subject: [PATCH 12/18] Do not publicly expose iterator --- .../Nethermind.Db.Rocks/ColumnDb.cs | 10 ----- .../Nethermind.Db.Rocks/DbOnTheRocks.cs | 39 +------------------ .../RocksDbIteratorWrapper.cs | 21 ---------- src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs | 10 ----- src/Nethermind/Nethermind.Db/CompressingDb.cs | 10 ----- .../FullPruning/FullPruningDb.cs | 10 ----- src/Nethermind/Nethermind.Db/IDb.cs | 15 ------- src/Nethermind/Nethermind.Db/IIterator.cs | 18 --------- src/Nethermind/Nethermind.Db/MemDb.cs | 10 ----- src/Nethermind/Nethermind.Db/NullDb.cs | 10 ----- src/Nethermind/Nethermind.Db/ReadOnlyDb.cs | 10 ----- .../Nethermind.Db/SimpleFilePublicKeyDb.cs | 10 ----- 12 files changed, 2 insertions(+), 171 deletions(-) delete mode 100644 src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs delete mode 100644 src/Nethermind/Nethermind.Db/IIterator.cs diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs index 60697a8d8f33..f1271ebe84e2 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs @@ -193,14 +193,4 @@ public ISortedView GetViewBetween(ReadOnlySpan firstKey, ReadOnlySpan key) return GetAllCore(iterator); } - protected internal Iterator CreateIterator(bool ordered, ColumnFamilyHandle? ch = null) - { - return CreateIterator(new IteratorOptions { Ordered = ordered }, ch); - } - - protected internal Iterator CreateIterator(IteratorOptions options, ColumnFamilyHandle? ch = null) + protected internal Iterator CreateIterator(bool ordered = false, ColumnFamilyHandle? ch = null) { ReadOptions readOptions = new(); - readOptions.SetTailing(!options.Ordered); - - if (options.LowerBound is { } lowerBound) - readOptions.SetIterateLowerBound(lowerBound); - - if (options.UpperBound is { } upperBound) - readOptions.SetIterateUpperBound(upperBound); - + readOptions.SetTailing(!ordered); return CreateIterator(readOptions, ch); } @@ -1759,29 +1747,6 @@ private static IDictionary GetBlobFilesOptions() }; } - public IIterator GetIterator(bool ordered = false) - { - var iterator = CreateIterator(ordered); - return new RocksDbIteratorWrapper(iterator); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - return GetIterator(ref options, null); - } - - public IIterator GetIterator(bool ordered, ColumnFamilyHandle familyHandle) - { - var options = new IteratorOptions { Ordered = ordered }; - return GetIterator(ref options, familyHandle); - } - - public IIterator GetIterator(ref IteratorOptions options, ColumnFamilyHandle? familyHandle) - { - var iterator = CreateIterator(options, familyHandle); - return new RocksDbIteratorWrapper(iterator); - } - /// /// Iterators should not be kept for long as it will pin some memory block and sst file. This would show up as /// temporary higher disk usage or memory usage. diff --git a/src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs b/src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs deleted file mode 100644 index 9909a4283720..000000000000 --- a/src/Nethermind/Nethermind.Db.Rocks/RocksDbIteratorWrapper.cs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using RocksDbSharp; -using System; - -namespace Nethermind.Db.Rocks; - -public sealed class RocksDbIteratorWrapper(Iterator iterator) : IIterator -{ - public void SeekToFirst() => iterator.SeekToFirst(); - public void Seek(ReadOnlySpan key) => iterator.Seek(key); - public void SeekForPrev(ReadOnlySpan key) => iterator.SeekForPrev(key); - public void Next() => iterator.Next(); - public void Prev() => iterator.Prev(); - public bool Valid() => iterator.Valid(); - public ReadOnlySpan Key() => iterator.GetKeySpan(); - public ReadOnlySpan Value() => iterator.GetValueSpan(); - public void Dispose() => iterator.Dispose(); -} - diff --git a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs index 4ed4478be963..98279f33c4ad 100644 --- a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs +++ b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs @@ -119,15 +119,5 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags public void DangerousReleaseMemory(in ReadOnlySpan span) { } - - public IIterator GetIterator(bool ordered = false) - { - throw new InvalidOperationException("RPC DB does not support iteration"); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - throw new InvalidOperationException("RPC DB does not support iteration"); - } } } diff --git a/src/Nethermind/Nethermind.Db/CompressingDb.cs b/src/Nethermind/Nethermind.Db/CompressingDb.cs index f3456ff6aaa5..fdfa590993f7 100644 --- a/src/Nethermind/Nethermind.Db/CompressingDb.cs +++ b/src/Nethermind/Nethermind.Db/CompressingDb.cs @@ -176,16 +176,6 @@ public void Tune(ITunableDb.TuneType type) if (_wrapped is ITunableDb tunable) tunable.Tune(type); } - - public IIterator GetIterator(bool ordered = false) - { - return _wrapped.GetIterator(ordered); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - return _wrapped.GetIterator(ref options); - } } } } diff --git a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs index 28b51264190b..f79c010c9889 100755 --- a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs +++ b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs @@ -349,15 +349,5 @@ public void Tune(ITunableDb.TuneType type) tunableDb.Tune(type); } } - - public IIterator GetIterator(bool ordered = false) - { - return _currentDb.GetIterator(ordered); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - return _currentDb.GetIterator(ref options); - } } } diff --git a/src/Nethermind/Nethermind.Db/IDb.cs b/src/Nethermind/Nethermind.Db/IDb.cs index 8c0db4b03e55..4ef54c1f85d7 100644 --- a/src/Nethermind/Nethermind.Db/IDb.cs +++ b/src/Nethermind/Nethermind.Db/IDb.cs @@ -14,24 +14,9 @@ public interface IDb : IKeyValueStoreWithBatching, IDbMeta, IDisposable IEnumerable> GetAll(bool ordered = false); IEnumerable GetAllKeys(bool ordered = false); IEnumerable GetAllValues(bool ordered = false); - IIterator GetIterator(bool ordered = false); - IIterator GetIterator(ref IteratorOptions options); public IReadOnlyDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyDb(this, createInMemWriteStore); } - public ref struct IteratorOptions - { - public byte[]? LowerBound { get; init; } - public byte[]? UpperBound { get; init; } - - /// - /// If false will create a tailing operator, - /// otherwise will use a regular one. - /// - /// https://github.com/facebook/rocksdb/wiki/Tailing-Iterator - public bool Ordered { get; init; } - } - // Some metadata options public interface IDbMeta { diff --git a/src/Nethermind/Nethermind.Db/IIterator.cs b/src/Nethermind/Nethermind.Db/IIterator.cs deleted file mode 100644 index fcde9e0dfebe..000000000000 --- a/src/Nethermind/Nethermind.Db/IIterator.cs +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; - -namespace Nethermind.Db; - -public interface IIterator : IDisposable -{ - void SeekToFirst(); - void Seek(ReadOnlySpan key); - void SeekForPrev(ReadOnlySpan key); - void Next(); - void Prev(); - bool Valid(); - ReadOnlySpan Key(); - ReadOnlySpan Value(); -} diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index 7c783ddfceeb..af6f4313e070 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -168,16 +168,6 @@ public IDbMeta.DbMetric GatherMetric(bool includeSharedCache = false) }; } - public IIterator GetIterator(bool ordered = false) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } - private IEnumerable> OrderedDb => _db.OrderBy(kvp => kvp.Key, Bytes.Comparer); } } diff --git a/src/Nethermind/Nethermind.Db/NullDb.cs b/src/Nethermind/Nethermind.Db/NullDb.cs index 8af73f9a3189..1bb3948fda07 100644 --- a/src/Nethermind/Nethermind.Db/NullDb.cs +++ b/src/Nethermind/Nethermind.Db/NullDb.cs @@ -60,15 +60,5 @@ public IWriteBatch StartWriteBatch() public void Dispose() { } - - public IIterator GetIterator(bool ordered = false) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } } } diff --git a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs index bfe495b2c805..ff73f260a3a5 100644 --- a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs +++ b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs @@ -87,15 +87,5 @@ public void PutSpan(ReadOnlySpan keyBytes, ReadOnlySpan value, Write public void DangerousReleaseMemory(in ReadOnlySpan span) { } public bool PreferWriteByArray => true; // Because of memdb buffer - - public IIterator GetIterator(bool ordered = false) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } } } diff --git a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs index 7a176cd678e5..979ff10526d7 100644 --- a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs +++ b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs @@ -325,15 +325,5 @@ private byte[] Add(byte[] value) public void Dispose() { } - - public IIterator GetIterator(bool ordered = false) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } - - public IIterator GetIterator(ref IteratorOptions options) - { - throw new NotSupportedException("Iteration is not supported by this implementation."); - } } } From d58e55662b6f8fead6c1c33c7481cc048e978f8f Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sun, 19 Oct 2025 01:35:23 +0300 Subject: [PATCH 13/18] Code cleanup --- src/Nethermind/Nethermind.Db/IDb.cs | 1 + .../Nethermind.Trie.Test/Pruning/TreeStoreTests.cs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/IDb.cs b/src/Nethermind/Nethermind.Db/IDb.cs index 4ef54c1f85d7..73f9e4ba1718 100644 --- a/src/Nethermind/Nethermind.Db/IDb.cs +++ b/src/Nethermind/Nethermind.Db/IDb.cs @@ -14,6 +14,7 @@ public interface IDb : IKeyValueStoreWithBatching, IDbMeta, IDisposable IEnumerable> GetAll(bool ordered = false); IEnumerable GetAllKeys(bool ordered = false); IEnumerable GetAllValues(bool ordered = false); + public IReadOnlyDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyDb(this, createInMemWriteStore); } diff --git a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs index ae8f2ac02ec4..ded8904a96ce 100644 --- a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; @@ -604,12 +605,13 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) { - throw new NotSupportedException("Merging is not supported by this implementation."); + var keyArr = key.ToArray(); + _inBatched[keyArr] = (_inBatched.GetValueOrDefault(keyArr) ?? []).Concat(value.ToArray()).ToArray(); } public void Clear() { - throw new NotSupportedException("Clearing is not supported by this implementation."); + _inBatched.Clear(); } } } From 330e6a019a2edbb79118933c5eda0af28cdbafed Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sun, 19 Oct 2025 01:40:56 +0300 Subject: [PATCH 14/18] Revert changes to DB config reading --- src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index 548bde045abe..e6e7b9501831 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -38,7 +38,7 @@ private void EnsureConfigIsAvailable(string propertyName) foreach (var prefix in _prefixes) { string prefixed = string.Concat(prefix, propertyName); - if (GetProperty(type, prefixed) is null) + if (GetProperty(type, prefixed, caseSensitive: true) is null) { throw new InvalidConfigurationException($"Configuration {propertyName} not available with prefix {prefix}. Add {prefix}{propertyName} to {nameof(IDbConfig)}.", -1); } From 70156c23effaa3c025de76869c7161e3b3e7f08c Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sun, 19 Oct 2025 17:20:02 +0300 Subject: [PATCH 15/18] PR feedback --- .../Nethermind.Blockchain/Receipts/IReceiptStorage.cs | 7 +++++-- .../Receipts/InMemoryReceiptStorage.cs | 2 +- .../Nethermind.Blockchain/Receipts/NullReceiptStorage.cs | 2 +- .../Receipts/PersistentReceiptStorage.cs | 2 +- src/Nethermind/Nethermind.Core/IKeyValueStore.cs | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs index 92823392c424..0cb6d561c4e4 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs @@ -19,13 +19,16 @@ public interface IReceiptStorage : IReceiptFinder void RemoveReceipts(Block block); /// - /// Receipts for a new main block are inserted + /// Receipts for canonical chain changed. /// event EventHandler NewCanonicalReceipts; /// - /// Any receipts are inserted + /// Receipts for any block are inserted. /// + /// + /// This is invoked for both canonical and non-canonical blocks. + /// event EventHandler ReceiptsInserted; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs index 9aacb6cf9436..a8824cb66a4c 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs @@ -18,7 +18,7 @@ public class InMemoryReceiptStorage : IReceiptStorage private readonly ConcurrentDictionary _transactions = new(); #pragma warning disable CS0067 - public event EventHandler NewCanonicalReceipts; + public event EventHandler? NewCanonicalReceipts; public event EventHandler? ReceiptsInserted; #pragma warning restore CS0067 diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs index 0a36ecc2fc1c..7ed7bb5637ae 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs @@ -13,7 +13,7 @@ public class NullReceiptStorage : IReceiptStorage public static NullReceiptStorage Instance { get; } = new(); #pragma warning disable CS0067 - public event EventHandler NewCanonicalReceipts; + public event EventHandler? NewCanonicalReceipts; public event EventHandler? ReceiptsInserted; #pragma warning restore CS0067 diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 5a64d4a41320..5f2cbd740052 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -38,7 +38,7 @@ public class PersistentReceiptStorage : IReceiptStorage private const int CacheSize = 64; private readonly LruCache _receiptsCache = new(CacheSize, CacheSize, "receipts"); - public event EventHandler NewCanonicalReceipts; + public event EventHandler? NewCanonicalReceipts; public event EventHandler? ReceiptsInserted; public PersistentReceiptStorage( diff --git a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs index d60f58189b23..52a6b97b317c 100644 --- a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs @@ -69,7 +69,7 @@ public interface IWriteOnlyKeyValueStore void Remove(ReadOnlySpan key) => Set(key, null); } - public interface IMergeableKeyValueStore + public interface IMergeableKeyValueStore : IWriteOnlyKeyValueStore { void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None); } From 74f92522ceee57ac10f55942878e9eb0e9189f3a Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Sun, 19 Oct 2025 17:33:24 +0300 Subject: [PATCH 16/18] PR feedback --- src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index b5d4e26025f9..a69d62475f0c 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -69,7 +69,6 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor // Need to keep options from GC in case of merge operator applied, as they are used in callback // ReSharper disable once CollectionNeverQueried.Local private readonly List _doNotGcOptions = []; - private readonly Lock _doNotGcOptionsLocker = new(); private readonly IRocksDbConfig _perTableDbConfig; private ulong _maxBytesForLevelBase; @@ -586,7 +585,7 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio if (mergeOperator is not null) { options.SetMergeOperator(new MergeOperatorAdapter(mergeOperator)); - lock (_doNotGcOptionsLocker) + lock (_doNotGcOptions) _doNotGcOptions.Add(options); } From 06fef18161634438b792d3d38d89930c402737b7 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Mon, 20 Oct 2025 20:04:13 +0300 Subject: [PATCH 17/18] PR feedback # Conflicts: # src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs # src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs --- .../Receipts/IReceiptStorage.cs | 4 ++-- .../Nethermind.Core/Utils/ConcurrentWriteBatcher.cs | 11 +---------- src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs | 2 +- .../Nethermind.Db.Rocks/Config/PerTableDbConfig.cs | 12 ++++-------- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs index 0cb6d561c4e4..dd7caa7366b7 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs @@ -21,7 +21,7 @@ public interface IReceiptStorage : IReceiptFinder /// /// Receipts for canonical chain changed. /// - event EventHandler NewCanonicalReceipts; + event EventHandler? NewCanonicalReceipts; /// /// Receipts for any block are inserted. @@ -29,6 +29,6 @@ public interface IReceiptStorage : IReceiptFinder /// /// This is invoked for both canonical and non-canonical blocks. /// - event EventHandler ReceiptsInserted; + event EventHandler? ReceiptsInserted; } } diff --git a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs index 5d582ca4b319..a6c80ed33fc7 100644 --- a/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs +++ b/src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Concurrent; -using System.Diagnostics; using System.Threading; namespace Nethermind.Core.Utils; @@ -50,12 +49,7 @@ public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags f public void Clear() { - ThrowIfDisposed(); - - foreach (IWriteBatch batch in _batches) - { - batch.Clear(); - } + throw new NotSupportedException($"{nameof(ConcurrentWriteBatcher)} can not be cancelled."); } public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) @@ -88,7 +82,4 @@ private IWriteBatch RentWriteBatch() return currentBatch; } - - [StackTraceHidden] - private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposing, this); } diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs index 88d204be059f..d5a240d1e968 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs @@ -142,7 +142,7 @@ public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteF public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) { - _writeBatch._writeBatch.Merge(key, value, flags); + _writeBatch._writeBatch.Merge(key, value, _column._columnFamily, flags); } } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index e6e7b9501831..0efa1fe7553d 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -38,7 +38,7 @@ private void EnsureConfigIsAvailable(string propertyName) foreach (var prefix in _prefixes) { string prefixed = string.Concat(prefix, propertyName); - if (GetProperty(type, prefixed, caseSensitive: true) is null) + if (GetProperty(type, prefixed) is null) { throw new InvalidConfigurationException($"Configuration {propertyName} not available with prefix {prefix}. Add {prefix}{propertyName} to {nameof(IDbConfig)}.", -1); } @@ -90,7 +90,7 @@ private static string ReadRocksdbOptions(IDbConfig dbConfig, string propertyName Type type = dbConfig.GetType(); PropertyInfo? propertyInfo; - string val = (string)GetProperty(type, propertyName, caseSensitive: true)!.GetValue(dbConfig)!; + string val = (string)GetProperty(type, propertyName)!.GetValue(dbConfig)!; foreach (var prefix in prefixes) { @@ -156,10 +156,6 @@ private static string ReadRocksdbOptions(IDbConfig dbConfig, string propertyName } } - private static PropertyInfo? GetProperty(Type type, string name, bool caseSensitive = false) - { - BindingFlags flags = BindingFlags.Public | BindingFlags.Instance; - if (!caseSensitive) flags |= BindingFlags.IgnoreCase; - return type.GetProperty(name, flags); - } + private static PropertyInfo? GetProperty(Type type, string name) => + type.GetProperty(name, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase); } From 5b493a95d0a21c90d4b423c4758c6b93146a7529 Mon Sep 17 00:00:00 2001 From: Alex Bespalov Date: Tue, 21 Oct 2025 19:45:11 +0300 Subject: [PATCH 18/18] PR feedback --- .../Collections/DictionaryExtensions.cs | 11 +++++++++-- .../JsonConverters/ByteArrayConverter.cs | 9 +++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs b/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs index 1172180fc1b6..ddb32f7f8bf4 100644 --- a/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs @@ -15,14 +15,21 @@ public static void Increment(this Dictionary dictionary, TKey k res++; } - public static TValue GetOrAdd(this Dictionary dictionary, TKey key, Func factory) + public static TValue GetOrAdd(this Dictionary dictionary, + TKey key, Func factory, + out bool exists) where TKey : notnull { - ref TValue? existing = ref CollectionsMarshal.GetValueRefOrAddDefault(dictionary, key, out bool exists); + ref TValue? existing = ref CollectionsMarshal.GetValueRefOrAddDefault(dictionary, key, out exists); if (!exists) existing = factory(key); return existing!; } + + public static TValue GetOrAdd(this Dictionary dictionary, + TKey key, Func factory) + where TKey : notnull => + GetOrAdd(dictionary, key, factory, out _); } diff --git a/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs b/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs index ce19f03d1fbb..27b535d096f2 100644 --- a/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs +++ b/src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs @@ -177,7 +177,7 @@ public static void Convert(ref Utf8JsonReader reader, scoped Span span) private static void ThrowFormatException() => throw new FormatException(); [DoesNotReturn, StackTraceHidden] - private static Exception ThrowInvalidOperationException() => throw new InvalidOperationException(); + private static void ThrowInvalidOperationException() => throw new InvalidOperationException(); public override void Write( Utf8JsonWriter writer, @@ -253,7 +253,12 @@ public static void Convert( public override byte[] ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - return Convert(ref reader) ?? throw ThrowInvalidOperationException(); + byte[]? result = Convert(ref reader); + + if (result is null) + ThrowInvalidOperationException(); + + return result; } public override void WriteAsPropertyName(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options)