Log index (preparation)

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs

@@ -411,6 +411,11 @@ public void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteF
                 _context.Set(key, value, flags);
             }
 
+            public void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None)
+            {
+                _context.Merge(key, value, flags);
+            }
+
             public byte[]? Get(ReadOnlySpan<byte> key, ReadFlags flags = ReadFlags.None)
             {
                 return _context.Get(key, flags);

src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs

@@ -19,8 +19,13 @@ public interface IReceiptStorage : IReceiptFinder
         void RemoveReceipts(Block block);
 
         /// <summary>
-        /// Receipts for a block are inserted
+        /// Receipts for a new main block are inserted
         /// </summary>
         event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+
+        /// <summary>
+        /// Any receipts are inserted
+        /// </summary>
+        event EventHandler<ReceiptsEventArgs> AnyReceiptsInserted;
     }
 }

src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs

@@ -19,6 +19,7 @@ public class InMemoryReceiptStorage : IReceiptStorage
 
 #pragma warning disable CS0067
         public event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+        public event EventHandler<ReceiptsEventArgs>? AnyReceiptsInserted;
 #pragma warning restore CS0067
 
         public InMemoryReceiptStorage(bool allowReceiptIterator = true, IBlockTree? blockTree = null)
@@ -73,6 +74,8 @@ public void Insert(Block block, TxReceipt[] txReceipts, IReleaseSpec spec, bool
             {
                 EnsureCanonical(block);
             }
+
+            AnyReceiptsInserted?.Invoke(this, new(block.Header, txReceipts));
         }
 
         public bool HasBlock(long blockNumber, Hash256 hash)

src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs

@@ -14,6 +14,7 @@ public class NullReceiptStorage : IReceiptStorage
 
 #pragma warning disable CS0067
         public event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+        public event EventHandler<ReceiptsEventArgs>? AnyReceiptsInserted;
 #pragma warning restore CS0067
 
         public Hash256? FindBlockHash(Hash256 hash) => null;

src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs

@@ -39,6 +39,7 @@ public class PersistentReceiptStorage : IReceiptStorage
         private readonly LruCache<ValueHash256, TxReceipt[]> _receiptsCache = new(CacheSize, CacheSize, "receipts");
 
         public event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+        public event EventHandler<ReceiptsEventArgs>? AnyReceiptsInserted;
 
         public PersistentReceiptStorage(
             IColumnsDb<ReceiptsColumns> receiptsDb,
@@ -288,6 +289,8 @@ public void Insert(Block block, TxReceipt[]? txReceipts, IReleaseSpec spec, bool
             {
                 EnsureCanonical(block, lastBlockNumber);
             }
+
+            AnyReceiptsInserted?.Invoke(this, new(block.Header, txReceipts));
         }
 
         public long MigratedBlockNumber

src/Nethermind/Nethermind.Core.Test/Builders/RandomExtensions.cs

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using Nethermind.Crypto;
+
+namespace Nethermind.Core.Test.Builders;
+
+public static class RandomExtensions
+{
+    public static T NextFrom<T>(this Random random, IReadOnlyList<T> values) => values[random.Next(values.Count)];
+
+    public static T NextFrom<T>(this ICryptoRandom random, IReadOnlyList<T> values) => values[random.NextInt(values.Count)];
+}

src/Nethermind/Nethermind.Core.Test/Json/ByteArrayConverterTests.cs

@@ -8,6 +8,8 @@
 using System.Text.Json;
 using FluentAssertions;
 using Nethermind.Core.Extensions;
+using Nethermind.Core.Test.Builders;
+using Nethermind.Crypto;
 using Nethermind.Serialization.Json;
 using NUnit.Framework;
 
@@ -294,6 +296,22 @@ public void Fuzz_RandomHex_SegmentationInvariant()
         }
     }
 
+    [Test]
+    public void Test_DictionaryKey()
+    {
+        var random = new CryptoRandom();
+        var dictionary = new Dictionary<byte[], int?>
+        {
+            { Bytes.FromHexString("0x0"), null },
+            { Bytes.FromHexString("0x1"), random.NextInt(int.MaxValue) },
+            { Build.An.Address.TestObject.Bytes, random.NextInt(int.MaxValue) },
+            { random.GenerateRandomBytes(10), random.NextInt(int.MaxValue) },
+            { random.GenerateRandomBytes(32), random.NextInt(int.MaxValue) },
+        };
+
+        TestConverter(dictionary, new ByteArrayConverter());
+    }
+
     private static ReadOnlySequence<byte> JsonForLiteral(string literal) =>
         MakeSequence(Encoding.UTF8.GetBytes(literal));
 

src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Collections;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using NUnit.Framework;
@@ -10,7 +11,10 @@ namespace Nethermind.Core.Test.Json;
 
 public class ConverterTestBase<T>
 {
-    protected void TestConverter(T? item, Func<T, T, bool> equalityComparer, JsonConverter<T> converter)
+    protected void TestConverter(T? item, Func<T, T, bool> equalityComparer, JsonConverter<T> converter) =>
+        TestConverter(item, converter, equalityComparer);
+
+    protected void TestConverter<R>(R? item, JsonConverter<T> converter, Func<R, R, bool>? equalityComparer = null)
     {
         var options = new JsonSerializerOptions
         {
@@ -22,10 +26,15 @@ protected void TestConverter(T? item, Func<T, T, bool> equalityComparer, JsonCon
 
         string result = JsonSerializer.Serialize(item, options);
 
-        T? deserialized = JsonSerializer.Deserialize<T>(result, options);
+        R? deserialized = JsonSerializer.Deserialize<R>(result, options);
 
 #pragma warning disable CS8604
-        Assert.That(equalityComparer(item, deserialized), Is.True);
+        if (equalityComparer != null)
+            Assert.That(equalityComparer(item, deserialized), Is.True);
+        else if (item is IEnumerable itemE && deserialized is IEnumerable deserializedE)
+            Assert.That(deserializedE, Is.EquivalentTo(itemE));
+        else
+            Assert.That(deserialized, Is.EqualTo(item));
 #pragma warning restore CS8604
     }
 }

src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs

@@ -87,6 +87,21 @@ public static bool NoResizeClear<TKey, TValue>(this ConcurrentDictionary<TKey, T
             return true;
         }
 
+        public static TValue GetOrAdd<TKey, TValue>(this Dictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> factory)
+            where TKey : notnull
+        {
+            if (dictionary.TryGetValue(key, out TValue? value))
+            {
+                return value;
+            }
+            else
+            {
+                value = factory(key);
+                dictionary.Add(key, value);
+                return value;
+            }
+        }
+
         private static class ClearCache<TKey, TValue> where TKey : notnull
         {
             public static readonly Action<ConcurrentDictionary<TKey, TValue>> Clear = CreateNoResizeClearExpression();

src/Nethermind/Nethermind.Core/Extensions/Bytes.cs

@@ -262,7 +262,7 @@ public static byte[] PadLeft(this ReadOnlySpan<byte> bytes, int length, byte pad
             return result;
         }
 
-        public static byte[] PadRight(this byte[] bytes, int length)
+        public static byte[] PadRight(this byte[] bytes, int length, byte padding = 0)
         {
             if (bytes.Length == length)
             {
@@ -276,6 +276,15 @@ public static byte[] PadRight(this byte[] bytes, int length)
 
             byte[] result = new byte[length];
             Buffer.BlockCopy(bytes, 0, result, 0, bytes.Length);
+
+            if (padding != 0)
+            {
+                for (int i = bytes.Length; i < length; i++)
+                {
+                    result[i] = padding;
+                }
+            }
+
             return result;
         }
 

src/Nethermind/Nethermind.Core/Extensions/SizeExtensions.cs

@@ -67,7 +67,7 @@ public static long KiB(this int @this)
             return ((long)@this).KiB();
         }
 
-        public static string SizeToString(this long @this, bool useSi = false, int precision = 1)
+        public static string SizeToString(this long @this, bool useSi = false, bool addSpace = false, int precision = 1)
         {
             string[] suf = useSi ? ["B", "KB", "MB", "GB", "TB"] : ["B", "KiB", "MiB", "GiB", "TiB"];
             if (@this == 0)
@@ -77,7 +77,7 @@ public static string SizeToString(this long @this, bool useSi = false, int preci
             long bytes = Math.Abs(@this);
             int place = Math.Min(suf.Length - 1, Convert.ToInt32(Math.Floor(Math.Log(bytes, useSi ? 1000 : 1024))));
             double num = Math.Round(bytes / Math.Pow(useSi ? 1000 : 1024, place), precision);
-            return (Math.Sign(@this) * num).ToString() + suf[place];
+            return string.Concat(Math.Sign(@this) * num, addSpace ? " " : "", suf[place]);
         }
     }
 }

src/Nethermind/Nethermind.Core/Extensions/WaitHandleExtensions.cs

@@ -23,8 +23,12 @@ public static async Task<bool> WaitOneAsync(this WaitHandle handle, int millisec
                     millisecondsTimeout,
                     true);
                 tokenRegistration = cancellationToken.Register(
-                    static state => ((TaskCompletionSource<bool>)state!).TrySetCanceled(),
-                    tcs);
+                    static state =>
+                    {
+                        var (tcs, ct) = ((TaskCompletionSource<bool> tcs, CancellationToken ct))state!;
+                        tcs.TrySetCanceled(ct);
+                    },
+                    (tcs, cancellationToken));
 
                 return await tcs.Task;
             }

src/Nethermind/Nethermind.Core/FakeWriteBatch.cs

@@ -31,5 +31,15 @@ public void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteF
         {
             _storePretendingToSupportBatches.Set(key, value, flags);
         }
+
+        public void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None)
+        {
+            throw new NotSupportedException("Merging is not supported by this implementation.");
+        }
+
+        public void Clear()
+        {
+            throw new NotSupportedException("Clearing is not supported by this implementation.");
+        }
     }
 }

src/Nethermind/Nethermind.Core/IKeyValueStore.cs

@@ -59,6 +59,7 @@ public interface IWriteOnlyKeyValueStore
         }
 
         void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteFlags.None);
+        void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None);
 
         /// <summary>
         /// Some store keep the input array directly. (eg: CachingStore), and therefore passing the value by array

src/Nethermind/Nethermind.Core/IWriteBatch.cs

@@ -5,5 +5,8 @@
 
 namespace Nethermind.Core
 {
-    public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore;
+    public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore
+    {
+        void Clear();
+    }
 }

src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs

@@ -173,7 +173,7 @@ public static void Convert(ref Utf8JsonReader reader, scoped Span<byte> span)
     private static void ThrowFormatException() => throw new FormatException();
 
     [DoesNotReturn, StackTraceHidden]
-    private static void ThrowInvalidOperationException() => throw new InvalidOperationException();
+    private static Exception ThrowInvalidOperationException() => throw new InvalidOperationException();
 
     public override void Write(
         Utf8JsonWriter writer,
@@ -246,4 +246,14 @@ public static void Convert(
         if (array is not null)
             ArrayPool<byte>.Shared.Return(array);
     }
+
+    public override byte[] ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        return Convert(ref reader) ?? throw ThrowInvalidOperationException();
+    }
+
+    public override void WriteAsPropertyName(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options)
+    {
+        Convert(writer, value, static (w, h) => w.WritePropertyName(h), skipLeadingZeros: false, addQuotations: false, addHexPrefix: true);
+    }
 }

src/Nethermind/Nethermind.Core/ProgressLogger.cs

@@ -47,9 +47,9 @@ public void IncrementSkipped(int skipped = 1)
             Interlocked.Add(ref _skipped, skipped);
         }
 
-        public void SetMeasuringPoint()
+        public void SetMeasuringPoint(bool resetCompletion = true)
         {
-            UtcEndTime = null;
+            if (resetCompletion) UtcEndTime = null;
 
             if (UtcStartTime is not null)
             {
@@ -169,7 +169,7 @@ public void LogProgress()
                 _lastReportState = reportState;
                 _logger.Info(reportString);
             }
-            SetMeasuringPoint();
+            SetMeasuringPoint(resetCompletion: false);
         }
 
         private string DefaultFormatter()

src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs

@@ -40,6 +40,24 @@ public void PutSpan(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags
         ReturnWriteBatch(currentBatch);
     }
 
+    public void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None)
+    {
+        IWriteBatch currentBatch = RentWriteBatch();
+        currentBatch.Merge(key, value, flags);
+        ReturnWriteBatch(currentBatch);
+    }
+
+    public void Clear()
+    {
+        if (_disposing)
+            throw new InvalidOperationException("Batch is already disposed.");
+
+        foreach (IWriteBatch batch in _batches)
+        {
+            batch.Clear();
+        }
+    }
+
     public void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteFlags.None)
     {
         IWriteBatch currentBatch = RentWriteBatch();

src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs

@@ -57,6 +57,11 @@ public void PutSpan(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags
         _mainDb.SetWithColumnFamily(key, _columnFamily, value, writeFlags);
     }
 
+    public void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags writeFlags = WriteFlags.None)
+    {
+        _mainDb.MergeWithColumnFamily(key, _columnFamily, value, writeFlags);
+    }
+
     public KeyValuePair<byte[], byte[]?>[] this[byte[][] keys] =>
         _rocksDb.MultiGet(keys, keys.Select(k => _columnFamily).ToArray());
 
@@ -99,6 +104,11 @@ public void Dispose()
             _underlyingWriteBatch.Dispose();
         }
 
+        public void Clear()
+        {
+            _underlyingWriteBatch.Clear();
+        }
+
         public void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteFlags.None)
         {
             if (value is null)
@@ -115,6 +125,11 @@ public void PutSpan(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags
         {
             _underlyingWriteBatch.Set(key, value, _columnDb._columnFamily, flags);
         }
+
+        public void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None)
+        {
+            _underlyingWriteBatch.Merge(key, value, _columnDb._columnFamily, flags);
+        }
     }
 
     public void Remove(ReadOnlySpan<byte> key)
@@ -130,7 +145,7 @@ public bool KeyExists(ReadOnlySpan<byte> key)
 
     public void Flush(bool onlyWal)
     {
-        _mainDb.Flush(onlyWal);
+        _mainDb.FlushWithColumnFamily(_columnFamily);
     }
 
     public void Compact()
@@ -178,4 +193,14 @@ public ISortedView GetViewBetween(ReadOnlySpan<byte> firstKey, ReadOnlySpan<byte
     {
         return _mainDb.GetViewBetween(firstKey, lastKey, _columnFamily);
     }
+
+    public IIterator GetIterator(bool ordered = false)
+    {
+        return _mainDb.GetIterator(ordered, _columnFamily);
+    }
+
+    public IIterator GetIterator(ref IteratorOptions options)
+    {
+        return _mainDb.GetIterator(ref options, _columnFamily);
+    }
 }