Log index (preparation)

src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs

@@ -25,7 +25,7 @@ public ReceiptCanonicalityMonitor(IReceiptStorage? receiptStorage, ILogManager?
         {
             _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage));
             _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager));
-            _receiptStorage.ReceiptsInserted += OnBlockAddedToMain;
+            _receiptStorage.NewCanonicalReceipts += OnBlockAddedToMain;
         }
 
         private void OnBlockAddedToMain(object sender, BlockReplacementEventArgs e)
@@ -55,7 +55,7 @@ private void TriggerReceiptInsertedEvent(Block newBlock, Block? previousBlock)
 
         public void Dispose()
         {
-            _receiptStorage.ReceiptsInserted -= OnBlockAddedToMain;
+            _receiptStorage.NewCanonicalReceipts -= OnBlockAddedToMain;
         }
     }
 }

src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs

@@ -19,8 +19,13 @@ public interface IReceiptStorage : IReceiptFinder
         void RemoveReceipts(Block block);
 
         /// <summary>
-        /// Receipts for a block are inserted
+        /// Receipts for a new main block are inserted
         /// </summary>
-        event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+        event EventHandler<BlockReplacementEventArgs> NewCanonicalReceipts;
+
+        /// <summary>
+        /// Any receipts are inserted
+        /// </summary>
+        event EventHandler<ReceiptsEventArgs> ReceiptsInserted;
     }
 }

src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs

@@ -18,7 +18,8 @@ public class InMemoryReceiptStorage : IReceiptStorage
         private readonly ConcurrentDictionary<Hash256AsKey, TxReceipt> _transactions = new();
 
 #pragma warning disable CS0067
-        public event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+        public event EventHandler<BlockReplacementEventArgs> NewCanonicalReceipts;
+        public event EventHandler<ReceiptsEventArgs>? ReceiptsInserted;
 #pragma warning restore CS0067
 
         public InMemoryReceiptStorage(bool allowReceiptIterator = true, IBlockTree? blockTree = null)
@@ -32,7 +33,7 @@ public InMemoryReceiptStorage(bool allowReceiptIterator = true, IBlockTree? bloc
         private void BlockTree_BlockAddedToMain(object? sender, BlockReplacementEventArgs e)
         {
             EnsureCanonical(e.Block);
-            ReceiptsInserted?.Invoke(this, e);
+            NewCanonicalReceipts?.Invoke(this, e);
         }
 
         public Hash256 FindBlockHash(Hash256 txHash)
@@ -73,6 +74,8 @@ public void Insert(Block block, TxReceipt[] txReceipts, IReleaseSpec spec, bool
             {
                 EnsureCanonical(block);
             }
+
+            ReceiptsInserted?.Invoke(this, new(block.Header, txReceipts));
         }
 
         public bool HasBlock(long blockNumber, Hash256 hash)

src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs

@@ -13,7 +13,8 @@ public class NullReceiptStorage : IReceiptStorage
         public static NullReceiptStorage Instance { get; } = new();
 
 #pragma warning disable CS0067
-        public event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+        public event EventHandler<BlockReplacementEventArgs> NewCanonicalReceipts;
+        public event EventHandler<ReceiptsEventArgs>? ReceiptsInserted;
 #pragma warning restore CS0067
 
         public Hash256? FindBlockHash(Hash256 hash) => null;

src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs

@@ -38,7 +38,8 @@ public class PersistentReceiptStorage : IReceiptStorage
         private const int CacheSize = 64;
         private readonly LruCache<ValueHash256, TxReceipt[]> _receiptsCache = new(CacheSize, CacheSize, "receipts");
 
-        public event EventHandler<BlockReplacementEventArgs> ReceiptsInserted;
+        public event EventHandler<BlockReplacementEventArgs> NewCanonicalReceipts;
+        public event EventHandler<ReceiptsEventArgs>? ReceiptsInserted;
 
         public PersistentReceiptStorage(
             IColumnsDb<ReceiptsColumns> receiptsDb,
@@ -73,7 +74,7 @@ public PersistentReceiptStorage(
         private void BlockTreeOnBlockAddedToMain(object? sender, BlockReplacementEventArgs e)
         {
             EnsureCanonical(e.Block);
-            ReceiptsInserted?.Invoke(this, e);
+            NewCanonicalReceipts?.Invoke(this, e);
 
             // Dont block main loop
             Task.Run(() =>
@@ -288,6 +289,8 @@ public void Insert(Block block, TxReceipt[]? txReceipts, IReleaseSpec spec, bool
             {
                 EnsureCanonical(block, lastBlockNumber);
             }
+
+            ReceiptsInserted?.Invoke(this, new(block.Header, txReceipts));
         }
 
         public long MigratedBlockNumber

src/Nethermind/Nethermind.Core.Test/Builders/RandomExtensions.cs

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using Nethermind.Crypto;
+
+namespace Nethermind.Core.Test.Builders;
+
+public static class RandomExtensions
+{
+    public static T NextFrom<T>(this Random random, IReadOnlyList<T> values) => values[random.Next(values.Count)];
+
+    public static T NextFrom<T>(this ICryptoRandom random, IReadOnlyList<T> values) => values[random.NextInt(values.Count)];
+}

src/Nethermind/Nethermind.Core.Test/Json/ByteArrayConverterTests.cs

@@ -8,6 +8,8 @@
 using System.Text.Json;
 using FluentAssertions;
 using Nethermind.Core.Extensions;
+using Nethermind.Core.Test.Builders;
+using Nethermind.Crypto;
 using Nethermind.Serialization.Json;
 using NUnit.Framework;
 
@@ -294,6 +296,22 @@ public void Fuzz_RandomHex_SegmentationInvariant()
         }
     }
 
+    [Test]
+    public void Test_DictionaryKey()
+    {
+        var random = new CryptoRandom();
+        var dictionary = new Dictionary<byte[], int?>
+        {
+            { Bytes.FromHexString("0x0"), null },
+            { Bytes.FromHexString("0x1"), random.NextInt(int.MaxValue) },
+            { Build.An.Address.TestObject.Bytes, random.NextInt(int.MaxValue) },
+            { random.GenerateRandomBytes(10), random.NextInt(int.MaxValue) },
+            { random.GenerateRandomBytes(32), random.NextInt(int.MaxValue) },
+        };
+
+        TestConverter(dictionary, new ByteArrayConverter());
+    }
+
     private static ReadOnlySequence<byte> JsonForLiteral(string literal) =>
         MakeSequence(Encoding.UTF8.GetBytes(literal));
 

src/Nethermind/Nethermind.Core.Test/Json/ConverterTestBase.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Collections;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using NUnit.Framework;
@@ -10,7 +11,10 @@ namespace Nethermind.Core.Test.Json;
 
 public class ConverterTestBase<T>
 {
-    protected void TestConverter(T? item, Func<T, T, bool> equalityComparer, JsonConverter<T> converter)
+    protected void TestConverter(T? item, Func<T, T, bool> equalityComparer, JsonConverter<T> converter) =>
+        TestConverter(item, converter, equalityComparer);
+
+    protected void TestConverter<TItem>(TItem? item, JsonConverter<T> converter, Func<TItem, TItem, bool>? equalityComparer = null)
     {
         var options = new JsonSerializerOptions
         {
@@ -22,10 +26,15 @@ protected void TestConverter(T? item, Func<T, T, bool> equalityComparer, JsonCon
 
         string result = JsonSerializer.Serialize(item, options);
 
-        T? deserialized = JsonSerializer.Deserialize<T>(result, options);
+        TItem? deserialized = JsonSerializer.Deserialize<TItem>(result, options);
 
 #pragma warning disable CS8604
-        Assert.That(equalityComparer(item, deserialized), Is.True);
+        if (equalityComparer is not null)
+            Assert.That(equalityComparer(item, deserialized), Is.True);
+        else if (item is IEnumerable itemE && deserialized is IEnumerable deserializedE)
+            Assert.That(deserializedE, Is.EquivalentTo(itemE));
+        else
+            Assert.That(deserialized, Is.EqualTo(item));
 #pragma warning restore CS8604
     }
 }

src/Nethermind/Nethermind.Core/Collections/DictionaryExtensions.cs

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System;
 using System.Collections.Generic;
 using System.Runtime.InteropServices;
 
@@ -13,4 +14,15 @@ public static void Increment<TKey>(this Dictionary<TKey, int> dictionary, TKey k
         ref int res = ref CollectionsMarshal.GetValueRefOrAddDefault(dictionary, key, out bool _);
         res++;
     }
+
+    public static TValue GetOrAdd<TKey, TValue>(this Dictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> factory)
+        where TKey : notnull
+    {
+        ref TValue? existing = ref CollectionsMarshal.GetValueRefOrAddDefault(dictionary, key, out bool exists);
+
+        if (!exists)
+            existing = factory(key);
+
+        return existing!;
+    }
 }

src/Nethermind/Nethermind.Core/Extensions/Bytes.cs

@@ -262,7 +262,7 @@ public static byte[] PadLeft(this ReadOnlySpan<byte> bytes, int length, byte pad
             return result;
         }
 
-        public static byte[] PadRight(this byte[] bytes, int length)
+        public static byte[] PadRight(this byte[] bytes, int length, byte padding = 0)
         {
             if (bytes.Length == length)
             {
@@ -276,6 +276,12 @@ public static byte[] PadRight(this byte[] bytes, int length)
 
             byte[] result = new byte[length];
             Buffer.BlockCopy(bytes, 0, result, 0, bytes.Length);
+
+            if (padding != 0)
+            {
+                result.AsSpan(bytes.Length, length - bytes.Length).Fill(padding);
+            }
+
             return result;
         }
 

src/Nethermind/Nethermind.Core/Extensions/SizeExtensions.cs

@@ -67,7 +67,7 @@ public static long KiB(this int @this)
             return ((long)@this).KiB();
         }
 
-        public static string SizeToString(this long @this, bool useSi = false, int precision = 1)
+        public static string SizeToString(this long @this, bool useSi = false, bool addSpace = false, int precision = 1)
         {
             string[] suf = useSi ? ["B", "KB", "MB", "GB", "TB"] : ["B", "KiB", "MiB", "GiB", "TiB"];
             if (@this == 0)
@@ -77,7 +77,7 @@ public static string SizeToString(this long @this, bool useSi = false, int preci
             long bytes = Math.Abs(@this);
             int place = Math.Min(suf.Length - 1, Convert.ToInt32(Math.Floor(Math.Log(bytes, useSi ? 1000 : 1024))));
             double num = Math.Round(bytes / Math.Pow(useSi ? 1000 : 1024, place), precision);
-            return (Math.Sign(@this) * num).ToString() + suf[place];
+            return string.Concat(Math.Sign(@this) * num, addSpace ? " " : "", suf[place]);
         }
     }
 }

src/Nethermind/Nethermind.Core/Extensions/WaitHandleExtensions.cs

@@ -23,8 +23,12 @@ public static async Task<bool> WaitOneAsync(this WaitHandle handle, int millisec
                     millisecondsTimeout,
                     true);
                 tokenRegistration = cancellationToken.Register(
-                    static state => ((TaskCompletionSource<bool>)state!).TrySetCanceled(),
-                    tcs);
+                    static state =>
+                    {
+                        var (tcs, ct) = ((TaskCompletionSource<bool> tcs, CancellationToken ct))state!;
+                        tcs.TrySetCanceled(ct);
+                    },
+                    (tcs, cancellationToken));
 
                 return await tcs.Task;
             }

src/Nethermind/Nethermind.Core/FakeWriteBatch.cs

@@ -31,5 +31,12 @@ public void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteF
         {
             _storePretendingToSupportBatches.Set(key, value, flags);
         }
+
+        public void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None)
+        {
+            throw new NotSupportedException("Merging is not supported by this implementation.");
+        }
+
+        public void Clear() { }
     }
 }

src/Nethermind/Nethermind.Core/IKeyValueStore.cs

@@ -69,6 +69,11 @@ public interface IWriteOnlyKeyValueStore
         void Remove(ReadOnlySpan<byte> key) => Set(key, null);
     }
 
+    public interface IMergeableKeyValueStore
+    {
+        void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None);
+    }
+
     public interface ISortedKeyValueStore : IKeyValueStore
     {
         byte[]? FirstKey { get; }

src/Nethermind/Nethermind.Core/IWriteBatch.cs

@@ -5,5 +5,8 @@
 
 namespace Nethermind.Core
 {
-    public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore;
+    public interface IWriteBatch : IDisposable, IWriteOnlyKeyValueStore, IMergeableKeyValueStore
+    {
+        void Clear();
+    }
 }

src/Nethermind/Nethermind.Core/JsonConverters/ByteArrayConverter.cs

@@ -173,7 +173,7 @@ public static void Convert(ref Utf8JsonReader reader, scoped Span<byte> span)
     private static void ThrowFormatException() => throw new FormatException();
 
     [DoesNotReturn, StackTraceHidden]
-    private static void ThrowInvalidOperationException() => throw new InvalidOperationException();
+    private static Exception ThrowInvalidOperationException() => throw new InvalidOperationException();
 
     public override void Write(
         Utf8JsonWriter writer,
@@ -246,4 +246,14 @@ public static void Convert(
         if (array is not null)
             ArrayPool<byte>.Shared.Return(array);
     }
+
+    public override byte[] ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        return Convert(ref reader) ?? throw ThrowInvalidOperationException();
+    }
+
+    public override void WriteAsPropertyName(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options)
+    {
+        Convert(writer, value, static (w, h) => w.WritePropertyName(h), skipLeadingZeros: false, addQuotations: false, addHexPrefix: true);
+    }
 }

src/Nethermind/Nethermind.Core/ProgressLogger.cs

@@ -47,9 +47,9 @@ public void IncrementSkipped(int skipped = 1)
             Interlocked.Add(ref _skipped, skipped);
         }
 
-        public void SetMeasuringPoint()
+        public void SetMeasuringPoint(bool resetCompletion = true)
         {
-            UtcEndTime = null;
+            if (resetCompletion) UtcEndTime = null;
 
             if (UtcStartTime is not null)
             {
@@ -169,7 +169,7 @@ public void LogProgress()
                 _lastReportState = reportState;
                 _logger.Info(reportString);
             }
-            SetMeasuringPoint();
+            SetMeasuringPoint(resetCompletion: false);
         }
 
         private string DefaultFormatter()

src/Nethermind/Nethermind.Core/Utils/ConcurrentWriteBatcher.cs

@@ -40,6 +40,23 @@ public void PutSpan(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags
         ReturnWriteBatch(currentBatch);
     }
 
+    public void Merge(ReadOnlySpan<byte> key, ReadOnlySpan<byte> value, WriteFlags flags = WriteFlags.None)
+    {
+        IWriteBatch currentBatch = RentWriteBatch();
+        currentBatch.Merge(key, value, flags);
+        ReturnWriteBatch(currentBatch);
+    }
+
+    public void Clear()
+    {
+        ThrowIfDisposed();
+
+        foreach (IWriteBatch batch in _batches)
+        {
+            batch.Clear();
+        }
+    }
+
     public void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteFlags.None)
     {
         IWriteBatch currentBatch = RentWriteBatch();
@@ -70,4 +87,6 @@ private IWriteBatch RentWriteBatch()
 
         return currentBatch;
     }
+
+    private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposing, this);
 }