Optimize engine_getBlobs more

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs

@@ -28,6 +28,7 @@
 using Nethermind.Merge.Plugin.Data;
 using Nethermind.Merge.Plugin.GC;
 using Nethermind.Merge.Plugin.Handlers;
+using Nethermind.TxPool.Collections;
 using Nethermind.Merge.Plugin.Synchronization;
 using Nethermind.Serialization.Json;
 using Nethermind.Serialization.Rlp;
@@ -602,8 +603,8 @@ public async Task GetBlobsV1_should_return_requested_blobs([Values(1, 2, 3, 4, 5
         ResultWrapper<IEnumerable<BlobAndProofV1?>> result = await rpcModule.engine_getBlobsV1(blobTx.BlobVersionedHashes!);
 
         ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!;
-        result.Data.Select(static b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs);
-        result.Data.Select(static b => b!.Proof).Should().BeEquivalentTo(wrapper.Proofs);
+        result.Data.Select(static b => b!.Value.Blob).Should().BeEquivalentTo(wrapper.Blobs);
+        result.Data.Select(static b => b!.Value.Proof).Should().BeEquivalentTo(wrapper.Proofs);
     }
 
     [Test]
@@ -673,8 +674,8 @@ public async Task GetBlobsV1_should_return_mix_of_blobs_and_nulls([Values(1, 2,
         {
             if (i % 10 == 0)
             {
-                resultBlobsAndProofs[i]!.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i / 10]);
-                resultBlobsAndProofs[i]!.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i / 10]);
+                resultBlobsAndProofs[i]!.Value.Blob.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Blobs[i / 10]);
+                resultBlobsAndProofs[i]!.Value.Proof.Should().BeEquivalentTo(((ShardBlobNetworkWrapper)blobTx.NetworkWrapper!).Proofs[i / 10]);
             }
             else
             {

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V5.cs

@@ -16,6 +16,7 @@
 using Nethermind.Merge.Plugin.Data;
 using Nethermind.Specs.Forks;
 using Nethermind.TxPool;
+using Nethermind.TxPool.Collections;
 using NUnit.Framework;
 
 namespace Nethermind.Merge.Plugin.Test;
@@ -104,9 +105,9 @@ public async Task GetBlobsV2_should_return_requested_blobs([Values(1, 2, 3, 4, 5
         ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!;
 
         result.Data.Should().NotBeNull();
-        result.Data!.Select(static b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs);
-        result.Data!.Select(static b => b!.Proofs.Length).Should().HaveCount(numberOfBlobs);
-        result.Data!.Select(static b => b!.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
+        result.Data!.Select(static b => b!.Value.Blob).Should().BeEquivalentTo(wrapper.Blobs);
+        result.Data!.Select(static b => b!.Value.Proofs!.Length).Should().HaveCount(numberOfBlobs);
+        result.Data!.Select(static b => b!.Value.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
     }
 
     [Test]
@@ -173,9 +174,9 @@ public async Task GetBlobsV2_should_return_empty_array_when_only_some_blobs_foun
             ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!;
 
             result.Data.Should().NotBeNull();
-            result.Data!.Select(static b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs);
-            result.Data!.Select(static b => b!.Proofs.Length).Should().HaveCount(numberOfBlobs);
-            result.Data!.Select(static b => b!.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
+            result.Data!.Select(static b => b!.Value.Blob).Should().BeEquivalentTo(wrapper.Blobs);
+            result.Data!.Select(static b => b!.Value.Proofs!.Length).Should().HaveCount(numberOfBlobs);
+            result.Data!.Select(static b => b!.Value.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
         }
     }
 
@@ -224,8 +225,8 @@ public async Task GetBlobsV3_should_return_partial_results_with_nulls_for_missin
             if (shouldBeFound)
             {
                 result.Data!.ElementAt(i).Should().NotBeNull();
-                result.Data!.ElementAt(i)!.Blob.Should().BeEquivalentTo(wrapper.Blobs[foundIndex]);
-                result.Data!.ElementAt(i)!.Proofs.Should().BeEquivalentTo(wrapper.Proofs.Skip(foundIndex * 128).Take(128));
+                result.Data!.ElementAt(i)!.Value.Blob.Should().BeEquivalentTo(wrapper.Blobs[foundIndex]);
+                result.Data!.ElementAt(i)!.Value.Proofs.Should().BeEquivalentTo(wrapper.Proofs.Skip(foundIndex * 128).Take(128));
                 foundIndex++;
             }
             else

src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Cancun.cs

@@ -9,6 +9,7 @@
 using Nethermind.JsonRpc;
 using Nethermind.Merge.Plugin.Data;
 using Nethermind.Merge.Plugin.Handlers;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin;
 

src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Osaka.cs

@@ -6,6 +6,7 @@
 using Nethermind.JsonRpc;
 using Nethermind.Merge.Plugin.Data;
 using Nethermind.Merge.Plugin.Handlers;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin;
 

src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs

@@ -11,6 +11,7 @@
 using Nethermind.Merge.Plugin.Data;
 using Nethermind.Merge.Plugin.GC;
 using Nethermind.Merge.Plugin.Handlers;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin;
 

src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandler.cs → src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandlerV1.cs

@@ -3,16 +3,18 @@
 
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Nethermind.Core.Collections;
 using Nethermind.Core.Specs;
 using Nethermind.JsonRpc;
-using Nethermind.Merge.Plugin.Data;
 using Nethermind.TxPool;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin.Handlers;
 
-public class GetBlobsHandler(ITxPool txPool, IChainHeadSpecProvider chainHeadSpecProvider) : IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>
+public class GetBlobsHandlerV1(ITxPool txPool, IChainHeadSpecProvider chainHeadSpecProvider) : IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>
 {
     private const int MaxRequest = 128;
+    private static string _overMaxRequestError = $"The number of requested blobs must not exceed {MaxRequest}";
 
     public Task<ResultWrapper<IEnumerable<BlobAndProofV1?>>> HandleAsync(byte[][] request)
     {
@@ -21,31 +23,27 @@ public class GetBlobsHandler(ITxPool txPool, IChainHeadSpecProvider chainHeadSpe
             return ResultWrapper<IEnumerable<BlobAndProofV1?>>.Fail(MergeErrorMessages.UnsupportedFork, MergeErrorCodes.UnsupportedFork);
         }
 
-        if (request.Length > MaxRequest)
+        int length = request.Length;
+        if (length > MaxRequest)
         {
-            var error = $"The number of requested blobs must not exceed {MaxRequest}";
-            return ResultWrapper<IEnumerable<BlobAndProofV1?>>.Fail(error, MergeErrorCodes.TooLargeRequest);
+            return ResultWrapper<IEnumerable<BlobAndProofV1?>>.Fail(_overMaxRequestError, MergeErrorCodes.TooLargeRequest);
         }
 
-        return ResultWrapper<IEnumerable<BlobAndProofV1?>>.Success(GetBlobsAndProofs(request));
-    }
+        Metrics.NumberOfRequestedBlobs += length;
 
-    private IEnumerable<BlobAndProofV1?> GetBlobsAndProofs(byte[][] request)
-    {
-        bool allBlobsAvailable = true;
-        Metrics.NumberOfRequestedBlobs += request.Length;
+        ArrayPoolList<BlobAndProofV1?> response = new(length);
+        txPool.TryGetBlobsAndProofsV1(request, response);
 
-        foreach (byte[] requestedBlobVersionedHash in request)
+        bool allBlobsAvailable = true;
+        for (int i = 0; i < response.Count; i++)
         {
-            if (txPool.TryGetBlobAndProofV0(requestedBlobVersionedHash, out byte[]? blob, out byte[]? proof))
+            if (response[i] is not null)
             {
                 Metrics.NumberOfSentBlobs++;
-                yield return new BlobAndProofV1(blob, proof);
             }
             else
             {
                 allBlobsAvailable = false;
-                yield return null;
             }
         }
 
@@ -57,5 +55,7 @@ public class GetBlobsHandler(ITxPool txPool, IChainHeadSpecProvider chainHeadSpe
         {
             Metrics.GetBlobsRequestsFailureTotal++;
         }
+
+        return ResultWrapper<IEnumerable<BlobAndProofV1?>>.Success(response);
     }
 }

src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandlerV2.cs

@@ -5,58 +5,47 @@
 using System.Threading.Tasks;
 using Nethermind.Core.Collections;
 using Nethermind.JsonRpc;
-using Nethermind.Merge.Plugin.Data;
 using Nethermind.TxPool;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin.Handlers;
 
 public class GetBlobsHandlerV2(ITxPool txPool) : IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?>
 {
     private const int MaxRequest = 128;
+    private static string _overMaxRequestError = $"The number of requested blobs must not exceed {MaxRequest}";
 
     private static readonly Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> NotFound = Task.FromResult(ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Success(null));
 
     public Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> HandleAsync(GetBlobsHandlerV2Request request)
     {
-        if (request.BlobVersionedHashes.Length > MaxRequest)
+        int length = request.BlobVersionedHashes.Length;
+
+        if (length > MaxRequest)
         {
-            var error = $"The number of requested blobs must not exceed {MaxRequest}";
-            return ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Fail(error, MergeErrorCodes.TooLargeRequest);
+            return ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Fail(_overMaxRequestError, MergeErrorCodes.TooLargeRequest);
         }
 
-        Metrics.GetBlobsRequestsTotal += request.BlobVersionedHashes.Length;
+        Metrics.GetBlobsRequestsTotal += length;
 
-        ArrayPoolList<BlobAndProofV2?> response = new(request.BlobVersionedHashes.Length);
+        ArrayPoolList<BlobAndProofV2?> response = new(length);
+        txPool.TryGetBlobsAndProofsV2(request.BlobVersionedHashes, response);
 
-        try
+        for (int i = 0; i < response.Count; i++)
         {
-            foreach (byte[] requestedBlobVersionedHash in request.BlobVersionedHashes)
+            if (response[i] is not null)
             {
-                if (txPool.TryGetBlobAndProofV1(requestedBlobVersionedHash, out byte[]? blob, out byte[][]? cellProofs))
-                {
-                    response.Add(new BlobAndProofV2(blob, cellProofs));
-                    Metrics.GetBlobsRequestsInBlobpoolTotal++;
-                }
-                else if (request.AllowPartialReturn)
-                {
-                    response.Add(null);
-                }
-                else
-                {
-                    // fail if we were not able to collect full blob data
-                    response.Dispose();
-                    return ReturnEmptyArray();
-                }
+                Metrics.GetBlobsRequestsInBlobpoolTotal++;
+            }
+            else if (!request.AllowPartialReturn)
+            {
+                response.Dispose();
+                return ReturnEmptyArray();
             }
-
-            Metrics.GetBlobsRequestsSuccessTotal++;
-            return ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Success(response);
-        }
-        catch
-        {
-            response.Dispose();
-            throw;
         }
+
+        Metrics.GetBlobsRequestsSuccessTotal++;
+        return ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Success(response);
     }
 
     private Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> ReturnEmptyArray()

src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs

@@ -8,6 +8,7 @@
 using Nethermind.JsonRpc;
 using Nethermind.JsonRpc.Modules;
 using Nethermind.Merge.Plugin.Data;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin;
 

src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Osaka.cs

@@ -6,6 +6,7 @@
 using Nethermind.JsonRpc;
 using Nethermind.JsonRpc.Modules;
 using Nethermind.Merge.Plugin.Data;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin;
 

src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs

@@ -43,6 +43,7 @@
 using Nethermind.Synchronization.ParallelSync;
 using Nethermind.Trie.Pruning;
 using Nethermind.TxPool;
+using Nethermind.TxPool.Collections;
 
 namespace Nethermind.Merge.Plugin;
 
@@ -329,7 +330,7 @@ protected override void Load(ContainerBuilder builder)
                 .AddSingleton<IHandler<TransitionConfigurationV1, TransitionConfigurationV1>, ExchangeTransitionConfigurationV1Handler>()
                 .AddSingleton<IHandler<IEnumerable<string>, IEnumerable<string>>, ExchangeCapabilitiesHandler>()
                     .AddSingleton<IRpcCapabilitiesProvider, EngineRpcCapabilitiesProvider>()
-                .AddSingleton<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>, GetBlobsHandler>()
+                .AddSingleton<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>, GetBlobsHandlerV1>()
                 .AddSingleton<IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?>, GetBlobsHandlerV2>()
 
                 .AddSingleton<NoSyncGcRegionStrategy>()

src/Nethermind/Nethermind.Taiko.Test/TxPoolContentListsTests.cs

@@ -7,6 +7,7 @@
 using Nethermind.Logging;
 using Nethermind.Taiko.Rpc;
 using Nethermind.TxPool;
+using Nethermind.TxPool.Collections;
 using NSubstitute;
 using NUnit.Framework;
 using Nethermind.Core;

src/Nethermind/Nethermind.Taiko/Rpc/TaikoEngineRpcModule.cs

@@ -29,6 +29,7 @@
 using Nethermind.Merge.Plugin.Handlers;
 using Nethermind.Serialization.Rlp;
 using Nethermind.TxPool;
+using Nethermind.TxPool.Collections;
 using Nethermind.Evm.State;
 using Nethermind.Taiko.Config;
 

src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs

@@ -20,6 +20,7 @@
 using Nethermind.Specs.ChainSpecStyle.Json;
 using Nethermind.Specs.Forks;
 using Nethermind.Specs.Test;
+using Nethermind.Core.Collections;
 using Nethermind.TxPool.Collections;
 using NSubstitute;
 using NUnit.Framework;
@@ -865,9 +866,12 @@ public void should_handle_indexing_blobs_when_adding_txs_in_parallel([Values(tru
 
                     blobPool.TryInsert(tx.Hash, tx, out _).Should().BeTrue();
 
+                    using ArrayPoolList<BlobAndProofV1?> results = new(1);
                     for (int j = 0; j < 100; j++)
                     {
-                        blobPool.TryGetBlobAndProofV0(expectedBlobVersionedHash.ToBytes(), out _, out _).Should().BeTrue();
+                        results.Clear();
+                        blobPool.TryGetBlobsAndProofsV1([expectedBlobVersionedHash], results);
+                        results[0].Should().NotBeNull();
                     }
 
                     // removing 50% of txs

src/Nethermind/Nethermind.TxPool/Collections/BlobAndProof.cs

@@ -0,0 +1,14 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.TxPool.Collections;
+
+/// <summary>
+/// Blob and single KZG proof for engine_getBlobsV1 (pre-PeerDAS format).
+/// </summary>
+public readonly record struct BlobAndProofV1(byte[] Blob, byte[] Proof);
+
+/// <summary>
+/// Blob and cell proofs for engine_getBlobsV2 (PeerDAS format).
+/// </summary>
+public readonly record struct BlobAndProofV2(byte[] Blob, byte[][] Proofs);