test: add engine blockchain pyspec fixtures and fix BAL gas budget tracking

src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/AmsterdamTestFixture.cs

@@ -29,6 +29,25 @@ public static IEnumerable<BlockchainTest> LoadTests() =>
         }, "fixtures/blockchain_tests/for_amsterdam", typeof(TSelf).GetCustomAttribute<EipWildcardAttribute>()!.Wildcard).LoadTests<BlockchainTest>();
 }
 
+/// <summary>
+/// Generic base for Amsterdam EIP engine blockchain tests.
+/// Wildcard is read from <see cref="EipWildcardAttribute"/> on <typeparamref name="TSelf"/>.
+/// </summary>
+[TestFixture]
+[Parallelizable(ParallelScope.All)]
+public abstract class AmsterdamEngineBlockChainTestFixture<TSelf> : BlockchainTestBase
+{
+    [TestCaseSource(nameof(LoadTests))]
+    public async Task Test(BlockchainTest test) => await RunTest(test);
+
+    public static IEnumerable<BlockchainTest> LoadTests() =>
+        new TestsSourceLoader(new LoadPyspecTestsStrategy
+        {
+            ArchiveVersion = Constants.BalArchiveVersion,
+            ArchiveName = Constants.BalArchiveName
+        }, "fixtures/blockchain_tests_engine/for_amsterdam", typeof(TSelf).GetCustomAttribute<EipWildcardAttribute>()!.Wildcard).LoadTests<BlockchainTest>();
+}
+
 /// <summary>
 /// Generic base for Amsterdam EIP state tests.
 /// Wildcard is read from <see cref="EipWildcardAttribute"/> on <typeparamref name="TSelf"/>.

src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/Tests.cs

@@ -8,24 +8,45 @@ namespace Ethereum.Blockchain.Pyspec.Test.Amsterdam;
 [EipWildcard("eip7708_eth_transfer_logs")]
 public class Eip7708BlockChainTests : AmsterdamBlockChainTestFixture<Eip7708BlockChainTests>;
 
+[EipWildcard("eip7708_eth_transfer_logs")]
+public class Eip7708EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture<Eip7708EngineBlockChainTests>;
+
 [EipWildcard("eip7778_block_gas_accounting_without_refunds")]
 public class Eip7778BlockChainTests : AmsterdamBlockChainTestFixture<Eip7778BlockChainTests>;
 
+[EipWildcard("eip7778_block_gas_accounting_without_refunds")]
+public class Eip7778EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture<Eip7778EngineBlockChainTests>;
+
 [EipWildcard("eip7843_slotnum")]
 public class Eip7843BlockChainTests : AmsterdamBlockChainTestFixture<Eip7843BlockChainTests>;
 
+[EipWildcard("eip7843_slotnum")]
+public class Eip7843EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture<Eip7843EngineBlockChainTests>;
+
 [EipWildcard("eip7928_block_level_access_lists")]
 public class Eip7928BlockChainTests : AmsterdamBlockChainTestFixture<Eip7928BlockChainTests>;
 
+[EipWildcard("eip7928_block_level_access_lists")]
+public class Eip7928EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture<Eip7928EngineBlockChainTests>;
+
 [EipWildcard("eip7954_increase_max_contract_size")]
 public class Eip7954BlockChainTests : AmsterdamBlockChainTestFixture<Eip7954BlockChainTests>;
 
+[EipWildcard("eip7954_increase_max_contract_size")]
+public class Eip7954EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture<Eip7954EngineBlockChainTests>;
+
 [EipWildcard("eip8024_dupn_swapn_exchange")]
 public class Eip8024BlockChainTests : AmsterdamBlockChainTestFixture<Eip8024BlockChainTests>;
 
+[EipWildcard("eip8024_dupn_swapn_exchange")]
+public class Eip8024EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture<Eip8024EngineBlockChainTests>;
+
 [EipWildcard("eip8037_state_creation_gas_cost_increase")]
 public class Eip8037BlockChainTests : AmsterdamBlockChainTestFixture<Eip8037BlockChainTests>;
 
+[EipWildcard("eip8037_state_creation_gas_cost_increase")]
+public class Eip8037EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture<Eip8037EngineBlockChainTests>;
+
 // State tests
 
 [EipWildcard("eip7708_eth_transfer_logs")]

src/Nethermind/Ethereum.Blockchain.Pyspec.Test/PyspecTestFixture.cs

@@ -25,6 +25,22 @@ public static IEnumerable<BlockchainTest> LoadTests() =>
             $"fixtures/blockchain_tests/for_{TestDirectoryHelper.GetDirectoryByConvention<TSelf>("BlockchainTests")}").LoadTests<BlockchainTest>();
 }
 
+/// <summary>
+/// Generic base for pyspec engine blockchain tests using <see cref="LoadPyspecTestsStrategy"/>.
+/// Directory is derived by convention: strip "EngineBlockchainTests" suffix, lowercase.
+/// </summary>
+[TestFixture]
+[Parallelizable(ParallelScope.All)]
+public abstract class PyspecEngineBlockchainTestFixture<TSelf> : BlockchainTestBase
+{
+    [TestCaseSource(nameof(LoadTests))]
+    public async Task Test(BlockchainTest test) => (await RunTest(test)).Pass.Should().BeTrue();
+
+    public static IEnumerable<BlockchainTest> LoadTests() =>
+        new TestsSourceLoader(new LoadPyspecTestsStrategy(),
+            $"fixtures/blockchain_tests_engine/for_{TestDirectoryHelper.GetDirectoryByConvention<TSelf>("EngineBlockchainTests")}").LoadTests<BlockchainTest>();
+}
+
 /// <summary>
 /// Generic base for pyspec state tests using <see cref="LoadPyspecTestsStrategy"/>.
 /// Directory is derived by convention: strip "StateTests" suffix, lowercase.

src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Tests.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using Ethereum.Test.Base;
+using NUnit.Framework;
 
 namespace Ethereum.Blockchain.Pyspec.Test;
 
@@ -27,6 +28,28 @@ public class PragueBlockchainTests : PyspecBlockchainTestFixture<PragueBlockchai
 
 public class OsakaBlockchainTests : PyspecBlockchainTestFixture<OsakaBlockchainTests>;
 
+// Engine blockchain tests - directory derived from class name by convention (strip "EngineBlockchainTests", lowercase)
+
+public class FrontierEngineBlockchainTests : PyspecEngineBlockchainTestFixture<FrontierEngineBlockchainTests>;
+
+public class HomesteadEngineBlockchainTests : PyspecEngineBlockchainTestFixture<HomesteadEngineBlockchainTests>;
+
+public class ByzantiumEngineBlockchainTests : PyspecEngineBlockchainTestFixture<ByzantiumEngineBlockchainTests>;
+
+public class IstanbulEngineBlockchainTests : PyspecEngineBlockchainTestFixture<IstanbulEngineBlockchainTests>;
+
+public class BerlinEngineBlockchainTests : PyspecEngineBlockchainTestFixture<BerlinEngineBlockchainTests>;
+
+public class ParisEngineBlockchainTests : PyspecEngineBlockchainTestFixture<ParisEngineBlockchainTests>;
+
+public class ShanghaiEngineBlockchainTests : PyspecEngineBlockchainTestFixture<ShanghaiEngineBlockchainTests>;
+
+public class CancunEngineBlockchainTests : PyspecEngineBlockchainTestFixture<CancunEngineBlockchainTests>;
+
+public class PragueEngineBlockchainTests : PyspecEngineBlockchainTestFixture<PragueEngineBlockchainTests>;
+
+public class OsakaEngineBlockchainTests : PyspecEngineBlockchainTestFixture<OsakaEngineBlockchainTests>;
+
 // State tests - directory derived from class name by convention (strip "StateTests", lowercase)
 
 public class FrontierStateTests : PyspecStateTestFixture<FrontierStateTests>;

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -313,7 +313,7 @@ private static BlockHeader SuggestBlocks(BlockchainTest test, bool failOnInvalid
 
     private async static Task RunNewPayloads(TestEngineNewPayloadsJson[]? newPayloads, IEngineRpcModule engineRpcModule)
     {
-        (ExecutionPayloadV4, string[]?, string[]?, int, int)[] payloads = [.. JsonToEthereumTest.Convert(newPayloads)];
+        (ExecutionPayload, string[]?, string[]?, int, int)[] payloads = [.. JsonToEthereumTest.Convert(newPayloads)];
 
         // blockchain test engine
         foreach ((ExecutionPayload executionPayload, string[]? blobVersionedHashes, string[]? validationError, int newPayloadVersion, int fcuVersion) in payloads)

src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs

@@ -86,7 +86,7 @@ public static BlockHeader Convert(TestBlockHeaderJson? headerJson)
             return header;
         }
 
-        public static IEnumerable<(ExecutionPayloadV4, string[]?, string[]?, int, int)> Convert(TestEngineNewPayloadsJson[]? executionPayloadsJson)
+        public static IEnumerable<(ExecutionPayload, string[]?, string[]?, int, int)> Convert(TestEngineNewPayloadsJson[]? executionPayloadsJson)
         {
             if (executionPayloadsJson is null)
             {
@@ -99,31 +99,57 @@ public static BlockHeader Convert(TestBlockHeaderJson? headerJson)
                 string[]? blobVersionedHashes = engineNewPayload.Params.Length > 1 ? engineNewPayload.Params[1].Deserialize<string[]?>(EthereumJsonSerializer.JsonOptions) : null;
                 string? parentBeaconBlockRoot = engineNewPayload.Params.Length > 2 ? engineNewPayload.Params[2].Deserialize<string?>(EthereumJsonSerializer.JsonOptions) : null;
                 string[]? validationError = engineNewPayload.Params.Length > 3 ? engineNewPayload.Params[3].Deserialize<string[]?>(EthereumJsonSerializer.JsonOptions) : null;
-                yield return (new ExecutionPayloadV4()
-                {
-                    BaseFeePerGas = (ulong)Bytes.FromHexString(executionPayload.BaseFeePerGas).ToUnsignedBigInteger(),
-                    BlockHash = new(executionPayload.BlockHash),
-                    BlockNumber = (long)Bytes.FromHexString(executionPayload.BlockNumber).ToUnsignedBigInteger(),
-                    ExtraData = Bytes.FromHexString(executionPayload.ExtraData),
-                    FeeRecipient = new(executionPayload.FeeRecipient),
-                    GasLimit = (long)Bytes.FromHexString(executionPayload.GasLimit).ToUnsignedBigInteger(),
-                    GasUsed = (long)Bytes.FromHexString(executionPayload.GasUsed).ToUnsignedBigInteger(),
-                    LogsBloom = new(Bytes.FromHexString(executionPayload.LogsBloom)),
-                    ParentHash = new(executionPayload.ParentHash),
-                    PrevRandao = new(executionPayload.PrevRandao),
-                    ReceiptsRoot = new(executionPayload.ReceiptsRoot),
-                    StateRoot = new(executionPayload.StateRoot),
-                    Timestamp = (ulong)Bytes.FromHexString(executionPayload.Timestamp).ToUnsignedBigInteger(),
-                    BlockAccessList = executionPayload.BlockAccessList is null ? null : Bytes.FromHexString(executionPayload.BlockAccessList),
-                    BlobGasUsed = executionPayload.BlobGasUsed is null ? null : (ulong)Bytes.FromHexString(executionPayload.BlobGasUsed).ToUnsignedBigInteger(),
-                    ExcessBlobGas = executionPayload.ExcessBlobGas is null ? null : (ulong)Bytes.FromHexString(executionPayload.ExcessBlobGas).ToUnsignedBigInteger(),
-                    ParentBeaconBlockRoot = parentBeaconBlockRoot is null ? null : new(parentBeaconBlockRoot),
-                    Withdrawals = executionPayload.Withdrawals is null ? null : [.. executionPayload.Withdrawals.Select(x => Rlp.Decode<Withdrawal>(Bytes.FromHexString(x)))],
-                    SlotNumber = executionPayload.SlotNumber is null ? null : (ulong)Bytes.FromHexString(executionPayload.SlotNumber).ToUnsignedBigInteger(),
-                    Transactions = [.. executionPayload.Transactions.Select(x => Bytes.FromHexString(x))],
-                    ExecutionRequests = []
-                }, blobVersionedHashes, validationError, int.Parse(engineNewPayload.NewPayloadVersion ?? "4"), int.Parse(engineNewPayload.ForkChoiceUpdatedVersion ?? "3"));
+                int newPayloadVersion = int.Parse(engineNewPayload.NewPayloadVersion ?? "4");
+                yield return (CreateExecutionPayload(executionPayload, parentBeaconBlockRoot, newPayloadVersion), blobVersionedHashes, validationError, newPayloadVersion, int.Parse(engineNewPayload.ForkChoiceUpdatedVersion ?? "3"));
+            }
+        }
+
+        private static ExecutionPayload CreateExecutionPayload(
+            TestEngineNewPayloadsJson.ParamsExecutionPayload executionPayload,
+            string? parentBeaconBlockRoot,
+            int newPayloadVersion)
+        {
+            ExecutionPayload payload = newPayloadVersion switch
+            {
+                >= 5 => new ExecutionPayloadV4(),
+                >= 3 => new ExecutionPayloadV3(),
+                _ => new ExecutionPayload(),
+            };
+
+            payload.BaseFeePerGas = (ulong)Bytes.FromHexString(executionPayload.BaseFeePerGas).ToUnsignedBigInteger();
+            payload.BlockHash = new(executionPayload.BlockHash);
+            payload.BlockNumber = (long)Bytes.FromHexString(executionPayload.BlockNumber).ToUnsignedBigInteger();
+            payload.ExtraData = Bytes.FromHexString(executionPayload.ExtraData);
+            payload.FeeRecipient = new(executionPayload.FeeRecipient);
+            payload.GasLimit = (long)Bytes.FromHexString(executionPayload.GasLimit).ToUnsignedBigInteger();
+            payload.GasUsed = (long)Bytes.FromHexString(executionPayload.GasUsed).ToUnsignedBigInteger();
+            payload.LogsBloom = new(Bytes.FromHexString(executionPayload.LogsBloom));
+            payload.ParentHash = new(executionPayload.ParentHash);
+            payload.PrevRandao = new(executionPayload.PrevRandao);
+            payload.ReceiptsRoot = new(executionPayload.ReceiptsRoot);
+            payload.StateRoot = new(executionPayload.StateRoot);
+            payload.Timestamp = (ulong)Bytes.FromHexString(executionPayload.Timestamp).ToUnsignedBigInteger();
+            payload.Withdrawals = executionPayload.Withdrawals is null ? null : [.. executionPayload.Withdrawals.Select(x => Rlp.Decode<Withdrawal>(Bytes.FromHexString(x)))];
+            payload.Transactions = [.. executionPayload.Transactions.Select(x => Bytes.FromHexString(x))];
+            payload.ParentBeaconBlockRoot = parentBeaconBlockRoot is null ? null : new(parentBeaconBlockRoot);
+
+            // Match most-derived first: V4 extends V3, so V4 arm sets both V4 and V3 fields.
+            switch (payload)
+            {
+                case ExecutionPayloadV4 v4:
+                    v4.BlobGasUsed = executionPayload.BlobGasUsed is null ? null : (ulong)Bytes.FromHexString(executionPayload.BlobGasUsed).ToUnsignedBigInteger();
+                    v4.ExcessBlobGas = executionPayload.ExcessBlobGas is null ? null : (ulong)Bytes.FromHexString(executionPayload.ExcessBlobGas).ToUnsignedBigInteger();
+                    v4.BlockAccessList = executionPayload.BlockAccessList is null ? null : Bytes.FromHexString(executionPayload.BlockAccessList);
+                    v4.SlotNumber = executionPayload.SlotNumber is null ? null : (ulong)Bytes.FromHexString(executionPayload.SlotNumber).ToUnsignedBigInteger();
+                    v4.ExecutionRequests = [];
+                    break;
+                case ExecutionPayloadV3 v3:
+                    v3.BlobGasUsed = executionPayload.BlobGasUsed is null ? null : (ulong)Bytes.FromHexString(executionPayload.BlobGasUsed).ToUnsignedBigInteger();
+                    v3.ExcessBlobGas = executionPayload.ExcessBlobGas is null ? null : (ulong)Bytes.FromHexString(executionPayload.ExcessBlobGas).ToUnsignedBigInteger();
+                    break;
             }
+
+            return payload;
         }
 
         public static Transaction Convert(PostStateJson postStateJson, TransactionJson transactionJson)

src/Nethermind/Ethereum.Transaction.Test/TransactionJsonTest.cs

@@ -2,12 +2,16 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using Ethereum.Test.Base;
+using System;
+using System.Linq;
+using System.Text.Json;
 using FluentAssertions;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Eip2930;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Int256;
+using Nethermind.Merge.Plugin.Data;
 using Nethermind.Serialization.Json;
 using Nethermind.Specs.Forks;
 using NUnit.Framework;
@@ -54,6 +58,27 @@ public void Convert_sets_AccessList_type_when_accessLists_field_present_but_empt
             "presence of accessLists field (even empty) should set Type 1");
     }
 
+    [TestCase(1, typeof(ExecutionPayload))]
+    [TestCase(3, typeof(ExecutionPayloadV3))]
+    [TestCase(4, typeof(ExecutionPayloadV3))]
+    [TestCase(5, typeof(ExecutionPayloadV4))]
+    public void Convert_engine_payloads_uses_declared_payload_version(int newPayloadVersion, Type expectedPayloadType)
+    {
+        TestEngineNewPayloadsJson[] payloads =
+        [
+            new()
+            {
+                NewPayloadVersion = newPayloadVersion.ToString(),
+                ForkChoiceUpdatedVersion = "3",
+                Params = [CreatePayloadJson(newPayloadVersion)]
+            }
+        ];
+
+        var converted = JsonToEthereumTest.Convert(payloads).Single();
+
+        converted.Item1.Should().BeOfType(expectedPayloadType);
+    }
+
     /// <summary>
     /// An AccessList transaction with an empty access list sent against Istanbul (pre-Berlin)
     /// must be rejected. The post-state root must equal the pre-state root - the invalid tx
@@ -120,4 +145,33 @@ public void Invalid_pre_berlin_access_list_tx_with_empty_list_preserves_prestate
             "invalid AccessList tx on pre-Berlin fork should not mutate state");
         result.Pass.Should().BeTrue();
     }
+
+    private static JsonElement CreatePayloadJson(int newPayloadVersion)
+    {
+        string payloadJson = $$"""
+            {
+              "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+              "feeRecipient": "0x0000000000000000000000000000000000000001",
+              "stateRoot": "0x0000000000000000000000000000000000000000000000000000000000000002",
+              "receiptsRoot": "0x0000000000000000000000000000000000000000000000000000000000000003",
+              "logsBloom": "0x{{new string('0', 512)}}",
+              "blockNumber": "0x1",
+              "gasLimit": "0x5208",
+              "gasUsed": "0x5208",
+              "timestamp": "0x1",
+              "extraData": "0x",
+              "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000004",
+              "baseFeePerGas": "0x7",
+              "blobGasUsed": "0x0",
+              "excessBlobGas": "0x0",
+              "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000005",
+              "transactions": [],
+              "withdrawals": []
+              {{(newPayloadVersion >= 5 ? ",\n  \"blockAccessList\": \"0xc0\",\n  \"slotNumber\": \"0x0\"" : string.Empty)}}
+            }
+            """;
+
+        using JsonDocument document = JsonDocument.Parse(payloadJson);
+        return document.RootElement.Clone();
+    }
 }