Test updates for cancun execution-spec-tests (hyperledger#5670)

- support legacy V values (larger V value) and type 1+ (v is recId only)
- new fields
- shared transaction extraction
- rejection detection

Signed-off-by: Danno Ferrin <[email protected]>

Author: shemnon

ethereum/evmtool/build.gradle

@@ -56,8 +56,6 @@ dependencies {
   implementation 'info.picocli:picocli'
   implementation 'io.vertx:vertx-core'
 
-  runtimeOnly 'org.slf4j:slf4j-nop'
-
   annotationProcessor 'com.google.dagger:dagger-compiler'
   annotationProcessor 'info.picocli:picocli-codegen'
 
@@ -67,6 +65,9 @@ dependencies {
   testImplementation 'org.mockito:mockito-core'
 
   testRuntimeOnly 'org.junit.vintage:junit-vintage-engine'
+
+  // No logging in grallvm EvmTool
+  nativeImageClasspath 'org.slf4j:slf4j-nop'
 }
 
 mainClassName = 'org.hyperledger.besu.evmtool.EvmTool'

ethereum/evmtool/src/main/graal/reflection-config.json

@@ -102,7 +102,7 @@
     "unsafeAllocated": true
   },
   {
-    "name": "org.hyperledger.besu.evmtool.T8nSubCommand$RejectedTransaction",
+    "name": "org.hyperledger.besu.evmtool.T8nExecutor$RejectedTransaction",
     "queryAllPublicConstructors": true,
     "queryAllPublicMethods": true,
     "allDeclaredConstructors": true,

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java

@@ -18,6 +18,9 @@
 import static org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules.shouldClearEmptyAccounts;
 
 import org.hyperledger.besu.config.StubGenesisConfigOptions;
+import org.hyperledger.besu.crypto.KeyPair;
+import org.hyperledger.besu.crypto.SignatureAlgorithm;
+import org.hyperledger.besu.crypto.SignatureAlgorithmFactory;
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.DataGas;
 import org.hyperledger.besu.datatypes.Hash;
@@ -36,24 +39,33 @@
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestBlockchain;
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv;
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules;
+import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
 import org.hyperledger.besu.ethereum.rlp.RLP;
 import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState;
+import org.hyperledger.besu.evm.AccessListEntry;
 import org.hyperledger.besu.evm.account.Account;
 import org.hyperledger.besu.evm.account.AccountStorageEntry;
 import org.hyperledger.besu.evm.log.Log;
 import org.hyperledger.besu.evm.tracing.OperationTracer;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.evmtool.exception.UnsupportedForkException;
+import org.hyperledger.besu.plugin.data.TransactionType;
 
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.NavigableMap;
+import java.util.Spliterator;
+import java.util.Spliterators;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.StreamSupport;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -64,6 +76,149 @@
 import org.apache.tuweni.units.bigints.UInt256;
 
 public class T8nExecutor {
+
+  record RejectedTransaction(int index, String error) {}
+
+  protected static List<Transaction> extractTransactions(
+      final PrintWriter out,
+      final Iterator<JsonNode> it,
+      final List<Transaction> transactions,
+      final List<RejectedTransaction> rejections) {
+    int i = 0;
+    while (it.hasNext()) {
+      try {
+        JsonNode txNode = it.next();
+        if (txNode.isTextual()) {
+          BytesValueRLPInput rlpInput =
+              new BytesValueRLPInput(Bytes.fromHexString(txNode.asText()), false);
+          rlpInput.enterList();
+          while (!rlpInput.isEndOfCurrentList()) {
+            Transaction tx = Transaction.readFrom(rlpInput);
+            transactions.add(tx);
+          }
+        } else if (txNode.isObject()) {
+          if (txNode.has("txBytes")) {
+            Transaction tx =
+                Transaction.readFrom(Bytes.fromHexString(txNode.get("txbytes").asText()));
+            transactions.add(tx);
+          } else {
+            Transaction.Builder builder = Transaction.builder();
+            int type = Bytes.fromHexStringLenient(txNode.get("type").textValue()).toInt();
+            TransactionType transactionType = TransactionType.of(type == 0 ? 0xf8 : type);
+            builder.type(transactionType);
+            builder.nonce(Bytes.fromHexStringLenient(txNode.get("nonce").textValue()).toLong());
+            builder.gasLimit(Bytes.fromHexStringLenient(txNode.get("gas").textValue()).toLong());
+            builder.value(Wei.fromHexString(txNode.get("value").textValue()));
+            builder.payload(Bytes.fromHexString(txNode.get("input").textValue()));
+
+            if (txNode.has("gasPrice")) {
+              builder.gasPrice(Wei.fromHexString(txNode.get("gasPrice").textValue()));
+            }
+            if (txNode.has("maxPriorityFeePerGas")) {
+              builder.maxPriorityFeePerGas(
+                  Wei.fromHexString(txNode.get("maxPriorityFeePerGas").textValue()));
+            }
+            if (txNode.has("maxFeePerGas")) {
+              builder.maxFeePerGas(Wei.fromHexString(txNode.get("maxFeePerGas").textValue()));
+            }
+            if (txNode.has("maxFeePerDataGas")) {
+              builder.maxFeePerDataGas(
+                  Wei.fromHexString(txNode.get("maxFeePerDataGas").textValue()));
+            }
+
+            if (txNode.has("to")) {
+              builder.to(Address.fromHexString(txNode.get("to").textValue()));
+            }
+
+            if (transactionType.requiresChainId()
+                || !txNode.has("protected")
+                || txNode.get("protected").booleanValue()) {
+              // chainid if protected
+              builder.chainId(
+                  new BigInteger(
+                      1,
+                      Bytes.fromHexStringLenient(txNode.get("chainId").textValue())
+                          .toArrayUnsafe()));
+            }
+
+            if (txNode.has("accessList")) {
+              JsonNode accessList = txNode.get("accessList");
+              if (!accessList.isArray()) {
+                out.printf(
+                    "TX json node unparseable: expected accessList to be an array - %s%n", txNode);
+                continue;
+              }
+              List<AccessListEntry> entries = new ArrayList<>(accessList.size());
+              for (JsonNode entryAsJson : accessList) {
+                Address address = Address.fromHexString(entryAsJson.get("address").textValue());
+                List<String> storageKeys =
+                    StreamSupport.stream(
+                            Spliterators.spliteratorUnknownSize(
+                                entryAsJson.get("storageKeys").elements(), Spliterator.ORDERED),
+                            false)
+                        .map(JsonNode::textValue)
+                        .toList();
+                var accessListEntry = AccessListEntry.createAccessListEntry(address, storageKeys);
+                entries.add(accessListEntry);
+              }
+              builder.accessList(entries);
+            }
+
+            if (txNode.has("blobVersionedHashes")) {
+              JsonNode blobVersionedHashes = txNode.get("blobVersionedHashes");
+              if (!blobVersionedHashes.isArray()) {
+                out.printf(
+                    "TX json node unparseable: expected blobVersionedHashes to be an array - %s%n",
+                    txNode);
+                continue;
+              }
+              // FUTURE: placeholder code until 4844 PR merges
+              // List<VersionedHash> entries = new ArrayList<>(blobVersionedHashes.size());
+              // for (JsonNode versionedHashNode : blobVersionedHashes) {
+              //   entries.add(
+              //       new VersionedHash(Bytes32.fromHexString(versionedHashNode.textValue())));
+              // }
+              // builder.versionedHashes(entries);
+            }
+
+            if (txNode.has("secretKey")) {
+              SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance();
+              KeyPair keys =
+                  signatureAlgorithm.createKeyPair(
+                      signatureAlgorithm.createPrivateKey(
+                          Bytes32.fromHexString(txNode.get("secretKey").textValue())));
+
+              transactions.add(builder.signAndBuild(keys));
+            } else {
+              BigInteger v =
+                  Bytes.fromHexStringLenient(txNode.get("v").textValue()).toUnsignedBigInteger();
+              if (v.compareTo(BigInteger.valueOf(35)) >= 0) {
+                v = v.subtract(BigInteger.valueOf(35)).mod(BigInteger.TWO);
+              } else if (v.compareTo(BigInteger.valueOf(27)) >= 0) {
+                v = v.subtract(BigInteger.valueOf(27)).mod(BigInteger.TWO);
+              }
+              builder.signature(
+                  SignatureAlgorithmFactory.getInstance()
+                      .createSignature(
+                          Bytes.fromHexStringLenient(txNode.get("r").textValue())
+                              .toUnsignedBigInteger(),
+                          Bytes.fromHexStringLenient(txNode.get("s").textValue())
+                              .toUnsignedBigInteger(),
+                          v.byteValueExact()));
+              transactions.add(builder.build());
+            }
+          }
+        } else {
+          out.printf("TX json node unparseable: %s%n", txNode);
+        }
+      } catch (IllegalArgumentException iae) {
+        rejections.add(new RejectedTransaction(i, iae.getMessage()));
+      }
+      i++;
+    }
+    return transactions;
+  }
+
   static T8nResult runTest(
       final Long chainId,
       final String fork,
@@ -72,6 +227,7 @@ static T8nResult runTest(
       final ReferenceTestEnv referenceTestEnv,
       final MutableWorldState initialWorldState,
       final List<Transaction> transactions,
+      final List<RejectedTransaction> rejections,
       final TracerManager tracerManager) {
 
     final ReferenceTestProtocolSchedules referenceTestProtocolSchedules =
@@ -95,7 +251,7 @@ static T8nResult runTest(
     final Wei dataGasPrice = protocolSpec.getFeeMarket().dataPrice(DataGas.ZERO);
 
     List<TransactionReceipt> receipts = new ArrayList<>();
-    List<T8nSubCommand.RejectedTransaction> invalidTransactions = new ArrayList<>();
+    List<RejectedTransaction> invalidTransactions = new ArrayList<>(rejections);
     List<Transaction> validTransactions = new ArrayList<>();
     ArrayNode receiptsArray = objectMapper.createArrayNode();
     long gasUsed = 0;
@@ -138,8 +294,7 @@ static T8nResult runTest(
       }
       if (result.isInvalid()) {
         invalidTransactions.add(
-            new T8nSubCommand.RejectedTransaction(
-                i, result.getValidationResult().getErrorMessage()));
+            new RejectedTransaction(i, result.getValidationResult().getErrorMessage()));
       } else {
         validTransactions.add(transaction);