diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index aeead13c362..020c8e4adc9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -134,6 +134,8 @@ public static Transaction readFrom(final RLPInput rlpInput) { /** * Instantiates a transaction instance. * + * @param forCopy true when using to create a copy of an already validated transaction avoid to + * redo the validation * @param transactionType the transaction type * @param nonce the nonce * @param gasPrice the gas price @@ -154,7 +156,8 @@ public static Transaction readFrom(final RLPInput rlpInput) { *

The {@code chainId} must be greater than 0 to be applied to a specific chain; otherwise * it will default to any chain. */ - public Transaction( + private Transaction( + final boolean forCopy, final TransactionType transactionType, final long nonce, final Optional gasPrice, @@ -172,36 +175,40 @@ public Transaction( final Optional> versionedHashes, final Optional blobsWithCommitments) { - if (transactionType.requiresChainId()) { - checkArgument( - chainId.isPresent(), "Chain id must be present for transaction type %s", transactionType); - } + if (!forCopy) { + if (transactionType.requiresChainId()) { + checkArgument( + chainId.isPresent(), + "Chain id must be present for transaction type %s", + transactionType); + } - if (maybeAccessList.isPresent()) { - checkArgument( - transactionType.supportsAccessList(), - "Must not specify access list for transaction not supporting it"); - } + if (maybeAccessList.isPresent()) { + checkArgument( + transactionType.supportsAccessList(), + "Must not specify access list for transaction not supporting it"); + } - if (Objects.equals(transactionType, TransactionType.ACCESS_LIST)) { - checkArgument( - maybeAccessList.isPresent(), "Must specify access list for access list transaction"); - } + if (Objects.equals(transactionType, TransactionType.ACCESS_LIST)) { + checkArgument( + maybeAccessList.isPresent(), "Must specify access list for access list transaction"); + } - if (versionedHashes.isPresent() || maxFeePerBlobGas.isPresent()) { - checkArgument( - transactionType.supportsBlob(), - "Must not specify blob versioned hashes or max fee per blob gas for transaction not supporting it"); - } + if (versionedHashes.isPresent() || maxFeePerBlobGas.isPresent()) { + checkArgument( + transactionType.supportsBlob(), + "Must not specify blob versioned hashes or max fee per blob gas for transaction not supporting it"); + } - if (transactionType.supportsBlob()) { - checkArgument( - versionedHashes.isPresent(), "Must specify blob versioned hashes for blob transaction"); - checkArgument( - !versionedHashes.get().isEmpty(), - "Blob transaction must have at least one versioned hash"); - checkArgument( - maxFeePerBlobGas.isPresent(), "Must specify max fee per blob gas for blob transaction"); + if (transactionType.supportsBlob()) { + checkArgument( + versionedHashes.isPresent(), "Must specify blob versioned hashes for blob transaction"); + checkArgument( + !versionedHashes.get().isEmpty(), + "Blob transaction must have at least one versioned hash"); + checkArgument( + maxFeePerBlobGas.isPresent(), "Must specify max fee per blob gas for blob transaction"); + } } this.transactionType = transactionType; @@ -221,7 +228,7 @@ public Transaction( this.versionedHashes = versionedHashes; this.blobsWithCommitments = blobsWithCommitments; - if (isUpfrontGasCostTooHigh()) { + if (!forCopy && isUpfrontGasCostTooHigh()) { throw new IllegalArgumentException("Upfront gas cost exceeds UInt256"); } } @@ -998,6 +1005,84 @@ public Optional

contractAddress() { return Optional.empty(); } + /** + * Creates a copy of this transaction that does not share any underlying byte array. + * + *

This is useful in case the transaction is built from a block body and fields, like to or + * payload, are wrapping (and so keeping references) sections of the large RPL encoded block body, + * and we plan to keep the transaction around for some time, like in the txpool in case of a + * reorg, and do not want to keep all the block body in memory for a long time, but only the + * actual transaction. + * + * @return a copy of the transaction + */ + public Transaction detachedCopy() { + final Optional

detachedTo = + to.isEmpty() ? to : Optional.of(Address.wrap(to.get().copy())); + final Optional> detachedAccessList = + maybeAccessList.isEmpty() + ? maybeAccessList + : Optional.of( + maybeAccessList.get().stream().map(this::accessListDetachedCopy).toList()); + final Optional> detachedVersionedHashes = + versionedHashes.isEmpty() + ? versionedHashes + : Optional.of( + versionedHashes.get().stream() + .map(vh -> new VersionedHash(vh.toBytes().copy())) + .toList()); + final Optional detachedBlobsWithCommitments = + blobsWithCommitments.isEmpty() + ? blobsWithCommitments + : Optional.of( + blobsWithCommitmentsDetachedCopy( + blobsWithCommitments.get(), detachedVersionedHashes.get())); + + return new Transaction( + true, + transactionType, + nonce, + gasPrice, + maxPriorityFeePerGas, + maxFeePerGas, + maxFeePerBlobGas, + gasLimit, + detachedTo, + value, + signature, + payload.copy(), + detachedAccessList, + sender, + chainId, + detachedVersionedHashes, + detachedBlobsWithCommitments); + } + + private AccessListEntry accessListDetachedCopy(final AccessListEntry accessListEntry) { + final Address detachedAddress = Address.wrap(accessListEntry.address().copy()); + final var detachedStorage = accessListEntry.storageKeys().stream().map(Bytes32::copy).toList(); + return new AccessListEntry(detachedAddress, detachedStorage); + } + + private BlobsWithCommitments blobsWithCommitmentsDetachedCopy( + final BlobsWithCommitments blobsWithCommitments, final List versionedHashes) { + final var detachedCommitments = + blobsWithCommitments.getKzgCommitments().stream() + .map(kc -> new KZGCommitment(kc.getData().copy())) + .toList(); + final var detachedBlobs = + blobsWithCommitments.getBlobs().stream() + .map(blob -> new Blob(blob.getData().copy())) + .toList(); + final var detachedProofs = + blobsWithCommitments.getKzgProofs().stream() + .map(proof -> new KZGProof(proof.getData().copy())) + .toList(); + + return new BlobsWithCommitments( + detachedCommitments, detachedBlobs, detachedProofs, versionedHashes); + } + public static class Builder { private static final Optional> EMPTY_ACCESS_LIST = Optional.of(List.of()); @@ -1134,6 +1219,7 @@ public TransactionType getTransactionType() { public Transaction build() { if (transactionType == null) guessType(); return new Transaction( + false, transactionType, nonce, Optional.ofNullable(gasPrice), diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/BlobPooledTransactionDecoder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/BlobPooledTransactionDecoder.java index 8f2efde53f8..1ccd5c4ad42 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/BlobPooledTransactionDecoder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/BlobPooledTransactionDecoder.java @@ -24,9 +24,9 @@ /** * Class responsible for decoding blob transactions from the transaction pool. Blob transactions - * have two network representations. During transaction gossip responses (PooledTransactions), the - * EIP-2718 TransactionPayload of the blob transaction is wrapped to become: rlp([tx_payload_body, - * blobs, commitments, proofs]). + * have two representations. The network representation is used during transaction gossip responses + * (PooledTransactions), the EIP-2718 TransactionPayload of the blob transaction is wrapped to + * become: rlp([tx_payload_body, blobs, commitments, proofs]). */ public class BlobPooledTransactionDecoder { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java index a716b1675e3..03e6ed69009 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java @@ -31,14 +31,20 @@ public abstract class PendingTransaction implements org.hyperledger.besu.datatypes.PendingTransaction { static final int NOT_INITIALIZED = -1; - static final int FRONTIER_BASE_MEMORY_SIZE = 944; - static final int ACCESS_LIST_BASE_MEMORY_SIZE = 944; - static final int EIP1559_BASE_MEMORY_SIZE = 1056; - static final int OPTIONAL_TO_MEMORY_SIZE = 92; + static final int FRONTIER_AND_ACCESS_LIST_BASE_MEMORY_SIZE = 872; + static final int EIP1559_AND_EIP4844_BASE_MEMORY_SIZE = 984; + static final int OPTIONAL_TO_MEMORY_SIZE = 112; + static final int OPTIONAL_CHAIN_ID_MEMORY_SIZE = 80; static final int PAYLOAD_BASE_MEMORY_SIZE = 32; static final int ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE = 32; - static final int ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE = 128; + static final int ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE = 248; static final int OPTIONAL_ACCESS_LIST_MEMORY_SIZE = 24; + static final int VERSIONED_HASH_SIZE = 96; + static final int BASE_LIST_SIZE = 48; + static final int BASE_OPTIONAL_SIZE = 16; + static final int KZG_COMMITMENT_OR_PROOF_SIZE = 112; + static final int BLOB_SIZE = 131136; + static final int BLOBS_WITH_COMMITMENTS_SIZE = 32; static final int PENDING_TRANSACTION_MEMORY_SIZE = 40; private static final AtomicLong TRANSACTIONS_ADDED = new AtomicLong(); private final Transaction transaction; @@ -47,10 +53,15 @@ public abstract class PendingTransaction private int memorySize = NOT_INITIALIZED; - protected PendingTransaction(final Transaction transaction, final long addedAt) { + private PendingTransaction( + final Transaction transaction, final long addedAt, final long sequence) { this.transaction = transaction; this.addedAt = addedAt; - this.sequence = TRANSACTIONS_ADDED.getAndIncrement(); + this.sequence = sequence; + } + + private PendingTransaction(final Transaction transaction, final long addedAt) { + this(transaction, addedAt, TRANSACTIONS_ADDED.getAndIncrement()); } @Override @@ -90,6 +101,8 @@ public int memorySize() { return memorySize; } + public abstract PendingTransaction detachedCopy(); + private int computeMemorySize() { return switch (transaction.getType()) { case FRONTIER -> computeFrontierMemorySize(); @@ -101,30 +114,49 @@ private int computeMemorySize() { } private int computeFrontierMemorySize() { - return FRONTIER_BASE_MEMORY_SIZE + computePayloadMemorySize() + computeToMemorySize(); + return FRONTIER_AND_ACCESS_LIST_BASE_MEMORY_SIZE + + computePayloadMemorySize() + + computeToMemorySize() + + computeChainIdMemorySize(); } private int computeAccessListMemorySize() { - return ACCESS_LIST_BASE_MEMORY_SIZE + return FRONTIER_AND_ACCESS_LIST_BASE_MEMORY_SIZE + computePayloadMemorySize() + computeToMemorySize() + + computeChainIdMemorySize() + computeAccessListEntriesMemorySize(); } private int computeEIP1559MemorySize() { - return EIP1559_BASE_MEMORY_SIZE + return EIP1559_AND_EIP4844_BASE_MEMORY_SIZE + computePayloadMemorySize() + computeToMemorySize() + + computeChainIdMemorySize() + computeAccessListEntriesMemorySize(); } private int computeBlobMemorySize() { - // ToDo 4844: adapt for blobs - return computeEIP1559MemorySize(); + return computeEIP1559MemorySize() + + BASE_OPTIONAL_SIZE // for the versionedHashes field + + computeBlobWithCommitmentsMemorySize(); + } + + private int computeBlobWithCommitmentsMemorySize() { + final int blobCount = transaction.getBlobCount(); + + return BASE_OPTIONAL_SIZE + + BLOBS_WITH_COMMITMENTS_SIZE + + (BASE_LIST_SIZE * 4) + + (KZG_COMMITMENT_OR_PROOF_SIZE * blobCount * 2) + + (VERSIONED_HASH_SIZE * blobCount) + + (BLOB_SIZE * blobCount); } private int computePayloadMemorySize() { - return PAYLOAD_BASE_MEMORY_SIZE + transaction.getPayload().size(); + return transaction.getPayload().size() > 0 + ? PAYLOAD_BASE_MEMORY_SIZE + transaction.getPayload().size() + : 0; } private int computeToMemorySize() { @@ -134,6 +166,13 @@ private int computeToMemorySize() { return 0; } + private int computeChainIdMemorySize() { + if (transaction.getChainId().isPresent()) { + return OPTIONAL_CHAIN_ID_MEMORY_SIZE; + } + return 0; + } + private int computeAccessListEntriesMemorySize() { return transaction .getAccessList() @@ -212,6 +251,15 @@ public Local(final Transaction transaction) { this(transaction, System.currentTimeMillis()); } + private Local(final long sequence, final Transaction transaction) { + super(transaction, System.currentTimeMillis(), sequence); + } + + @Override + public PendingTransaction detachedCopy() { + return new Local(getSequence(), getTransaction().detachedCopy()); + } + @Override public boolean isReceivedFromLocalSource() { return true; @@ -228,6 +276,15 @@ public Remote(final Transaction transaction) { this(transaction, System.currentTimeMillis()); } + private Remote(final long sequence, final Transaction transaction) { + super(transaction, System.currentTimeMillis(), sequence); + } + + @Override + public PendingTransaction detachedCopy() { + return new Remote(getSequence(), getTransaction().detachedCopy()); + } + @Override public boolean isReceivedFromLocalSource() { return false; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java index 026d1bf4f72..12ffd91f590 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java @@ -150,7 +150,7 @@ public TransactionAddedResult add(final PendingTransaction pendingTransaction, f } if (addStatus.isSuccess()) { - processAdded(pendingTransaction); + processAdded(pendingTransaction.detachedCopy()); addStatus.maybeReplacedTransaction().ifPresent(this::replaced); nextLayer.notifyAdded(pendingTransaction); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java index 54cbe804ab7..dde45d0a95b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java @@ -19,52 +19,63 @@ import org.hyperledger.besu.crypto.SignatureAlgorithm; import org.hyperledger.besu.datatypes.AccessListEntry; import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.BlobsWithCommitments; import org.hyperledger.besu.datatypes.TransactionType; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.core.encoding.EncodingContext; +import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder; +import org.hyperledger.besu.ethereum.core.encoding.TransactionEncoder; import org.hyperledger.besu.ethereum.eth.transactions.layered.BaseTransactionPoolTest; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import java.math.BigInteger; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; +import com.google.common.collect.Sets; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; import org.openjdk.jol.info.ClassLayout; import org.openjdk.jol.info.GraphPathRecord; import org.openjdk.jol.info.GraphVisitor; import org.openjdk.jol.info.GraphWalker; -@Disabled("Need to handle different results on different OS") +@EnabledOnOs(OS.LINUX) public class PendingTransactionEstimatedMemorySizeTest extends BaseTransactionPoolTest { private static final Set> SHARED_CLASSES = Set.of(SignatureAlgorithm.class, TransactionType.class); - private static final Set EIP1559_CONSTANT_FIELD_PATHS = Set.of(".gasPrice"); - private static final Set EIP1559_VARIABLE_SIZE_PATHS = - Set.of(".to", ".payload", ".maybeAccessList"); - + private static final Set COMMON_CONSTANT_FIELD_PATHS = + Set.of(".value.ctor", ".hashNoSignature"); + private static final Set EIP1559_EIP4844_CONSTANT_FIELD_PATHS = + Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".gasPrice")); private static final Set FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS = - Set.of(".maxFeePerGas", ".maxPriorityFeePerGas"); - private static final Set FRONTIER_ACCESS_LIST_VARIABLE_SIZE_PATHS = - Set.of(".to", ".payload", ".maybeAccessList"); + Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".maxFeePerGas", ".maxPriorityFeePerGas")); + private static final Set VARIABLE_SIZE_PATHS = + Set.of(".chainId", ".to", ".payload", ".maybeAccessList"); @Test public void toSize() { TransactionTestFixture preparedTx = - prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10); + prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10, 0); Transaction txTo = preparedTx.to(Optional.of(Address.extract(Bytes32.random()))).createTransaction(KEYS1); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); txTo.writeTo(rlpOut); - txTo = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + txTo = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); System.out.println(txTo.getSender()); System.out.println(txTo.getHash()); System.out.println(txTo.getSize()); @@ -78,34 +89,17 @@ public void toSize() { GraphVisitor gv = gpr -> { - // byte[] is shared so only count the specific part for each field - if (gpr.path().endsWith(".bytes")) { - if (gpr.path().contains("delegate")) { - size.add(20); - System.out.println( - "(" - + size - + ")[20 = fixed address size; overrides: " - + gpr.size() - + ", " - + gpr.path() - + ", " - + gpr.klass().toString() - + "]"); - } - } else { - size.add(gpr.size()); - System.out.println( - "(" - + size - + ")[" - + gpr.size() - + ", " - + gpr.path() - + ", " - + gpr.klass().toString() - + "]"); - } + size.add(gpr.size()); + System.out.println( + "(" + + size + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); }; GraphWalker gw = new GraphWalker(gv); @@ -121,12 +115,13 @@ public void toSize() { public void payloadSize() { TransactionTestFixture preparedTx = - prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10); + prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10, 0); Transaction txPayload = preparedTx.createTransaction(KEYS1); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); txPayload.writeTo(rlpOut); - txPayload = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + txPayload = + Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); System.out.println(txPayload.getSender()); System.out.println(txPayload.getHash()); System.out.println(txPayload.getSize()); @@ -141,16 +136,141 @@ public void payloadSize() { assertThat(size.sum()).isEqualTo(PendingTransaction.PAYLOAD_BASE_MEMORY_SIZE); } + @Test + public void chainIdSize() { + + BigInteger chainId = BigInteger.valueOf(1); + Optional maybeChainId = Optional.of(chainId); + + final ClassLayout cl = ClassLayout.parseInstance(maybeChainId); + System.out.println(cl.toPrintable()); + LongAdder size = new LongAdder(); + size.add(cl.instanceSize()); + System.out.println("Base chainId size: " + size); + + GraphVisitor gv = + gpr -> { + size.add(gpr.size()); + System.out.println( + "(" + + size + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + }; + + GraphWalker gw = new GraphWalker(gv); + + gw.walk(maybeChainId); + + assertThat(size.sum()).isEqualTo(PendingTransaction.OPTIONAL_CHAIN_ID_MEMORY_SIZE); + } + + @Test + public void kgzCommitmentsSize() { + blobsWithCommitmentsFieldSize( + t -> t.getBlobsWithCommitments().get().getKzgCommitments(), + PendingTransaction.BASE_LIST_SIZE, + PendingTransaction.KZG_COMMITMENT_OR_PROOF_SIZE); + } + + @Test + public void kgzProofsSize() { + blobsWithCommitmentsFieldSize( + t -> t.getBlobsWithCommitments().get().getKzgProofs(), + PendingTransaction.BASE_LIST_SIZE, + PendingTransaction.KZG_COMMITMENT_OR_PROOF_SIZE); + } + + @Test + public void blobsSize() { + blobsWithCommitmentsFieldSize( + t -> t.getBlobsWithCommitments().get().getBlobs(), + PendingTransaction.BASE_LIST_SIZE, + PendingTransaction.BLOB_SIZE); + } + + @Test + public void versionedHashesSize() { + blobsWithCommitmentsFieldSize( + t -> t.getBlobsWithCommitments().get().getVersionedHashes(), + PendingTransaction.BASE_LIST_SIZE, + PendingTransaction.VERSIONED_HASH_SIZE); + } + + private void blobsWithCommitmentsFieldSize( + final Function> containerExtractor, + final long containerSize, + final long itemSize) { + TransactionTestFixture preparedTx = + prepareTransaction(TransactionType.BLOB, 10, Wei.of(500), 10, 1); + Transaction txBlob = preparedTx.createTransaction(KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + TransactionEncoder.encodeRLP(txBlob, rlpOut, EncodingContext.POOLED_TRANSACTION); + + txBlob = + TransactionDecoder.decodeRLP( + new BytesValueRLPInput(rlpOut.encoded(), false), EncodingContext.POOLED_TRANSACTION) + .detachedCopy(); + System.out.println(txBlob.getSender()); + System.out.println(txBlob.getHash()); + System.out.println(txBlob.getSize()); + + final List list = containerExtractor.apply(txBlob); + + final long cSize = sizeOfField(list, ".elements["); + + System.out.println("Container size: " + cSize); + + assertThat(cSize).isEqualTo(containerSize); + + final Object item = list.get(0); + final long iSize = sizeOfField(item); + + System.out.println("Item size: " + iSize); + + assertThat(iSize).isEqualTo(itemSize); + } + + @Test + public void blobsWithCommitmentsSize() { + TransactionTestFixture preparedTx = + prepareTransaction(TransactionType.BLOB, 10, Wei.of(500), 10, 1); + Transaction txBlob = preparedTx.createTransaction(KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + TransactionEncoder.encodeRLP(txBlob, rlpOut, EncodingContext.POOLED_TRANSACTION); + + txBlob = + TransactionDecoder.decodeRLP( + new BytesValueRLPInput(rlpOut.encoded(), false), EncodingContext.POOLED_TRANSACTION) + .detachedCopy(); + System.out.println(txBlob.getSender()); + System.out.println(txBlob.getHash()); + System.out.println(txBlob.getSize()); + + final BlobsWithCommitments bwc = txBlob.getBlobsWithCommitments().get(); + final ClassLayout cl = ClassLayout.parseInstance(bwc); + System.out.println(cl.toPrintable()); + System.out.println("BlobsWithCommitments size: " + cl.instanceSize()); + + assertThat(cl.instanceSize()).isEqualTo(PendingTransaction.BLOBS_WITH_COMMITMENTS_SIZE); + } + @Test public void pendingTransactionSize() { TransactionTestFixture preparedTx = - prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10); + prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10, 0); Transaction txPayload = preparedTx.createTransaction(KEYS1); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); txPayload.writeTo(rlpOut); - txPayload = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + txPayload = + Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); System.out.println(txPayload.getSender()); System.out.println(txPayload.getHash()); System.out.println(txPayload.getSize()); @@ -176,12 +296,13 @@ public void accessListSize() { final List ales = List.of(ale1); TransactionTestFixture preparedTx = - prepareTransaction(TransactionType.ACCESS_LIST, 0, Wei.of(500), 0); + prepareTransaction(TransactionType.ACCESS_LIST, 0, Wei.of(500), 0, 0); Transaction txAccessList = preparedTx.accessList(ales).createTransaction(KEYS1); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); txAccessList.writeTo(rlpOut); - txAccessList = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + txAccessList = + Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); System.out.println(txAccessList.getSender()); System.out.println(txAccessList.getHash()); System.out.println(txAccessList.getSize()); @@ -200,55 +321,11 @@ public void accessListSize() { final AccessListEntry ale = optAL.get().get(0); - final ClassLayout cl3 = ClassLayout.parseInstance(ale); - System.out.println(cl3.toPrintable()); - System.out.println("AccessListEntry size: " + cl3.instanceSize()); - - LongAdder size = new LongAdder(); - size.add(cl3.instanceSize()); - - GraphVisitor gv = - gpr -> { - // byte[] is shared so only count the specific part for each field - if (gpr.path().endsWith(".bytes")) { - if (gpr.path().contains("address")) { - size.add(20); - System.out.println( - "(" - + size - + ")[20 = fixed address size; overrides: " - + gpr.size() - + ", " - + gpr.path() - + ", " - + gpr.klass().toString() - + "]"); - } - } else if (!gpr.path() - .contains( - "storageKeys.elementData[")) { // exclude elements since we want the container - // size - size.add(gpr.size()); - System.out.println( - "(" - + size - + ")[" - + gpr.size() - + ", " - + gpr.path() - + ", " - + gpr.klass().toString() - + "]"); - } - }; - - GraphWalker gw = new GraphWalker(gv); - - gw.walk(ale); + long aleSize = sizeOfField(ale, "storageKeys.elementData["); - System.out.println("AccessListEntry container size: " + size); + System.out.println("AccessListEntry container size: " + aleSize); - assertThat(size.sum()).isEqualTo(PendingTransaction.ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE); + assertThat(aleSize).isEqualTo(PendingTransaction.ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE); final Bytes32 storageKey = ale.storageKeys().get(0); final ClassLayout cl4 = ClassLayout.parseInstance(storageKey); @@ -260,13 +337,14 @@ public void accessListSize() { } @Test - public void baseEIP1559TransactionMemorySize() { + public void baseEIP1559AndEIP4844TransactionMemorySize() { System.setProperty("jol.magicFieldOffset", "true"); Transaction txEip1559 = createEIP1559Transaction(1, KEYS1, 10); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); txEip1559.writeTo(rlpOut); - txEip1559 = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + txEip1559 = + Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); System.out.println(txEip1559.getSender()); System.out.println(txEip1559.getHash()); System.out.println(txEip1559.getSize()); @@ -277,138 +355,141 @@ public void baseEIP1559TransactionMemorySize() { eip1559size.add(cl.instanceSize()); System.out.println(eip1559size); - final Set skipPrefixes = new HashSet<>(); - - GraphVisitor gv = - gpr -> { - if (!skipPrefixes.stream().anyMatch(sp -> gpr.path().startsWith(sp))) { - if (SHARED_CLASSES.stream().anyMatch(scz -> scz.isAssignableFrom(gpr.klass()))) { - skipPrefixes.add(gpr.path()); - } else if (!startWithAnyOf(EIP1559_CONSTANT_FIELD_PATHS, gpr) - && !startWithAnyOf(EIP1559_VARIABLE_SIZE_PATHS, gpr)) { - eip1559size.add(gpr.size()); - System.out.println( - "(" - + eip1559size - + ")[" - + gpr.size() - + ", " - + gpr.path() - + ", " - + gpr.klass().toString() - + "]"); - } - } - }; - - GraphWalker gw = new GraphWalker(gv); + final SortedSet fieldSizes = new TreeSet<>(); + GraphWalker gw = getGraphWalker(EIP1559_EIP4844_CONSTANT_FIELD_PATHS, fieldSizes); gw.walk(txEip1559); + fieldSizes.forEach( + fieldSize -> { + eip1559size.add(fieldSize.size()); + System.out.println( + "(" + + eip1559size + + ")[" + + fieldSize.size() + + ", " + + fieldSize.path() + + ", " + + fieldSize + + "]"); + }); + System.out.println("Base EIP1559 size: " + eip1559size); - assertThat(eip1559size.sum()).isEqualTo(PendingTransaction.EIP1559_BASE_MEMORY_SIZE); + assertThat(eip1559size.sum()) + .isEqualTo(PendingTransaction.EIP1559_AND_EIP4844_BASE_MEMORY_SIZE); } @Test - public void baseAccessListTransactionMemorySize() { + public void baseFrontierAndAccessListTransactionMemorySize() { System.setProperty("jol.magicFieldOffset", "true"); - Transaction txAccessList = - createTransaction(TransactionType.ACCESS_LIST, 1, Wei.of(500), 0, KEYS1); + Transaction txFrontier = createTransaction(TransactionType.FRONTIER, 1, Wei.of(500), 0, KEYS1); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); - txAccessList.writeTo(rlpOut); + txFrontier.writeTo(rlpOut); - txAccessList = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); - System.out.println(txAccessList.getSender()); - System.out.println(txAccessList.getHash()); - System.out.println(txAccessList.getSize()); + txFrontier = + Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); + System.out.println(txFrontier.getSender()); + System.out.println(txFrontier.getHash()); + System.out.println(txFrontier.getSize()); - final ClassLayout cl = ClassLayout.parseInstance(txAccessList); + final ClassLayout cl = ClassLayout.parseInstance(txFrontier); System.out.println(cl.toPrintable()); - LongAdder accessListSize = new LongAdder(); - accessListSize.add(cl.instanceSize()); - System.out.println(accessListSize); + LongAdder frontierSize = new LongAdder(); + frontierSize.add(cl.instanceSize()); + System.out.println(frontierSize); - final Set skipPrefixes = new HashSet<>(); + final SortedSet fieldSizes = new TreeSet<>(); + + GraphWalker gw = getGraphWalker(FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS, fieldSizes); + gw.walk(txFrontier); + + fieldSizes.forEach( + fieldSize -> { + frontierSize.add(fieldSize.size()); + System.out.println( + "(" + + frontierSize + + ")[" + + fieldSize.size() + + ", " + + fieldSize.path() + + ", " + + fieldSize + + "]"); + }); + + System.out.println("Base Frontier size: " + frontierSize); + assertThat(frontierSize.sum()) + .isEqualTo(PendingTransaction.FRONTIER_AND_ACCESS_LIST_BASE_MEMORY_SIZE); + } + + private GraphWalker getGraphWalker( + final Set constantFieldPaths, final SortedSet fieldSizes) { + final Set skipPrefixes = new HashSet<>(); GraphVisitor gv = gpr -> { if (!skipPrefixes.stream().anyMatch(sp -> gpr.path().startsWith(sp))) { if (SHARED_CLASSES.stream().anyMatch(scz -> scz.isAssignableFrom(gpr.klass()))) { skipPrefixes.add(gpr.path()); - } else if (!startWithAnyOf(FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS, gpr) - && !startWithAnyOf(FRONTIER_ACCESS_LIST_VARIABLE_SIZE_PATHS, gpr)) { - accessListSize.add(gpr.size()); - System.out.println( - "(" - + accessListSize - + ")[" - + gpr.size() - + ", " - + gpr.path() - + ", " - + gpr.klass().toString() - + "]"); + } else if (!startWithAnyOf(constantFieldPaths, gpr) + && !startWithAnyOf(VARIABLE_SIZE_PATHS, gpr)) { + + fieldSizes.add(new FieldSize(gpr.path(), gpr.klass(), gpr.size())); } } }; GraphWalker gw = new GraphWalker(gv); - - gw.walk(txAccessList); - System.out.println("Base Access List size: " + accessListSize); - assertThat(accessListSize.sum()).isEqualTo(PendingTransaction.ACCESS_LIST_BASE_MEMORY_SIZE); + return gw; } - @Test - public void baseFrontierTransactionMemorySize() { - System.setProperty("jol.magicFieldOffset", "true"); - Transaction txFrontier = createTransaction(TransactionType.FRONTIER, 1, Wei.of(500), 0, KEYS1); - BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); - txFrontier.writeTo(rlpOut); - - txFrontier = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); - System.out.println(txFrontier.getSender()); - System.out.println(txFrontier.getHash()); - System.out.println(txFrontier.getSize()); + private boolean startWithAnyOf(final Set prefixes, final GraphPathRecord path) { + return prefixes.stream().anyMatch(prefix -> path.path().startsWith(prefix)); + } - final ClassLayout cl = ClassLayout.parseInstance(txFrontier); + private long sizeOfField(final Object container, final String... excludePaths) { + final ClassLayout cl = ClassLayout.parseInstance(container); System.out.println(cl.toPrintable()); - LongAdder frontierSize = new LongAdder(); - frontierSize.add(cl.instanceSize()); - System.out.println(frontierSize); + System.out.println("Base container size: " + cl.instanceSize()); - final Set skipPrefixes = new HashSet<>(); + LongAdder size = new LongAdder(); + size.add(cl.instanceSize()); GraphVisitor gv = gpr -> { - if (!skipPrefixes.stream().anyMatch(sp -> gpr.path().startsWith(sp))) { - if (SHARED_CLASSES.stream().anyMatch(scz -> scz.isAssignableFrom(gpr.klass()))) { - skipPrefixes.add(gpr.path()); - } else if (!startWithAnyOf(FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS, gpr) - && !startWithAnyOf(FRONTIER_ACCESS_LIST_VARIABLE_SIZE_PATHS, gpr)) { - frontierSize.add(gpr.size()); - System.out.println( - "(" - + frontierSize - + ")[" - + gpr.size() - + ", " - + gpr.path() - + ", " - + gpr.klass().toString() - + "]"); - } + if (Arrays.stream(excludePaths) + .anyMatch(excludePath -> gpr.path().contains(excludePath))) { + System.out.println("Excluded path " + gpr.path()); + } else { + size.add(gpr.size()); + System.out.println( + "(" + + size + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); } }; GraphWalker gw = new GraphWalker(gv); - gw.walk(txFrontier); - System.out.println("Base Frontier size: " + frontierSize); - assertThat(frontierSize.sum()).isEqualTo(PendingTransaction.FRONTIER_BASE_MEMORY_SIZE); + gw.walk(container); + + System.out.println("Container size: " + size); + return size.sum(); } - private boolean startWithAnyOf(final Set prefixes, final GraphPathRecord path) { - return prefixes.stream().anyMatch(prefix -> path.path().startsWith(prefix)); + record FieldSize(String path, Class clazz, long size) implements Comparable { + + @Override + public int compareTo(final FieldSize o) { + return path.compareTo(o.path); + } } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java index b735055445a..4cf6a3bd264 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java @@ -20,7 +20,13 @@ import org.hyperledger.besu.crypto.SignatureAlgorithm; import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Blob; +import org.hyperledger.besu.datatypes.BlobsWithCommitments; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.KZGCommitment; +import org.hyperledger.besu.datatypes.KZGProof; import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.VersionedHash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; @@ -33,10 +39,12 @@ import java.util.Optional; import java.util.Random; +import java.util.stream.IntStream; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes48; public class BaseTransactionPoolTest { @@ -82,6 +90,12 @@ protected Transaction createEIP1559Transaction( TransactionType.EIP1559, nonce, Wei.of(5000L).multiply(gasFeeMultiplier), 0, keys); } + protected Transaction createEIP4844Transaction( + final long nonce, final KeyPair keys, final int gasFeeMultiplier, final int blobCount) { + return createTransaction( + TransactionType.BLOB, nonce, Wei.of(5000L).multiply(gasFeeMultiplier), 0, blobCount, keys); + } + protected Transaction createTransaction( final long nonce, final Wei maxGasPrice, final int payloadSize, final KeyPair keys) { @@ -97,11 +111,26 @@ protected Transaction createTransaction( final Wei maxGasPrice, final int payloadSize, final KeyPair keys) { - return prepareTransaction(type, nonce, maxGasPrice, payloadSize).createTransaction(keys); + return createTransaction(type, nonce, maxGasPrice, payloadSize, 0, keys); + } + + protected Transaction createTransaction( + final TransactionType type, + final long nonce, + final Wei maxGasPrice, + final int payloadSize, + final int blobCount, + final KeyPair keys) { + return prepareTransaction(type, nonce, maxGasPrice, payloadSize, blobCount) + .createTransaction(keys); } protected TransactionTestFixture prepareTransaction( - final TransactionType type, final long nonce, final Wei maxGasPrice, final int payloadSize) { + final TransactionType type, + final long nonce, + final Wei maxGasPrice, + final int payloadSize, + final int blobCount) { var tx = new TransactionTestFixture() @@ -116,6 +145,24 @@ protected TransactionTestFixture prepareTransaction( if (type.supports1559FeeMarket()) { tx.maxFeePerGas(Optional.of(maxGasPrice)) .maxPriorityFeePerGas(Optional.of(maxGasPrice.divide(10))); + if (type.supportsBlob() && blobCount > 0) { + final var versionHashes = + IntStream.range(0, blobCount) + .mapToObj(i -> new VersionedHash((byte) 1, Hash.ZERO)) + .toList(); + final var kgzCommitments = + IntStream.range(0, blobCount) + .mapToObj(i -> new KZGCommitment(Bytes48.random())) + .toList(); + final var kzgProofs = + IntStream.range(0, blobCount).mapToObj(i -> new KZGProof(Bytes48.random())).toList(); + final var blobs = + IntStream.range(0, blobCount).mapToObj(i -> new Blob(Bytes.random(32 * 4096))).toList(); + tx.versionedHashes(Optional.of(versionHashes)); + final var blobsWithCommitments = + new BlobsWithCommitments(kgzCommitments, blobs, kzgProofs, versionHashes); + tx.blobsWithCommitments(Optional.of(blobsWithCommitments)); + } } else { tx.gasPrice(maxGasPrice); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java index d68c586c8d9..864efcd7f26 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; @@ -103,6 +104,7 @@ public class ReplayTest { @Test @Disabled("Provide a replay file to run the test on demand") public void replay() throws IOException { + SignatureAlgorithmFactory.setDefaultInstance(); try (BufferedReader br = new BufferedReader( new InputStreamReader(