simulators/ethereum/engine: Test with PoW Re-Org With PREVRANDAO Transactions

simulators/ethereum/engine/README.md

@@ -226,6 +226,17 @@ ForkchoiceUpdated is sent to Client 1 with B as Head.
 ForkchoiceUpdated is sent to Client 1 with C as Head.  
 Verification is made that Client 1 Re-orgs to chain G -> C.  
 
+- Two Block PoW Re-org to Lower-Height Chain, Transaction Overwrite
+Client 1 starts with chain G -> A -> B, Client 2 starts with chain G -> C. 
+Blocks B and C reach TTD, but block B has higher height than C.  
+Block A and C contain TX1, and Block B contains TX2.
+TX1 and TX2 use the DIFFICULTY/PREVRANDAO opcode into storage. 
+ForkchoiceUpdated is sent to Client 1 with B as Head.  
+ForkchoiceUpdated is sent to Client 1 with C as Head.  
+Verification is made that Client 1 Re-orgs to chain G -> C.  
+PoS chain on top of C contains TX2, which was originally part of the PoW chain.
+Verification is made that the resulting PoS storage contains PREVRANDAO instead of DIFFICULTY.
+
 - Two Block Post-PoS Re-org to Higher-Total-Difficulty PoW Chain:
 Client 1 starts with chain G -> A.
 ForkchoiceUpdated is sent to Client 1 with A as Head.

simulators/ethereum/engine/chains/README.md

@@ -0,0 +1,98 @@
+# Proof of Work Pre-computed Chains
+
+Chains Included:
+
+- blocks_1_td_196608.rlp
+    Block 1, 548b3a20d40c9b29fe6bf89b86bc3e1d52c1fadd60d1590627f486b6cd56635c, difficulty: 196608, Tx: 1
+    Chain Total Difficulty 196608
+
+- blocks_1_td_196704.rlp
+    Block 1, 509261eb35ed0e4c07645c6f4403a347bd8245e51a1dee314e15e91fbca8c0a7, difficulty: 196704, Tx: 1
+    Chain Total Difficulty 196704
+
+- blocks_2_td_393120.rlp
+    Block 1, 548b3a20d40c9b29fe6bf89b86bc3e1d52c1fadd60d1590627f486b6cd56635c, difficulty: 196608, Tx: 1
+    Block 2, 2d4f65a26a8d5ca4f8000010b4741652decb89c8ccf19119b5d9f58846efbbd4, difficulty: 196512, Tx: 1
+    Chain Total Difficulty 393120
+
+- blocks_2_td_393504.rlp
+    Block 1, 509261eb35ed0e4c07645c6f4403a347bd8245e51a1dee314e15e91fbca8c0a7, difficulty: 196704, Tx: 1
+    Block 2, b22fd8a8b1841e6b020a8626180cdb5e395c7b4eceb53a691f8c3e67557aac33, difficulty: 196800, Tx: 1
+    Chain Total Difficulty 393504
+
+- blocks_1024_td_135112316.rlp
+    Block 1, 2a9b7a5a209a58f672fa23a3ad9c831958d37dd74a960f69c0075b5c92be457e, difficulty: 196416, Tx: 0
+    (Details of every block use `hivechain print`)
+    Chain Total Difficulty 135112316
+
+- blocks_1_td_196416.rlp
+    Block 1, 2a9b7a5a209a58f672fa23a3ad9c831958d37dd74a960f69c0075b5c92be457e, difficulty: 196416, Tx: 0
+    Chain Total Difficulty 196416
+
+- blocks_10_td_1971072_1.rlp
+    Block 1, de3a2a51b26b591495ace78793dc6a44288f912db59a04d155994237370fd318, difficulty: 196704, Tx: 2
+    Block 2, b97614ff71ad6c28145be39fde22d1b0410e4d072c425038f111530edee9af53, difficulty: 196800, Tx: 2
+    Block 3, c808dd0a22812c618d2ec50532a5a3bb5871043056784bc3148ad4379289ed04, difficulty: 196896, Tx: 2
+    Block 4, 1c29067fa2a5a6e23e635a4ca4a1420f80235f2141b69750a8fb3268559e5d12, difficulty: 196992, Tx: 2
+    Block 5, fffbc0a1c906855660d539d6c04da8eb8fe4e28acc5bcc9bb54560375b17653e, difficulty: 197088, Tx: 2
+    Block 6, f45296743964bde4a2f9141d322e7313a8a2472b2f2b518aa57917e62f97b94a, difficulty: 197184, Tx: 2
+    Block 7, ee11efa0870fc0950f44fc2767bec30829b17c308b7bdc47ab0d495af33d4d07, difficulty: 197280, Tx: 2
+    Block 8, 092d3f12996ca51ec93d43d2aedbdb578e3a3735492ca740e0df5bde73f5a042, difficulty: 197376, Tx: 2
+    Block 9, 2d30b49532c6ad67fe7751bb6c0893c8349f96c344167fd9db583c1ec4d27f72, difficulty: 197376, Tx: 2
+    Block 10, 0eb132b42bd47de3ffe0486bf568a476ee1ba17c37a3cd061f44c030ab61c4bd, difficulty: 197376, Tx: 2
+    Chain Total Difficulty 1971072
+
+- blocks_10_td_1971072_2.rlp
+    Block 1, f5b50675ee55ae3605bdc558dc988eab60bcfe5b9c8868aee03c4db101cbf4eb, difficulty: 196704, Tx: 2
+    Block 2, 0d3e37982030a5c641d91b77673d5c4193c3aafeaf68cddfcd796b84b1b89c92, difficulty: 196800, Tx: 2
+    Block 3, 3e7f66616b80d612ff2044202d5d12ada55cbb7ab4c59f5738bfcbf1a903ccc8, difficulty: 196896, Tx: 2
+    Block 4, fe33be274a100ff275407210cc6e5c07e3b936caf5f5b75e2415f92720277947, difficulty: 196992, Tx: 2
+    Block 5, 37eaab4452fbe6f4d36190f8676be2f7711e57475d18000dba23db69f5f75ad6, difficulty: 197088, Tx: 2
+    Block 6, d0112b28eb45ccd7c8387890098992943737ca85d5561ac8810ac5a2b456deb3, difficulty: 197184, Tx: 2
+    Block 7, 63ae8245a99aa3cade137089646ae9be583859ec599a75ef4389912c3524b96b, difficulty: 197280, Tx: 2
+    Block 8, d8c134a27c8fe2130ae84be2ffa65ae958155148f9d7acaf41fab2a78b47a0dc, difficulty: 197376, Tx: 2
+    Block 9, 9aabc2ab99f7a67245ff72e5f28c61bdfc16e00d860a96592c233e3b6367dc03, difficulty: 197376, Tx: 2
+    Block 10, 55ab665e11fc9395d042f0039d0d4f351b968e289dcdf62755caa5515addf5a2, difficulty: 197376, Tx: 2
+    Chain Total Difficulty 1971072
+
+- blocks_10_td_1971072_3.rlp
+    Block 1, e1ecffd3ace7657f8e236dbfa64069f9e178ffe464a7d09ba693f5d4858600d7, difficulty: 196704, Tx: 2
+    Block 2, 16ab63f8431d2aa266a4bc397c958e73eee87017ff5b14e9ed6eb126dc0e62f6, difficulty: 196800, Tx: 2
+    Block 3, e4c801ef8dfb13fb17522d238d343ed3536ce1ee1f7f3f6ebf9a9285f8f43fbb, difficulty: 196896, Tx: 2
+    Block 4, 2b07cfbfe868bf268ce0f7df7651887ad2d507be9e4f71161c0f7b25642b2540, difficulty: 196992, Tx: 2
+    Block 5, 3e79d8e7efb50830d01850d3936ffa22154691e1c5d0407c0e99cd3d419733a9, difficulty: 197088, Tx: 2
+    Block 6, dbee50a39659a935e977433594ac572dbb0576dcd6c319f142803a9e220a8f7d, difficulty: 197184, Tx: 2
+    Block 7, 90ecdff850685248ed16db8dae923b0e33023a229158003d647fb640255d268c, difficulty: 197280, Tx: 2
+    Block 8, 46ff0b8e2ad915b4e1b9035b57cd556c726b1f25313cd1c20b939a073c8e0cf3, difficulty: 197376, Tx: 2
+    Block 9, 6c93230c31d837220100d5ab295b6c3433b48b32f5ac3b4ba9cb98e2ef8c728f, difficulty: 197376, Tx: 2
+    Block 10, fc11761cd3e009f408cfe396d1492496f2b758c69a29a755148c03b734859431, difficulty: 197376, Tx: 2
+    Chain Total Difficulty 1971072
+
+- blocks_10_td_1971072_4.rlp
+    Block 1, 218ae446091aa23cff5bd42399783842c3679d47379f323f91ab01e16fc419b0, difficulty: 196704, Tx: 2
+    Block 2, 98ab26945e4b884c11f6efde04952479ef857f8d0a888b340c1276c52b45f77a, difficulty: 196800, Tx: 2
+    Block 3, 479d43266fdf45e16e73fec777d359b5264de9547bd90420b9f34b298d1f87f7, difficulty: 196896, Tx: 2
+    Block 4, 9d965795ee0e2bab2764d74be40e13bdc5799e89412330c294ed3a7bf83890d0, difficulty: 196992, Tx: 2
+    Block 5, 02934887ade02dfb1447287df9678050fd2a4b75209a8e9240d714403a37f830, difficulty: 197088, Tx: 2
+    Block 6, e6c78538577071ecc8e614c1ebff6c3f1edf42dae445256597c0a9802950f41f, difficulty: 197184, Tx: 2
+    Block 7, d1cc9a185a3945775045ded54674a2e1e5cde0de342234d468c917b015245eb3, difficulty: 197280, Tx: 2
+    Block 8, 451adf40b85dc9c39a9ad95594dd762e506d9fc6eaba7ff80d6466dfe5d66cf4, difficulty: 197376, Tx: 2
+    Block 9, 03e79ac09bb66810bdb3eaf0366d44aca92a338b31bf3ef7d34cc7930cb52839, difficulty: 197376, Tx: 2
+    Block 10, bc4ba3ed16e2fe3ae7f016788007044730a5fc4c5779d0dc06d63165fc9b1897, difficulty: 197376, Tx: 2
+    Chain Total Difficulty 1971072
+
+- blocks_10_td_1971072_5.rlp
+    Block 1, ae0f4bc591d55871c692c7d29f71fa274084648fd0ec1fac6d8f59e1e5b5b44e, difficulty: 196704, Tx: 2
+    Block 2, 0cb80cafec0062f8806ec30e62ae8b6dfa3c2e9a3004b17fc6b624bdbcae6f2c, difficulty: 196800, Tx: 2
+    Block 3, 0b52d79e6dd0d58c182866ff18656ab31e275dff8aa76f7f3782d2003348e42a, difficulty: 196896, Tx: 2
+    Block 4, cfc68655021271c2d6e0decfc959aff0de2ae2602ee682f1064355003206caf4, difficulty: 196992, Tx: 2
+    Block 5, 56e5850869e273bf3a05ab62e5121abcf0f2a0fee29ca60020bbfcfca4f8e69d, difficulty: 197088, Tx: 2
+    Block 6, 76870baeff8d9d5fa018049970e977666e90ff53dabc6214e1e833a2fced28c6, difficulty: 197184, Tx: 2
+    Block 7, 426270f8917ffb393227d46e8cb4581171227b7dffca19f04f47d0b54ed0d3d6, difficulty: 197280, Tx: 2
+    Block 8, a935304ff29b2c5ad8b6527b5dc2b96d65cd5f5337abeec3cfeb6b9d060d0861, difficulty: 197376, Tx: 2
+    Block 9, e817b97653bdbec402ea9c2ff0bbd1ce03e241b4e6431389b7cfd458a80be6b5, difficulty: 197376, Tx: 2
+    Block 10, 8476e7bff6b7f650f71d50639436907265e796d3d04d73c83e30bc734696072d, difficulty: 197376, Tx: 2
+    Chain Total Difficulty 1971072
+
+Chains were generated using the following command:
+    `hivechain generate -genesis ./init/genesis.json -mine -length 2|1`.

simulators/ethereum/engine/clmock.go

@@ -170,25 +170,16 @@ func (cl *CLMocker) pickNextPayloadProducer() {
 		ec_id := (int(cl.LatestFinalizedNumber.Int64()) + i) % len(cl.EngineClients)
 		cl.NextBlockProducer = cl.EngineClients[ec_id]
 
-		lastBlockNumber, err := cl.NextBlockProducer.Eth.BlockNumber(cl.NextBlockProducer.Ctx())
+		// Get latest header. Number and hash must coincide with our view of the chain,
+		// and only then we can build on top of this client's chain
+		latestHeader, err := cl.NextBlockProducer.Eth.HeaderByNumber(cl.NextBlockProducer.Ctx(), nil)
 		if err != nil {
-			cl.Fatalf("CLMocker: Could not get block number while selecting client for payload production (%v): %v", cl.NextBlockProducer.Client.Container, err)
-		}
-
-		if cl.LatestFinalizedNumber.Int64() != int64(lastBlockNumber) {
-			// Selected client is not synced to the last block number, try again
-			cl.NextBlockProducer = nil
-			continue
-		}
-
-		latestHeader, err := cl.NextBlockProducer.Eth.HeaderByNumber(cl.NextBlockProducer.Ctx(), big.NewInt(int64(lastBlockNumber)))
-		if err != nil {
-			cl.Fatalf("CLMocker: Could not get block header while selecting client for payload production (%v): %v", cl.NextBlockProducer.Client.Container, err)
+			cl.Fatalf("CLMocker: Could not get latest block header while selecting client for payload production (%v): %v", cl.NextBlockProducer.Client.Container, err)
 		}
 
 		lastBlockHash := latestHeader.Hash()
 
-		if cl.LatestFinalizedHeader.Hash() != lastBlockHash {
+		if cl.LatestFinalizedHeader.Hash() != lastBlockHash || cl.LatestFinalizedNumber.Cmp(latestHeader.Number) != 0 {
 			// Selected client latest block hash does not match canonical chain, try again
 			cl.NextBlockProducer = nil
 			continue

simulators/ethereum/engine/mergetests.go

@@ -5,6 +5,8 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/hive/hivesim"
 )
 
@@ -54,6 +56,11 @@ type MergeTestSpec struct {
 	// Chain file to initialize the main client.
 	MainChainFile string
 
+	// Introduce PREVRANDAO transactions on the PoS blocks, including transition,
+	// which could overwrite an existing transaction in the PoW chain (if re-org
+	// occurred to a lower-height chain)
+	PrevRandaoTransactions bool
+
 	// Whether or not to send a forkchoiceUpdated directive on the main client before any attempts to re-org
 	// to secondary clients happen.
 	SkipMainClientFcU bool
@@ -70,38 +77,6 @@ type MergeTestSpec struct {
 	SecondaryClientSpecs SecondaryClientSpecs
 }
 
-/*
-	Chains Included:
-		- blocks_1_td_196608.rlp
-			Block 1, ab5d6faf8d4460e2f58cff55055411bd66c2bbce7436d54bbc962576ce84605f, difficulty: 196608
-			Chain Total Difficulty 196608
-
-		- blocks_1_td_196704.rlp
-			Block 1, f37dca584c0f6abde6e9c0bb486ac42a8ac091156ab97aeb26136ee37a3aaef7, difficulty: 196704
-			Chain Total Difficulty 196704
-
-		- blocks_2_td_393120.rlp
-			Block 1, ab5d6faf8d4460e2f58cff55055411bd66c2bbce7436d54bbc962576ce84605f, difficulty: 196608
-			Block 2, 6e98e14794478cbfe8feaa24d4463b1a4e5d3743a66a49a851a3029d9ff3de60, difficulty: 196512
-			Chain Total Difficulty 393120
-
-		- blocks_2_td_393504.rlp
-			Block 1, f37dca584c0f6abde6e9c0bb486ac42a8ac091156ab97aeb26136ee37a3aaef7, difficulty: 196704
-			Block 2, ee5dbba4ccfdc75800bf98804be90d2ffe7fa9b2dce37b6fe31c891ca5fb87b8, difficulty: 196800
-			Chain Total Difficulty 393504
-
-		- blocks_1024_td_135112316.rlp
-			Block 1, 2a9b7a5a209a58f672fa23a3ad9c831958d37dd74a960f69c0075b5c92be457e, difficulty: 196416
-			* Details of every block use `hivechain print`
-			Chain Total Difficulty 135112316
-
-		- blocks_1_td_196416.rlp
-			Block 1, 2a9b7a5a209a58f672fa23a3ad9c831958d37dd74a960f69c0075b5c92be457e, difficulty: 196416
-			Chain Total Difficulty 196416
-
-	Chains were generated using the following command:
-		`hivechain generate -genesis ./init/genesis.json -mine -length 2|1`.
-*/
 var mergeTestSpecs = []MergeTestSpec{
 	{
 		Name:          "Single Block PoW Re-org to Higher-Total-Difficulty Chain, Equal Height",
@@ -175,6 +150,20 @@ var mergeTestSpecs = []MergeTestSpec{
 			},
 		},
 	},
+	{
+		Name:                     "Two Block PoW Re-org to Lower-Height Chain, Transaction Overwrite",
+		TTD:                      196704,
+		MainChainFile:            "blocks_2_td_393120.rlp",
+		KeepCheckingUntilTimeout: true,
+		PrevRandaoTransactions:   true,
+		SecondaryClientSpecs: []SecondaryClientSpec{
+			{
+				ChainFile:           "blocks_1_td_196704.rlp",
+				BuildPoSChainOnTop:  true,
+				MainClientShallSync: true,
+			},
+		},
+	},
 	{
 		Name:                "Two Block Post-PoS Re-org to Higher-Total-Difficulty PoW Chain",
 		TTD:                 196608,
@@ -385,11 +374,53 @@ func GenerateMergeTestSpec(mergeTestSpec MergeTestSpec) TestSpec {
 			}
 		}
 
+		// We are going to send PREVRANDAO transactions if the test requires so.
+		// These transactions might overwrite some of the PoW chain transactions if we re-org'd into a lower height chain.
+		prevRandaoTxs := make([]*types.Transaction, 0)
+		prevRandaoFunc := func() {
+			if mergeTestSpec.PrevRandaoTransactions {
+				// Get the address nonce:
+				// This is because we could have included transactions in the PoW chain of the block
+				// producer, or re-orged.
+				nonce, err := t.CLMock.NextBlockProducer.Eth.NonceAt(t.CLMock.NextBlockProducer.Ctx(), vaultAccountAddr, nil)
+				if err != nil {
+					t.Logf("INFO (%s): Unable to obtain address [%s] latest nonce: %v", t.TestName, vaultAccountAddr, err)
+					return
+				}
+				t.nonce = nonce
+				tx := t.makeNextTransaction(prevRandaoContractAddr, big0, nil)
+				err = t.CLMock.NextBlockProducer.Eth.SendTransaction(t.CLMock.NextBlockProducer.Ctx(), tx)
+				if err != nil {
+					t.Logf("INFO (%s): Unable to send tx (address=%v): %v", t.TestName, vaultAccountAddr, err)
+				}
+				prevRandaoTxs = append(prevRandaoTxs, tx)
+			}
+		}
+		if mergeTestSpec.PrevRandaoTransactions {
+			// At the end of the test we are going to verify that these transactions did produce the post-merge expected
+			// outcome, even if they had been previously executed on the PoW chain.
+			defer func() {
+				for _, tx := range prevRandaoTxs {
+					// Get the receipt of the transaction
+					r, err := t.Eth.TransactionReceipt(t.Ctx(), tx.Hash())
+					if err != nil {
+						t.Fatalf("FAIL (%s): Unable to obtain tx [%v] receipt: %v", t.TestName, tx.Hash(), err)
+					}
+
+					blockNumberAsStorageKey := common.BytesToHash(r.BlockNumber.Bytes())
+					s := t.TestEth.TestStorageAt(prevRandaoContractAddr, blockNumberAsStorageKey, nil)
+					s.ExpectStorageEqual(t.CLMock.PrevRandaoHistory[r.BlockNumber.Uint64()])
+				}
+			}()
+		}
+
 		// Test end state of the main client
 		for {
 			if mergeTestSpec.SecondaryClientSpecs.AnyPoSChainOnTop() {
 				// Build a block and check whether the main client switches
-				t.CLMock.produceSingleBlock(BlockProcessCallbacks{})
+				t.CLMock.produceSingleBlock(BlockProcessCallbacks{
+					OnPayloadProducerSelected: prevRandaoFunc,
+				})
 
 				// If the main client should follow the PoS chain, update the mustHeadHash
 				if mustHeadHash == t.CLMock.LatestFinalizedHeader.ParentHash {
@@ -421,7 +452,9 @@ func GenerateMergeTestSpec(mergeTestSpec MergeTestSpec) TestSpec {
 			for {
 				if mergeTestSpec.SecondaryClientSpecs.AnyPoSChainOnTop() {
 					// Build a block and check whether the main client switches
-					t.CLMock.produceSingleBlock(BlockProcessCallbacks{})
+					t.CLMock.produceSingleBlock(BlockProcessCallbacks{
+						OnPayloadProducerSelected: prevRandaoFunc,
+					})
 
 					// If the main client should follow the PoS chain, update the mustHeadHash
 					if mustHeadHash == t.CLMock.LatestFinalizedHeader.ParentHash {