diff --git a/simulators/ethereum/engine/clmock.go b/simulators/ethereum/engine/clmock.go index bbd8f574b8..0e53d2fc81 100644 --- a/simulators/ethereum/engine/clmock.go +++ b/simulators/ethereum/engine/clmock.go @@ -127,7 +127,6 @@ func (cl *CLMocker) setTTDBlockClient(ec *EngineClient) { if td == nil { cl.Fatalf("CLMocker: Returned TotalDifficultyHeader is invalid: %v", td) } - cl.Logf("CLMocker: Returned TotalDifficultyHeader: %v", td) if td.TotalDifficulty.ToInt().Cmp(ec.TerminalTotalDifficulty) < 0 { cl.Fatalf("CLMocker: Attempted to set TTD Block when TTD had not been reached: %v > %v", ec.TerminalTotalDifficulty, td.TotalDifficulty.ToInt()) @@ -173,9 +172,9 @@ func (cl *CLMocker) setTTDBlockClient(ec *EngineClient) { // This method waits for TTD in at least one of the clients, then sends the // initial forkchoiceUpdated with the info obtained from the client. func (cl *CLMocker) waitForTTD() { - ec := cl.EngineClients.waitForTTD(cl.Timeout) - if ec == nil { - cl.Fatalf("CLMocker: Timeout while waiting for TTD") + ec, err := cl.EngineClients.waitForTTD(cl.Timeout) + if ec == nil || err != nil { + cl.Fatalf("CLMocker: Error while waiting for TTD: %v", err) } // One of the clients has reached TTD, send the initial fcU with this client as head cl.setTTDBlockClient(ec) diff --git a/simulators/ethereum/engine/engineclient.go b/simulators/ethereum/engine/engineclient.go index 423144381b..204c965560 100644 --- a/simulators/ethereum/engine/engineclient.go +++ b/simulators/ethereum/engine/engineclient.go @@ -80,23 +80,35 @@ func (ec *EngineClient) Equals(ec2 *EngineClient) bool { return ec.Container == ec2.Container } -func (ec *EngineClient) checkTTD() bool { +func (ec *EngineClient) checkTTD() (bool, error) { var td *TotalDifficultyHeader if err := ec.cEth.CallContext(ec.Ctx(), &td, "eth_getBlockByNumber", "latest", false); err != nil { - panic(err) + return false, err } - return td.TotalDifficulty.ToInt().Cmp(ec.TerminalTotalDifficulty) >= 0 + return td.TotalDifficulty.ToInt().Cmp(ec.TerminalTotalDifficulty) >= 0, nil +} + +type WaitTTDResponse struct { + ec *EngineClient + err error } // Wait until the TTD is reached by a single client. -func (ec *EngineClient) waitForTTD(wg *sync.WaitGroup, done chan<- *EngineClient, cancel <-chan interface{}) { +func (ec *EngineClient) waitForTTD(wg *sync.WaitGroup, done chan<- WaitTTDResponse, cancel <-chan interface{}) { defer wg.Done() for { select { case <-time.After(tTDCheckPeriod): - if ec.checkTTD() { + ttdReached, err := ec.checkTTD() + if err != nil { + + } + if err != nil || ttdReached { select { - case done <- ec: + case done <- WaitTTDResponse{ + ec: ec, + err: err, + }: case <-cancel: } return @@ -109,24 +121,28 @@ func (ec *EngineClient) waitForTTD(wg *sync.WaitGroup, done chan<- *EngineClient // Wait until the TTD is reached by a single client with a timeout. // Returns true if the TTD has been reached, false when timeout occurred. -func (ec *EngineClient) waitForTTDWithTimeout(timeout <-chan time.Time) bool { +func (ec *EngineClient) waitForTTDWithTimeout(timeout <-chan time.Time) error { for { select { case <-time.After(tTDCheckPeriod): - if ec.checkTTD() { - return true + ttdReached, err := ec.checkTTD() + if err != nil { + return err + } + if ttdReached { + return nil } case <-timeout: - return false + return fmt.Errorf("Timeout") } } } // Wait until the TTD is reached by any of the engine clients -func (ecs EngineClients) waitForTTD(timeout <-chan time.Time) *EngineClient { +func (ecs EngineClients) waitForTTD(timeout <-chan time.Time) (*EngineClient, error) { var wg sync.WaitGroup defer wg.Wait() - done := make(chan *EngineClient) + done := make(chan WaitTTDResponse) cancel := make(chan interface{}) defer close(cancel) for _, ec := range ecs { @@ -134,10 +150,10 @@ func (ecs EngineClients) waitForTTD(timeout <-chan time.Time) *EngineClient { go ec.waitForTTD(&wg, done, cancel) } select { - case ec := <-done: - return ec + case r := <-done: + return r.ec, r.err case <-timeout: - return nil + return nil, fmt.Errorf("Timeout") } } diff --git a/simulators/ethereum/engine/enginetests.go b/simulators/ethereum/engine/enginetests.go index 80cd9d1bd9..c29f9b6471 100644 --- a/simulators/ethereum/engine/enginetests.go +++ b/simulators/ethereum/engine/enginetests.go @@ -259,12 +259,15 @@ var engineTests = []TestSpec{ SlotsToFinalized: big.NewInt(20), Run: invalidMissingAncestorReOrgGenSync(8, InvalidReceiptsRoot, false), }, - { - Name: "Invalid Ancestor Chain Re-Org, Invalid Number, Invalid P9', Reveal using sync", - TimeoutSeconds: 30, - SlotsToFinalized: big.NewInt(20), - Run: invalidMissingAncestorReOrgGenSync(9, InvalidNumber, false), - }, + /* + TODO, RE-ENABLE: Test is causing a panic on the secondary node, disabling for now. + { + Name: "Invalid Ancestor Chain Re-Org, Invalid Number, Invalid P9', Reveal using sync", + TimeoutSeconds: 30, + SlotsToFinalized: big.NewInt(20), + Run: invalidMissingAncestorReOrgGenSync(9, InvalidNumber, false), + }, + */ { Name: "Invalid Ancestor Chain Re-Org, Invalid GasLimit, Invalid P9', Reveal using sync", TimeoutSeconds: 30, @@ -283,12 +286,15 @@ var engineTests = []TestSpec{ SlotsToFinalized: big.NewInt(20), Run: invalidMissingAncestorReOrgGenSync(8, InvalidTimestamp, false), }, - { - Name: "Invalid Ancestor Chain Re-Org, Invalid PrevRandao, Invalid P9', Reveal using sync", - TimeoutSeconds: 30, - SlotsToFinalized: big.NewInt(20), - Run: invalidMissingAncestorReOrgGenSync(8, InvalidPrevRandao, false), - }, + /* + TODO, RE-ENABLE: Test consistently fails with Failed to set invalid block: missing trie node. + { + Name: "Invalid Ancestor Chain Re-Org, Invalid PrevRandao, Invalid P9', Reveal using sync", + TimeoutSeconds: 30, + SlotsToFinalized: big.NewInt(20), + Run: invalidMissingAncestorReOrgGenSync(8, InvalidPrevRandao, false), + }, + */ { Name: "Invalid Ancestor Chain Re-Org, Incomplete Transactions, Invalid P9', Reveal using sync", TimeoutSeconds: 30, @@ -1248,6 +1254,13 @@ func invalidMissingAncestorReOrgGenSync(invalid_index int, payloadField InvalidP genesis := loadGenesis("init/genesis.json") genesis.Config.TerminalTotalDifficulty = t.Engine.TerminalTotalDifficulty secondaryClient, err := newNode(enode, &genesis) + defer func() { + if err := secondaryClient.Stop(); err != nil { + t.Logf("WARN (%s): Unable to stop secondary node: %v", t.TestName, err) + } else { + t.Logf("INFO (%s): Successfully stopped secondary node", t.TestName) + } + }() if err != nil { t.Fatalf("FAIL (%s): Unable to spawn a secondary client: %v", t.TestName, err) } @@ -1332,10 +1345,10 @@ func invalidMissingAncestorReOrgGenSync(invalid_index int, payloadField InvalidP if i < invalid_index { status, err := secondaryClient.sendNewPayload(altChainPayloads[i]) if err != nil { - t.Fatalf("FAIL (%s): Unable to send new payload: %v", t.TestName, err) + t.Fatalf("FAIL (%s): TEST ISSUE - Unable to send new payload: %v", t.TestName, err) } if status.Status != "VALID" && status.Status != "ACCEPTED" { - t.Fatalf("FAIL (%s): Invalid payload status, expected VALID, ACCEPTED: %v", t.TestName, status.Status) + t.Fatalf("FAIL (%s): TEST ISSUE - Invalid payload status, expected VALID, ACCEPTED: %v", t.TestName, status.Status) } status2, err := secondaryClient.sendFCU(&ForkchoiceStateV1{ @@ -1344,20 +1357,20 @@ func invalidMissingAncestorReOrgGenSync(invalid_index int, payloadField InvalidP FinalizedBlockHash: cA.BlockHash, }, nil) if err != nil { - t.Fatalf("FAIL (%s): Unable to send new payload: %v", t.TestName, err) + t.Fatalf("FAIL (%s): TEST ISSUE - Unable to send new payload: %v", t.TestName, err) } if status2.PayloadStatus.Status != "VALID" && status2.PayloadStatus.Status != "SYNCING" { - t.Fatalf("FAIL (%s): Invalid payload status, expected VALID: %v", t.TestName, status2.PayloadStatus.Status) + t.Fatalf("FAIL (%s): TEST ISSUE - Invalid payload status, expected VALID: %v", t.TestName, status2.PayloadStatus.Status) } } else { invalid_block, err := beacon.ExecutableDataToBlock(execData(altChainPayloads[i])) if err != nil { - t.Fatalf("FAIL (%s): Failed to create block from payload: %v", t.TestName, err) + t.Fatalf("FAIL (%s): TEST ISSUE - Failed to create block from payload: %v", t.TestName, err) } if err := secondaryClient.setBlock(invalid_block, altChainPayloads[i-1].Number, altChainPayloads[i-1].StateRoot); err != nil { - t.Fatalf("FAIL (%s): Failed to set invalid block: %v", t.TestName, err) + t.Fatalf("FAIL (%s): TEST ISSUE - Failed to set invalid block: %v", t.TestName, err) } t.Logf("INFO (%s): Invalid block successfully set %d (%s): %v", t.TestName, i, payloadValidStr, altChainPayloads[i].BlockHash) } @@ -1365,9 +1378,9 @@ func invalidMissingAncestorReOrgGenSync(invalid_index int, payloadField InvalidP // Check that the second node has the correct head head := secondaryClient.eth.APIBackend.CurrentBlock() if head.Hash() != altChainPayloads[n-1].BlockHash { - t.Fatalf("Secondary Node has invalid blockhash got %v want %v gotNum %v wantNum %v", head.Hash(), altChainPayloads[n-1].BlockHash, head.Number(), altChainPayloads[n].Number) + t.Fatalf("FAIL (%s): TEST ISSUE - Secondary Node has invalid blockhash got %v want %v gotNum %v wantNum %v", t.TestName, head.Hash(), altChainPayloads[n-1].BlockHash, head.Number(), altChainPayloads[n].Number) } else { - t.Logf("Secondary Node has correct block") + t.Logf("INFO (%s): Secondary Node has correct block", t.TestName) } // If we are syncing through p2p, we need to keep polling until the client syncs the missing payloads diff --git a/simulators/ethereum/engine/main.go b/simulators/ethereum/engine/main.go index b1ed9035f1..6acb97bc18 100644 --- a/simulators/ethereum/engine/main.go +++ b/simulators/ethereum/engine/main.go @@ -137,6 +137,10 @@ have reached the Terminal Total Difficulty.`[1:], Parameters: newParams, Files: testFiles, Run: func(t *hivesim.T, c *hivesim.Client) { + t.Logf("Start test: %s", currentTest.Name) + defer func() { + t.Logf("End test: %s", currentTest.Name) + }() timeout := DefaultTestCaseTimeout // If a TestSpec specifies a timeout, use that instead if currentTest.TimeoutSeconds != 0 { diff --git a/simulators/ethereum/engine/mergetests.go b/simulators/ethereum/engine/mergetests.go index 1d5949360f..2471cb82d7 100644 --- a/simulators/ethereum/engine/mergetests.go +++ b/simulators/ethereum/engine/mergetests.go @@ -372,8 +372,8 @@ func GenerateMergeTestSpec(mergeTestSpec MergeTestSpec) TestSpec { runFunc := func(t *TestEnv) { // The first client waits for TTD, which ideally should be reached immediately using loaded chain if !mergeTestSpec.SkipMainClientTTDWait { - if !t.Engine.waitForTTDWithTimeout(t.Timeout) { - t.Fatalf("FAIL (%s): Timeout waiting for EngineClient (%s) to reach TTD", t.TestName, t.Engine.Container) + if err := t.Engine.waitForTTDWithTimeout(t.Timeout); err != nil { + t.Fatalf("FAIL (%s): Error while waiting for EngineClient (%s) to reach TTD: %v", t.TestName, t.Engine.Container, err) } if !mergeTestSpec.SkipMainClientFcU { @@ -411,8 +411,8 @@ func GenerateMergeTestSpec(mergeTestSpec MergeTestSpec) TestSpec { t.CLMock.AddEngineClient(t.T, secondaryClient.Client, big.NewInt(secondaryClientSpec.TTD)) if secondaryClientSpec.BuildPoSChainOnTop { - if !secondaryClient.waitForTTDWithTimeout(t.Timeout) { - t.Fatalf("FAIL (%s): Timeout waiting for EngineClient (%s) to reach TTD", t.TestName, secondaryClient.Client.Container) + if err := secondaryClient.waitForTTDWithTimeout(t.Timeout); err != nil { + t.Fatalf("FAIL (%s): Error while waiting for EngineClient (%s) to reach TTD: %v", t.TestName, secondaryClient.Client.Container, err) } t.CLMock.setTTDBlockClient(secondaryClient) } diff --git a/simulators/ethereum/engine/node.go b/simulators/ethereum/engine/node.go index 8b642fbf47..9e69bb727a 100644 --- a/simulators/ethereum/engine/node.go +++ b/simulators/ethereum/engine/node.go @@ -98,6 +98,10 @@ func restart(bootnode, datadir string, genesis *core.Genesis) (*gethNode, error) }, err } +func (n *gethNode) Stop() error { + return n.eth.Stop() +} + type validator struct{} func (v *validator) ValidateBody(block *types.Block) error { diff --git a/simulators/ethereum/engine/testenv.go b/simulators/ethereum/engine/testenv.go index f753e79f69..630d1d1f6b 100644 --- a/simulators/ethereum/engine/testenv.go +++ b/simulators/ethereum/engine/testenv.go @@ -143,21 +143,21 @@ func (t *TestEnv) StartClient(clientType string, params hivesim.Params, ttd *big func (t *TestEnv) makeNextTransaction(recipient common.Address, amount *big.Int, payload []byte) *types.Transaction { gasLimit := uint64(75000) - tx := types.NewTransaction(t.nonce, recipient, amount, gasLimit, gasPrice, payload) signer := types.NewEIP155Signer(chainID) signedTx, err := types.SignTx(tx, signer, vaultKey) if err != nil { t.Fatal("FAIL (%s): could not sign new tx: %v", t.TestName, err) } + t.Logf("INFO (%s): Built next transaction: hash=%s, nonce=%d, recipient=%s", t.TestName, signedTx.Hash(), t.nonce, recipient) t.nonce++ return signedTx } -func (t *TestEnv) sendNextTransaction(sender *EngineClient, recipient common.Address, amount *big.Int, payload []byte) *types.Transaction { +func (t *TestEnv) sendNextTransaction(node *EngineClient, recipient common.Address, amount *big.Int, payload []byte) *types.Transaction { tx := t.makeNextTransaction(recipient, amount, payload) for { - err := sender.Eth.SendTransaction(sender.Ctx(), tx) + err := node.Eth.SendTransaction(node.Ctx(), tx) if err == nil { return tx }