Skip to content

simulators/ethereum/engine: CLMock safe/finalized Slot Delay #541

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fjl merged 2 commits into from
May 19, 2022
Merged

simulators/ethereum/engine: CLMock safe/finalized Slot Delay #541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b95ace4
simulators/ethereum/engine: clmock safe/finalized slot delay
marioevz May 17, 2022
3c1c8d9
simulators/ethereum/engine: merge tests finalized slots increase
marioevz May 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 81 additions & 48 deletions simulators/ethereum/engine/clmock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,21 @@ import (
"github.com/ethereum/hive/hivesim"
)

var (
DefaultSlotsToSafe = big.NewInt(1)
DefaultSlotsToFinalized = big.NewInt(2)
)

// Consensus Layer Client Mock used to sync the Execution Clients once the TTD has been reached
type CLMocker struct {
*hivesim.T
// List of Engine Clients being served by the CL Mocker
EngineClients EngineClients
// Lock required so no client is offboarded during block production.
EngineClientsLock sync.Mutex
// Number of required slots before a block which was set as Head moves to `safe` and `finalized` respectively
SlotsToSafe *big.Int
SlotsToFinalized *big.Int

// Block Production State
NextBlockProducer *EngineClient
Expand All @@ -27,10 +35,11 @@ type CLMocker struct {
// PoS Chain History Information
PrevRandaoHistory map[uint64]common.Hash
ExecutedPayloadHistory map[uint64]ExecutableDataV1
HeadHashHistory []common.Hash

// Latest broadcasted data using the PoS Engine API
LatestFinalizedNumber *big.Int
LatestFinalizedHeader *types.Header
LatestHeadNumber *big.Int
LatestHeader *types.Header
LatestPayloadBuilt ExecutableDataV1
LatestPayloadAttributes PayloadAttributesV1
LatestExecutedPayload ExecutableDataV1
Expand All @@ -44,21 +53,31 @@ type CLMocker struct {
Timeout <-chan time.Time
}

func NewCLMocker(t *hivesim.T, tTD *big.Int) *CLMocker {
func NewCLMocker(t *hivesim.T, slotsToSafe *big.Int, slotsToFinalized *big.Int) *CLMocker {
// Init random seed for different purposes
seed := time.Now().Unix()
t.Logf("Randomness seed: %v\n", seed)
rand.Seed(seed)

if slotsToSafe == nil {
// Use default
slotsToSafe = DefaultSlotsToSafe
}
if slotsToFinalized == nil {
// Use default
slotsToFinalized = DefaultSlotsToFinalized
}
// Create the new CL mocker
newCLMocker := &CLMocker{
T: t,
EngineClients: make([]*EngineClient, 0),
PrevRandaoHistory: map[uint64]common.Hash{},
ExecutedPayloadHistory: map[uint64]ExecutableDataV1{},
LatestFinalizedHeader: nil,
SlotsToSafe: slotsToSafe,
SlotsToFinalized: slotsToFinalized,
LatestHeader: nil,
FirstPoSBlockNumber: nil,
LatestFinalizedNumber: nil,
LatestHeadNumber: nil,
TTDReached: false,
NextFeeRecipient: common.Address{},
LatestForkchoice: ForkchoiceStateV1{
Expand Down Expand Up @@ -114,15 +133,26 @@ func (cl *CLMocker) setTTDBlockClient(ec *EngineClient) {
cl.Fatalf("CLMocker: Attempted to set TTD Block when TTD had not been reached: %v > %v", ec.TerminalTotalDifficulty, td.TotalDifficulty.ToInt())
}

cl.LatestFinalizedHeader = &td.Header
cl.LatestHeader = &td.Header
cl.TTDReached = true
cl.Logf("CLMocker: TTD has been reached at block %v (%v>=%v)\n", cl.LatestFinalizedHeader.Number, td.TotalDifficulty.ToInt(), ec.TerminalTotalDifficulty)
cl.Logf("CLMocker: TTD has been reached at block %v (%v>=%v)\n", cl.LatestHeader.Number, td.TotalDifficulty.ToInt(), ec.TerminalTotalDifficulty)

// Reset transition values
cl.LatestHeadNumber = cl.LatestHeader.Number
cl.HeadHashHistory = []common.Hash{cl.LatestHeader.Hash()}
cl.FirstPoSBlockNumber = nil

// Prepare initial forkchoice
cl.LatestForkchoice = ForkchoiceStateV1{}
cl.LatestForkchoice.HeadBlockHash = cl.LatestHeader.Hash()
if cl.SlotsToSafe.Cmp(big0) == 0 {
cl.LatestForkchoice.SafeBlockHash = cl.LatestHeader.Hash()
}
if cl.SlotsToFinalized.Cmp(big0) == 0 {
cl.LatestForkchoice.FinalizedBlockHash = cl.LatestHeader.Hash()
}

// Broadcast initial ForkchoiceUpdated
cl.LatestForkchoice.HeadBlockHash = cl.LatestFinalizedHeader.Hash()
cl.LatestForkchoice.SafeBlockHash = cl.LatestFinalizedHeader.Hash()
cl.LatestForkchoice.FinalizedBlockHash = cl.LatestFinalizedHeader.Hash()
cl.LatestFinalizedNumber = cl.LatestFinalizedHeader.Number
anySuccess := false
for _, resp := range cl.broadcastForkchoiceUpdated(&cl.LatestForkchoice, nil) {
if resp.Error != nil {
Expand Down Expand Up @@ -167,7 +197,7 @@ func (cl *CLMocker) pickNextPayloadProducer() {

for i := 0; i < len(cl.EngineClients); i++ {
// Get a client to generate the payload
ec_id := (int(cl.LatestFinalizedNumber.Int64()) + i) % len(cl.EngineClients)
ec_id := (int(cl.LatestHeadNumber.Int64()) + i) % len(cl.EngineClients)
cl.NextBlockProducer = cl.EngineClients[ec_id]

// Get latest header. Number and hash must coincide with our view of the chain,
Expand All @@ -179,7 +209,7 @@ func (cl *CLMocker) pickNextPayloadProducer() {

lastBlockHash := latestHeader.Hash()

if cl.LatestFinalizedHeader.Hash() != lastBlockHash || cl.LatestFinalizedNumber.Cmp(latestHeader.Number) != 0 {
if cl.LatestHeader.Hash() != lastBlockHash || cl.LatestHeadNumber.Cmp(latestHeader.Number) != 0 {
// Selected client latest block hash does not match canonical chain, try again
cl.NextBlockProducer = nil
continue
Expand All @@ -200,13 +230,13 @@ func (cl *CLMocker) getNextPayloadID() {
rand.Read(nextPrevRandao[:])

cl.LatestPayloadAttributes = PayloadAttributesV1{
Timestamp: cl.LatestFinalizedHeader.Time + 1,
Timestamp: cl.LatestHeader.Time + 1,
PrevRandao: nextPrevRandao,
SuggestedFeeRecipient: cl.NextFeeRecipient,
}

// Save random value
cl.PrevRandaoHistory[cl.LatestFinalizedHeader.Number.Uint64()+1] = nextPrevRandao
cl.PrevRandaoHistory[cl.LatestHeader.Number.Uint64()+1] = nextPrevRandao

resp, err := cl.NextBlockProducer.EngineForkchoiceUpdatedV1(cl.NextBlockProducer.Ctx(), &cl.LatestForkchoice, &cl.LatestPayloadAttributes)
if err != nil {
Expand Down Expand Up @@ -236,11 +266,11 @@ func (cl *CLMocker) getNextPayload() {
if cl.LatestPayloadBuilt.PrevRandao != cl.LatestPayloadAttributes.PrevRandao {
cl.Fatalf("CLMocker: Incorrect PrevRandao on payload built: %v != %v", cl.LatestPayloadBuilt.PrevRandao, cl.LatestPayloadAttributes.PrevRandao)
}
if cl.LatestPayloadBuilt.ParentHash != cl.LatestFinalizedHeader.Hash() {
cl.Fatalf("CLMocker: Incorrect ParentHash on payload built: %v != %v", cl.LatestPayloadBuilt.ParentHash, cl.LatestFinalizedHeader.Hash())
if cl.LatestPayloadBuilt.ParentHash != cl.LatestHeader.Hash() {
cl.Fatalf("CLMocker: Incorrect ParentHash on payload built: %v != %v", cl.LatestPayloadBuilt.ParentHash, cl.LatestHeader.Hash())
}
if cl.LatestPayloadBuilt.Number != cl.LatestFinalizedHeader.Number.Uint64()+1 {
cl.Fatalf("CLMocker: Incorrect Number on payload built: %v != %v", cl.LatestPayloadBuilt.Number, cl.LatestFinalizedHeader.Number.Uint64()+1)
if cl.LatestPayloadBuilt.Number != cl.LatestHeader.Number.Uint64()+1 {
cl.Fatalf("CLMocker: Incorrect Number on payload built: %v != %v", cl.LatestPayloadBuilt.Number, cl.LatestHeader.Number.Uint64()+1)
}
}

Expand Down Expand Up @@ -304,13 +334,13 @@ func (cl *CLMocker) broadcastLatestForkchoice() {
}

type BlockProcessCallbacks struct {
OnPayloadProducerSelected func()
OnGetPayloadID func()
OnGetPayload func()
OnNewPayloadBroadcast func()
OnHeadBlockForkchoiceBroadcast func()
OnSafeBlockForkchoiceBroadcast func()
OnFinalizedBlockForkchoiceBroadcast func()
OnPayloadProducerSelected func()
OnGetPayloadID func()
OnGetPayload func()
OnNewPayloadBroadcast func()
OnForkchoiceBroadcast func()
OnSafeBlockChange func()
OnFinalizedBlockChange func()
}

func (cl *CLMocker) produceSingleBlock(callbacks BlockProcessCallbacks) {
Expand Down Expand Up @@ -351,37 +381,44 @@ func (cl *CLMocker) produceSingleBlock(callbacks BlockProcessCallbacks) {
}

// Broadcast forkchoice updated with new HeadBlock to all clients
previousForkchoice := cl.LatestForkchoice
cl.HeadHashHistory = append(cl.HeadHashHistory, cl.LatestPayloadBuilt.BlockHash)

cl.LatestForkchoice = ForkchoiceStateV1{}
cl.LatestForkchoice.HeadBlockHash = cl.LatestPayloadBuilt.BlockHash
if len(cl.HeadHashHistory) > int(cl.SlotsToSafe.Int64()) {
cl.LatestForkchoice.SafeBlockHash = cl.HeadHashHistory[len(cl.HeadHashHistory)-int(cl.SlotsToSafe.Int64())-1]
}
if len(cl.HeadHashHistory) > int(cl.SlotsToFinalized.Int64()) {
cl.LatestForkchoice.FinalizedBlockHash = cl.HeadHashHistory[len(cl.HeadHashHistory)-int(cl.SlotsToFinalized.Int64())-1]
}
cl.broadcastLatestForkchoice()

if callbacks.OnHeadBlockForkchoiceBroadcast != nil {
callbacks.OnHeadBlockForkchoiceBroadcast()
if callbacks.OnForkchoiceBroadcast != nil {
callbacks.OnForkchoiceBroadcast()
}

// Broadcast forkchoice updated with new SafeBlock to all clients
cl.LatestForkchoice.SafeBlockHash = cl.LatestPayloadBuilt.BlockHash
cl.broadcastLatestForkchoice()

if callbacks.OnSafeBlockForkchoiceBroadcast != nil {
callbacks.OnSafeBlockForkchoiceBroadcast()
if callbacks.OnSafeBlockChange != nil && previousForkchoice.SafeBlockHash != cl.LatestForkchoice.SafeBlockHash {
callbacks.OnSafeBlockChange()
}

// Broadcast forkchoice updated with new FinalizedBlock to all clients
cl.LatestForkchoice.FinalizedBlockHash = cl.LatestPayloadBuilt.BlockHash
cl.broadcastLatestForkchoice()
if callbacks.OnFinalizedBlockChange != nil && previousForkchoice.FinalizedBlockHash != cl.LatestForkchoice.FinalizedBlockHash {
callbacks.OnFinalizedBlockChange()
}

// Save the number of the first PoS block
if cl.FirstPoSBlockNumber == nil {
cl.FirstPoSBlockNumber = big.NewInt(int64(cl.LatestFinalizedHeader.Number.Uint64() + 1))
cl.FirstPoSBlockNumber = big.NewInt(int64(cl.LatestHeader.Number.Uint64() + 1))
}

// Save the header of the latest block in the PoS chain
cl.LatestFinalizedNumber = cl.LatestFinalizedNumber.Add(cl.LatestFinalizedNumber, big1)
cl.LatestHeadNumber = cl.LatestHeadNumber.Add(cl.LatestHeadNumber, big1)

// Check if any of the clients accepted the new payload
cl.LatestFinalizedHeader = nil
cl.LatestHeader = nil
for _, ec := range cl.EngineClients {
newHeader, err := ec.Eth.HeaderByNumber(cl.NextBlockProducer.Ctx(), cl.LatestFinalizedNumber)
newHeader, err := ec.Eth.HeaderByNumber(cl.NextBlockProducer.Ctx(), cl.LatestHeadNumber)
if err != nil {
continue
}
Expand All @@ -398,8 +435,8 @@ func (cl *CLMocker) produceSingleBlock(callbacks BlockProcessCallbacks) {
cl.Fatalf("CLMocker: Client %v produced a new header with incorrect difficulty: %v", ec.Container, newHeader.Difficulty)
}
// mixHash == prevRandao
if newHeader.MixDigest != cl.PrevRandaoHistory[cl.LatestFinalizedNumber.Uint64()] {
cl.Fatalf("CLMocker: Client %v produced a new header with incorrect mixHash: %v != %v", ec.Container, newHeader.MixDigest, cl.PrevRandaoHistory[cl.LatestFinalizedNumber.Uint64()])
if newHeader.MixDigest != cl.PrevRandaoHistory[cl.LatestHeadNumber.Uint64()] {
cl.Fatalf("CLMocker: Client %v produced a new header with incorrect mixHash: %v != %v", ec.Container, newHeader.MixDigest, cl.PrevRandaoHistory[cl.LatestHeadNumber.Uint64()])
}
// nonce == 0x0000000000000000
if newHeader.Nonce != (types.BlockNonce{}) {
Expand All @@ -408,16 +445,12 @@ func (cl *CLMocker) produceSingleBlock(callbacks BlockProcessCallbacks) {
if len(newHeader.Extra) > 32 {
cl.Fatalf("CLMocker: Client %v produced a new header with incorrect extraData (len > 32): %v", ec.Container, newHeader.Extra)
}
cl.LatestFinalizedHeader = newHeader
cl.LatestHeader = newHeader
}
if cl.LatestFinalizedHeader == nil {
if cl.LatestHeader == nil {
cl.Fatalf("CLMocker: None of the clients accepted the newly constructed payload")
}

if callbacks.OnFinalizedBlockForkchoiceBroadcast != nil {
callbacks.OnFinalizedBlockForkchoiceBroadcast()
}

}

// Loop produce PoS blocks by using the Engine API
Expand Down
32 changes: 16 additions & 16 deletions simulators/ethereum/engine/enginetests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -761,15 +761,15 @@ func badHashOnNewPayloadGen(syncing bool, sidechain bool) func(*TestEnv) {
// We alter the payload by setting the parent to a known past block in the
// canonical chain, which makes this payload a side chain payload, and also an invalid block hash
// (because we did not update the block hash appropriately)
alteredPayload.ParentHash = t.CLMock.LatestFinalizedHeader.ParentHash
alteredPayload.ParentHash = t.CLMock.LatestHeader.ParentHash
} else if syncing {
// We need to send an fcU to put the client in SYNCING state.
randomHeadBlock := common.Hash{}
rand.Read(randomHeadBlock[:])
fcU := ForkchoiceStateV1{
HeadBlockHash: randomHeadBlock,
SafeBlockHash: t.CLMock.LatestFinalizedHeader.Hash(),
FinalizedBlockHash: t.CLMock.LatestFinalizedHeader.Hash(),
SafeBlockHash: t.CLMock.LatestHeader.Hash(),
FinalizedBlockHash: t.CLMock.LatestHeader.Hash(),
}
r := t.TestEngine.TestEngineForkchoiceUpdatedV1(&fcU, nil)
r.ExpectPayloadStatus(Syncing)
Expand All @@ -778,7 +778,7 @@ func badHashOnNewPayloadGen(syncing bool, sidechain bool) func(*TestEnv) {
// Syncing and sidechain, the caonincal head is an unknown payload to us,
// but this specific bad hash payload is in theory part of a side chain.
// Therefore the parent we use is the head hash.
alteredPayload.ParentHash = t.CLMock.LatestFinalizedHeader.Hash()
alteredPayload.ParentHash = t.CLMock.LatestHeader.Hash()
} else {
// The invalid bad-hash payload points to the unknown head, but we know it is
// indeed canonical because the head was set using forkchoiceUpdated.
Expand Down Expand Up @@ -1257,7 +1257,7 @@ func blockStatusExecPayloadGen(transitionBlock bool) func(t *TestEnv) {
r.ExpectHash(t.CLMock.LatestForkchoice.HeadBlockHash)

s := t.TestEth.TestBlockNumber()
s.ExpectNumber(t.CLMock.LatestFinalizedNumber.Uint64())
s.ExpectNumber(t.CLMock.LatestHeadNumber.Uint64())

p := t.TestEth.TestBlockByNumber(nil)
p.ExpectHash(t.CLMock.LatestForkchoice.HeadBlockHash)
Expand Down Expand Up @@ -1287,7 +1287,7 @@ func blockStatusHeadBlockGen(transitionBlock bool) func(t *TestEnv) {
tx = t.sendNextTransaction(t.TestEngine.Engine, (common.Address{}), big1, nil)
},
// Run test after a forkchoice with new HeadBlockHash has been broadcasted
OnHeadBlockForkchoiceBroadcast: func() {
OnForkchoiceBroadcast: func() {
r := t.TestEth.TestHeaderByNumber(nil)
r.ExpectHash(t.CLMock.LatestForkchoice.HeadBlockHash)

Expand Down Expand Up @@ -1315,7 +1315,7 @@ func blockStatusSafeBlockGen(transitionBlock bool) func(t *TestEnv) {
tx = t.sendNextTransaction(t.TestEngine.Engine, (common.Address{}), big1, nil)
},
// Run test after a forkchoice with new SafeBlockHash has been broadcasted
OnSafeBlockForkchoiceBroadcast: func() {
OnSafeBlockChange: func() {
r := t.TestEth.TestHeaderByNumber(nil)
r.ExpectHash(t.CLMock.LatestForkchoice.HeadBlockHash)

Expand Down Expand Up @@ -1343,7 +1343,7 @@ func blockStatusFinalizedBlockGen(transitionBlock bool) func(t *TestEnv) {
tx = t.sendNextTransaction(t.TestEngine.Engine, (common.Address{}), big1, nil)
},
// Run test after a forkchoice with new FinalizedBlockHash has been broadcasted
OnFinalizedBlockForkchoiceBroadcast: func() {
OnFinalizedBlockChange: func() {
r := t.TestEth.TestHeaderByNumber(nil)
r.ExpectHash(t.CLMock.LatestForkchoice.HeadBlockHash)

Expand Down Expand Up @@ -1388,7 +1388,7 @@ func blockStatusReorg(t *TestEnv) {
r.ExpectHash(customizedPayload.BlockHash)

},
OnHeadBlockForkchoiceBroadcast: func() {
OnForkchoiceBroadcast: func() {
// At this point, we have re-org'd to the payload that the CLMocker was originally planning to send,
// verify that the client is serving the latest HeadBlock.
r := t.TestEth.TestHeaderByNumber(nil)
Expand All @@ -1412,7 +1412,7 @@ func reorgBack(t *TestEnv) {

// Produce blocks before starting the test (So we don't try to reorg back to the genesis block)
t.CLMock.produceBlocks(5, BlockProcessCallbacks{
OnHeadBlockForkchoiceBroadcast: func() {
OnForkchoiceBroadcast: func() {
// Send a fcU with the HeadBlockHash pointing back to the previous block
forkchoiceUpdatedBack := ForkchoiceStateV1{
HeadBlockHash: previousHash,
Expand Down Expand Up @@ -1530,7 +1530,7 @@ func transactionReorg(t *TestEnv) {
t.Fatalf("FAIL (%s): Payload built does not contain the transaction: %v", t.TestName, t.CLMock.LatestPayloadBuilt)
}
},
OnHeadBlockForkchoiceBroadcast: func() {
OnForkchoiceBroadcast: func() {
// Transaction is now in the head of the canonical chain, re-org and verify it's removed
// Get the receipt
_, err := t.Eth.TransactionReceipt(t.Ctx(), tx.Hash())
Expand Down Expand Up @@ -1595,7 +1595,7 @@ func sidechainReorg(t *TestEnv) {

r := t.TestEngine.TestEngineForkchoiceUpdatedV1(&t.CLMock.LatestForkchoice,
&PayloadAttributesV1{
Timestamp: t.CLMock.LatestFinalizedHeader.Time + 1,
Timestamp: t.CLMock.LatestHeader.Time + 1,
PrevRandao: alternativePrevRandao,
SuggestedFeeRecipient: t.CLMock.NextFeeRecipient,
})
Expand Down Expand Up @@ -1628,7 +1628,7 @@ func sidechainReorg(t *TestEnv) {
})
// The reorg actually happens after the CLMocker continues,
// verify here that the reorg was successful
latestBlockNum := t.CLMock.LatestFinalizedNumber.Uint64()
latestBlockNum := t.CLMock.LatestHeadNumber.Uint64()
checkPrevRandaoValue(t, t.CLMock.PrevRandaoHistory[latestBlockNum], latestBlockNum)

}
Expand Down Expand Up @@ -1976,9 +1976,9 @@ func prevRandaoOpcodeTx(t *TestEnv) {
txs = append(txs, tx)
currentTxIndex++
},
OnHeadBlockForkchoiceBroadcast: func() {
OnForkchoiceBroadcast: func() {
// Check the transaction tracing, which is client specific
expectedPrevRandao := t.CLMock.PrevRandaoHistory[t.CLMock.LatestFinalizedHeader.Number.Uint64()+1]
expectedPrevRandao := t.CLMock.PrevRandaoHistory[t.CLMock.LatestHeader.Number.Uint64()+1]
if err := debugPrevRandaoTransaction(t.Engine.Ctx(), t.RPC, t.Engine.Client.Type, txs[currentTxIndex-1],
&expectedPrevRandao); err != nil {
t.Fatalf("FAIL (%s): Error during transaction tracing: %v", t.TestName, err)
Expand Down Expand Up @@ -2040,7 +2040,7 @@ func postMergeSync(t *TestEnv) {
if err != nil {
t.Fatalf("FAIL (%s): Unable to obtain latest header: %v", t.TestName, err)
}
if t.CLMock.LatestFinalizedHeader != nil && latestHeader.Hash() == t.CLMock.LatestFinalizedHeader.Hash() {
if t.CLMock.LatestHeader != nil && latestHeader.Hash() == t.CLMock.LatestHeader.Hash() {
t.Logf("INFO (%v): Client (%v) is now synced to latest PoS block: %v", t.TestName, c.Container, latestHeader.Hash())
break syncLoop
}
Expand Down
Loading