simulators/eth2/withdrawals: Withdrawals Tests Fixes

simulators/eth2/common/clients/beacon.go

@@ -38,9 +38,7 @@ const (
 	PortValidatorAPI = 5000
 )
 
-var (
-	EMPTY_TREE_ROOT = tree.Root{}
-)
+var EMPTY_TREE_ROOT = tree.Root{}
 
 type BeaconClient struct {
 	T                     *hivesim.T
@@ -98,6 +96,11 @@ func (bn *BeaconClient) Start(extraOptions ...hivesim.StartOption) error {
 		Cli:   &http.Client{},
 		Codec: eth2api.JSONCodec{},
 	}
+	bn.T.Logf(
+		"Started client %s, container: %s",
+		bn.ClientType,
+		bn.HiveClient.Container,
+	)
 	return nil
 }
 
@@ -223,6 +226,14 @@ func (versionedBlock *VersionedSignedBeaconBlock) ExecutionPayload() (api.Execut
 	return result, nil
 }
 
+func (versionedBlock *VersionedSignedBeaconBlock) Withdrawals() (common.Withdrawals, error) {
+	switch v := versionedBlock.Data.(type) {
+	case *capella.SignedBeaconBlock:
+		return v.Message.Body.ExecutionPayload.Withdrawals, nil
+	}
+	return nil, nil
+}
+
 func (b *VersionedSignedBeaconBlock) StateRoot() tree.Root {
 	switch v := b.Data.(type) {
 	case *phase0.SignedBeaconBlock:
@@ -445,6 +456,20 @@ type VersionedBeaconStateResponse struct {
 	spec *common.Spec
 }
 
+func (vbs *VersionedBeaconStateResponse) Root() tree.Root {
+	switch state := vbs.Data.(type) {
+	case *phase0.BeaconState:
+		return state.HashTreeRoot(vbs.spec, tree.GetHashFn())
+	case *altair.BeaconState:
+		return state.HashTreeRoot(vbs.spec, tree.GetHashFn())
+	case *bellatrix.BeaconState:
+		return state.HashTreeRoot(vbs.spec, tree.GetHashFn())
+	case *capella.BeaconState:
+		return state.HashTreeRoot(vbs.spec, tree.GetHashFn())
+	}
+	panic("badly formatted beacon state")
+}
+
 func (vbs *VersionedBeaconStateResponse) CurrentVersion() common.Version {
 	switch state := vbs.Data.(type) {
 	case *phase0.BeaconState:
@@ -565,6 +590,24 @@ func (vbs *VersionedBeaconStateResponse) LatestExecutionPayloadHeaderHash() tree
 	panic("badly formatted beacon state")
 }
 
+func (vbs *VersionedBeaconStateResponse) NextWithdrawalIndex() (common.WithdrawalIndex, error) {
+	var wIndex common.WithdrawalIndex
+	switch state := vbs.Data.(type) {
+	case *capella.BeaconState:
+		wIndex = state.NextWithdrawalIndex
+	}
+	return wIndex, nil
+}
+
+func (vbs *VersionedBeaconStateResponse) NextWithdrawalValidatorIndex() (common.ValidatorIndex, error) {
+	var wIndex common.ValidatorIndex
+	switch state := vbs.Data.(type) {
+	case *capella.BeaconState:
+		wIndex = state.NextWithdrawalValidatorIndex
+	}
+	return wIndex, nil
+}
+
 func (vbs *VersionedBeaconStateResponse) NextWithdrawals(
 	slot common.Slot,
 ) (common.Withdrawals, error) {

simulators/eth2/common/clients/execution.go

@@ -123,6 +123,11 @@ func (en *ExecutionClient) Start(extraOptions ...hivesim.StartOption) error {
 	opts = append(opts, extraOptions...)
 
 	en.HiveClient = en.T.StartClient(en.ClientType, opts...)
+	en.T.Logf(
+		"Started client %s, container: %s",
+		en.ClientType,
+		en.HiveClient.Container,
+	)
 
 	// Prepare Eth/Engine RPCs
 	engineRPCAddress, err := en.EngineRPCAddress()

simulators/eth2/common/clients/validator.go

@@ -58,6 +58,11 @@ func (vc *ValidatorClient) Start(extraOptions ...hivesim.StartOption) error {
 	}
 
 	vc.HiveClient = vc.T.StartClient(vc.ClientType, opts...)
+	vc.T.Logf(
+		"Started client %s, container: %s",
+		vc.ClientType,
+		vc.HiveClient.Container,
+	)
 	return nil
 }
 

simulators/eth2/common/config/consensus/genesis.go

@@ -60,6 +60,59 @@ func genesisPayloadHeader(
 	}, nil
 }
 
+func genesisPayloadHeaderCapella(
+	eth1GenesisBlock *types.Block,
+	spec *common.Spec,
+) (*capella.ExecutionPayloadHeader, error) {
+	extra := eth1GenesisBlock.Extra()
+	if len(extra) > common.MAX_EXTRA_DATA_BYTES {
+		return nil, fmt.Errorf(
+			"extra data is %d bytes, max is %d",
+			len(extra),
+			common.MAX_EXTRA_DATA_BYTES,
+		)
+	}
+	if len(eth1GenesisBlock.Transactions()) != 0 {
+		return nil, fmt.Errorf(
+			"expected no transactions in genesis execution payload",
+		)
+	}
+	if len(eth1GenesisBlock.Withdrawals()) != 0 {
+		return nil, fmt.Errorf(
+			"expected no withdrawals in genesis execution payload",
+		)
+	}
+
+	baseFee, overflow := uint256.FromBig(eth1GenesisBlock.BaseFee())
+	if overflow {
+		return nil, fmt.Errorf("basefee larger than 2^256-1")
+	}
+
+	return &capella.ExecutionPayloadHeader{
+		ParentHash:    common.Root(eth1GenesisBlock.ParentHash()),
+		FeeRecipient:  common.Eth1Address(eth1GenesisBlock.Coinbase()),
+		StateRoot:     common.Bytes32(eth1GenesisBlock.Root()),
+		ReceiptsRoot:  common.Bytes32(eth1GenesisBlock.ReceiptHash()),
+		LogsBloom:     common.LogsBloom(eth1GenesisBlock.Bloom()),
+		PrevRandao:    common.Bytes32{},
+		BlockNumber:   view.Uint64View(eth1GenesisBlock.NumberU64()),
+		GasLimit:      view.Uint64View(eth1GenesisBlock.GasLimit()),
+		GasUsed:       view.Uint64View(eth1GenesisBlock.GasUsed()),
+		Timestamp:     common.Timestamp(eth1GenesisBlock.Time()),
+		ExtraData:     extra,
+		BaseFeePerGas: view.Uint256View(*baseFee),
+		BlockHash:     common.Root(eth1GenesisBlock.Hash()),
+		// empty transactions root
+		TransactionsRoot: common.PayloadTransactionsType(spec).
+			DefaultNode().
+			MerkleRoot(tree.GetHashFn()),
+		// empty withdrawals root
+		WithdrawalsRoot: common.WithdrawalsType(spec).
+			DefaultNode().
+			MerkleRoot(tree.GetHashFn()),
+	}, nil
+}
+
 func createValidators(
 	spec *common.Spec,
 	keys []*KeyDetails,
@@ -267,10 +320,10 @@ func BuildBeaconState(
 		}
 	}
 
-	if st, ok := state.(*bellatrix.BeaconStateView); ok {
-		// did we hit the TTD at genesis block?
+	switch st := state.(type) {
+	case *bellatrix.BeaconStateView:
 		tdd := uint256.Int(spec.TERMINAL_TOTAL_DIFFICULTY)
-		embedExecAtGenesis := tdd.ToBig().Cmp(eth1Genesis.Difficulty) < 0
+		embedExecAtGenesis := tdd.ToBig().Cmp(eth1Genesis.Difficulty) <= 0
 
 		var execPayloadHeader *bellatrix.ExecutionPayloadHeader
 		if embedExecAtGenesis {
@@ -287,6 +340,29 @@ func BuildBeaconState(
 			execPayloadHeader = new(bellatrix.ExecutionPayloadHeader)
 		}
 
+		if err := st.SetLatestExecutionPayloadHeader(execPayloadHeader); err != nil {
+			return nil, err
+		}
+	case *capella.BeaconStateView:
+		// did we hit the TTD at genesis block?
+		tdd := uint256.Int(spec.TERMINAL_TOTAL_DIFFICULTY)
+		embedExecAtGenesis := tdd.ToBig().Cmp(eth1Genesis.Difficulty) <= 0
+
+		var execPayloadHeader *capella.ExecutionPayloadHeader
+		if embedExecAtGenesis {
+			execPayloadHeader, err = genesisPayloadHeaderCapella(
+				eth1GenesisBlock,
+				spec,
+			)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			// we didn't build any on the eth1 chain though,
+			// so we just put the genesis hash here (it could be any block from eth1 chain before TTD that is not ahead of eth2)
+			execPayloadHeader = new(capella.ExecutionPayloadHeader)
+		}
+
 		if err := st.SetLatestExecutionPayloadHeader(execPayloadHeader); err != nil {
 			return nil, err
 		}

simulators/eth2/common/config/execution/execution_config.go

@@ -126,6 +126,26 @@ func (c ExecutionPreChain) SecondsPerBlock() uint64 {
 	return 1
 }
 
+// A pre-existing chain is imported by the client, and it is not
+// expected that the client mines or produces any blocks.
+type ExecutionPostMergeGenesis struct{}
+
+func (c ExecutionPostMergeGenesis) Configure(*ExecutionGenesis) error {
+	return nil
+}
+
+func (c ExecutionPostMergeGenesis) HiveParams(node int) hivesim.Params {
+	return hivesim.Params{}
+}
+
+func (c ExecutionPostMergeGenesis) DifficultyPerBlock() *big.Int {
+	return big.NewInt(0)
+}
+
+func (c ExecutionPostMergeGenesis) SecondsPerBlock() uint64 {
+	return 12
+}
+
 type ExecutionCliqueConsensus struct {
 	CliquePeriod uint64
 	PrivateKey   string

simulators/eth2/withdrawals/helper.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"crypto/ecdsa"
 	"fmt"
 	"math/big"
+	"sort"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -76,6 +78,20 @@ type BeaconBlockState struct {
 
 type BeaconCache map[tree.Root]BeaconBlockState
 
+// Clear the cache for when there was a known/expected re-org to query everything again
+func (c BeaconCache) Clear() error {
+	roots := make([]tree.Root, len(c))
+	i := 0
+	for s := range c {
+		roots[i] = s
+		i++
+	}
+	for _, s := range roots {
+		delete(c, s)
+	}
+	return nil
+}
+
 func (c BeaconCache) GetBlockStateByRoot(
 	ctx context.Context,
 	bc *clients.BeaconClient,
@@ -88,10 +104,17 @@ func (c BeaconCache) GetBlockStateByRoot(
 	if err != nil {
 		return BeaconBlockState{}, err
 	}
-	s, err := bc.BeaconStateV2(ctx, eth2api.StateIdRoot(b.StateRoot()))
+	s, err := bc.BeaconStateV2(ctx, eth2api.StateIdSlot(b.Slot()))
 	if err != nil {
 		return BeaconBlockState{}, err
 	}
+	blockStateRoot := b.StateRoot()
+	stateRoot := s.Root()
+	if !bytes.Equal(blockStateRoot[:], stateRoot[:]) {
+		return BeaconBlockState{}, fmt.Errorf(
+			"state root missmatch while fetching state",
+		)
+	}
 	both := BeaconBlockState{
 		VersionedBeaconStateResponse: s,
 		VersionedSignedBeaconBlock:   b,
@@ -128,6 +151,42 @@ func (c BeaconCache) GetBlockStateBySlotFromHeadRoot(
 	}
 }
 
+func PrintWithdrawalHistory(c BeaconCache) error {
+	slotMap := make(map[beacon.Slot]tree.Root)
+	slots := make([]beacon.Slot, 0)
+	for r, s := range c {
+		slot := s.StateSlot()
+		slotMap[slot] = r
+		slots = append(slots, slot)
+	}
+
+	sort.Slice(slots, func(i, j int) bool { return slots[j] > slots[i] })
+
+	for _, slot := range slots {
+		root := slotMap[slot]
+		s := c[root]
+		nextWithdrawalIndex, _ := s.NextWithdrawalIndex()
+		nextWithdrawalValidatorIndex, _ := s.NextWithdrawalValidatorIndex()
+		fmt.Printf(
+			"Slot=%d, NextWithdrawalIndex=%d, NextWithdrawalValidatorIndex=%d\n",
+			slot,
+			nextWithdrawalIndex,
+			nextWithdrawalValidatorIndex,
+		)
+		fmt.Printf("Withdrawals:\n")
+		ws, _ := s.Withdrawals()
+		for i, w := range ws {
+			fmt.Printf(
+				"%d: Validator Index: %s, Amount: %d\n",
+				i,
+				w.ValidatorIndex,
+				w.Amount,
+			)
+		}
+	}
+	return nil
+}
+
 // Helper struct to keep track of current status of a validator withdrawal state
 type Validator struct {
 	Index                      beacon.ValidatorIndex

simulators/eth2/withdrawals/scenarios.go

@@ -165,9 +165,9 @@ func (ts BaseWithdrawalsTestSpec) Execute(
 	// Get the beacon state and verify the credentials were updated
 	var versionedBeaconState *clients.VersionedBeaconStateResponse
 	for _, bn := range testnet.BeaconClients().Running() {
-		versionedBeaconState, err = bn.BeaconStateV2ByBlock(
+		versionedBeaconState, err = bn.BeaconStateV2(
 			ctx,
-			eth2api.BlockHead,
+			eth2api.StateHead,
 		)
 		if err != nil || versionedBeaconState == nil {
 			t.Logf("WARN: Unable to get latest beacon state: %v", err)
@@ -231,6 +231,7 @@ loop:
 	for {
 		select {
 		case <-slotCtx.Done():
+			PrintWithdrawalHistory(allValidators[0].BlockStateCache)
 			t.Fatalf("FAIL: Timeout waiting on all accounts to withdraw")
 		case <-time.After(time.Duration(testnet.Spec().SECONDS_PER_SLOT) * time.Second):
 			// Print all info
@@ -256,6 +257,8 @@ loop:
 		}
 	}
 
+	PrintWithdrawalHistory(allValidators[0].BlockStateCache)
+
 	// Lastly check all clients are on the same head
 	testnet.VerifyELHeads(ctx)
 

simulators/eth2/withdrawals/specs.go

@@ -59,7 +59,7 @@ var (
 		AltairForkEpoch:         common.Big0,
 		BellatrixForkEpoch:      common.Big0,
 		CapellaForkEpoch:        common.Big1,
-		Eth1Consensus:           &el.ExecutionCliqueConsensus{},
+		Eth1Consensus:           &el.ExecutionPostMergeGenesis{},
 	}
 
 	MINIMAL_SLOT_TIME_CLIENTS = []string{
@@ -204,8 +204,9 @@ func (ts BaseWithdrawalsTestSpec) GetValidatorKeys(
 	}
 
 	for index, key := range keys {
-		// All validators have idiosyncratic balance amounts to identify them
-		key.ExtraInitialBalance = beacon.Gwei(index+1) + ts.ExtraGwei
+		// All validators have idiosyncratic balance amounts to identify them.
+		// Also include a high amount in order to guarantee withdrawals.
+		key.ExtraInitialBalance = beacon.Gwei((index+1)*1000000) + ts.ExtraGwei
 
 		if ts.GenesisExecutionWithdrawalCredentialsShares > 0 &&
 			(index%ts.GenesisExecutionWithdrawalCredentialsShares) == 0 {