From d0bef2a1be43edc463fc3329ed146f310bbda1b9 Mon Sep 17 00:00:00 2001 From: marioevz Date: Tue, 24 Jan 2023 05:08:44 -0600 Subject: [PATCH 1/5] simulators/eth2/common: Add parentRoot, blockRoot --- simulators/eth2/common/clients/beacon.go | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/simulators/eth2/common/clients/beacon.go b/simulators/eth2/common/clients/beacon.go index f99e952729..e4fc1f46e0 100644 --- a/simulators/eth2/common/clients/beacon.go +++ b/simulators/eth2/common/clients/beacon.go @@ -218,6 +218,20 @@ func (b *VersionedSignedBeaconBlock) StateRoot() tree.Root { panic("badly formatted beacon block") } +func (b *VersionedSignedBeaconBlock) ParentRoot() tree.Root { + switch v := b.Data.(type) { + case *phase0.SignedBeaconBlock: + return v.Message.ParentRoot + case *altair.SignedBeaconBlock: + return v.Message.ParentRoot + case *bellatrix.SignedBeaconBlock: + return v.Message.ParentRoot + case *capella.SignedBeaconBlock: + return v.Message.ParentRoot + } + panic("badly formatted beacon block") +} + func (b *VersionedSignedBeaconBlock) Slot() common.Slot { switch v := b.Data.(type) { case *phase0.SignedBeaconBlock: @@ -246,6 +260,26 @@ func (b *VersionedSignedBeaconBlock) ProposerIndex() common.ValidatorIndex { panic("badly formatted beacon block") } +func (bn *BeaconClient) BlockV2Root( + parentCtx context.Context, + blockId eth2api.BlockId, +) (tree.Root, error) { + var ( + root tree.Root + exists bool + err error + ) + ctx, cancel := utils.ContextTimeoutRPC(parentCtx) + defer cancel() + root, exists, err = beaconapi.BlockRoot(ctx, bn.API, blockId) + if !exists { + return root, fmt.Errorf( + "endpoint not found on beacon client", + ) + } + return root, err +} + func (bn *BeaconClient) BlockV2( parentCtx context.Context, blockId eth2api.BlockId, @@ -445,6 +479,16 @@ func (vbs *VersionedBeaconStateResponse) Balances() phase0.Balances { panic("badly formatted beacon state") } +func (vbs *VersionedBeaconStateResponse) Balance( + id common.ValidatorIndex, +) common.Gwei { + balances := vbs.Balances() + if int(id) >= len(balances) { + panic("invalid validator requested") + } + return balances[id] +} + func (vbs *VersionedBeaconStateResponse) Validators() phase0.ValidatorRegistry { switch state := vbs.Data.(type) { case *phase0.BeaconState: From ec68e6b00318b9af606168644f765751653d2ac1 Mon Sep 17 00:00:00 2001 From: marioevz Date: Tue, 24 Jan 2023 05:09:13 -0600 Subject: [PATCH 2/5] simulators/eth2/withdrawals: Verify partial withdrawals --- simulators/eth2/withdrawals/helper.go | 174 +++++++++++++++++++++-- simulators/eth2/withdrawals/scenarios.go | 11 +- 2 files changed, 171 insertions(+), 14 deletions(-) diff --git a/simulators/eth2/withdrawals/helper.go b/simulators/eth2/withdrawals/helper.go index 4332a40793..fd361a5298 100644 --- a/simulators/eth2/withdrawals/helper.go +++ b/simulators/eth2/withdrawals/helper.go @@ -12,6 +12,7 @@ import ( cl "github.com/ethereum/hive/simulators/eth2/common/config/consensus" "github.com/ethereum/hive/simulators/eth2/common/testnet" blsu "github.com/protolambda/bls12-381-util" + "github.com/protolambda/eth2api" beacon "github.com/protolambda/zrnt/eth2/beacon/common" "github.com/protolambda/ztyp/tree" ) @@ -56,6 +57,76 @@ var VaultSigner = Signer{ PrivateKey: VAULT_KEY, } +func WithdrawalsContainValidator( + ws []*types.Withdrawal, + vId beacon.ValidatorIndex, +) bool { + for _, w := range ws { + if w.Validator == uint64(vId) { + return true + } + } + return false +} + +type BeaconBlockState struct { + *clients.VersionedBeaconStateResponse + *clients.VersionedSignedBeaconBlock +} + +type BeaconCache map[tree.Root]BeaconBlockState + +func (c BeaconCache) GetBlockStateByRoot( + ctx context.Context, + bc *clients.BeaconClient, + blockroot tree.Root, +) (BeaconBlockState, error) { + if s, ok := c[blockroot]; ok { + return s, nil + } + b, err := bc.BlockV2(ctx, eth2api.BlockIdRoot(blockroot)) + if err != nil { + return BeaconBlockState{}, err + } + s, err := bc.BeaconStateV2(ctx, eth2api.StateIdRoot(b.StateRoot())) + if err != nil { + return BeaconBlockState{}, err + } + both := BeaconBlockState{ + VersionedBeaconStateResponse: s, + VersionedSignedBeaconBlock: b, + } + c[blockroot] = both + return both, nil +} + +func (c BeaconCache) GetBlockStateBySlotFromHeadRoot( + ctx context.Context, + bc *clients.BeaconClient, + headblockroot tree.Root, + slot beacon.Slot, +) (BeaconBlockState, error) { + current, err := c.GetBlockStateByRoot(ctx, bc, headblockroot) + if err != nil { + return BeaconBlockState{}, err + } + if current.Slot() < slot { + return BeaconBlockState{}, fmt.Errorf("requested for slot above head") + } + for { + if current.Slot() == slot { + return current, nil + } + if current.Slot() == 0 { + return BeaconBlockState{}, fmt.Errorf("couldn't find slot") + } + current, err = c.GetBlockStateByRoot(ctx, bc, current.ParentRoot()) + if err != nil { + return BeaconBlockState{}, err + } + } +} + // Helper struct to keep track of current status of a validator withdrawal state type Validator struct { Index beacon.ValidatorIndex @@ -66,11 +137,16 @@ type Validator struct { Keys *cl.KeyDetails BLSToExecutionChangeDomain *beacon.BLSDomain Verified bool + InitialBalance beacon.Gwei + Spec beacon.Spec + BlockStateCache BeaconCache } func (v *Validator) VerifyWithdrawnBalance( ctx context.Context, + bc *clients.BeaconClient, ec *clients.ExecutionClient, + headBlockRoot tree.Root, ) (bool, error) { if v.Verified { // Validator already verified on a previous iteration @@ -84,8 +160,31 @@ func (v *Validator) VerifyWithdrawnBalance( } execAddress := *v.WithdrawAddress - // First get the balance - balance, err := ec.BalanceAt(ctx, execAddress, nil) + // First get the head block + headBlockState, err := v.BlockStateCache.GetBlockStateByRoot( + ctx, + bc, + headBlockRoot, + ) + if err != nil { + return false, err + } + fmt.Printf( + "INFO: Verifying balance validator %d on slot %d\n", + v.Index, + headBlockState.Slot(), + ) + + // Then get the balance + execPayload, err := headBlockState.ExecutionPayload() + if err != nil { + return false, err + } + balance, err := ec.BalanceAt( + ctx, + execAddress, + big.NewInt(int64(execPayload.Number)), + ) if err != nil { return false, err } @@ -110,16 +209,52 @@ func (v *Validator) VerifyWithdrawnBalance( } else { return true, fmt.Errorf("unexepected balance: want=%d, got=%d", v.ExactWithdrawableBalance, balance) } - } + } else { + // We need to traverse the beacon state history to be able to compute + // the expected partial balance withdrawn + previousBalance := v.InitialBalance + expectedPartialWithdrawnBalance := beacon.Gwei(0) + for slot := beacon.Slot(0); slot <= headBlockState.Slot(); slot++ { + block, err := bc.BlockV2(ctx, eth2api.BlockIdSlot(slot)) + if err != nil { + return false, err + } + + execPayload, err := block.ExecutionPayload() + if err != nil { + return false, err + } + + if WithdrawalsContainValidator(execPayload.Withdrawals, v.Index) { + expectedPartialWithdrawnBalance += previousBalance - (v.Spec.MAX_EFFECTIVE_BALANCE) + } + + slotState, err := v.BlockStateCache.GetBlockStateBySlotFromHeadRoot(ctx, bc, headBlockRoot, slot) + if err != nil { + return false, err + } + previousBalance = slotState.Balance(v.Index) + } - // Otherwise simply return true, as a signal that a - potentially partial - withdrawal has taken place - fmt.Printf( - "INFO: Validator %d partially withdrawn: %d\n", - v.Index, - balance, - ) - v.Verified = true - return true, nil + if expectedPartialWithdrawnBalance != 0 { + expectedBalanceWei := new(big.Int).SetUint64(uint64(expectedPartialWithdrawnBalance)) + expectedBalanceWei.Mul(expectedBalanceWei, big.NewInt(1e9)) + if balance.Cmp(expectedBalanceWei) == 0 { + fmt.Printf( + "INFO: Validator %d partially withdrawn: %d\n", + v.Index, + balance, + ) + v.Verified = true + return true, nil + } else { + return true, fmt.Errorf("unexepected balance: want=%d, got=%d", expectedBalanceWei, balance) + } + + } + + } + return false, nil } // Signs the BLS-to-execution-change for the given address @@ -183,10 +318,12 @@ type Validators []*Validator // Verify all validators have withdrawn func (vs Validators) VerifyWithdrawnBalance( ctx context.Context, + bc *clients.BeaconClient, ec *clients.ExecutionClient, + headBlockRoot tree.Root, ) (bool, error) { for i, v := range vs { - if withdrawn, err := v.VerifyWithdrawnBalance(ctx, ec); err != nil { + if withdrawn, err := v.VerifyWithdrawnBalance(ctx, bc, ec, headBlockRoot); err != nil { return withdrawn, fmt.Errorf( "error verifying validator %d balance: %v", i, @@ -253,20 +390,24 @@ func (vs Validators) Chunks(totalShares int) []Validators { } func ValidatorFromBeaconValidator( + spec beacon.Spec, index beacon.ValidatorIndex, source beacon.Validator, balance beacon.Gwei, keys *cl.KeyDetails, domain *beacon.BLSDomain, + beaconCache BeaconCache, ) (*Validator, error) { // Assume genesis state currentEpoch := beacon.Epoch(0) v := new(Validator) + v.Spec = spec v.Index = index v.Keys = keys v.BLSToExecutionChangeDomain = domain + v.BlockStateCache = beaconCache wc, err := source.WithdrawalCredentials() if err != nil { @@ -302,14 +443,17 @@ func ValidatorFromBeaconValidator( big.NewInt(1e9), ) } + v.InitialBalance = balance return v, nil } func ValidatorFromBeaconState( + spec beacon.Spec, state beacon.BeaconState, index beacon.ValidatorIndex, keys *cl.KeyDetails, domain *beacon.BLSDomain, + beaconCache BeaconCache, ) (*Validator, error) { stateVals, err := state.Validators() if err != nil { @@ -328,16 +472,19 @@ func ValidatorFromBeaconState( return nil, err } return ValidatorFromBeaconValidator( + spec, index, beaconVal, balance, keys, domain, + beaconCache, ) } func ValidatorsFromBeaconState( state beacon.BeaconState, + spec beacon.Spec, keys []*cl.KeyDetails, domain *beacon.BLSDomain, ) (Validators, error) { @@ -357,6 +504,7 @@ func ValidatorsFromBeaconState( } else if validatorCount != uint64(len(keys)) { return nil, fmt.Errorf("incorrect amount of keys: want=%d, got=%d", validatorCount, len(keys)) } + beaconCache := make(BeaconCache) validators := make(Validators, 0) for i := beacon.ValidatorIndex(0); i < beacon.ValidatorIndex(validatorCount); i++ { beaconVal, err := stateVals.Validator(beacon.ValidatorIndex(i)) @@ -368,11 +516,13 @@ func ValidatorsFromBeaconState( return nil, err } validator, err := ValidatorFromBeaconValidator( + spec, i, beaconVal, balance, keys[i], domain, + beaconCache, ) if err != nil { return nil, err diff --git a/simulators/eth2/withdrawals/scenarios.go b/simulators/eth2/withdrawals/scenarios.go index 67871a88c7..fe8f369b70 100644 --- a/simulators/eth2/withdrawals/scenarios.go +++ b/simulators/eth2/withdrawals/scenarios.go @@ -36,6 +36,7 @@ func (ts BaseWithdrawalsTestSpec) Execute( // Get all validators info allValidators, err := ValidatorsFromBeaconState( testnet.GenesisBeaconState(), + *testnet.Spec().Spec, env.Keys, &blsDomain, ) @@ -197,8 +198,14 @@ loop: testnet.BeaconClients().Running().PrintStatus(slotCtx, t) // Check all accounts - for _, ec := range testnet.ExecutionClients().Running() { - if allAccountsWithdrawn, err := allValidators.Withdrawable().VerifyWithdrawnBalance(ctx, ec); err != nil { + for _, n := range testnet.Nodes.Running() { + ec := n.ExecutionClient + bc := n.BeaconClient + headBlockRoot, err := bc.BlockV2Root(ctx, eth2api.BlockHead) + if err != nil { + t.Fatalf("FAIL: Error getting head block: %v", err) + } + if allAccountsWithdrawn, err := allValidators.Withdrawable().VerifyWithdrawnBalance(ctx, bc, ec, headBlockRoot); err != nil { t.Fatalf("FAIL: %v", err) } else if allAccountsWithdrawn { t.Logf("INFO: All accounts have successfully withdrawn") From 509ce1e4a0f743e3ae00b917d8af958ce4a3a19f Mon Sep 17 00:00:00 2001 From: marioevz Date: Tue, 24 Jan 2023 07:05:16 -0600 Subject: [PATCH 3/5] simulators/eth2/withdrawals: Optimize getting balance --- simulators/eth2/withdrawals/helper.go | 40 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/simulators/eth2/withdrawals/helper.go b/simulators/eth2/withdrawals/helper.go index fd361a5298..5c192abf86 100644 --- a/simulators/eth2/withdrawals/helper.go +++ b/simulators/eth2/withdrawals/helper.go @@ -105,24 +105,25 @@ func (c BeaconCache) GetBlockStateBySlotFromHeadRoot( bc *clients.BeaconClient, headblockroot tree.Root, slot beacon.Slot, -) (BeaconBlockState, error) { +) (*BeaconBlockState, error) { current, err := c.GetBlockStateByRoot(ctx, bc, headblockroot) if err != nil { - return BeaconBlockState{}, err + return nil, err } if current.Slot() < slot { - return BeaconBlockState{}, fmt.Errorf("requested for slot above head") + return nil, fmt.Errorf("requested for slot above head") } for { if current.Slot() == slot { - return current, nil + return ¤t, nil } - if current.Slot() == 0 { - return BeaconBlockState{}, fmt.Errorf("couldn't find slot") + if current.Slot() < slot || current.Slot() == 0 { + // Skipped slot probably, not a fatal error + return nil, nil } current, err = c.GetBlockStateByRoot(ctx, bc, current.ParentRoot()) if err != nil { - return BeaconBlockState{}, err + return nil, err } } } @@ -189,6 +190,13 @@ func (v *Validator) VerifyWithdrawnBalance( return false, err } + fmt.Printf( + "INFO: Balance of validator %d in the execution chain (block %d): %d\n", + v.Index, + execPayload.Number, + balance, + ) + // If balance is zero, there have not been any withdrawals yet, // but this is not an error if balance.Cmp(common.Big0) == 0 { @@ -215,12 +223,15 @@ func (v *Validator) VerifyWithdrawnBalance( previousBalance := v.InitialBalance expectedPartialWithdrawnBalance := beacon.Gwei(0) for slot := beacon.Slot(0); slot <= headBlockState.Slot(); slot++ { - block, err := bc.BlockV2(ctx, eth2api.BlockIdSlot(slot)) + blockState, err := v.BlockStateCache.GetBlockStateBySlotFromHeadRoot(ctx, bc, headBlockRoot, slot) if err != nil { return false, err } + if blockState == nil { + return false, nil + } - execPayload, err := block.ExecutionPayload() + execPayload, err := blockState.ExecutionPayload() if err != nil { return false, err } @@ -229,11 +240,7 @@ func (v *Validator) VerifyWithdrawnBalance( expectedPartialWithdrawnBalance += previousBalance - (v.Spec.MAX_EFFECTIVE_BALANCE) } - slotState, err := v.BlockStateCache.GetBlockStateBySlotFromHeadRoot(ctx, bc, headBlockRoot, slot) - if err != nil { - return false, err - } - previousBalance = slotState.Balance(v.Index) + previousBalance = blockState.Balance(v.Index) } if expectedPartialWithdrawnBalance != 0 { @@ -251,6 +258,11 @@ func (v *Validator) VerifyWithdrawnBalance( return true, fmt.Errorf("unexepected balance: want=%d, got=%d", expectedBalanceWei, balance) } + } else { + fmt.Printf( + "INFO: Validator %d expected withdraw balance is zero\n", + v.Index, + ) } } From c6420cc791690fcc5d67333bdf0a014431dc71a9 Mon Sep 17 00:00:00 2001 From: marioevz Date: Tue, 24 Jan 2023 07:50:35 -0600 Subject: [PATCH 4/5] simulators/eth2/withdrawals: minor optimization --- simulators/eth2/withdrawals/scenarios.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/simulators/eth2/withdrawals/scenarios.go b/simulators/eth2/withdrawals/scenarios.go index fe8f369b70..6b987997b4 100644 --- a/simulators/eth2/withdrawals/scenarios.go +++ b/simulators/eth2/withdrawals/scenarios.go @@ -119,18 +119,18 @@ func (ts BaseWithdrawalsTestSpec) Execute( ) } } + + // Wait for all BLS to execution to be included + slotsForAllBlsInclusion := beacon.Slot( + len(genesisNonWithdrawable)/int( + testnet.Spec().MAX_BLS_TO_EXECUTION_CHANGES, + ) + 1, + ) + testnet.WaitSlots(ctx, slotsForAllBlsInclusion) } else { t.Logf("INFO: no validators left on BLS credentials") } - // Wait for all BLS to execution to be included - slotsForAllBlsInclusion := beacon.Slot( - len(genesisNonWithdrawable)/int( - testnet.Spec().MAX_BLS_TO_EXECUTION_CHANGES, - ) + 1, - ) - testnet.WaitSlots(ctx, slotsForAllBlsInclusion) - // Get the beacon state and verify the credentials were updated var versionedBeaconState *clients.VersionedBeaconStateResponse for _, bn := range testnet.BeaconClients().Running() { @@ -140,6 +140,8 @@ func (ts BaseWithdrawalsTestSpec) Execute( ) if err != nil || versionedBeaconState == nil { t.Logf("WARN: Unable to get latest beacon state: %v", err) + } else { + break } } if versionedBeaconState == nil { From 973495345b1910055e26c73cb162869a915ccfe3 Mon Sep 17 00:00:00 2001 From: marioevz Date: Tue, 24 Jan 2023 09:10:10 -0600 Subject: [PATCH 5/5] simulators/eth2/common: fix p2paddr --- simulators/eth2/common/clients/beacon.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/simulators/eth2/common/clients/beacon.go b/simulators/eth2/common/clients/beacon.go index e4fc1f46e0..e36f303492 100644 --- a/simulators/eth2/common/clients/beacon.go +++ b/simulators/eth2/common/clients/beacon.go @@ -109,6 +109,7 @@ func (bn *BeaconClient) ENR(parentCtx context.Context) (string, error) { } fmt.Printf("p2p addrs: %v\n", out.P2PAddresses) fmt.Printf("peer id: %s\n", out.PeerID) + fmt.Printf("enr: %s\n", out.ENR) return out.ENR, nil } @@ -119,7 +120,12 @@ func (bn *BeaconClient) P2PAddr(parentCtx context.Context) (string, error) { if err := nodeapi.Identity(ctx, bn.API, &out); err != nil { return "", err } - return out.P2PAddresses[0], nil + return fmt.Sprintf( + "/ip4/%s/tcp/%d/p2p/%s", + bn.HiveClient.IP.String(), + PortBeaconTCP, + out.PeerID, + ), nil } func (bn *BeaconClient) EnodeURL() (string, error) {