diff --git a/clients/lighthouse-bn/lighthouse_bn.sh b/clients/lighthouse-bn/lighthouse_bn.sh index d1897a2f00..dea46b4ec3 100755 --- a/clients/lighthouse-bn/lighthouse_bn.sh +++ b/clients/lighthouse-bn/lighthouse_bn.sh @@ -63,13 +63,14 @@ esac echo "bootnodes: ${HIVE_ETH2_BOOTNODE_ENRS}" CONTAINER_IP=`hostname -i | awk '{print $1;}'` -eth1_option=$([[ "$HIVE_ETH2_ETH1_RPC_ADDRS" == "" ]] && echo "--dummy-eth1" || echo "--eth1-endpoints=$HIVE_ETH2_ETH1_RPC_ADDRS") metrics_option=$([[ "$HIVE_ETH2_METRICS_PORT" == "" ]] && echo "" || echo "--metrics --metrics-address=0.0.0.0 --metrics-port=$HIVE_ETH2_METRICS_PORT --metrics-allow-origin=*") if [ "$HIVE_ETH2_MERGE_ENABLED" != "" ]; then echo -n "0x7365637265747365637265747365637265747365637265747365637265747365" > /jwtsecret merge_option="--execution-endpoints=$HIVE_ETH2_ETH1_ENGINE_RPC_ADDRS --jwt-secrets=/jwtsecret" fi opt_sync_option=$([[ "$HIVE_ETH2_SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY" == "" ]] && echo "" || echo "--safe-slots-to-import-optimistically=$HIVE_ETH2_SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY") +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--builder=$HIVE_ETH2_BUILDER_ENDPOINT") +echo BUILDER=$builder_option lighthouse \ --debug-level="$LOG" \ @@ -77,7 +78,7 @@ lighthouse \ --testnet-dir=/data/testnet_setup \ bn \ --network-dir=/data/network \ - $metrics_option $eth1_option $merge_option $opt_sync_option \ + $metrics_option $builder_option $merge_option $opt_sync_option \ --enr-tcp-port="${HIVE_ETH2_P2P_TCP_PORT:-9000}" \ --enr-udp-port="${HIVE_ETH2_P2P_UDP_PORT:-9000}" \ --enr-address="${CONTAINER_IP}" \ diff --git a/clients/lighthouse-vc/lighthouse_vc.sh b/clients/lighthouse-vc/lighthouse_vc.sh index 6843c88a19..dd5f91ba22 100755 --- a/clients/lighthouse-vc/lighthouse_vc.sh +++ b/clients/lighthouse-vc/lighthouse_vc.sh @@ -30,6 +30,8 @@ case "$HIVE_LOGLEVEL" in 5) LOG=trace ;; esac +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--builder-proposals") + lighthouse \ --debug-level="$LOG" \ --datadir=/data/vc \ @@ -39,4 +41,5 @@ lighthouse \ --secrets-dir="/data/secrets" \ --init-slashing-protection \ --beacon-nodes="http://$HIVE_ETH2_BN_API_IP:$HIVE_ETH2_BN_API_PORT" \ - --suggested-fee-recipient="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + --suggested-fee-recipient="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" \ + $builder_option diff --git a/clients/lodestar-bn/lodestar_bn.sh b/clients/lodestar-bn/lodestar_bn.sh index 349b9e5679..3837b60ad4 100755 --- a/clients/lodestar-bn/lodestar_bn.sh +++ b/clients/lodestar-bn/lodestar_bn.sh @@ -36,6 +36,8 @@ CONTAINER_IP=`hostname -i | awk '{print $1;}'` echo Container IP: $CONTAINER_IP bootnodes_option=$([[ "$HIVE_ETH2_BOOTNODE_ENRS" == "" ]] && echo "" || echo "--bootnodes ${HIVE_ETH2_BOOTNODE_ENRS//,/ }") metrics_option=$([[ "$HIVE_ETH2_METRICS_PORT" == "" ]] && echo "" || echo "--metrics --metrics.address=$CONTAINER_IP --metrics.port=$HIVE_ETH2_METRICS_PORT") +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--builder --builder.urls $HIVE_ETH2_BUILDER_ENDPOINT") +echo BUILDER=$builder_option echo "bootnodes option : ${bootnodes_option}" @@ -61,6 +63,7 @@ node /usr/app/node_modules/.bin/lodestar \ --jwt-secret=/jwtsecret \ $metrics_option \ $bootnodes_option \ + $builder_option \ --enr.ip="${CONTAINER_IP}" \ --enr.tcp="${HIVE_ETH2_P2P_TCP_PORT:-9000}" \ --enr.udp="${HIVE_ETH2_P2P_UDP_PORT:-9000}" \ diff --git a/clients/lodestar-vc/lodestar_vc.sh b/clients/lodestar-vc/lodestar_vc.sh index cc1d01fc70..c79fe604c9 100755 --- a/clients/lodestar-vc/lodestar_vc.sh +++ b/clients/lodestar-vc/lodestar_vc.sh @@ -32,6 +32,9 @@ case "$HIVE_LOGLEVEL" in 5) LOG=trace ;; esac +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--builder --suggestedFeeRecipient 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B") +echo BUILDER=$builder_option + echo Starting Lodestar Validator Client node /usr/app/node_modules/.bin/lodestar \ @@ -41,6 +44,6 @@ node /usr/app/node_modules/.bin/lodestar \ --paramsFile=/hive/input/config.yaml \ --keystoresDir="/data/validators" \ --secretsDir="/data/secrets" \ - $metrics_option \ + $metrics_option $builder_option \ --beaconNodes="http://$HIVE_ETH2_BN_API_IP:$HIVE_ETH2_BN_API_PORT" diff --git a/clients/nimbus-bn/nimbus_bn.sh b/clients/nimbus-bn/nimbus_bn.sh index 28859249ac..4a75ef52f0 100755 --- a/clients/nimbus-bn/nimbus_bn.sh +++ b/clients/nimbus-bn/nimbus_bn.sh @@ -56,6 +56,9 @@ else fi metrics_option=$([[ "$HIVE_ETH2_METRICS_PORT" == "" ]] && echo "" || echo "--metrics --metrics-address=0.0.0.0 --metrics-port=$HIVE_ETH2_METRICS_PORT") +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--payload-builder=true --payload-builder-url=$HIVE_ETH2_BUILDER_ENDPOINT") +echo BUILDER=$builder_option + echo -n "0x7365637265747365637265747365637265747365637265747365637265747365" > /jwtsecret echo Starting Nimbus Beacon Node @@ -68,7 +71,7 @@ echo Starting Nimbus Beacon Node --web3-url="$HIVE_ETH2_ETH1_ENGINE_RPC_ADDRS" \ --jwt-secret=/jwtsecret \ --num-threads=4 \ - $bootnodes_option $metrics_option \ + $bootnodes_option $metrics_option $builder_option \ --nat="extip:${CONTAINER_IP}" \ --listen-address=0.0.0.0 \ --tcp-port="${HIVE_ETH2_P2P_TCP_PORT:-9000}" \ diff --git a/clients/nimbus-vc/nimbus_vc.sh b/clients/nimbus-vc/nimbus_vc.sh index 1c715f550c..20bc8592e9 100755 --- a/clients/nimbus-vc/nimbus_vc.sh +++ b/clients/nimbus-vc/nimbus_vc.sh @@ -17,6 +17,9 @@ case "$HIVE_LOGLEVEL" in 5) LOG=TRACE ;; esac +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--payload-builder=true") +echo BUILDER=$builder_option + echo Starting Nimbus Validator Client /usr/bin/nimbus_validator_client \ @@ -26,3 +29,4 @@ echo Starting Nimbus Validator Client --beacon-node="http://$HIVE_ETH2_BN_API_IP:$HIVE_ETH2_BN_API_PORT" \ --validators-dir="/hive/input/keystores" \ --secrets-dir="/hive/input/secrets" \ + $builder_option diff --git a/clients/prysm-bn/prysm_bn.sh b/clients/prysm-bn/prysm_bn.sh index edbd3a6548..35b7666121 100755 --- a/clients/prysm-bn/prysm_bn.sh +++ b/clients/prysm-bn/prysm_bn.sh @@ -56,6 +56,9 @@ else bootnode_option="$bootnode_option --bootstrap-node=$trimmed_bn" done fi +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--http-mev-relay=$HIVE_ETH2_BUILDER_ENDPOINT") +echo BUILDER=$builder_option + echo Starting Prysm Beacon Node /beacon-chain \ @@ -75,6 +78,7 @@ echo Starting Prysm Beacon Node --subscribe-all-subnets=true \ --enable-debug-rpc-endpoints=true \ $metrics_option \ + $builder_option \ --deposit-contract="${HIVE_ETH2_CONFIG_DEPOSIT_CONTRACT_ADDRESS:-0x1111111111111111111111111111111111111111}" \ --contract-deployment-block="${HIVE_ETH2_DEPOSIT_DEPLOY_BLOCK_NUMBER:-0}" \ --rpc-host=0.0.0.0 --rpc-port="${HIVE_ETH2_BN_GRPC_PORT:-3500}" \ diff --git a/clients/prysm-vc/prysm_vc.sh b/clients/prysm-vc/prysm_vc.sh index 5c555bb7b9..ebffd9f089 100755 --- a/clients/prysm-vc/prysm_vc.sh +++ b/clients/prysm-vc/prysm_vc.sh @@ -41,6 +41,9 @@ case "$HIVE_LOGLEVEL" in 5) LOG=trace ;; esac +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--enable-builder --suggested-fee-recipient=0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b") +echo BUILDER=$builder_option + echo Starting Prysm Validator Client /validator \ @@ -52,5 +55,6 @@ echo Starting Prysm Validator Client --datadir="/data/vc" \ --wallet-dir="/data/validators" \ --wallet-password-file="/wallet.pass" \ - --chain-config-file="/hive/input/config.yaml" + --chain-config-file="/hive/input/config.yaml" \ + $builder_option # NOTE: gRPC/RPC ports are inverted to allow the simulator to access the REST API \ No newline at end of file diff --git a/clients/teku-bn/teku_bn.sh b/clients/teku-bn/teku_bn.sh index 9d6de90755..34eb6af186 100755 --- a/clients/teku-bn/teku_bn.sh +++ b/clients/teku-bn/teku_bn.sh @@ -46,6 +46,8 @@ metrics_option=$([[ "$HIVE_ETH2_METRICS_PORT" == "" ]] && echo "" || echo "--met enr_option=$([[ "$HIVE_ETH2_BOOTNODE_ENRS" == "" ]] && echo "" || echo --p2p-discovery-bootnodes="$HIVE_ETH2_BOOTNODE_ENRS") static_option=$([[ "$HIVE_ETH2_STATIC_PEERS" == "" ]] && echo "" || echo --p2p-static-peers="$HIVE_ETH2_STATIC_PEERS") opt_sync_option=$([[ "$HIVE_ETH2_SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY" == "" ]] && echo "" || echo "--Xnetwork-safe-slots-to-import-optimistically=$HIVE_ETH2_SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY") +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--validators-builder-registration-default-enabled=true --validators-proposer-blinded-blocks-enabled=true --builder-endpoint=$HIVE_ETH2_BUILDER_ENDPOINT") +echo $builder_option if [ "$HIVE_ETH2_MERGE_ENABLED" != "" ]; then echo -n "0x7365637265747365637265747365637265747365637265747365637265747365" > /jwtsecret @@ -61,7 +63,7 @@ echo Starting Teku Beacon Node --eth1-deposit-contract-address="${HIVE_ETH2_CONFIG_DEPOSIT_CONTRACT_ADDRESS:-0x1111111111111111111111111111111111111111}" \ --log-destination console \ --logging="$LOG" \ - $metrics_option $eth1_option $merge_option $enr_option $static_option $opt_sync_option \ + $metrics_option $eth1_option $merge_option $enr_option $static_option $opt_sync_option $builder_option \ --validators-proposer-default-fee-recipient="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" \ --p2p-port="${HIVE_ETH2_P2P_TCP_PORT:-9000}" \ --p2p-udp-port="${HIVE_ETH2_P2P_UDP_PORT:-9000}" \ diff --git a/clients/teku-vc/teku_vc.sh b/clients/teku-vc/teku_vc.sh index aeb326c25c..ad10dc6e66 100755 --- a/clients/teku-vc/teku_vc.sh +++ b/clients/teku-vc/teku_vc.sh @@ -24,6 +24,9 @@ case "$HIVE_LOGLEVEL" in 5) LOG=TRACE ;; esac +builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--validators-proposer-blinded-blocks-enabled=true --validators-builder-registration-default-enabled=true") +echo $builder_option + echo Starting Teku Validator Client /opt/teku/bin/teku vc \ @@ -34,4 +37,5 @@ echo Starting Teku Validator Client --logging="$LOG" \ --validator-keys=/data/validators/keys:/data/validators/passwords \ --validators-proposer-default-fee-recipient="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" \ - --validators-external-signer-slashing-protection-enabled=true + --validators-external-signer-slashing-protection-enabled=true \ + $builder_option diff --git a/simulators/eth2/common/builder/builder.go b/simulators/eth2/common/builder/builder.go new file mode 100644 index 0000000000..144a09c291 --- /dev/null +++ b/simulators/eth2/common/builder/builder.go @@ -0,0 +1,15 @@ +package builder + +import ( + api "github.com/ethereum/go-ethereum/core/beacon" + beacon "github.com/protolambda/zrnt/eth2/beacon/common" +) + +type Builder interface { + Address() string + Cancel() error + GetBuiltPayloadsCount() int + GetSignedBeaconBlockCount() int + GetModifiedPayloads() map[beacon.Slot]*api.ExecutableData + GetBuiltPayloads() map[beacon.Slot]*api.ExecutableData +} diff --git a/simulators/eth2/common/builder/mock/README.md b/simulators/eth2/common/builder/mock/README.md new file mode 100644 index 0000000000..7dd06a343b --- /dev/null +++ b/simulators/eth2/common/builder/mock/README.md @@ -0,0 +1,30 @@ +## Hive Configurable Builder API Mock Server + +Instantiates a server that listens for Builder API (https://github.com/ethereum/builder-specs/) directives and responds with payloads built using an execution client. + +The builder can inject modifications into the built payloads at predefined slots by using configurable callbacks: +- Before sending the ForkchoiceUpdated directive to the execution client, by modifying the payload attributes, using `WithPayloadAttributesModifier` option +- Before responding with the build payload to the consensus client by modifying the any field in the payload, using `WithPayloadModifier` option + +Both callbacks are supplied with either the `PayloadAttributesV1`/`PayloadAttributesV2` or the `ExecutionPayloadV1`/`ExecutionPayloadV2` object, and the beacon slot number of the payload request. + +The callbacks must respond with a boolean indicating whether any modification was performed, and an error, if any. + +The builder can also be configured to insert an error on: +- `/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey}` using `WithErrorOnHeaderRequest` option +- `/eth/v1/builder/blinded_blocks` using `WithErrorOnPayloadReveal` option + +Both callbacks are supplied with the beacon slot number of the payload/blinded block request. + +The callback can then use the slot number to determine whether to throw an error or not. + +Currently, the builder will produce payloads with the following correct fields: +- PrevRandao +- Timestamp +- SuggestedFeeRecipient +- Withdrawals + +For the builder to function properly, the following parameters are necessary: +- Execution client: Required to build the payloads +- Beacon client: Required to fetch the state of the previous slot, and calculate, e.g., the prevrandao value +- Beacon Spec: Required so the builder is aware of fork specific changes in the built payloads, as well as the beacon blocks \ No newline at end of file diff --git a/simulators/eth2/common/builder/mock/mock_builder.go b/simulators/eth2/common/builder/mock/mock_builder.go new file mode 100644 index 0000000000..ba729b20e6 --- /dev/null +++ b/simulators/eth2/common/builder/mock/mock_builder.go @@ -0,0 +1,922 @@ +package mock_builder + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net" + "net/http" + "sync" + "time" + + el_common "github.com/ethereum/go-ethereum/common" + api "github.com/ethereum/go-ethereum/core/beacon" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/hive/simulators/eth2/common/builder/types/bellatrix" + "github.com/ethereum/hive/simulators/eth2/common/builder/types/capella" + "github.com/ethereum/hive/simulators/eth2/common/builder/types/common" + "github.com/ethereum/hive/simulators/eth2/common/clients" + "github.com/gorilla/mux" + blsu "github.com/protolambda/bls12-381-util" + "github.com/protolambda/eth2api" + beacon "github.com/protolambda/zrnt/eth2/beacon/common" + "github.com/protolambda/ztyp/tree" + "github.com/sirupsen/logrus" +) + +var ( + DOMAIN_APPLICATION_BUILDER = beacon.BLSDomainType{0x00, 0x00, 0x00, 0x01} + EMPTY_HASH = el_common.Hash{} +) + +type MockBuilder struct { + // Execution and consensus clients + el *clients.ExecutionClient + cl *clients.BeaconClient + + // General properties + srv *http.Server + sk *blsu.SecretKey + pk *blsu.Pubkey + pkBeacon beacon.BLSPubkey + + address string + spec *beacon.Spec + cancel context.CancelFunc + + // Payload/Blocks history maps + suggestedFeeRecipients map[beacon.BLSPubkey]el_common.Address + suggestedFeeRecipientsMutex sync.Mutex + builtPayloads map[beacon.Slot]*api.ExecutableData + builtPayloadsMutex sync.Mutex + modifiedPayloads map[beacon.Slot]*api.ExecutableData + modifiedPayloadsMutex sync.Mutex + signedBeaconBlock map[tree.Root]bool + signedBeaconBlockMutex sync.Mutex + + // Configuration object + cfg *config +} + +const ( + DEFAULT_BUILDER_HOST = "0.0.0.0" + DEFAULT_BUILDER_PORT = 18550 +) + +func NewMockBuilder( + el *clients.ExecutionClient, + cl *clients.BeaconClient, + spec *beacon.Spec, + opts ...Option, +) (*MockBuilder, error) { + if el == nil { + panic(fmt.Errorf("invalid EL provided: nil")) + } + var ( + err error + ) + + m := &MockBuilder{ + el: el, + cl: cl, + spec: spec, + + suggestedFeeRecipients: make(map[beacon.BLSPubkey]el_common.Address), + builtPayloads: make(map[beacon.Slot]*api.ExecutableData), + modifiedPayloads: make(map[beacon.Slot]*api.ExecutableData), + signedBeaconBlock: make(map[tree.Root]bool), + + cfg: &config{ + host: DEFAULT_BUILDER_HOST, + port: DEFAULT_BUILDER_PORT, + builderApiDomain: beacon.ComputeDomain( + DOMAIN_APPLICATION_BUILDER, + spec.GENESIS_FORK_VERSION, + tree.Root{}, + ), + }, + } + + for _, o := range opts { + if err = o(m); err != nil { + return nil, err + } + } + + // builder key + skByte := [32]byte{} + sk := blsu.SecretKey{} + rand.Read(skByte[:]) + (&sk).Deserialize(&skByte) + m.sk = &sk + if m.pk, err = blsu.SkToPk(m.sk); err != nil { + panic(err) + } + pkBytes := m.pk.Serialize() + copy(m.pkBeacon[:], pkBytes[:]) + + router := mux.NewRouter() + router.HandleFunc("/eth/v1/builder/validators", m.HandleValidators). + Methods("POST") + router.HandleFunc("/eth/v1/builder/header/{slot}/{parenthash}/{pubkey}", m.HandleGetExecutionPayloadHeader). + Methods("GET") + router.HandleFunc("/eth/v1/builder/blinded_blocks", m.HandleSubmitBlindedBlock). + Methods("POST") + router.HandleFunc("/eth/v1/builder/status", m.HandleStatus).Methods("GET") + + m.srv = &http.Server{ + Handler: router, + Addr: fmt.Sprintf("%s:%d", m.cfg.host, m.cfg.port), + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + if err := m.Start(ctx); err != nil && err != context.Canceled { + panic(err) + } + }() + m.cancel = cancel + + return m, nil +} + +func (m *MockBuilder) Cancel() error { + if m.cancel != nil { + m.cancel() + } + return nil +} + +// Start a proxy server. +func (m *MockBuilder) Start(ctx context.Context) error { + m.srv.BaseContext = func(listener net.Listener) context.Context { + return ctx + } + el_address := "unknown yet" + + if addr, err := m.el.UserRPCAddress(); err == nil { + el_address = addr + } + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "address": m.address, + "port": m.cfg.port, + "el_address": el_address, + }).Info("Builder now listening") + go func() { + if err := m.srv.ListenAndServe(); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + }).Error(err) + } + }() + for { + <-ctx.Done() + return m.srv.Shutdown(ctx) + } +} + +func (m *MockBuilder) Address() string { + return fmt.Sprintf( + "http://%s@%v:%d", + m.pkBeacon.String(), + m.cfg.externalIP, + m.cfg.port, + ) +} + +func (m *MockBuilder) GetBuiltPayloadsCount() int { + return len(m.builtPayloads) +} + +func (m *MockBuilder) GetSignedBeaconBlockCount() int { + return len(m.signedBeaconBlock) +} + +func (m *MockBuilder) GetBuiltPayloads() map[beacon.Slot]*api.ExecutableData { + mapCopy := make(map[beacon.Slot]*api.ExecutableData) + for k, v := range m.builtPayloads { + mapCopy[k] = v + } + return mapCopy +} + +func (m *MockBuilder) GetModifiedPayloads() map[beacon.Slot]*api.ExecutableData { + mapCopy := make(map[beacon.Slot]*api.ExecutableData) + for k, v := range m.modifiedPayloads { + mapCopy[k] = v + } + return mapCopy +} + +func (m *MockBuilder) HandleValidators( + w http.ResponseWriter, + req *http.Request, +) { + requestBytes, err := ioutil.ReadAll(req.Body) + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to read request body") + http.Error(w, "Unable to read request body", http.StatusBadRequest) + return + } + var signedValidatorRegistrations []common.SignedValidatorRegistrationV1 + if err := json.Unmarshal(requestBytes, &signedValidatorRegistrations); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to parse request body") + http.Error(w, "Unable to parse request body", http.StatusBadRequest) + return + } + + for _, vr := range signedValidatorRegistrations { + // Verify signature + signingRoot := beacon.ComputeSigningRoot( + vr.Message.HashTreeRoot(tree.GetHashFn()), + m.cfg.builderApiDomain, + ) + + pk, err := vr.Message.PubKey.Pubkey() + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to deserialize pubkey") + http.Error( + w, + "Unable to deserialize pubkey", + http.StatusBadRequest, + ) + return + } + + sig, err := vr.Signature.Signature() + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to deserialize signature") + http.Error( + w, + "Unable to deserialize signature", + http.StatusBadRequest, + ) + return + } + + if !blsu.Verify(pk, signingRoot[:], sig) { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "pubkey": vr.Message.PubKey, + "fee_recipient": vr.Message.FeeRecipient, + "timestamp": vr.Message.Timestamp, + "gas_limit": vr.Message.GasLimit, + "signature": vr.Signature, + }).Error("Unable to verify signature") + http.Error( + w, + "Unable to verify signature", + http.StatusBadRequest, + ) + return + } + var addr el_common.Address + copy(addr[:], vr.Message.FeeRecipient[:]) + m.suggestedFeeRecipientsMutex.Lock() + m.suggestedFeeRecipients[vr.Message.PubKey] = addr + m.suggestedFeeRecipientsMutex.Unlock() + } + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "validator_count": len(signedValidatorRegistrations), + }).Info( + "Received validator registrations", + ) + w.WriteHeader(http.StatusOK) + +} + +func (m *MockBuilder) SlotToTimestamp(slot beacon.Slot) uint64 { + return uint64( + m.cfg.beaconGenesisTime + beacon.Timestamp( + slot, + )*beacon.Timestamp( + m.spec.SECONDS_PER_SLOT, + ), + ) +} + +type PayloadHeaderRequestVarsParser map[string]string + +func (vars PayloadHeaderRequestVarsParser) Slot() (slot beacon.Slot, err error) { + if slotStr, ok := vars["slot"]; ok { + err = (&slot).UnmarshalJSON([]byte(slotStr)) + } else { + err = fmt.Errorf("no slot") + } + return slot, err +} + +func (vars PayloadHeaderRequestVarsParser) PubKey() (pubkey beacon.BLSPubkey, err error) { + if pubkeyStr, ok := vars["pubkey"]; ok { + err = (&pubkey).UnmarshalText([]byte(pubkeyStr)) + } else { + err = fmt.Errorf("no pubkey") + } + return pubkey, err +} + +func (vars PayloadHeaderRequestVarsParser) ParentHash() (el_common.Hash, error) { + if parentHashStr, ok := vars["parenthash"]; ok { + return el_common.HexToHash(parentHashStr), nil + } + return el_common.Hash{}, fmt.Errorf("no parent_hash") +} + +func (m *MockBuilder) HandleGetExecutionPayloadHeader( + w http.ResponseWriter, req *http.Request, +) { + var ( + prevRandao el_common.Hash + payloadModified = false + vars = PayloadHeaderRequestVarsParser(mux.Vars(req)) + ) + + slot, err := vars.Slot() + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to parse request url") + http.Error( + w, + "Unable to parse request url", + http.StatusBadRequest, + ) + return + } + + parentHash, err := vars.ParentHash() + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to parse request url") + http.Error( + w, + "Unable to parse request url", + http.StatusBadRequest, + ) + return + } + + pubkey, err := vars.PubKey() + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to parse request url") + http.Error( + w, + "Unable to parse request url", + http.StatusBadRequest, + ) + return + } + + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "slot": slot, + "parent_hash": parentHash, + "pubkey": pubkey, + }).Info( + "Received request for header", + ) + // Request head state from the CL + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + state, err := m.cl.BeaconStateV2(ctx, eth2api.StateHead) + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "slot": slot, + "err": err, + }).Error("Error getting beacon state from CL") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + var forkchoiceState *api.ForkchoiceStateV1 + if bytes.Equal(parentHash[:], EMPTY_HASH[:]) { + // Edge case where the CL is requesting us to build the very first block + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + genesis, err := m.el.BlockByNumber(ctx, nil) + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Error getting latest block from the EL") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + forkchoiceState = &api.ForkchoiceStateV1{ + HeadBlockHash: genesis.Hash(), + } + } else { + // Check if we have the correct beacon state + latestExecPayloadHeaderHash := state.LatestExecutionPayloadHeaderHash() + if !bytes.Equal(latestExecPayloadHeaderHash[:], parentHash[:]) { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "latestExecPayloadHeaderHash": latestExecPayloadHeaderHash.String(), + "parentHash": parentHash.String(), + "err": "beacon state latest execution payload hash and parent hash requested don't match", + }).Error("Unable to respond to header request") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + // Check if we know the latest forkchoice updated + if m.el.LatestForkchoice == nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": "last fcu is unknown", + }).Error("Unable to respond to header request") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + // Check if the requested parent matches the last fcu + if !bytes.Equal(m.el.LatestForkchoice.HeadBlockHash[:], parentHash[:]) { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": "last fcu head and requested parent don't match", + }).Error("Unable to respond to header request") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + forkchoiceState = m.el.LatestForkchoice + } + + // Build payload attributes + + // PrevRandao + prevRandaoMixes := state.RandaoMixes() + prevRandaoRoot := prevRandaoMixes[m.spec.SlotToEpoch(slot-1)] + copy(prevRandao[:], prevRandaoRoot[:]) + + // Timestamp + timestamp := m.SlotToTimestamp(slot) + + // Suggested Fee Recipient + suggestedFeeRecipient := m.suggestedFeeRecipients[pubkey] + + // Withdrawals + var withdrawals types.Withdrawals + if m.spec.SlotToEpoch(slot) >= m.spec.CAPELLA_FORK_EPOCH { + wSsz, err := state.NextWithdrawals(slot) + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to obtain correct list of withdrawals") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + withdrawals = make(types.Withdrawals, len(wSsz)) + for i, w := range wSsz { + newWithdrawal := types.Withdrawal{} + copy(newWithdrawal.Address[:], w.Address[:]) + newWithdrawal.Amount = uint64(w.Amount) + newWithdrawal.Index = uint64(w.Index) + newWithdrawal.Validator = uint64(w.ValidatorIndex) + withdrawals[i] = &newWithdrawal + } + } + + pAttr := api.PayloadAttributes{ + Timestamp: timestamp, + Random: prevRandao, + SuggestedFeeRecipient: suggestedFeeRecipient, + Withdrawals: withdrawals, + } + + if m.cfg.payloadAttrModifier != nil { + if mod, err := m.cfg.payloadAttrModifier(&pAttr, slot); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to modify payload attributes using modifier") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } else if mod { + payloadModified = true + } + } + + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "Timestamp": timestamp, + "PrevRandao": prevRandao, + "SuggestedFeeRecipient": suggestedFeeRecipient, + "Withdrawals": withdrawals, + }).Info("Built payload attributes for header") + + // Request a payload from the execution client + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := m.el.EngineForkchoiceUpdated( + ctx, + forkchoiceState, + &pAttr, + 2, + ) + if err != nil || r.PayloadID == nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + "payloadID": r.PayloadID, + }).Error("Error on ForkchoiceUpdated to EL") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + // Wait for EL to produce payload + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "payloadID": r.PayloadID.String(), + }).Info("Waiting for payload from EL") + + time.Sleep(200 * time.Millisecond) + + // Request payload from the EL + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + p, bValue, err := m.el.EngineGetPayload(ctx, r.PayloadID, 2) + if err != nil || p == nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + "payload": p, + }).Error("Error on GetPayload to EL") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + // Watermark payload + if err := ModifyExtraData(p, []byte("builder payload")); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Error modifying payload") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + // Modify the payload if necessary + if m.cfg.payloadModifier != nil { + if mod, err := m.cfg.payloadModifier(p, slot); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Error modifying payload") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } else if mod { + payloadModified = true + } + } + + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "payload": p.BlockHash.String(), + }).Info("Built payload from EL") + + // We are ready to respond to the CL + var ( + builderBid common.BuilderBid + version string + ) + + if m.spec.SlotToEpoch(slot) >= m.spec.CAPELLA_FORK_EPOCH { + builderBid = &capella.BuilderBid{} + version = "capella" + } else if m.spec.SlotToEpoch(slot) >= m.spec.BELLATRIX_FORK_EPOCH { + builderBid = &bellatrix.BuilderBid{} + version = "bellatrix" + } else { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": fmt.Errorf("payload requested from improper fork"), + }).Error("Invalid slot requested") + http.Error( + w, + "Unable to respond to header request", + http.StatusBadRequest, + ) + return + } + + if err := builderBid.FromExecutableData(m.spec, p); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Error building bid from execution data") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + if m.cfg.payloadWeiValueBump != nil { + // If requested, fake a higher gwei so the CL always takes the bid + bValue = bValue.Add(bValue, m.cfg.payloadWeiValueBump) + } + builderBid.SetValue(bValue) + builderBid.SetPubKey(m.pkBeacon) + + signedBid, err := builderBid.Sign(m.cfg.builderApiDomain, m.sk, m.pk) + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Error signing bid from execution data") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + // Check if we are supposed to simulate an error + if m.cfg.errorOnHeaderRequest != nil { + if err := m.cfg.errorOnHeaderRequest(slot); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "slot": slot, + "err": err, + }).Error("Simulated error") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + } + + versionedSignedBid := signedBid.Versioned(version) + if err := serveJSON(w, versionedSignedBid); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Error versioning bid from execution data") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + + // Finally add the execution payload to the cache + m.builtPayloadsMutex.Lock() + m.builtPayloads[slot] = p + m.builtPayloadsMutex.Unlock() + if payloadModified { + m.modifiedPayloadsMutex.Lock() + m.modifiedPayloads[slot] = p + m.modifiedPayloadsMutex.Unlock() + } +} + +type SlotEnvelope struct { + Slot beacon.Slot `json:"slot" yaml:"slot"` +} + +type MessageSlotEnvelope struct { + SlotEnvelope SlotEnvelope `json:"message" yaml:"message"` +} + +func (m *MockBuilder) HandleSubmitBlindedBlock( + w http.ResponseWriter, req *http.Request, +) { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + }).Info( + "Received submission for blinded blocks", + ) + requestBytes, err := ioutil.ReadAll(req.Body) + if err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to read request body") + http.Error(w, "Unable to read request body", http.StatusBadRequest) + return + } + + // First try to find out the slot to get the version of the block + var messageSlotEnvelope MessageSlotEnvelope + if err := json.Unmarshal(requestBytes, &messageSlotEnvelope); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to parse request body") + http.Error(w, "Unable to parse request body", http.StatusBadRequest) + return + } + + var ( + signedBeaconBlock common.SignedBlindedBeacon + executionPayloadResp common.ExecutionPayloadResponse + ) + if m.spec.SlotToEpoch( + messageSlotEnvelope.SlotEnvelope.Slot, + ) >= m.spec.CAPELLA_FORK_EPOCH { + signedBeaconBlock = &capella.SignedBeaconBlock{} + executionPayloadResp.Version = "capella" + executionPayloadResp.Data = &capella.ExecutionPayload{} + } else if m.spec.SlotToEpoch(messageSlotEnvelope.SlotEnvelope.Slot) >= m.spec.BELLATRIX_FORK_EPOCH { + signedBeaconBlock = &bellatrix.SignedBeaconBlock{} + executionPayloadResp.Version = "bellatrix" + executionPayloadResp.Data = &bellatrix.ExecutionPayload{} + } else { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": fmt.Errorf("received signed beacon blinded block of unknown fork"), + }).Error("Invalid slot requested") + http.Error( + w, + "Unable to respond to header request", + http.StatusBadRequest, + ) + return + } + // Unmarshall the full signed beacon block + if err := json.Unmarshal(requestBytes, &signedBeaconBlock); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Unable to parse request body") + http.Error(w, "Unable to parse request body", http.StatusBadRequest) + return + } + + // Look up the payload in the history of payloads + p, ok := m.builtPayloads[messageSlotEnvelope.SlotEnvelope.Slot] + if !ok { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "slot": messageSlotEnvelope.SlotEnvelope.Slot, + }).Error("Could not find payload in history") + http.Error(w, "Unable to get payload", http.StatusInternalServerError) + return + } + + // Prepare response + executionPayloadResp.Data.FromExecutableData(p) + + // Embed the execution payload in the block to obtain correct root + signedBeaconBlock.SetExecutionPayload( + executionPayloadResp.Data, + ) + + // Record the signed beacon block + signedBeaconBlockRoot := signedBeaconBlock.Root(m.spec) + m.signedBeaconBlockMutex.Lock() + m.signedBeaconBlock[signedBeaconBlockRoot] = true + m.signedBeaconBlockMutex.Unlock() + + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "root": signedBeaconBlock.Root(m.spec).String(), + "stateRoot": signedBeaconBlock.StateRoot().String(), + "slot": signedBeaconBlock.Slot().String(), + }).Info("Received signed beacon block") + + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "payload": p.BlockHash.String(), + }).Info("Built payload sent to CL") + + // Check if we are supposed to simulate an error + if m.cfg.errorOnPayloadReveal != nil { + if err := m.cfg.errorOnPayloadReveal(messageSlotEnvelope.SlotEnvelope.Slot); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "slot": messageSlotEnvelope.SlotEnvelope.Slot, + "err": err, + }).Error("Simulated error") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } + } + + if err := serveJSON(w, executionPayloadResp); err != nil { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + "err": err, + }).Error("Error preparing response from payload") + http.Error( + w, + "Unable to respond to header request", + http.StatusInternalServerError, + ) + return + } +} + +func (m *MockBuilder) HandleStatus( + w http.ResponseWriter, req *http.Request, +) { + logrus.WithFields(logrus.Fields{ + "builder_id": m.cfg.id, + }).Info( + "Received request for status", + ) + w.WriteHeader(http.StatusOK) +} + +func serveJSON(w http.ResponseWriter, value interface{}) error { + resp, err := json.Marshal(value) + if err != nil { + return err + } + w.Header().Set("content-type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(resp) + return nil +} + +func ModifyExtraData(p *api.ExecutableData, newExtraData []byte) error { + if p == nil { + return fmt.Errorf("nil payload") + } + if b, err := api.ExecutableDataToBlock(*p); err != nil { + return err + } else { + h := b.Header() + h.Extra = newExtraData + p.ExtraData = newExtraData + p.BlockHash = h.Hash() + } + return nil +} diff --git a/simulators/eth2/common/builder/mock/options.go b/simulators/eth2/common/builder/mock/options.go new file mode 100644 index 0000000000..7192bb4f20 --- /dev/null +++ b/simulators/eth2/common/builder/mock/options.go @@ -0,0 +1,113 @@ +package mock_builder + +import ( + "math/big" + "net" + + api "github.com/ethereum/go-ethereum/core/beacon" + beacon "github.com/protolambda/zrnt/eth2/beacon/common" +) + +type PayloadAttributesModifier func(*api.PayloadAttributes, beacon.Slot) (bool, error) +type PayloadModifier func(*api.ExecutableData, beacon.Slot) (bool, error) +type ErrorProducer func(beacon.Slot) error + +type config struct { + id int + port int + host string + externalIP net.IP + builderApiDomain beacon.BLSDomain + beaconGenesisTime beacon.Timestamp + payloadWeiValueBump *big.Int + + payloadAttrModifier PayloadAttributesModifier + payloadModifier PayloadModifier + errorOnHeaderRequest ErrorProducer + errorOnPayloadReveal ErrorProducer +} + +type Option func(m *MockBuilder) error + +func WithID(id int) Option { + return func(m *MockBuilder) error { + m.cfg.id = id + return nil + } +} + +func WithHost(host string) Option { + return func(m *MockBuilder) error { + m.cfg.host = host + return nil + } +} + +func WithPort(port int) Option { + return func(m *MockBuilder) error { + m.cfg.port = port + return nil + } +} + +func WithExternalIP(ip net.IP) Option { + return func(m *MockBuilder) error { + m.cfg.externalIP = ip + return nil + } +} + +func WithExecutionClient() Option { + return func(m *MockBuilder) error { + return nil + } +} + +func WithBuilderApiDomain(domain beacon.BLSDomain) Option { + return func(m *MockBuilder) error { + m.cfg.builderApiDomain = domain + return nil + } +} + +func WithBeaconGenesisTime(t beacon.Timestamp) Option { + return func(m *MockBuilder) error { + m.cfg.beaconGenesisTime = t + return nil + } +} + +func WithPayloadWeiValueBump(wei *big.Int) Option { + return func(m *MockBuilder) error { + m.cfg.payloadWeiValueBump = wei + return nil + } +} + +func WithPayloadAttributesModifier(pam PayloadAttributesModifier) Option { + return func(m *MockBuilder) error { + m.cfg.payloadAttrModifier = pam + return nil + } +} + +func WithPayloadModifier(pm PayloadModifier) Option { + return func(m *MockBuilder) error { + m.cfg.payloadModifier = pm + return nil + } +} + +func WithErrorOnHeaderRequest(e ErrorProducer) Option { + return func(m *MockBuilder) error { + m.cfg.errorOnHeaderRequest = e + return nil + } +} + +func WithErrorOnPayloadReveal(e ErrorProducer) Option { + return func(m *MockBuilder) error { + m.cfg.errorOnPayloadReveal = e + return nil + } +} diff --git a/simulators/eth2/common/builder/types/bellatrix/bellatrix.go b/simulators/eth2/common/builder/types/bellatrix/bellatrix.go new file mode 100644 index 0000000000..669c7cd617 --- /dev/null +++ b/simulators/eth2/common/builder/types/bellatrix/bellatrix.go @@ -0,0 +1,221 @@ +package bellatrix + +import ( + "fmt" + "math/big" + + el_common "github.com/ethereum/go-ethereum/common" + api "github.com/ethereum/go-ethereum/core/beacon" + "github.com/ethereum/hive/simulators/eth2/common/builder/types/common" + blsu "github.com/protolambda/bls12-381-util" + "github.com/protolambda/zrnt/eth2/beacon/bellatrix" + beacon "github.com/protolambda/zrnt/eth2/beacon/common" + "github.com/protolambda/ztyp/tree" + "github.com/protolambda/ztyp/view" +) + +type SignedBeaconBlock bellatrix.SignedBeaconBlock + +func (s *SignedBeaconBlock) ExecutionPayloadHash() el_common.Hash { + var hash el_common.Hash + copy(hash[:], s.Message.Body.ExecutionPayload.BlockHash[:]) + return hash +} + +func (s *SignedBeaconBlock) Root(spec *beacon.Spec) tree.Root { + return s.Message.HashTreeRoot(spec, tree.GetHashFn()) +} + +func (s *SignedBeaconBlock) StateRoot() tree.Root { + return s.Message.StateRoot +} + +func (s *SignedBeaconBlock) Slot() beacon.Slot { + return s.Message.Slot +} + +func (s *SignedBeaconBlock) SetExecutionPayload( + ep common.ExecutionPayload, +) error { + s.Message.Body.ExecutionPayload.ParentHash = ep.GetParentHash() + s.Message.Body.ExecutionPayload.FeeRecipient = ep.GetFeeRecipient() + s.Message.Body.ExecutionPayload.StateRoot = ep.GetStateRoot() + s.Message.Body.ExecutionPayload.ReceiptsRoot = ep.GetReceiptsRoot() + s.Message.Body.ExecutionPayload.LogsBloom = ep.GetLogsBloom() + s.Message.Body.ExecutionPayload.PrevRandao = ep.GetPrevRandao() + s.Message.Body.ExecutionPayload.BlockNumber = ep.GetBlockNumber() + s.Message.Body.ExecutionPayload.GasLimit = ep.GetGasLimit() + s.Message.Body.ExecutionPayload.GasUsed = ep.GetGasUsed() + s.Message.Body.ExecutionPayload.Timestamp = ep.GetTimestamp() + s.Message.Body.ExecutionPayload.ExtraData = ep.GetExtraData() + s.Message.Body.ExecutionPayload.BaseFeePerGas = ep.GetBaseFeePerGas() + s.Message.Body.ExecutionPayload.BlockHash = ep.GetBlockHash() + s.Message.Body.ExecutionPayload.Transactions = ep.GetTransactions() + return nil +} + +type BuilderBid struct { + Header bellatrix.ExecutionPayloadHeader `json:"header" yaml:"header"` + Value view.Uint256View `json:"value" yaml:"value"` + PubKey beacon.BLSPubkey `json:"pubkey" yaml:"pubkey"` +} + +func (b *BuilderBid) HashTreeRoot(hFn tree.HashFn) tree.Root { + return hFn.HashTreeRoot( + &b.Header, + &b.Value, + &b.PubKey, + ) +} + +func (b *BuilderBid) FromExecutableData( + spec *beacon.Spec, + ed *api.ExecutableData, +) error { + if ed == nil { + return fmt.Errorf("nil execution payload") + } + if ed.Withdrawals != nil { + return fmt.Errorf("execution data contains withdrawals") + } + copy(b.Header.ParentHash[:], ed.ParentHash[:]) + copy(b.Header.FeeRecipient[:], ed.FeeRecipient[:]) + copy(b.Header.StateRoot[:], ed.StateRoot[:]) + copy(b.Header.ReceiptsRoot[:], ed.ReceiptsRoot[:]) + copy(b.Header.LogsBloom[:], ed.LogsBloom[:]) + copy(b.Header.PrevRandao[:], ed.Random[:]) + + b.Header.BlockNumber = view.Uint64View(ed.Number) + b.Header.GasLimit = view.Uint64View(ed.GasLimit) + b.Header.GasUsed = view.Uint64View(ed.GasUsed) + b.Header.Timestamp = beacon.Timestamp(ed.Timestamp) + + b.Header.ExtraData = make(beacon.ExtraData, len(ed.ExtraData)) + copy(b.Header.ExtraData[:], ed.ExtraData[:]) + b.Header.BaseFeePerGas.SetFromBig(ed.BaseFeePerGas) + copy(b.Header.BlockHash[:], ed.BlockHash[:]) + + txs := make(beacon.PayloadTransactions, len(ed.Transactions)) + for i, tx := range ed.Transactions { + txs[i] = make(beacon.Transaction, len(tx)) + copy(txs[i][:], tx[:]) + } + txRoot := txs.HashTreeRoot(spec, tree.GetHashFn()) + copy(b.Header.TransactionsRoot[:], txRoot[:]) + + return nil +} + +func (b *BuilderBid) SetValue(value *big.Int) { + b.Value.SetFromBig(value) +} + +func (b *BuilderBid) SetPubKey(pk beacon.BLSPubkey) { + b.PubKey = pk +} + +func (b *BuilderBid) Sign( + domain beacon.BLSDomain, + sk *blsu.SecretKey, + pk *blsu.Pubkey, +) (*common.SignedBuilderBid, error) { + pkBytes := pk.Serialize() + copy(b.PubKey[:], pkBytes[:]) + sigRoot := beacon.ComputeSigningRoot( + b.HashTreeRoot(tree.GetHashFn()), + domain, + ) + return &common.SignedBuilderBid{ + Message: b, + Signature: beacon.BLSSignature(blsu.Sign(sk, sigRoot[:]).Serialize()), + }, nil +} + +type ExecutionPayload bellatrix.ExecutionPayload + +func (p *ExecutionPayload) FromExecutableData(ed *api.ExecutableData) error { + if ed == nil { + return fmt.Errorf("nil execution payload") + } + if ed.Withdrawals != nil { + return fmt.Errorf("execution data contains withdrawals") + } + copy(p.ParentHash[:], ed.ParentHash[:]) + copy(p.FeeRecipient[:], ed.FeeRecipient[:]) + copy(p.StateRoot[:], ed.StateRoot[:]) + copy(p.ReceiptsRoot[:], ed.ReceiptsRoot[:]) + copy(p.LogsBloom[:], ed.LogsBloom[:]) + copy(p.PrevRandao[:], ed.Random[:]) + + p.BlockNumber = view.Uint64View(ed.Number) + p.GasLimit = view.Uint64View(ed.GasLimit) + p.GasUsed = view.Uint64View(ed.GasUsed) + p.Timestamp = beacon.Timestamp(ed.Timestamp) + + p.ExtraData = make(beacon.ExtraData, len(ed.ExtraData)) + copy(p.ExtraData[:], ed.ExtraData[:]) + p.BaseFeePerGas.SetFromBig(ed.BaseFeePerGas) + copy(p.BlockHash[:], ed.BlockHash[:]) + p.Transactions = make(beacon.PayloadTransactions, len(ed.Transactions)) + for i, tx := range ed.Transactions { + p.Transactions[i] = make(beacon.Transaction, len(tx)) + copy(p.Transactions[i][:], tx[:]) + } + return nil +} + +func (p *ExecutionPayload) GetParentHash() beacon.Hash32 { + return p.ParentHash +} + +func (p *ExecutionPayload) GetFeeRecipient() beacon.Eth1Address { + return p.FeeRecipient +} + +func (p *ExecutionPayload) GetStateRoot() beacon.Bytes32 { + return p.StateRoot +} + +func (p *ExecutionPayload) GetReceiptsRoot() beacon.Bytes32 { + return p.ReceiptsRoot +} + +func (p *ExecutionPayload) GetLogsBloom() beacon.LogsBloom { + return p.LogsBloom +} + +func (p *ExecutionPayload) GetPrevRandao() beacon.Bytes32 { + return p.PrevRandao +} + +func (p *ExecutionPayload) GetBlockNumber() view.Uint64View { + return p.BlockNumber +} + +func (p *ExecutionPayload) GetGasLimit() view.Uint64View { + return p.GasLimit +} + +func (p *ExecutionPayload) GetGasUsed() view.Uint64View { + return p.GasUsed +} + +func (p *ExecutionPayload) GetTimestamp() beacon.Timestamp { + return p.Timestamp +} + +func (p *ExecutionPayload) GetExtraData() beacon.ExtraData { + return p.ExtraData +} + +func (p *ExecutionPayload) GetBaseFeePerGas() view.Uint256View { + return p.BaseFeePerGas +} + +func (p *ExecutionPayload) GetBlockHash() beacon.Hash32 { + return p.BlockHash +} + +func (p *ExecutionPayload) GetTransactions() beacon.PayloadTransactions { + return p.Transactions +} diff --git a/simulators/eth2/common/builder/types/capella/capella.go b/simulators/eth2/common/builder/types/capella/capella.go new file mode 100644 index 0000000000..08cefa0071 --- /dev/null +++ b/simulators/eth2/common/builder/types/capella/capella.go @@ -0,0 +1,247 @@ +package capella + +import ( + "fmt" + "math/big" + + el_common "github.com/ethereum/go-ethereum/common" + api "github.com/ethereum/go-ethereum/core/beacon" + "github.com/ethereum/hive/simulators/eth2/common/builder/types/common" + blsu "github.com/protolambda/bls12-381-util" + "github.com/protolambda/zrnt/eth2/beacon/capella" + beacon "github.com/protolambda/zrnt/eth2/beacon/common" + "github.com/protolambda/ztyp/tree" + "github.com/protolambda/ztyp/view" +) + +type SignedBeaconBlock capella.SignedBeaconBlock + +func (s *SignedBeaconBlock) ExecutionPayloadHash() el_common.Hash { + var hash el_common.Hash + copy(hash[:], s.Message.Body.ExecutionPayload.BlockHash[:]) + return hash +} + +func (s *SignedBeaconBlock) Root(spec *beacon.Spec) tree.Root { + return s.Message.HashTreeRoot(spec, tree.GetHashFn()) +} + +func (s *SignedBeaconBlock) StateRoot() tree.Root { + return s.Message.StateRoot +} + +func (s *SignedBeaconBlock) Slot() beacon.Slot { + return s.Message.Slot +} + +func (s *SignedBeaconBlock) SetExecutionPayload( + ep common.ExecutionPayload, +) error { + if ep, ok := ep.(common.ExecutionPayloadWithdrawals); ok { + s.Message.Body.ExecutionPayload.ParentHash = ep.GetParentHash() + s.Message.Body.ExecutionPayload.FeeRecipient = ep.GetFeeRecipient() + s.Message.Body.ExecutionPayload.StateRoot = ep.GetStateRoot() + s.Message.Body.ExecutionPayload.ReceiptsRoot = ep.GetReceiptsRoot() + s.Message.Body.ExecutionPayload.LogsBloom = ep.GetLogsBloom() + s.Message.Body.ExecutionPayload.PrevRandao = ep.GetPrevRandao() + s.Message.Body.ExecutionPayload.BlockNumber = ep.GetBlockNumber() + s.Message.Body.ExecutionPayload.GasLimit = ep.GetGasLimit() + s.Message.Body.ExecutionPayload.GasUsed = ep.GetGasUsed() + s.Message.Body.ExecutionPayload.Timestamp = ep.GetTimestamp() + s.Message.Body.ExecutionPayload.ExtraData = ep.GetExtraData() + s.Message.Body.ExecutionPayload.BaseFeePerGas = ep.GetBaseFeePerGas() + s.Message.Body.ExecutionPayload.BlockHash = ep.GetBlockHash() + s.Message.Body.ExecutionPayload.Transactions = ep.GetTransactions() + s.Message.Body.ExecutionPayload.Withdrawals = ep.GetWithdrawals() + return nil + } else { + return fmt.Errorf("invalid payload for capella") + } +} + +type BuilderBid struct { + Header capella.ExecutionPayloadHeader `json:"header" yaml:"header"` + Value view.Uint256View `json:"value" yaml:"value"` + PubKey beacon.BLSPubkey `json:"pubkey" yaml:"pubkey"` +} + +func (b *BuilderBid) HashTreeRoot(hFn tree.HashFn) tree.Root { + return hFn.HashTreeRoot( + &b.Header, + &b.Value, + &b.PubKey, + ) +} + +func (b *BuilderBid) FromExecutableData( + spec *beacon.Spec, + ed *api.ExecutableData, +) error { + if ed == nil { + return fmt.Errorf("nil execution payload") + } + if ed.Withdrawals == nil { + return fmt.Errorf("execution data does not contain withdrawals") + } + copy(b.Header.ParentHash[:], ed.ParentHash[:]) + copy(b.Header.FeeRecipient[:], ed.FeeRecipient[:]) + copy(b.Header.StateRoot[:], ed.StateRoot[:]) + copy(b.Header.ReceiptsRoot[:], ed.ReceiptsRoot[:]) + copy(b.Header.LogsBloom[:], ed.LogsBloom[:]) + copy(b.Header.PrevRandao[:], ed.Random[:]) + + b.Header.BlockNumber = view.Uint64View(ed.Number) + b.Header.GasLimit = view.Uint64View(ed.GasLimit) + b.Header.GasUsed = view.Uint64View(ed.GasUsed) + b.Header.Timestamp = beacon.Timestamp(ed.Timestamp) + + b.Header.ExtraData = make(beacon.ExtraData, len(ed.ExtraData)) + copy(b.Header.ExtraData[:], ed.ExtraData[:]) + b.Header.BaseFeePerGas.SetFromBig(ed.BaseFeePerGas) + copy(b.Header.BlockHash[:], ed.BlockHash[:]) + + txs := make(beacon.PayloadTransactions, len(ed.Transactions)) + for i, tx := range ed.Transactions { + txs[i] = make(beacon.Transaction, len(tx)) + copy(txs[i][:], tx[:]) + } + txRoot := txs.HashTreeRoot(spec, tree.GetHashFn()) + copy(b.Header.TransactionsRoot[:], txRoot[:]) + + withdrawals := make(beacon.Withdrawals, len(ed.Withdrawals)) + for i, w := range ed.Withdrawals { + withdrawals[i].Index = beacon.WithdrawalIndex(w.Index) + withdrawals[i].ValidatorIndex = beacon.ValidatorIndex(w.Validator) + copy(withdrawals[i].Address[:], w.Address[:]) + withdrawals[i].Amount = beacon.Gwei(w.Amount) + } + withdrawalsRoot := withdrawals.HashTreeRoot(spec, tree.GetHashFn()) + copy(b.Header.WithdrawalsRoot[:], withdrawalsRoot[:]) + + return nil +} + +func (b *BuilderBid) SetValue(value *big.Int) { + b.Value.SetFromBig(value) +} + +func (b *BuilderBid) SetPubKey(pk beacon.BLSPubkey) { + b.PubKey = pk +} + +func (b *BuilderBid) Sign( + domain beacon.BLSDomain, + sk *blsu.SecretKey, + pk *blsu.Pubkey, +) (*common.SignedBuilderBid, error) { + pkBytes := pk.Serialize() + copy(b.PubKey[:], pkBytes[:]) + sigRoot := beacon.ComputeSigningRoot( + b.HashTreeRoot(tree.GetHashFn()), + domain, + ) + return &common.SignedBuilderBid{ + Message: b, + Signature: beacon.BLSSignature(blsu.Sign(sk, sigRoot[:]).Serialize()), + }, nil +} + +type ExecutionPayload capella.ExecutionPayload + +func (p *ExecutionPayload) FromExecutableData(ed *api.ExecutableData) error { + if ed == nil { + return fmt.Errorf("nil execution payload") + } + if ed.Withdrawals == nil { + return fmt.Errorf("execution data does not contain withdrawals") + } + copy(p.ParentHash[:], ed.ParentHash[:]) + copy(p.FeeRecipient[:], ed.FeeRecipient[:]) + copy(p.StateRoot[:], ed.StateRoot[:]) + copy(p.ReceiptsRoot[:], ed.ReceiptsRoot[:]) + copy(p.LogsBloom[:], ed.LogsBloom[:]) + copy(p.PrevRandao[:], ed.Random[:]) + + p.BlockNumber = view.Uint64View(ed.Number) + p.GasLimit = view.Uint64View(ed.GasLimit) + p.GasUsed = view.Uint64View(ed.GasUsed) + p.Timestamp = beacon.Timestamp(ed.Timestamp) + + p.ExtraData = make(beacon.ExtraData, len(ed.ExtraData)) + copy(p.ExtraData[:], ed.ExtraData[:]) + p.BaseFeePerGas.SetFromBig(ed.BaseFeePerGas) + copy(p.BlockHash[:], ed.BlockHash[:]) + p.Transactions = make(beacon.PayloadTransactions, len(ed.Transactions)) + for i, tx := range ed.Transactions { + p.Transactions[i] = make(beacon.Transaction, len(tx)) + copy(p.Transactions[i][:], tx[:]) + } + p.Withdrawals = make(beacon.Withdrawals, len(ed.Withdrawals)) + for i, w := range ed.Withdrawals { + p.Withdrawals[i].Index = beacon.WithdrawalIndex(w.Index) + p.Withdrawals[i].ValidatorIndex = beacon.ValidatorIndex(w.Validator) + copy(p.Withdrawals[i].Address[:], w.Address[:]) + p.Withdrawals[i].Amount = beacon.Gwei(w.Amount) + } + return nil +} + +func (p *ExecutionPayload) GetParentHash() beacon.Hash32 { + return p.ParentHash +} + +func (p *ExecutionPayload) GetFeeRecipient() beacon.Eth1Address { + return p.FeeRecipient +} + +func (p *ExecutionPayload) GetStateRoot() beacon.Bytes32 { + return p.StateRoot +} + +func (p *ExecutionPayload) GetReceiptsRoot() beacon.Bytes32 { + return p.ReceiptsRoot +} + +func (p *ExecutionPayload) GetLogsBloom() beacon.LogsBloom { + return p.LogsBloom +} + +func (p *ExecutionPayload) GetPrevRandao() beacon.Bytes32 { + return p.PrevRandao +} + +func (p *ExecutionPayload) GetBlockNumber() view.Uint64View { + return p.BlockNumber +} + +func (p *ExecutionPayload) GetGasLimit() view.Uint64View { + return p.GasLimit +} + +func (p *ExecutionPayload) GetGasUsed() view.Uint64View { + return p.GasUsed +} + +func (p *ExecutionPayload) GetTimestamp() beacon.Timestamp { + return p.Timestamp +} + +func (p *ExecutionPayload) GetExtraData() beacon.ExtraData { + return p.ExtraData +} + +func (p *ExecutionPayload) GetBaseFeePerGas() view.Uint256View { + return p.BaseFeePerGas +} + +func (p *ExecutionPayload) GetBlockHash() beacon.Hash32 { + return p.BlockHash +} + +func (p *ExecutionPayload) GetTransactions() beacon.PayloadTransactions { + return p.Transactions +} + +func (p *ExecutionPayload) GetWithdrawals() beacon.Withdrawals { + return p.Withdrawals +} diff --git a/simulators/eth2/common/builder/types/common/common.go b/simulators/eth2/common/builder/types/common/common.go new file mode 100644 index 0000000000..c8e468a3d4 --- /dev/null +++ b/simulators/eth2/common/builder/types/common/common.go @@ -0,0 +1,98 @@ +package common + +import ( + "math/big" + + el_common "github.com/ethereum/go-ethereum/common" + api "github.com/ethereum/go-ethereum/core/beacon" + blsu "github.com/protolambda/bls12-381-util" + "github.com/protolambda/zrnt/eth2/beacon/common" + beacon "github.com/protolambda/zrnt/eth2/beacon/common" + "github.com/protolambda/ztyp/tree" + "github.com/protolambda/ztyp/view" +) + +type ValidatorRegistrationV1 struct { + FeeRecipient common.Eth1Address `json:"fee_recipient" yaml:"fee_recipient"` + GasLimit view.Uint64View `json:"gas_limit" yaml:"gas_limit"` + Timestamp view.Uint64View `json:"timestamp" yaml:"timestamp"` + PubKey common.BLSPubkey `json:"pubkey" yaml:"pubkey"` +} + +func (vr *ValidatorRegistrationV1) HashTreeRoot(hFn tree.HashFn) tree.Root { + return hFn.HashTreeRoot( + &vr.FeeRecipient, + &vr.GasLimit, + &vr.Timestamp, + &vr.PubKey, + ) +} + +type SignedValidatorRegistrationV1 struct { + Message ValidatorRegistrationV1 `json:"message" yaml:"message"` + Signature common.BLSSignature `json:"signature" yaml:"signature"` +} + +type BuilderBid interface { + FromExecutableData(*beacon.Spec, *api.ExecutableData) error + SetValue(*big.Int) + SetPubKey(beacon.BLSPubkey) + Sign(domain beacon.BLSDomain, + sk *blsu.SecretKey, + pk *blsu.Pubkey) (*SignedBuilderBid, error) +} + +type SignedBuilderBid struct { + Message BuilderBid `json:"message" yaml:"message"` + Signature common.BLSSignature `json:"signature" yaml:"signature"` +} + +func (s *SignedBuilderBid) Versioned( + version string, +) *VersionedSignedBuilderBid { + return &VersionedSignedBuilderBid{ + Version: version, + Data: s, + } +} + +type VersionedSignedBuilderBid struct { + Version string `json:"version" yaml:"version"` + Data *SignedBuilderBid `json:"data" yaml:"data"` +} + +type SignedBlindedBeacon interface { + ExecutionPayloadHash() el_common.Hash + Root(*beacon.Spec) tree.Root + StateRoot() tree.Root + SetExecutionPayload(ExecutionPayload) error + Slot() beacon.Slot +} + +type ExecutionPayload interface { + FromExecutableData(*api.ExecutableData) error + GetParentHash() beacon.Hash32 + GetFeeRecipient() beacon.Eth1Address + GetStateRoot() beacon.Bytes32 + GetReceiptsRoot() beacon.Bytes32 + GetLogsBloom() beacon.LogsBloom + GetPrevRandao() beacon.Bytes32 + GetBlockNumber() view.Uint64View + GetGasLimit() view.Uint64View + GetGasUsed() view.Uint64View + GetTimestamp() beacon.Timestamp + GetExtraData() beacon.ExtraData + GetBaseFeePerGas() view.Uint256View + GetBlockHash() beacon.Hash32 + GetTransactions() beacon.PayloadTransactions +} + +type ExecutionPayloadWithdrawals interface { + ExecutionPayload + GetWithdrawals() beacon.Withdrawals +} + +type ExecutionPayloadResponse struct { + Version string `json:"version" yaml:"version"` + Data ExecutionPayload `json:"data" yaml:"data"` +} diff --git a/simulators/eth2/common/clients/beacon.go b/simulators/eth2/common/clients/beacon.go index ff06f63e26..d1f0cc1f80 100644 --- a/simulators/eth2/common/clients/beacon.go +++ b/simulators/eth2/common/clients/beacon.go @@ -13,6 +13,8 @@ import ( api "github.com/ethereum/go-ethereum/core/beacon" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/hive/hivesim" + + "github.com/ethereum/hive/simulators/eth2/common/builder" "github.com/ethereum/hive/simulators/eth2/common/utils" "github.com/holiman/uint256" "github.com/protolambda/eth2api" @@ -36,6 +38,10 @@ const ( PortValidatorAPI = 5000 ) +var ( + EMPTY_TREE_ROOT = tree.Root{} +) + type BeaconClient struct { T *hivesim.T HiveClient *hivesim.Client @@ -46,6 +52,7 @@ type BeaconClient struct { spec *common.Spec index int genesisValidatorsRoot tree.Root + Builder builder.Builder } func NewBeaconClient( @@ -79,6 +86,12 @@ func (bn *BeaconClient) Start(extraOptions ...hivesim.StartOption) error { } opts = append(opts, extraOptions...) + if bn.Builder != nil { + opts = append(opts, hivesim.Params{ + "HIVE_ETH2_BUILDER_ENDPOINT": bn.Builder.Address(), + }) + } + bn.HiveClient = bn.T.StartClient(bn.ClientType, opts...) bn.API = ð2api.Eth2HttpClient{ Addr: fmt.Sprintf("http://%s:%d", bn.HiveClient.IP, PortBeaconAPI), @@ -429,6 +442,7 @@ func (bn *BeaconClient) BlockFinalityCheckpoints( type VersionedBeaconStateResponse struct { *eth2api.VersionedBeaconState + spec *common.Spec } func (vbs *VersionedBeaconStateResponse) CurrentVersion() common.Version { @@ -509,6 +523,20 @@ func (vbs *VersionedBeaconStateResponse) Validators() phase0.ValidatorRegistry { panic("badly formatted beacon state") } +func (vbs *VersionedBeaconStateResponse) RandaoMixes() phase0.RandaoMixes { + switch state := vbs.Data.(type) { + case *phase0.BeaconState: + return state.RandaoMixes + case *altair.BeaconState: + return state.RandaoMixes + case *bellatrix.BeaconState: + return state.RandaoMixes + case *capella.BeaconState: + return state.RandaoMixes + } + panic("badly formatted beacon state") +} + func (vbs *VersionedBeaconStateResponse) StateSlot() common.Slot { switch state := vbs.Data.(type) { case *phase0.BeaconState: @@ -523,6 +551,125 @@ func (vbs *VersionedBeaconStateResponse) StateSlot() common.Slot { panic("badly formatted beacon state") } +func (vbs *VersionedBeaconStateResponse) LatestExecutionPayloadHeaderHash() tree.Root { + switch state := vbs.Data.(type) { + case *phase0.BeaconState: + return tree.Root{} + case *altair.BeaconState: + return tree.Root{} + case *bellatrix.BeaconState: + return state.LatestExecutionPayloadHeader.BlockHash + case *capella.BeaconState: + return state.LatestExecutionPayloadHeader.BlockHash + } + panic("badly formatted beacon state") +} + +func (vbs *VersionedBeaconStateResponse) NextWithdrawals( + slot common.Slot, +) (common.Withdrawals, error) { + var ( + withdrawalIndex common.WithdrawalIndex + validatorIndex common.ValidatorIndex + validators phase0.ValidatorRegistry + balances phase0.Balances + epoch = vbs.spec.SlotToEpoch(slot) + ) + switch state := vbs.Data.(type) { + case *bellatrix.BeaconState: + // withdrawalIndex and validatorIndex start at zero + validators = state.Validators + balances = state.Balances + case *capella.BeaconState: + withdrawalIndex = state.NextWithdrawalIndex + validatorIndex = state.NextWithdrawalValidatorIndex + validators = state.Validators + balances = state.Balances + default: + return nil, fmt.Errorf("badly formatted beacon state") + } + validatorCount := uint64(len(validators)) + withdrawals := make(common.Withdrawals, 0) + + i := uint64(0) + for { + if validatorIndex >= common.ValidatorIndex(validatorCount) || + validatorIndex >= common.ValidatorIndex(len(balances)) { + return nil, fmt.Errorf("invalid validator index") + } + validator := validators[validatorIndex] + if validator == nil { + return nil, fmt.Errorf("invalid validator") + } + balance := balances[validatorIndex] + if i >= validatorCount || + i >= uint64(vbs.spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) { + break + } + if IsFullyWithdrawableValidator(validator, balance, epoch) { + withdrawals = append(withdrawals, common.Withdrawal{ + Index: withdrawalIndex, + ValidatorIndex: validatorIndex, + Address: Eth1WithdrawalCredential(validator), + Amount: balance, + }) + withdrawalIndex += 1 + } else if IsPartiallyWithdrawableValidator(vbs.spec, validator, balance, epoch) { + withdrawals = append(withdrawals, common.Withdrawal{ + Index: withdrawalIndex, + ValidatorIndex: validatorIndex, + Address: Eth1WithdrawalCredential(validator), + Amount: balance - vbs.spec.MAX_EFFECTIVE_BALANCE, + }) + withdrawalIndex += 1 + } + if len(withdrawals) == int(vbs.spec.MAX_WITHDRAWALS_PER_PAYLOAD) { + break + } + validatorIndex = common.ValidatorIndex( + uint64(validatorIndex+1) % validatorCount, + ) + i += 1 + } + return withdrawals, nil +} + +func Eth1WithdrawalCredential(validator *phase0.Validator) common.Eth1Address { + var address common.Eth1Address + copy(address[:], validator.WithdrawalCredentials[12:]) + return address +} + +func IsFullyWithdrawableValidator( + validator *phase0.Validator, + balance common.Gwei, + epoch common.Epoch, +) bool { + return HasEth1WithdrawalCredential(validator) && + validator.WithdrawableEpoch <= epoch && + balance > 0 +} + +func IsPartiallyWithdrawableValidator( + spec *common.Spec, + validator *phase0.Validator, + balance common.Gwei, + epoch common.Epoch, +) bool { + effectiveBalance := validator.EffectiveBalance + hasMaxEffectiveBalance := effectiveBalance == spec.MAX_EFFECTIVE_BALANCE + hasExcessBalance := balance > spec.MAX_EFFECTIVE_BALANCE + return HasEth1WithdrawalCredential(validator) && hasMaxEffectiveBalance && + hasExcessBalance +} + +func HasEth1WithdrawalCredential(validator *phase0.Validator) bool { + return bytes.Equal( + validator.WithdrawalCredentials[:1], + []byte{common.ETH1_ADDRESS_WITHDRAWAL_PREFIX}, + ) +} + func (bn *BeaconClient) BeaconStateV2( parentCtx context.Context, stateId eth2api.StateId, @@ -545,6 +692,7 @@ func (bn *BeaconClient) BeaconStateV2( } return &VersionedBeaconStateResponse{ VersionedBeaconState: versionedBeaconStateResponse, + spec: bn.spec, }, err } @@ -779,8 +927,10 @@ func (bn *BeaconClient) GetLatestExecutionBeaconBlock( return nil, fmt.Errorf("failed to retrieve block: %v", err) } if executionPayload, err := versionedBlock.ExecutionPayload(); err == nil { - emptyRoot := tree.Root{} - if !bytes.Equal(executionPayload.BlockHash[:], emptyRoot[:]) { + if !bytes.Equal( + executionPayload.BlockHash[:], + EMPTY_TREE_ROOT[:], + ) { return versionedBlock, nil } } @@ -801,8 +951,10 @@ func (bn *BeaconClient) GetFirstExecutionBeaconBlock( continue } if executionPayload, err := versionedBlock.ExecutionPayload(); err == nil { - emptyRoot := tree.Root{} - if !bytes.Equal(executionPayload.BlockHash[:], emptyRoot[:]) { + if !bytes.Equal( + executionPayload.BlockHash[:], + EMPTY_TREE_ROOT[:], + ) { return versionedBlock, nil } } @@ -832,6 +984,38 @@ func (bn *BeaconClient) GetBeaconBlockByExecutionHash( return nil, nil } +func (bn *BeaconClient) GetFilledSlotsCountPerEpoch( + parentCtx context.Context, +) (map[common.Epoch]uint64, error) { + headInfo, err := bn.BlockHeader(parentCtx, eth2api.BlockHead) + epochMap := make(map[common.Epoch]uint64) + for { + if err != nil { + return nil, fmt.Errorf("failed to poll head: %v", err) + } + epoch := common.Epoch( + headInfo.Header.Message.Slot / bn.spec.SLOTS_PER_EPOCH, + ) + if prev, ok := epochMap[epoch]; ok { + epochMap[epoch] = prev + 1 + } else { + epochMap[epoch] = 1 + } + if bytes.Equal( + headInfo.Header.Message.ParentRoot[:], + EMPTY_TREE_ROOT[:], + ) { + break + } + headInfo, err = bn.BlockHeader( + parentCtx, + eth2api.BlockIdRoot(headInfo.Header.Message.ParentRoot), + ) + } + + return epochMap, nil +} + type BeaconClients []*BeaconClient // Return subset of clients that are currently running diff --git a/simulators/eth2/common/clients/execution.go b/simulators/eth2/common/clients/execution.go index de2dd879ba..a32a23ee12 100644 --- a/simulators/eth2/common/clients/execution.go +++ b/simulators/eth2/common/clients/execution.go @@ -29,6 +29,11 @@ const ( PortEngineRPC = 8551 ) +var AllForkchoiceUpdatedCalls = []string{ + "engine_forkchoiceUpdatedV1", + "engine_forkchoiceUpdatedV2", +} + var AllEngineCallsLog = []string{ "engine_forkchoiceUpdatedV1", "engine_forkchoiceUpdatedV2", @@ -43,6 +48,8 @@ type ExecutionClient struct { HiveClient *hivesim.Client ClientType string OptionsGenerator func() ([]hivesim.StartOption, error) + LatestForkchoice *api.ForkchoiceStateV1 + trackFcU bool proxy **proxy.Proxy proxyPort int subnet string @@ -63,12 +70,14 @@ func NewExecutionClient( proxyPort int, subnet string, ttd *big.Int, + trackFcu bool, logEngineCalls bool, ) *ExecutionClient { return &ExecutionClient{ T: t, ClientType: eth1Def.Name, OptionsGenerator: optionsGenerator, + trackFcU: trackFcu, proxyPort: proxyPort, proxy: new(*proxy.Proxy), subnet: subnet, @@ -79,6 +88,9 @@ func NewExecutionClient( } func (en *ExecutionClient) UserRPCAddress() (string, error) { + if en.HiveClient == nil { + return "", fmt.Errorf("el hive client not yet launched") + } return fmt.Sprintf("http://%v:%d", en.HiveClient.IP, PortUserRPC), nil } @@ -158,12 +170,31 @@ func (en *ExecutionClient) Start(extraOptions ...hivesim.StartOption) error { panic(err) } - proxy := proxy.NewProxy( + p := proxy.NewProxy( net.ParseIP(simIP), en.proxyPort, dest, secret, ) + + if en.trackFcU { + logCallback := func(req []byte) *spoof.Spoof { + var ( + fcState api.ForkchoiceStateV1 + pAttr api.PayloadAttributes + err error + ) + err = proxy.UnmarshalFromJsonRPCRequest(req, &fcState, &pAttr) + if err == nil { + en.LatestForkchoice = &fcState + } + return nil + } + for _, c := range AllForkchoiceUpdatedCalls { + p.AddRequestCallback(c, logCallback) + } + } + if en.logEngineCalls { logCallback := func(res []byte, req []byte) *spoof.Spoof { en.T.Logf( @@ -179,11 +210,11 @@ func (en *ExecutionClient) Start(extraOptions ...hivesim.StartOption) error { return nil } for _, c := range AllEngineCallsLog { - proxy.AddResponseCallback(c, logCallback) + p.AddResponseCallback(c, logCallback) } } - *en.proxy = proxy + *en.proxy = p return nil } @@ -271,16 +302,36 @@ func (en *ExecutionClient) EngineGetPayload( parentCtx context.Context, payloadID *api.PayloadID, version int, -) (*api.ExecutableData, error) { - var result api.ExecutableData +) (*api.ExecutableData, *big.Int, error) { + + var ( + rpcString = fmt.Sprintf("engine_getPayloadV%d", version) + ) + if err := en.PrepareDefaultAuthCallToken(); err != nil { - return nil, err + return nil, nil, err } - request := fmt.Sprintf("engine_getPayload%d", version) + ctx, cancel := context.WithTimeout(parentCtx, time.Second*10) defer cancel() - err := en.engineRpcClient.CallContext(ctx, &result, request, payloadID) - return &result, err + if version == 2 { + type ExecutionPayloadEnvelope struct { + ExecutionPayload *api.ExecutableData `json:"executionPayload" gencodec:"required"` + BlockValue *hexutil.Big `json:"blockValue" gencodec:"required"` + } + var response ExecutionPayloadEnvelope + err := en.engineRpcClient.CallContext( + ctx, + &response, + rpcString, + payloadID, + ) + return response.ExecutionPayload, (*big.Int)(response.BlockValue), err + } else { + var executableData api.ExecutableData + err := en.engineRpcClient.CallContext(ctx, &executableData, rpcString, payloadID) + return &executableData, common.Big0, err + } } func (en *ExecutionClient) EngineNewPayload( @@ -292,7 +343,7 @@ func (en *ExecutionClient) EngineNewPayload( if err := en.PrepareDefaultAuthCallToken(); err != nil { return nil, err } - request := fmt.Sprintf("engine_newPayload%d", version) + request := fmt.Sprintf("engine_newPayloadV%d", version) ctx, cancel := context.WithTimeout(parentCtx, time.Second*10) defer cancel() err := en.engineRpcClient.CallContext(ctx, &result, request, payload) @@ -446,6 +497,15 @@ func (ec *ExecutionClient) BalanceAt( return ec.eth.BalanceAt(ctx, account, n) } +func (ec *ExecutionClient) SendTransaction( + parentCtx context.Context, + tx *types.Transaction, +) error { + ctx, cancel := utils.ContextTimeoutRPC(parentCtx) + defer cancel() + return ec.eth.SendTransaction(ctx, tx) +} + type ExecutionClients []*ExecutionClient // Return subset of clients that are currently running diff --git a/simulators/eth2/common/clients/validator.go b/simulators/eth2/common/clients/validator.go index 2d29edc050..98988dfc01 100644 --- a/simulators/eth2/common/clients/validator.go +++ b/simulators/eth2/common/clients/validator.go @@ -17,6 +17,7 @@ type ValidatorClient struct { ClientType string OptionsGenerator func(map[common.ValidatorIndex]*consensus_config.KeyDetails) ([]hivesim.StartOption, error) Keys map[common.ValidatorIndex]*consensus_config.KeyDetails + beacon *BeaconClient } func NewValidatorClient( @@ -24,12 +25,14 @@ func NewValidatorClient( validatorDef *hivesim.ClientDefinition, optionsGenerator func(map[common.ValidatorIndex]*consensus_config.KeyDetails) ([]hivesim.StartOption, error), keys map[common.ValidatorIndex]*consensus_config.KeyDetails, + bn *BeaconClient, ) *ValidatorClient { return &ValidatorClient{ T: t, ClientType: validatorDef.Name, OptionsGenerator: optionsGenerator, Keys: keys, + beacon: bn, } } @@ -48,6 +51,12 @@ func (vc *ValidatorClient) Start(extraOptions ...hivesim.StartOption) error { } opts = append(opts, extraOptions...) + if vc.beacon.Builder != nil { + opts = append(opts, hivesim.Params{ + "HIVE_ETH2_BUILDER_ENDPOINT": vc.beacon.Builder.Address(), + }) + } + vc.HiveClient = vc.T.StartClient(vc.ClientType, opts...) return nil } diff --git a/simulators/eth2/common/config/execution/execution_config.go b/simulators/eth2/common/config/execution/execution_config.go index 2d99f26281..ee0fda9627 100644 --- a/simulators/eth2/common/config/execution/execution_config.go +++ b/simulators/eth2/common/config/execution/execution_config.go @@ -187,6 +187,7 @@ func BuildExecutionGenesis( genesisTime uint64, consensus ExecutionConsensus, forkConfig *params.ChainConfig, + genesisExecAccounts map[common.Address]core.GenesisAccount, ) *ExecutionGenesis { depositContractAddr := common.HexToAddress( "0x4242424242424242424242424242424242424242", @@ -233,6 +234,14 @@ func BuildExecutionGenesis( NetworkID: 7, } + for addr, acc := range genesisExecAccounts { + acc := acc + if acc.Balance == nil { + acc.Balance = common.Big0 + } + genesis.Genesis.Alloc[addr] = acc + } + // Configure post-merge forks if forkConfig.ShanghaiTime != nil { genesis.Genesis.Config.ShanghaiTime = forkConfig.ShanghaiTime diff --git a/simulators/eth2/common/go.mod b/simulators/eth2/common/go.mod index 37573c8d0e..4403d9fb49 100644 --- a/simulators/eth2/common/go.mod +++ b/simulators/eth2/common/go.mod @@ -7,6 +7,7 @@ require ( github.com/ethereum/hive v0.0.0-20221214152536-bfabd993ae7b github.com/golang-jwt/jwt/v4 v4.3.0 github.com/google/uuid v1.3.0 + github.com/gorilla/mux v1.8.0 github.com/herumi/bls-eth-go-binary v1.28.1 github.com/holiman/uint256 v1.2.1 github.com/pkg/errors v0.9.1 @@ -16,6 +17,7 @@ require ( github.com/protolambda/zrnt v0.30.0 github.com/protolambda/ztyp v0.2.2 github.com/rauljordan/engine-proxy v0.0.0-20220517190449-e62b2e2f6e27 + github.com/sirupsen/logrus v1.9.0 github.com/tyler-smith/go-bip39 v1.1.0 github.com/wealdtech/go-eth2-util v1.8.0 golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a @@ -49,7 +51,6 @@ require ( github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/rs/cors v1.8.2 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect - github.com/sirupsen/logrus v1.9.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect diff --git a/simulators/eth2/common/go.sum b/simulators/eth2/common/go.sum index 9b4d9a309f..0f0cfdd393 100644 --- a/simulators/eth2/common/go.sum +++ b/simulators/eth2/common/go.sum @@ -76,6 +76,7 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= diff --git a/simulators/eth2/common/testnet/config.go b/simulators/eth2/common/testnet/config.go index 59fd990ddb..e357a90bd3 100644 --- a/simulators/eth2/common/testnet/config.go +++ b/simulators/eth2/common/testnet/config.go @@ -3,6 +3,9 @@ package testnet import ( "math/big" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + mock_builder "github.com/ethereum/hive/simulators/eth2/common/builder/mock" "github.com/ethereum/hive/simulators/eth2/common/clients" execution_config "github.com/ethereum/hive/simulators/eth2/common/config/execution" ) @@ -29,7 +32,12 @@ type Config struct { Eth1Consensus execution_config.ExecutionConsensus // Execution Layer specific config - InitialBaseFeePerGas *big.Int + InitialBaseFeePerGas *big.Int + GenesisExecutionAccounts map[common.Address]core.GenesisAccount + + // Builders + EnableBuilders bool + BuilderOptions []mock_builder.Option } // Choose a configuration value. `b` takes precedence @@ -83,6 +91,14 @@ func (a *Config) Join(b *Config) *Config { c.Eth1Consensus = a.Eth1Consensus } + if b.GenesisExecutionAccounts != nil { + c.GenesisExecutionAccounts = b.GenesisExecutionAccounts + } else { + c.GenesisExecutionAccounts = a.GenesisExecutionAccounts + } + + c.EnableBuilders = b.EnableBuilders || a.EnableBuilders + return &c } diff --git a/simulators/eth2/common/testnet/prepared_testnet.go b/simulators/eth2/common/testnet/prepared_testnet.go index b6fe3b4467..365dd48335 100644 --- a/simulators/eth2/common/testnet/prepared_testnet.go +++ b/simulators/eth2/common/testnet/prepared_testnet.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "math/big" + "net" "os" "strings" "time" @@ -20,6 +21,7 @@ import ( "github.com/protolambda/zrnt/eth2/configs" "github.com/ethereum/hive/hivesim" + mock_builder "github.com/ethereum/hive/simulators/eth2/common/builder/mock" "github.com/ethereum/hive/simulators/eth2/common/clients" cl "github.com/ethereum/hive/simulators/eth2/common/config/consensus" el "github.com/ethereum/hive/simulators/eth2/common/config/execution" @@ -69,7 +71,7 @@ func prepareExecutionForkConfig( if config.CapellaForkEpoch.Uint64() == 0 { chainConfig.ShanghaiTime = big.NewInt(int64(eth2GenesisTime)) } else { - chainConfig.ShanghaiTime = big.NewInt(int64(eth2GenesisTime) + config.SlotTime.Int64()*32) + chainConfig.ShanghaiTime = big.NewInt(int64(eth2GenesisTime) + config.CapellaForkEpoch.Int64()*config.SlotTime.Int64()*32) } } return &chainConfig @@ -90,6 +92,7 @@ func prepareTestnet( uint64(eth1GenesisTime), config.Eth1Consensus, prepareExecutionForkConfig(eth2GenesisTime, config), + config.GenesisExecutionAccounts, ) if config.InitialBaseFeePerGas != nil { eth1Genesis.Genesis.BaseFee = config.InitialBaseFeePerGas @@ -399,6 +402,7 @@ func (p *PreparedTestnet) prepareExecutionNode( clients.PortEngineRPC+executionIndex, subnet, ttd, + true, logEngineCalls, ) } @@ -411,6 +415,8 @@ func (p *PreparedTestnet) prepareBeaconNode( beaconDef *hivesim.ClientDefinition, ttd *big.Int, beaconIndex int, + enableBuilders bool, + builderOptions []mock_builder.Option, eth1Endpoints ...*clients.ExecutionClient, ) *clients.BeaconClient { testnet.Logf( @@ -491,7 +497,7 @@ func (p *PreparedTestnet) prepareBeaconNode( //if p.configName != "mainnet" && hasBuildTarget(beaconDef, p.configName) { // opts = append(opts, hivesim.WithBuildTarget(p.configName)) //} - return clients.NewBeaconClient( + cl := clients.NewBeaconClient( testnet.T, beaconDef, optionsGenerator, @@ -500,6 +506,43 @@ func (p *PreparedTestnet) prepareBeaconNode( beaconIndex, testnet.genesisValidatorsRoot, ) + + if enableBuilders { + simIP, err := testnet.T.Sim.ContainerNetworkIP( + testnet.T.SuiteID, + "bridge", + "simulation", + ) + if err != nil { + panic(err) + } + + options := []mock_builder.Option{ + mock_builder.WithExternalIP(net.ParseIP(simIP)), + mock_builder.WithPort( + mock_builder.DEFAULT_BUILDER_PORT + beaconIndex, + ), + mock_builder.WithID(beaconIndex), + mock_builder.WithBeaconGenesisTime(testnet.genesisTime), + } + + if builderOptions != nil { + options = append(options, builderOptions...) + } + + mockBuilder, err := mock_builder.NewMockBuilder( + eth1Endpoints[0], + cl, + p.spec, + options..., + ) + if err != nil { + panic(err) + } + cl.Builder = mockBuilder + } + + return cl } // Prepares a validator client object with all the necessary information @@ -548,5 +591,6 @@ func (p *PreparedTestnet) prepareValidatorClient( validatorDef, optionsGenerator, p.keyTranches[keyIndex], + bn, ) } diff --git a/simulators/eth2/common/testnet/running_testnet.go b/simulators/eth2/common/testnet/running_testnet.go index c39f71e888..8ade39b3cf 100644 --- a/simulators/eth2/common/testnet/running_testnet.go +++ b/simulators/eth2/common/testnet/running_testnet.go @@ -10,12 +10,14 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/pkg/errors" "github.com/protolambda/eth2api" "github.com/protolambda/zrnt/eth2/beacon/altair" "github.com/protolambda/zrnt/eth2/beacon/common" "github.com/protolambda/zrnt/eth2/beacon/phase0" "github.com/protolambda/zrnt/eth2/util/math" + "github.com/protolambda/ztyp/tree" "github.com/ethereum/hive/hivesim" "github.com/ethereum/hive/simulators/eth2/common/clients" @@ -23,7 +25,14 @@ import ( "github.com/ethereum/hive/simulators/eth2/common/utils" ) -var MAX_PARTICIPATION_SCORE = 7 +const ( + MAX_PARTICIPATION_SCORE = 7 +) + +var ( + EMPTY_EXEC_HASH = ethcommon.Hash{} + EMPTY_TREE_ROOT = tree.Root{} +) type Testnet struct { *hivesim.T @@ -38,6 +47,9 @@ type Testnet struct { eth1Genesis *execution_config.ExecutionGenesis // Consensus genesis state eth2GenesisState common.BeaconState + + // Test configuration + maxConsecutiveErrorsOnWaits int } type ActiveSpec struct { @@ -187,6 +199,8 @@ func StartTestnet( beaconDef, node.BeaconNodeTTD, nodeIndex, + config.EnableBuilders, + config.BuilderOptions, nodeClient.ExecutionClient, ) @@ -210,6 +224,9 @@ func StartTestnet( } } + // Default config + testnet.maxConsecutiveErrorsOnWaits = 3 + return testnet } @@ -217,6 +234,11 @@ func (t *Testnet) Stop() { for _, p := range t.Proxies().Running() { p.Cancel() } + for _, b := range t.BeaconClients() { + if b.Builder != nil { + b.Builder.Cancel() + } + } } func (t *Testnet) ValidatorClientIndex(pk [48]byte) (int, error) { @@ -253,17 +275,18 @@ func (t *Testnet) WaitSlots(ctx context.Context, slots common.Slot) error { // WaitForFork blocks until a beacon client reaches specified fork, // or context finalizes, whichever happens first. func (t *Testnet) WaitForFork(ctx context.Context, fork string) error { - genesis := t.GenesisTimeUnix() - slotDuration := time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second - timer := time.NewTicker(slotDuration) - runningNodes := t.VerificationNodes().Running() - done := make(chan error, len(runningNodes)) + var ( + genesis = t.GenesisTimeUnix() + slotDuration = time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second + timer = time.NewTicker(slotDuration) + runningNodes = t.VerificationNodes().Running() + results = makeResults(runningNodes, t.maxConsecutiveErrorsOnWaits) + ) + for { select { case <-ctx.Done(): return ctx.Err() - case err := <-done: - return err case tim := <-timer.C: // start polling after first slot of genesis if tim.Before(genesis.Add(slotDuration)) { @@ -272,44 +295,29 @@ func (t *Testnet) WaitForFork(ctx context.Context, fork string) error { } // new slot, log and check status of all beacon nodes - type res struct { - idx int - msg string - err error - } var ( - wg sync.WaitGroup - ch = make(chan res, len(runningNodes)) + wg sync.WaitGroup + clockSlot = t.spec.TimeToSlot( + common.Timestamp(time.Now().Unix()), + t.GenesisTime(), + ) ) + results.Clear() + for i, n := range runningNodes { wg.Add(1) go func( ctx context.Context, - i int, n *clients.Node, - ch chan res, + r *result, ) { defer wg.Done() - var ( - b = n.BeaconClient - slot common.Slot - head string - justified string - finalized string - execution = "0x0000..0000" - ) + b := n.BeaconClient headInfo, err := b.BlockHeader(ctx, eth2api.BlockHead) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s): failed to poll head: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap(err, "failed to poll head") return } @@ -318,7 +326,10 @@ func (t *Testnet) WaitForFork(ctx context.Context, fork string) error { eth2api.BlockHead, ) if err != nil { - ch <- res{err: fmt.Errorf("node %d (%s) failed to poll finality checkpoint: %v", i, n.ClientNames(), err)} + r.err = errors.Wrap( + err, + "failed to poll finality checkpoint", + ) return } @@ -327,63 +338,50 @@ func (t *Testnet) WaitForFork(ctx context.Context, fork string) error { eth2api.BlockIdRoot(headInfo.Root), ) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to retrieve block: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap(err, "failed to retrieve block") return } + + execution := ethcommon.Hash{} if executionPayload, err := versionedBlock.ExecutionPayload(); err == nil { - execution = utils.Shorten( - executionPayload.BlockHash.String(), - ) + execution = executionPayload.BlockHash } - slot = headInfo.Header.Message.Slot - head = utils.Shorten(headInfo.Root.String()) - justified = utils.Shorten( - checkpoints.CurrentJustified.String(), - ) - finalized = utils.Shorten(checkpoints.Finalized.String()) - - ch <- res{ - i, - fmt.Sprintf( - "node %d (%s) fork=%s, slot=%d, head=%s, exec_payload=%s, justified=%s, finalized=%s", - i, - n.ClientNames(), - versionedBlock.Version, + slot := headInfo.Header.Message.Slot + if clockSlot > slot && + (clockSlot-slot) >= t.spec.SLOTS_PER_EPOCH { + r.fatal = fmt.Errorf( + "unable to sync for an entire epoch: clockSlot=%d, slot=%d", + clockSlot, slot, - head, - execution, - justified, - finalized, - ), - nil, + ) + return } + r.msg = fmt.Sprintf( + "fork=%s, clock_slot=%s, slot=%d, head=%s, exec_payload=%s, justified=%s, finalized=%s", + versionedBlock.Version, + clockSlot, + slot, + utils.Shorten(headInfo.Root.String()), + utils.Shorten(execution.String()), + utils.Shorten(checkpoints.CurrentJustified.String()), + utils.Shorten(checkpoints.Finalized.String()), + ) + if versionedBlock.Version == fork { - done <- nil + r.done = true } - }(ctx, i, n, ch) + }(ctx, n, results[i]) } wg.Wait() - close(ch) - // print out logs in ascending idx order - sorted := make([]string, len(runningNodes)) - for out := range ch { - if out.err != nil { - return out.err - } - sorted[out.idx] = out.msg + if err := results.CheckError(); err != nil { + return err } - for _, msg := range sorted { - t.Logf(msg) + results.PrintMessages(t.Logf) + if results.AllDone() { + return nil } } } @@ -394,17 +392,18 @@ func (t *Testnet) WaitForFork(ctx context.Context, fork string) error { func (t *Testnet) WaitForFinality(ctx context.Context) ( common.Checkpoint, error, ) { - genesis := t.GenesisTimeUnix() - slotDuration := time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second - timer := time.NewTicker(slotDuration) - runningNodes := t.VerificationNodes().Running() - done := make(chan common.Checkpoint, len(runningNodes)) + var ( + genesis = t.GenesisTimeUnix() + slotDuration = time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second + timer = time.NewTicker(slotDuration) + runningNodes = t.VerificationNodes().Running() + results = makeResults(runningNodes, t.maxConsecutiveErrorsOnWaits) + ) + for { select { case <-ctx.Done(): return common.Checkpoint{}, ctx.Err() - case finalized := <-done: - return finalized, nil case tim := <-timer.C: // start polling after first slot of genesis if tim.Before(genesis.Add(slotDuration)) { @@ -413,45 +412,25 @@ func (t *Testnet) WaitForFinality(ctx context.Context) ( } // new slot, log and check status of all beacon nodes - type res struct { - idx int - msg string - err error - } var ( - wg sync.WaitGroup - ch = make(chan res, len(runningNodes)) + wg sync.WaitGroup + clockSlot = t.spec.TimeToSlot( + common.Timestamp(time.Now().Unix()), + t.GenesisTime(), + ) ) + results.Clear() + for i, n := range runningNodes { wg.Add(1) - go func( - ctx context.Context, - i int, - n *clients.Node, - ch chan res, - ) { + go func(ctx context.Context, n *clients.Node, r *result) { defer wg.Done() - var ( - b = n.BeaconClient - slot common.Slot - head string - justified string - finalized string - health float64 - execution = "0x0000..0000" - ) + b := n.BeaconClient headInfo, err := b.BlockHeader(ctx, eth2api.BlockHead) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to poll head: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap(err, "failed to poll head") return } @@ -460,14 +439,10 @@ func (t *Testnet) WaitForFinality(ctx context.Context) ( eth2api.BlockHead, ) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to poll finality checkpoint: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap( + err, + "failed to poll finality checkpoint", + ) return } @@ -476,90 +451,57 @@ func (t *Testnet) WaitForFinality(ctx context.Context) ( eth2api.BlockIdRoot(headInfo.Root), ) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to retrieve block: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap(err, "failed to retrieve block") return } + execution := ethcommon.Hash{} if executionPayload, err := versionedBlock.ExecutionPayload(); err == nil { - execution = utils.Shorten( - executionPayload.BlockHash.String(), - ) + execution = executionPayload.BlockHash } - slot = headInfo.Header.Message.Slot - head = utils.Shorten(headInfo.Root.String()) - justified = utils.Shorten( - checkpoints.CurrentJustified.String(), - ) - finalized = utils.Shorten(checkpoints.Finalized.String()) - health, err = GetHealth(ctx, b, t.spec, slot) - if err != nil { - // warning is printed here instead because some clients - // don't support the required REST endpoint. - fmt.Printf( - "WARN: node %d (%s) %s\n", - i, - n.ClientNames(), - err, + slot := headInfo.Header.Message.Slot + if clockSlot > slot && + (clockSlot-slot) >= t.spec.SLOTS_PER_EPOCH { + r.fatal = fmt.Errorf( + "unable to sync for an entire epoch: clockSlot=%d, slot=%d", + clockSlot, + slot, ) + return } - ep := t.spec.SlotToEpoch(slot) - if ep > 4 && ep > checkpoints.Finalized.Epoch+2 { - ch <- res{ - err: fmt.Errorf( - "failed to finalize, head slot %d (epoch %d) "+ - "is more than 2 ahead of finality checkpoint %d", - slot, - ep, - checkpoints.Finalized.Epoch, - ), - } - } else { - ch <- res{ - i, - fmt.Sprintf( - "node %d (%s) fork=%s, slot=%d, head=%s, "+ - "health=%.2f, exec_payload=%s, justified=%s, "+ - "finalized=%s", - i, - n.ClientNames(), - versionedBlock.Version, - slot, - head, - health, - execution, - justified, - finalized, - ), - nil, - } - } + health, _ := GetHealth(ctx, b, t.spec, slot) + + r.msg = fmt.Sprintf( + "fork=%s, clock_slot=%d, slot=%d, head=%s, "+ + "health=%.2f, exec_payload=%s, justified=%s, "+ + "finalized=%s", + versionedBlock.Version, + clockSlot, + slot, + utils.Shorten(headInfo.Root.String()), + health, + utils.Shorten(execution.String()), + utils.Shorten(checkpoints.CurrentJustified.String()), + utils.Shorten(checkpoints.Finalized.String()), + ) if (checkpoints.Finalized != common.Checkpoint{}) { - done <- checkpoints.Finalized + r.done = true + r.result = checkpoints.Finalized } - }(ctx, i, n, ch) + }(ctx, n, results[i]) } wg.Wait() - close(ch) - // print out logs in ascending idx order - sorted := make([]string, len(runningNodes)) - for out := range ch { - if out.err != nil { - return common.Checkpoint{}, out.err - } - sorted[out.idx] = out.msg + if err := results.CheckError(); err != nil { + return common.Checkpoint{}, err } - for _, msg := range sorted { - t.Logf(msg) + results.PrintMessages(t.Logf) + if results.AllDone() { + if cp, ok := results[0].result.(common.Checkpoint); ok { + return cp, nil + } } } } @@ -571,17 +513,18 @@ func (t *Testnet) WaitForFinality(ctx context.Context) ( func (t *Testnet) WaitForExecutionFinality( ctx context.Context, ) (common.Checkpoint, error) { - genesis := t.GenesisTimeUnix() - slotDuration := time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second - timer := time.NewTicker(slotDuration) - runningNodes := t.VerificationNodes().Running() - done := make(chan common.Checkpoint, len(runningNodes)) + var ( + genesis = t.GenesisTimeUnix() + slotDuration = time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second + timer = time.NewTicker(slotDuration) + runningNodes = t.VerificationNodes().Running() + results = makeResults(runningNodes, t.maxConsecutiveErrorsOnWaits) + ) + for { select { case <-ctx.Done(): return common.Checkpoint{}, ctx.Err() - case finalized := <-done: - return finalized, nil case tim := <-timer.C: // start polling after first slot of genesis if tim.Before(genesis.Add(slotDuration)) { @@ -590,129 +533,103 @@ func (t *Testnet) WaitForExecutionFinality( } // new slot, log and check status of all beacon nodes - type res struct { - idx int - msg string - err error - } var ( - wg sync.WaitGroup - ch = make(chan res, len(runningNodes)) + wg sync.WaitGroup + clockSlot = t.spec.TimeToSlot( + common.Timestamp(time.Now().Unix()), + t.GenesisTime(), + ) ) + results.Clear() + for i, n := range runningNodes { wg.Add(1) - go func(ctx context.Context, i int, n *clients.Node, ch chan res) { + go func(ctx context.Context, n *clients.Node, r *result) { defer wg.Done() var ( - b = n.BeaconClient - slot common.Slot - version string - head string - justified string - finalized string + b = n.BeaconClient + version string ) headInfo, err := b.BlockHeader(ctx, eth2api.BlockHead) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to poll head: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap(err, "failed to poll head") + return + } + slot := headInfo.Header.Message.Slot + if clockSlot > slot && + (clockSlot-slot) >= t.spec.SLOTS_PER_EPOCH { + r.fatal = fmt.Errorf( + "unable to sync for an entire epoch: clockSlot=%d, slot=%d", + clockSlot, + slot, + ) return } - slot = headInfo.Header.Message.Slot - head = utils.Shorten(headInfo.Root.String()) checkpoints, err := b.BlockFinalityCheckpoints( ctx, eth2api.BlockHead, ) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to poll finality checkpoint: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap( + err, + "failed to poll finality checkpoint", + ) return } - justified = utils.Shorten( - checkpoints.CurrentJustified.String(), - ) - finalized = utils.Shorten(checkpoints.Finalized.String()) - - var ( - execution ethcommon.Hash - executionStr = "0x0000..0000" - ) + execution := ethcommon.Hash{} if (checkpoints.Finalized != common.Checkpoint{}) { if versionedBlock, err := b.BlockV2( ctx, eth2api.BlockIdRoot(checkpoints.Finalized.Root), ); err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to retrieve block: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap( + err, + "failed to retrieve block", + ) return } else { version = versionedBlock.Version if exeuctionPayload, err := versionedBlock.ExecutionPayload(); err == nil { execution = exeuctionPayload.BlockHash - executionStr = utils.Shorten(execution.Hex()) } } } - ch <- res{ - i, - fmt.Sprintf("node %d (%s) fork=%s, slot=%d, head=%s, "+"finalized_exec_payload=%s, justified=%s, finalized=%s", - i, - n.ClientNames(), - version, - slot, - head, - executionStr, - justified, - finalized, - ), - nil, - } - emptyHash := ethcommon.Hash{} - if !bytes.Equal(execution[:], emptyHash[:]) { - done <- checkpoints.Finalized + r.msg = fmt.Sprintf( + "fork=%s, clock_slot=%s, slot=%d, head=%s, "+ + "exec_payload=%s, justified=%s, finalized=%s", + version, + clockSlot, + slot, + utils.Shorten(headInfo.Root.String()), + utils.Shorten(execution.Hex()), + utils.Shorten(checkpoints.CurrentJustified.String()), + utils.Shorten(checkpoints.Finalized.String()), + ) + + if !bytes.Equal(execution[:], EMPTY_EXEC_HASH[:]) { + r.done = true + r.result = checkpoints.Finalized } }( ctx, - i, n, - ch, + results[i], ) } wg.Wait() - close(ch) - // print out logs in ascending idx order - sorted := make([]string, len(runningNodes)) - for out := range ch { - if out.err != nil { - return common.Checkpoint{}, out.err - } - sorted[out.idx] = out.msg + if err := results.CheckError(); err != nil { + return common.Checkpoint{}, err } - for _, msg := range sorted { - t.Logf(msg) + results.PrintMessages(t.Logf) + if results.AllDone() { + if cp, ok := results[0].result.(common.Checkpoint); ok { + return cp, nil + } } } } @@ -722,28 +639,27 @@ func (t *Testnet) WaitForExecutionFinality( func (t *Testnet) WaitForCurrentEpochFinalization( ctx context.Context, ) (common.Checkpoint, error) { - genesis := t.GenesisTimeUnix() - slotDuration := time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second - timer := time.NewTicker(slotDuration) - runningNodes := t.VerificationNodes().Running() - done := make(chan common.Checkpoint, len(runningNodes)) - - // Get the current head root which must be finalized - headInfo, err := runningNodes[0].BeaconClient.BlockHeader( - ctx, - eth2api.BlockHead, + var ( + genesis = t.GenesisTimeUnix() + slotDuration = time.Duration( + t.spec.SECONDS_PER_SLOT, + ) * time.Second + timer = time.NewTicker(slotDuration) + runningNodes = t.VerificationNodes().Running() + results = makeResults( + runningNodes, + t.maxConsecutiveErrorsOnWaits, + ) + epochToBeFinalized = t.spec.SlotToEpoch(t.spec.TimeToSlot( + common.Timestamp(time.Now().Unix()), + t.GenesisTime(), + )) ) - if err != nil { - return common.Checkpoint{}, fmt.Errorf("failed to poll head: %v", err) - } - epochToBeFinalized := t.spec.SlotToEpoch(headInfo.Header.Message.Slot) for { select { case <-ctx.Done(): return common.Checkpoint{}, ctx.Err() - case finalized := <-done: - return finalized, nil case tim := <-timer.C: // start polling after first slot of genesis if tim.Before(genesis.Add(slotDuration)) { @@ -752,103 +668,82 @@ func (t *Testnet) WaitForCurrentEpochFinalization( } // new slot, log and check status of all beacon nodes - type res struct { - idx int - msg string - err error - } var ( - wg sync.WaitGroup - ch = make(chan res, len(runningNodes)) + wg sync.WaitGroup + clockSlot = t.spec.TimeToSlot( + common.Timestamp(time.Now().Unix()), + t.GenesisTime(), + ) ) + results.Clear() + for i, n := range runningNodes { + i := i wg.Add(1) - go func(ctx context.Context, i int, n *clients.Node, ch chan res) { + go func(ctx context.Context, n *clients.Node, r *result) { defer wg.Done() - var ( - b = n.BeaconClient - slot common.Slot - head string - justified string - finalized string - ) + b := n.BeaconClient headInfo, err := b.BlockHeader(ctx, eth2api.BlockHead) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to poll head: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap(err, "failed to poll head") + return + } + + slot := headInfo.Header.Message.Slot + if clockSlot > slot && + (clockSlot-slot) >= t.spec.SLOTS_PER_EPOCH { + r.fatal = fmt.Errorf( + "unable to sync for an entire epoch: clockSlot=%d, slot=%d", + clockSlot, + slot, + ) return } - slot = headInfo.Header.Message.Slot - head = utils.Shorten(headInfo.Root.String()) checkpoints, err := b.BlockFinalityCheckpoints( ctx, eth2api.BlockHead, ) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s) failed to poll finality checkpoint: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap( + err, + "failed to poll finality checkpoint", + ) return } - justified = utils.Shorten( - checkpoints.CurrentJustified.String(), + + r.msg = fmt.Sprintf( + "clock_slot=%d, slot=%d, head=%s justified=%s, "+ + "finalized=%s, epoch_to_finalize=%d", + clockSlot, + slot, + utils.Shorten(headInfo.Root.String()), + utils.Shorten(checkpoints.CurrentJustified.String()), + utils.Shorten(checkpoints.Finalized.String()), + epochToBeFinalized, ) - finalized = utils.Shorten(checkpoints.Finalized.String()) - - ch <- res{ - i, - fmt.Sprintf( - "node %d (%s) slot=%d, head=%s justified=%s, "+ - "finalized=%s, epoch_to_finalize=%d", - i, - n.ClientNames(), - slot, - head, - justified, - finalized, - epochToBeFinalized, - ), - nil, - } if checkpoints.Finalized != (common.Checkpoint{}) && checkpoints.Finalized.Epoch >= epochToBeFinalized { - done <- checkpoints.Finalized + r.done = true + r.result = checkpoints.Finalized } - }( - ctx, - i, - n, - ch, - ) + }(ctx, n, results[i]) + } wg.Wait() - close(ch) - // print out logs in ascending idx order - sorted := make([]string, len(runningNodes)) - for out := range ch { - if out.err != nil { - return common.Checkpoint{}, out.err - } - sorted[out.idx] = out.msg + if err := results.CheckError(); err != nil { + return common.Checkpoint{}, err } - for _, msg := range sorted { - t.Logf(msg) + results.PrintMessages(t.Logf) + if results.AllDone() { + t.Logf("INFO: Epoch %d finalized", epochToBeFinalized) + if cp, ok := results[0].result.(common.Checkpoint); ok { + return cp, nil + } } } } @@ -859,20 +754,23 @@ func (t *Testnet) WaitForCurrentEpochFinalization( func (t *Testnet) WaitForExecutionPayload( ctx context.Context, ) (ethcommon.Hash, error) { - genesis := t.GenesisTimeUnix() - slotDuration := time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second - timer := time.NewTicker(slotDuration) - runningNodes := t.VerificationNodes().Running() - done := make(chan ethcommon.Hash, len(runningNodes)) - executionClient := runningNodes[0].ExecutionClient - ttdReached := false + var ( + genesis = t.GenesisTimeUnix() + slotDuration = time.Duration(t.spec.SECONDS_PER_SLOT) * time.Second + timer = time.NewTicker(slotDuration) + runningNodes = t.VerificationNodes().Running() + results = makeResults( + runningNodes, + t.maxConsecutiveErrorsOnWaits, + ) + executionClient = runningNodes[0].ExecutionClient + ttdReached = false + ) for { select { case <-ctx.Done(): return ethcommon.Hash{}, ctx.Err() - case result := <-done: - return result, nil case tim := <-timer.C: // start polling after first slot of genesis if tim.Before(genesis.Add(slotDuration)) { @@ -882,137 +780,96 @@ func (t *Testnet) WaitForExecutionPayload( if !ttdReached { // Check if TTD has been reached - if td, err := executionClient.TotalDifficultyByNumber(ctx, nil); err == nil && - td.Cmp(t.eth1Genesis.Genesis.Config.TerminalTotalDifficulty) >= 0 { - ttdReached = true + if td, err := executionClient.TotalDifficultyByNumber(ctx, nil); err == nil { + if td.Cmp( + t.eth1Genesis.Genesis.Config.TerminalTotalDifficulty, + ) >= 0 { + ttdReached = true + } else { + continue + } } else { t.Logf("Error querying eth1 for TTD: %v", err) } } // new slot, log and check status of all beacon nodes - type res struct { - idx int - msg string - err error - } - var ( - wg sync.WaitGroup - ch = make(chan res, len(runningNodes)) + wg sync.WaitGroup + clockSlot = t.spec.TimeToSlot( + common.Timestamp(time.Now().Unix()), + t.GenesisTime(), + ) ) + results.Clear() + for i, n := range runningNodes { wg.Add(1) - go func(ctx context.Context, i int, n *clients.Node, ch chan res) { + go func(ctx context.Context, n *clients.Node, r *result) { defer wg.Done() - var ( - b = n.BeaconClient - slot common.Slot - version string - head string - health float64 - ) + b := n.BeaconClient headInfo, err := b.BlockHeader(ctx, eth2api.BlockHead) if err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s): failed to poll head: %v", - i, - n.ClientNames(), - err, - ), - } + r.err = errors.Wrap(err, "failed to poll head") + return + } + + slot := headInfo.Header.Message.Slot + if clockSlot > slot && + (clockSlot-slot) >= t.spec.SLOTS_PER_EPOCH { + r.fatal = fmt.Errorf( + "unable to sync for an entire epoch: clockSlot=%d, slot=%d", + clockSlot, + slot, + ) return } - slot = headInfo.Header.Message.Slot - head = utils.Shorten(headInfo.Root.String()) - if versionedBlock, err := b.BlockV2( + versionedBlock, err := b.BlockV2( ctx, eth2api.BlockIdRoot(headInfo.Root), - ); err != nil { - ch <- res{ - err: fmt.Errorf( - "node %d (%s): failed to retrieve block: %v", - i, - n.ClientNames(), - err, - ), - } + ) + if err != nil { + r.err = errors.Wrap(err, "failed to retrieve block") return - } else { - version = versionedBlock.Version - if executionPayload, err := versionedBlock.ExecutionPayload(); err == nil { - emptyHash := ethcommon.Hash{} - if !bytes.Equal(executionPayload.BlockHash[:], emptyHash[:]) { - ch <- res{ - i, - fmt.Sprintf( - "node %d (%s): fork=%s, slot=%d, "+ - "head=%s, health=%.2f, exec_payload=%s", - i, - n.ClientNames(), - version, - slot, - head, - health, - utils.Shorten(executionPayload.BlockHash.Hex()), - ), - nil, - } - done <- executionPayload.BlockHash - } - } } - health, err = GetHealth(ctx, b, t.spec, slot) - if err != nil { - // warning is printed here instead because some clients - // don't support the required REST endpoint. - fmt.Printf( - "WARN: node %d (%s): %s\n", - i, - n.ClientNames(), - err, - ) + executionHash := ethcommon.Hash{} + if executionPayload, err := versionedBlock.ExecutionPayload(); err == nil { + executionHash = executionPayload.BlockHash } - ch <- res{ - i, - fmt.Sprintf( - "node %d (%s): fork=%s, slot=%d, head=%s, "+ - "health=%.2f, exec_payload=0x000..000", - i, - n.ClientNames(), - version, - slot, - head, - health, - ), - nil, + health, _ := GetHealth(ctx, b, t.spec, slot) + + r.msg = fmt.Sprintf( + "fork=%s, clock_slot=%d, slot=%d, "+ + "head=%s, health=%.2f, exec_payload=%s", + versionedBlock.Version, + clockSlot, + slot, + utils.Shorten(headInfo.Root.String()), + health, + utils.Shorten(executionHash.Hex()), + ) + + if !bytes.Equal(executionHash[:], EMPTY_EXEC_HASH[:]) { + r.done = true + r.result = executionHash } - }( - ctx, - i, - n, - ch, - ) + }(ctx, n, results[i]) } wg.Wait() - close(ch) - // print out logs in ascending idx order - sorted := make([]string, len(runningNodes)) - for out := range ch { - if out.err != nil { - return ethcommon.Hash{}, out.err - } - sorted[out.idx] = out.msg + if err := results.CheckError(); err != nil { + return ethcommon.Hash{}, err } - for _, msg := range sorted { - t.Logf(msg) + results.PrintMessages(t.Logf) + if results.AllDone() { + if h, ok := results[0].result.(ethcommon.Hash); ok { + return h, nil + } } } diff --git a/simulators/eth2/common/testnet/utils.go b/simulators/eth2/common/testnet/utils.go new file mode 100644 index 0000000000..e24a3ddcca --- /dev/null +++ b/simulators/eth2/common/testnet/utils.go @@ -0,0 +1,92 @@ +package testnet + +import ( + "fmt" + + "github.com/pkg/errors" + + "github.com/ethereum/hive/simulators/eth2/common/clients" +) + +// result object used to get a result/error from each node +type result struct { + idx int + name string + msg string + err error + errCount int + fatal error + maxErr int + done bool + result interface{} +} + +func (r *result) Clear() { + r.msg = "" + r.err = nil + r.fatal = nil + r.done = false + r.result = nil +} + +type resultsArr []*result + +func (rs resultsArr) Clear() { + for _, r := range rs { + r.Clear() + } +} + +func (rs resultsArr) CheckError() error { + for _, r := range rs { + if r.fatal != nil { + return errors.Wrap( + r.fatal, + fmt.Sprintf("node %d (%s)", r.idx, r.name), + ) + } else if r.err != nil && r.errCount >= r.maxErr { + return errors.Wrap( + r.err, + fmt.Sprintf("node %d (%s)", r.idx, r.name), + ) + } else if r.err != nil { + r.msg = fmt.Sprintf("WARN: node %d (%s): error %d/%d: %v", r.idx, r.name, r.errCount, r.maxErr, r.err) + r.errCount++ + } else { + r.errCount = 0 + } + } + return nil +} + +func (rs resultsArr) PrintMessages( + logf func(fmt string, values ...interface{}), +) { + for _, r := range rs { + if r.msg != "" { + logf("node %d (%s): %s", r.idx, r.name, r.msg) + } + } +} + +func (rs resultsArr) AllDone() bool { + for _, r := range rs { + if !r.done { + return false + } + } + return true +} + +func makeResults(nodes clients.Nodes, maxErr int) resultsArr { + res := make(resultsArr, len(nodes)) + for i, n := range nodes { + r := result{ + idx: i, + name: n.ClientNames(), + maxErr: maxErr, + } + res[i] = &r + } + return res +} diff --git a/simulators/eth2/go.work.sum b/simulators/eth2/go.work.sum index 9914de4ca4..2ee762c679 100644 --- a/simulators/eth2/go.work.sum +++ b/simulators/eth2/go.work.sum @@ -126,6 +126,7 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= @@ -420,7 +421,6 @@ github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -661,6 +661,7 @@ github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= @@ -707,6 +708,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= @@ -764,6 +767,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -1191,10 +1195,12 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inconshreveable/log15.v2 v2.0.0-20200109203555-b30bc20e4fd1/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= diff --git a/simulators/eth2/withdrawals/README.md b/simulators/eth2/withdrawals/README.md index ba59de351d..3c50f389f6 100644 --- a/simulators/eth2/withdrawals/README.md +++ b/simulators/eth2/withdrawals/README.md @@ -160,3 +160,66 @@ document. - Withdrawal addresses specified on node `A` and `B` are fully withdrawing + + +### Builder API Fallback for Withdrawals + +* [x] Builder API Constructs Payloads with Invalid Withdrawals List +
+ Click for details + + - Start two validating nodes on Bellatrix/Paris genesis + - Total of 128 Validators, 64 for each validating node + - All genesis validators have Execution address withdrawal credentials + - Both validating nodes are connected to a builder API mock server + - Builder API server is configured to return payloads with an invalid withdrawals list, starting from capella + - Wait for finalization, and verify at least one block was built by the builder API on each node + - Wait for capella and verify that the invalid payloads are correctly rejected from the canonical chain + - Verify that the chain is able to finalize even after the builder API returns payloads with invalid withdrawals on every request + +
+ +* [x] Builder API Returns Error on Header Request Starting from Capella +
+ Click for details + + - Start two validating nodes on Bellatrix/Paris genesis + - Total of 128 Validators, 64 for each validating node + - All genesis validators have Execution address withdrawal credentials + - Both validating nodes are connected to a builder API mock server + - Builder API server is configured to return error on header request, starting from capella + - Wait for capella + - Wait for finalization, and verify at least one block was built by the builder API on each node + - Verify that the chain is able to finalize even after the builder API returns error on every header request + +
+ +* [x] Builder API Returns Error on Unblinded Block Request Starting from Capella +
+ Click for details + + - Start two validating nodes on Bellatrix/Paris genesis + - Total of 128 Validators, 64 for each validating node + - All genesis validators have Execution address withdrawal credentials + - Both validating nodes are connected to a builder API mock server + - Builder API server is configured to return error on unblinded block request, starting from capella + - Wait for capella + - Wait for finalization, and verify at least one block was built by the builder API on each node + - Verify that the chain is able to finalize even after the builder API returns error on every unblinded block request + +
+ +* [x] Builder API Returns Constructs Valid Withdrawals/Invalid StateRoot Payload Starting from Capella +
+ Click for details + + - Start two validating nodes on Bellatrix/Paris genesis + - Total of 128 Validators, 64 for each validating node + - All genesis validators have Execution address withdrawal credentials + - Both validating nodes are connected to a builder API mock server + - Builder API server is configured to produce payloads with valid withdrawals list, but invalid state root, starting from capella + - Wait for capella + - Verify that the consensus clients correctly circuit break the builder when the empty slots are detected + - Verify that the chain is able to finalize + +
\ No newline at end of file diff --git a/simulators/eth2/withdrawals/helper.go b/simulators/eth2/withdrawals/helper.go index 3842bd6cbe..6cebffd68d 100644 --- a/simulators/eth2/withdrawals/helper.go +++ b/simulators/eth2/withdrawals/helper.go @@ -555,3 +555,44 @@ func ComputeBLSToExecutionDomain( t.GenesisValidatorsRoot(), ) } + +type BaseTransactionCreator struct { + Recipient *common.Address + GasLimit uint64 + Amount *big.Int + Payload []byte + PrivateKey *ecdsa.PrivateKey +} + +func (tc *BaseTransactionCreator) MakeTransaction( + nonce uint64, +) (*types.Transaction, error) { + var newTxData types.TxData + + gasFeeCap := new(big.Int).Set(GasPrice) + gasTipCap := new(big.Int).Set(GasTipPrice) + newTxData = &types.DynamicFeeTx{ + Nonce: nonce, + Gas: tc.GasLimit, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + To: tc.Recipient, + Value: tc.Amount, + Data: tc.Payload, + } + + tx := types.NewTx(newTxData) + key := tc.PrivateKey + if key == nil { + key = VaultKey + } + signedTx, err := types.SignTx( + tx, + types.NewLondonSigner(ChainID), + key, + ) + if err != nil { + return nil, err + } + return signedTx, nil +} diff --git a/simulators/eth2/withdrawals/main.go b/simulators/eth2/withdrawals/main.go index 8af01c1f39..8cd34932f6 100644 --- a/simulators/eth2/withdrawals/main.go +++ b/simulators/eth2/withdrawals/main.go @@ -90,6 +90,72 @@ var tests = []TestSpec{ }, } +var builderTests = []TestSpec{ + BuilderWithdrawalsTestSpec{ + BaseWithdrawalsTestSpec: BaseWithdrawalsTestSpec{ + Name: "test-builders-capella-invalid-withdrawals", + Description: ` + Test canonical chain can still finalize if the builders start + building payloads with invalid withdrawals list. + `, + // All validators can withdraw from the start + GenesisExecutionWithdrawalCredentialsShares: 1, + }, + BuilderTestError: INVALID_WITHDRAWALS, + }, + BuilderWithdrawalsTestSpec{ + BaseWithdrawalsTestSpec: BaseWithdrawalsTestSpec{ + Name: "test-builders-capella-error-on-capella-header-request", + Description: ` + Test canonical chain can still finalize if the builders start + returning error on header request after capella transition. + `, + // All validators can withdraw from the start + GenesisExecutionWithdrawalCredentialsShares: 1, + }, + BuilderTestError: ERROR_ON_HEADER_REQUEST, + }, + BuilderWithdrawalsTestSpec{ + BaseWithdrawalsTestSpec: BaseWithdrawalsTestSpec{ + Name: "test-builders-capella-error-on-capella-unblind-payload-requestr", + Description: ` + Test canonical chain can still finalize if the builders start + returning error on unblinded payload request after capella transition. + `, + // All validators can withdraw from the start + GenesisExecutionWithdrawalCredentialsShares: 1, + }, + BuilderTestError: ERROR_ON_UNBLINDED_PAYLOAD_REQUEST, + }, + BuilderWithdrawalsTestSpec{ + BaseWithdrawalsTestSpec: BaseWithdrawalsTestSpec{ + Name: "test-builders-capella-invalid-payload", + Description: ` + Test consensus clients correctly circuit break builder after a + period of empty blocks due to invalid unblinded blocks. + The payloads are built using an invalid state root, which can only + be caught after unblinding the entire payload and running it in the + local execution client, at which point another payload cannot be + produced locally and results in an empty slot. + `, + // All validators can withdraw from the start + GenesisExecutionWithdrawalCredentialsShares: 1, + }, + BuilderTestError: VALID_WITHDRAWALS_INVALID_STATE_ROOT, + }, + BuilderWithdrawalsTestSpec{ + BaseWithdrawalsTestSpec: BaseWithdrawalsTestSpec{ + Name: "test-builders-capella-correct-withdrawals", + Description: ` + Test canonical chain includes capella payloads built by the builder api. + `, + // All validators can withdraw from the start + GenesisExecutionWithdrawalCredentialsShares: 1, + }, + BuilderTestError: NO_ERROR, + }, +} + func main() { // Create simulator that runs all tests sim := hivesim.New() @@ -101,16 +167,22 @@ func main() { c := clients.ClientsByRole(clientTypes) // Create the test suites - engineSuite := hivesim.Suite{ + withdrawalsSuite := hivesim.Suite{ Name: "eth2-withdrawals", - Description: `Collection of test vectors that use a ExecutionClient+BeaconNode+ValidatorClient testnet.`, + Description: `Collection of test vectors that use a ExecutionClient+BeaconNode+ValidatorClient testnet for Shanghai+Capella.`, + } + builderSuite := hivesim.Suite{ + Name: "eth2-withdrawals-builder", + Description: `Collection of test vectors that use a ExecutionClient+BeaconNode+ValidatorClient testnet and builder API for Shanghai+Capella.`, } // Add all tests to the suites - addAllTests(&engineSuite, c, tests) + addAllTests(&withdrawalsSuite, c, tests) + addAllTests(&builderSuite, c, builderTests) // Mark suites for execution - hivesim.MustRunSuite(sim, engineSuite) + hivesim.MustRunSuite(sim, withdrawalsSuite) + hivesim.MustRunSuite(sim, builderSuite) } func addAllTests( diff --git a/simulators/eth2/withdrawals/scenarios.go b/simulators/eth2/withdrawals/scenarios.go index e37890064a..4f10731714 100644 --- a/simulators/eth2/withdrawals/scenarios.go +++ b/simulators/eth2/withdrawals/scenarios.go @@ -3,10 +3,16 @@ package main import ( "bytes" "context" + "crypto/rand" + "fmt" + "math/big" "time" "github.com/ethereum/go-ethereum/common" + api "github.com/ethereum/go-ethereum/core/beacon" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/hive/hivesim" + mock_builder "github.com/ethereum/hive/simulators/eth2/common/builder/mock" "github.com/ethereum/hive/simulators/eth2/common/clients" beacon_verification "github.com/ethereum/hive/simulators/eth2/common/spoofing/beacon" @@ -271,3 +277,300 @@ loop: testnet.WaitForFinality(ctx) } } + +var ( + slotsPerEpoch = uint64(32) + withdrawalsPerInvalidList = uint64(16) +) + +// Builder testnet. +func (ts BuilderWithdrawalsTestSpec) Execute( + t *hivesim.T, + env *tn.Environment, + n []clients.NodeDefinition, +) { + config := ts.GetTestnetConfig(n) + ctx := context.Background() + + capellaSlot := beacon.Slot( + config.CapellaForkEpoch.Uint64() * slotsPerEpoch, + ) + + // Configure the builder according to the error + config.BuilderOptions = make([]mock_builder.Option, 0) + + // Bump the built payloads value + config.BuilderOptions = append( + config.BuilderOptions, + mock_builder.WithPayloadWeiValueBump(big.NewInt(10000)), + ) + + // Inject test error + switch ts.BuilderTestError { + case INVALID_WITHDRAWALS: + config.BuilderOptions = append( + config.BuilderOptions, + mock_builder.WithPayloadAttributesModifier( + func(pa *api.PayloadAttributes, s beacon.Slot) (bool, error) { + // Only modify once we reached capella + if s >= capellaSlot { + // Create a list of invalid (random) withdrawals within the length limit + pa.Withdrawals = make( + []*types.Withdrawal, + withdrawalsPerInvalidList, + ) + for i := uint64(0); i < withdrawalsPerInvalidList; i++ { + w := types.Withdrawal{} + w.Index = i + (uint64(s-capellaSlot) * withdrawalsPerInvalidList) + w.Validator = i + 1 + w.Amount = i + 1 + rand.Read(w.Address[:]) + pa.Withdrawals[i] = &w + } + return true, nil + } + return false, nil + }, + ), + ) + case INVALIDATE_SINGLE_WITHDRAWAL_ADDRESS, + INVALIDATE_SINGLE_WITHDRAWAL_AMOUNT, + INVALIDATE_SINGLE_WITHDRAWAL_VALIDATOR_INDEX, + INVALIDATE_SINGLE_WITHDRAWAL_INDEX: + config.BuilderOptions = append( + config.BuilderOptions, + mock_builder.WithPayloadAttributesModifier( + func(pa *api.PayloadAttributes, s beacon.Slot) (bool, error) { + // Only modify once we reached capella + if s >= capellaSlot { + // We need to invalidate a single withdrawal + if len(pa.Withdrawals) > 0 { + switch ts.BuilderTestError { + case INVALIDATE_SINGLE_WITHDRAWAL_ADDRESS: + pa.Withdrawals[0].Address[0]++ + case INVALIDATE_SINGLE_WITHDRAWAL_AMOUNT: + pa.Withdrawals[0].Amount++ + case INVALIDATE_SINGLE_WITHDRAWAL_VALIDATOR_INDEX: + pa.Withdrawals[0].Validator++ + case INVALIDATE_SINGLE_WITHDRAWAL_INDEX: + pa.Withdrawals[0].Index++ + } + return true, nil + } + } + return false, nil + }, + ), + ) + case VALID_WITHDRAWALS_INVALID_STATE_ROOT: + config.BuilderOptions = append( + config.BuilderOptions, + mock_builder.WithPayloadModifier( + func(ed *api.ExecutableData, s beacon.Slot) (bool, error) { + // Only modify once we reached capella + if s >= capellaSlot { + var ( + originalHash = ed.BlockHash + originalStateRoot = ed.StateRoot + modifiedStateRoot = common.Hash{} + ) + // We need to simulate the builder producing an invalid + // execution payload by modifying its state root + rand.Read(modifiedStateRoot[:]) + if b, err := api.ExecutableDataToBlock(*ed); err != nil { + return false, err + } else { + header := b.Header() + header.Root = modifiedStateRoot + modifiedHash := header.Hash() + copy(ed.BlockHash[:], modifiedHash[:]) + copy(ed.StateRoot[:], modifiedStateRoot[:]) + } + t.Logf( + "INFO: Modified payload %d: hash:%s->%s, stateRoot:%s->%s, parentHash:%s", + ed.Number, + originalHash, + ed.BlockHash, + originalStateRoot, + ed.StateRoot, + ed.ParentHash, + ) + return true, nil + } + return false, nil + }, + ), + ) + case ERROR_ON_HEADER_REQUEST: + config.BuilderOptions = append( + config.BuilderOptions, + mock_builder.WithErrorOnHeaderRequest( + func(s beacon.Slot) error { + if s >= capellaSlot { + return fmt.Errorf("error produced by test") + } + return nil + }, + ), + ) + case ERROR_ON_UNBLINDED_PAYLOAD_REQUEST: + config.BuilderOptions = append( + config.BuilderOptions, + mock_builder.WithErrorOnPayloadReveal( + func(s beacon.Slot) error { + if s >= capellaSlot { + return fmt.Errorf("error produced by test") + } + return nil + }, + ), + ) + } + + testnet := tn.StartTestnet(ctx, t, env, config) + defer testnet.Stop() + + go func() { + lastNonce := uint64(0) + txPerIteration := 5 + txCreator := BaseTransactionCreator{ + GasLimit: 500000, + Amount: common.Big1, + PrivateKey: VaultKey, + } + // Send some transactions constantly in the bg + for { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + for i := 0; i < txPerIteration; i++ { + txCreator.Recipient = &CodeContractAddress + tx, err := txCreator.MakeTransaction(lastNonce) + if err != nil { + panic(err) + } + if err := testnet.ExecutionClients().Running()[0].SendTransaction( + ctx, + tx, + ); err != nil { + t.Logf("INFO: Error sending tx: %v", err) + } + lastNonce++ + } + } + } + }() + + // Wait for capella + forkCtx, cancel := testnet.Spec(). + EpochTimeoutContext(ctx, beacon.Epoch(config.CapellaForkEpoch.Uint64())+1) + defer cancel() + if err := testnet.WaitForFork(forkCtx, "capella"); err != nil { + t.Fatalf("FAIL: error while waiting for capella: %v", err) + } + + // Check that the builder was working properly until now + for i, b := range testnet.BeaconClients().Running() { + builder := b.Builder + if builder.GetBuiltPayloadsCount() == 0 { + t.Fatalf("FAIL: builder %d did not build any payloads", i) + } + if builder.GetSignedBeaconBlockCount() == 0 { + t.Fatalf( + "FAIL: builder %d did not produce any signed beacon blocks", + i, + ) + } + } + + // Wait for finalization, to verify that builder modifications + // did not affect the network + finalityCtx, cancel := testnet.Spec().EpochTimeoutContext(ctx, 5) + defer cancel() + if _, err := testnet.WaitForCurrentEpochFinalization(finalityCtx); err != nil { + t.Fatalf("FAIL: error waiting for epoch finalization: %v", err) + } + + // Verify any modified payloads did not make it into the + // canonical chain + switch ts.BuilderTestError { + case NO_ERROR: + // Simply verify that builder's capella payloads were included in the + // canonical chain + for i, n := range testnet.Nodes.Running() { + b := n.BeaconClient.Builder + ec := n.ExecutionClient + includedPayloads := 0 + for _, p := range b.GetBuiltPayloads() { + if p.Withdrawals != nil { + if h, err := ec.HeaderByNumber(ctx, big.NewInt(int64(p.Number))); err != nil { + t.Fatalf( + "FAIL: error getting execution header from node %d: %v", + i, + err, + ) + } else if h != nil { + hash := h.Hash() + if bytes.Equal(hash[:], p.BlockHash[:]) { + includedPayloads++ + } + } + } + } + if includedPayloads == 0 { + t.Fatalf( + "FAIL: builder %d did not produce capella payloads included in the canonical chain", + i, + ) + } + } + case INVALID_WITHDRAWALS, + INVALIDATE_SINGLE_WITHDRAWAL_ADDRESS, + INVALIDATE_SINGLE_WITHDRAWAL_AMOUNT, + INVALIDATE_SINGLE_WITHDRAWAL_VALIDATOR_INDEX, + INVALIDATE_SINGLE_WITHDRAWAL_INDEX: + for i, n := range testnet.VerificationNodes().Running() { + modifiedPayloads := n.BeaconClient.Builder.GetModifiedPayloads() + if len(modifiedPayloads) == 0 { + t.Fatalf("FAIL: No payloads were modified by builder %d", i) + } + for _, p := range modifiedPayloads { + for _, ec := range testnet.ExecutionClients().Running() { + b, err := ec.BlockByNumber( + ctx, + big.NewInt(int64(p.Number)), + ) + if err != nil { + t.Fatalf( + "FAIL: Error getting execution block %d: %v", + p.Number, + err, + ) + } + h := b.Hash() + if bytes.Equal(h[:], p.BlockHash[:]) { + t.Fatalf( + "FAIL: Modified payload included in canonical chain: %d (%s)", + p.Number, + p.BlockHash, + ) + } + } + } + t.Logf( + "INFO: No modified payloads were included in canonical chain of node %d", + i, + ) + } + } + + // Count and print missed slots + if count, err := testnet.BeaconClients().Running()[0].GetFilledSlotsCountPerEpoch(ctx); err != nil { + t.Fatalf("FAIL: unable to obtain slot count per epoch: %v", err) + } else { + for ep, slots := range count { + t.Logf("INFO: Epoch %d, filled slots=%d", ep, slots) + } + } +} diff --git a/simulators/eth2/withdrawals/specs.go b/simulators/eth2/withdrawals/specs.go index 1d00bb32c5..10c7246db3 100644 --- a/simulators/eth2/withdrawals/specs.go +++ b/simulators/eth2/withdrawals/specs.go @@ -5,6 +5,9 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/hive/simulators/eth2/common/clients" cl "github.com/ethereum/hive/simulators/eth2/common/config/consensus" el "github.com/ethereum/hive/simulators/eth2/common/config/execution" @@ -80,6 +83,24 @@ var ( } */ + // This is the account that sends vault funding transactions. + VaultAccountAddress = common.HexToAddress( + "0xcf49fda3be353c69b41ed96333cd24302da4556f", + ) + VaultKey, _ = crypto.HexToECDSA( + "63b508a03c3b5937ceb903af8b1b0c191012ef6eb7e9c3fb7afa94e5d214d376", + ) + VaultStartAmount, _ = new(big.Int).SetString("d3c21bcecceda1000000", 16) + + CodeContractAddress = common.HexToAddress( + "0xcccccccccccccccccccccccccccccccccccccccc", + ) + CodeContract = common.Hex2Bytes("0x328043558043600080a250") + + GasPrice = big.NewInt(30 * params.GWei) + GasTipPrice = big.NewInt(1 * params.GWei) + + ChainID = big.NewInt(7) ) func (ts BaseWithdrawalsTestSpec) GetTestnetConfig( @@ -130,6 +151,19 @@ func (ts BaseWithdrawalsTestSpec) GetTestnetConfig( } nodeDefinitions = append(nodeDefinitions, n) } + + // Fund execution layer account for transactions + + config.GenesisExecutionAccounts = map[common.Address]core.GenesisAccount{ + VaultAccountAddress: { + Balance: VaultStartAmount, + }, + CodeContractAddress: { + Balance: common.Big0, + Code: CodeContract, + }, + } + return config.Join(&testnet.Config{ NodeDefinitions: nodeDefinitions, }) @@ -197,3 +231,51 @@ func (ts BaseWithdrawalsTestSpec) GetValidatorKeys( return keys } + +type BuilderTestError int + +const ( + NO_ERROR BuilderTestError = iota + ERROR_ON_HEADER_REQUEST + ERROR_ON_UNBLINDED_PAYLOAD_REQUEST + INVALID_WITHDRAWALS + INVALIDATE_SINGLE_WITHDRAWAL_ADDRESS + INVALIDATE_SINGLE_WITHDRAWAL_AMOUNT + INVALIDATE_SINGLE_WITHDRAWAL_VALIDATOR_INDEX + INVALIDATE_SINGLE_WITHDRAWAL_INDEX + VALID_WITHDRAWALS_INVALID_STATE_ROOT + TIMEOUT +) + +var REQUIRES_FINALIZATION_TO_ACTIVATE_BUILDER = []string{ + "lighthouse", + "teku", +} + +type BuilderWithdrawalsTestSpec struct { + BaseWithdrawalsTestSpec + BuilderTestError BuilderTestError +} + +func (ts BuilderWithdrawalsTestSpec) GetTestnetConfig( + allNodeDefinitions clients.NodeDefinitions, +) *testnet.Config { + tc := ts.BaseWithdrawalsTestSpec.GetTestnetConfig(allNodeDefinitions) + + tc.CapellaForkEpoch = big.NewInt(2) + + if len( + allNodeDefinitions.FilterByCL( + REQUIRES_FINALIZATION_TO_ACTIVATE_BUILDER, + ), + ) > 0 { + // At least one of the CLs require finalization to start requesting + // headers from the builder + tc.CapellaForkEpoch = big.NewInt(5) + } + + // Builders are always enabled for these tests + tc.EnableBuilders = true + + return tc +}