simulators/ethereum/engine: tests for getPayloadBodies*, exchangeCapabilities

simulators/ethereum/engine/client/engine.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+
+	client_types "github.com/ethereum/hive/simulators/ethereum/engine/client/types"
 )
 
 type Eth interface {
@@ -31,9 +33,13 @@ type Engine interface {
 
 	GetPayloadV1(ctx context.Context, payloadId *api.PayloadID) (api.ExecutableData, error)
 	GetPayloadV2(ctx context.Context, payloadId *api.PayloadID) (api.ExecutableData, *big.Int, error)
+
 	NewPayloadV1(ctx context.Context, payload *api.ExecutableData) (api.PayloadStatusV1, error)
 	NewPayloadV2(ctx context.Context, payload *api.ExecutableData) (api.PayloadStatusV1, error)
 
+	GetPayloadBodiesByRangeV1(ctx context.Context, start uint64, count uint64) ([]*client_types.ExecutionPayloadBodyV1, error)
+	GetPayloadBodiesByHashV1(ctx context.Context, hashes []common.Hash) ([]*client_types.ExecutionPayloadBodyV1, error)
+
 	LatestForkchoiceSent() (fcState *api.ForkchoiceStateV1, pAttributes *api.PayloadAttributes)
 	LatestNewPayloadSent() (payload *api.ExecutableData)
 

simulators/ethereum/engine/client/hive_rpc/hive_rpc.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/hive/hivesim"
 	"github.com/ethereum/hive/simulators/ethereum/engine/client"
+	client_types "github.com/ethereum/hive/simulators/ethereum/engine/client/types"
 	"github.com/ethereum/hive/simulators/ethereum/engine/globals"
 	"github.com/ethereum/hive/simulators/ethereum/engine/helper"
 	"github.com/golang-jwt/jwt/v4"
@@ -334,7 +335,7 @@ func (ec *HiveRPCEngineClient) GetPayload(ctx context.Context, version int, payl
 	}
 
 	if version == 2 {
-		var response api.ExecutableDataV2
+		var response api.ExecutionPayloadEnvelope
 		err = ec.c.CallContext(ctx, &response, rpcString, payloadId)
 		if response.ExecutionPayload != nil {
 			executableData = *response.ExecutionPayload
@@ -356,6 +357,32 @@ func (ec *HiveRPCEngineClient) GetPayloadV2(ctx context.Context, payloadId *api.
 	return ec.GetPayload(ctx, 2, payloadId)
 }
 
+func (ec *HiveRPCEngineClient) GetPayloadBodiesByRangeV1(ctx context.Context, start uint64, count uint64) ([]*client_types.ExecutionPayloadBodyV1, error) {
+	var (
+		result []*client_types.ExecutionPayloadBodyV1
+		err    error
+	)
+	if err = ec.PrepareDefaultAuthCallToken(); err != nil {
+		return nil, err
+	}
+
+	err = ec.c.CallContext(ctx, &result, "engine_getPayloadBodiesByRangeV1", start, count)
+	return result, err
+}
+
+func (ec *HiveRPCEngineClient) GetPayloadBodiesByHashV1(ctx context.Context, hashes []common.Hash) ([]*client_types.ExecutionPayloadBodyV1, error) {
+	var (
+		result []*client_types.ExecutionPayloadBodyV1
+		err    error
+	)
+	if err = ec.PrepareDefaultAuthCallToken(); err != nil {
+		return nil, err
+	}
+
+	err = ec.c.CallContext(ctx, &result, "engine_getPayloadBodiesByHashV1", hashes)
+	return result, err
+}
+
 func (ec *HiveRPCEngineClient) NewPayload(ctx context.Context, version int, payload *api.ExecutableData) (api.PayloadStatusV1, error) {
 	var result api.PayloadStatusV1
 	if err := ec.PrepareDefaultAuthCallToken(); err != nil {
@@ -381,6 +408,15 @@ func (ec *HiveRPCEngineClient) ExchangeTransitionConfigurationV1(ctx context.Con
 	return result, err
 }
 
+func (ec *HiveRPCEngineClient) ExchangeCapabilities(ctx context.Context, clCapabilities []string) ([]string, error) {
+	var result []string
+	if err := ec.PrepareDefaultAuthCallToken(); err != nil {
+		return result, err
+	}
+	err := ec.c.CallContext(ctx, &result, "engine_exchangeCapabilities", clCapabilities)
+	return result, err
+}
+
 func (ec *HiveRPCEngineClient) GetNextAccountNonce(testCtx context.Context, account common.Address) (uint64, error) {
 	// First get the current head of the client where we will send the tx
 	ctx, cancel := context.WithTimeout(testCtx, globals.RPCTimeout)

simulators/ethereum/engine/client/node/node.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/hive/hivesim"
 	"github.com/ethereum/hive/simulators/ethereum/engine/client"
+	client_types "github.com/ethereum/hive/simulators/ethereum/engine/client/types"
 	"github.com/ethereum/hive/simulators/ethereum/engine/globals"
 	"github.com/ethereum/hive/simulators/ethereum/engine/helper"
 )
@@ -734,6 +735,14 @@ func (n *GethNode) GetPayloadV2(ctx context.Context, payloadId *beacon.PayloadID
 	return *p.ExecutionPayload, p.BlockValue, err
 }
 
+func (n *GethNode) GetPayloadBodiesByRangeV1(ctx context.Context, start uint64, count uint64) ([]*client_types.ExecutionPayloadBodyV1, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (n *GethNode) GetPayloadBodiesByHashV1(ctx context.Context, hashes []common.Hash) ([]*client_types.ExecutionPayloadBodyV1, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 // Eth JSON RPC
 const (
 	SafeBlockNumber      = rpc.BlockNumber(-4) // This is not yet true

simulators/ethereum/engine/client/types/types.go

@@ -0,0 +1,17 @@
+package types
+
+import (
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+//go:generate go run github.com/fjl/gencodec -type ExecutionPayloadBodyV1 -field-override executionPayloadBodyV1Marshaling -out gen_epbv1.go
+type ExecutionPayloadBodyV1 struct {
+	Transactions [][]byte            `json:"transactions"  gencodec:"required"`
+	Withdrawals  []*types.Withdrawal `json:"withdrawals"`
+}
+
+// JSON type overrides for executableData.
+type executionPayloadBodyV1Marshaling struct {
+	Transactions []hexutil.Bytes
+}

simulators/ethereum/engine/go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/ethereum/hive v0.0.0-20230124094404-5b561920e19b
 	github.com/golang-jwt/jwt/v4 v4.4.3
+	golang.org/x/exp v0.0.0-20230126173853-a67bb567ff2e
 )
 
 require (
@@ -69,4 +70,4 @@ require (
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.26 => github.com/lightclient/go-ethereum v1.10.10-0.20230116085521-6ab6d738866f
+replace github.com/ethereum/go-ethereum v1.10.26 => github.com/ethereum/go-ethereum v1.10.24-0.20230127123926-a63875bf4d7f

simulators/ethereum/engine/go.sum

@@ -76,6 +76,8 @@ github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ethereum/go-ethereum v1.10.24-0.20230127123926-a63875bf4d7f h1:4zLYUPuCDwBfQSCY1QWJfVn9fmOFmX1D4GCnMC8UlU8=
+github.com/ethereum/go-ethereum v1.10.24-0.20230127123926-a63875bf4d7f/go.mod h1:C5Xij/C5W1rXkm7pCvr7p2HWUiVk6RmTF3EUvp8LXMY=
 github.com/ethereum/hive v0.0.0-20230124094404-5b561920e19b h1:TW8iKjNf7tZZkleHIGNG2wkawQC8BzGce++cxRw10lU=
 github.com/ethereum/hive v0.0.0-20230124094404-5b561920e19b/go.mod h1:pqToex2uV+Tg7bhBla/uvB3APKZ6FogJMMpow33nUIA=
 github.com/fjl/memsize v0.0.1 h1:+zhkb+dhUgx0/e+M8sF0QqiouvMQUiKR+QYvdxIOKcQ=
@@ -216,8 +218,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lightclient/go-ethereum v1.10.10-0.20230116085521-6ab6d738866f h1:Pox2oxBYKWVEw2JUCyiEybqEIurSkq8VHjRp6s9jdf8=
-github.com/lightclient/go-ethereum v1.10.10-0.20230116085521-6ab6d738866f/go.mod h1:n7VlOgCwYheLB/mi+V8ni2yf8K2qM3N9WAmalxkhk+c=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
@@ -363,6 +363,8 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20230126173853-a67bb567ff2e h1:nEzRHNOazEST44vMvEwxGxnYGrzXEmxJmnti5mKSWTk=
+golang.org/x/exp v0.0.0-20230126173853-a67bb567ff2e/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

simulators/ethereum/engine/main.go

@@ -12,6 +12,7 @@ import (
 
 	suite_auth "github.com/ethereum/hive/simulators/ethereum/engine/suites/auth"
 	suite_engine "github.com/ethereum/hive/simulators/ethereum/engine/suites/engine"
+	suite_ex_cap "github.com/ethereum/hive/simulators/ethereum/engine/suites/exchange_capabilities"
 	suite_transition "github.com/ethereum/hive/simulators/ethereum/engine/suites/transition"
 	suite_withdrawals "github.com/ethereum/hive/simulators/ethereum/engine/suites/withdrawals"
 )
@@ -34,6 +35,11 @@ func main() {
 			Name: "engine-auth",
 			Description: `
 	Test Engine API authentication features.`[1:],
+		}
+		excap = hivesim.Suite{
+			Name: "engine-exchange-capabilities",
+			Description: `
+	Test Engine API exchange capabilities.`[1:],
 		}
 		sync = hivesim.Suite{
 			Name: "engine-sync",
@@ -52,13 +58,15 @@ func main() {
 	addTestsToSuite(simulator, &engine, specToInterface(suite_engine.Tests), "full")
 	addTestsToSuite(simulator, &transition, specToInterface(suite_transition.Tests), "full")
 	addTestsToSuite(simulator, &auth, specToInterface(suite_auth.Tests), "full")
+	addTestsToSuite(simulator, &excap, specToInterface(suite_ex_cap.Tests), "full")
 	//suite_sync.AddSyncTestsToSuite(simulator, &sync, suite_sync.Tests)
 	addTestsToSuite(simulator, &withdrawals, suite_withdrawals.Tests, "full")
 
 	// Mark suites for execution
 	hivesim.MustRunSuite(simulator, engine)
 	hivesim.MustRunSuite(simulator, transition)
 	hivesim.MustRunSuite(simulator, auth)
+	hivesim.MustRunSuite(simulator, excap)
 	hivesim.MustRunSuite(simulator, sync)
 	hivesim.MustRunSuite(simulator, withdrawals)
 }
@@ -118,6 +126,7 @@ func addTestsToSuite(sim *hivesim.Simulation, suite *hivesim.Suite, tests []test
 			delete(newParams, "HIVE_CLIQUE_PRIVATEKEY")
 			delete(newParams, "HIVE_CLIQUE_PERIOD")
 			delete(newParams, "HIVE_MINER")
+			newParams = newParams.Set("HIVE_POST_MERGE_GENESIS", "true")
 		}
 
 		if clientTypes, err := sim.ClientTypes(); err == nil {

simulators/ethereum/engine/suites/exchange_capabilities/tests.go

@@ -0,0 +1,35 @@
+package suite_exchange_capabilities
+
+import (
+	"github.com/ethereum/hive/simulators/ethereum/engine/test"
+	"golang.org/x/exp/slices"
+)
+
+var Tests = []test.Spec{
+	{
+		Name: "Exchange Capabilities",
+		Run:  exCapTests,
+	},
+}
+
+var minimalSetExpectedSupportedELCapabilities = []string{
+	"engine_newPayloadV1",
+	"engine_newPayloadV2",
+	"engine_forkchoiceUpdatedV1",
+	"engine_forkchoiceUpdatedV2",
+	"engine_getPayloadV1",
+	"engine_getPayloadV2",
+	// "engine_getPayloadBodiesByRangeV1",
+}
+
+func exCapTests(t *test.Env) {
+	if returnedCapabilities, err := t.HiveEngine.ExchangeCapabilities(t.TestContext, minimalSetExpectedSupportedELCapabilities); err != nil {
+		t.Fatalf("FAIL (%s): Unable request capabilities: %v", t.TestName, err)
+	} else {
+		for _, cap := range minimalSetExpectedSupportedELCapabilities {
+			if !slices.Contains(returnedCapabilities, cap) {
+				t.Fatalf("FAIL (%s): Expected capability (%s) not found", t.TestName, cap)
+			}
+		}
+	}
+}

simulators/ethereum/engine/suites/withdrawals/README.md

@@ -151,4 +151,28 @@ This test suite verifies behavior of the Engine API on each client after the Sha
   - Verify that `TxB` returns error on `eth_sendRawTransaction` and also should be absent from the transaction pool of the client
   - Request a new payload from the client and verify that the payload built only includes `TxA`, and `TxB` is not included, nor the contract it could create is present in the `stateRoot`.
   - Create a modified payload where `TxA` is replaced by `TxB` and send using `engine_newPayloadV2`
-  - Verify that `engine_newPayloadV2` returns `INVALID` nad `latestValidHash` points to the latest valid payload in the canonical chain.
+  - Verify that `engine_newPayloadV2` returns `INVALID` nad `latestValidHash` points to the latest valid payload in the canonical chain.
+
+## GetPayloadBodies Tests
+
+- Payload Bodies By Range - Shanghai Fork on Block 16 - 16 Withdrawal Blocks
+  - Launch client `A` and create a canonical chain consisting of 32 blocks, where the first shanghai block is number 17
+  - Payloads produced of the following characteristics
+    - [x] 16 Transactions, 16 Withdrawals
+    - [x] 0 Transactions, 0 Withdrawals
+  - Make multiple requests to obtain the payload bodies from the canonical chain (see `./tests.go` for full list).
+  - Verify that:
+    - Payload bodies of blocks before the Shanghai fork contain `withdrawals==null`
+    - All transactions and withdrawals are in the correct format and order.
+    - Requested payload bodies past the highest known block are ignored and absent from the returned list
+
+- Payload Bodies By Hash - Shanghai Fork on Block 16 - 16 Withdrawal Blocks
+  - Launch client `A` and create a canonical chain consisting of 32 blocks, where the first shanghai block is number 17
+  - Payloads produced of the following characteristics
+    - [x] 16 Transactions, 16 Withdrawals
+    - [x] 0 Transactions, 0 Withdrawals
+  - Make multiple requests to obtain the payload bodies from the canonical chain (see `./tests.go` for full list).
+  - Verify that:
+    - Payload bodies of blocks before the Shanghai fork contain `withdrawals==null`
+    - All transactions and withdrawals are in the correct format and order.
+    - Requested payload bodies of unknown hashes are returned as null in the returned list