op-devstack: L2EL Engine API support for op-geth

op-acceptance-tests/tests/sync/init_test.go → op-acceptance-tests/tests/sync/elsync/gap_clp2p/init_test.go

@@ -1,4 +1,4 @@
-package elsync
+package gap_clp2p
 
 import (
 	"testing"

op-acceptance-tests/tests/sync/sync_test.go → op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go

@@ -1,4 +1,4 @@
-package elsync
+package gap_clp2p
 
 import (
 	"testing"

op-acceptance-tests/tests/sync/manual/init_test.go

@@ -0,0 +1,22 @@
+package manual
+
+import (
+	"testing"
+
+	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
+	"github.com/ethereum-optimism/optimism/op-devstack/compat"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
+)
+
+func TestMain(m *testing.M) {
+	// No ELP2P, CLP2P to control the supply of unsafe payload to the CL
+	presets.DoMain(m, presets.WithSingleChainMultiNodeWithoutP2P(),
+		presets.WithCompatibleTypes(compat.SysGo),
+		stack.MakeCommon(sysgo.WithBatcherOption(func(id stack.L2BatcherID, cfg *bss.CLIConfig) {
+			// For stopping derivation, not to advance safe heads
+			cfg.Stopped = true
+		})),
+	)
+}

op-acceptance-tests/tests/sync/manual/sync_test.go

@@ -0,0 +1,60 @@
+package manual
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
+	"github.com/ethereum/go-ethereum"
+)
+
+func TestVerifierManualSync(gt *testing.T) {
+	t := devtest.SerialT(gt)
+
+	// Disable ELP2P and Batcher
+	sys := presets.NewSingleChainMultiNodeWithoutCheck(t)
+	require := t.Require()
+	logger := t.Logger()
+
+	delta := uint64(7)
+	sys.L2CL.Advanced(types.LocalUnsafe, delta, 30)
+
+	// Disable Derivation
+	sys.L2CLB.Stop()
+
+	startBlockNum := sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number
+
+	// Manual Block insertion using engine APIs
+	for i := uint64(1); i <= delta; i++ {
+		blockNum := startBlockNum + i
+		block := sys.L2EL.BlockRefByNumber(blockNum)
+		// Validator does not have canonical nor noncanonical block for blockNum
+		_, err := sys.L2ELB.Escape().EthClient().BlockRefByNumber(t.Ctx(), blockNum)
+		require.Error(err, ethereum.NotFound)
+		_, err = sys.L2ELB.Escape().EthClient().BlockRefByHash(t.Ctx(), block.Hash)
+		require.Error(err, ethereum.NotFound)
+
+		// Insert payload
+		logger.Info("NewPayload", "target", blockNum)
+		sys.L2ELB.NewPayload(sys.L2EL, blockNum).IsValid()
+		// Payload valid but not canonicalized. Cannot fetch block by number
+		_, err = sys.L2ELB.Escape().EthClient().BlockRefByNumber(t.Ctx(), blockNum)
+		require.Error(err, ethereum.NotFound)
+		// Now fetchable by hash
+		require.Equal(blockNum, sys.L2ELB.BlockRefByHash(block.Hash).Number)
+
+		// FCU
+		logger.Info("ForkchoiceUpdate", "target", blockNum)
+		sys.L2ELB.ForkchoiceUpdate(sys.L2EL, blockNum, 0, 0, nil).IsValid()
+		// Payload valid and canonicalized
+		require.Equal(block.Hash, sys.L2ELB.BlockRefByNumber(blockNum).Hash)
+		require.Equal(blockNum, sys.L2ELB.BlockRefByHash(block.Hash).Number)
+	}
+
+	// Check correctly synced by comparing with sequencer EL
+	res := sys.L2ELB.BlockRefByLabel(eth.Unsafe)
+	require.Equal(startBlockNum+delta, res.Number)
+	require.Equal(sys.L2EL.BlockRefByNumber(startBlockNum+delta).Hash, res.Hash)
+}

op-devstack/dsl/engine.go

@@ -0,0 +1,54 @@
+package dsl
+
+import (
+	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+)
+
+type NewPayloadResult struct {
+	T      devtest.T
+	Status *eth.PayloadStatusV1
+	Err    error
+}
+
+func (r *NewPayloadResult) IsPayloadStatus(status eth.ExecutePayloadStatus) *NewPayloadResult {
+	r.T.Require().NotNil(r.Status, "payload status nil")
+	r.T.Require().Equal(status, r.Status.Status)
+	return r
+}
+
+func (r *NewPayloadResult) IsSyncing() *NewPayloadResult {
+	r.IsPayloadStatus(eth.ExecutionSyncing)
+	r.T.Require().NoError(r.Err)
+	return r
+}
+
+func (r *NewPayloadResult) IsValid() *NewPayloadResult {
+	r.IsPayloadStatus(eth.ExecutionValid)
+	r.T.Require().NoError(r.Err)
+	return r
+}
+
+type ForkchoiceUpdateResult struct {
+	T      devtest.T
+	Result *eth.ForkchoiceUpdatedResult
+	Err    error
+}
+
+func (r *ForkchoiceUpdateResult) IsForkchoiceUpdatedStatus(status eth.ExecutePayloadStatus) *ForkchoiceUpdateResult {
+	r.T.Require().NotNil(r.Result, "fcu result nil")
+	r.T.Require().Equal(status, r.Result.PayloadStatus.Status)
+	return r
+}
+
+func (r *ForkchoiceUpdateResult) IsSyncing() *ForkchoiceUpdateResult {
+	r.IsForkchoiceUpdatedStatus(eth.ExecutionSyncing)
+	r.T.Require().NoError(r.Err)
+	return r
+}
+
+func (r *ForkchoiceUpdateResult) IsValid() *ForkchoiceUpdateResult {
+	r.IsForkchoiceUpdatedStatus(eth.ExecutionValid)
+	r.T.Require().NoError(r.Err)
+	return r
+}

op-devstack/dsl/l2_el.go

@@ -53,6 +53,14 @@ func (el *L2ELNode) BlockRefByLabel(label eth.BlockLabel) eth.L2BlockRef {
 	return block
 }
 
+func (el *L2ELNode) BlockRefByHash(hash common.Hash) eth.L2BlockRef {
+	ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout)
+	defer cancel()
+	block, err := el.inner.L2EthClient().L2BlockRefByHash(ctx, hash)
+	el.require.NoError(err, "block not found using block hash")
+	return block
+}
+
 func (el *L2ELNode) AdvancedFn(label eth.BlockLabel, block uint64) CheckFunc {
 	return func() error {
 		initial := el.BlockRefByLabel(label)
@@ -215,3 +223,21 @@ func (el *L2ELNode) PayloadByNumber(number uint64) *eth.ExecutionPayloadEnvelope
 	el.require.NoError(err, "failed to get payload")
 	return payload
 }
+
+// NewPayload fetches payload for target number from the reference EL Node, and inserts the payload
+func (el *L2ELNode) NewPayload(refNode *L2ELNode, number uint64) *NewPayloadResult {
+	payload := refNode.PayloadByNumber(number)
+	status, err := el.inner.L2EngineClient().NewPayload(el.ctx, payload.ExecutionPayload, payload.ParentBeaconBlockRoot)
+	return &NewPayloadResult{T: el.t, Status: status, Err: err}
+}
+
+// ForkchoiceUpdate fetches FCU target hashes from the reference EL node, and FCU update with attributes
+func (el *L2ELNode) ForkchoiceUpdate(refNode *L2ELNode, unsafe, safe, finalized uint64, attr *eth.PayloadAttributes) *ForkchoiceUpdateResult {
+	state := &eth.ForkchoiceState{
+		HeadBlockHash:      refNode.BlockRefByNumber(unsafe).Hash,
+		SafeBlockHash:      refNode.BlockRefByNumber(safe).Hash,
+		FinalizedBlockHash: refNode.BlockRefByNumber(finalized).Hash,
+	}
+	result, err := el.inner.L2EngineClient().ForkchoiceUpdate(el.ctx, state, attr)
+	return &ForkchoiceUpdateResult{T: el.t, Result: result, Err: err}
+}

op-devstack/shim/l2_el.go

@@ -6,18 +6,21 @@ import (
 	"github.com/ethereum-optimism/optimism/op-devstack/stack"
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
 	"github.com/ethereum-optimism/optimism/op-service/apis"
+	"github.com/ethereum-optimism/optimism/op-service/client"
 	"github.com/ethereum-optimism/optimism/op-service/sources"
 )
 
 type L2ELNodeConfig struct {
 	ELNodeConfig
-	RollupCfg *rollup.Config
-	ID        stack.L2ELNodeID
+	EngineClient client.RPC
+	RollupCfg    *rollup.Config
+	ID           stack.L2ELNodeID
 }
 
 type rpcL2ELNode struct {
 	rpcELNode
-	l2Client *sources.L2Client
+	l2Client       *sources.L2Client
+	l2EngineClient *sources.EngineClient
 
 	id stack.L2ELNodeID
 }
@@ -30,11 +33,17 @@ func NewL2ELNode(cfg L2ELNodeConfig) stack.L2ELNode {
 	require.NotNil(cfg.T, cfg.RollupCfg, "rollup config must be configured")
 	l2Client, err := sources.NewL2Client(cfg.ELNodeConfig.Client, cfg.T.Logger(), nil, sources.L2ClientSimpleConfig(cfg.RollupCfg, false, 10, 10))
 	require.NoError(cfg.T, err)
-
+	engineClientConfig := &sources.EngineClientConfig{
+		L2ClientConfig: *sources.L2ClientSimpleConfig(cfg.RollupCfg, false, 10, 10),
+	}
+	// initialize engine API client using different client
+	engineClient, err := sources.NewEngineClient(cfg.EngineClient, cfg.T.Logger(), nil, engineClientConfig)
+	require.NoError(cfg.T, err)
 	return &rpcL2ELNode{
-		rpcELNode: newRpcELNode(cfg.ELNodeConfig),
-		l2Client:  l2Client,
-		id:        cfg.ID,
+		rpcELNode:      newRpcELNode(cfg.ELNodeConfig),
+		l2Client:       l2Client,
+		l2EngineClient: engineClient,
+		id:             cfg.ID,
 	}
 }
 
@@ -49,3 +58,7 @@ func (r *rpcL2ELNode) L2EthClient() apis.L2EthClient {
 func (r *rpcL2ELNode) L2EthExtendedClient() apis.L2EthExtendedClient {
 	return r.l2Client
 }
+
+func (r *rpcL2ELNode) L2EngineClient() apis.EngineClient {
+	return r.l2EngineClient.EngineAPIClient
+}

op-devstack/stack/l2_el.go

@@ -72,6 +72,7 @@ type L2ELNode interface {
 	ID() L2ELNodeID
 	L2EthClient() apis.L2EthClient
 	L2EthExtendedClient() apis.L2EthExtendedClient
+	L2EngineClient() apis.EngineClient
 
 	ELNode
 }

op-devstack/sysgo/l2_el.go

@@ -80,8 +80,9 @@ func WithExtL2Node(id stack.L2ELNodeID, elRPCEndpoint string) stack.Option[*Orch
 
 		// Create L2 EL node with external RPC
 		l2ELNode := &OpGeth{
-			id:      id,
-			userRPC: elRPCEndpoint,
+			id:       id,
+			userRPC:  elRPCEndpoint,
+			readOnly: true,
 		}
 		require.True(orch.l2ELs.SetIfMissing(id, l2ELNode), "must not already exist")
 	})

op-devstack/sysgo/l2_el_opgeth.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	gn "github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
 
 	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
 	"github.com/ethereum-optimism/optimism/op-devstack/shim"
@@ -25,8 +26,10 @@ type OpGeth struct {
 	id            stack.L2ELNodeID
 	l2Net         *L2Network
 	jwtPath       string
+	jwtSecret     [32]byte
 	supervisorRPC string
 	l2Geth        *geth.GethInstance
+	readOnly      bool
 
 	authRPC string
 	userRPC string
@@ -55,6 +58,15 @@ func (n *OpGeth) hydrate(system stack.ExtensibleSystem) {
 	require.NoError(err)
 	system.T().Cleanup(rpcCl.Close)
 
+	// ReadOnly cannot expose auth RPC
+	var engineCl client.RPC
+	if !n.readOnly {
+		auth := rpc.WithHTTPAuth(gn.NewJWTAuth(n.jwtSecret))
+		engineCl, err = client.NewRPC(system.T().Ctx(), system.Logger(), n.authRPC, client.WithGethRPCOptions(auth))
+		require.NoError(err)
+		system.T().Cleanup(engineCl.Close)
+	}
+
 	l2Net := system.L2Network(stack.L2NetworkID(n.id.ChainID()))
 	sysL2EL := shim.NewL2ELNode(shim.L2ELNodeConfig{
 		RollupCfg: l2Net.RollupConfig(),
@@ -63,7 +75,8 @@ func (n *OpGeth) hydrate(system stack.ExtensibleSystem) {
 			Client:       rpcCl,
 			ChainID:      n.id.ChainID(),
 		},
-		ID: n.id,
+		EngineClient: engineCl,
+		ID:           n.id,
 	})
 	sysL2EL.SetLabel(match.LabelVendor, string(match.OpGeth))
 	l2Net.(stack.ExtensibleL2Network).AddL2ELNode(sysL2EL)
@@ -138,7 +151,7 @@ func WithOpGeth(id stack.L2ELNodeID, opts ...L2ELOption) stack.Option[*Orchestra
 		orch.l2ELOptions.Apply(p, id, cfg)       // apply global options
 		L2ELOptionBundle(opts).Apply(p, id, cfg) // apply specific options
 
-		jwtPath, _ := orch.writeDefaultJWT()
+		jwtPath, jwtSecret := orch.writeDefaultJWT()
 
 		useInterop := l2Net.genesis.Config.InteropTime != nil
 
@@ -158,6 +171,7 @@ func WithOpGeth(id stack.L2ELNodeID, opts ...L2ELOption) stack.Option[*Orchestra
 			logger:        logger,
 			l2Net:         l2Net,
 			jwtPath:       jwtPath,
+			jwtSecret:     jwtSecret,
 			supervisorRPC: supervisorRPC,
 		}
 		l2EL.Start()

op-service/apis/engine.go

@@ -0,0 +1,14 @@
+package apis
+
+import (
+	"context"
+
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type EngineClient interface {
+	GetPayload(ctx context.Context, payloadInfo eth.PayloadInfo) (*eth.ExecutionPayloadEnvelope, error)
+	ForkchoiceUpdate(ctx context.Context, state *eth.ForkchoiceState, attr *eth.PayloadAttributes) (*eth.ForkchoiceUpdatedResult, error)
+	NewPayload(ctx context.Context, payload *eth.ExecutionPayload, parentBeaconBlockRoot *common.Hash) (*eth.PayloadStatusV1, error)
+}