op-{acceptance-tests|devstack}: sync test: op-supervisor ahead of op-node, unsafe chain known with DSL

op-acceptance-tests/tests/interop/message/interop_msg_test.go

@@ -22,7 +22,7 @@ func TestInitExecMsg(gt *testing.T) {
 	// Trigger random init message at chain A
 	initIntent, _ := alice.SendInitMessage(interop.RandomInitTrigger(rng, eventLoggerAddress, rng.Intn(5), rng.Intn(30)))
 	// Make sure supervisor indexs block which includes init message
-	sys.Supervisor.AdvanceUnsafeHead(alice.ChainID(), 2)
+	sys.Supervisor.AdvancedUnsafeHead(alice.ChainID(), 2)
 	// Single event in tx so index is 0
 	bob.SendExecMessage(initIntent, 0)
 }

op-acceptance-tests/tests/interop/sync/redundant_interop/init_test.go

@@ -0,0 +1,11 @@
+package sync
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+)
+
+func TestMain(m *testing.M) {
+	presets.DoMain(m, presets.WithRedundantInterop())
+}

op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go

@@ -0,0 +1,87 @@
+package sync
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
+	"github.com/ethereum-optimism/optimism/op-devstack/dsl"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
+)
+
+// TestUnsafeChainKnownToL2CL tests the below scenario:
+// supervisor cross-safe ahead of L2CL cross-safe, aka L2CL can "skip" forward to match safety of supervisor.
+// To create this out-of-sync scenario, we follow the steps below:
+// 1. Make sequencer (L2CL), verifier (L2CL), and supervisor sync for a few blocks.
+// - Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync.
+// - Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor.
+// - Wait enough for both L2CLs advance safe heads.
+// 2. Disconnect the P2P connection between the sequencer and verifier.
+// - The verifier will not receive unsafe heads via P2P, and can only update unsafe heads matching with safe heads by reading L1 batches.
+// - The verifier safe head will lag behind or match the sequencer and supervisor because all three components share the same L1 view.
+// 3. Stop verifier L2CL
+// - The verifier will not be able to advance unsafe head and safe head.
+// - The sequencer will advance unsafe head and safe head, as well as synced with supervisor.
+// 4. Wait until sequencer and supervisor diverged enough from the verifier.
+// - To make the verifier held unsafe blocks which are already viewed as safe by sequencer and supervisor, we wait.
+// - Wait until supervisor viewed safe head number is large enough than the stopped verifier's safe head view.
+// 5. Restart the verifier.
+// - The verifier will not sync via P2P but only able to advance unsafe and safe heads by reading L1 batches.
+// - The verifier will quickly catch up with the sequencer safe head as well as the supervisor.
+// - The verifier will "skip" processing already known unsafe blocks, and consolidate them into safe blocks.
+func TestUnsafeChainKnownToL2CL(gt *testing.T) {
+	t := devtest.SerialT(gt)
+
+	sys := presets.NewRedundantInterop(t)
+	logger := sys.Log.With("Test", "TestUnsafeChainKnownToL2CL")
+	require := sys.T.Require()
+
+	logger.Info("make sure verifier safe head advances")
+	dsl.CheckAll(t,
+		sys.L2CLA.Advanced(types.CrossSafe, 5, 30),
+		sys.L2CLA2.Advanced(types.CrossSafe, 5, 30),
+	)
+
+	safeA2 := sys.L2ELA2.BlockRefByLabel(eth.Safe)
+	logger.Info("verifier advanced safe head", "number", safeA2.Number)
+	unsafeA2 := sys.L2ELA2.BlockRefByLabel(eth.Unsafe)
+	logger.Info("verifier advanced unsafe head", "number", unsafeA2.Number)
+
+	// For making verifier stop advancing unsafe head via P2P
+	logger.Info("disconnect p2p between L2CLs")
+	sys.L2CLA.DisconnectPeer(sys.L2CLA2)
+	sys.L2CLA2.DisconnectPeer(sys.L2CLA)
+
+	// For making verifer not sync at all
+	logger.Info("stop verifier")
+	sys.L2CLA2.Stop()
+
+	delta := uint64(10)
+	logger.Info("wait until supervisor reaches safe head", "delta", delta)
+	sys.Supervisor.AdvancedSafeHead(sys.L2ChainA.ChainID(), delta, 30)
+
+	// Restarted verifier will advance its unsafe head by reading L1 but not by P2P
+	logger.Info("restart verifier")
+	sys.L2CLA2.Start()
+
+	safeA2 = sys.L2ELA2.BlockRefByLabel(eth.Safe)
+	logger.Info("verifier safe head after restart", "number", safeA2.Number)
+	unsafeA2 = sys.L2ELA2.BlockRefByLabel(eth.Unsafe)
+	logger.Info("verifier unsafe head after restart", "number", unsafeA2.Number)
+
+	// Make sure there are unsafe blocks to be consolidated:
+	// To check verifier does not have to process blocks since unsafe blocks are already processed
+	require.Greater(unsafeA2.Number, safeA2.Number)
+
+	logger.Info("make sure verifier unsafe head was consolidated to safe")
+	dsl.CheckAll(t, sys.L2CLA2.Reached(types.CrossSafe, unsafeA2.Number, 30))
+
+	safeA := sys.L2ELA.BlockRefByLabel(eth.Safe)
+	target := safeA.Number + delta
+	logger.Info("make sure verifier unsafe head advances due to safe head advances", "target", target, "delta", delta)
+	dsl.CheckAll(t, sys.L2CLA2.Reached(types.LocalUnsafe, target, 30))
+
+	block := sys.L2ELA2.BlockRefByNumber(unsafeA2.Number)
+	require.Equal(unsafeA2.Hash, block.Hash)
+}

op-devstack/dsl/l2_cl.go

@@ -1,23 +1,33 @@
 package dsl
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
 	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-service/apis"
 	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-service/retry"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
 )
 
 // L2CLNode wraps a stack.L2CLNode interface for DSL operations
 type L2CLNode struct {
 	commonImpl
 	inner   stack.L2CLNode
 	control stack.ControlPlane
+	chainID eth.ChainID
 }
 
 // NewL2CLNode creates a new L2CLNode DSL wrapper
-func NewL2CLNode(inner stack.L2CLNode, control stack.ControlPlane) *L2CLNode {
+func NewL2CLNode(inner stack.L2CLNode, control stack.ControlPlane, chainID eth.ChainID) *L2CLNode {
 	return &L2CLNode{
 		commonImpl: commonFromT(inner.T()),
 		inner:      inner,
 		control:    control,
+		chainID:    chainID,
 	}
 }
 
@@ -31,10 +41,7 @@ func (cl *L2CLNode) Escape() stack.L2CLNode {
 }
 
 func (cl *L2CLNode) SafeL2BlockRef() eth.L2BlockRef {
-	syncStatus, err := cl.Escape().RollupAPI().SyncStatus(cl.ctx)
-	cl.require.NoError(err, "Expected to get sync status")
-
-	return syncStatus.SafeL2
+	return cl.HeadBlockRef(types.CrossSafe)
 }
 
 func (cl *L2CLNode) Start() {
@@ -44,3 +51,83 @@ func (cl *L2CLNode) Start() {
 func (cl *L2CLNode) Stop() {
 	cl.control.L2CLNodeState(cl.inner.ID(), stack.Stop)
 }
+
+func (cl *L2CLNode) SyncStatus() *eth.SyncStatus {
+	ctx, cancel := context.WithTimeout(cl.ctx, DefaultTimeout)
+	defer cancel()
+	syncStatus, err := cl.inner.RollupAPI().SyncStatus(ctx)
+	cl.require.NoError(err)
+	return syncStatus
+}
+
+// HeadBlockRef fetches L2CL sync status and returns block ref with given safety level
+func (cl *L2CLNode) HeadBlockRef(lvl types.SafetyLevel) eth.L2BlockRef {
+	syncStatus := cl.SyncStatus()
+	var blockRef eth.L2BlockRef
+	switch lvl {
+	case types.Finalized:
+		blockRef = syncStatus.FinalizedL2
+	case types.CrossSafe:
+		blockRef = syncStatus.SafeL2
+	case types.LocalSafe:
+		blockRef = syncStatus.LocalSafeL2
+	case types.CrossUnsafe:
+		blockRef = syncStatus.CrossUnsafeL2
+	case types.LocalUnsafe:
+		blockRef = syncStatus.UnsafeL2
+	default:
+		cl.require.NoError(errors.New("invalid safety level"))
+	}
+	return blockRef
+}
+
+func (cl *L2CLNode) ChainID() eth.ChainID {
+	return cl.chainID
+}
+
+// Advanced returns a lambda that checks the L2CL chain head with given safety level advanced more than delta block number
+// Composable with other lambdas to wait in parallel
+func (cl *L2CLNode) Advanced(lvl types.SafetyLevel, delta uint64, attempts int) CheckFunc {
+	return func() error {
+		initial := cl.HeadBlockRef(lvl)
+		target := initial.Number + delta
+		cl.log.Info("expecting chain to advance", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "delta", delta)
+		return cl.Reached(lvl, target, attempts)()
+	}
+}
+
+// Reached returns a lambda that checks the L2CL chain head with given safety level reaches the target block number
+// Composable with other lambdas to wait in parallel
+func (cl *L2CLNode) Reached(lvl types.SafetyLevel, target uint64, attempts int) CheckFunc {
+	return func() error {
+		cl.log.Info("expecting chain to reach", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target)
+		return retry.Do0(cl.ctx, attempts, &retry.FixedStrategy{Dur: 2 * time.Second},
+			func() error {
+				head := cl.HeadBlockRef(lvl)
+				if head.Number >= target {
+					cl.log.Info("chain advanced", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target)
+					return nil
+				}
+				cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target, "current", head.Number)
+				return fmt.Errorf("expected head to advance: %s", lvl)
+			})
+	}
+}
+
+func (cl *L2CLNode) PeerInfo() *apis.PeerInfo {
+	peerInfo, err := cl.inner.P2PAPI().Self(cl.ctx)
+	cl.require.NoError(err, "failed to get peer info")
+	return peerInfo
+}
+
+func (cl *L2CLNode) Peers() *apis.PeerDump {
+	peerDump, err := cl.inner.P2PAPI().Peers(cl.ctx, true)
+	cl.require.NoError(err, "failed to get peers")
+	return peerDump
+}
+
+func (cl *L2CLNode) DisconnectPeer(peer *L2CLNode) {
+	peerInfo := peer.PeerInfo()
+	err := cl.inner.P2PAPI().DisconnectPeer(cl.ctx, peerInfo.PeerID)
+	cl.require.NoError(err, "failed to disconnect peer")
+}

op-devstack/dsl/l2_el.go

@@ -79,3 +79,11 @@ func (el *L2ELNode) DoesNotAdvance(label eth.BlockLabel) CheckFunc {
 		return nil
 	}
 }
+
+func (el *L2ELNode) BlockRefByNumber(num uint64) eth.BlockRef {
+	ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout)
+	defer cancel()
+	block, err := el.inner.EthClient().BlockRefByNumber(ctx, num)
+	el.require.NoError(err, "block not found using block label")
+	return block
+}

op-devstack/dsl/supervisor.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-service/eth"
 	"github.com/ethereum-optimism/optimism/op-service/retry"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/backend/status"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
 )
 
 type Supervisor struct {
@@ -88,36 +89,55 @@ func (s *Supervisor) FetchSyncStatus() eth.SupervisorSyncStatus {
 }
 
 func (s *Supervisor) SafeBlockID(chainID eth.ChainID) eth.BlockID {
-	ctx, cancel := context.WithTimeout(s.ctx, DefaultTimeout)
-	defer cancel()
-	syncStatus, err := retry.Do[eth.SupervisorSyncStatus](ctx, 2, retry.Fixed(500*time.Millisecond), func() (eth.SupervisorSyncStatus, error) {
-		syncStatus, err := s.inner.QueryAPI().SyncStatus(s.ctx)
-		if errors.Is(err, status.ErrStatusTrackerNotReady) {
-			s.log.Debug("Sync status not ready from supervisor")
-		}
-		return syncStatus, err
-	})
-	s.require.NoError(err, "Failed to fetch sync status")
+	return s.L2HeadBlockID(chainID, types.CrossSafe)
+}
 
-	return syncStatus.Chains[chainID].CrossSafe
+// L2HeadBlockID fetches supervisor sync status and returns block id with given safety level
+func (s *Supervisor) L2HeadBlockID(chainID eth.ChainID, lvl types.SafetyLevel) eth.BlockID {
+	supervisorSyncStatus := s.FetchSyncStatus()
+	supervisorChainSyncStatus, ok := supervisorSyncStatus.Chains[chainID]
+	s.require.True(ok, "chain id not found in supervisor sync status")
+	var blockID eth.BlockID
+	switch lvl {
+	case types.Finalized:
+		blockID = supervisorChainSyncStatus.Finalized
+	case types.CrossSafe:
+		blockID = supervisorChainSyncStatus.CrossSafe
+	case types.LocalSafe:
+		blockID = supervisorChainSyncStatus.LocalSafe
+	case types.CrossUnsafe:
+		blockID = supervisorChainSyncStatus.CrossUnsafe
+	case types.LocalUnsafe:
+		blockID = supervisorChainSyncStatus.LocalUnsafe.ID()
+	default:
+		s.require.NoError(errors.New("invalid safety level"))
+	}
+	return blockID
 }
 
-func (s *Supervisor) AdvanceUnsafeHead(chainID eth.ChainID, block uint64) {
-	initial := s.FetchSyncStatus()
-	chInitial, ok := initial.Chains[chainID]
-	s.require.True(ok, fmt.Sprintf("chain sync status not found: chain id: %d", chainID))
-	required := chInitial.LocalUnsafe.Number + block
-	attempts := int(block + 3) // intentionally allow few more attempts for avoid flaking
+// AdvancedL2Head checks the supervisor view of L2CL chain head with given safety level advanced more than delta block number
+func (s *Supervisor) AdvancedL2Head(chainID eth.ChainID, delta uint64, lvl types.SafetyLevel, attempts int) {
+	chInitial := s.L2HeadBlockID(chainID, lvl)
+	target := chInitial.Number + delta
 	err := retry.Do0(s.ctx, attempts, &retry.FixedStrategy{Dur: 2 * time.Second},
 		func() error {
-			chStatus := s.FetchSyncStatus().Chains[chainID]
-			s.log.Info("Supervisor view of unsafe head", "chain", chainID, "unsafe", chStatus.LocalUnsafe)
-			if chStatus.LocalUnsafe.Number < required {
-				s.log.Info("Unsafe head sync status not ready",
-					"chain", chainID, "initialUnsafe", chInitial.LocalUnsafe, "currentUnsafe", chStatus.LocalUnsafe, "minRequired", required)
-				return fmt.Errorf("expected head to advance")
+			chStatus := s.L2HeadBlockID(chainID, lvl)
+			s.log.Info("Supervisor view",
+				"chain", chainID, "label", lvl, "initial", chInitial.Number, "current", chStatus.Number, "target", target)
+			if chStatus.Number >= target {
+				s.log.Info("Supervisor view advanced", "chain", chainID, "label", lvl, "target", target)
+				return nil
 			}
-			return nil
+			return fmt.Errorf("expected head to advance: %s", lvl)
 		})
 	s.require.NoError(err)
 }
+
+func (s *Supervisor) AdvancedUnsafeHead(chainID eth.ChainID, block uint64) {
+	attempts := int(block + 3) // intentionally allow few more attempts for avoid flaking
+	s.AdvancedL2Head(chainID, block, types.LocalUnsafe, attempts)
+}
+
+func (s *Supervisor) AdvancedSafeHead(chainID eth.ChainID, block uint64, attempts int) {
+	s.AdvancedL2Head(chainID, block, types.CrossSafe, attempts)
+}

op-devstack/presets/interop.go

@@ -80,8 +80,8 @@ func NewSimpleInterop(t devtest.T) *SimpleInterop {
 		L2ChainB:     dsl.NewL2Network(l2B),
 		L2ELA:        dsl.NewL2ELNode(l2A.L2ELNode(match.Assume(t, match.FirstL2EL))),
 		L2ELB:        dsl.NewL2ELNode(l2B.L2ELNode(match.Assume(t, match.FirstL2EL))),
-		L2CLA:        dsl.NewL2CLNode(l2A.L2CLNode(match.Assume(t, match.FirstL2CL)), orch.ControlPlane()),
-		L2CLB:        dsl.NewL2CLNode(l2B.L2CLNode(match.Assume(t, match.FirstL2CL)), orch.ControlPlane()),
+		L2CLA:        dsl.NewL2CLNode(l2A.L2CLNode(match.Assume(t, match.FirstL2CL)), orch.ControlPlane(), l2A.ChainID()),
+		L2CLB:        dsl.NewL2CLNode(l2B.L2CLNode(match.Assume(t, match.FirstL2CL)), orch.ControlPlane(), l2B.ChainID()),
 		Wallet:       dsl.NewHDWallet(t, devkeys.TestMnemonic, 30),
 		FaucetA:      dsl.NewFaucet(l2A.Faucet(match.Assume(t, match.FirstFaucet))),
 		FaucetB:      dsl.NewFaucet(l2B.Faucet(match.Assume(t, match.FirstFaucet))),
@@ -116,3 +116,26 @@ func WithInteropNotAtGenesis() stack.CommonOption {
 		}
 	})
 }
+
+type RedundantInterop struct {
+	SimpleInterop
+
+	L2ELA2 *dsl.L2ELNode
+	L2CLA2 *dsl.L2CLNode
+}
+
+func WithRedundantInterop() stack.CommonOption {
+	return stack.MakeCommon(sysgo.RedundantInteropSystem(&sysgo.RedundantInteropSystemIDs{}))
+}
+
+func NewRedundantInterop(t devtest.T) *RedundantInterop {
+	simpleInterop := NewSimpleInterop(t)
+	orch := Orchestrator()
+	l2A := simpleInterop.L2ChainA.Escape()
+	out := &RedundantInterop{
+		SimpleInterop: *simpleInterop,
+		L2ELA2:        dsl.NewL2ELNode(l2A.L2ELNode(match.Assume(t, match.SecondL2EL))),
+		L2CLA2:        dsl.NewL2CLNode(l2A.L2CLNode(match.Assume(t, match.SecondL2CL)), orch.ControlPlane(), l2A.ChainID()),
+	}
+	return out
+}

op-devstack/stack/match/second.go

@@ -0,0 +1,6 @@
+package match
+
+import "github.com/ethereum-optimism/optimism/op-devstack/stack"
+
+var SecondL2EL = Second[stack.L2ELNodeID, stack.L2ELNode]()
+var SecondL2CL = Second[stack.L2CLNodeID, stack.L2CLNode]()

op-devstack/sysgo/contracts.go

@@ -33,9 +33,9 @@ func ensureDir(dirPath string) error {
 // Different tests might be nested in subdirectories of the op-e2e dir.
 func findMonorepoRoot(testPath string) (string, error) {
 	path := "./"
-	// Only search up 5 directories
+	// Only search up 6 directories
 	// Avoids infinite recursion if the root isn't found for some reason
-	for i := 0; i < 5; i++ {
+	for i := 0; i < 6; i++ {
 		_, err := os.Stat(path + testPath)
 		if errors.Is(err, os.ErrNotExist) {
 			path = path + "../"