op-{acceptance-tests|devstack}: sync test: op-node ahead of op-supervisor with DSL

op-acceptance-tests/tests/interop/sync/multisupervisor_interop/init_test.go

@@ -0,0 +1,11 @@
+package sync
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+)
+
+func TestMain(m *testing.M) {
+	presets.DoMain(m, presets.WithMultiSupervisorInterop())
+}

op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go

@@ -0,0 +1,119 @@
+package sync
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
+	"github.com/ethereum-optimism/optimism/op-devstack/dsl"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
+)
+
+// TestL2CLAheadOfSupervisor tests the below scenario:
+// L2CL ahead of supervisor, aka supervisor needs to reset the L2CL, to reproduce old data. Currently supervisor has only managed mode implemented, so the supervisor will ask the L2CL to reset back.
+func TestL2CLAheadOfSupervisor(gt *testing.T) {
+	t := devtest.SerialT(gt)
+
+	// Two supervisor initialized, each managing two L2CLs per chains.
+	// Primary supervisor manages sequencer L2CLs for chain A, B.
+	// Secondary supervisor manages verifier L2CLs for chain A, B.
+	// Each L2CLs per chain is connected via P2P.
+	sys := presets.NewMultiSupervisorInterop(t)
+	logger := sys.Log.With("Test", "TestL2CLAheadOfSupervisor")
+	require := sys.T.Require()
+
+	// Make sequencers (L2CL), verifiers (L2CL), and supervisors sync for a few blocks.
+	// Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync.
+	// Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor.
+	delta := uint64(10)
+	logger.Info("Make sure verifiers advances unsafe head", "delta", delta)
+	dsl.CheckAll(t,
+		sys.L2CLA.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLA2.Advanced(types.LocalUnsafe, delta, 30),
+		sys.L2CLB.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB2.Advanced(types.LocalUnsafe, delta, 30),
+	)
+
+	safeHeadViewA2 := sys.SupervisorSecondary.SafeBlockID(sys.L2CLA.ChainID())
+	safeHeadViewB2 := sys.SupervisorSecondary.SafeBlockID(sys.L2CLB.ChainID())
+
+	logger.Info("Stop secondary supervisor")
+	sys.SupervisorSecondary.Stop()
+
+	safeHeadA2 := sys.L2CLA2.SafeL2BlockRef()
+	safeHeadB2 := sys.L2CLB2.SafeL2BlockRef()
+	require.Equal(safeHeadViewA2.Hash, safeHeadA2.Hash)
+	require.Equal(safeHeadViewB2.Hash, safeHeadB2.Hash)
+	logger.Info("Secondary supervisor(stopped) safe head view", "chainA", safeHeadA2, "chainB", safeHeadB2)
+
+	// Wait enough to make sequencers and primary supervisor advance safe head enough.
+	logger.Info("Sequencers advances safe heads but not verifiers", "delta", delta)
+	dsl.CheckAll(t,
+		// verifier CLs cannot advance their safe head because secondary supervisor is down, no supervisor to provide them L1 data.
+		sys.L2CLA2.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30),
+		// sequencer CLs advance their safe heads
+		sys.L2CLA.Advanced(types.CrossSafe, delta, 30), sys.L2CLB.Advanced(types.CrossSafe, delta, 30),
+		// All the L2CLs advance their unsafe heads
+		// Verifiers advances unsafe head because they still have P2P connection with each sequencers
+		sys.L2CLA.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB.Advanced(types.LocalUnsafe, delta, 30),
+		sys.L2CLA2.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB2.Advanced(types.LocalUnsafe, delta, 30),
+	)
+
+	// Primary supervisor has safe heads synced with sequencers.
+	// After connection, verifiers will sync with primary supervisor, matching supervisor safe head view.
+	logger.Info("Connect verifier CLs to primary supervisor to advance verifier safe heads")
+	sys.Supervisor.AddManagedL2CL(sys.L2CLA2)
+	sys.Supervisor.AddManagedL2CL(sys.L2CLB2)
+
+	// Secondary supervisor and verifiers becomes out-of-sync with safe heads.
+	target := max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta
+	logger.Info("Every CLs advance safe heads", "delta", delta, "target", target)
+	dsl.CheckAll(t,
+		sys.L2CLA.Reached(types.CrossSafe, target, 30), sys.L2CLA2.Reached(types.CrossSafe, target, 30),
+		sys.L2CLB.Reached(types.CrossSafe, target, 30), sys.L2CLB2.Reached(types.CrossSafe, target, 30),
+	)
+
+	logger.Info("Stop primary supervisor to disconnect every CL connection")
+	sys.Supervisor.Stop()
+
+	logger.Info("Restart primary supervisor")
+	sys.Supervisor.Start()
+
+	logger.Info("No CL connected to supervisor so every CL safe head will not advance")
+	dsl.CheckAll(t,
+		sys.L2CLA.NotAdvanced(types.CrossSafe, 30), sys.L2CLA2.NotAdvanced(types.CrossSafe, 30),
+		sys.L2CLB.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30),
+	)
+
+	// Sequencers will resume advancing safe heads, but not verifiers.
+	logger.Info("Reconnect sequencer CLs to primary supervisor")
+	sys.Supervisor.AddManagedL2CL(sys.L2CLA)
+	sys.Supervisor.AddManagedL2CL(sys.L2CLB)
+
+	logger.Info("Restart secondary supervisor")
+	sys.SupervisorSecondary.Start()
+
+	logger.Info("Reconnect verifier CLs to secondary supervisor")
+	sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLA2)
+	sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLB2)
+
+	// Secondary supervisor will compare its safe head knowledge with L2CLs, and find out L2CLs are ahead of the Secondary supervisor.
+	// Secondary supervisor asks the verifiers (L2CL) to rewind(reset) back to match Secondary supervisor safe head view.
+	rewind := uint64(3)
+	logger.Info("Check verifier CLs safe head rewinded", "rewind", rewind)
+	dsl.CheckAll(t,
+		sys.L2CLA2.Rewinded(types.CrossSafe, rewind, 60),
+		sys.L2CLB2.Rewinded(types.CrossSafe, rewind, 60),
+	)
+
+	// After rewinding(reset), verifier will advance safe heads again because Secondary supervisor gives L1 data to the verifiers.
+	// Wait until verifiers advance safe head enough
+	target = max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta
+	logger.Info("Every CLs advance safe heads", "delta", delta, "target", target)
+	dsl.CheckAll(t,
+		sys.L2CLA.Reached(types.CrossSafe, target, 30), sys.L2CLA2.Reached(types.CrossSafe, target, 30),
+		sys.L2CLB.Reached(types.CrossSafe, target, 30), sys.L2CLB2.Reached(types.CrossSafe, target, 30),
+	)
+
+	// Make sure each chain did not diverge
+	require.Equal(sys.L2ELA.BlockRefByNumber(target).Hash, sys.L2ELA2.BlockRefByNumber(target).Hash)
+	require.Equal(sys.L2ELB.BlockRefByNumber(target).Hash, sys.L2ELB2.BlockRefByNumber(target).Hash)
+}

op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go

@@ -12,74 +12,64 @@ import (
 
 // TestUnsafeChainKnownToL2CL tests the below scenario:
 // supervisor cross-safe ahead of L2CL cross-safe, aka L2CL can "skip" forward to match safety of supervisor.
-// To create this out-of-sync scenario, we follow the steps below:
-// 1. Make sequencer (L2CL), verifier (L2CL), and supervisor sync for a few blocks.
-// - Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync.
-// - Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor.
-// - Wait enough for both L2CLs advance safe heads.
-// 2. Disconnect the P2P connection between the sequencer and verifier.
-// - The verifier will not receive unsafe heads via P2P, and can only update unsafe heads matching with safe heads by reading L1 batches.
-// - The verifier safe head will lag behind or match the sequencer and supervisor because all three components share the same L1 view.
-// 3. Stop verifier L2CL
-// - The verifier will not be able to advance unsafe head and safe head.
-// - The sequencer will advance unsafe head and safe head, as well as synced with supervisor.
-// 4. Wait until sequencer and supervisor diverged enough from the verifier.
-// - To make the verifier held unsafe blocks which are already viewed as safe by sequencer and supervisor, we wait.
-// - Wait until supervisor viewed safe head number is large enough than the stopped verifier's safe head view.
-// 5. Restart the verifier.
-// - The verifier will not sync via P2P but only able to advance unsafe and safe heads by reading L1 batches.
-// - The verifier will quickly catch up with the sequencer safe head as well as the supervisor.
-// - The verifier will "skip" processing already known unsafe blocks, and consolidate them into safe blocks.
 func TestUnsafeChainKnownToL2CL(gt *testing.T) {
 	t := devtest.SerialT(gt)
 
+	// Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync.
+	// Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor.
 	sys := presets.NewRedundantInterop(t)
 	logger := sys.Log.With("Test", "TestUnsafeChainKnownToL2CL")
 	require := sys.T.Require()
 
-	logger.Info("make sure verifier safe head advances")
+	logger.Info("Make sure verifier safe head advances")
 	dsl.CheckAll(t,
 		sys.L2CLA.Advanced(types.CrossSafe, 5, 30),
 		sys.L2CLA2.Advanced(types.CrossSafe, 5, 30),
 	)
 
 	safeA2 := sys.L2ELA2.BlockRefByLabel(eth.Safe)
-	logger.Info("verifier advanced safe head", "number", safeA2.Number)
+	logger.Info("Verifier advanced safe head", "number", safeA2.Number)
 	unsafeA2 := sys.L2ELA2.BlockRefByLabel(eth.Unsafe)
-	logger.Info("verifier advanced unsafe head", "number", unsafeA2.Number)
+	logger.Info("Verifier advanced unsafe head", "number", unsafeA2.Number)
 
-	// For making verifier stop advancing unsafe head via P2P
-	logger.Info("disconnect p2p between L2CLs")
+	// The verifier stops advancing unsafe head because it will not receive unsafe heads via P2P, and can only update unsafe heads matching with safe heads by reading L1 batches,
+	// The verifier safe head will lag behind or match the sequencer and supervisor because all three components share the same L1 view.
+	logger.Info("Disconnect p2p between L2CLs")
 	sys.L2CLA.DisconnectPeer(sys.L2CLA2)
 	sys.L2CLA2.DisconnectPeer(sys.L2CLA)
 
-	// For making verifer not sync at all
+	// For making verifer not sync at all, both unsafe haead and safe head
+	// The sequencer will advance unsafe head and safe head, as well as synced with supervisor.
 	logger.Info("stop verifier")
 	sys.L2CLA2.Stop()
 
+	// Wait until sequencer and supervisor diverged enough from the verifier.
+	// To make the verifier held unsafe blocks are already as safe by sequencer and supervisor, we wait.
 	delta := uint64(10)
-	logger.Info("wait until supervisor reaches safe head", "delta", delta)
+	logger.Info("Wait until supervisor reaches safe head", "delta", delta)
 	sys.Supervisor.AdvancedSafeHead(sys.L2ChainA.ChainID(), delta, 30)
 
-	// Restarted verifier will advance its unsafe head by reading L1 but not by P2P
-	logger.Info("restart verifier")
+	// Restarted verifier will advance its unsafe head and safe head by reading L1 but not by P2P
+	logger.Info("Restart verifier")
 	sys.L2CLA2.Start()
 
 	safeA2 = sys.L2ELA2.BlockRefByLabel(eth.Safe)
-	logger.Info("verifier safe head after restart", "number", safeA2.Number)
+	logger.Info("Verifier safe head after restart", "number", safeA2.Number)
 	unsafeA2 = sys.L2ELA2.BlockRefByLabel(eth.Unsafe)
-	logger.Info("verifier unsafe head after restart", "number", unsafeA2.Number)
+	logger.Info("Verifier unsafe head after restart", "number", unsafeA2.Number)
 
 	// Make sure there are unsafe blocks to be consolidated:
 	// To check verifier does not have to process blocks since unsafe blocks are already processed
 	require.Greater(unsafeA2.Number, safeA2.Number)
 
-	logger.Info("make sure verifier unsafe head was consolidated to safe")
+	// The verifier will quickly catch up with the sequencer safe head as well as the supervisor.
+	// The verifier will "skip" processing already known unsafe blocks, and consolidate them into safe blocks.
+	logger.Info("Make sure verifier unsafe head was consolidated to safe")
 	dsl.CheckAll(t, sys.L2CLA2.Reached(types.CrossSafe, unsafeA2.Number, 30))
 
 	safeA := sys.L2ELA.BlockRefByLabel(eth.Safe)
 	target := safeA.Number + delta
-	logger.Info("make sure verifier unsafe head advances due to safe head advances", "target", target, "delta", delta)
+	logger.Info("Make sure verifier unsafe head advances due to safe head advances", "target", target, "delta", delta)
 	dsl.CheckAll(t, sys.L2CLA2.Reached(types.LocalUnsafe, target, 30))
 
 	block := sys.L2ELA2.BlockRefByNumber(unsafeA2.Number)

op-acceptance-tests/tests/interop/sync/simple_interop/interop_sync_test.go

@@ -16,36 +16,36 @@ func TestL2CLResync(gt *testing.T) {
 	sys := presets.NewSimpleInterop(t)
 	logger := sys.Log.With("Test", "TestL2CLResync")
 
-	logger.Info("check unsafe chains are advancing")
+	logger.Info("Check unsafe chains are advancing")
 	dsl.CheckAll(t,
 		sys.L2ELA.Advance(eth.Unsafe, 5),
 		sys.L2ELB.Advance(eth.Unsafe, 5),
 	)
 
-	logger.Info("stop L2CL nodes")
+	logger.Info("Stop L2CL nodes")
 	sys.L2CLA.Stop()
 	sys.L2CLB.Stop()
 
-	logger.Info("make sure L2ELs does not advance")
+	logger.Info("Make sure L2ELs does not advance")
 	dsl.CheckAll(t,
 		sys.L2ELA.DoesNotAdvance(eth.Unsafe),
 		sys.L2ELB.DoesNotAdvance(eth.Unsafe),
 	)
 
-	logger.Info("restart L2CL nodes")
+	logger.Info("Restart L2CL nodes")
 	sys.L2CLA.Start()
 	sys.L2CLB.Start()
 
 	// L2CL may advance a few blocks without supervisor connection, but eventually it will stop without the connection
 	// we must check that unsafe head is advancing due to reconnection
-	logger.Info("boot up L2CL nodes")
+	logger.Info("Boot up L2CL nodes")
 	dsl.CheckAll(t,
 		sys.L2ELA.Advance(eth.Unsafe, 10),
 		sys.L2ELB.Advance(eth.Unsafe, 10),
 	)
 
 	// supervisor will attempt to reconnect with L2CLs at this point because L2CL ws endpoint is recovered
-	logger.Info("check unsafe chains are advancing again")
+	logger.Info("Check unsafe chains are advancing again")
 	dsl.CheckAll(t,
 		sys.L2ELA.Advance(eth.Unsafe, 10),
 		sys.L2ELB.Advance(eth.Unsafe, 10),

op-devstack/dsl/l2_cl.go

@@ -96,6 +96,23 @@ func (cl *L2CLNode) Advanced(lvl types.SafetyLevel, delta uint64, attempts int)
 	}
 }
 
+func (cl *L2CLNode) NotAdvanced(lvl types.SafetyLevel, attempts int) CheckFunc {
+	return func() error {
+		initial := cl.HeadBlockRef(lvl)
+		cl.log.Info("expecting chain not to advance", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", initial.Number)
+		for range attempts {
+			time.Sleep(2 * time.Second)
+			head := cl.HeadBlockRef(lvl)
+			cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", initial.Number, "current", head.Number)
+			if head.Hash == initial.Hash {
+				continue
+			}
+			return fmt.Errorf("expected head not to advance: %s", lvl)
+		}
+		return nil
+	}
+}
+
 // Reached returns a lambda that checks the L2CL chain head with given safety level reaches the target block number
 // Composable with other lambdas to wait in parallel
 func (cl *L2CLNode) Reached(lvl types.SafetyLevel, target uint64, attempts int) CheckFunc {
@@ -114,6 +131,28 @@ func (cl *L2CLNode) Reached(lvl types.SafetyLevel, target uint64, attempts int)
 	}
 }
 
+// Rewinded returns a lambda that checks the L2CL chain head with given safety level rewinded more than the delta block number
+// Composable with other lambdas to wait in parallel
+func (cl *L2CLNode) Rewinded(lvl types.SafetyLevel, delta uint64, attempts int) CheckFunc {
+	return func() error {
+		initial := cl.HeadBlockRef(lvl)
+		cl.require.GreaterOrEqual(initial.Number, delta, "cannot rewind before genesis")
+		target := initial.Number - delta
+		cl.log.Info("expecting chain to rewind", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target, "delta", delta)
+		// check rewind more aggressively, in shorter interval
+		return retry.Do0(cl.ctx, attempts, &retry.FixedStrategy{Dur: 500 * time.Millisecond},
+			func() error {
+				head := cl.HeadBlockRef(lvl)
+				if head.Number <= target {
+					cl.log.Info("chain rewinded", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target)
+					return nil
+				}
+				cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target, "current", head.Number)
+				return fmt.Errorf("expected head to rewind: %s", lvl)
+			})
+	}
+}
+
 func (cl *L2CLNode) PeerInfo() *apis.PeerInfo {
 	peerInfo, err := cl.inner.P2PAPI().Self(cl.ctx)
 	cl.require.NoError(err, "failed to get peer info")

op-devstack/dsl/supervisor.go

@@ -16,13 +16,15 @@ import (
 
 type Supervisor struct {
 	commonImpl
-	inner stack.Supervisor
+	inner   stack.Supervisor
+	control stack.ControlPlane
 }
 
-func NewSupervisor(inner stack.Supervisor) *Supervisor {
+func NewSupervisor(inner stack.Supervisor, control stack.ControlPlane) *Supervisor {
 	return &Supervisor{
 		commonImpl: commonFromT(inner.T()),
 		inner:      inner,
+		control:    control,
 	}
 }
 
@@ -141,3 +143,17 @@ func (s *Supervisor) AdvancedUnsafeHead(chainID eth.ChainID, block uint64) {
 func (s *Supervisor) AdvancedSafeHead(chainID eth.ChainID, block uint64, attempts int) {
 	s.AdvancedL2Head(chainID, block, types.CrossSafe, attempts)
 }
+
+func (s *Supervisor) Start() {
+	s.control.SupervisorState(s.inner.ID(), stack.Start)
+}
+
+func (s *Supervisor) Stop() {
+	s.control.SupervisorState(s.inner.ID(), stack.Stop)
+}
+
+func (s *Supervisor) AddManagedL2CL(cl *L2CLNode) {
+	interopEndpoint, secret := cl.inner.InteropRPC()
+	err := s.inner.AdminAPI().AddL2RPC(s.ctx, interopEndpoint, secret)
+	s.require.NoError(err, "failed to connect L2CL to supervisor")
+}