From 0dfb321bc2a5a8102cfeec5def3acc40ac4c2bdd Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Tue, 13 May 2025 22:37:46 +0900 Subject: [PATCH 01/13] DSL for l2 components + supervisor --- op-devstack/dsl/supervisor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/op-devstack/dsl/supervisor.go b/op-devstack/dsl/supervisor.go index 8aaa095acb50b..1fb905d9de4d5 100644 --- a/op-devstack/dsl/supervisor.go +++ b/op-devstack/dsl/supervisor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "time" "github.com/ethereum-optimism/optimism/op-devstack/stack" From b1ddb64840839baaebc0dde532f450c34058b161 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 21:55:51 +0900 Subject: [PATCH 02/13] type safety --- op-devstack/dsl/supervisor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/op-devstack/dsl/supervisor.go b/op-devstack/dsl/supervisor.go index 1fb905d9de4d5..8aaa095acb50b 100644 --- a/op-devstack/dsl/supervisor.go +++ b/op-devstack/dsl/supervisor.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "reflect" "time" "github.com/ethereum-optimism/optimism/op-devstack/stack" From ff738700214ca54e159a07d695bada607b883568 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Wed, 14 May 2025 20:17:03 +0900 Subject: [PATCH 03/13] support multisupervisor system --- op-devstack/presets/interop.go | 38 +++++++++++++++++--- op-devstack/shim/l2_cl.go | 23 +++++++++--- op-devstack/stack/match/second.go | 2 ++ op-devstack/sysgo/system.go | 58 ++++++++++++++++++++++++++++++- 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/op-devstack/presets/interop.go b/op-devstack/presets/interop.go index 36390d9ee0b69..c02867f3edddd 100644 --- a/op-devstack/presets/interop.go +++ b/op-devstack/presets/interop.go @@ -15,8 +15,10 @@ import ( ) type SimpleInterop struct { - Log log.Logger - T devtest.T + Log log.Logger + T devtest.T + system stack.ExtensibleSystem + Supervisor *dsl.Supervisor Sequencer *dsl.Sequencer ControlPlane stack.ControlPlane @@ -72,8 +74,9 @@ func NewSimpleInterop(t devtest.T) *SimpleInterop { out := &SimpleInterop{ Log: t.Logger(), T: t, + system: system, Sequencer: dsl.NewSequencer(system.Sequencer(match.Assume(t, match.FirstSequencer))), - Supervisor: dsl.NewSupervisor(system.Supervisor(match.Assume(t, match.FirstSupervisor))), + Supervisor: dsl.NewSupervisor(system.Supervisor(match.Assume(t, match.FirstSupervisor)), orch.ControlPlane()), ControlPlane: orch.ControlPlane(), L1Network: dsl.NewL1Network(system.L1Network(match.FirstL1Network)), L2ChainA: dsl.NewL2Network(l2A), @@ -131,7 +134,7 @@ func WithRedundantInterop() stack.CommonOption { func NewRedundantInterop(t devtest.T) *RedundantInterop { simpleInterop := NewSimpleInterop(t) orch := Orchestrator() - l2A := simpleInterop.L2ChainA.Escape() + l2A := simpleInterop.system.L2Network(match.Assume(t, match.L2ChainA)) out := &RedundantInterop{ SimpleInterop: *simpleInterop, L2ELA2: dsl.NewL2ELNode(l2A.L2ELNode(match.Assume(t, match.SecondL2EL))), @@ -139,3 +142,30 @@ func NewRedundantInterop(t devtest.T) *RedundantInterop { } return out } + +type MultiSupervisorInterop struct { + RedundancyInterop + + SupervisorSecondary *dsl.Supervisor + + L2ELB2 *dsl.L2ELNode + L2CLB2 *dsl.L2CLNode +} + +func WithMultiSupervisorInterop() stack.CommonOption { + return stack.MakeCommon(sysgo.MultiSupervisorInteropSystem(&sysgo.MultiSupervisorInteropSystemIDs{})) +} + +func NewMultiSupervisorInterop(t devtest.T) *MultiSupervisorInterop { + redundancyInterop := NewRedundancyInterop(t) + orch := Orchestrator() + + l2B := redundancyInterop.system.L2Network(match.Assume(t, match.L2ChainB)) + out := &MultiSupervisorInterop{ + RedundancyInterop: *redundancyInterop, + SupervisorSecondary: dsl.NewSupervisor(redundancyInterop.system.Supervisor(match.Assume(t, match.SecondSupervisor)), orch.ControlPlane()), + L2ELB2: dsl.NewL2ELNode(l2B.L2ELNode(match.Assume(t, match.SecondL2EL))), + L2CLB2: dsl.NewL2CLNode(l2B.L2CLNode(match.Assume(t, match.SecondL2CL)), orch.ControlPlane(), l2B.ChainID()), + } + return out +} diff --git a/op-devstack/shim/l2_cl.go b/op-devstack/shim/l2_cl.go index 3e20df9b024f1..41c7619f17087 100644 --- a/op-devstack/shim/l2_cl.go +++ b/op-devstack/shim/l2_cl.go @@ -4,6 +4,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-service/apis" "github.com/ethereum-optimism/optimism/op-service/client" + "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/locks" "github.com/ethereum-optimism/optimism/op-service/sources" ) @@ -12,6 +13,9 @@ type L2CLNodeConfig struct { CommonConfig ID stack.L2CLNodeID Client client.RPC + + InteropEndpoint string + InteropJwtSecret eth.Bytes32 } type rpcL2CLNode struct { @@ -21,6 +25,9 @@ type rpcL2CLNode struct { rollupClient apis.RollupClient p2pClient apis.P2PClient els locks.RWMap[stack.L2ELNodeID, stack.L2ELNode] + + interopEndpoint string + interopJwtSecret eth.Bytes32 } var _ stack.L2CLNode = (*rpcL2CLNode)(nil) @@ -29,11 +36,13 @@ var _ stack.LinkableL2CLNode = (*rpcL2CLNode)(nil) func NewL2CLNode(cfg L2CLNodeConfig) stack.L2CLNode { cfg.Log = cfg.Log.New("chainID", cfg.ID.ChainID, "id", cfg.ID) return &rpcL2CLNode{ - commonImpl: newCommon(cfg.CommonConfig), - id: cfg.ID, - client: cfg.Client, - rollupClient: sources.NewRollupClient(cfg.Client), - p2pClient: sources.NewP2PClient(cfg.Client), + commonImpl: newCommon(cfg.CommonConfig), + id: cfg.ID, + client: cfg.Client, + rollupClient: sources.NewRollupClient(cfg.Client), + p2pClient: sources.NewP2PClient(cfg.Client), + interopEndpoint: cfg.InteropEndpoint, + interopJwtSecret: cfg.InteropJwtSecret, } } @@ -56,3 +65,7 @@ func (r *rpcL2CLNode) LinkEL(el stack.L2ELNode) { func (r *rpcL2CLNode) ELs() []stack.L2ELNode { return stack.SortL2ELNodes(r.els.Values()) } + +func (r *rpcL2CLNode) InteropRPC() (endpoint string, jwtSecret eth.Bytes32) { + return r.interopEndpoint, r.interopJwtSecret +} diff --git a/op-devstack/stack/match/second.go b/op-devstack/stack/match/second.go index 1fa5e81ec6548..20e6c2a96077c 100644 --- a/op-devstack/stack/match/second.go +++ b/op-devstack/stack/match/second.go @@ -4,3 +4,5 @@ import "github.com/ethereum-optimism/optimism/op-devstack/stack" var SecondL2EL = Second[stack.L2ELNodeID, stack.L2ELNode]() var SecondL2CL = Second[stack.L2CLNodeID, stack.L2CLNode]() + +var SecondSupervisor = Second[stack.SupervisorID, stack.Supervisor]() diff --git a/op-devstack/sysgo/system.go b/op-devstack/sysgo/system.go index 6510b5cdc5aad..b0df32bf8d5aa 100644 --- a/op-devstack/sysgo/system.go +++ b/op-devstack/sysgo/system.go @@ -40,7 +40,7 @@ func NewDefaultInteropSystemIDs(l1ID, l2AID, l2BID eth.ChainID) DefaultInteropSy L1CL: stack.L1CLNodeID{Key: "l1", ChainID: l1ID}, Superchain: "main", // TODO(#15244): hardcoded to match the deployer default ID Cluster: "main", - Supervisor: "dev", + Supervisor: "primary", Sequencer: "dev", L2A: stack.L2NetworkID(l2AID), L2ACL: stack.L2CLNodeID{Key: "sequencer", ChainID: l2AID}, @@ -155,3 +155,59 @@ func RedundantInteropSystem(dest *RedundantInteropSystemIDs) stack.Option[*Orche return opt } + +type MultiSupervisorInteropSystemIDs struct { + DefaultRedundancyInteropSystemIDs + + SupervisorSecondary stack.SupervisorID + + L2B2CL stack.L2CLNodeID + L2B2EL stack.L2ELNodeID +} + +func MultiSupervisorInteropSystem(dest *MultiSupervisorInteropSystemIDs) stack.Option[*Orchestrator] { + l1ID := eth.ChainIDFromUInt64(900) + l2AID := eth.ChainIDFromUInt64(901) + l2BID := eth.ChainIDFromUInt64(902) + ids := MultiSupervisorInteropSystemIDs{ + DefaultRedundancyInteropSystemIDs: DefaultRedundancyInteropSystemIDs{ + DefaultInteropSystemIDs: NewDefaultInteropSystemIDs(l1ID, l2AID, l2BID), + L2A2CL: stack.L2CLNodeID{Key: "verifier", ChainID: l2AID}, + L2A2EL: stack.L2ELNodeID{Key: "verifier", ChainID: l2AID}, + }, + SupervisorSecondary: "secondary", + L2B2CL: stack.L2CLNodeID{Key: "verifier", ChainID: l2BID}, + L2B2EL: stack.L2ELNodeID{Key: "verifier", ChainID: l2BID}, + } + + // start with default interop system + var parentIds DefaultInteropSystemIDs + opt := stack.Combine[*Orchestrator]() + opt.Add(DefaultInteropSystem(&parentIds)) + + // add backup supervisor + opt.Add(WithSupervisor(ids.SupervisorSecondary, ids.Cluster, ids.L1EL)) + + opt.Add(WithL2ELNode(ids.L2A2EL, &ids.SupervisorSecondary)) + opt.Add(WithL2CLNode(ids.L2A2CL, false, ids.L1CL, ids.L1EL, ids.L2A2EL)) + + opt.Add(WithL2ELNode(ids.L2B2EL, &ids.SupervisorSecondary)) + opt.Add(WithL2CLNode(ids.L2B2CL, false, ids.L1CL, ids.L1EL, ids.L2B2EL)) + + // verifier must be also managed or it cannot advance + // we attach verifer L2CL with backup supervisor + opt.Add(WithManagedBySupervisor(ids.L2A2CL, ids.SupervisorSecondary)) + opt.Add(WithManagedBySupervisor(ids.L2B2CL, ids.SupervisorSecondary)) + + // P2P connect L2CL nodes + opt.Add(WithL2CLP2PConnection(ids.L2ACL, ids.L2A2CL)) + opt.Add(WithL2CLP2PConnection(ids.L2BCL, ids.L2B2CL)) + + // Upon evaluation of the option, export the contents we created. + // Ids here are static, but other things may be exported too. + opt.Add(stack.Finally(func(orch *Orchestrator, hook stack.SystemHook) { + *dest = ids + })) + + return opt +} From db026ca70e162049a41ed72241c37e2ebe32bd2b Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Wed, 14 May 2025 20:18:06 +0900 Subject: [PATCH 04/13] expose interopRPC towards DSL --- op-devstack/dsl/l2_cl.go | 37 +++++++++++++++++++++++++++++++++++ op-devstack/dsl/supervisor.go | 20 +++++++++++++++++-- op-devstack/stack/l2_cl.go | 2 ++ op-devstack/sysgo/l2_cl.go | 31 +++++++++++++++++------------ 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/op-devstack/dsl/l2_cl.go b/op-devstack/dsl/l2_cl.go index af6757eacc29b..5f007283ec9ef 100644 --- a/op-devstack/dsl/l2_cl.go +++ b/op-devstack/dsl/l2_cl.go @@ -96,6 +96,23 @@ func (cl *L2CLNode) Advanced(lvl types.SafetyLevel, delta uint64, attempts int) } } +func (cl *L2CLNode) DoesNotAdvance(label string, attempts int) CheckFunc { + return func() error { + initial := cl.HeadBlockRef(label) + cl.log.Info("expecting chain not to advance", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", initial.Number) + for range attempts { + time.Sleep(2 * time.Second) + head := cl.HeadBlockRef(label) + cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", initial.Number, "current", head.Number) + if head.Hash == initial.Hash { + continue + } + return fmt.Errorf("expected head not to advance: %s", label) + } + return nil + } +} + // Reached returns a lambda that checks the L2CL chain head with given safety level reaches the target block number // Composable with other lambdas to wait in parallel func (cl *L2CLNode) Reached(lvl types.SafetyLevel, target uint64, attempts int) CheckFunc { @@ -114,6 +131,26 @@ func (cl *L2CLNode) Reached(lvl types.SafetyLevel, target uint64, attempts int) } } +func (cl *L2CLNode) Rewind(label string, delta uint64, attempts int) CheckFunc { + return func() error { + initial := cl.HeadBlockRef(label) + cl.require.GreaterOrEqual(initial.Number, delta, "cannot rewind before genesis") + target := initial.Number - delta + cl.log.Info("expecting chain to rewind", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", target, "delta", delta) + // check rewind more aggressively, in shorter interval + return retry.Do0(cl.ctx, attempts, &retry.FixedStrategy{Dur: 500 * time.Millisecond}, + func() error { + head := cl.HeadBlockRef(label) + if head.Number <= target { + cl.log.Info("chain rewinded", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", target) + return nil + } + cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", target, "current", head.Number) + return fmt.Errorf("expected head to rewind: %s", label) + }) + } +} + func (cl *L2CLNode) PeerInfo() *apis.PeerInfo { peerInfo, err := cl.inner.P2PAPI().Self(cl.ctx) cl.require.NoError(err, "failed to get peer info") diff --git a/op-devstack/dsl/supervisor.go b/op-devstack/dsl/supervisor.go index 8aaa095acb50b..da7cb62f2dc82 100644 --- a/op-devstack/dsl/supervisor.go +++ b/op-devstack/dsl/supervisor.go @@ -16,13 +16,15 @@ import ( type Supervisor struct { commonImpl - inner stack.Supervisor + inner stack.Supervisor + control stack.ControlPlane } -func NewSupervisor(inner stack.Supervisor) *Supervisor { +func NewSupervisor(inner stack.Supervisor, control stack.ControlPlane) *Supervisor { return &Supervisor{ commonImpl: commonFromT(inner.T()), inner: inner, + control: control, } } @@ -141,3 +143,17 @@ func (s *Supervisor) AdvancedUnsafeHead(chainID eth.ChainID, block uint64) { func (s *Supervisor) AdvancedSafeHead(chainID eth.ChainID, block uint64, attempts int) { s.AdvancedL2Head(chainID, block, types.CrossSafe, attempts) } + +func (s *Supervisor) Start() { + s.control.SupervisorState(s.inner.ID(), stack.Start) +} + +func (s *Supervisor) Stop() { + s.control.SupervisorState(s.inner.ID(), stack.Stop) +} + +func (s *Supervisor) AddManagedL2CL(cl *L2CLNode) { + interopEndpoint, secret := cl.inner.InteropRPC() + err := s.inner.AdminAPI().AddL2RPC(s.ctx, interopEndpoint, secret) + s.require.NoError(err, "failed to connect L2CL to supervisor") +} diff --git a/op-devstack/stack/l2_cl.go b/op-devstack/stack/l2_cl.go index 0173ad2928cc9..0dbedab0f0c8e 100644 --- a/op-devstack/stack/l2_cl.go +++ b/op-devstack/stack/l2_cl.go @@ -2,6 +2,7 @@ package stack import ( "github.com/ethereum-optimism/optimism/op-service/apis" + "github.com/ethereum-optimism/optimism/op-service/eth" ) // L2CLNodeID identifies a L2CLNode by name and chainID, is type-safe, and can be value-copied and used as map key. @@ -46,6 +47,7 @@ type L2CLNode interface { RollupAPI() apis.RollupClient P2PAPI() apis.P2PClient + InteropRPC() (endpoint string, jwtSecret eth.Bytes32) // ELs returns the engine(s) that this L2CLNode is connected to. // This may be empty, if the L2CL is not connected to any. diff --git a/op-devstack/sysgo/l2_cl.go b/op-devstack/sysgo/l2_cl.go index e0cebe9eebdca..23acc8aea4f4f 100644 --- a/op-devstack/sysgo/l2_cl.go +++ b/op-devstack/sysgo/l2_cl.go @@ -23,6 +23,7 @@ import ( nodeSync "github.com/ethereum-optimism/optimism/op-node/rollup/sync" "github.com/ethereum-optimism/optimism/op-service/apis" "github.com/ethereum-optimism/optimism/op-service/client" + "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/oppprof" "github.com/ethereum-optimism/optimism/op-service/retry" "github.com/ethereum-optimism/optimism/op-service/sources" @@ -36,14 +37,15 @@ import ( type L2CLNode struct { mu sync.Mutex - id stack.L2CLNodeID - opNode *opnode.Opnode - userRPC string - interopRPC string - cfg *node.Config - p devtest.P - logger log.Logger - el stack.L2ELNodeID + id stack.L2CLNodeID + opNode *opnode.Opnode + userRPC string + interopEndpoint string + interopJwtSecret eth.Bytes32 + cfg *node.Config + p devtest.P + logger log.Logger + el stack.L2ELNodeID } var _ stack.Lifecycle = (*L2CLNode)(nil) @@ -55,9 +57,11 @@ func (n *L2CLNode) hydrate(system stack.ExtensibleSystem) { system.T().Cleanup(rpcCl.Close) sysL2CL := shim.NewL2CLNode(shim.L2CLNodeConfig{ - CommonConfig: shim.NewCommonConfig(system.T()), - ID: n.id, - Client: rpcCl, + CommonConfig: shim.NewCommonConfig(system.T()), + ID: n.id, + Client: rpcCl, + InteropEndpoint: n.interopEndpoint, + InteropJwtSecret: n.interopJwtSecret, }) l2Net := system.L2Network(stack.L2NetworkID(n.id.ChainID)) l2Net.(stack.ExtensibleL2Network).AddL2CLNode(sysL2CL) @@ -93,8 +97,9 @@ func (n *L2CLNode) Start() { // store endpoints to reuse when restart n.userRPC = opNode.UserRPC().RPC() - interopRPC, _ := opNode.InteropRPC() - n.interopRPC = interopRPC + interopEndpoint, interopJwtSecret := opNode.InteropRPC() + n.interopEndpoint = interopEndpoint + n.interopJwtSecret = interopJwtSecret // for p2p endpoints / node keys, they are already persistent, stored at p2p configs n.rememberPort() From 968c215fb0e12f4f808ebfdf32e4b8a8b569c18d Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Wed, 14 May 2025 20:18:25 +0900 Subject: [PATCH 05/13] Add TestL2CLAheadOfSupervisor with DSL --- .../sync/multisupervisor_interop/init_test.go | 11 ++ .../interop_sync_test.go | 127 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 op-acceptance-tests/tests/interop/sync/multisupervisor_interop/init_test.go create mode 100644 op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go diff --git a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/init_test.go b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/init_test.go new file mode 100644 index 0000000000000..3ebaea92b5001 --- /dev/null +++ b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/init_test.go @@ -0,0 +1,11 @@ +package sync + +import ( + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/presets" +) + +func TestMain(m *testing.M) { + presets.DoMain(m, presets.WithMultiSupervisorInterop()) +} diff --git a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go new file mode 100644 index 0000000000000..9bc834d38933c --- /dev/null +++ b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go @@ -0,0 +1,127 @@ +package sync + +import ( + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/presets" +) + +// TestL2CLAheadOfSupervisor tests the below scenario: +// L2CL ahead of supervisor, aka supervisor needs to reset the L2CL, to reproduce old data. Currently supervisor has only managed mode implemented, so the supervisor will ask the L2CL to reset back. +// To create this out-of-sync scenario, we follow the steps below: +// 0. System setup +// - Two supervisor initialized, each managing two L2CLs per chains. +// - Primary supervisor manages sequencer L2CLs for chain A, B. +// - Secondary supervisor manages verifier L2CLs for chain A, B. +// - Each L2CLs per chain is connected via P2P. +// 1. Make sequencers (L2CL), verifiers (L2CL), and supervisors sync for a few blocks. +// - Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync. +// - Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor. +// - Wait enough for both L2CLs advance unsafe heads. +// 2. Stop Secondary supervisor. +// - Verifiers stops advancing safe heads because there is no supervisor to provide them L1 data. +// - Verifiers advances unsafe head because they still have P2P connection with each sequencers. +// - Wait enough to make sequencers and primary supervisor advance safe head enough. +// 3. Connect verifiers (L2CL) to primary supervisor. +// - Primary supervisor has safe heads synced with sequencers. +// - After connection, verifiers will sync with primary supervisor, matching supervisor safe head view. +// - Secondary supervisor and verifiers becomes out-of-sync with safe heads. +// - Every L2CLs advance safe head. +// 4. Stop primary supervisor. +// - Every L2CL safe heads will stop advancing. +// - For disconnecting every L2CLs from the supervisor. +// 5. Restart primary supervisor and reconnect sequencers (L2CL) to primary supervisor. +// - Sequencers will resume advancing safe heads, but not verifiers. +// 6. Restart Secondary supervisor and reconnect verifiers (L2CL) to Secondary supervisor. +// - Secondary supervisor will compare its safe head knowledge with L2CLs, and find out L2CLs are ahead of the Secondary supervisor. +// - Secondary supervisor asks the verifiers (L2CL) to rewind(reset) back to match Secondary supervisor safe head view. +// - After rewinding(reset), verifier will advance safe heads again because Secondary supervisor gives L1 data to the verifiers. +// - Wait until verifiers advance safe head enough +func TestL2CLAheadOfSupervisor(gt *testing.T) { + t := devtest.SerialT(gt) + + sys := presets.NewMultiSupervisorInterop(t) + logger := sys.Log.With("Test", "TestL2CLAheadOfSupervisor") + require := sys.T.Require() + + delta := uint64(10) + logger.Info("make sure verifiers advances unsafe head", "delta", delta) + dsl.CheckAll(t, + sys.L2CLA.Advance("UnsafeL2", delta, 30), sys.L2CLA2.Advance("UnsafeL2", delta, 30), + sys.L2CLB.Advance("UnsafeL2", delta, 30), sys.L2CLB2.Advance("UnsafeL2", delta, 30), + ) + + safeHeadViewA2 := sys.SupervisorSecondary.SyncView(sys.L2CLA.ChainID(), "CrossSafe") + safeHeadViewB2 := sys.SupervisorSecondary.SyncView(sys.L2CLB.ChainID(), "CrossSafe") + + logger.Info("stop secondary supervisor") + sys.SupervisorSecondary.Stop() + + safeHeadA2 := sys.L2CLA2.SafeL2BlockRef() + safeHeadB2 := sys.L2CLB2.SafeL2BlockRef() + require.Equal(safeHeadViewA2.Hash, safeHeadA2.Hash) + require.Equal(safeHeadViewB2.Hash, safeHeadB2.Hash) + logger.Info("secondary supervisor(stopped) safe head view", "chainA", safeHeadA2, "chainB", safeHeadB2) + + logger.Info("sequencers advances safe heads but not verifiers", "delta", delta) + dsl.CheckAll(t, + // verifier CLs cannot advance their safe head because secondary supervisor is down + sys.L2CLA2.DoesNotAdvance("SafeL2", 30), sys.L2CLB2.DoesNotAdvance("SafeL2", 30), + // sequencer CLs advance + sys.L2CLA.Advance("SafeL2", delta, 30), sys.L2CLB.Advance("SafeL2", delta, 30), + ) + + logger.Info("connect verifier CLs to primary supervisor to advance verifier safe heads") + sys.Supervisor.AddManagedL2CL(sys.L2CLA2) + sys.Supervisor.AddManagedL2CL(sys.L2CLB2) + + target := max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta + logger.Info("every CLs advance safe heads", "delta", delta, "target", target) + dsl.CheckAll(t, + sys.L2CLA.Reach("SafeL2", target, 30), sys.L2CLA2.Reach("SafeL2", target, 30), + sys.L2CLB.Reach("SafeL2", target, 30), sys.L2CLB2.Reach("SafeL2", target, 30), + ) + + logger.Info("stop primary supervisor to disconnect every CL connection") + sys.Supervisor.Stop() + + logger.Info("restart primary supervisor") + sys.Supervisor.Start() + + logger.Info("no CL connected to supervisor so every CL safe head will not advance") + dsl.CheckAll(t, + sys.L2CLA.DoesNotAdvance("SafeL2", 30), sys.L2CLA2.DoesNotAdvance("SafeL2", 30), + sys.L2CLB.DoesNotAdvance("SafeL2", 30), sys.L2CLB2.DoesNotAdvance("SafeL2", 30), + ) + + logger.Info("reconnect sequencer CLs to primary supervisor") + sys.Supervisor.AddManagedL2CL(sys.L2CLA) + sys.Supervisor.AddManagedL2CL(sys.L2CLB) + + logger.Info("restart secondary supervisor") + sys.SupervisorSecondary.Start() + + logger.Info("reconnect verifier CLs to secondary supervisor") + sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLA2) + sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLB2) + + rewind := uint64(3) + logger.Info("check verifier CLs safe head rewinded", "rewind", rewind) + dsl.CheckAll(t, + sys.L2CLA2.Rewind("SafeL2", rewind, 30), + sys.L2CLB2.Rewind("SafeL2", rewind, 30), + ) + + target = max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta + logger.Info("every CLs advance safe heads", "delta", delta, "target", target) + dsl.CheckAll(t, + sys.L2CLA.Reach("SafeL2", target, 30), sys.L2CLA2.Reach("SafeL2", target, 30), + sys.L2CLB.Reach("SafeL2", target, 30), sys.L2CLB2.Reach("SafeL2", target, 30), + ) + + // Make sure each chain did not diverge + require.Equal(sys.L2ELA.BlockRefByNumber(target).Hash, sys.L2ELA2.BlockRefByNumber(target).Hash) + require.Equal(sys.L2ELB.BlockRefByNumber(target).Hash, sys.L2ELB2.BlockRefByNumber(target).Hash) +} From fbcdd6d327232d0eff81f8923f2f1c0e4f384a8f Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 22:35:09 +0900 Subject: [PATCH 06/13] better naming --- op-devstack/presets/interop.go | 6 +++--- op-devstack/sysgo/system.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/op-devstack/presets/interop.go b/op-devstack/presets/interop.go index c02867f3edddd..7a64d348f8bc4 100644 --- a/op-devstack/presets/interop.go +++ b/op-devstack/presets/interop.go @@ -144,7 +144,7 @@ func NewRedundantInterop(t devtest.T) *RedundantInterop { } type MultiSupervisorInterop struct { - RedundancyInterop + RedundantInterop SupervisorSecondary *dsl.Supervisor @@ -157,12 +157,12 @@ func WithMultiSupervisorInterop() stack.CommonOption { } func NewMultiSupervisorInterop(t devtest.T) *MultiSupervisorInterop { - redundancyInterop := NewRedundancyInterop(t) + redundancyInterop := NewRedundantInterop(t) orch := Orchestrator() l2B := redundancyInterop.system.L2Network(match.Assume(t, match.L2ChainB)) out := &MultiSupervisorInterop{ - RedundancyInterop: *redundancyInterop, + RedundantInterop: *redundancyInterop, SupervisorSecondary: dsl.NewSupervisor(redundancyInterop.system.Supervisor(match.Assume(t, match.SecondSupervisor)), orch.ControlPlane()), L2ELB2: dsl.NewL2ELNode(l2B.L2ELNode(match.Assume(t, match.SecondL2EL))), L2CLB2: dsl.NewL2CLNode(l2B.L2CLNode(match.Assume(t, match.SecondL2CL)), orch.ControlPlane(), l2B.ChainID()), diff --git a/op-devstack/sysgo/system.go b/op-devstack/sysgo/system.go index b0df32bf8d5aa..7ff2f8af2abb7 100644 --- a/op-devstack/sysgo/system.go +++ b/op-devstack/sysgo/system.go @@ -157,7 +157,7 @@ func RedundantInteropSystem(dest *RedundantInteropSystemIDs) stack.Option[*Orche } type MultiSupervisorInteropSystemIDs struct { - DefaultRedundancyInteropSystemIDs + RedundantInteropSystemIDs SupervisorSecondary stack.SupervisorID @@ -170,7 +170,7 @@ func MultiSupervisorInteropSystem(dest *MultiSupervisorInteropSystemIDs) stack.O l2AID := eth.ChainIDFromUInt64(901) l2BID := eth.ChainIDFromUInt64(902) ids := MultiSupervisorInteropSystemIDs{ - DefaultRedundancyInteropSystemIDs: DefaultRedundancyInteropSystemIDs{ + RedundantInteropSystemIDs: RedundantInteropSystemIDs{ DefaultInteropSystemIDs: NewDefaultInteropSystemIDs(l1ID, l2AID, l2BID), L2A2CL: stack.L2CLNodeID{Key: "verifier", ChainID: l2AID}, L2A2EL: stack.L2ELNodeID{Key: "verifier", ChainID: l2AID}, From f9c6b2ce8d7f8b3183cf02a5564d8bee8e68570c Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 22:35:57 +0900 Subject: [PATCH 07/13] godoc and better DSL naming and typing --- .../interop_sync_test.go | 29 ++++++++++--------- op-devstack/dsl/l2_cl.go | 28 +++++++++--------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go index 9bc834d38933c..c2ad46279497e 100644 --- a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go +++ b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go @@ -6,6 +6,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" ) // TestL2CLAheadOfSupervisor tests the below scenario: @@ -49,12 +50,12 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { delta := uint64(10) logger.Info("make sure verifiers advances unsafe head", "delta", delta) dsl.CheckAll(t, - sys.L2CLA.Advance("UnsafeL2", delta, 30), sys.L2CLA2.Advance("UnsafeL2", delta, 30), - sys.L2CLB.Advance("UnsafeL2", delta, 30), sys.L2CLB2.Advance("UnsafeL2", delta, 30), + sys.L2CLA.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLA2.Advanced(types.LocalUnsafe, delta, 30), + sys.L2CLB.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB2.Advanced(types.LocalUnsafe, delta, 30), ) - safeHeadViewA2 := sys.SupervisorSecondary.SyncView(sys.L2CLA.ChainID(), "CrossSafe") - safeHeadViewB2 := sys.SupervisorSecondary.SyncView(sys.L2CLB.ChainID(), "CrossSafe") + safeHeadViewA2 := sys.SupervisorSecondary.SafeBlockID(sys.L2CLA.ChainID()) + safeHeadViewB2 := sys.SupervisorSecondary.SafeBlockID(sys.L2CLB.ChainID()) logger.Info("stop secondary supervisor") sys.SupervisorSecondary.Stop() @@ -68,9 +69,9 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { logger.Info("sequencers advances safe heads but not verifiers", "delta", delta) dsl.CheckAll(t, // verifier CLs cannot advance their safe head because secondary supervisor is down - sys.L2CLA2.DoesNotAdvance("SafeL2", 30), sys.L2CLB2.DoesNotAdvance("SafeL2", 30), + sys.L2CLA2.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30), // sequencer CLs advance - sys.L2CLA.Advance("SafeL2", delta, 30), sys.L2CLB.Advance("SafeL2", delta, 30), + sys.L2CLA.Advanced(types.CrossSafe, delta, 30), sys.L2CLB.Advanced(types.CrossSafe, delta, 30), ) logger.Info("connect verifier CLs to primary supervisor to advance verifier safe heads") @@ -80,8 +81,8 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { target := max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta logger.Info("every CLs advance safe heads", "delta", delta, "target", target) dsl.CheckAll(t, - sys.L2CLA.Reach("SafeL2", target, 30), sys.L2CLA2.Reach("SafeL2", target, 30), - sys.L2CLB.Reach("SafeL2", target, 30), sys.L2CLB2.Reach("SafeL2", target, 30), + sys.L2CLA.Reached(types.CrossSafe, target, 30), sys.L2CLA2.Reached(types.CrossSafe, target, 30), + sys.L2CLB.Reached(types.CrossSafe, target, 30), sys.L2CLB2.Reached(types.CrossSafe, target, 30), ) logger.Info("stop primary supervisor to disconnect every CL connection") @@ -92,8 +93,8 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { logger.Info("no CL connected to supervisor so every CL safe head will not advance") dsl.CheckAll(t, - sys.L2CLA.DoesNotAdvance("SafeL2", 30), sys.L2CLA2.DoesNotAdvance("SafeL2", 30), - sys.L2CLB.DoesNotAdvance("SafeL2", 30), sys.L2CLB2.DoesNotAdvance("SafeL2", 30), + sys.L2CLA.NotAdvanced(types.CrossSafe, 30), sys.L2CLA2.NotAdvanced(types.CrossSafe, 30), + sys.L2CLB.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30), ) logger.Info("reconnect sequencer CLs to primary supervisor") @@ -110,15 +111,15 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { rewind := uint64(3) logger.Info("check verifier CLs safe head rewinded", "rewind", rewind) dsl.CheckAll(t, - sys.L2CLA2.Rewind("SafeL2", rewind, 30), - sys.L2CLB2.Rewind("SafeL2", rewind, 30), + sys.L2CLA2.Rewinded(types.CrossSafe, rewind, 30), + sys.L2CLB2.Rewinded(types.CrossSafe, rewind, 30), ) target = max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta logger.Info("every CLs advance safe heads", "delta", delta, "target", target) dsl.CheckAll(t, - sys.L2CLA.Reach("SafeL2", target, 30), sys.L2CLA2.Reach("SafeL2", target, 30), - sys.L2CLB.Reach("SafeL2", target, 30), sys.L2CLB2.Reach("SafeL2", target, 30), + sys.L2CLA.Reached(types.CrossSafe, target, 30), sys.L2CLA2.Reached(types.CrossSafe, target, 30), + sys.L2CLB.Reached(types.CrossSafe, target, 30), sys.L2CLB2.Reached(types.CrossSafe, target, 30), ) // Make sure each chain did not diverge diff --git a/op-devstack/dsl/l2_cl.go b/op-devstack/dsl/l2_cl.go index 5f007283ec9ef..0ab7a61a73f1f 100644 --- a/op-devstack/dsl/l2_cl.go +++ b/op-devstack/dsl/l2_cl.go @@ -96,18 +96,18 @@ func (cl *L2CLNode) Advanced(lvl types.SafetyLevel, delta uint64, attempts int) } } -func (cl *L2CLNode) DoesNotAdvance(label string, attempts int) CheckFunc { +func (cl *L2CLNode) NotAdvanced(lvl types.SafetyLevel, attempts int) CheckFunc { return func() error { - initial := cl.HeadBlockRef(label) - cl.log.Info("expecting chain not to advance", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", initial.Number) + initial := cl.HeadBlockRef(lvl) + cl.log.Info("expecting chain not to advance", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", initial.Number) for range attempts { time.Sleep(2 * time.Second) - head := cl.HeadBlockRef(label) - cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", initial.Number, "current", head.Number) + head := cl.HeadBlockRef(lvl) + cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", initial.Number, "current", head.Number) if head.Hash == initial.Hash { continue } - return fmt.Errorf("expected head not to advance: %s", label) + return fmt.Errorf("expected head not to advance: %s", lvl) } return nil } @@ -131,22 +131,24 @@ func (cl *L2CLNode) Reached(lvl types.SafetyLevel, target uint64, attempts int) } } -func (cl *L2CLNode) Rewind(label string, delta uint64, attempts int) CheckFunc { +// Rewinded returns a lambda that checks the L2CL chain head with given safety level rewinded more than the delta block number +// Composable with other lambdas to wait in parallel +func (cl *L2CLNode) Rewinded(lvl types.SafetyLevel, delta uint64, attempts int) CheckFunc { return func() error { - initial := cl.HeadBlockRef(label) + initial := cl.HeadBlockRef(lvl) cl.require.GreaterOrEqual(initial.Number, delta, "cannot rewind before genesis") target := initial.Number - delta - cl.log.Info("expecting chain to rewind", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", target, "delta", delta) + cl.log.Info("expecting chain to rewind", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target, "delta", delta) // check rewind more aggressively, in shorter interval return retry.Do0(cl.ctx, attempts, &retry.FixedStrategy{Dur: 500 * time.Millisecond}, func() error { - head := cl.HeadBlockRef(label) + head := cl.HeadBlockRef(lvl) if head.Number <= target { - cl.log.Info("chain rewinded", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", target) + cl.log.Info("chain rewinded", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target) return nil } - cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", label, "target", target, "current", head.Number) - return fmt.Errorf("expected head to rewind: %s", label) + cl.log.Info("Chain sync status", "id", cl.inner.ID(), "chain", cl.chainID, "label", lvl, "target", target, "current", head.Number) + return fmt.Errorf("expected head to rewind: %s", lvl) }) } } From 96040164d04edc00cdae3f118c58940151e03cb2 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 22:37:13 +0900 Subject: [PATCH 08/13] prefix with decimal for supervisor ordering --- op-devstack/sysgo/system.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/op-devstack/sysgo/system.go b/op-devstack/sysgo/system.go index 7ff2f8af2abb7..6bd270b2df49c 100644 --- a/op-devstack/sysgo/system.go +++ b/op-devstack/sysgo/system.go @@ -40,7 +40,7 @@ func NewDefaultInteropSystemIDs(l1ID, l2AID, l2BID eth.ChainID) DefaultInteropSy L1CL: stack.L1CLNodeID{Key: "l1", ChainID: l1ID}, Superchain: "main", // TODO(#15244): hardcoded to match the deployer default ID Cluster: "main", - Supervisor: "primary", + Supervisor: "1-primary", // prefix with number for ordering of supervisors Sequencer: "dev", L2A: stack.L2NetworkID(l2AID), L2ACL: stack.L2CLNodeID{Key: "sequencer", ChainID: l2AID}, @@ -175,7 +175,7 @@ func MultiSupervisorInteropSystem(dest *MultiSupervisorInteropSystemIDs) stack.O L2A2CL: stack.L2CLNodeID{Key: "verifier", ChainID: l2AID}, L2A2EL: stack.L2ELNodeID{Key: "verifier", ChainID: l2AID}, }, - SupervisorSecondary: "secondary", + SupervisorSecondary: "2-secondary", // prefix with number for ordering of supervisors L2B2CL: stack.L2CLNodeID{Key: "verifier", ChainID: l2BID}, L2B2EL: stack.L2ELNodeID{Key: "verifier", ChainID: l2BID}, } From 364ff4ae1d352f8b2030ce459c8f738e36076694 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 22:38:57 +0900 Subject: [PATCH 09/13] Capitalize test log msgs --- .../interop_sync_test.go | 28 +++++++++---------- .../redundant_interop/interop_sync_test.go | 20 ++++++------- .../sync/simple_interop/interop_sync_test.go | 12 ++++---- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go index c2ad46279497e..843eb05b17769 100644 --- a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go +++ b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go @@ -48,7 +48,7 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { require := sys.T.Require() delta := uint64(10) - logger.Info("make sure verifiers advances unsafe head", "delta", delta) + logger.Info("Make sure verifiers advances unsafe head", "delta", delta) dsl.CheckAll(t, sys.L2CLA.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLA2.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB2.Advanced(types.LocalUnsafe, delta, 30), @@ -57,16 +57,16 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { safeHeadViewA2 := sys.SupervisorSecondary.SafeBlockID(sys.L2CLA.ChainID()) safeHeadViewB2 := sys.SupervisorSecondary.SafeBlockID(sys.L2CLB.ChainID()) - logger.Info("stop secondary supervisor") + logger.Info("Stop secondary supervisor") sys.SupervisorSecondary.Stop() safeHeadA2 := sys.L2CLA2.SafeL2BlockRef() safeHeadB2 := sys.L2CLB2.SafeL2BlockRef() require.Equal(safeHeadViewA2.Hash, safeHeadA2.Hash) require.Equal(safeHeadViewB2.Hash, safeHeadB2.Hash) - logger.Info("secondary supervisor(stopped) safe head view", "chainA", safeHeadA2, "chainB", safeHeadB2) + logger.Info("Secondary supervisor(stopped) safe head view", "chainA", safeHeadA2, "chainB", safeHeadB2) - logger.Info("sequencers advances safe heads but not verifiers", "delta", delta) + logger.Info("Sequencers advances safe heads but not verifiers", "delta", delta) dsl.CheckAll(t, // verifier CLs cannot advance their safe head because secondary supervisor is down sys.L2CLA2.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30), @@ -74,49 +74,49 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { sys.L2CLA.Advanced(types.CrossSafe, delta, 30), sys.L2CLB.Advanced(types.CrossSafe, delta, 30), ) - logger.Info("connect verifier CLs to primary supervisor to advance verifier safe heads") + logger.Info("Connect verifier CLs to primary supervisor to advance verifier safe heads") sys.Supervisor.AddManagedL2CL(sys.L2CLA2) sys.Supervisor.AddManagedL2CL(sys.L2CLB2) target := max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta - logger.Info("every CLs advance safe heads", "delta", delta, "target", target) + logger.Info("Every CLs advance safe heads", "delta", delta, "target", target) dsl.CheckAll(t, sys.L2CLA.Reached(types.CrossSafe, target, 30), sys.L2CLA2.Reached(types.CrossSafe, target, 30), sys.L2CLB.Reached(types.CrossSafe, target, 30), sys.L2CLB2.Reached(types.CrossSafe, target, 30), ) - logger.Info("stop primary supervisor to disconnect every CL connection") + logger.Info("Stop primary supervisor to disconnect every CL connection") sys.Supervisor.Stop() - logger.Info("restart primary supervisor") + logger.Info("Restart primary supervisor") sys.Supervisor.Start() - logger.Info("no CL connected to supervisor so every CL safe head will not advance") + logger.Info("No CL connected to supervisor so every CL safe head will not advance") dsl.CheckAll(t, sys.L2CLA.NotAdvanced(types.CrossSafe, 30), sys.L2CLA2.NotAdvanced(types.CrossSafe, 30), sys.L2CLB.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30), ) - logger.Info("reconnect sequencer CLs to primary supervisor") + logger.Info("Reconnect sequencer CLs to primary supervisor") sys.Supervisor.AddManagedL2CL(sys.L2CLA) sys.Supervisor.AddManagedL2CL(sys.L2CLB) - logger.Info("restart secondary supervisor") + logger.Info("Restart secondary supervisor") sys.SupervisorSecondary.Start() - logger.Info("reconnect verifier CLs to secondary supervisor") + logger.Info("Reconnect verifier CLs to secondary supervisor") sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLA2) sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLB2) rewind := uint64(3) - logger.Info("check verifier CLs safe head rewinded", "rewind", rewind) + logger.Info("Check verifier CLs safe head rewinded", "rewind", rewind) dsl.CheckAll(t, sys.L2CLA2.Rewinded(types.CrossSafe, rewind, 30), sys.L2CLB2.Rewinded(types.CrossSafe, rewind, 30), ) target = max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta - logger.Info("every CLs advance safe heads", "delta", delta, "target", target) + logger.Info("Every CLs advance safe heads", "delta", delta, "target", target) dsl.CheckAll(t, sys.L2CLA.Reached(types.CrossSafe, target, 30), sys.L2CLA2.Reached(types.CrossSafe, target, 30), sys.L2CLB.Reached(types.CrossSafe, target, 30), sys.L2CLB2.Reached(types.CrossSafe, target, 30), diff --git a/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go index 0004309facd39..f0065a66cf3c6 100644 --- a/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go +++ b/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go @@ -37,19 +37,19 @@ func TestUnsafeChainKnownToL2CL(gt *testing.T) { logger := sys.Log.With("Test", "TestUnsafeChainKnownToL2CL") require := sys.T.Require() - logger.Info("make sure verifier safe head advances") + logger.Info("Make sure verifier safe head advances") dsl.CheckAll(t, sys.L2CLA.Advanced(types.CrossSafe, 5, 30), sys.L2CLA2.Advanced(types.CrossSafe, 5, 30), ) safeA2 := sys.L2ELA2.BlockRefByLabel(eth.Safe) - logger.Info("verifier advanced safe head", "number", safeA2.Number) + logger.Info("Verifier advanced safe head", "number", safeA2.Number) unsafeA2 := sys.L2ELA2.BlockRefByLabel(eth.Unsafe) - logger.Info("verifier advanced unsafe head", "number", unsafeA2.Number) + logger.Info("Verifier advanced unsafe head", "number", unsafeA2.Number) // For making verifier stop advancing unsafe head via P2P - logger.Info("disconnect p2p between L2CLs") + logger.Info("Disconnect p2p between L2CLs") sys.L2CLA.DisconnectPeer(sys.L2CLA2) sys.L2CLA2.DisconnectPeer(sys.L2CLA) @@ -58,28 +58,28 @@ func TestUnsafeChainKnownToL2CL(gt *testing.T) { sys.L2CLA2.Stop() delta := uint64(10) - logger.Info("wait until supervisor reaches safe head", "delta", delta) + logger.Info("Wait until supervisor reaches safe head", "delta", delta) sys.Supervisor.AdvancedSafeHead(sys.L2ChainA.ChainID(), delta, 30) // Restarted verifier will advance its unsafe head by reading L1 but not by P2P - logger.Info("restart verifier") + logger.Info("Restart verifier") sys.L2CLA2.Start() safeA2 = sys.L2ELA2.BlockRefByLabel(eth.Safe) - logger.Info("verifier safe head after restart", "number", safeA2.Number) + logger.Info("Verifier safe head after restart", "number", safeA2.Number) unsafeA2 = sys.L2ELA2.BlockRefByLabel(eth.Unsafe) - logger.Info("verifier unsafe head after restart", "number", unsafeA2.Number) + logger.Info("Verifier unsafe head after restart", "number", unsafeA2.Number) // Make sure there are unsafe blocks to be consolidated: // To check verifier does not have to process blocks since unsafe blocks are already processed require.Greater(unsafeA2.Number, safeA2.Number) - logger.Info("make sure verifier unsafe head was consolidated to safe") + logger.Info("Make sure verifier unsafe head was consolidated to safe") dsl.CheckAll(t, sys.L2CLA2.Reached(types.CrossSafe, unsafeA2.Number, 30)) safeA := sys.L2ELA.BlockRefByLabel(eth.Safe) target := safeA.Number + delta - logger.Info("make sure verifier unsafe head advances due to safe head advances", "target", target, "delta", delta) + logger.Info("Make sure verifier unsafe head advances due to safe head advances", "target", target, "delta", delta) dsl.CheckAll(t, sys.L2CLA2.Reached(types.LocalUnsafe, target, 30)) block := sys.L2ELA2.BlockRefByNumber(unsafeA2.Number) diff --git a/op-acceptance-tests/tests/interop/sync/simple_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/simple_interop/interop_sync_test.go index 7e81b6a845695..1446aae6b9484 100644 --- a/op-acceptance-tests/tests/interop/sync/simple_interop/interop_sync_test.go +++ b/op-acceptance-tests/tests/interop/sync/simple_interop/interop_sync_test.go @@ -16,36 +16,36 @@ func TestL2CLResync(gt *testing.T) { sys := presets.NewSimpleInterop(t) logger := sys.Log.With("Test", "TestL2CLResync") - logger.Info("check unsafe chains are advancing") + logger.Info("Check unsafe chains are advancing") dsl.CheckAll(t, sys.L2ELA.Advance(eth.Unsafe, 5), sys.L2ELB.Advance(eth.Unsafe, 5), ) - logger.Info("stop L2CL nodes") + logger.Info("Stop L2CL nodes") sys.L2CLA.Stop() sys.L2CLB.Stop() - logger.Info("make sure L2ELs does not advance") + logger.Info("Make sure L2ELs does not advance") dsl.CheckAll(t, sys.L2ELA.DoesNotAdvance(eth.Unsafe), sys.L2ELB.DoesNotAdvance(eth.Unsafe), ) - logger.Info("restart L2CL nodes") + logger.Info("Restart L2CL nodes") sys.L2CLA.Start() sys.L2CLB.Start() // L2CL may advance a few blocks without supervisor connection, but eventually it will stop without the connection // we must check that unsafe head is advancing due to reconnection - logger.Info("boot up L2CL nodes") + logger.Info("Boot up L2CL nodes") dsl.CheckAll(t, sys.L2ELA.Advance(eth.Unsafe, 10), sys.L2ELB.Advance(eth.Unsafe, 10), ) // supervisor will attempt to reconnect with L2CLs at this point because L2CL ws endpoint is recovered - logger.Info("check unsafe chains are advancing again") + logger.Info("Check unsafe chains are advancing again") dsl.CheckAll(t, sys.L2ELA.Advance(eth.Unsafe, 10), sys.L2ELB.Advance(eth.Unsafe, 10), From 97739f29ddda2861950a6b3153a396bf9cc65f31 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 23:17:26 +0900 Subject: [PATCH 10/13] rebase fix --- op-devstack/sysgo/system.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-devstack/sysgo/system.go b/op-devstack/sysgo/system.go index 6bd270b2df49c..c6ca56bf4f6d5 100644 --- a/op-devstack/sysgo/system.go +++ b/op-devstack/sysgo/system.go @@ -205,7 +205,7 @@ func MultiSupervisorInteropSystem(dest *MultiSupervisorInteropSystemIDs) stack.O // Upon evaluation of the option, export the contents we created. // Ids here are static, but other things may be exported too. - opt.Add(stack.Finally(func(orch *Orchestrator, hook stack.SystemHook) { + opt.Add(stack.Finally(func(orch *Orchestrator) { *dest = ids })) From be6ad183445ecc828c7bfe14df1279563d332f35 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 23:28:02 +0900 Subject: [PATCH 11/13] More graceful rewind --- .../interop/sync/multisupervisor_interop/interop_sync_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go index 843eb05b17769..bea57107d8bda 100644 --- a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go +++ b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go @@ -111,8 +111,8 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { rewind := uint64(3) logger.Info("Check verifier CLs safe head rewinded", "rewind", rewind) dsl.CheckAll(t, - sys.L2CLA2.Rewinded(types.CrossSafe, rewind, 30), - sys.L2CLB2.Rewinded(types.CrossSafe, rewind, 30), + sys.L2CLA2.Rewinded(types.CrossSafe, rewind, 60), + sys.L2CLB2.Rewinded(types.CrossSafe, rewind, 60), ) target = max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta From 415e359607887371b6a96e799192c5fdb0d42742 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Thu, 15 May 2025 23:51:30 +0900 Subject: [PATCH 12/13] spread out godoc scenario in test --- .../interop_sync_test.go | 53 ++++++++----------- .../redundant_interop/interop_sync_test.go | 32 ++++------- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go index bea57107d8bda..b2bb8ce7da40c 100644 --- a/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go +++ b/op-acceptance-tests/tests/interop/sync/multisupervisor_interop/interop_sync_test.go @@ -11,42 +11,20 @@ import ( // TestL2CLAheadOfSupervisor tests the below scenario: // L2CL ahead of supervisor, aka supervisor needs to reset the L2CL, to reproduce old data. Currently supervisor has only managed mode implemented, so the supervisor will ask the L2CL to reset back. -// To create this out-of-sync scenario, we follow the steps below: -// 0. System setup -// - Two supervisor initialized, each managing two L2CLs per chains. -// - Primary supervisor manages sequencer L2CLs for chain A, B. -// - Secondary supervisor manages verifier L2CLs for chain A, B. -// - Each L2CLs per chain is connected via P2P. -// 1. Make sequencers (L2CL), verifiers (L2CL), and supervisors sync for a few blocks. -// - Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync. -// - Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor. -// - Wait enough for both L2CLs advance unsafe heads. -// 2. Stop Secondary supervisor. -// - Verifiers stops advancing safe heads because there is no supervisor to provide them L1 data. -// - Verifiers advances unsafe head because they still have P2P connection with each sequencers. -// - Wait enough to make sequencers and primary supervisor advance safe head enough. -// 3. Connect verifiers (L2CL) to primary supervisor. -// - Primary supervisor has safe heads synced with sequencers. -// - After connection, verifiers will sync with primary supervisor, matching supervisor safe head view. -// - Secondary supervisor and verifiers becomes out-of-sync with safe heads. -// - Every L2CLs advance safe head. -// 4. Stop primary supervisor. -// - Every L2CL safe heads will stop advancing. -// - For disconnecting every L2CLs from the supervisor. -// 5. Restart primary supervisor and reconnect sequencers (L2CL) to primary supervisor. -// - Sequencers will resume advancing safe heads, but not verifiers. -// 6. Restart Secondary supervisor and reconnect verifiers (L2CL) to Secondary supervisor. -// - Secondary supervisor will compare its safe head knowledge with L2CLs, and find out L2CLs are ahead of the Secondary supervisor. -// - Secondary supervisor asks the verifiers (L2CL) to rewind(reset) back to match Secondary supervisor safe head view. -// - After rewinding(reset), verifier will advance safe heads again because Secondary supervisor gives L1 data to the verifiers. -// - Wait until verifiers advance safe head enough func TestL2CLAheadOfSupervisor(gt *testing.T) { t := devtest.SerialT(gt) + // Two supervisor initialized, each managing two L2CLs per chains. + // Primary supervisor manages sequencer L2CLs for chain A, B. + // Secondary supervisor manages verifier L2CLs for chain A, B. + // Each L2CLs per chain is connected via P2P. sys := presets.NewMultiSupervisorInterop(t) logger := sys.Log.With("Test", "TestL2CLAheadOfSupervisor") require := sys.T.Require() + // Make sequencers (L2CL), verifiers (L2CL), and supervisors sync for a few blocks. + // Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync. + // Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor. delta := uint64(10) logger.Info("Make sure verifiers advances unsafe head", "delta", delta) dsl.CheckAll(t, @@ -66,18 +44,26 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { require.Equal(safeHeadViewB2.Hash, safeHeadB2.Hash) logger.Info("Secondary supervisor(stopped) safe head view", "chainA", safeHeadA2, "chainB", safeHeadB2) + // Wait enough to make sequencers and primary supervisor advance safe head enough. logger.Info("Sequencers advances safe heads but not verifiers", "delta", delta) dsl.CheckAll(t, - // verifier CLs cannot advance their safe head because secondary supervisor is down + // verifier CLs cannot advance their safe head because secondary supervisor is down, no supervisor to provide them L1 data. sys.L2CLA2.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30), - // sequencer CLs advance + // sequencer CLs advance their safe heads sys.L2CLA.Advanced(types.CrossSafe, delta, 30), sys.L2CLB.Advanced(types.CrossSafe, delta, 30), + // All the L2CLs advance their unsafe heads + // Verifiers advances unsafe head because they still have P2P connection with each sequencers + sys.L2CLA.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB.Advanced(types.LocalUnsafe, delta, 30), + sys.L2CLA2.Advanced(types.LocalUnsafe, delta, 30), sys.L2CLB2.Advanced(types.LocalUnsafe, delta, 30), ) + // Primary supervisor has safe heads synced with sequencers. + // After connection, verifiers will sync with primary supervisor, matching supervisor safe head view. logger.Info("Connect verifier CLs to primary supervisor to advance verifier safe heads") sys.Supervisor.AddManagedL2CL(sys.L2CLA2) sys.Supervisor.AddManagedL2CL(sys.L2CLB2) + // Secondary supervisor and verifiers becomes out-of-sync with safe heads. target := max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta logger.Info("Every CLs advance safe heads", "delta", delta, "target", target) dsl.CheckAll(t, @@ -97,6 +83,7 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { sys.L2CLB.NotAdvanced(types.CrossSafe, 30), sys.L2CLB2.NotAdvanced(types.CrossSafe, 30), ) + // Sequencers will resume advancing safe heads, but not verifiers. logger.Info("Reconnect sequencer CLs to primary supervisor") sys.Supervisor.AddManagedL2CL(sys.L2CLA) sys.Supervisor.AddManagedL2CL(sys.L2CLB) @@ -108,6 +95,8 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLA2) sys.SupervisorSecondary.AddManagedL2CL(sys.L2CLB2) + // Secondary supervisor will compare its safe head knowledge with L2CLs, and find out L2CLs are ahead of the Secondary supervisor. + // Secondary supervisor asks the verifiers (L2CL) to rewind(reset) back to match Secondary supervisor safe head view. rewind := uint64(3) logger.Info("Check verifier CLs safe head rewinded", "rewind", rewind) dsl.CheckAll(t, @@ -115,6 +104,8 @@ func TestL2CLAheadOfSupervisor(gt *testing.T) { sys.L2CLB2.Rewinded(types.CrossSafe, rewind, 60), ) + // After rewinding(reset), verifier will advance safe heads again because Secondary supervisor gives L1 data to the verifiers. + // Wait until verifiers advance safe head enough target = max(sys.L2CLA.SafeL2BlockRef().Number, sys.L2CLB.SafeL2BlockRef().Number) + delta logger.Info("Every CLs advance safe heads", "delta", delta, "target", target) dsl.CheckAll(t, diff --git a/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go b/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go index f0065a66cf3c6..395479b4ea3b4 100644 --- a/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go +++ b/op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go @@ -12,27 +12,11 @@ import ( // TestUnsafeChainKnownToL2CL tests the below scenario: // supervisor cross-safe ahead of L2CL cross-safe, aka L2CL can "skip" forward to match safety of supervisor. -// To create this out-of-sync scenario, we follow the steps below: -// 1. Make sequencer (L2CL), verifier (L2CL), and supervisor sync for a few blocks. -// - Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync. -// - Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor. -// - Wait enough for both L2CLs advance safe heads. -// 2. Disconnect the P2P connection between the sequencer and verifier. -// - The verifier will not receive unsafe heads via P2P, and can only update unsafe heads matching with safe heads by reading L1 batches. -// - The verifier safe head will lag behind or match the sequencer and supervisor because all three components share the same L1 view. -// 3. Stop verifier L2CL -// - The verifier will not be able to advance unsafe head and safe head. -// - The sequencer will advance unsafe head and safe head, as well as synced with supervisor. -// 4. Wait until sequencer and supervisor diverged enough from the verifier. -// - To make the verifier held unsafe blocks which are already viewed as safe by sequencer and supervisor, we wait. -// - Wait until supervisor viewed safe head number is large enough than the stopped verifier's safe head view. -// 5. Restart the verifier. -// - The verifier will not sync via P2P but only able to advance unsafe and safe heads by reading L1 batches. -// - The verifier will quickly catch up with the sequencer safe head as well as the supervisor. -// - The verifier will "skip" processing already known unsafe blocks, and consolidate them into safe blocks. func TestUnsafeChainKnownToL2CL(gt *testing.T) { t := devtest.SerialT(gt) + // Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync. + // Both L2CLs are in managed mode, digesting L1 blocks from the supervisor and reporting unsafe and safe blocks back to the supervisor. sys := presets.NewRedundantInterop(t) logger := sys.Log.With("Test", "TestUnsafeChainKnownToL2CL") require := sys.T.Require() @@ -48,20 +32,24 @@ func TestUnsafeChainKnownToL2CL(gt *testing.T) { unsafeA2 := sys.L2ELA2.BlockRefByLabel(eth.Unsafe) logger.Info("Verifier advanced unsafe head", "number", unsafeA2.Number) - // For making verifier stop advancing unsafe head via P2P + // The verifier stops advancing unsafe head because it will not receive unsafe heads via P2P, and can only update unsafe heads matching with safe heads by reading L1 batches, + // The verifier safe head will lag behind or match the sequencer and supervisor because all three components share the same L1 view. logger.Info("Disconnect p2p between L2CLs") sys.L2CLA.DisconnectPeer(sys.L2CLA2) sys.L2CLA2.DisconnectPeer(sys.L2CLA) - // For making verifer not sync at all + // For making verifer not sync at all, both unsafe haead and safe head + // The sequencer will advance unsafe head and safe head, as well as synced with supervisor. logger.Info("stop verifier") sys.L2CLA2.Stop() + // Wait until sequencer and supervisor diverged enough from the verifier. + // To make the verifier held unsafe blocks are already as safe by sequencer and supervisor, we wait. delta := uint64(10) logger.Info("Wait until supervisor reaches safe head", "delta", delta) sys.Supervisor.AdvancedSafeHead(sys.L2ChainA.ChainID(), delta, 30) - // Restarted verifier will advance its unsafe head by reading L1 but not by P2P + // Restarted verifier will advance its unsafe head and safe head by reading L1 but not by P2P logger.Info("Restart verifier") sys.L2CLA2.Start() @@ -74,6 +62,8 @@ func TestUnsafeChainKnownToL2CL(gt *testing.T) { // To check verifier does not have to process blocks since unsafe blocks are already processed require.Greater(unsafeA2.Number, safeA2.Number) + // The verifier will quickly catch up with the sequencer safe head as well as the supervisor. + // The verifier will "skip" processing already known unsafe blocks, and consolidate them into safe blocks. logger.Info("Make sure verifier unsafe head was consolidated to safe") dsl.CheckAll(t, sys.L2CLA2.Reached(types.CrossSafe, unsafeA2.Number, 30)) From a60d279cfd756b16da1429bc8fde6be4db4de7c7 Mon Sep 17 00:00:00 2001 From: Park Changwan Date: Fri, 16 May 2025 00:36:32 +0900 Subject: [PATCH 13/13] Add comment about shim L2CL interop fields --- op-devstack/shim/l2_cl.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/op-devstack/shim/l2_cl.go b/op-devstack/shim/l2_cl.go index 41c7619f17087..89eaef37105b1 100644 --- a/op-devstack/shim/l2_cl.go +++ b/op-devstack/shim/l2_cl.go @@ -26,6 +26,9 @@ type rpcL2CLNode struct { p2pClient apis.P2PClient els locks.RWMap[stack.L2ELNodeID, stack.L2ELNode] + // Store interop ws endpoints and secrets to provide to the supervisor, + // when reconnection happens using the supervisor's admin_addL2RPC method. + // These fields are not intended for manual dial-in or initializing client.RPC interopEndpoint string interopJwtSecret eth.Bytes32 }