diff --git a/op-acceptance-tests/tests/interop/reorgs/init_exec_msg_test.go b/op-acceptance-tests/tests/interop/reorgs/init_exec_msg_test.go new file mode 100644 index 0000000000000..757c854e07256 --- /dev/null +++ b/op-acceptance-tests/tests/interop/reorgs/init_exec_msg_test.go @@ -0,0 +1,349 @@ +package reorgs + +import ( + "math/rand" + "strings" + "testing" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/ethereum-optimism/optimism/devnet-sdk/contracts/bindings" + "github.com/ethereum-optimism/optimism/devnet-sdk/contracts/constants" + "github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop" + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/stack/match" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/retry" + "github.com/ethereum-optimism/optimism/op-service/txintent" + "github.com/ethereum-optimism/optimism/op-service/txplan" + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/stretchr/testify/require" +) + +func TestReorgInitExecMsg(gt *testing.T) { + t := devtest.SerialT(gt) + ctx := t.Ctx() + + sys := SimpleInterop(t) + l := sys.Log + + ia := sys.Sequencer.Escape().IndividualAPI(sys.L2ChainA.ChainID()) + + // three EOAs for triggering the init and exec interop txs, as well as a simple transfer tx + var alice, bob, cathrine *dsl.EOA + { + // alice is on chain A + pk, err := crypto.GenerateKey() + require.NoError(t, err) + alice = dsl.NewEOA(dsl.NewKey(t, pk), sys.L2ELA) + sys.FaucetA.Fund(alice.Address(), eth.ThousandEther) + + // bob is on chain B + pk, err = crypto.GenerateKey() + require.NoError(t, err) + bob = dsl.NewEOA(dsl.NewKey(t, pk), sys.L2ELB) + sys.FaucetB.Fund(bob.Address(), eth.ThousandEther) + + // cathrine is on chain A + pk, err = crypto.GenerateKey() + require.NoError(t, err) + cathrine = dsl.NewEOA(dsl.NewKey(t, pk), sys.L2ELA) + sys.FaucetA.Fund(cathrine.Address(), eth.ThousandEther) + + l.Info("alice", "address", alice.Address()) + l.Info("bob", "address", bob.Address()) + l.Info("cathrine", "address", cathrine.Address()) + } + + sys.L1Network.WaitForBlock() + sys.L2ChainA.WaitForBlock() + + // stop batchers on chain A and on chain B + { + err := retry.Do0(ctx, 3, retry.Exponential(), func() error { + err := sys.L2BatcherA.Escape().ActivityAPI().StopBatcher(ctx) + if err != nil && strings.Contains(err.Error(), "batcher is not running") { + return nil + } + return err + }) + require.NoError(t, err, "Expected to be able to call StopBatcher API on chain A, but got error") + + err = retry.Do0(ctx, 3, retry.Exponential(), func() error { + err := sys.L2BatcherB.Escape().ActivityAPI().StopBatcher(ctx) + if err != nil && strings.Contains(err.Error(), "batcher is not running") { + return nil + } + return err + }) + require.NoError(t, err, "Expected to be able to call StopBatcher API on chain B, but got error") + } + + // deploy event logger on chain A + var eventLoggerAddress common.Address + { + tx := txplan.NewPlannedTx(txplan.Combine( + alice.Plan(), + txplan.WithData(common.FromHex(bindings.EventloggerBin)), + )) + res, err := tx.Included.Eval(ctx) + require.NoError(t, err) + + eventLoggerAddress = res.ContractAddress + l.Info("deployed EventLogger", "chainID", tx.ChainID.Value(), "address", eventLoggerAddress) + } + + sys.L1Network.WaitForBlock() + + var initTrigger *txintent.InitTrigger + // prepare init trigger (i.e. what logs to emit on chain A) + { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + nTopics := 3 + lenData := 10 + initTrigger = interop.RandomInitTrigger(rng, eventLoggerAddress, nTopics, lenData) + + l.Info("created init trigger", "address", eventLoggerAddress, "topics", nTopics, "lenData", lenData) + } + + // wait for chain B to catch up to chain A if necessary + sys.L2ChainB.CatchUpTo(sys.L2ChainA) + + var initTx *txintent.IntentTx[*txintent.InitTrigger, *txintent.InteropOutput] + var initReceipt *types.Receipt + // prepare and include initiating message on chain A + { + initTx = txintent.NewIntent[*txintent.InitTrigger, *txintent.InteropOutput](alice.Plan()) + initTx.Content.Set(initTrigger) + var err error + initReceipt, err = initTx.PlannedTx.Included.Eval(ctx) + require.NoError(t, err) + + l.Info("initiating message included", "chain", sys.L2ChainA.ChainID(), "block_number", initReceipt.BlockNumber, "block_hash", initReceipt.BlockHash, "now", time.Now().Unix()) + } + + // stop sequencer on chain A so that we later force a reorg/removal of the init msg + { + unsafeHead, err := sys.L2CLNodeA.Escape().RollupAPI().StopSequencer(ctx) + require.NoError(t, err, "expected to be able to call StopSequencer API, but got error") + + // wait for the sequencer to become inactive + var active bool + err = wait.For(ctx, 1*time.Second, func() (bool, error) { + active, err = sys.L2CLNodeA.Escape().RollupAPI().SequencerActive(ctx) + return !active, err + }) + require.NoError(t, err, "expected to be able to call SequencerActive API, and wait for inactive state for sequencer, but got error") + + l.Info("rollup node sequencer status", "chain", sys.L2ChainA.ChainID(), "active", active, "unsafeHead", unsafeHead) + } + + // at least one block between the init tx on chain A and the exec tx on chain B + sys.L2ChainB.WaitForBlock() + + var execTx *txintent.IntentTx[*txintent.ExecTrigger, *txintent.InteropOutput] + var execReceipt *types.Receipt + // prepare and include executing message on chain B + { + execTx = txintent.NewIntent[*txintent.ExecTrigger, *txintent.InteropOutput](bob.Plan()) + execTx.Content.DependOn(&initTx.Result) + // single event in tx so index is 0. ExecuteIndexed returns a lambda to transform InteropOutput to a new ExecTrigger + execTx.Content.Fn(txintent.ExecuteIndexed(constants.CrossL2Inbox, &initTx.Result, 0)) + var err error + execReceipt, err = execTx.PlannedTx.Included.Eval(ctx) + require.NoError(t, err) + require.Equal(t, 1, len(execReceipt.Logs)) + + l.Info("executing message included", "chain", sys.L2ChainB.ChainID(), "block_number", execReceipt.BlockNumber, "block_hash", execReceipt.BlockHash, "now", time.Now().Unix()) + } + + // record divergence block numbers and original refs for future validation checks + var divergenceBlockNumber_A, divergenceBlockNumber_B uint64 + var originalRef_A, originalRef_B eth.L2BlockRef + + // sequence a conflicting block with a simple transfer tx, based on the parent of the parent of the unsafe head + { + var err error + divergenceBlockNumber_B = execReceipt.BlockNumber.Uint64() + originalRef_B, err = sys.L2ELB.Escape().L2EthClient().L2BlockRefByHash(ctx, execReceipt.BlockHash) + require.NoError(t, err, "Expected to be able to call L2BlockRefByHash API, but got error") + + headToReorgA := initReceipt.BlockHash + headToReorgARef, err := sys.L2ELA.Escape().L2EthClient().L2BlockRefByHash(ctx, headToReorgA) + require.NoError(t, err, "Expected to be able to call L2BlockRefByHash API, but got error") + + divergenceBlockNumber_A = headToReorgARef.Number + originalRef_A = headToReorgARef + + parentOfHeadToReorgA := headToReorgARef.ParentID() + parentsL1Origin, err := sys.L2ELA.Escape().L2EthClient().L2BlockRefByHash(ctx, parentOfHeadToReorgA.Hash) + require.NoError(t, err, "Expected to be able to call L2BlockRefByHash API, but got error") + + nextL1Origin := parentsL1Origin.L1Origin.Number + 1 + l1Origin, err := sys.L1Network.Escape().L1ELNode(match.FirstL1EL).EthClient().InfoByNumber(ctx, nextL1Origin) + require.NoError(t, err, "Expected to get block number %v from L1 execution client", nextL1Origin) + l1OriginHash := l1Origin.Hash() + + l.Info("Sequencing a conflicting block", "chain", sys.L2ChainA.ChainID(), "newL1Origin", eth.ToBlockID(l1Origin), "headToReorgA", headToReorgARef, "parent", parentOfHeadToReorgA, "parent_l1_origin", parentsL1Origin.L1Origin) + + err = ia.New(ctx, seqtypes.BuildOpts{ + Parent: parentOfHeadToReorgA.Hash, + L1Origin: &l1OriginHash, + }) + require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") + + // include simple transfer tx in opened block + { + to := cathrine.PlanTransfer(alice.Address(), eth.OneEther) + opt := txplan.Combine(to) + ptx := txplan.NewPlannedTx(opt) + signed_tx, err := ptx.Signed.Eval(ctx) + require.NoError(t, err, "Expected to be able to evaluate a planned transaction on op-test-sequencer, but got error") + txdata, err := signed_tx.MarshalBinary() + require.NoError(t, err, "Expected to be able to marshal a signed transaction on op-test-sequencer, but got error") + + err = ia.IncludeTx(ctx, txdata) + require.NoError(t, err, "Expected to be able to include a signed transaction on op-test-sequencer, but got error") + } + + err = ia.Next(ctx) + require.NoError(t, err, "Expected to be able to call Next() after New() on op-test-sequencer, but got error") + } + + // sequence a second block with op-test-sequencer + { + currentUnsafeRef := sys.L2ChainA.UnsafeHeadRef() + l.Info("Current unsafe ref", "unsafeHead", currentUnsafeRef) + err := ia.New(ctx, seqtypes.BuildOpts{ + Parent: currentUnsafeRef.Hash, + L1Origin: nil, + }) + require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") + time.Sleep(2 * time.Second) + + err = ia.Next(ctx) + require.NoError(t, err, "Expected to be able to call Next() after New() on op-test-sequencer, but got error") + time.Sleep(2 * time.Second) + } + + // continue sequencing with op-node + { + newUnsafeHeadRef := sys.L2ChainA.UnsafeHeadRef() + l.Info("Continue sequencing with consensus node (op-node)", "unsafeHead", newUnsafeHeadRef) + + err := sys.L2CLNodeA.Escape().RollupAPI().StartSequencer(ctx, newUnsafeHeadRef.Hash) + require.NoError(t, err, "Expected to be able to start sequencer on rollup node") + + // wait for the sequencer to become active + var active bool + err = wait.For(ctx, 1*time.Second, func() (bool, error) { + active, err = sys.L2CLNodeA.Escape().RollupAPI().SequencerActive(ctx) + return active, err + }) + require.NoError(t, err, "Expected to be able to call SequencerActive API, and wait for an active state for sequencer, but got error") + + l.Info("Rollup node sequencer", "active", active) + } + + // start batchers on chain A and on chain B + { + err := retry.Do0(ctx, 3, retry.Exponential(), func() error { + return sys.L2BatcherA.Escape().ActivityAPI().StartBatcher(ctx) + }) + require.NoError(t, err, "Expected to be able to call StartBatcher API on chain A, but got error") + + err = retry.Do0(ctx, 3, retry.Exponential(), func() error { + return sys.L2BatcherB.Escape().ActivityAPI().StartBatcher(ctx) + }) + require.NoError(t, err, "Expected to be able to call StartBatcher API on chain B, but got error") + } + + // confirm reorg on chain A + { + reorgedRef_A, err := sys.L2ELA.Escape().EthClient().BlockRefByNumber(ctx, divergenceBlockNumber_A) + require.NoError(t, err, "Expected to be able to call BlockRefByNumber API, but got error") + + l.Info("Reorged chain A on divergence block number (prior the reorg)", "chain", sys.L2ChainA.ChainID(), "number", divergenceBlockNumber_A, "head", originalRef_A.Hash, "parent", originalRef_A.ParentID().Hash) + l.Info("Reorged chain A on divergence block number (after the reorg)", "chain", sys.L2ChainA.ChainID(), "number", divergenceBlockNumber_A, "head", reorgedRef_A.Hash, "parent", reorgedRef_A.ParentID().Hash) + require.NotEqual(t, originalRef_A.Hash, reorgedRef_A.Hash, "Expected to get different heads on divergence block A number, but got the same hash, so no reorg happened") + require.Equal(t, originalRef_A.ParentID().Hash, reorgedRef_A.ParentHash, "Expected to get same parent hashes on divergence block A number, but got different hashes") + } + + // wait for reorg on chain B + require.Eventually(t, func() bool { + reorgedRef_B, err := sys.L2ELB.Escape().EthClient().BlockRefByNumber(ctx, divergenceBlockNumber_B) + if err != nil { + if strings.Contains(err.Error(), "not found") { // reorg is happening wait a bit longer + l.Info("Supervisor still hasn't reorged chain B", "error", err) + return false + } + require.NoError(t, err, "Expected to be able to call BlockRefByNumber API, but got error") + } + + if originalRef_B.Hash.Cmp(reorgedRef_B.Hash) == 0 { // want not equal + l.Info("Supervisor still hasn't reorged chain B", "ref", originalRef_B) + return false + } + + l.Info("Reorged chain B on divergence block number (prior the reorg)", "chain", sys.L2ChainB.ChainID(), "number", divergenceBlockNumber_B, "head", originalRef_B.Hash, "parent", originalRef_B.ParentID().Hash) + l.Info("Reorged chain B on divergence block number (after the reorg)", "chain", sys.L2ChainB.ChainID(), "number", divergenceBlockNumber_B, "head", reorgedRef_B.Hash, "parent", reorgedRef_B.ParentID().Hash) + return true + }, 180*time.Second, 10*time.Second, "No reorg happened on chain B. Should have been triggered by the supervisor.") + + // executing tx should eventually be no longer confirmed on chain B + require.Eventually(t, func() bool { + receipt, err := sys.L2ELB.Escape().EthClient().TransactionReceipt(ctx, execReceipt.TxHash) + if err == nil || err.Error() != "not found" { // want to get "not found" error + return false + } + if receipt != nil { // want to get nil receipt + return false + } + return true + }, 60*time.Second, 3*time.Second, "Expected for the executing tx to be removed from chain B") + + err := wait.For(ctx, 5*time.Second, func() (bool, error) { + safeL2Head_supervisor_A := sys.Supervisor.SafeBlockID(sys.L2ChainA.ChainID()).Hash + safeL2Head_supervisor_B := sys.Supervisor.SafeBlockID(sys.L2ChainB.ChainID()).Hash + safeL2Head_sequencer_A := sys.L2CLNodeA.SafeL2BlockRef() + safeL2Head_sequencer_B := sys.L2CLNodeB.SafeL2BlockRef() + + if safeL2Head_sequencer_A.Number < divergenceBlockNumber_A { + l.Info("Safe ref number is still behind divergence block A number", "divergence", divergenceBlockNumber_A, "safe", safeL2Head_sequencer_A.Number) + return false, nil + } + + if safeL2Head_sequencer_B.Number < divergenceBlockNumber_B { + l.Info("Safe ref number is still behind divergence block B number", "divergence", divergenceBlockNumber_B, "safe", safeL2Head_sequencer_B.Number) + return false, nil + } + + if safeL2Head_sequencer_A.Hash.Cmp(safeL2Head_supervisor_A) != 0 { + l.Info("Safe ref still not the same on supervisor and sequencer A", "supervisor", safeL2Head_supervisor_A, "sequencer", safeL2Head_sequencer_A.Hash) + return false, nil + } + + if safeL2Head_sequencer_B.Hash.Cmp(safeL2Head_supervisor_B) != 0 { + l.Info("Safe ref still not the same on supervisor and sequencer B", "supervisor", safeL2Head_supervisor_B, "sequencer", safeL2Head_sequencer_B.Hash) + return false, nil + } + + l.Info("Safe ref the same across supervisor and sequencers", + "supervisor_A", safeL2Head_supervisor_A, + "sequencer_A", safeL2Head_sequencer_A.Hash, + "supervisor_B", safeL2Head_supervisor_B, + "sequencer_B", safeL2Head_sequencer_B.Hash) + + return true, nil + }) + require.NoError(t, err, "Expected to get same safe ref on both supervisor and sequencer eventually") + + sys.L2ChainA.PrintChain() + sys.L2ChainB.PrintChain() + spew.Dump(sys.Supervisor.FetchSyncStatus()) +} diff --git a/op-acceptance-tests/tests/interop/reorgs/interop_reorg_test.go b/op-acceptance-tests/tests/interop/reorgs/unsafe_head_test.go similarity index 60% rename from op-acceptance-tests/tests/interop/reorgs/interop_reorg_test.go rename to op-acceptance-tests/tests/interop/reorgs/unsafe_head_test.go index 64a8c9585c3a4..311df4530f5a8 100644 --- a/op-acceptance-tests/tests/interop/reorgs/interop_reorg_test.go +++ b/op-acceptance-tests/tests/interop/reorgs/unsafe_head_test.go @@ -1,17 +1,17 @@ package reorgs import ( + "strings" "testing" "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/presets" - "github.com/ethereum-optimism/optimism/op-devstack/stack/match" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/retry" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" - "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" ) @@ -33,18 +33,22 @@ func TestReorgUnsafeHead(gt *testing.T) { ia := sys.Sequencer.Escape().IndividualAPI(sys.L2ChainA.ChainID()) - l.Info("Stopping batcher") - err := sys.L2BatcherA.Escape().ActivityAPI().StopBatcher(ctx) - require.NoError(t, err, "Expected to be able to call StopBatcher API, but got error") + // stop batcher + { + err := retry.Do0(ctx, 3, retry.Exponential(), func() error { + err := sys.L2BatcherA.Escape().ActivityAPI().StopBatcher(ctx) + if err != nil && strings.Contains(err.Error(), "batcher is not running") { + return nil + } + return err + }) + require.NoError(t, err, "Expected to be able to call StopBatcher API, but got error") + } - // two eoas for a sample transfer tx used later in a conflicting block + // two EOAs for a sample transfer tx used later in a conflicting block alice := sys.FunderA.NewFundedEOA(eth.ThousandEther) bob := sys.Wallet.NewEOA(sys.L2ELA) - active, err := sys.L2CLNodeA.Escape().RollupAPI().SequencerActive(ctx) - require.NoError(t, err, "Expected to be able to call SequencerActive API, but got error") - l.Info("Rollup node sequencer status", "active", active) - sys.L1Network.WaitForBlock() sys.L2ChainA.WaitForBlock() @@ -56,6 +60,7 @@ func TestReorgUnsafeHead(gt *testing.T) { require.NoError(t, err, "Expected to be able to call StopSequencer API, but got error") // wait for the sequencer to become inactive + var active bool err = wait.For(ctx, 1*time.Second, func() (bool, error) { active, err = sys.L2CLNodeA.Escape().RollupAPI().SequencerActive(ctx) return !active, err @@ -64,8 +69,8 @@ func TestReorgUnsafeHead(gt *testing.T) { l.Info("Rollup node sequencer status", "active", active, "unsafeHead", unsafeHead) - var divergenceBlockNumber uint64 - var originalRef eth.L2BlockRef + var divergenceBlockNumber_A uint64 + var originalRef_A eth.L2BlockRef // prepare and sequencer a conflicting block for the L2A chain { unsafeHeadRef, err := sys.L2ELA.Escape().L2EthClient().L2BlockRefByHash(ctx, unsafeHead) @@ -74,37 +79,20 @@ func TestReorgUnsafeHead(gt *testing.T) { l.Info("Current unsafe ref", "unsafeHead", unsafeHead, "parent", unsafeHeadRef.ParentID().Hash, "l1_origin", unsafeHeadRef.L1Origin) l.Info("Expect to reorg the chain on current unsafe block", "number", unsafeHeadRef.Number, "head", unsafeHead, "parent", unsafeHeadRef.ParentID().Hash) - divergenceBlockNumber = unsafeHeadRef.Number - originalRef = unsafeHeadRef + divergenceBlockNumber_A = unsafeHeadRef.Number + originalRef_A = unsafeHeadRef sys.L2ChainA.PrintChain() - l1Origin, err := sys.L1Network.Escape().L1ELNode(match.FirstL1EL).EthClient().InfoByLabel(ctx, "latest") - require.NoError(t, err, "Expected to get latest block from L1 execution client") - - l1OriginHash := l1Origin.Hash() - parentOfUnsafeHead := unsafeHeadRef.ParentID() - parentsL1Origin, err := sys.L2ELA.Escape().L2EthClient().L2BlockRefByHash(ctx, parentOfUnsafeHead.Hash) - require.NoError(t, err, "Expected to be able to call L2BlockRefByHash API, but got error") - - if l1Origin.NumberU64() == parentsL1Origin.L1Origin.Number { - l.Info("Wait for a new L1 block, as current L1 head is the same as the parent of the unsafe head") - sys.L1Network.WaitForBlock() - - l1Origin, err := sys.L1Network.Escape().L1ELNode(match.FirstL1EL).EthClient().InfoByLabel(ctx, "latest") - require.NoError(t, err, "Expected to get latest block from L1 execution client") - - l1OriginHash = l1Origin.Hash() - } - l.Info("Sequencing a conflicting block", "unsafeHead", unsafeHeadRef, "parent", parentOfUnsafeHead, "l1_origin", eth.InfoToL1BlockRef(l1Origin)) + l.Info("Sequencing a conflicting block", "unsafeHead", unsafeHeadRef, "parent", parentOfUnsafeHead) // sequence a conflicting block with a simple transfer tx, based on the parent of the parent of the unsafe head { err = ia.New(ctx, seqtypes.BuildOpts{ Parent: parentOfUnsafeHead.Hash, - L1Origin: &l1OriginHash, + L1Origin: nil, }) require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") @@ -127,19 +115,19 @@ func TestReorgUnsafeHead(gt *testing.T) { } } - l.Info("Conflicting block has been produced, sequence a second block with op-test-sequencer") - + // start batcher { - currentUnsafeRef := sys.L2ChainA.UnsafeHeadRef() - l.Info("Current unsafe ref", "unsafeHead", currentUnsafeRef) - - l.Info("Starting batcher") - err = sys.L2BatcherA.Escape().ActivityAPI().StartBatcher(ctx) + err = retry.Do0(ctx, 3, retry.Exponential(), func() error { + return sys.L2BatcherA.Escape().ActivityAPI().StartBatcher(ctx) + }) require.NoError(t, err, "Expected to be able to call StartBatcher API, but got error") + } + // sequence a second block with op-test-sequencer (no L1 origin override) + { l.Info("Sequencing with op-test-sequencer (no L1 origin override)") err = ia.New(ctx, seqtypes.BuildOpts{ - Parent: currentUnsafeRef.Hash, + Parent: sys.L2ChainA.UnsafeHeadRef().Hash, L1Origin: nil, }) require.NoError(t, err, "Expected to be able to create a new block job for sequencing on op-test-sequencer, but got error") @@ -167,42 +155,31 @@ func TestReorgUnsafeHead(gt *testing.T) { sys.L2ChainA.WaitForBlock() - reorgedRef, err := sys.L2ELA.Escape().EthClient().BlockRefByNumber(ctx, divergenceBlockNumber) + reorgedRef_A, err := sys.L2ELA.Escape().EthClient().BlockRefByNumber(ctx, divergenceBlockNumber_A) require.NoError(t, err, "Expected to be able to call BlockRefByNumber API, but got error") sys.L2ChainA.PrintChain() - l.Info("Reorged chain on divergence block number (prior the reorg)", "number", divergenceBlockNumber, "head", originalRef.Hash, "parent", originalRef.ParentID().Hash) - l.Info("Reorged chain on divergence block number (after the reorg)", "number", divergenceBlockNumber, "head", reorgedRef.Hash, "parent", reorgedRef.ParentID().Hash) - require.NotEqual(t, originalRef.Hash, reorgedRef.Hash, "Expected to get different heads on divergence block number, but got the same hash, so no reorg happened") - require.Equal(t, originalRef.ParentID().Hash, reorgedRef.ParentHash, "Expected to get same parent hashes on divergence block number, but got different hashes") + l.Info("Reorged chain A on divergence block number (prior the reorg)", "number", divergenceBlockNumber_A, "head", originalRef_A.Hash, "parent", originalRef_A.ParentID().Hash) + l.Info("Reorged chain A on divergence block number (after the reorg)", "number", divergenceBlockNumber_A, "head", reorgedRef_A.Hash, "parent", reorgedRef_A.ParentID().Hash) + require.NotEqual(t, originalRef_A.Hash, reorgedRef_A.Hash, "Expected to get different heads on divergence block number, but got the same hash, so no reorg happened on chain A") + require.Equal(t, originalRef_A.ParentID().Hash, reorgedRef_A.ParentHash, "Expected to get same parent hashes on divergence block number, but got different hashes") err = wait.For(ctx, 5*time.Second, func() (bool, error) { - var safeL2Head_supervisor common.Hash - var safeL2Head_sequencer eth.L2BlockRef - - // get supervisor safe L2 head - { - safeBlockID := sys.Supervisor.SafeBlockID(sys.L2ChainA.ChainID()) - safeL2Head_supervisor = safeBlockID.Hash - } - // get sequencer safe L2 head - { - safeL2Head_sequencer = sys.L2CLNodeA.SafeL2BlockRef() - } + safeL2Head_A_supervisor := sys.Supervisor.SafeBlockID(sys.L2ChainA.ChainID()).Hash + safeL2Head_A_sequencer := sys.L2CLNodeA.SafeL2BlockRef() - if safeL2Head_sequencer.Number <= divergenceBlockNumber { - l.Info("Safe ref number is still behind divergence block number", "divergence", divergenceBlockNumber, "safe", safeL2Head_sequencer.Number) + if safeL2Head_A_sequencer.Number <= divergenceBlockNumber_A { + l.Info("Safe ref number is still behind divergence block number", "divergence", divergenceBlockNumber_A, "safe", safeL2Head_A_sequencer.Number) return false, nil } - - if safeL2Head_sequencer.Hash.Cmp(safeL2Head_supervisor) == 0 { - l.Info("Safe ref is the same on both supervisor and sequencer", "supervisor", safeL2Head_supervisor, "sequencer", safeL2Head_sequencer.Hash) - return true, nil + if safeL2Head_A_sequencer.Hash.Cmp(safeL2Head_A_supervisor) != 0 { + l.Info("Safe ref still not the same on supervisor and sequencer", "supervisor", safeL2Head_A_supervisor, "sequencer", safeL2Head_A_sequencer.Hash) + return false, nil } + l.Info("Safe ref is the same on both supervisor and sequencer", "supervisor", safeL2Head_A_supervisor, "sequencer", safeL2Head_A_sequencer.Hash) - l.Info("Safe ref still not the same on supervisor and sequencer", "supervisor", safeL2Head_supervisor, "sequencer", safeL2Head_sequencer.Hash) - return false, nil + return true, nil }) require.NoError(t, err, "Expected to get same safe ref on both supervisor and sequencer eventually") sys.L2ChainA.PrintChain() diff --git a/op-devstack/dsl/l2_network.go b/op-devstack/dsl/l2_network.go index d36000836279c..0dd22dec54a92 100644 --- a/op-devstack/dsl/l2_network.go +++ b/op-devstack/dsl/l2_network.go @@ -2,6 +2,7 @@ package dsl import ( "fmt" + "math" "time" "github.com/davecgh/go-spew/spew" @@ -39,6 +40,32 @@ func (n *L2Network) Escape() stack.L2Network { return n.inner } +func (n *L2Network) CatchUpTo(o *L2Network) { + this := n.inner.L2ELNode(match.FirstL2EL) + other := o.inner.L2ELNode(match.FirstL2EL) + + err := wait.For(n.ctx, 5*time.Second, func() (bool, error) { + a, err := this.EthClient().InfoByLabel(n.ctx, "latest") + if err != nil { + return false, err + } + + b, err := other.EthClient().InfoByLabel(n.ctx, "latest") + if err != nil { + return false, err + } + + eps := 6.0 // 6 seconds + if math.Abs(float64(a.Time()-b.Time())) > eps { + n.log.Warn("L2 networks too far off each other", n.String(), a.Time(), o.String(), b.Time()) + return false, nil + } + + return true, nil + }) + n.require.NoError(err, "Expected to get latest block from L2 execution clients") +} + func (n *L2Network) WaitForBlock() { l2_el := n.inner.L2ELNode(match.FirstL2EL) @@ -47,7 +74,7 @@ func (n *L2Network) WaitForBlock() { initialHash := initial.Hash() - err = wait.For(n.ctx, 500*time.Millisecond, func() (bool, error) { + err = wait.For(n.ctx, 1000*time.Millisecond, func() (bool, error) { latest, err := l2_el.EthClient().InfoByLabel(n.ctx, "latest") if err != nil { return false, err @@ -56,12 +83,12 @@ func (n *L2Network) WaitForBlock() { newHash := latest.Hash() if initialHash.Cmp(newHash) == 0 { - n.log.Info("Still same block detected", "initial_block_hash", initialHash, "new_block_hash", newHash) + n.log.Info("Still same block detected", "number", latest.NumberU64(), "chain", n.ChainID(), "initial_block_hash", initialHash, "new_block_hash", newHash) return false, nil } - n.log.Info("New block detected", "prev_block_hash", initialHash, "new_block_hash", newHash) + n.log.Info("New block detected", "chain", n.ChainID(), "prev_block_hash", initialHash, "new_block_hash", newHash, "time", latest.Time()) return true, nil }) n.require.NoError(err, "Expected to get latest block from L2 execution client for comparison") @@ -79,7 +106,10 @@ func (n *L2Network) PrintChain() { ref, err := l2_el.EthClient().BlockRefByNumber(n.ctx, i) n.require.NoError(err, "Expected to get block ref by number") - entries = append(entries, fmt.Sprintln("Number: ", ref.Number, "Hash: ", ref.Hash.Hex(), "Parent: ", ref.ParentID().Hash.Hex())) + l2blockref, err := l2_el.L2EthClient().L2BlockRefByHash(n.ctx, ref.Hash) + n.require.NoError(err, "Expected to get block ref by hash") + + entries = append(entries, fmt.Sprintln("Time: ", ref.Time, "Number: ", ref.Number, "Hash: ", ref.Hash.Hex(), "Parent: ", ref.ParentID().Hash.Hex(), "L1 Origin: ", l2blockref.L1Origin)) } syncStatus, err := l2_cl.RollupAPI().SyncStatus(n.ctx) @@ -87,7 +117,7 @@ func (n *L2Network) PrintChain() { entries = append(entries, spew.Sdump(syncStatus)) - n.log.Info("Printing block hashes and parent hashes") + n.log.Info("Printing block hashes and parent hashes", "network", n.String(), "chain", n.ChainID()) spew.Dump(entries) } diff --git a/op-devstack/dsl/supervisor.go b/op-devstack/dsl/supervisor.go index b63ac7c475dcc..d565ebe0347dd 100644 --- a/op-devstack/dsl/supervisor.go +++ b/op-devstack/dsl/supervisor.go @@ -47,11 +47,11 @@ func WithAllLocalUnsafeHeadsAdvancedBy(blocks uint64) func(cfg *VerifySyncStatus // VerifySyncStatus performs assertions based on the supervisor's SyncStatus endpoint. func (s *Supervisor) VerifySyncStatus(opts ...func(config *VerifySyncStatusConfig)) { cfg := applyOpts(VerifySyncStatusConfig{}, opts...) - initial := s.fetchSyncStatus() + initial := s.FetchSyncStatus() ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) defer cancel() err := wait.For(ctx, 1*time.Second, func() (bool, error) { - status := s.fetchSyncStatus() + status := s.FetchSyncStatus() s.require.Equalf(len(initial.Chains), len(status.Chains), "Expected %d chains in status but got %d", len(initial.Chains), len(status.Chains)) for chID, chStatus := range status.Chains { chInitial := initial.Chains[chID] @@ -67,7 +67,7 @@ func (s *Supervisor) VerifySyncStatus(opts ...func(config *VerifySyncStatusConfi s.require.NoError(err, "Expected sync status not found") } -func (s *Supervisor) fetchSyncStatus() eth.SupervisorSyncStatus { +func (s *Supervisor) FetchSyncStatus() eth.SupervisorSyncStatus { s.log.Debug("Fetching supervisor sync status") ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) defer cancel() @@ -98,5 +98,5 @@ func (s *Supervisor) SafeBlockID(chainID eth.ChainID) eth.BlockID { }) s.require.NoError(err, "Failed to fetch sync status") - return syncStatus.Chains[chainID].Safe + return syncStatus.Chains[chainID].CrossSafe } diff --git a/op-service/eth/supervisor_status.go b/op-service/eth/supervisor_status.go index d506c712cd592..20a56bf3de096 100644 --- a/op-service/eth/supervisor_status.go +++ b/op-service/eth/supervisor_status.go @@ -15,6 +15,9 @@ type SupervisorSyncStatus struct { type SupervisorChainSyncStatus struct { // LocalUnsafe is the latest L2 block that has been processed by the supervisor. LocalUnsafe BlockRef `json:"localUnsafe"` - Safe BlockID `json:"safe"` - Finalized BlockID `json:"finalized"` + LocalSafe BlockID `json:"localSafe"` + CrossUnsafe BlockID `json:"crossUnsafe"` + // Some fault-proof releases may already depend on `safe`, so we keep JSON field name as `safe`. + CrossSafe BlockID `json:"safe"` + Finalized BlockID `json:"finalized"` } diff --git a/op-supervisor/supervisor/backend/db/update.go b/op-supervisor/supervisor/backend/db/update.go index 1cd1775999682..9b55cc488d583 100644 --- a/op-supervisor/supervisor/backend/db/update.go +++ b/op-supervisor/supervisor/backend/db/update.go @@ -189,6 +189,29 @@ func (db *ChainsDB) initializedUpdateCrossSafe(chain eth.ChainID, l1View eth.Blo }, }) db.m.RecordCrossSafeRef(chain, lastCrossDerived) + + // compare new cross-safe to recorded cross-unsafe + crossUnsafe, err := db.CrossUnsafe(chain) + if err != nil { + db.logger.Warn("cannot get CrossUnsafe ref from db", "err", err) + return nil // log error for cross-unsafe call, but ignore it, as this call is for cross-safe + } + if crossUnsafe.Number > lastCrossDerived.Number { // nothing to do + return nil + } + + // if cross-unsafe block number is same or smaller than new cross-safe, make sure to update cross-unsafe to new cross-safe + if crossUnsafe.Hash.Cmp(lastCrossDerived.Hash) != 0 { + db.logger.Warn("Updated cross-unsafe due to cross-safe update", "chain", chain, "new cross-safe", lastCrossDerived, "current cross-unsafe", crossUnsafe) + err := db.UpdateCrossUnsafe(chain, types.BlockSeal{ + Hash: lastCrossDerived.Hash, + Number: lastCrossDerived.Number, + Timestamp: lastCrossDerived.Time, + }) + if err != nil { + return fmt.Errorf("failed to update cross-unsafe after processing a new cross-safe block: %w", err) + } + } return nil } diff --git a/op-supervisor/supervisor/backend/status/status.go b/op-supervisor/supervisor/backend/status/status.go index 5b31fdb51386f..1816df68717f8 100644 --- a/op-supervisor/supervisor/backend/status/status.go +++ b/op-supervisor/supervisor/backend/status/status.go @@ -24,6 +24,8 @@ type StatusTracker struct { type NodeSyncStatus struct { CurrentL1 eth.L1BlockRef LocalUnsafe eth.BlockRef + LocalSafe types.BlockSeal + CrossUnsafe types.BlockSeal CrossSafe types.BlockSeal Finalized types.BlockSeal } @@ -57,6 +59,12 @@ func (su *StatusTracker) OnEvent(ev event.Event) bool { case superevents.LocalUnsafeUpdateEvent: status := loadStatusRef(x.ChainID) status.LocalUnsafe = x.NewLocalUnsafe + case superevents.LocalSafeUpdateEvent: + status := loadStatusRef(x.ChainID) + status.LocalSafe = x.NewLocalSafe.Derived + case superevents.CrossUnsafeUpdateEvent: + status := loadStatusRef(x.ChainID) + status.CrossUnsafe = x.NewCrossUnsafe case superevents.CrossSafeUpdateEvent: status := loadStatusRef(x.ChainID) status.CrossSafe = x.NewCrossSafe.Derived @@ -120,7 +128,9 @@ func (su *StatusTracker) SyncStatus() (eth.SupervisorSyncStatus, error) { supervisorStatus.Chains[chainID] = ð.SupervisorChainSyncStatus{ LocalUnsafe: nodeStatus.LocalUnsafe, - Safe: nodeStatus.CrossSafe.ID(), + LocalSafe: nodeStatus.LocalSafe.ID(), + CrossUnsafe: nodeStatus.CrossUnsafe.ID(), + CrossSafe: nodeStatus.CrossSafe.ID(), Finalized: nodeStatus.Finalized.ID(), } firstChain = false diff --git a/op-supervisor/supervisor/backend/status/status_test.go b/op-supervisor/supervisor/backend/status/status_test.go index 08eab3e20993d..954be9787f328 100644 --- a/op-supervisor/supervisor/backend/status/status_test.go +++ b/op-supervisor/supervisor/backend/status/status_test.go @@ -87,8 +87,8 @@ func TestUpdateCrossSafe(t *testing.T) { status, err := tracker.SyncStatus() require.NoError(t, err) require.Equal(t, chain1Safe.Derived.Timestamp, status.SafeTimestamp) - require.Equal(t, chain1Safe.Derived.ID(), status.Chains[chain1].Safe) - require.Equal(t, chain2Safe.Derived.ID(), status.Chains[chain2].Safe) + require.Equal(t, chain1Safe.Derived.ID(), status.Chains[chain1].CrossSafe) + require.Equal(t, chain2Safe.Derived.ID(), status.Chains[chain2].CrossSafe) } func TestUpdateFinalized(t *testing.T) {