diff --git a/justfile b/justfile index 14665b5f726a7..00822da84c73d 100644 --- a/justfile +++ b/justfile @@ -39,4 +39,3 @@ update-op-geth ref: go mod edit -replace=github.com/ethereum/go-ethereum=github.com/ethereum-optimism/op-geth@"$ver"; \ go mod tidy; \ echo "Updated op-geth to $ver" - diff --git a/op-acceptance-tests/tests/sequencer/init_test.go b/op-acceptance-tests/tests/sequencer/init_test.go new file mode 100644 index 0000000000000..ed9fb2d955542 --- /dev/null +++ b/op-acceptance-tests/tests/sequencer/init_test.go @@ -0,0 +1,17 @@ +package sequencer + +import ( + "log/slog" + "testing" + + "github.com/ethereum-optimism/optimism/op-devstack/compat" + "github.com/ethereum-optimism/optimism/op-devstack/presets" +) + +// TestMain creates the test-setups against the shared backend +func TestMain(m *testing.M) { + presets.DoMain(m, presets.WithMinimal(), + presets.WithCompatibleTypes(compat.SysGo), + presets.WithLogLevel(slog.LevelDebug), + ) +} diff --git a/op-acceptance-tests/tests/sequencer/recover_mode_test.go b/op-acceptance-tests/tests/sequencer/recover_mode_test.go new file mode 100644 index 0000000000000..548218716ab74 --- /dev/null +++ b/op-acceptance-tests/tests/sequencer/recover_mode_test.go @@ -0,0 +1,44 @@ +package sequencer + +import ( + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/stretchr/testify/require" +) + +// TestRecoverModeWhenChainHealthy checks that the chain +// can progress as normal when recover mode is activated. +// Recover mode is designed to recover from a sequencing +// window expiry when there are ample L1 blocks to eagerly +// progress the l1 origin to. But when the l1 origin is +// close to the tip of the l1 chain, the eagerness would cause +// a delay in unsafe block production while the sequencer waits +// for the next l1 origin to become available. Recover mode +// has since been patched, and the sequencer will not demand the +// next l1 origin until it is actually available. This tests +// protects against a regeression in that behavior. +func TestRecoverModeWhenChainHealthy(gt *testing.T) { + t := devtest.ParallelT(gt) + sys := presets.NewMinimal(t) + tracer := t.Tracer() + ctx := t.Ctx() + + sys.L2CL.SetSequencerRecoverMode(true) + blockTime := sys.L2Chain.Escape().RollupConfig().BlockTime + numL2Blocks := uint64(20) + waitTime := time.Duration(blockTime*numL2Blocks+5) * time.Second + + num := sys.L2CL.SyncStatus().UnsafeL2.Number + new_num := num + require.Eventually(t, func() bool { + ctx, span := tracer.Start(ctx, "check head") + defer span.End() + + new_num, num = sys.L2CL.SyncStatus().UnsafeL2.Number, new_num + t.Logger().InfoContext(ctx, "unsafe head", "number", new_num, "safe head", sys.L2CL.SyncStatus().SafeL2.Number) + return new_num >= numL2Blocks + }, waitTime, time.Duration(blockTime)*time.Second) +} diff --git a/op-devstack/dsl/l2_cl.go b/op-devstack/dsl/l2_cl.go index 862394fc522c4..2d9874d3987c2 100644 --- a/op-devstack/dsl/l2_cl.go +++ b/op-devstack/dsl/l2_cl.go @@ -91,6 +91,10 @@ func (cl *L2CLNode) StopSequencer() common.Hash { return unsafeHead } +func (cl *L2CLNode) SetSequencerRecoverMode(b bool) error { + return cl.inner.RollupAPI().SetRecoverMode(cl.ctx, b) +} + func (cl *L2CLNode) SyncStatus() *eth.SyncStatus { ctx, cancel := context.WithTimeout(cl.ctx, DefaultTimeout) defer cancel() diff --git a/op-e2e/actions/helpers/l2_batcher.go b/op-e2e/actions/helpers/l2_batcher.go index e0d8eac5a284e..2e1a941721d39 100644 --- a/op-e2e/actions/helpers/l2_batcher.go +++ b/op-e2e/actions/helpers/l2_batcher.go @@ -270,7 +270,7 @@ func (s *L2Batcher) Buffer(t Testing, bufferOpts ...BufferOption) error { } } - s.ActCreateChannel(t, s.rollupCfg.IsDelta(block.Time()), options.channelModifiers...) + s.ActCreateChannel(t, s.rollupCfg.IsDelta(block.Time()) && !s.l2BatcherCfg.ForceSubmitSingularBatch, options.channelModifiers...) if _, err := s.L2ChannelOut.AddBlock(s.rollupCfg, block); err != nil { return err diff --git a/op-e2e/actions/helpers/l2_sequencer.go b/op-e2e/actions/helpers/l2_sequencer.go index 97d8100b1a5d6..ef3d4e29dfbfc 100644 --- a/op-e2e/actions/helpers/l2_sequencer.go +++ b/op-e2e/actions/helpers/l2_sequencer.go @@ -40,8 +40,8 @@ func (m *MockL1OriginSelector) FindL1Origin(ctx context.Context, l2Head eth.L2Bl return m.actual.FindL1Origin(ctx, l2Head) } -func (m *MockL1OriginSelector) SetRecoverMode(bool) { - // noop +func (m *MockL1OriginSelector) SetRecoverMode(b bool) { + m.actual.SetRecoverMode(b) } // L2Sequencer is an actor that functions like a rollup node, @@ -98,20 +98,28 @@ func NewL2Sequencer(t Testing, log log.Logger, l1 derive.L1Fetcher, blobSrc deri // ActL2StartBlock starts building of a new L2 block on top of the head func (s *L2Sequencer) ActL2StartBlock(t Testing) { + err := s.ActMaybeL2StartBlock(t) + require.NoError(t, err, "failed to start block building") +} + +// ActMaybeL2StartBlock tries to start building a new L2 block on top of the head +func (s *L2Sequencer) ActMaybeL2StartBlock(t Testing) error { require.NoError(t, s.drainer.Drain()) // can't build when other work is still blocking if !s.L2PipelineIdle { t.InvalidAction("cannot start L2 build when derivation is not idle") - return + return nil } if s.l2Building { t.InvalidAction("already started building L2 block") - return + return nil } s.synchronousEvents.Emit(t.Ctx(), sequencing.SequencerActionEvent{}) - require.NoError(t, s.drainer.DrainUntil(event.Is[engine.BuildStartedEvent], false), - "failed to start block building") - + err := s.drainer.DrainUntil(event.Is[engine.BuildStartedEvent], false) + if err != nil { + return err + } s.l2Building = true + return nil } // ActL2EndBlock completes a new L2 block and applies it to the L2 chain as new canonical unsafe head @@ -272,3 +280,7 @@ func (s *L2Sequencer) ActBuildL2ToInterop(t Testing) { s.ActL2EmptyBlock(t) } } + +func (s *L2Sequencer) ActSetRecoverMode(t Testing, b bool) { + s.sequencer.SetRecoverMode(b) +} diff --git a/op-e2e/actions/helpers/utils.go b/op-e2e/actions/helpers/utils.go index 9fa5d0438741c..e8dc404ea164a 100644 --- a/op-e2e/actions/helpers/utils.go +++ b/op-e2e/actions/helpers/utils.go @@ -15,7 +15,7 @@ func DefaultRollupTestParams() *e2eutils.TestParams { MaxSequencerDrift: 40, SequencerWindowSize: 120, ChannelTimeout: 120, - L1BlockTime: 15, + L1BlockTime: 12, // Many of the action helpers assume a 12s L1 block time AllocType: config.DefaultAllocType, } } diff --git a/op-e2e/actions/proofs/helpers/env.go b/op-e2e/actions/proofs/helpers/env.go index 6872c1d0e1efa..d700763d7f79f 100644 --- a/op-e2e/actions/proofs/helpers/env.go +++ b/op-e2e/actions/proofs/helpers/env.go @@ -243,15 +243,12 @@ func (env *L2FaultProofEnv) BatchAndMine(t helpers.Testing) { // Returns the L2 Safe Block Reference func (env *L2FaultProofEnv) BatchMineAndSync(t helpers.Testing) eth.L2BlockRef { t.Helper() - id := env.Miner.UnsafeID() env.BatchAndMine(t) env.Sequencer.ActL1HeadSignal(t) env.Sequencer.ActL2PipelineFull(t) // Assertions - syncStatus := env.Sequencer.SyncStatus() - require.Equal(t, syncStatus.UnsafeL2.L1Origin, id, "UnsafeL2.L1Origin should equal L1 Unsafe ID before batch submitted") require.Equal(t, syncStatus.UnsafeL2, syncStatus.SafeL2, "UnsafeL2 should equal SafeL2") return syncStatus.SafeL2 diff --git a/op-e2e/actions/proofs/sequence_window_expiry_test.go b/op-e2e/actions/proofs/sequence_window_expiry_test.go index cb702fe8eb6a4..6e85163d9b074 100644 --- a/op-e2e/actions/proofs/sequence_window_expiry_test.go +++ b/op-e2e/actions/proofs/sequence_window_expiry_test.go @@ -5,18 +5,38 @@ import ( actionsHelpers "github.com/ethereum-optimism/optimism/op-e2e/actions/helpers" "github.com/ethereum-optimism/optimism/op-e2e/actions/proofs/helpers" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils" "github.com/ethereum-optimism/optimism/op-program/client/claim" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" ) -// Run a test that proves a deposit-only block generated due to sequence window expiry. +// Run a test that proves a deposit-only block generated due to sequence window expiry, +// and then recovers the chain using sequencer recover mode. func runSequenceWindowExpireTest(gt *testing.T, testCfg *helpers.TestCfg[any]) { t := actionsHelpers.NewDefaultTesting(gt) - tp := helpers.NewTestParams() - env := helpers.NewL2FaultProofEnv(t, testCfg, tp, helpers.NewBatcherCfg()) + const SEQUENCER_WINDOW_SIZE = 50 // (short, to keep test fast) + tp := helpers.NewTestParams(func(p *e2eutils.TestParams) { + p.SequencerWindowSize = SEQUENCER_WINDOW_SIZE + }) + + // It seems more difficult (almost impossible) to recover from sequencing window expiry with span batches, + // since the singular batches within are invalidated _atomically_. + // That is to say, if the oldest batch in the span batch fails the sequencing window check + // (l1 origin + seq window < l1 inclusion) + // All following batches are invalidated / dropped as well. + // https://github.com/ethereum-optimism/optimism/blob/73339162d78a1ebf2daadab01736382eed6f4527/op-node/rollup/derive/batches.go#L96-L100 + // + // If the same blocks were batched with singular batches, the validation rules are different + // https://github.com/ethereum-optimism/optimism/blob/73339162d78a1ebf2daadab01736382eed6f4527/op-node/rollup/derive/batches.go#L83-L86 + // In the case of recover mode, the noTxPool=true condition means autoderviation actually fills + // the gap with identical blocks anyway, meaning the following batches are actually still valid. + bc := helpers.NewBatcherCfg() + bc.ForceSubmitSingularBatch = true - // Mine an empty block for gas estimation purposes. + env := helpers.NewL2FaultProofEnv(t, testCfg, tp, bc) + + // Mine an empty L1 block for gas estimation purposes. env.Miner.ActEmptyBlock(t) // Expire the sequence window by building `SequenceWindow + 1` empty blocks on L1. @@ -24,7 +44,7 @@ func runSequenceWindowExpireTest(gt *testing.T, testCfg *helpers.TestCfg[any]) { env.Alice.L1.ActResetTxOpts(t) env.Alice.ActDeposit(t) - env.Miner.ActL1StartBlock(12)(t) + env.Miner.ActL1StartBlock(tp.L1BlockTime)(t) env.Miner.ActL1IncludeTx(env.Alice.Address())(t) env.Miner.ActL1EndBlock(t) @@ -44,8 +64,104 @@ func runSequenceWindowExpireTest(gt *testing.T, testCfg *helpers.TestCfg[any]) { l2SafeHead = env.Engine.L2Chain().CurrentSafeBlock() require.Greater(t, l2SafeHead.Number.Uint64(), uint64(0)) - // Run the FPP on one of the auto-derived blocks. env.RunFaultProofProgram(t, l2SafeHead.Number.Uint64()/2, testCfg.CheckResult, testCfg.InputParams...) + + // Set recover mode on the sequencer: + env.Sequencer.ActSetRecoverMode(t, true) + // Since recover mode only affects the L2 CL (op-node), + // it won't stop the test environment injecting transactions + // directly into the engine. So we will force the engine + // to ignore such injections if recover mode is enabled. + env.Engine.EngineApi.SetForceEmpty(true) + + // Define "lag" as the difference between the current L1 block number and the safe L2 block's L1 origin number. + computeLag := func() int { + ss := env.Sequencer.SyncStatus() + return int(ss.CurrentL1.Number - ss.SafeL2.L1Origin.Number) + } + + // Define "drift" as the difference between the current L2 block's timestamp and the unsafe L2 block's L1 origin's timestamp. + computeDrift := func() int { + ss := env.Sequencer.SyncStatus() + l2header, err := env.Engine.EthClient().HeaderByHash(t.Ctx(), ss.UnsafeL2.Hash) + require.NoError(t, err) + l1header, err := env.Miner.EthClient().HeaderByHash(t.Ctx(), ss.UnsafeL2.L1Origin.Hash) + require.NoError(t, err) + t.Log("l2header.Time", l2header.Time) + t.Log("l1header.Time", l1header.Time) + return int(l2header.Time) - int(l1header.Time) + } + + // Build both chains and assert the L1 origin catches back up with the tip of the L1 chain. + lag := computeLag() + t.Log("lag", lag) + drift := computeDrift() + t.Log("drift", drift) + require.GreaterOrEqual(t, uint64(lag), tp.SequencerWindowSize, "Lag is less than sequencing window size") + numL1Blocks := 0 + timeout := tp.SequencerWindowSize * 50 + + for numL1Blocks < int(timeout) { + for range 100 * tp.L1BlockTime / env.Sd.RollupCfg.BlockTime { // go at 100x real time + err := env.Sequencer.ActMaybeL2StartBlock(t) + if err != nil { + break + } + env.Bob.L2.ActResetTxOpts(t) + env.Bob.L2.ActMakeTx(t) + env.Engine.ActL2IncludeTx(env.Bob.Address())(t) + // RecoverMode (enabled above) should prevent this + // transaction from being included in the block, which + // is critical for recover mode to work. + env.Sequencer.ActL2EndBlock(t) + drift = computeDrift() + t.Log("drift", drift) + } + env.BatchMineAndSync(t) // Mines 1 block on L1 + numL1Blocks++ + lag = computeLag() + t.Log("lag", lag) + drift = computeDrift() + t.Log("drift", drift) + if lag == 1 { // A lag of 1 is the minimum possible. + break + } + } + + if uint64(numL1Blocks) >= timeout { + t.Fatal("L1 Origin did not catch up to tip within %d L1 blocks (lag is %d)", numL1Blocks, lag) + } else { + t.Logf("L1 Origin caught up to within %d blocks of the tip within %d L1 blocks (sequencing window size %d)", + lag, numL1Blocks, tp.SequencerWindowSize) + } + + switch { + case drift == 0: + t.Fatal("drift is zero, this implies the unsafe l2 head is pinned to the l1 head") + case drift > int(tp.MaxSequencerDrift): + t.Fatal("drift is too high") + default: + t.Log("drift", drift) + } + + // Disable recover mode so we can get some user transactions in again. + env.Sequencer.ActSetRecoverMode(t, false) + env.Engine.EngineApi.SetForceEmpty(false) + l2SafeBefore := env.Sequencer.L2Safe() + env.Sequencer.ActL2StartBlock(t) + env.Bob.L2.ActResetTxOpts(t) + env.Bob.L2.ActMakeTx(t) + env.Engine.ActL2IncludeTx(env.Bob.Address())(t) + env.Sequencer.ActL2EndBlock(t) + env.BatchMineAndSync(t) + l2Safe := env.Sequencer.L2Safe() + require.Equal(t, l2Safe.Number, l2SafeBefore.Number+1, "safe chain did not progress with user transactions") + l2SafeBlock, err := env.Engine.EthClient().BlockByHash(t.Ctx(), l2Safe.Hash) + require.NoError(t, err) + // Assert safe block has at least two transactions + require.GreaterOrEqual(t, len(l2SafeBlock.Transactions()), 2, "safe block did not have at least two transactions") + + env.RunFaultProofProgram(t, l2Safe.Number, testCfg.CheckResult, testCfg.InputParams...) } // Runs a that proves a block in a chain where the batcher opens a channel, the sequence window expires, and then the diff --git a/op-node/rollup/sequencing/origin_selector.go b/op-node/rollup/sequencing/origin_selector.go index c5f23407d7652..6c91351b5bc4a 100644 --- a/op-node/rollup/sequencing/origin_selector.go +++ b/op-node/rollup/sequencing/origin_selector.go @@ -135,7 +135,13 @@ func (los *L1OriginSelector) CurrentAndNextOrigin(ctx context.Context, l2Head et } los.currentOrigin = currentOrigin nextOrigin, err := los.l1.L1BlockRefByNumber(ctx, currentOrigin.Number+1) - if err != nil { + if errors.Is(err, ethereum.NotFound) { + // If the next origin is not found, it means we are at the end of the chain. + // In this case, we set the next origin to an empty block reference. + los.log.Error("next L1 origin not found, recover mode likely brought L1 origin up to the tip of the chain", "error", err) + los.nextOrigin = eth.L1BlockRef{} + return los.currentOrigin, los.nextOrigin, nil + } else if err != nil { return eth.L1BlockRef{}, eth.L1BlockRef{}, derive.NewTemporaryError(fmt.Errorf("failed to fetch next L1 origin: %w", err)) } diff --git a/op-node/rollup/sequencing/origin_selector_test.go b/op-node/rollup/sequencing/origin_selector_test.go index be8a0dc76255a..2f2e12f147f01 100644 --- a/op-node/rollup/sequencing/origin_selector_test.go +++ b/op-node/rollup/sequencing/origin_selector_test.go @@ -7,7 +7,6 @@ import ( "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/confdepth" - "github.com/ethereum-optimism/optimism/op-node/rollup/derive" "github.com/ethereum-optimism/optimism/op-node/rollup/engine" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/testlog" @@ -213,9 +212,7 @@ func TestOriginSelectorAdvances(t *testing.T) { // is not ready yet by simulating a NotFound error. l1.ExpectL1BlockRefByHash(c.Hash, c, nil) l1.ExpectL1BlockRefByNumber(d.Number, eth.BlockRef{}, ethereum.NotFound) - _, err := s.FindL1Origin(ctx, l2Head) - require.ErrorIs(t, err, derive.ErrTemporary) - require.ErrorIs(t, err, ethereum.NotFound) + requireL1OriginAt(l2Head, c) // Now, simulate the block being ready, and ensure // that the origin advances to the next block.