From 93aabb4fe6de40db722aed15939d39bf89067e8d Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 26 Aug 2022 15:56:54 +0200 Subject: [PATCH] op-node: fix heads finding algorithm to handle finalized/safe/unsafe correctly, and fix off-by-1 in epoch window boundary --- op-node/rollup/derive/engine_queue.go | 54 +-- op-node/rollup/derive/engine_queue_test.go | 94 ++++-- op-node/rollup/sync/start.go | 315 ++++++++++-------- op-node/rollup/sync/start_test.go | 365 ++++++++++++--------- op-node/testutils/fake_chain.go | 75 ++++- 5 files changed, 551 insertions(+), 352 deletions(-) diff --git a/op-node/rollup/derive/engine_queue.go b/op-node/rollup/derive/engine_queue.go index 8faba6abffe21..cb34a9e156c84 100644 --- a/op-node/rollup/derive/engine_queue.go +++ b/op-node/rollup/derive/engine_queue.go @@ -10,7 +10,6 @@ import ( "github.com/ethereum-optimism/optimism/op-node/eth" "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/sync" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -72,6 +71,8 @@ type EngineQueue struct { // Tracks which L2 blocks where last derived from which L1 block. At most finalityLookback large. finalityData []FinalityData + syncStart *sync.FindSyncStart + engine Engine metrics Metrics @@ -175,6 +176,10 @@ func (eq *EngineQueue) postProcessSafeL2() { if len(eq.finalityData) >= finalityLookback { eq.finalityData = append(eq.finalityData[:0], eq.finalityData[1:finalityLookback]...) } + // after resets we have a safe head that is equal or later than the L1 chain, we can't count that as derived. + if eq.safeHead.L1Origin.Number >= eq.progress.Origin.Number { + return + } // remember the last L2 block that we fully derived from the given finality data if len(eq.finalityData) == 0 || eq.finalityData[len(eq.finalityData)-1].L1Block.Number < eq.progress.Origin.Number { // append entry for new L1 block @@ -369,38 +374,37 @@ func (eq *EngineQueue) forceNextSafeAttributes(ctx context.Context) error { // ResetStep Walks the L2 chain backwards until it finds an L2 block whose L1 origin is canonical. // The unsafe head is set to the head of the L2 chain, unless the existing safe head is not canonical. func (eq *EngineQueue) ResetStep(ctx context.Context, l1Fetcher L1Fetcher) error { - finalized, err := eq.engine.L2BlockRefByLabel(ctx, eth.Finalized) - if errors.Is(err, ethereum.NotFound) { - // default to genesis if we have not finalized anything before. - finalized, err = eq.engine.L2BlockRefByHash(ctx, eq.cfg.Genesis.L2.Hash) - } - if err != nil { - return NewTemporaryError(fmt.Errorf("failed to find the finalized L2 block: %w", err)) - } - // TODO: this should be resetting using the safe head instead. Out of scope for L2 client bindings PR. - prevUnsafe, err := eq.engine.L2BlockRefByLabel(ctx, eth.Unsafe) - if err != nil { - return NewTemporaryError(fmt.Errorf("failed to find the L2 Head block: %w", err)) - } - unsafe, safe, err := sync.FindL2Heads(ctx, prevUnsafe, eq.cfg.SeqWindowSize, l1Fetcher, eq.engine, &eq.cfg.Genesis) - if err != nil { - return NewTemporaryError(fmt.Errorf("failed to find the L2 Heads to start from: %w", err)) + if eq.syncStart == nil { + eq.syncStart = sync.NewFindSyncStart(eq.cfg, l1Fetcher, eq.engine) + return nil } - l1Origin, err := l1Fetcher.L1BlockRefByHash(ctx, safe.L1Origin.Hash) - if err != nil { - return NewTemporaryError(fmt.Errorf("failed to fetch the new L1 progress: origin: %v; err: %v", safe.L1Origin, err)) + + err := eq.syncStart.Step(ctx) + if err == nil { + return nil // more sync start steps left to do } - if safe.Time < l1Origin.Time { - return NewResetError(fmt.Errorf("cannot reset block derivation to start at L2 block %s with time %d older than its L1 origin %s with time %d, time invariant is broken", - safe, safe.Time, l1Origin, l1Origin.Time)) + if err != io.EOF { + if errors.Is(err, sync.TooDeepReorgErr) { + return NewCriticalError(err) + } + if errors.Is(err, sync.WrongChainErr) { + return NewCriticalError(err) + } + if errors.Is(err, sync.ReorgFinalizedErr) { + return NewResetError(err) + } + return NewTemporaryError(fmt.Errorf("sync start step failed: %w", err)) } - eq.log.Debug("Reset engine queue", "safeHead", safe, "unsafe", unsafe, "safe_timestamp", safe.Time, "unsafe_timestamp", unsafe.Time, "l1Origin", l1Origin) + + finalized, safe, unsafe, startL1 := eq.syncStart.Result() + + eq.log.Debug("Reset engine queue", "l2_finalized", finalized, "l2_safe", safe, "l2_unsafe", unsafe, "l1_start", startL1) eq.unsafeHead = unsafe eq.safeHead = safe eq.finalized = finalized eq.finalityData = eq.finalityData[:0] eq.progress = Progress{ - Origin: l1Origin, + Origin: startL1, Closed: false, } eq.metrics.RecordL2Ref("l2_finalized", finalized) diff --git a/op-node/rollup/derive/engine_queue_test.go b/op-node/rollup/derive/engine_queue_test.go index 103d44d4f663c..dc52905dc8fe4 100644 --- a/op-node/rollup/derive/engine_queue_test.go +++ b/op-node/rollup/derive/engine_queue_test.go @@ -1,6 +1,7 @@ package derive import ( + "fmt" "math/rand" "testing" @@ -33,8 +34,12 @@ func TestEngineQueue_Finalize(t *testing.T) { // D: unsafe, not yet referenced by L2 l1Time := uint64(2) - refA := testutils.RandomBlockRef(rng) - + refA := eth.L1BlockRef{ + Hash: testutils.RandomHash(rng), + Number: 10000, + ParentHash: testutils.RandomHash(rng), + Time: 2000000, + } refB := eth.L1BlockRef{ Hash: testutils.RandomHash(rng), Number: refA.Number + 1, @@ -103,42 +108,85 @@ func TestEngineQueue_Finalize(t *testing.T) { L1Origin: refC.ID(), SequenceNumber: 0, } + refC1 := eth.L2BlockRef{ + Hash: testutils.RandomHash(rng), + Number: refC0.Number + 1, + ParentHash: refC0.Hash, + Time: refC0.Time + cfg.BlockTime, + L1Origin: refC.ID(), + SequenceNumber: 1, + } + refD0 := eth.L2BlockRef{ + Hash: testutils.RandomHash(rng), + Number: refC1.Number + 1, + ParentHash: refC1.Hash, + Time: refC1.Time + cfg.BlockTime, + L1Origin: refD.ID(), + SequenceNumber: 0, + } + refD1 := eth.L2BlockRef{ + Hash: testutils.RandomHash(rng), + Number: refD0.Number + 1, + ParentHash: refD0.Hash, + Time: refD0.Time + cfg.BlockTime, + L1Origin: refD.ID(), + SequenceNumber: 1, + } + fmt.Println("refA", refA.Hash) + fmt.Println("refB", refB.Hash) + fmt.Println("refC", refC.Hash) + fmt.Println("refD", refD.Hash) + + fmt.Println("refA0", refA0.Hash) + fmt.Println("refA1", refA1.Hash) + fmt.Println("refB0", refB0.Hash) + fmt.Println("refB1", refB1.Hash) + fmt.Println("refC0", refC0.Hash) + fmt.Println("refC1", refC1.Hash) + fmt.Println("refD0", refD0.Hash) metrics := &TestMetrics{} eng := &testutils.MockEngine{} - eng.ExpectL2BlockRefByLabel(eth.Finalized, refA1, nil) - // TODO(Proto): update expectation once we're using safe block label properly for sync starting point - eng.ExpectL2BlockRefByLabel(eth.Unsafe, refC0, nil) - // we find the common point to initialize to by comparing the L1 origins in the L2 chain with the L1 chain l1F := &testutils.MockL1Source{} - l1F.ExpectL1BlockRefByLabel(eth.Unsafe, refD, nil) - l1F.ExpectL1BlockRefByNumber(refC0.L1Origin.Number, refC, nil) - eng.ExpectL2BlockRefByHash(refC0.ParentHash, refB1, nil) // good L1 origin - eng.ExpectL2BlockRefByHash(refB1.ParentHash, refB0, nil) // need a block with seqnr == 0, don't stop at above - l1F.ExpectL1BlockRefByHash(refB0.L1Origin.Hash, refB, nil) // the origin of the safe L2 head will be the L1 starting point for derivation. - eq := NewEngineQueue(logger, cfg, eng, metrics) - require.NoError(t, RepeatResetStep(t, eq.ResetStep, l1F, 3)) + eng.ExpectL2BlockRefByLabel(eth.Finalized, refA1, nil) + eng.ExpectL2BlockRefByLabel(eth.Safe, refD0, nil) + eng.ExpectL2BlockRefByLabel(eth.Unsafe, refD1, nil) - // TODO(proto): this is changing, needs to be a sequence window ago, but starting traversal back from safe block, - // safe blocks with canon origin are good, but we go back a full window to ensure they are all included in L1, - // by forcing them to be consolidated with L1 again. - require.Equal(t, eq.SafeL2Head(), refB0, "L2 reset should go back to sequence window ago") + l1F.ExpectL1BlockRefByNumber(refD.Number, refD, nil) // fetch L1 origin of head, it's canon + eng.ExpectL2BlockRefByHash(refD1.ParentHash, refD0, nil) // traverse L2 chain, find safe head D0 + eng.ExpectL2BlockRefByHash(refD0.ParentHash, refC1, nil) // traverse back full seq window + l1F.ExpectL1BlockRefByNumber(refC.Number, refC, nil) + eng.ExpectL2BlockRefByHash(refC1.ParentHash, refC0, nil) + eng.ExpectL2BlockRefByHash(refC0.ParentHash, refB1, nil) + l1F.ExpectL1BlockRefByNumber(refB.Number, refB, nil) + l1F.ExpectL1BlockRefByHash(refB.Hash, refB, nil) + eq := NewEngineQueue(logger, cfg, eng, metrics) + require.NoError(t, RepeatResetStep(t, eq.ResetStep, l1F, 20)) + + require.Equal(t, refB1, eq.SafeL2Head(), "L2 reset should go back to sequence window ago: blocks with origin D and C are not safe until we reconcile") + require.Equal(t, refB, eq.Progress().Origin, "Expecting to be set back derivation L1 progress to B") require.Equal(t, refA1, eq.Finalized(), "A1 is recognized as finalized before we run any steps") - // we are not adding blocks in this test, - // but we can still trigger post-processing for the already existing safe head, - // so the engine can prepare to finalize that. + // now say B1 was included in C and became the new safe head + eq.progress.Origin = refC + eq.safeHead = refB1 eq.postProcessSafeL2() - // let's finalize C, which included B0, but not B1 + + // now say C0 was included in D and became the new safe head + eq.progress.Origin = refD + eq.safeHead = refC0 + eq.postProcessSafeL2() + + // let's finalize C (current L1), from which we fully derived B1, but not C0 eq.Finalize(refC.ID()) // Now a few steps later, without consuming any additional L1 inputs, - // we should be able to resolve that B0 is now finalized + // we should be able to resolve that B1 is now finalized require.NoError(t, RepeatStep(t, eq.Step, eq.progress, 10)) - require.Equal(t, refB0, eq.Finalized(), "B0 was included in finalized C, and should now be finalized") + require.Equal(t, refB1, eq.Finalized(), "B1 was included in finalized C, and should now be finalized") l1F.AssertExpectations(t) eng.AssertExpectations(t) diff --git a/op-node/rollup/sync/start.go b/op-node/rollup/sync/start.go index 155442945d3e1..cfc0281306c44 100644 --- a/op-node/rollup/sync/start.go +++ b/op-node/rollup/sync/start.go @@ -1,45 +1,52 @@ -// The sync package is responsible for reconciling L1 and L2. +// Package sync is responsible for reconciling L1 and L2. // // The Ethereum chain is a DAG of blocks with the root block being the genesis block. At any given // time, the head (or tip) of the chain can change if an offshoot/branch of the chain has a higher -// total difficulty. This is known as a re-organization of the canonical chain. Each block points to -// a parent block and the node is responsible for deciding which block is the head and thus the -// mapping from block number to canonical block. +// total difficulty or PoS attestation weight. +// This is known as a re-organization of the canonical chain. +// Each block points to a parent block and the node is responsible for deciding which block is the +// head and thus the mapping from block number to canonical block. // // The Optimism (L2) chain has similar properties, but also retains references to the Ethereum (L1) // chain. Each L2 block retains a reference to an L1 block (its "L1 origin", i.e. L1 block -// associated with the epoch that the L2 block belongs to) and to its parent L2 block. The L2 chain -// node must satisfy the following validity rules: +// associated with the epoch that the L2 block belongs to) and to its parent L2 block. +// The L2 chain node must satisfy the following validity rules: // // 1. l2block.number == l2block.l2parent.block.number + 1 -// 2. l2block.l1Origin.number >= l2block.l2parent.l1Origin.number +// 2. l2block.l1Origin.number == l2block.l2parent.l1Origin.number +// OR l2block.l1Origin.number == l2block.l2parent.l1Origin.number + 1 // 3. l2block.l1Origin is in the canonical chain on L1 // 4. l1_rollup_genesis is an ancestor of l2block.l1Origin // // During normal operation, both the L1 and L2 canonical chains can change, due to a re-organisation // or due to an extension (new L1 or L2 block). // -// When one of these changes occurs, the rollup node needs to determine what the new L2 head blocks -// should be. We track two L2 head blocks: +// When one of these changes occurs, the rollup node needs to determine what the new L2 sync status +// should be. We track the following attributes: // // - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a plausible (1) // extension of the canonical L1 chain (as known to the op-node). -// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is -// complete within the canonical L1 chain (as known to the op-node). +// - The *safe L2 block*: This is the highest L2 block which is certain to be fully derived from the L1 chain. +// Being derived from the L1 chain requires inclusion in L1, not just references of L1 in L2. +// Inclusion is guaranteed by rewinding back a full sequence window before the L2 block that has +// a canonical origin and is before or equal the previous safe L2 block. +// - The *finalized L2 block*: This is the highest L2 block which is fully derived from finalized L1 data. +// This block does not change upon a reorg, assuming L1 cannot reorg finalized data. +// Additionally, the safe block and unsafe block cannot rewind past the finalized L2 block. +// - The *starting L1 origin*: This is the L1 origin to restart the derivation process at. +// This may be behind the L1 origin of the safe L2 block, +// since the safe L2 block cannot be reset further than the finalized L2 block. // -// (1) Plausible meaning that the blockhash of the L2 block's L1 origin (as reported in the L1 -// -// Attributes deposit within the L2 block) is not canonical at another height in the L1 chain, -// and the same holds for all its ancestors. -// -// In particular, in the case of L1 extension, the L2 unsafe head will generally remain the same, -// but in the case of an L1 re-org, we need to search for the new safe and unsafe L2 block. +// (1) Plausible meaning that the blockhash of the L2 block's L1 origin (as reported in the L1) package sync import ( "context" "errors" "fmt" + "io" + + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -50,154 +57,194 @@ import ( type L1Chain interface { L1BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L1BlockRef, error) L1BlockRefByNumber(ctx context.Context, number uint64) (eth.L1BlockRef, error) + L1BlockRefByHash(ctx context.Context, hash common.Hash) (eth.L1BlockRef, error) } type L2Chain interface { L2BlockRefByHash(ctx context.Context, l2Hash common.Hash) (eth.L2BlockRef, error) + L2BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L2BlockRef, error) } +const MaxReorgDepth = 1000 + +var ReorgFinalizedErr = errors.New("cannot reorg finalized block") var WrongChainErr = errors.New("wrong chain") var TooDeepReorgErr = errors.New("reorg is too deep") -const MaxReorgDepth = 500 - -// isCanonical returns the following values: -// - `aheadOrCanonical: true if the supplied block is ahead of the known head of the L1 chain, -// or canonical in the L1 chain. -// - `canonical`: true if the block is canonical in the L1 chain. -func isAheadOrCanonical(ctx context.Context, l1 L1Chain, block eth.BlockID) (aheadOrCanonical bool, canonical bool, err error) { - if l1Head, err := l1.L1BlockRefByLabel(ctx, eth.Unsafe); err != nil { - return false, false, err - } else if block.Number > l1Head.Number { - return true, false, nil - } else if canonical, err := l1.L1BlockRefByNumber(ctx, block.Number); err != nil { - return false, false, err - } else { - canonical := canonical.Hash == block.Hash - return canonical, canonical, nil +type FindSyncStart struct { + cfg *rollup.Config + + l1 L1Chain + l2 L2Chain + + finalized eth.L2BlockRef + safeMaybe eth.L2BlockRef + safe eth.L2BlockRef + unsafe eth.L2BlockRef + + // currentL2 is used to traverse from unsafe L2 head all the way down to the L2 block that we can fully derive from L1, + // to use its origin as starting point. Along the traversal we update the other values. + currentL2 eth.L2BlockRef + // currentL1 represents the L1 data, when available + currentL1 eth.L1BlockRef + // currentL1 may not always exist, so we traverse the chain by needed number, + // and get the full reference data when we can + currentL1Needed uint64 + + startL1 eth.L1BlockRef +} + +func NewFindSyncStart(cfg *rollup.Config, l1 L1Chain, l2 L2Chain) *FindSyncStart { + return &FindSyncStart{ + cfg: cfg, + l1: l1, + l2: l2, } } -// FindL2Heads walks back from `start` (the previous unsafe L2 block) and finds the unsafe and safe -// L2 blocks. -// -// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a plausible (1) -// extension of the canonical L1 chain (as known to the op-node). -// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is -// complete within the canonical L1 chain (as known to the op-node). -// -// (1) Plausible meaning that the blockhash of the L2 block's L1 origin (as reported in the L1 -// -// Attributes deposit within the L2 block) is not canonical at another height in the L1 chain, -// and the same holds for all its ancestors. -func FindL2Heads(ctx context.Context, start eth.L2BlockRef, seqWindowSize uint64, - l1 L1Chain, l2 L2Chain, genesis *rollup.Genesis) (unsafe eth.L2BlockRef, safe eth.L2BlockRef, err error) { - - // Loop 1. Walk the L2 chain backwards until we find an L2 block whose L1 origin is canonical. - - // Current L2 block. - n := start - - // Number of blocks between n and start. - reorgDepth := 0 - - // Blockhash of L1 origin hash for the L2 block during the previous iteration, 0 for first - // iteration. When this changes as we walk the L2 chain backwards, it means we're seeing a different - // (earlier) epoch. - var prevL1OriginHash common.Hash - - // The highest L2 ancestor of `start` (or `start` itself) whose ancestors are not (yet) known - // to have a non-canonical L1 origin. Empty if no such candidate is known yet. Guaranteed to be - // set after exiting from Loop 1. - var highestPlausibleCanonicalOrigin eth.L2BlockRef - - for { - // Check if l1Origin is canonical when we get to a new epoch. - if prevL1OriginHash != n.L1Origin.Hash { - prevL1OriginHash = n.L1Origin.Hash - - if plausible, canonical, err := isAheadOrCanonical(ctx, l1, n.L1Origin); err != nil { - return eth.L2BlockRef{}, eth.L2BlockRef{}, err - } else if !plausible { - // L1 origin nor ahead of L1 head nor canonical, discard previous candidate and - // keep looking. - highestPlausibleCanonicalOrigin = eth.L2BlockRef{} - } else { - if highestPlausibleCanonicalOrigin == (eth.L2BlockRef{}) { - // No highest plausible candidate, make L2 block new candidate. - highestPlausibleCanonicalOrigin = n - } - if canonical { - break - } - } +func (fss *FindSyncStart) Step(ctx context.Context) error { + if fss.finalized == (eth.L2BlockRef{}) { + finalized, err := fss.l2.L2BlockRefByLabel(ctx, eth.Finalized) + if errors.Is(err, ethereum.NotFound) { + // default to genesis if we have not finalized anything before. + finalized, err = fss.l2.L2BlockRefByHash(ctx, fss.cfg.Genesis.L2.Hash) } + if err != nil { + return fmt.Errorf("failed to find the finalized L2 block: %w", err) + } + fss.finalized = finalized + return nil + } - // Don't walk past genesis. If we were at the L2 genesis, but could not find its L1 origin, - // the L2 chain is building on the wrong L1 branch. - if n.Hash == genesis.L2.Hash || n.Number == genesis.L2.Number { - return eth.L2BlockRef{}, eth.L2BlockRef{}, WrongChainErr + if fss.safeMaybe == (eth.L2BlockRef{}) { + safe, err := fss.l2.L2BlockRefByLabel(ctx, eth.Safe) + if errors.Is(err, ethereum.NotFound) { + safe = fss.finalized + } else if err != nil { + return fmt.Errorf("failed to find the safe L2 block: %w", err) } + fss.safeMaybe = safe + return nil + } - // Pull L2 parent for next iteration - n, err = l2.L2BlockRefByHash(ctx, n.ParentHash) + if fss.unsafe == (eth.L2BlockRef{}) { + unsafe, err := fss.l2.L2BlockRefByLabel(ctx, eth.Unsafe) if err != nil { - return eth.L2BlockRef{}, eth.L2BlockRef{}, - fmt.Errorf("failed to fetch L2 block by hash %v: %w", n.ParentHash, err) + return fmt.Errorf("failed to find the L2 head block: %w", err) } + fss.unsafe = unsafe + fss.currentL2 = unsafe + fss.currentL1Needed = unsafe.L1Origin.Number + return nil + } - reorgDepth++ - if reorgDepth >= MaxReorgDepth { - // If the reorg depth is too large, something is fishy. - // This can legitimately happen if L1 goes down for a while. But in that case, - // restarting the L2 node with a bigger configured MaxReorgDepth is an acceptable - // stopgap solution. - // Currently this can also happen if the L2 node is down for a while, but in the future - // state sync should prevent this issue. - return eth.L2BlockRef{}, eth.L2BlockRef{}, TooDeepReorgErr + if fss.currentL1 == (eth.L1BlockRef{}) || fss.currentL1Needed != fss.currentL1.Number { + currentL1, err := fss.l1.L1BlockRefByNumber(ctx, fss.currentL1Needed) + if err == ethereum.NotFound { + if fss.currentL1Needed == fss.cfg.Genesis.L1.Number { + return fmt.Errorf("rollup data starting at L1 block %s is not available on provided L1 node, maybe the L1 node is syncing", fss.cfg.Genesis.L1) + } + // Plausible still, the L1 source may be lagging behind the L1 origins in the L2 unsafe blocks. + fss.currentL1Needed -= 1 + return nil + } else if err != nil { + return fmt.Errorf("failed to get L1 block %d: %w", fss.currentL1Needed, err) + } else { + fss.currentL1 = currentL1 + return nil } } - // Loop 2. Walk from the L1 origin of the `n` block (*) back to the L1 block that starts the - // sequencing window ending at that block. Instead of iterating on L1 blocks, we actually - // iterate on L2 blocks, because we want to find the safe L2 head, i.e. the highest L2 block - // whose L1 origin is the start of the sequencing window. + // Check traversal against Genesis data + if (fss.currentL2.Number == fss.cfg.Genesis.L2.Number) && (fss.currentL2.Hash != fss.cfg.Genesis.L2.Hash) { + return fmt.Errorf("%w L2: genesis: %s, got %s", WrongChainErr, fss.cfg.Genesis.L2, fss.currentL2) + } + + if (fss.currentL1.Number == fss.cfg.Genesis.L1.Number) && (fss.currentL1.Hash != fss.cfg.Genesis.L1.Hash) { + return fmt.Errorf("%w L1: genesis: %s, got %s", WrongChainErr, fss.cfg.Genesis.L1, fss.currentL1) + } - // (*) `n` being at this stage the highest L2 block whose L1 origin is canonical. + // Check traversal against finalized data + if (fss.currentL2.Number == fss.finalized.Number) && (fss.currentL2.Hash != fss.finalized.Hash) { + return fmt.Errorf("%w: finalized %s, got: %s", ReorgFinalizedErr, fss.finalized, fss.currentL2) + } - // Depth counter: we need to walk back `seqWindowSize` L1 blocks in order to find the start - // of the sequencing window. - depth := uint64(1) + // Check we are not reorging incredibly deep + if fss.currentL2.Number+MaxReorgDepth < fss.safe.Number { + return fmt.Errorf("%w: traversed back to L2 block %s, but too deep compared to previous safe block %s", TooDeepReorgErr, fss.currentL2, fss.safe) + } - // Before entering the loop: `prevL1OriginHash == n.L1Origin.Hash` - // The original definitions of `n` and `prevL1OriginHash` still hold. - for { - // Advance depth if we change to a different (earlier) epoch. - if n.L1Origin.Hash != prevL1OriginHash { - depth++ - prevL1OriginHash = n.L1Origin.Hash + // if the L2 chain references a yet unknown L1 origin, then traverse back L2 to find a L1 reference within view + if fss.currentL2.L1Origin.Number > fss.currentL1.Number { + parentL2, err := fss.l2.L2BlockRefByHash(ctx, fss.currentL2.ParentHash) + if err != nil { + return fmt.Errorf("failed to retrieve parent %s of L2 block %s with origin %s to get towards older L1 origin %s: %w", + fss.currentL2.ParentHash, fss.currentL2, fss.currentL2.L1Origin, fss.currentL1, err) } + fss.currentL2 = parentL2 + fss.currentL1Needed = fss.currentL2.L1Origin.Number + return nil + } - // Found an L2 block whose L1 origin is the start of the sequencing window. - // Note: We also ensure that we are on the block number with the 0 seq number. - // This is a little hacky, but kinda works. The issue is about where the - // batch queue should start building. - if depth == seqWindowSize && n.SequenceNumber == 0 { - return highestPlausibleCanonicalOrigin, n, nil + // if the origin of the current block is not canonical, then we have to revert to the parent of this L2 block + if fss.currentL1.Hash != fss.currentL2.L1Origin.Hash { + // and traverse back further + parentL2, err := fss.l2.L2BlockRefByHash(ctx, fss.currentL2.ParentHash) + if err != nil { + return fmt.Errorf("failed to retrieve parent %s of L2 block %s to reorg different L1 origin %s (expected %s): %w", + fss.currentL2.ParentHash, fss.currentL2, fss.currentL2.L1Origin, fss.currentL1, err) } + fss.currentL2 = parentL2 + fss.currentL1Needed = fss.currentL2.L1Origin.Number - // Genesis is always safe. - if n.Hash == genesis.L2.Hash || n.Number == genesis.L2.Number { - safe = eth.L2BlockRef{Hash: genesis.L2.Hash, Number: genesis.L2.Number, - Time: genesis.L2Time, L1Origin: genesis.L1, SequenceNumber: 0} - return highestPlausibleCanonicalOrigin, safe, nil + // it not being canonical also means we have to reorg whatever we previously determined as potential L2 chain + fss.unsafe = parentL2 + if parentL2.Number <= fss.safeMaybe.Number { + fss.safeMaybe = parentL2 } + return nil + } + + // At this point we've initialized everything, and found a L2 block with known L1 origin. + // However, we still have to traverse back a full sequence window, + // to ensure the L2 block not just has the correct origin, but was also derived from L1 data, and thus still safe. + // The confirmed L2 data on the L1 chain can then be reconciled against the now temporarily unsafe L2 chain, + // this is not necessarily a reorg. - // Pull L2 parent for next iteration. - n, err = l2.L2BlockRefByHash(ctx, n.ParentHash) + // initialize the default the safe head to what we know + if fss.safe == (eth.L2BlockRef{}) { + fss.safe = fss.safeMaybe + } + + // Keep resetting back the safe head until the L1 origin is deep enough to guarantee it was fully derived from L1 + if fss.currentL2.L1Origin.Number+fss.cfg.SeqWindowSize > fss.safeMaybe.L1Origin.Number && fss.currentL2.Number > fss.cfg.Genesis.L2.Number { + // and traverse back further + parentL2, err := fss.l2.L2BlockRefByHash(ctx, fss.currentL2.ParentHash) + if err != nil { + return fmt.Errorf("failed to retrieve parent %s of L2 block %s with origin %s to find block %d blocks before last safe L1 block %s: %w", + fss.currentL2.ParentHash, fss.currentL2, fss.currentL2.L1Origin, fss.cfg.SeqWindowSize, fss.safeMaybe.L1Origin, err) + } + fss.currentL2 = parentL2 + fss.currentL1Needed = fss.currentL2.L1Origin.Number + // don't reset back the safe head beyond the finalized head + if parentL2.Number >= fss.finalized.Number { + fss.safe = parentL2 + } + return nil + } + + if fss.startL1 == (eth.L1BlockRef{}) { + ref, err := fss.l1.L1BlockRefByHash(ctx, fss.currentL2.L1Origin.Hash) if err != nil { - return eth.L2BlockRef{}, eth.L2BlockRef{}, - fmt.Errorf("failed to fetch L2 block by hash %v: %w", n.ParentHash, err) + return fmt.Errorf("failed to find full L1 block contents for L1 starting point: %w", err) } + fss.startL1 = ref } + + // finished + return io.EOF +} + +func (fss *FindSyncStart) Result() (finalized, safe, unsafe eth.L2BlockRef, startL1 eth.L1BlockRef) { + return fss.finalized, fss.safe, fss.unsafe, fss.startL1 } diff --git a/op-node/rollup/sync/start_test.go b/op-node/rollup/sync/start_test.go index 3946ccec2724c..903aff85f8659 100644 --- a/op-node/rollup/sync/start_test.go +++ b/op-node/rollup/sync/start_test.go @@ -2,8 +2,11 @@ package sync import ( "context" + "io" "testing" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum-optimism/optimism/op-node/testlog" "github.com/ethereum-optimism/optimism/op-node/testutils" @@ -20,19 +23,23 @@ var _ L2Chain = (*testutils.FakeChainSource)(nil) // - The L2 chain is based off of the L1 chain // - The actual L1 chain is the New L1 chain // - Both heads are at the tip of their respective chains -func (c *syncStartTestCase) generateFakeL2(t *testing.T) (*testutils.FakeChainSource, eth.L2BlockRef, rollup.Genesis) { +func (c *syncStartTestCase) generateFakeL2(t *testing.T) (*testutils.FakeChainSource, rollup.Genesis) { log := testlog.Logger(t, log.LvlError) chain := testutils.NewFakeChainSource([]string{c.L1, c.NewL1}, []string{c.L2}, int(c.GenesisL1Num), log) chain.SetL2Head(len(c.L2) - 1) genesis := testutils.FakeGenesis(c.GenesisL1, c.GenesisL2, int(c.GenesisL1Num)) - head, err := chain.L2BlockRefByNumber(context.Background(), nil) - require.Nil(t, err) chain.ReorgL1() for i := 0; i < len(c.NewL1)-1; i++ { chain.AdvanceL1() } - return chain, head, genesis + return chain, genesis + +} +func runeToHash(id rune) common.Hash { + var h common.Hash + copy(h[:], string(id)) + return h } type syncStartTestCase struct { @@ -42,6 +49,9 @@ type syncStartTestCase struct { L2 string // L2 Chain that follows from L1Chain NewL1 string // New L1 chain + PreFinalizedL2 rune + PreSafeL2 rune + GenesisL1 rune GenesisL1Num uint64 GenesisL2 rune @@ -57,188 +67,239 @@ func refToRune(r eth.BlockID) rune { } func (c *syncStartTestCase) Run(t *testing.T) { - chain, l2Head, genesis := c.generateFakeL2(t) + chain, genesis := c.generateFakeL2(t) + chain.SetL2Finalized(runeToHash(c.PreFinalizedL2)) + chain.SetL2Safe(runeToHash(c.PreSafeL2)) - unsafeL2Head, safeHead, err := FindL2Heads(context.Background(), l2Head, c.SeqWindowSize, chain, chain, &genesis) + cfg := &rollup.Config{ + Genesis: genesis, + SeqWindowSize: c.SeqWindowSize, + } + syncStart := NewFindSyncStart(cfg, chain, chain) + ctx := context.Background() + i := 0 + for { + err := syncStart.Step(ctx) + if err == io.EOF { + break + } + if err != nil { + if c.ExpectedErr != nil { + require.Error(t, err, "Expecting an error in this test case") + require.ErrorIs(t, err, c.ExpectedErr, "Unexpected error") + return + } else { + t.Fatalf("unexpected error: %v", err) + } + } + i++ + if i > 1000 { + t.Fatal("too many steps to find sync start, infinite loop?") + } + } + finalized, safe, unsafe, startL1 := syncStart.Result() if c.ExpectedErr != nil { - require.Error(t, err, "Expecting an error in this test case") - require.ErrorIs(t, c.ExpectedErr, err, "Unexpected error") - } else { + t.Fatal("expected error, but got sync result", finalized, safe, unsafe, startL1) + } - require.NoError(t, err) - expectedUnsafeHead := refToRune(unsafeL2Head.ID()) - require.Equal(t, string(c.UnsafeL2Head), string(expectedUnsafeHead), "Unsafe L2 Head not equal") + gotUnsafeHead := refToRune(unsafe.ID()) + require.Equal(t, string(c.UnsafeL2Head), string(gotUnsafeHead), "Unsafe L2 Head not equal") - expectedSafeHead := refToRune(safeHead.ID()) - require.Equal(t, string(c.SafeL2Head), string(expectedSafeHead), "Safe L2 Head not equal") - } + gotSafeHead := refToRune(safe.ID()) + require.Equal(t, string(c.SafeL2Head), string(gotSafeHead), "Safe L2 Head not equal") } func TestFindSyncStart(t *testing.T) { testCases := []syncStartTestCase{ { - Name: "already synced", - GenesisL1Num: 0, - L1: "ab", - L2: "AB", - NewL1: "ab", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'B', - SeqWindowSize: 2, - SafeL2Head: 'A', - ExpectedErr: nil, + Name: "already synced", + GenesisL1Num: 0, + L1: "ab", + L2: "AB", + NewL1: "ab", + PreFinalizedL2: 'A', + PreSafeL2: 'A', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'B', + SeqWindowSize: 2, + SafeL2Head: 'A', + ExpectedErr: nil, }, { - Name: "small reorg long chain", - GenesisL1Num: 0, - L1: "abcdefgh", - L2: "ABCDEFGH", - NewL1: "abcdefgx", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'G', - SeqWindowSize: 2, - SafeL2Head: 'F', - ExpectedErr: nil, + Name: "small reorg long chain", + GenesisL1Num: 0, + L1: "abcdefgh", + L2: "ABCDEFGH", + NewL1: "abcdefgx", + PreFinalizedL2: 'B', + PreSafeL2: 'H', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'G', + SeqWindowSize: 2, + SafeL2Head: 'E', + ExpectedErr: nil, }, { - Name: "L1 Chain ahead", - GenesisL1Num: 0, - L1: "abcde", - L2: "ABCD", - NewL1: "abcde", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'D', - SeqWindowSize: 3, - SafeL2Head: 'B', - ExpectedErr: nil, + Name: "L1 Chain ahead", + GenesisL1Num: 0, + L1: "abcdef", + L2: "ABCDE", + NewL1: "abcdef", + PreFinalizedL2: 'A', + PreSafeL2: 'D', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'E', + SeqWindowSize: 2, + SafeL2Head: 'B', + ExpectedErr: nil, }, { - Name: "L2 Chain ahead after reorg", - GenesisL1Num: 0, - L1: "abxyz", - L2: "ABXYZ", - NewL1: "abx", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'Z', - SeqWindowSize: 2, - SafeL2Head: 'B', - ExpectedErr: nil, + Name: "L2 Chain ahead after reorg", + GenesisL1Num: 0, + L1: "abcxyz", + L2: "ABCXYZ", + NewL1: "abcx", + PreFinalizedL2: 'B', + PreSafeL2: 'X', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'Z', + SeqWindowSize: 2, + SafeL2Head: 'B', + ExpectedErr: nil, }, { - Name: "genesis", - GenesisL1Num: 0, - L1: "a", - L2: "A", - NewL1: "a", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'A', - SeqWindowSize: 2, - SafeL2Head: 'A', - ExpectedErr: nil, + Name: "genesis", + GenesisL1Num: 0, + L1: "a", + L2: "A", + NewL1: "a", + PreFinalizedL2: 'A', + PreSafeL2: 'A', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'A', + SeqWindowSize: 2, + SafeL2Head: 'A', + ExpectedErr: nil, }, { - Name: "reorg one step back", - GenesisL1Num: 0, - L1: "abcd", - L2: "ABCD", - NewL1: "abcx", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'C', - SeqWindowSize: 3, - SafeL2Head: 'A', - ExpectedErr: nil, + Name: "reorg one step back", + GenesisL1Num: 0, + L1: "abcdefg", + L2: "ABCDEFG", + NewL1: "abcdefx", + PreFinalizedL2: 'A', + PreSafeL2: 'E', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'F', + SeqWindowSize: 3, + SafeL2Head: 'B', + ExpectedErr: nil, }, { - Name: "reorg two steps back", - GenesisL1Num: 0, - L1: "abc", - L2: "ABC", - NewL1: "axy", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'A', - SeqWindowSize: 2, - SafeL2Head: 'A', - ExpectedErr: nil, + Name: "reorg two steps back, clip genesis and finalized", + GenesisL1Num: 0, + L1: "abc", + L2: "ABC", + PreFinalizedL2: 'A', + PreSafeL2: 'B', + NewL1: "axy", + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'A', + SeqWindowSize: 2, + SafeL2Head: 'A', + ExpectedErr: nil, }, { - Name: "reorg three steps back", - GenesisL1Num: 0, - L1: "abcdef", - L2: "ABCDEF", - NewL1: "abcxyz", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 'C', - SeqWindowSize: 2, - SafeL2Head: 'B', - ExpectedErr: nil, + Name: "reorg three steps back", + GenesisL1Num: 0, + L1: "abcdefg", + L2: "ABCDEFG", + NewL1: "abcdxyz", + PreFinalizedL2: 'A', + PreSafeL2: 'D', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 'D', + SeqWindowSize: 2, + SafeL2Head: 'B', + ExpectedErr: nil, }, { - Name: "unexpected L1 chain", - GenesisL1Num: 0, - L1: "abcdef", - L2: "ABCDEF", - NewL1: "xyzwio", - GenesisL1: 'a', - GenesisL2: 'A', - UnsafeL2Head: 0, - ExpectedErr: WrongChainErr, + Name: "unexpected L1 chain", + GenesisL1Num: 0, + L1: "abcdef", + L2: "ABCDEF", + NewL1: "xyzwio", + PreFinalizedL2: 'A', + PreSafeL2: 'B', + GenesisL1: 'a', + GenesisL2: 'A', + UnsafeL2Head: 0, + ExpectedErr: WrongChainErr, }, { - Name: "unexpected L2 chain", - GenesisL1Num: 0, - L1: "abcdef", - L2: "ABCDEF", - NewL1: "xyzwio", - GenesisL1: 'a', - GenesisL2: 'X', - UnsafeL2Head: 0, - ExpectedErr: WrongChainErr, + Name: "unexpected L2 chain", + GenesisL1Num: 0, + L1: "abcdef", + L2: "ABCDEF", + NewL1: "xyzwio", + PreFinalizedL2: 'A', + PreSafeL2: 'B', + GenesisL1: 'a', + GenesisL2: 'X', + UnsafeL2Head: 0, + ExpectedErr: WrongChainErr, }, { - Name: "offset L2 genesis", - GenesisL1Num: 3, - L1: "abcdef", - L2: "DEF", - NewL1: "abcdef", - GenesisL1: 'd', - GenesisL2: 'D', - UnsafeL2Head: 'F', - SeqWindowSize: 2, - SafeL2Head: 'E', - ExpectedErr: nil, + Name: "offset L2 genesis", + GenesisL1Num: 3, + L1: "abcdefghi", + L2: "DEFGHI", + NewL1: "abcdefghi", + PreFinalizedL2: 'E', + PreSafeL2: 'H', + GenesisL1: 'd', + GenesisL2: 'D', + UnsafeL2Head: 'I', + SeqWindowSize: 2, + SafeL2Head: 'F', + ExpectedErr: nil, }, { - Name: "offset L2 genesis reorg", - GenesisL1Num: 3, - L1: "abcdefgh", - L2: "DEFGH", - NewL1: "abcdxyzw", - GenesisL1: 'd', - GenesisL2: 'D', - UnsafeL2Head: 'D', - SeqWindowSize: 2, - SafeL2Head: 'D', - ExpectedErr: nil, + Name: "offset L2 genesis reorg", + GenesisL1Num: 3, + L1: "abcdefgh", + L2: "DEFGH", + NewL1: "abcdxyzw", + PreFinalizedL2: 'D', + PreSafeL2: 'D', + GenesisL1: 'd', + GenesisL2: 'D', + UnsafeL2Head: 'D', + SeqWindowSize: 2, + SafeL2Head: 'D', + ExpectedErr: nil, }, { - Name: "reorg past offset genesis", - GenesisL1Num: 3, - L1: "abcdefgh", - L2: "DEFGH", - NewL1: "abxyzwio", - GenesisL1: 'd', - GenesisL2: 'D', - UnsafeL2Head: 0, - ExpectedErr: WrongChainErr, + Name: "reorg past offset genesis", + GenesisL1Num: 3, + L1: "abcdefgh", + L2: "DEFGH", + NewL1: "abxyzwio", + PreFinalizedL2: 'D', + PreSafeL2: 'D', + GenesisL1: 'd', + GenesisL2: 'D', + UnsafeL2Head: 0, + ExpectedErr: WrongChainErr, }, } diff --git a/op-node/testutils/fake_chain.go b/op-node/testutils/fake_chain.go index 4a46cb3dde469..da2d5f643cc78 100644 --- a/op-node/testutils/fake_chain.go +++ b/op-node/testutils/fake_chain.go @@ -85,13 +85,17 @@ func NewFakeChainSource(l1 []string, l2 []string, l1GenesisNumber int, log log.L // what the head block is of the L1 and L2 chains. In addition, it enables re-orgs // to easily be implemented type FakeChainSource struct { - l1reorg int // Index of the L1 chain to be operating on - l2reorg int // Index of the L2 chain to be operating on - l1head int // Head block of the L1 chain - l2head int // Head block of the L2 chain - l1s [][]eth.L1BlockRef // l1s[reorg] is the L1 chain in that specific re-org configuration - l2s [][]eth.L2BlockRef // l2s[reorg] is the L2 chain in that specific re-org configuration - log log.Logger + l1reorg int // Index of the L1 chain to be operating on + l2reorg int // Index of the L2 chain to be operating on + l1head int // Head block of the L1 chain + l2head int // Head block of the L2 chain + l1safe int + l2safe int + l1finalized int + l2finalized int + l1s [][]eth.L1BlockRef // l1s[reorg] is the L1 chain in that specific re-org configuration + l2s [][]eth.L2BlockRef // l2s[reorg] is the L2 chain in that specific re-org configuration + log log.Logger } func (m *FakeChainSource) L1Range(ctx context.Context, base eth.BlockID, max uint64) ([]eth.BlockID, error) { @@ -134,26 +138,39 @@ func (m *FakeChainSource) L1BlockRefByHash(ctx context.Context, l1Hash common.Ha } func (m *FakeChainSource) L1BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L1BlockRef, error) { - if label != eth.Unsafe { - return eth.L1BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L1BlockRefByLabel(%s)", label) - } - m.log.Trace("L1HeadBlockRef", "l1Head", m.l1head, "reorg", m.l1reorg) + m.log.Trace("L1BlockRefByLabel", "l1Head", m.l1head, "l1Safe", m.l1safe, "l1Finalized", m.l1finalized, "reorg", m.l1reorg) l := len(m.l1s[m.l1reorg]) if l == 0 { return eth.L1BlockRef{}, ethereum.NotFound } - return m.l1s[m.l1reorg][m.l1head], nil + switch label { + case eth.Unsafe: + return m.l1s[m.l1reorg][m.l1head], nil + case eth.Safe: + return m.l1s[m.l1reorg][m.l1safe], nil + case eth.Finalized: + return m.l1s[m.l1reorg][m.l1finalized], nil + default: + return eth.L1BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L1BlockRefByLabel(%s)", label) + } } func (m *FakeChainSource) L2BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L2BlockRef, error) { - if label != eth.Unsafe { - return eth.L2BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L2BlockRefByLabel(%s)", label) + m.log.Trace("L2BlockRefByLabel", "l2Head", m.l2head, "l2Safe", m.l2safe, "l2Finalized", m.l2finalized, "reorg", m.l2reorg) + l := len(m.l2s[m.l2reorg]) + if l == 0 { + return eth.L2BlockRef{}, ethereum.NotFound } - m.log.Trace("L2BlockRefHead", "l2Head", m.l2head, "reorg", m.l2reorg) - if len(m.l2s[m.l2reorg]) == 0 { - panic("bad test, no l2 chain") + switch label { + case eth.Unsafe: + return m.l2s[m.l2reorg][m.l2head], nil + case eth.Safe: + return m.l2s[m.l2reorg][m.l2safe], nil + case eth.Finalized: + return m.l2s[m.l2reorg][m.l2finalized], nil + default: + return eth.L2BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L2BlockRefByLabel(%s)", label) } - return m.l2s[m.l2reorg][m.l2head], nil } func (m *FakeChainSource) L2BlockRefByNumber(ctx context.Context, l2Num *big.Int) (eth.L2BlockRef, error) { @@ -204,6 +221,28 @@ func (m *FakeChainSource) ReorgL1() { } } +func (m *FakeChainSource) SetL2Safe(safe common.Hash) { + m.log.Trace("Set L2 safe head", "new_safe", safe, "old_safe", m.l2safe) + for i, v := range m.l2s[m.l2reorg] { + if v.Hash == safe { + m.l2safe = i + return + } + } + panic(fmt.Errorf("unknown safe block: %s", safe)) +} + +func (m *FakeChainSource) SetL2Finalized(finalized common.Hash) { + m.log.Trace("Set L2 finalized head", "new_finalized", finalized, "old_finalized", m.l2finalized) + for i, v := range m.l2s[m.l2reorg] { + if v.Hash == finalized { + m.l2finalized = i + return + } + } + panic(fmt.Errorf("unknown finalized block: %s", finalized)) +} + func (m *FakeChainSource) SetL2Head(head int) eth.L2BlockRef { m.log.Trace("Set L2 head", "new_head", head, "old_head", m.l2head) m.l2head = head