op-node: update OnUnsafeL2Payload to perform ELSync gap filling

op-acceptance-tests/tests/depreqres/common/common.go

@@ -0,0 +1,103 @@
+package common
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
+	"github.com/ethereum-optimism/optimism/op-devstack/dsl"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
+)
+
+func UnsafeChainNotStalling_Disconnect(gt *testing.T, syncMode sync.Mode, sleep time.Duration) {
+	t := devtest.SerialT(gt)
+	sys := presets.NewSingleChainMultiNodeWithoutCheck(t)
+	require := t.Require()
+	l := t.Logger().With("syncmode", syncMode)
+
+	l.Info("Confirm that the CL nodes are progressing the unsafe chain")
+	target := uint64(3)
+	dsl.CheckAll(t,
+		sys.L2CL.AdvancedFn(types.LocalUnsafe, target, 30),
+		sys.L2CLB.AdvancedFn(types.LocalUnsafe, target, 30),
+	)
+
+	l.Info("Disconnect L2CL from L2CLB, and vice versa")
+	sys.L2CLB.DisconnectPeer(sys.L2CL)
+	sys.L2CL.DisconnectPeer(sys.L2CLB)
+
+	ssA_before := sys.L2CL.SyncStatus()
+	ssB_before := sys.L2CLB.SyncStatus()
+
+	l.Info("L2CL status before delay", "unsafeL2", ssA_before.UnsafeL2.ID(), "safeL2", ssA_before.SafeL2.ID())
+	l.Info("L2CLB status before delay", "unsafeL2", ssB_before.UnsafeL2.ID(), "safeL2", ssB_before.SafeL2.ID())
+
+	time.Sleep(sleep)
+
+	ssA_after := sys.L2CL.SyncStatus()
+	ssB_after := sys.L2CLB.SyncStatus()
+
+	l.Info("L2CL status after delay", "unsafeL2", ssA_after.UnsafeL2.ID(), "safeL2", ssA_after.SafeL2.ID())
+	l.Info("L2CLB status after delay", "unsafeL2", ssB_after.UnsafeL2.ID(), "safeL2", ssB_after.SafeL2.ID())
+
+	require.Greater(ssA_after.UnsafeL2.Number, ssA_before.UnsafeL2.Number, "unsafe chain for L2CL should have advanced")
+	require.Equal(ssB_after.UnsafeL2.Number, ssB_before.UnsafeL2.Number, "unsafe chain for L2CLB should have stalled")
+
+	l.Info("Re-connect L2CL to L2CLB")
+	sys.L2CLB.ConnectPeer(sys.L2CL)
+	sys.L2CL.ConnectPeer(sys.L2CLB)
+
+	l.Info("Confirm that the unsafe chain for L2CLB is not stalled")
+	sys.L2CLB.Reached(types.LocalUnsafe, ssA_after.UnsafeL2.Number, 30)
+	sys.L2ELB.Reached(eth.Unsafe, ssA_after.UnsafeL2.Number, 30)
+}
+
+func UnsafeChainNotStalling_RestartOpNode(gt *testing.T, syncMode sync.Mode, sleep time.Duration) {
+	t := devtest.SerialT(gt)
+	sys := presets.NewSingleChainMultiNodeWithoutCheck(t)
+	require := t.Require()
+	l := t.Logger().With("syncmode", syncMode)
+
+	l.Info("Confirm that the CL nodes are progressing the unsafe chain")
+	target := uint64(3)
+	dsl.CheckAll(t,
+		sys.L2CL.AdvancedFn(types.LocalUnsafe, target, 30),
+		sys.L2CLB.AdvancedFn(types.LocalUnsafe, target, 30),
+	)
+
+	l.Info("Disconnect L2CL from L2CLB, and vice versa")
+	sys.L2CLB.DisconnectPeer(sys.L2CL)
+	sys.L2CL.DisconnectPeer(sys.L2CLB)
+
+	ssA_before := sys.L2CL.SyncStatus()
+	ssB_before := sys.L2CLB.SyncStatus()
+
+	l.Info("L2CL status before delay", "unsafeL2", ssA_before.UnsafeL2.ID(), "safeL2", ssA_before.SafeL2.ID())
+	l.Info("L2CLB status before delay", "unsafeL2", ssB_before.UnsafeL2.ID(), "safeL2", ssB_before.SafeL2.ID())
+
+	sys.L2CLB.Stop()
+
+	time.Sleep(sleep)
+
+	sys.L2CLB.Start()
+
+	ssA_after := sys.L2CL.SyncStatus()
+	ssB_after := sys.L2CLB.SyncStatus()
+
+	l.Info("L2CL status after delay", "unsafeL2", ssA_after.UnsafeL2.ID(), "safeL2", ssA_after.SafeL2.ID())
+	l.Info("L2CLB status after delay", "unsafeL2", ssB_after.UnsafeL2.ID(), "safeL2", ssB_after.SafeL2.ID())
+
+	require.Greater(ssA_after.UnsafeL2.Number, ssA_before.UnsafeL2.Number, "unsafe chain for L2CL should have advanced")
+	require.LessOrEqual(ssB_after.UnsafeL2.Number, ssB_before.UnsafeL2.Number, "unsafe chain for L2CLB should have stalled")
+
+	l.Info("Re-connect L2CL to L2CLB")
+	sys.L2CLB.ConnectPeer(sys.L2CL)
+	sys.L2CL.ConnectPeer(sys.L2CLB)
+
+	l.Info("Confirm that the unsafe chain for L2CLB is not stalled")
+	sys.L2CLB.Reached(types.LocalUnsafe, ssA_after.UnsafeL2.Number, 30)
+	sys.L2ELB.Reached(eth.Unsafe, ssA_after.UnsafeL2.Number, 30)
+}

op-acceptance-tests/tests/depreqres/reqressyncdisabled/clsync/clsync_test.go

@@ -0,0 +1,21 @@
+package clsync
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/common"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
+)
+
+func TestUnsafeChainNotStalling_CLSync_Short(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.CLSync, 20*time.Second)
+}
+
+func TestUnsafeChainNotStalling_CLSync_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.CLSync, 95*time.Second)
+}
+
+func TestUnsafeChainNotStalling_CLSync_RestartOpNode_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_RestartOpNode(gt, sync.CLSync, 95*time.Second)
+}

op-acceptance-tests/tests/depreqres/reqressyncdisabled/clsync/init_test.go

@@ -0,0 +1,23 @@
+package clsync
+
+import (
+	"testing"
+
+	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
+	"github.com/ethereum-optimism/optimism/op-devstack/compat"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
+)
+
+func TestMain(m *testing.M) {
+	presets.DoMain(m, presets.WithSingleChainMultiNode(),
+		presets.WithConsensusLayerSync(),
+		presets.WithCompatibleTypes(compat.SysGo),
+		presets.WithReqRespSyncDisabled(),
+		presets.WithNoDiscovery(),
+		stack.MakeCommon(sysgo.WithBatcherOption(func(id stack.L2BatcherID, cfg *bss.CLIConfig) {
+			cfg.Stopped = true
+		})),
+	)
+}

op-acceptance-tests/tests/depreqres/depreqres_test.go → op-acceptance-tests/tests/depreqres/reqressyncdisabled/depreqres_test.go

@@ -7,25 +7,23 @@ import (
 	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
 	"github.com/ethereum-optimism/optimism/op-devstack/dsl"
 	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
 )
 
-func TestUnsafeChainStalling_DisabledReqRespSync(gt *testing.T) {
+func TestUnsafeChainNotStalling_DisabledReqRespSync(gt *testing.T) {
 	t := devtest.SerialT(gt)
-	sys := presets.NewSingleChainMultiNode(t)
+	sys := presets.NewSingleChainMultiNodeWithoutCheck(t)
 	require := t.Require()
 	l := t.Logger()
 
 	l.Info("Confirm that the CL nodes are progressing the unsafe chain")
-	target := uint64(10)
+	delta := uint64(3)
 	dsl.CheckAll(t,
-		sys.L2CL.AdvancedFn(types.LocalUnsafe, target, 30),
-		sys.L2CLB.AdvancedFn(types.LocalUnsafe, target, 30),
+		sys.L2CL.AdvancedFn(types.LocalUnsafe, delta, 30),
+		sys.L2CLB.AdvancedFn(types.LocalUnsafe, delta, 30),
 	)
 
-	l.Info("Stop the L2 batcher")
-	sys.L2Batcher.Stop()
-
 	l.Info("Disconnect L2CL from L2CLB, and vice versa")
 	sys.L2CLB.DisconnectPeer(sys.L2CL)
 	sys.L2CL.DisconnectPeer(sys.L2CLB)
@@ -51,6 +49,9 @@ func TestUnsafeChainStalling_DisabledReqRespSync(gt *testing.T) {
 	sys.L2CLB.ConnectPeer(sys.L2CL)
 	sys.L2CL.ConnectPeer(sys.L2CLB)
 
-	l.Info("Confirm that the unsafe chain for L2CLB is stalled")
-	sys.L2CLB.NotAdvanced(types.LocalUnsafe, 10)
+	l.Info("Confirm that the unsafe chain for L2CLB can advance")
+	dsl.CheckAll(t,
+		sys.L2CLB.AdvancedFn(types.LocalUnsafe, delta, 30),
+		sys.L2ELB.AdvancedFn(eth.Unsafe, delta),
+	)
 }

op-acceptance-tests/tests/depreqres/reqressyncdisabled/divergence/divergence_test.go

@@ -0,0 +1,70 @@
+package divergence
+
+import (
+	"testing"
+
+	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
+	"github.com/ethereum-optimism/optimism/op-devstack/compat"
+	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
+	"github.com/ethereum/go-ethereum"
+)
+
+func TestMain(m *testing.M) {
+	// No ELP2P, CLP2P to control the supply of unsafe payload to the CL
+	presets.DoMain(m, presets.WithSingleChainMultiNodeWithoutP2P(),
+		presets.WithCompatibleTypes(compat.SysGo),
+		presets.WithExecutionLayerSyncOnVerifiers(),
+		presets.WithReqRespSyncDisabled(),
+		presets.WithNoDiscovery(),
+		stack.MakeCommon(sysgo.WithBatcherOption(func(id stack.L2BatcherID, cfg *bss.CLIConfig) {
+			cfg.Stopped = true
+		})),
+	)
+}
+
+// TestCLELDivergence tests that the CL and EL diverge when the CL advances the unsafe head, due to accepting SYNCING response from the EL, but the EL cannot validate the block (yet), does not canonicalize it, and doesn't serve it.
+func TestCLELDivergence(gt *testing.T) {
+	t := devtest.SerialT(gt)
+	sys := presets.NewSingleChainMultiNodeWithoutCheck(t)
+	require := t.Require()
+	l := t.Logger()
+
+	sys.L2CL.Advanced(types.LocalUnsafe, 8, 30)
+
+	// batcher down so safe not advanced
+	require.Equal(uint64(0), sys.L2CL.HeadBlockRef(types.LocalSafe).Number)
+	require.Equal(uint64(0), sys.L2CLB.HeadBlockRef(types.LocalSafe).Number)
+
+	startNum := sys.L2CLB.HeadBlockRef(types.LocalUnsafe).Number
+
+	// Finish EL sync by supplying the first block
+	// EL Sync finished because underlying EL has states to validate the payload for block startNum+1
+	sys.L2CLB.SignalTarget(sys.L2EL, startNum+1)
+	require.Equal(startNum+1, sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number)
+
+	for _, delta := range []uint64{3, 4, 5} {
+		targetNumber := startNum + delta
+		l.Info("Sending payload ", "target", targetNumber, "startNum", startNum)
+		sys.L2CLB.SignalTarget(sys.L2EL, targetNumber)
+
+		// Canonical unsafe head never advances because of the gap
+		require.Equal(startNum+1, sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number)
+
+		// Unsafe head on CL advanced, but on EL we cannot fetch state for the unsafe block hash yet
+		targetBlock := sys.L2EL.BlockRefByNumber(targetNumber)
+
+		// Confirm that L2CLB SyncStatus returns the newest unsafe block number and hash
+		ss := sys.L2CLB.SyncStatus()
+		require.Equal(targetNumber, ss.UnsafeL2.Number)
+		require.Equal(targetBlock.Hash, ss.UnsafeL2.Hash)
+
+		// Confirm that L2ELB cannot fetch the block by hash yet, because the block is not canonicalized, even though the CL reference is set to it.
+		_, err := sys.L2ELB.Escape().L2EthClient().L2BlockRefByHash(t.Ctx(), ss.UnsafeL2.Hash)
+		require.Error(err, ethereum.NotFound)
+	}
+}

op-acceptance-tests/tests/depreqres/reqressyncdisabled/elsync/elsync_test.go

@@ -0,0 +1,21 @@
+package elsync
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/common"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
+)
+
+func TestUnsafeChainNotStalling_ELSync_Short(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.ELSync, 20*time.Second)
+}
+
+func TestUnsafeChainNotStalling_ELSync_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.ELSync, 95*time.Second)
+}
+
+func TestUnsafeChainNotStalling_ELSync_RestartOpNode_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_RestartOpNode(gt, sync.ELSync, 95*time.Second)
+}

op-acceptance-tests/tests/depreqres/reqressyncdisabled/elsync/init_test.go

@@ -0,0 +1,23 @@
+package elsync
+
+import (
+	"testing"
+
+	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
+	"github.com/ethereum-optimism/optimism/op-devstack/compat"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
+)
+
+func TestMain(m *testing.M) {
+	presets.DoMain(m, presets.WithSingleChainMultiNode(),
+		presets.WithExecutionLayerSyncOnVerifiers(),
+		presets.WithCompatibleTypes(compat.SysGo),
+		presets.WithReqRespSyncDisabled(),
+		presets.WithNoDiscovery(),
+		stack.MakeCommon(sysgo.WithBatcherOption(func(id stack.L2BatcherID, cfg *bss.CLIConfig) {
+			cfg.Stopped = true
+		})),
+	)
+}

op-acceptance-tests/tests/depreqres/init_test.go → op-acceptance-tests/tests/depreqres/reqressyncdisabled/init_test.go

@@ -3,8 +3,11 @@ package depreqres
 import (
 	"testing"
 
+	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
 	"github.com/ethereum-optimism/optimism/op-devstack/compat"
 	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
 )
 
 func TestMain(m *testing.M) {
@@ -13,5 +16,8 @@ func TestMain(m *testing.M) {
 		presets.WithCompatibleTypes(compat.SysGo),
 		presets.WithReqRespSyncDisabled(),
 		presets.WithNoDiscovery(),
+		stack.MakeCommon(sysgo.WithBatcherOption(func(id stack.L2BatcherID, cfg *bss.CLIConfig) {
+			cfg.Stopped = true
+		})),
 	)
 }

op-acceptance-tests/tests/depreqres/syncmodereqressync/clsync/clsync_test.go

@@ -0,0 +1,21 @@
+package clsync
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/common"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
+)
+
+func TestUnsafeChainNotStalling_CLSync_Short(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.CLSync, 20*time.Second)
+}
+
+func TestUnsafeChainNotStalling_CLSync_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.CLSync, 95*time.Second)
+}
+
+func TestUnsafeChainNotStalling_CLSync_RestartOpNode_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_RestartOpNode(gt, sync.CLSync, 95*time.Second)
+}

op-acceptance-tests/tests/depreqres/syncmodereqressync/clsync/init_test.go

@@ -0,0 +1,23 @@
+package clsync
+
+import (
+	"testing"
+
+	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
+	"github.com/ethereum-optimism/optimism/op-devstack/compat"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
+)
+
+func TestMain(m *testing.M) {
+	presets.DoMain(m, presets.WithSingleChainMultiNode(),
+		presets.WithConsensusLayerSync(),
+		presets.WithCompatibleTypes(compat.SysGo),
+		presets.WithSyncModeReqRespSync(),
+		presets.WithNoDiscovery(),
+		stack.MakeCommon(sysgo.WithBatcherOption(func(id stack.L2BatcherID, cfg *bss.CLIConfig) {
+			cfg.Stopped = true
+		})),
+	)
+}

op-acceptance-tests/tests/depreqres/syncmodereqressync/elsync/elsync_test.go

@@ -0,0 +1,21 @@
+package elsync
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/common"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
+)
+
+func TestUnsafeChainNotStalling_ELSync_Short(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.ELSync, 20*time.Second)
+}
+
+func TestUnsafeChainNotStalling_ELSync_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_Disconnect(gt, sync.ELSync, 95*time.Second)
+}
+
+func TestUnsafeChainNotStalling_ELSync_RestartOpNode_Long(gt *testing.T) {
+	common.UnsafeChainNotStalling_RestartOpNode(gt, sync.ELSync, 95*time.Second)
+}