diff --git a/.circleci/config.yml b/.circleci/config.yml index 12fd931fcd8..858e9bf3dd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3243,7 +3243,7 @@ workflows: - op-acceptance-tests: name: memory-all gate: "" # Empty gate = gateless mode - no_output_timeout: 80m # Keep this less than 90m to avoid CircleCI timeout + no_output_timeout: 120m # Allow longer runs for memory-all gate context: - circleci-repo-readonly-authenticated-github-token - slack diff --git a/op-acceptance-tests/justfile b/op-acceptance-tests/justfile index 266bd7bac28..633894f24d8 100644 --- a/op-acceptance-tests/justfile +++ b/op-acceptance-tests/justfile @@ -101,7 +101,7 @@ acceptance-test devnet="" gate="base": "--validators" "./acceptance-tests.yaml" "--exclude-gates" "flake-shake" "--allow-skips" - "--timeout" "60m" + "--timeout" "120m" "--orchestrator" "$ORCHESTRATOR" "--show-progress" ) diff --git a/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go b/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go index fd770988cf4..7e61b8931e7 100644 --- a/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go +++ b/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go @@ -1,6 +1,3 @@ -//go:build !ci - -// use a tag prefixed with "!". Such tag ensures that the default behaviour of this test would be to be built/run even when the go toolchain (go test) doesn't specify any tag filter. package conductor import ( diff --git a/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go b/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go index c534aae772f..cd08cb0e20c 100644 --- a/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go +++ b/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go @@ -6,16 +6,18 @@ package flashblocks import ( "context" "encoding/json" - "fmt" + "log/slog" "os" "strconv" "testing" "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" - "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" - "github.com/ethereum-optimism/optimism/op-service/testlog" + "github.com/ethereum-optimism/optimism/op-service/eth" + "github.com/ethereum-optimism/optimism/op-service/log/logfilter" + "github.com/ethereum-optimism/optimism/op-service/logmods" + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" "github.com/ethereum/go-ethereum/log" "github.com/stretchr/testify/require" ) @@ -25,14 +27,20 @@ var ( maxExpectedFlashblocks = 20 ) -// TestFlashblocksStream checks we can connect to the flashblocks stream +// TestFlashblocksStream checks we can connect to the flashblocks stream across multiple CL backends. func TestFlashblocksStream(gt *testing.T) { t := devtest.SerialT(gt) - sys := presets.NewSimpleFlashblocks(t) - logger := testlog.Logger(t, log.LevelInfo).With("Test", "TestFlashblocksStream") + logger := t.Logger() + sys := presets.NewSingleChainWithFlashblocks(t) + filterHandler, ok := logmods.FindHandler[logfilter.FilterHandler](logger.Handler()) + if ok { + filterHandler.Set(logfilter.DefaultMute( + logfilter.Level(slog.LevelError).Show(), + logfilter.Select("kind", "L2CLNode").Show(), + )) + } tracer := t.Tracer() ctx := t.Ctx() - logger.Info("Started Flashblocks Stream test") ctx, span := tracer.Start(ctx, "test chains") defer span.End() @@ -51,33 +59,91 @@ func TestFlashblocksStream(gt *testing.T) { logger.Info("Flashblocks stream rate", "rate", flashblocksStreamRateMs) // Test all L2 chains in the system - for l2Chain, flashblocksBuilderSet := range sys.FlashblocksBuilderSets { - _, span = tracer.Start(ctx, "test chain") - defer span.End() - - networkName := l2Chain.String() - t.Run(fmt.Sprintf("L2_Chain_%s", networkName), func(tt devtest.T) { - if len(flashblocksBuilderSet) == 0 { - tt.Skip("no flashblocks builders for chain", l2Chain.String()) - } + oprbuilderNode := sys.L2OPRBuilder + rollupBoostNode := sys.L2RollupBoost + _, span = tracer.Start(ctx, "test chain") + defer span.End() + + expectedChainID := sys.L2Chain.ChainID().ToBig() + require.Equal(t, oprbuilderNode.Escape().ChainID().ToBig(), expectedChainID, "flashblocks builder node chain id should match expected chain id") + + driveViaTestSequencer(t, sys, 3) + + // Test the presence / absence of a flashblocks stream operating at a 250ms rate from a flashblocks-websocket-proxy node. + // Allow a generous window for first flashblocks to appear. + testDuration := time.Duration(int64(flashblocksStreamRateMs*maxExpectedFlashblocks*2)) * time.Millisecond + // Allow up to 15% of expected flashblocks to be missing due to timing variations + failureTolerance := int(0.15 * float64(maxExpectedFlashblocks)) + + logger.Debug("Test duration", "duration", testDuration, "failure tolerance (of flashblocks)", failureTolerance) + + // Instrument builder stream separately to confirm flashblocks emission upstream. + builderOutput := make(chan []byte, maxExpectedFlashblocks) + defer close(builderOutput) + builderDone := make(chan struct{}) + go func() { + err := oprbuilderNode.FlashblocksClient().ListenFor(ctx, logger.With("stream_source", "op-rbuilder"), testDuration, builderOutput, builderDone) + require.NoError(t, err) + }() + builderMessages := make([]string, 0) + + output := make(chan []byte, maxExpectedFlashblocks) + defer close(output) + doneListening := make(chan struct{}) + streamedMessages := make([]string, 0) + go func() { + err := rollupBoostNode.FlashblocksClient().ListenFor(ctx, logger.With("stream_source", "rollup-boost"), testDuration, output, doneListening) + require.NoError(t, err) + }() + + listening := true + for listening { + select { + case <-doneListening: + doneListening = nil + case <-builderDone: + builderDone = nil + case msg := <-output: + streamedMessages = append(streamedMessages, string(msg)) + case msg := <-builderOutput: + builderMessages = append(builderMessages, string(msg)) + } - expectedChainID := l2Chain.ChainID().ToBig() - for _, flashblocksBuilderNode := range flashblocksBuilderSet { - require.Equal(t, flashblocksBuilderNode.Escape().ChainID().ToBig(), expectedChainID, "flashblocks builder node chain id should match expected chain id") + if doneListening == nil && builderDone == nil { + listening = false + } + } - mode := FlashblocksStreamMode_Follower - if dsl.NewConductor(flashblocksBuilderNode.Escape().Conductor()).IsLeader() { - mode = FlashblocksStreamMode_Leader - } + logger.Info("Completed WebSocket stream reading", "msg_count", len(streamedMessages), "builder_msg_count", len(builderMessages)) - testFlashblocksStreamRbuilder(tt, logger, flashblocksBuilderNode, mode, flashblocksStreamRateMs) - } + if len(builderMessages) > 0 { + logger.Info("Sample builder message", "payload", builderMessages[0]) + } - for _, flashblocksWebsocketProxy := range sys.FlashblocksWebsocketProxies[l2Chain] { - testFlashblocksStreamFbWsProxy(tt, logger, flashblocksWebsocketProxy, flashblocksStreamRateMs) - } - }) + totalFlashblocksProduced := evaluateFlashblocksStream(t, logger, streamedMessages, failureTolerance) + require.Greater(t, totalFlashblocksProduced, 0, "expected to receive flashblocks from rollup-boost stream") + logger.Info("Flashblocks stream validation completed", "total_flashblocks_produced", totalFlashblocksProduced) +} + +// driveViaTestSequencer explicitly builds a few blocks to ensure the builder/rollup-boost +// have payloads to serve before we start listening for flashblocks. +func driveViaTestSequencer(t devtest.T, sys *presets.SingleChainWithFlashblocks, count int) { + t.Helper() + ts := sys.TestSequencer.Escape().ControlAPI(sys.L2Chain.ChainID()) + ctx := t.Ctx() + + head := sys.L2EL.BlockRefByLabel(eth.Unsafe) + for i := 0; i < count; i++ { + require.NoError(t, ts.New(ctx, seqtypes.BuildOpts{Parent: head.Hash})) + require.NoError(t, ts.Next(ctx)) + head = sys.L2EL.BlockRefByLabel(eth.Unsafe) } + // Ensure the sequencer EL has produced at least one unsafe block before subscribing. + sys.L2EL.WaitForBlockNumber(1) + + // Log the latest unsafe head and L1 origin to confirm block production before listening. + head = sys.L2EL.BlockRefByLabel(eth.Unsafe) + sys.Log.Info("Pre-listen unsafe head", "unsafe", head) } func evaluateFlashblocksStream(t devtest.T, logger log.Logger, streamedMessages []string, failureTolerance int) int { @@ -133,102 +199,3 @@ func evaluateFlashblocksStream(t devtest.T, logger log.Logger, streamedMessages return totalFlashblocksProduced } - -// testFlashblocksStreamRbuilder tests the presence / absence of a flashblocks stream operating at a 250ms (configurable via env var FLASHBLOCKS_STREAM_RATE) rate from an rbuilder node -func testFlashblocksStreamRbuilder(t devtest.T, logger log.Logger, flashblocksBuilderNode *dsl.FlashblocksBuilderNode, mode FlashblocksStreamMode, expectedFlashblocksStreamRateMs int) { - t.Run(fmt.Sprintf("Flashblocks_Stream_Rbuilder_%s_%s", flashblocksBuilderNode.Escape().ID(), mode), func(t devtest.T) { - testDuration := time.Duration(int64(expectedFlashblocksStreamRateMs*maxExpectedFlashblocks)) * time.Millisecond - failureTolerance := int(0.15 * float64(maxExpectedFlashblocks)) - - logger.Debug("Test duration", "duration", testDuration, "failure tolerance (of flashblocks)", failureTolerance) - - require.Contains(t, []FlashblocksStreamMode{FlashblocksStreamMode_Leader, FlashblocksStreamMode_Follower}, mode, "mode should be either leader or follower") - require.NotNil(t, flashblocksBuilderNode, "flashblocksBuilderNode should not be nil") - - output := make(chan []byte, maxExpectedFlashblocks) - doneListening := make(chan struct{}) - streamedMessages := make([]string, 0) - go flashblocksBuilderNode.ListenFor(logger, testDuration, output, doneListening) //nolint:errcheck - - for { - select { - case <-doneListening: - goto done - case msg := <-output: - streamedMessages = append(streamedMessages, string(msg)) - } - } - done: - - defer close(output) - - logger.Info("Completed WebSocket stream reading", "message_count", len(streamedMessages)) - if mode == FlashblocksStreamMode_Follower { - require.Equal(t, len(streamedMessages), 0, "follower should not receive any messages") - return - } - - totalFlashblocksProduced := evaluateFlashblocksStream(t, logger, streamedMessages, failureTolerance) - - minExpectedFlashblocks := maxExpectedFlashblocks - failureTolerance - require.Greater(t, - totalFlashblocksProduced, minExpectedFlashblocks, - fmt.Sprintf("total flashblocks produced should be greater than %d (%d over %s with a %dms rate with a failure tolerance of %d flashblocks)", - minExpectedFlashblocks, - maxExpectedFlashblocks, - testDuration, - expectedFlashblocksStreamRateMs, - failureTolerance, - ), - ) - - logger.Info("Flashblocks stream validation completed", "total_flashblocks_produced", totalFlashblocksProduced) - }) -} - -// testFlashblocksStreamFbWsProxy tests the presence / absence of a flashblocks stream operating at a 250ms (configurable via env var FLASHBLOCKS_STREAM_RATE) rate from a flashblocks-websocket-proxy node -func testFlashblocksStreamFbWsProxy(t devtest.T, logger log.Logger, flashblocksWebsocketProxy *dsl.FlashblocksWebsocketProxy, expectedFlashblocksStreamRateMs int) { - t.Run(fmt.Sprintf("Flashblocks_Stream_FbWsProxy_%s", flashblocksWebsocketProxy.Escape().ID()), func(t devtest.T) { - testDuration := time.Duration(int64(expectedFlashblocksStreamRateMs*maxExpectedFlashblocks)) * time.Millisecond - failureTolerance := int(0.15 * float64(maxExpectedFlashblocks)) - - logger.Debug("Test duration", "duration", testDuration, "failure tolerance (of flashblocks)", failureTolerance) - - require.NotNil(t, flashblocksWebsocketProxy, "flashblocksWebsocketProxy should not be nil") - - output := make(chan []byte, maxExpectedFlashblocks) - doneListening := make(chan struct{}) - streamedMessages := make([]string, 0) - go flashblocksWebsocketProxy.ListenFor(logger, testDuration, output, doneListening) //nolint:errcheck - - for { - select { - case <-doneListening: - goto done - case msg := <-output: - streamedMessages = append(streamedMessages, string(msg)) - } - } - done: - - defer close(output) - - logger.Info("Completed WebSocket stream reading", "message_count", len(streamedMessages)) - - totalFlashblocksProduced := evaluateFlashblocksStream(t, logger, streamedMessages, failureTolerance) - - minExpectedFlashblocks := maxExpectedFlashblocks - failureTolerance - require.Greater(t, - totalFlashblocksProduced, minExpectedFlashblocks, - fmt.Sprintf("total flashblocks produced should be greater than %d (%d over %s with a %dms rate with a failure tolerance of %d flashblocks)", - minExpectedFlashblocks, - maxExpectedFlashblocks, - testDuration, - expectedFlashblocksStreamRateMs, - failureTolerance, - ), - ) - - logger.Info("Flashblocks stream validation completed", "total_flashblocks_produced", totalFlashblocksProduced) - }) -} diff --git a/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go b/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go index 0761c870ba9..77c925955c4 100644 --- a/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go +++ b/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go @@ -1,6 +1,3 @@ -//go:build !ci - -// use a tag prefixed with "!". Such tag ensures that the default behaviour of this test would be to be built/run even when the go toolchain (go test) doesn't specify any tag filter. package flashblocks import ( @@ -14,9 +11,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/presets" "github.com/ethereum-optimism/optimism/op-service/eth" - "github.com/ethereum-optimism/optimism/op-service/testlog" "github.com/ethereum-optimism/optimism/op-service/txplan" - "github.com/ethereum/go-ethereum/log" "github.com/stretchr/testify/require" ) @@ -37,117 +32,101 @@ type timedMessage struct { // - That Flashblock's time in nanoseconds must be before the approximated transaction confirmation time recorded previously. func TestFlashblocksTransfer(gt *testing.T) { t := devtest.SerialT(gt) - sys := presets.NewSimpleFlashblocks(t) - logger := testlog.Logger(t, log.LevelInfo).With("Test", "TestFlashblocksTransfer") + logger := t.Logger() tracer := t.Tracer() ctx := t.Ctx() - logger.Info("Started Flashblocks Transfer test") + sys := presets.NewSingleChainWithFlashblocks(t) topLevelCtx, span := tracer.Start(ctx, "test chains") defer span.End() - // Test all L2 chains in the system - for l2Chain, funder := range sys.Funders { - ctx, cancel := context.WithTimeout(topLevelCtx, 45*time.Second) - defer cancel() - _, span = tracer.Start(ctx, fmt.Sprintf("test chain %s", l2Chain.String())) - defer span.End() - - t.Run(fmt.Sprintf("L2_Chain_%s", l2Chain.String()), func(tt devtest.T) { - if len(sys.FlashblocksBuilderSets[l2Chain]) == 0 && len(sys.FlashblocksWebsocketProxies[l2Chain]) == 0 { - tt.Skip("no flashblocks builders or websocket proxies found for chain", l2Chain.String()) - } - - doneListening := make(chan struct{}) - output := make(chan []byte, 100) - - alice := funder.NewFundedEOA(eth.ThreeHundredthsEther) - bob := sys.Wallet.NewEOA(sys.L2ELNodes[l2Chain]) - bobAddress := bob.Address().Hex() - - // flashblocks listener - fbWsProxies := sys.FlashblocksWebsocketProxies[l2Chain] - if len(fbWsProxies) > 0 { - fbWsProxy := fbWsProxies[0] - logger.Info("Listening for flashblocks via websocket proxy", "proxy", fbWsProxy.String()) - go fbWsProxy.ListenFor(logger, 20*time.Second, output, doneListening) //nolint:errcheck - } else { - leaderFbBuilder := sys.FlashblocksBuilderSets[l2Chain].Leader() - require.NotNil(tt, leaderFbBuilder, "should have a leader", "chain", l2Chain.String()) - - logger.Info("Listening for flashblocks via flashblocks builder", "builder", leaderFbBuilder.String()) - - go leaderFbBuilder.ListenFor(logger, 20*time.Second, output, doneListening) //nolint:errcheck - } - - var executedTransaction *txplan.PlannedTx - var transactionApproxConfirmationTime time.Time - var expectedBobBalance string - - // transactor - go func() { - time.Sleep(6 * time.Second) // warm up for the websocket handshake to be established - bobBalance := bob.GetBalance() - - depositAmount := eth.OneHundredthEther - bobAddr := bob.Address() - executedTransaction = alice.Transact( - alice.Plan(), - txplan.WithTo(&bobAddr), - txplan.WithValue(depositAmount), - ) - transactionApproxConfirmationTime = time.Now() - newBobBalance := bobBalance.Add(depositAmount) - expectedBobBalance = newBobBalance.Hex() - bob.VerifyBalanceExact(newBobBalance) - }() - - streamedMessages := make([]timedMessage, 0) - for { - select { - case <-doneListening: - goto done - case msg := <-output: - streamedMessages = append(streamedMessages, timedMessage{message: msg, timestamp: time.Now()}) - } - } - done: - require.Greater(tt, len(streamedMessages), 0, "should have received at least one message from the flashblocks stream") - require.NotNil(tt, executedTransaction, "should have executed a transaction") - - var bobFlashblockTime time.Time - var bobFlashblock *Flashblock - var observedBobBalance string - - for _, msg := range streamedMessages { - flashblock := &Flashblock{} - err := json.Unmarshal(msg.message, flashblock) - require.NoError(tt, err, "should be able to unmarshal the message") - - bobBalance := flashblock.Metadata.NewAccountBalances[strings.ToLower(bobAddress)] - if bobBalance != "" && bobBalance != "0x0" { - bobFlashblockTime = msg.timestamp - bobFlashblock = flashblock - observedBobBalance = bobBalance - break - } - } - - require.NotNil(tt, bobFlashblock, "should have received a flashblock corresponding to Bob's receival of the funds") - - txBlock, err := executedTransaction.IncludedBlock.Eval(ctx) - require.NoError(tt, err, "should be able to evaluate the block in which the transaction was included") - - txBlockNum := int(txBlock.Number) // block number of the block in which the transaction was included / confirmed - flashblockParentBlockNum := int(bobFlashblock.Metadata.BlockNumber) // block number of the parent block of the flashblock which first recorded the update in Bob's account balance (representative of the flashblock which included this transaction) - - txBlockTimeSeconds := int64(txBlock.Time) // timestamp of the block in which the transaction was included / confirmed - txFlashblockTimeInSeconds := bobFlashblockTime.Unix() // timestamp of the flashblock which supposedly included Bob's transaction - - require.Equal(tt, observedBobBalance, expectedBobBalance, "Bob's balance must be correct as per exactly what Alice transferred to them") - require.Equal(tt, txBlockNum, flashblockParentBlockNum, "the transaction's block number should be the same as the flashblock's parent block number") - require.LessOrEqual(tt, txFlashblockTimeInSeconds, txBlockTimeSeconds, "the transaction's block time (in seconds) should be less than or equal to the flashblock's time (in seconds)") - require.Less(tt, bobFlashblockTime.UnixNano(), transactionApproxConfirmationTime.UnixNano(), "flashblock time should be before the transaction's (approximated) confirmation time") - }) + ctx, cancel := context.WithTimeout(topLevelCtx, 45*time.Second) + defer cancel() + _, span = tracer.Start(ctx, fmt.Sprintf("test chain %s", sys.L2Chain.String())) + defer span.End() + + doneListening := make(chan struct{}) + output := make(chan []byte, 100) + + // Drive a couple blocks on the test sequencer so the faucet L2 funding tx has a chance to land before we rely on it. + driveViaTestSequencer(t, sys, 2) + + alice := sys.FunderL2.NewFundedEOA(eth.ThreeHundredthsEther) + bob := sys.Wallet.NewEOA(sys.L2EL) + bobAddress := bob.Address().Hex() + + // flashblocks listener + flashblocksClient := sys.L2RollupBoost.FlashblocksClient() + go func() { + err := flashblocksClient.ListenFor(ctx, logger, 20*time.Second, output, doneListening) + t.Require().NoError(err, "failed to listen for flashblocks") + }() + + var executedTransaction *txplan.PlannedTx + var transactionApproxConfirmationTime time.Time + var expectedBobBalance string + + // transactor + go func() { + bobBalance := bob.GetBalance() + + depositAmount := eth.OneHundredthEther + bobAddr := bob.Address() + executedTransaction = alice.Transact( + alice.Plan(), + txplan.WithTo(&bobAddr), + txplan.WithValue(depositAmount), + ) + transactionApproxConfirmationTime = time.Now() + newBobBalance := bobBalance.Add(depositAmount) + expectedBobBalance = newBobBalance.Hex() + bob.VerifyBalanceExact(newBobBalance) + }() + + streamedMessages := make([]timedMessage, 0) + listening := true + for listening { + select { + case <-doneListening: + listening = false + case msg := <-output: + streamedMessages = append(streamedMessages, timedMessage{message: msg, timestamp: time.Now()}) + } } + require.Greater(t, len(streamedMessages), 0, "should have received at least one message from the flashblocks stream") + require.NotNil(t, executedTransaction, "should have executed a transaction") + + var bobFlashblockTime time.Time + var bobFlashblock *Flashblock + var observedBobBalance string + + for _, msg := range streamedMessages { + flashblock := &Flashblock{} + err := json.Unmarshal(msg.message, flashblock) + require.NoError(t, err, "should be able to unmarshal the message") + + bobBalance := flashblock.Metadata.NewAccountBalances[strings.ToLower(bobAddress)] + if bobBalance != "" && bobBalance != "0x0" { + bobFlashblockTime = msg.timestamp + bobFlashblock = flashblock + observedBobBalance = bobBalance + break + } + } + + require.NotNil(t, bobFlashblock, "should have received a flashblock corresponding to Bob's receival of the funds") + + txBlock, err := executedTransaction.IncludedBlock.Eval(ctx) + require.NoError(t, err, "should be able to evaluate the block in which the transaction was included") + + txBlockNum := int(txBlock.Number) // block number of the block in which the transaction was included / confirmed + flashblockParentBlockNum := int(bobFlashblock.Metadata.BlockNumber) // block number of the parent block of the flashblock which first recorded the update in Bob's account balance (representative of the flashblock which included this transaction) + + txBlockTimeSeconds := int64(txBlock.Time) // timestamp of the block in which the transaction was included / confirmed + txFlashblockTimeInSeconds := bobFlashblockTime.Unix() // timestamp of the flashblock which supposedly included Bob's transaction + + require.Equal(t, observedBobBalance, expectedBobBalance, "Bob's balance must be correct as per exactly what Alice transferred to them") + require.Equal(t, txBlockNum, flashblockParentBlockNum, "the transaction's block number should be the same as the flashblock's parent block number") + require.LessOrEqual(t, txFlashblockTimeInSeconds, txBlockTimeSeconds, "the transaction's block time (in seconds) should be less than or equal to the flashblock's time (in seconds)") + require.Less(t, bobFlashblockTime.UnixNano(), transactionApproxConfirmationTime.UnixNano(), "flashblock time should be before the transaction's (approximated) confirmation time") } diff --git a/op-acceptance-tests/tests/flashblocks/init_test.go b/op-acceptance-tests/tests/flashblocks/init_test.go index 3c4d36f9a39..6d2759e47bd 100644 --- a/op-acceptance-tests/tests/flashblocks/init_test.go +++ b/op-acceptance-tests/tests/flashblocks/init_test.go @@ -8,5 +8,5 @@ import ( // TestMain creates the test-setups against the shared backend func TestMain(m *testing.M) { - presets.DoMain(m, presets.WithSimpleFlashblocks()) + presets.DoMain(m, presets.WithSingleChainSystemWithFlashblocks()) } diff --git a/op-devstack/dsl/fb_builder.go b/op-devstack/dsl/fb_builder.go deleted file mode 100644 index 0820d9c46db..00000000000 --- a/op-devstack/dsl/fb_builder.go +++ /dev/null @@ -1,55 +0,0 @@ -package dsl - -import ( - "time" - - "github.com/ethereum-optimism/optimism/op-devstack/stack" - "github.com/ethereum/go-ethereum/log" -) - -type FlashblocksBuilderSet []*FlashblocksBuilderNode - -func (f FlashblocksBuilderSet) Leader() *FlashblocksBuilderNode { - for _, node := range f { - if node.Conductor().IsLeader() { - return node - } - } - return nil -} - -func NewFlashblocksBuilderSet(inner []stack.FlashblocksBuilderNode) FlashblocksBuilderSet { - flashblocksBuilders := make([]*FlashblocksBuilderNode, len(inner)) - for i, c := range inner { - flashblocksBuilders[i] = NewFlashblocksBuilderNode(c) - } - return flashblocksBuilders -} - -type FlashblocksBuilderNode struct { - commonImpl - inner stack.FlashblocksBuilderNode -} - -func NewFlashblocksBuilderNode(inner stack.FlashblocksBuilderNode) *FlashblocksBuilderNode { - return &FlashblocksBuilderNode{ - commonImpl: commonFromT(inner.T()), - inner: inner, - } -} - -func (c *FlashblocksBuilderNode) String() string { - return c.inner.ID().String() -} - -func (c *FlashblocksBuilderNode) Escape() stack.FlashblocksBuilderNode { - return c.inner -} - -func (c *FlashblocksBuilderNode) Conductor() *Conductor { - return NewConductor(c.inner.Conductor()) -} - -func (c *FlashblocksBuilderNode) ListenFor(logger log.Logger, duration time.Duration, output chan<- []byte, done chan<- struct{}) error { - return websocketListenFor(logger, c.inner.FlashblocksWsUrl(), c.inner.FlashblocksWsHeaders(), duration, output, done) -} diff --git a/op-devstack/dsl/fb_ws_client.go b/op-devstack/dsl/fb_ws_client.go new file mode 100644 index 00000000000..fad80c7aa59 --- /dev/null +++ b/op-devstack/dsl/fb_ws_client.go @@ -0,0 +1,146 @@ +package dsl + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum/go-ethereum/log" + "github.com/gorilla/websocket" +) + +type FlashblocksWSClientSet []*FlashblocksWSClient + +func NewFlashblocksWSClientSet(inner []stack.FlashblocksWSClient) FlashblocksWSClientSet { + flashblocksWSClients := make([]*FlashblocksWSClient, len(inner)) + for i, c := range inner { + flashblocksWSClients[i] = NewFlashblocksWSClient(c) + } + return flashblocksWSClients +} + +type FlashblocksWSClient struct { + commonImpl + inner stack.FlashblocksWSClient +} + +func NewFlashblocksWSClient(inner stack.FlashblocksWSClient) *FlashblocksWSClient { + return &FlashblocksWSClient{ + commonImpl: commonFromT(inner.T()), + inner: inner, + } +} + +func (c *FlashblocksWSClient) String() string { + return c.inner.ID().String() +} + +func (c *FlashblocksWSClient) Escape() stack.FlashblocksWSClient { + return c.inner +} + +func (c *FlashblocksWSClient) ListenFor(ctx context.Context, logger log.Logger, duration time.Duration, output chan<- []byte, done chan<- struct{}) error { + wsURL := c.Escape().WsUrl() + headers := c.Escape().WsHeaders() + return websocketListenFor(ctx, logger, wsURL, headers, duration, output, done) +} + +func websocketListenFor(ctx context.Context, logger log.Logger, wsURL string, headers http.Header, duration time.Duration, output chan<- []byte, done chan<- struct{}) error { + defer close(done) + + listenCtx, cancel := context.WithTimeout(ctx, duration) + defer cancel() + + logger.Debug("Testing WebSocket connection to", "url", wsURL, "headers", headers) + + // Log the headers for debug purposes + if headers != nil { + for key, values := range headers { + logger.Debug("Header", "key", key, "values", values) + } + } else { + logger.Debug("No headers provided") + } + + dialer := &websocket.Dialer{ + HandshakeTimeout: 6 * time.Second, + } + + // Always close the response body to prevent resource leaks + logger.Debug("Attempting WebSocket connection", "url", wsURL) + conn, resp, err := dialer.DialContext(listenCtx, wsURL, headers) + if err != nil { + if listenCtx.Err() != nil { + logger.Info("Context completed before WebSocket connection established", "reason", listenCtx.Err()) + return nil + } + logger.Error("WebSocket connection failed", "url", wsURL, "error", err) + if resp != nil { + logger.Error("HTTP response details", "status", resp.Status, "headers", resp.Header) + resp.Body.Close() + } + return fmt.Errorf("failed to connect to Flashblocks WebSocket endpoint %s: %w", wsURL, err) + } + + if resp != nil { + defer resp.Body.Close() + } + defer conn.Close() + + logger.Info("WebSocket connection established successfully", "url", wsURL, "reading_stream_for", duration) + go func() { + <-listenCtx.Done() + _ = conn.Close() + }() + + messageCount := 0 + for { + select { + case <-listenCtx.Done(): + logListenStop(logger, listenCtx.Err(), messageCount) + return nil + default: + } + + _, message, err := conn.ReadMessage() + if err != nil { + if listenCtx.Err() != nil { + logListenStop(logger, listenCtx.Err(), messageCount) + return nil + } + + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway, websocket.CloseNoStatusReceived) { + logger.Info("WebSocket connection closed by peer", "total_messages", messageCount) + return nil + } + + logger.Error("Error reading WebSocket message", "error", err, "message_count", messageCount) + return fmt.Errorf("error reading WebSocket message: %w", err) + } + + messageCount++ + logger.Debug("Received WebSocket message", "message_count", messageCount, "message_length", len(message)) + + select { + case output <- message: + logger.Debug("Message sent to output channel", "message_count", messageCount) + case <-listenCtx.Done(): + logListenStop(logger, listenCtx.Err(), messageCount) + return nil + } + } +} + +func logListenStop(logger log.Logger, reason error, messageCount int) { + switch { + case errors.Is(reason, context.DeadlineExceeded): + logger.Info("WebSocket read duration reached", "total_messages", messageCount) + case errors.Is(reason, context.Canceled): + logger.Info("WebSocket listener canceled", "total_messages", messageCount) + default: + logger.Info("WebSocket listener stopping", "total_messages", messageCount) + } +} diff --git a/op-devstack/dsl/fb_ws_proxy.go b/op-devstack/dsl/fb_ws_proxy.go deleted file mode 100644 index 72fa8c5e3a6..00000000000 --- a/op-devstack/dsl/fb_ws_proxy.go +++ /dev/null @@ -1,116 +0,0 @@ -package dsl - -import ( - "fmt" - "net/http" - "strings" - "time" - - "github.com/ethereum-optimism/optimism/op-devstack/stack" - "github.com/ethereum/go-ethereum/log" - "github.com/gorilla/websocket" -) - -type FlashblocksWebsocketProxySet []*FlashblocksWebsocketProxy - -func NewFlashblocksWebsocketProxySet(inner []stack.FlashblocksWebsocketProxy) FlashblocksWebsocketProxySet { - flashblocksWebsocketProxies := make([]*FlashblocksWebsocketProxy, len(inner)) - for i, c := range inner { - flashblocksWebsocketProxies[i] = NewFlashblocksWebsocketProxy(c) - } - return flashblocksWebsocketProxies -} - -type FlashblocksWebsocketProxy struct { - commonImpl - inner stack.FlashblocksWebsocketProxy -} - -func NewFlashblocksWebsocketProxy(inner stack.FlashblocksWebsocketProxy) *FlashblocksWebsocketProxy { - return &FlashblocksWebsocketProxy{ - commonImpl: commonFromT(inner.T()), - inner: inner, - } -} - -func (c *FlashblocksWebsocketProxy) String() string { - return c.inner.ID().String() -} - -func (c *FlashblocksWebsocketProxy) Escape() stack.FlashblocksWebsocketProxy { - return c.inner -} - -func (c *FlashblocksWebsocketProxy) ListenFor(logger log.Logger, duration time.Duration, output chan<- []byte, done chan<- struct{}) error { - wsURL := c.Escape().WsUrl() - headers := c.Escape().WsHeaders() - return websocketListenFor(logger, wsURL, headers, duration, output, done) -} - -func websocketListenFor(logger log.Logger, wsURL string, headers http.Header, duration time.Duration, output chan<- []byte, done chan<- struct{}) error { - defer close(done) - logger.Debug("Testing WebSocket connection to", "url", wsURL, "headers", headers) - - // Log the headers for debug purposes - if headers != nil { - for key, values := range headers { - logger.Debug("Header", "key", key, "values", values) - } - } else { - logger.Debug("No headers provided") - } - - dialer := &websocket.Dialer{ - HandshakeTimeout: 6 * time.Second, - } - - // Always close the response body to prevent resource leaks - logger.Debug("Attempting WebSocket connection", "url", wsURL) - conn, resp, err := dialer.Dial(wsURL, headers) - if err != nil { - logger.Error("WebSocket connection failed", "url", wsURL, "error", err) - if resp != nil { - logger.Error("HTTP response details", "status", resp.Status, "headers", resp.Header) - resp.Body.Close() - } - return fmt.Errorf("failed to connect to Flashblocks WebSocket endpoint %s: %w", wsURL, err) - } - - if resp != nil { - defer resp.Body.Close() - } - defer conn.Close() - - logger.Info("WebSocket connection established successfully", "url", wsURL, "reading stream for", duration) - - timeout := time.After(duration) - messageCount := 0 - for { - select { - case <-timeout: - logger.Info("WebSocket read timeout reached", "total_messages", messageCount) - return nil - default: - err = conn.SetReadDeadline(time.Now().Add(duration)) - if err != nil { - return fmt.Errorf("failed to set read deadline: %w", err) - } - _, message, err := conn.ReadMessage() - if err != nil && !strings.Contains(err.Error(), "timeout") { - logger.Error("Error reading WebSocket message", "error", err, "message_count", messageCount) - return fmt.Errorf("error reading WebSocket message: %w", err) - } - if err == nil { - messageCount++ - logger.Debug("Received WebSocket message", "message_count", messageCount, "message_length", len(message)) - select { - case output <- message: - logger.Debug("Message sent to output channel", "message_count", messageCount) - case <-timeout: // to avoid indefinite hang - logger.Info("Timeout while sending message to output channel", "total_messages", messageCount) - return nil - } - } - } - } -} diff --git a/op-devstack/dsl/l2_cl.go b/op-devstack/dsl/l2_cl.go index 1ca3556e0e2..20982773d5e 100644 --- a/op-devstack/dsl/l2_cl.go +++ b/op-devstack/dsl/l2_cl.go @@ -191,7 +191,8 @@ func (cl *L2CLNode) ReachedRefFn(lvl types.SafetyLevel, target eth.BlockID, atte if err != nil { return err } - ethclient := cl.inner.ELs()[0].EthClient() + + ethclient := cl.inner.ELClient() result, err := ethclient.BlockRefByNumber(cl.ctx, target.Number) if err != nil { return err diff --git a/op-devstack/dsl/l2_op_rbuilder.go b/op-devstack/dsl/l2_op_rbuilder.go new file mode 100644 index 00000000000..78f395c51e1 --- /dev/null +++ b/op-devstack/dsl/l2_op_rbuilder.go @@ -0,0 +1,60 @@ +package dsl + +import ( + "context" + "time" + + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum/go-ethereum/log" +) + +type OPRBuilderNodeSet []*OPRBuilderNode + +func NewOPRBuilderNodeSet(inner []stack.OPRBuilderNode, control stack.ControlPlane) OPRBuilderNodeSet { + oprbuilders := make([]*OPRBuilderNode, len(inner)) + for i, c := range inner { + oprbuilders[i] = NewOPRBuilderNode(c, control) + } + return oprbuilders +} + +type OPRBuilderNode struct { + commonImpl + inner stack.OPRBuilderNode + wsClient *FlashblocksWSClient + control stack.ControlPlane +} + +func NewOPRBuilderNode(inner stack.OPRBuilderNode, control stack.ControlPlane) *OPRBuilderNode { + return &OPRBuilderNode{ + commonImpl: commonFromT(inner.T()), + inner: inner, + wsClient: NewFlashblocksWSClient(inner.FlashblocksClient()), + control: control, + } +} + +func (c *OPRBuilderNode) String() string { + return c.inner.ID().String() +} + +func (c *OPRBuilderNode) Escape() stack.OPRBuilderNode { + return c.inner +} + +func (c *OPRBuilderNode) ListenFor(ctx context.Context, logger log.Logger, duration time.Duration, output chan<- []byte, done chan<- struct{}) error { + return c.wsClient.ListenFor(ctx, logger, duration, output, done) +} + +func (el *OPRBuilderNode) Stop() { + el.log.Info("Stopping", "id", el.inner.ID()) + el.control.OPRBuilderNodeState(el.inner.ID(), stack.Stop) +} + +func (el *OPRBuilderNode) Start() { + el.control.OPRBuilderNodeState(el.inner.ID(), stack.Start) +} + +func (el *OPRBuilderNode) FlashblocksClient() *FlashblocksWSClient { + return NewFlashblocksWSClient(el.inner.FlashblocksClient()) +} diff --git a/op-devstack/dsl/rollup_boost.go b/op-devstack/dsl/rollup_boost.go new file mode 100644 index 00000000000..1638944c1bb --- /dev/null +++ b/op-devstack/dsl/rollup_boost.go @@ -0,0 +1,35 @@ +package dsl + +import "github.com/ethereum-optimism/optimism/op-devstack/stack" + +type RollupBoostNodesSet []*RollupBoostNode + +func NewRollupBoostNodesSet(inner []stack.RollupBoostNode, control stack.ControlPlane) RollupBoostNodesSet { + rollupBoostNodes := make([]*RollupBoostNode, len(inner)) + for i, c := range inner { + rollupBoostNodes[i] = NewRollupBoostNode(c, control) + } + return rollupBoostNodes +} + +// RollupBoostNode wraps a stack.RollupBoostNode interface for DSL operations +type RollupBoostNode struct { + inner stack.RollupBoostNode + control stack.ControlPlane +} + +func (r *RollupBoostNode) Escape() stack.RollupBoostNode { + return r.inner +} + +// NewRollupBoostNode creates a new RollupBoostNode DSL wrapper +func NewRollupBoostNode(inner stack.RollupBoostNode, control stack.ControlPlane) *RollupBoostNode { + return &RollupBoostNode{ + inner, + control, + } +} + +func (r *RollupBoostNode) FlashblocksClient() *FlashblocksWSClient { + return NewFlashblocksWSClient(r.inner.FlashblocksClient()) +} diff --git a/op-devstack/presets/flashblocks.go b/op-devstack/presets/flashblocks.go index 43b9deb504d..60298b917f6 100644 --- a/op-devstack/presets/flashblocks.go +++ b/op-devstack/presets/flashblocks.go @@ -1,71 +1,85 @@ package presets import ( - "github.com/ethereum-optimism/optimism/op-devstack/compat" + "time" + + challengerConfig "github.com/ethereum-optimism/optimism/op-challenger/config" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/dsl/proofs" "github.com/ethereum-optimism/optimism/op-devstack/shim" "github.com/ethereum-optimism/optimism/op-devstack/stack" "github.com/ethereum-optimism/optimism/op-devstack/stack/match" "github.com/ethereum-optimism/optimism/op-devstack/sysgo" ) -type SimpleFlashblocks struct { +type SingleChainWithFlashblocks struct { *Minimal - ConductorSets map[stack.L2NetworkID]dsl.ConductorSet - FlashblocksBuilderSets map[stack.L2NetworkID]dsl.FlashblocksBuilderSet - FlashblocksWebsocketProxies map[stack.L2NetworkID]dsl.FlashblocksWebsocketProxySet - - Faucets map[stack.L2NetworkID]*dsl.Faucet - Funders map[stack.L2NetworkID]*dsl.Funder - L2ELNodes map[stack.L2NetworkID]*dsl.L2ELNode + L2OPRBuilder *dsl.OPRBuilderNode + L2RollupBoost *dsl.RollupBoostNode + TestSequencer *dsl.TestSequencer } -func WithSimpleFlashblocks() stack.CommonOption { - return stack.Combine( - stack.MakeCommon(sysgo.DefaultMinimalSystem(&sysgo.DefaultMinimalSystemIDs{})), - // TODO(#16450): add sysgo support for flashblocks - WithCompatibleTypes(compat.Persistent, compat.Kurtosis), - ) +func (m *SingleChainWithFlashblocks) L2Networks() []*dsl.L2Network { + return []*dsl.L2Network{ + m.L2Chain, + } } -func NewSimpleFlashblocks(t devtest.T) *SimpleFlashblocks { - system := shim.NewSystem(t) - orch := Orchestrator() - orch.Hydrate(system) - chains := system.L2Networks() - - minimalPreset := NewMinimal(t) +func (m *SingleChainWithFlashblocks) StandardBridge() *dsl.StandardBridge { + return dsl.NewStandardBridge(m.T, m.L2Chain, nil, m.L1EL) +} - conductorSets := make(map[stack.L2NetworkID]dsl.ConductorSet) - flashblocksBuilderSets := make(map[stack.L2NetworkID]dsl.FlashblocksBuilderSet) - faucets := make(map[stack.L2NetworkID]*dsl.Faucet) - funders := make(map[stack.L2NetworkID]*dsl.Funder) - l2ELNodes := make(map[stack.L2NetworkID]*dsl.L2ELNode) - flashblocksWebsocketProxies := make(map[stack.L2NetworkID]dsl.FlashblocksWebsocketProxySet) +func (m *SingleChainWithFlashblocks) DisputeGameFactory() *proofs.DisputeGameFactory { + return proofs.NewDisputeGameFactory(m.T, m.L1Network, m.L1EL.EthClient(), m.L2Chain.DisputeGameFactoryProxyAddr(), m.L2CL, m.L2EL, nil, m.challengerConfig) +} - for _, chain := range chains { - chainMatcher := match.L2ChainById(chain.ID()) - l2 := system.L2Network(match.Assume(t, chainMatcher)) - firstELNode := dsl.NewL2ELNode(l2.L2ELNode(match.FirstL2EL), orch.ControlPlane()) - firstFaucet := dsl.NewFaucet(l2.Faucet(match.Assume(t, match.FirstFaucet))) +func (m *SingleChainWithFlashblocks) AdvanceTime(amount time.Duration) { + ttSys, ok := m.system.(stack.TimeTravelSystem) + m.T.Require().True(ok, "attempting to advance time on incompatible system") + ttSys.AdvanceTime(amount) +} - conductorSets[chain.ID()] = dsl.NewConductorSet(l2.Conductors()) - flashblocksBuilderSets[chain.ID()] = dsl.NewFlashblocksBuilderSet(l2.FlashblocksBuilders()) - flashblocksWebsocketProxies[chain.ID()] = dsl.NewFlashblocksWebsocketProxySet(l2.FlashblocksWebsocketProxies()) +func WithSingleChainSystemWithFlashblocks() stack.CommonOption { + return stack.MakeCommon(sysgo.DefaultSingleChainSystemWithFlashblocks(&sysgo.SingleChainSystemWithFlashblocksIDs{})) +} - faucets[chain.ID()] = firstFaucet - funders[chain.ID()] = dsl.NewFunder(minimalPreset.Wallet, firstFaucet, firstELNode) - l2ELNodes[chain.ID()] = firstELNode +func NewSingleChainWithFlashblocks(t devtest.T) *SingleChainWithFlashblocks { + system := shim.NewSystem(t) + orch := Orchestrator() + orch.Hydrate(system) + l1Net := system.L1Network(match.FirstL1Network) + l2 := system.L2Network(match.Assume(t, match.L2ChainA)) + sequencerCL := l2.L2CLNode(match.Assume(t, match.WithSequencerActive(t.Ctx()))) + sequencerEL := l2.L2ELNode(match.Assume(t, match.EngineFor(sequencerCL))) + var challengerCfg *challengerConfig.Config + if len(l2.L2Challengers()) > 0 { + challengerCfg = l2.L2Challengers()[0].Config() } - return &SimpleFlashblocks{ - Minimal: minimalPreset, - ConductorSets: conductorSets, - FlashblocksBuilderSets: flashblocksBuilderSets, - FlashblocksWebsocketProxies: flashblocksWebsocketProxies, - Faucets: faucets, - Funders: funders, - L2ELNodes: l2ELNodes, + + out := &SingleChainWithFlashblocks{ + L2OPRBuilder: dsl.NewOPRBuilderNode(l2.OPRBuilderNode(match.Assume(t, match.FirstOPRBuilderNode)), orch.ControlPlane()), + L2RollupBoost: dsl.NewRollupBoostNode(l2.RollupBoostNode(match.Assume(t, match.FirstRollupBoostNode)), orch.ControlPlane()), + Minimal: &Minimal{ + Log: t.Logger(), + T: t, + ControlPlane: orch.ControlPlane(), + system: system, + L1Network: dsl.NewL1Network(system.L1Network(match.FirstL1Network)), + L1EL: dsl.NewL1ELNode(l1Net.L1ELNode(match.Assume(t, match.FirstL1EL))), + L2Chain: dsl.NewL2Network(l2, orch.ControlPlane()), + L2Batcher: dsl.NewL2Batcher(l2.L2Batcher(match.Assume(t, match.FirstL2Batcher))), + L2EL: dsl.NewL2ELNode(sequencerEL, orch.ControlPlane()), + L2CL: dsl.NewL2CLNode(sequencerCL, orch.ControlPlane()), + Wallet: dsl.NewRandomHDWallet(t, 30), // Random for test isolation + FaucetL2: dsl.NewFaucet(l2.Faucet(match.Assume(t, match.FirstFaucet))), + challengerConfig: challengerCfg, + }, + TestSequencer: dsl.NewTestSequencer(system.TestSequencer(match.Assume(t, match.FirstTestSequencer))), } + out.FaucetL1 = dsl.NewFaucet(out.L1Network.Escape().Faucet(match.Assume(t, match.FirstFaucet))) + out.FunderL1 = dsl.NewFunder(out.Wallet, out.FaucetL1, out.L1EL) + out.FunderL2 = dsl.NewFunder(out.Wallet, out.FaucetL2, out.L2EL) + return out } diff --git a/op-devstack/shim/fb_builder.go b/op-devstack/shim/fb_builder.go deleted file mode 100644 index 960747612e3..00000000000 --- a/op-devstack/shim/fb_builder.go +++ /dev/null @@ -1,68 +0,0 @@ -package shim - -import ( - "net/http" - - "github.com/stretchr/testify/require" - - "github.com/ethereum-optimism/optimism/op-devstack/stack" - "github.com/ethereum-optimism/optimism/op-service/apis" - "github.com/ethereum-optimism/optimism/op-service/sources" -) - -type FlashblocksBuilderNodeConfig struct { - ELNodeConfig - ID stack.FlashblocksBuilderID - Conductor stack.Conductor - FlashblocksWsUrl string - FlashblocksWsHeaders http.Header -} - -type flashblocksBuilderNode struct { - rpcELNode - l2Client *sources.L2Client - - id stack.FlashblocksBuilderID - conductor stack.Conductor - - flashblocksWsUrl string - flashblocksWsHeaders http.Header -} - -var _ stack.FlashblocksBuilderNode = (*flashblocksBuilderNode)(nil) - -func NewFlashblocksBuilderNode(cfg FlashblocksBuilderNodeConfig) stack.FlashblocksBuilderNode { - require.Equal(cfg.T, cfg.ID.ChainID(), cfg.ELNodeConfig.ChainID, "chainID must be configured to match node chainID") - cfg.T = cfg.T.WithCtx(stack.ContextWithID(cfg.T.Ctx(), cfg.ID)) - l2Client, err := sources.NewL2Client(cfg.ELNodeConfig.Client, cfg.T.Logger(), nil, sources.L2ClientSimpleConfig(nil, false, 10, 10)) - require.NoError(cfg.T, err) - - return &flashblocksBuilderNode{ - rpcELNode: newRpcELNode(cfg.ELNodeConfig), - l2Client: l2Client, - id: cfg.ID, - conductor: cfg.Conductor, - flashblocksWsUrl: cfg.FlashblocksWsUrl, - flashblocksWsHeaders: cfg.FlashblocksWsHeaders, - } -} - -func (r *flashblocksBuilderNode) ID() stack.FlashblocksBuilderID { - return r.id -} - -func (r *flashblocksBuilderNode) Conductor() stack.Conductor { - return r.conductor -} - -func (r *flashblocksBuilderNode) L2EthClient() apis.L2EthClient { - return r.l2Client -} - -func (r *flashblocksBuilderNode) FlashblocksWsUrl() string { - return r.flashblocksWsUrl -} - -func (r *flashblocksBuilderNode) FlashblocksWsHeaders() http.Header { - return r.flashblocksWsHeaders -} diff --git a/op-devstack/shim/fb_ws_client.go b/op-devstack/shim/fb_ws_client.go new file mode 100644 index 00000000000..b9275497a6e --- /dev/null +++ b/op-devstack/shim/fb_ws_client.go @@ -0,0 +1,50 @@ +package shim + +import ( + "net/http" + + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +type FlashblocksWSClientConfig struct { + CommonConfig + ID stack.FlashblocksWSClientID + WsUrl string + WsHeaders http.Header +} + +type flashblocksWSClient struct { + commonImpl + id stack.FlashblocksWSClientID + wsUrl string + wsHeaders http.Header +} + +var _ stack.FlashblocksWSClient = (*flashblocksWSClient)(nil) + +func NewFlashblocksWSClient(cfg FlashblocksWSClientConfig) stack.FlashblocksWSClient { + cfg.T = cfg.T.WithCtx(stack.ContextWithID(cfg.T.Ctx(), cfg.ID)) + return &flashblocksWSClient{ + commonImpl: newCommon(cfg.CommonConfig), + id: cfg.ID, + wsUrl: cfg.WsUrl, + wsHeaders: cfg.WsHeaders, + } +} + +func (r *flashblocksWSClient) ID() stack.FlashblocksWSClientID { + return r.id +} + +func (r *flashblocksWSClient) ChainID() eth.ChainID { + return r.id.ChainID() +} + +func (r *flashblocksWSClient) WsUrl() string { + return r.wsUrl +} + +func (r *flashblocksWSClient) WsHeaders() http.Header { + return r.wsHeaders +} diff --git a/op-devstack/shim/fb_ws_proxy.go b/op-devstack/shim/fb_ws_proxy.go deleted file mode 100644 index b01d10441ef..00000000000 --- a/op-devstack/shim/fb_ws_proxy.go +++ /dev/null @@ -1,50 +0,0 @@ -package shim - -import ( - "net/http" - - "github.com/ethereum-optimism/optimism/op-devstack/stack" - "github.com/ethereum-optimism/optimism/op-service/eth" -) - -type FlashblocksWebsocketProxyConfig struct { - CommonConfig - ID stack.FlashblocksWebsocketProxyID - WsUrl string - WsHeaders http.Header -} - -type flashblocksWebsocketProxy struct { - commonImpl - id stack.FlashblocksWebsocketProxyID - wsUrl string - wsHeaders http.Header -} - -var _ stack.FlashblocksWebsocketProxy = (*flashblocksWebsocketProxy)(nil) - -func NewFlashblocksWebsocketProxy(cfg FlashblocksWebsocketProxyConfig) stack.FlashblocksWebsocketProxy { - cfg.T = cfg.T.WithCtx(stack.ContextWithID(cfg.T.Ctx(), cfg.ID)) - return &flashblocksWebsocketProxy{ - commonImpl: newCommon(cfg.CommonConfig), - id: cfg.ID, - wsUrl: cfg.WsUrl, - wsHeaders: cfg.WsHeaders, - } -} - -func (r *flashblocksWebsocketProxy) ID() stack.FlashblocksWebsocketProxyID { - return r.id -} - -func (r *flashblocksWebsocketProxy) ChainID() eth.ChainID { - return r.id.ChainID() -} - -func (r *flashblocksWebsocketProxy) WsUrl() string { - return r.wsUrl -} - -func (r *flashblocksWebsocketProxy) WsHeaders() http.Header { - return r.wsHeaders -} diff --git a/op-devstack/shim/l2_cl.go b/op-devstack/shim/l2_cl.go index bcf02f78355..1421c54f229 100644 --- a/op-devstack/shim/l2_cl.go +++ b/op-devstack/shim/l2_cl.go @@ -22,11 +22,13 @@ type L2CLNodeConfig struct { type rpcL2CLNode struct { commonImpl - id stack.L2CLNodeID - client client.RPC - rollupClient apis.RollupClient - p2pClient apis.P2PClient - els locks.RWMap[stack.L2ELNodeID, stack.L2ELNode] + id stack.L2CLNodeID + client client.RPC + rollupClient apis.RollupClient + p2pClient apis.P2PClient + els locks.RWMap[stack.L2ELNodeID, stack.L2ELNode] + rollupBoostNodes locks.RWMap[stack.RollupBoostNodeID, stack.RollupBoostNode] + oprbuilderNodes locks.RWMap[stack.OPRBuilderNodeID, stack.OPRBuilderNode] userRPC string @@ -74,10 +76,38 @@ func (r *rpcL2CLNode) LinkEL(el stack.L2ELNode) { r.els.Set(el.ID(), el) } +func (r *rpcL2CLNode) LinkRollupBoostNode(rollupBoostNode stack.RollupBoostNode) { + r.rollupBoostNodes.Set(rollupBoostNode.ID(), rollupBoostNode) +} + +func (r *rpcL2CLNode) LinkOPRBuilderNode(oprb stack.OPRBuilderNode) { + r.oprbuilderNodes.Set(oprb.ID(), oprb) +} + func (r *rpcL2CLNode) ELs() []stack.L2ELNode { return stack.SortL2ELNodes(r.els.Values()) } +func (r *rpcL2CLNode) ELClient() apis.EthClient { + var ethclient apis.EthClient + if len(r.els.Values()) > 0 { + ethclient = r.els.Values()[0].EthClient() + } else if len(r.rollupBoostNodes.Values()) > 0 { + ethclient = r.rollupBoostNodes.Values()[0].EthClient() + } else if len(r.oprbuilderNodes.Values()) > 0 { + ethclient = r.oprbuilderNodes.Values()[0].EthClient() + } + return ethclient +} + +func (r *rpcL2CLNode) RollupBoostNodes() []stack.RollupBoostNode { + return stack.SortRollupBoostNodes(r.rollupBoostNodes.Values()) +} + +func (r *rpcL2CLNode) OPRBuilderNodes() []stack.OPRBuilderNode { + return stack.SortOPRBuilderNodes(r.oprbuilderNodes.Values()) +} + func (r *rpcL2CLNode) UserRPC() string { return r.userRPC } diff --git a/op-devstack/shim/l2_network.go b/op-devstack/shim/l2_network.go index 58ce9462161..914c35aac63 100644 --- a/op-devstack/shim/l2_network.go +++ b/op-devstack/shim/l2_network.go @@ -40,9 +40,9 @@ type presetL2Network struct { els locks.RWMap[stack.L2ELNodeID, stack.L2ELNode] cls locks.RWMap[stack.L2CLNodeID, stack.L2CLNode] - conductors locks.RWMap[stack.ConductorID, stack.Conductor] - fbBuilders locks.RWMap[stack.FlashblocksBuilderID, stack.FlashblocksBuilderNode] - fbWsProxies locks.RWMap[stack.FlashblocksWebsocketProxyID, stack.FlashblocksWebsocketProxy] + conductors locks.RWMap[stack.ConductorID, stack.Conductor] + rollupBoostNodes locks.RWMap[stack.RollupBoostNodeID, stack.RollupBoostNode] + oprBuilderNodes locks.RWMap[stack.OPRBuilderNodeID, stack.OPRBuilderNode] } var _ stack.L2Network = (*presetL2Network)(nil) @@ -122,17 +122,6 @@ func (p *presetL2Network) AddConductor(v stack.Conductor) { p.require().True(p.conductors.SetIfMissing(id, v), "conductor %s must not already exist", id) } -func (p *presetL2Network) FlashblocksBuilder(m stack.FlashblocksBuilderMatcher) stack.FlashblocksBuilderNode { - v, ok := findMatch(m, p.fbBuilders.Get, p.FlashblocksBuilders) - p.require().True(ok, "must find flashblocks builder %s", m) - return v -} - -func (p *presetL2Network) AddFlashblocksBuilder(v stack.FlashblocksBuilderNode) { - id := v.ID() - p.require().True(p.fbBuilders.SetIfMissing(id, v), "flashblocks builder %s must not already exist", id) -} - func (p *presetL2Network) L2Proposer(m stack.L2ProposerMatcher) stack.L2Proposer { v, ok := findMatch(m, p.proposers.Get, p.L2Proposers) p.require().True(ok, "must find L2 proposer %s", m) @@ -145,12 +134,6 @@ func (p *presetL2Network) AddL2Proposer(v stack.L2Proposer) { p.require().True(p.proposers.SetIfMissing(id, v), "l2 proposer %s must not already exist", id) } -func (p *presetL2Network) AddFlashblocksWebsocketProxy(v stack.FlashblocksWebsocketProxy) { - id := v.ID() - p.require().Equal(p.chainID, id.ChainID(), "flashblocks websocket proxy %s must be on chain %s", id, p.chainID) - p.require().True(p.fbWsProxies.SetIfMissing(id, v), "flashblocks websocket proxy %s must not already exist", id) -} - func (p *presetL2Network) L2Challenger(m stack.L2ChallengerMatcher) stack.L2Challenger { v, ok := findMatch(m, p.challengers.Get, p.L2Challengers) p.require().True(ok, "must find L2 challenger %s", m) @@ -203,14 +186,6 @@ func (p *presetL2Network) L2Proposers() []stack.L2Proposer { return stack.SortL2Proposers(p.proposers.Values()) } -func (p *presetL2Network) FlashblocksWebsocketProxies() []stack.FlashblocksWebsocketProxy { - return stack.SortFlashblocksWebsocketProxies(p.fbWsProxies.Values()) -} - -func (p *presetL2Network) FlashblocksWebsocketProxyIDs() []stack.FlashblocksWebsocketProxyID { - return stack.SortFlashblocksWebsocketProxyIDs(p.fbWsProxies.Keys()) -} - func (p *presetL2Network) L2ChallengerIDs() []stack.L2ChallengerID { return stack.SortL2ChallengerIDs(p.challengers.Keys()) } @@ -223,10 +198,6 @@ func (p *presetL2Network) Conductors() []stack.Conductor { return stack.SortConductors(p.conductors.Values()) } -func (p *presetL2Network) FlashblocksBuilders() []stack.FlashblocksBuilderNode { - return stack.SortFlashblocksBuilders(p.fbBuilders.Values()) -} - func (p *presetL2Network) L2CLNodeIDs() []stack.L2CLNodeID { return stack.SortL2CLNodeIDs(p.cls.Keys()) } @@ -242,3 +213,33 @@ func (p *presetL2Network) L2ELNodeIDs() []stack.L2ELNodeID { func (p *presetL2Network) L2ELNodes() []stack.L2ELNode { return stack.SortL2ELNodes(p.els.Values()) } + +func (p *presetL2Network) RollupBoostNodes() []stack.RollupBoostNode { + return stack.SortRollupBoostNodes(p.rollupBoostNodes.Values()) +} + +func (p *presetL2Network) OPRBuilderNodes() []stack.OPRBuilderNode { + return stack.SortOPRBuilderNodes(p.oprBuilderNodes.Values()) +} + +func (p *presetL2Network) AddRollupBoostNode(v stack.RollupBoostNode) { + id := v.ID() + p.require().True(p.rollupBoostNodes.SetIfMissing(id, v), "rollup boost node %s must not already exist", id) +} + +func (p *presetL2Network) AddOPRBuilderNode(v stack.OPRBuilderNode) { + id := v.ID() + p.require().True(p.oprBuilderNodes.SetIfMissing(id, v), "OPR builder node %s must not already exist", id) +} + +func (p *presetL2Network) OPRBuilderNode(m stack.OPRBuilderNodeMatcher) stack.OPRBuilderNode { + v, ok := findMatch(m, p.oprBuilderNodes.Get, p.OPRBuilderNodes) + p.require().True(ok, "must find OPR builder node %s", m) + return v +} + +func (p *presetL2Network) RollupBoostNode(m stack.RollupBoostNodeMatcher) stack.RollupBoostNode { + v, ok := findMatch(m, p.rollupBoostNodes.Get, p.RollupBoostNodes) + p.require().True(ok, "must find rollup boost node %s", m) + return v +} diff --git a/op-devstack/shim/op_rbuilder.go b/op-devstack/shim/op_rbuilder.go new file mode 100644 index 00000000000..8e931142136 --- /dev/null +++ b/op-devstack/shim/op_rbuilder.go @@ -0,0 +1,57 @@ +package shim + +import ( + "github.com/stretchr/testify/require" + + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-node/rollup" + "github.com/ethereum-optimism/optimism/op-service/apis" + "github.com/ethereum-optimism/optimism/op-service/sources" +) + +type OPRBuilderNodeConfig struct { + ELNodeConfig + RollupCfg *rollup.Config + ID stack.OPRBuilderNodeID + FlashblocksWsClient stack.FlashblocksWSClient +} + +type OPRBuilderNode struct { + rpcELNode + id stack.OPRBuilderNodeID + engineClient *sources.EngineClient + flashblocksWsClient stack.FlashblocksWSClient +} + +var _ stack.OPRBuilderNode = (*OPRBuilderNode)(nil) + +func NewOPRBuilderNode(cfg OPRBuilderNodeConfig) *OPRBuilderNode { + require.Equal(cfg.T, cfg.ID.ChainID(), cfg.ELNodeConfig.ChainID, "chainID must be configured to match node chainID") + cfg.T = cfg.T.WithCtx(stack.ContextWithID(cfg.T.Ctx(), cfg.ID)) + l2EngineClient, err := sources.NewEngineClient(cfg.ELNodeConfig.Client, cfg.T.Logger(), nil, sources.EngineClientDefaultConfig(cfg.RollupCfg)) + + require.NoError(cfg.T, err) + + return &OPRBuilderNode{ + rpcELNode: newRpcELNode(cfg.ELNodeConfig), + engineClient: l2EngineClient, + id: cfg.ID, + flashblocksWsClient: cfg.FlashblocksWsClient, + } +} + +func (r *OPRBuilderNode) ID() stack.OPRBuilderNodeID { + return r.id +} + +func (r *OPRBuilderNode) L2EthClient() apis.L2EthClient { + return r.engineClient.L2Client +} + +func (r *OPRBuilderNode) FlashblocksClient() stack.FlashblocksWSClient { + return r.flashblocksWsClient +} + +func (r *OPRBuilderNode) L2EngineClient() apis.EngineClient { + return r.engineClient.EngineAPIClient +} diff --git a/op-devstack/shim/rollup_boost.go b/op-devstack/shim/rollup_boost.go new file mode 100644 index 00000000000..6b2024f4f13 --- /dev/null +++ b/op-devstack/shim/rollup_boost.go @@ -0,0 +1,62 @@ +package shim + +import ( + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-node/rollup" + "github.com/ethereum-optimism/optimism/op-service/apis" + "github.com/ethereum-optimism/optimism/op-service/sources" + "github.com/stretchr/testify/require" +) + +type RollupBoostNodeConfig struct { + ELNodeConfig + RollupCfg *rollup.Config + ID stack.RollupBoostNodeID + FlashblocksWsClient stack.FlashblocksWSClient +} + +type RollupBoostNode struct { + rpcELNode + engineClient *sources.EngineClient + + id stack.RollupBoostNodeID + + flashblocksWsClient stack.FlashblocksWSClient +} + +var _ stack.RollupBoostNode = (*RollupBoostNode)(nil) + +func NewRollupBoostNode(cfg RollupBoostNodeConfig) *RollupBoostNode { + require.Equal(cfg.T, cfg.ID.ChainID(), cfg.ELNodeConfig.ChainID, "chainID must be configured to match node chainID") + cfg.T = cfg.T.WithCtx(stack.ContextWithID(cfg.T.Ctx(), cfg.ID)) + l2EngineClient, err := sources.NewEngineClient(cfg.ELNodeConfig.Client, cfg.T.Logger(), nil, sources.EngineClientDefaultConfig(cfg.RollupCfg)) + + require.NoError(cfg.T, err) + + return &RollupBoostNode{ + rpcELNode: newRpcELNode(cfg.ELNodeConfig), + engineClient: l2EngineClient, + id: cfg.ID, + flashblocksWsClient: cfg.FlashblocksWsClient, + } +} + +func (r *RollupBoostNode) ID() stack.RollupBoostNodeID { + return r.id +} + +func (r *RollupBoostNode) L2EthClient() apis.L2EthClient { + return r.engineClient.L2Client +} + +func (r *RollupBoostNode) FlashblocksClient() stack.FlashblocksWSClient { + return r.flashblocksWsClient +} + +func (r *RollupBoostNode) L2EngineClient() apis.EngineClient { + return r.engineClient.EngineAPIClient +} + +func (r *RollupBoostNode) ELNode() stack.ELNode { + return &r.rpcELNode +} diff --git a/op-devstack/stack/fb_builder.go b/op-devstack/stack/fb_builder.go deleted file mode 100644 index 1a67f5b1517..00000000000 --- a/op-devstack/stack/fb_builder.go +++ /dev/null @@ -1,67 +0,0 @@ -package stack - -import ( - "log/slog" - "net/http" - - "github.com/ethereum-optimism/optimism/op-service/apis" - "github.com/ethereum-optimism/optimism/op-service/eth" -) - -type FlashblocksBuilderNode interface { - ELNode - ID() FlashblocksBuilderID - Conductor() Conductor - L2EthClient() apis.L2EthClient - FlashblocksWsUrl() string - FlashblocksWsHeaders() http.Header -} - -type FlashblocksBuilderID idWithChain - -const FlashblocksBuilderKind Kind = "FlashblocksBuilder" - -func NewFlashblocksBuilderID(key string, chainID eth.ChainID) FlashblocksBuilderID { - return FlashblocksBuilderID{ - key: key, - chainID: chainID, - } -} - -func (id FlashblocksBuilderID) String() string { - return idWithChain(id).string(FlashblocksBuilderKind) -} - -func (id FlashblocksBuilderID) ChainID() eth.ChainID { - return idWithChain(id).chainID -} - -func (id FlashblocksBuilderID) MarshalText() ([]byte, error) { - return idWithChain(id).marshalText(FlashblocksBuilderKind) -} - -func (id FlashblocksBuilderID) LogValue() slog.Value { - return slog.StringValue(id.String()) -} - -func (id *FlashblocksBuilderID) UnmarshalText(data []byte) error { - return (*idWithChain)(id).unmarshalText(FlashblocksBuilderKind, data) -} - -func SortFlashblocksBuilderIDs(ids []FlashblocksBuilderID) []FlashblocksBuilderID { - return copyAndSort(ids, func(a, b FlashblocksBuilderID) bool { - return lessIDWithChain(idWithChain(a), idWithChain(b)) - }) -} - -func SortFlashblocksBuilders(elems []FlashblocksBuilderNode) []FlashblocksBuilderNode { - return copyAndSort(elems, func(a, b FlashblocksBuilderNode) bool { - return lessIDWithChain(idWithChain(a.ID()), idWithChain(b.ID())) - }) -} - -var _ FlashblocksBuilderMatcher = FlashblocksBuilderID{} - -func (id FlashblocksBuilderID) Match(elems []FlashblocksBuilderNode) []FlashblocksBuilderNode { - return findByID(id, elems) -} diff --git a/op-devstack/stack/fb_ws_client.go b/op-devstack/stack/fb_ws_client.go new file mode 100644 index 00000000000..9da423dcbf3 --- /dev/null +++ b/op-devstack/stack/fb_ws_client.go @@ -0,0 +1,63 @@ +package stack + +import ( + "log/slog" + "net/http" + + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +type FlashblocksWSClient interface { + Common + ChainID() eth.ChainID + ID() FlashblocksWSClientID + WsUrl() string + WsHeaders() http.Header +} + +type FlashblocksWSClientID idWithChain + +const FlashblocksWSClientKind Kind = "FlashblocksWSClient" + +func NewFlashblocksWSClientID(key string, chainID eth.ChainID) FlashblocksWSClientID { + return FlashblocksWSClientID{ + key: key, + chainID: chainID, + } +} + +func (id FlashblocksWSClientID) String() string { + return idWithChain(id).string(FlashblocksWSClientKind) +} + +func (id FlashblocksWSClientID) ChainID() eth.ChainID { + return idWithChain(id).chainID +} + +func (id FlashblocksWSClientID) MarshalText() ([]byte, error) { + return idWithChain(id).marshalText(FlashblocksWSClientKind) +} + +func (id FlashblocksWSClientID) LogValue() slog.Value { + return slog.StringValue(id.String()) +} + +func (id *FlashblocksWSClientID) UnmarshalText(data []byte) error { + return (*idWithChain)(id).unmarshalText(FlashblocksWSClientKind, data) +} + +func SortFlashblocksWSClientIDs(ids []FlashblocksWSClientID) []FlashblocksWSClientID { + return copyAndSort(ids, func(a, b FlashblocksWSClientID) bool { + return lessIDWithChain(idWithChain(a), idWithChain(b)) + }) +} + +func SortFlashblocksWSClients(elems []FlashblocksWSClient) []FlashblocksWSClient { + return copyAndSort(elems, func(a, b FlashblocksWSClient) bool { + return lessIDWithChain(idWithChain(a.ID()), idWithChain(b.ID())) + }) +} + +func (id FlashblocksWSClientID) Match(elems []FlashblocksWSClient) []FlashblocksWSClient { + return findByID(id, elems) +} diff --git a/op-devstack/stack/fb_ws_proxy.go b/op-devstack/stack/fb_ws_proxy.go deleted file mode 100644 index 4b0f1167bfa..00000000000 --- a/op-devstack/stack/fb_ws_proxy.go +++ /dev/null @@ -1,65 +0,0 @@ -package stack - -import ( - "log/slog" - "net/http" - - "github.com/ethereum-optimism/optimism/op-service/eth" -) - -type FlashblocksWebsocketProxy interface { - Common - ChainID() eth.ChainID - ID() FlashblocksWebsocketProxyID - WsUrl() string - WsHeaders() http.Header -} - -type FlashblocksWebsocketProxyID idWithChain - -const FlashblocksWebsocketProxyKind Kind = "FlashblocksWebsocketProxy" - -func NewFlashblocksWebsocketProxyID(key string, chainID eth.ChainID) FlashblocksWebsocketProxyID { - return FlashblocksWebsocketProxyID{ - key: key, - chainID: chainID, - } -} - -func (id FlashblocksWebsocketProxyID) String() string { - return idWithChain(id).string(FlashblocksWebsocketProxyKind) -} - -func (id FlashblocksWebsocketProxyID) ChainID() eth.ChainID { - return idWithChain(id).chainID -} - -func (id FlashblocksWebsocketProxyID) MarshalText() ([]byte, error) { - return idWithChain(id).marshalText(FlashblocksWebsocketProxyKind) -} - -func (id FlashblocksWebsocketProxyID) LogValue() slog.Value { - return slog.StringValue(id.String()) -} - -func (id *FlashblocksWebsocketProxyID) UnmarshalText(data []byte) error { - return (*idWithChain)(id).unmarshalText(FlashblocksWebsocketProxyKind, data) -} - -func SortFlashblocksWebsocketProxyIDs(ids []FlashblocksWebsocketProxyID) []FlashblocksWebsocketProxyID { - return copyAndSort(ids, func(a, b FlashblocksWebsocketProxyID) bool { - return lessIDWithChain(idWithChain(a), idWithChain(b)) - }) -} - -func SortFlashblocksWebsocketProxies(elems []FlashblocksWebsocketProxy) []FlashblocksWebsocketProxy { - return copyAndSort(elems, func(a, b FlashblocksWebsocketProxy) bool { - return lessIDWithChain(idWithChain(a.ID()), idWithChain(b.ID())) - }) -} - -var _ FlashblocksBuilderMatcher = FlashblocksBuilderID{} - -func (id FlashblocksWebsocketProxyID) Match(elems []FlashblocksWebsocketProxy) []FlashblocksWebsocketProxy { - return findByID(id, elems) -} diff --git a/op-devstack/stack/l2_cl.go b/op-devstack/stack/l2_cl.go index e5e5b042b34..8005cb3d28c 100644 --- a/op-devstack/stack/l2_cl.go +++ b/op-devstack/stack/l2_cl.go @@ -82,9 +82,15 @@ type L2CLNode interface { // ELs returns the engine(s) that this L2CLNode is connected to. // This may be empty, if the L2CL is not connected to any. ELs() []L2ELNode + RollupBoostNodes() []RollupBoostNode + OPRBuilderNodes() []OPRBuilderNode + + ELClient() apis.EthClient } type LinkableL2CLNode interface { // Links the nodes. Does not make any backend changes, just registers the EL as connected to this CL. LinkEL(el L2ELNode) + LinkRollupBoostNode(rollupBoostNode RollupBoostNode) + LinkOPRBuilderNode(oprb OPRBuilderNode) } diff --git a/op-devstack/stack/l2_network.go b/op-devstack/stack/l2_network.go index 296f0412c99..d49f26750b6 100644 --- a/op-devstack/stack/l2_network.go +++ b/op-devstack/stack/l2_network.go @@ -91,6 +91,8 @@ type L2Network interface { L2CLNode(m L2CLMatcher) L2CLNode L2ELNode(m L2ELMatcher) L2ELNode Conductor(m ConductorMatcher) Conductor + RollupBoostNode(m RollupBoostNodeMatcher) RollupBoostNode + OPRBuilderNode(m OPRBuilderNodeMatcher) OPRBuilderNode L2BatcherIDs() []L2BatcherID L2ProposerIDs() []L2ProposerID @@ -104,10 +106,8 @@ type L2Network interface { L2CLNodes() []L2CLNode L2ELNodes() []L2ELNode Conductors() []Conductor - FlashblocksBuilders() []FlashblocksBuilderNode - AddFlashblocksBuilder(v FlashblocksBuilderNode) - FlashblocksWebsocketProxies() []FlashblocksWebsocketProxy - AddFlashblocksWebsocketProxy(v FlashblocksWebsocketProxy) + RollupBoostNodes() []RollupBoostNode + OPRBuilderNodes() []OPRBuilderNode } // ExtensibleL2Network is an optional extension interface for L2Network, @@ -121,4 +121,6 @@ type ExtensibleL2Network interface { AddL2CLNode(v L2CLNode) AddL2ELNode(v L2ELNode) AddConductor(v Conductor) + AddRollupBoostNode(v RollupBoostNode) + AddOPRBuilderNode(v OPRBuilderNode) } diff --git a/op-devstack/stack/match/engine.go b/op-devstack/stack/match/engine.go index c5f0f36a7c9..a9bd93413e6 100644 --- a/op-devstack/stack/match/engine.go +++ b/op-devstack/stack/match/engine.go @@ -1,6 +1,8 @@ package match -import "github.com/ethereum-optimism/optimism/op-devstack/stack" +import ( + "github.com/ethereum-optimism/optimism/op-devstack/stack" +) func WithEngine(engine stack.L2ELNodeID) stack.Matcher[stack.L2CLNodeID, stack.L2CLNode] { return MatchElemFn[stack.L2CLNodeID, stack.L2CLNode](func(elem stack.L2CLNode) bool { @@ -9,6 +11,18 @@ func WithEngine(engine stack.L2ELNodeID) stack.Matcher[stack.L2CLNodeID, stack.L return true } } + rbID := stack.RollupBoostNodeID(engine) + for _, rb := range elem.RollupBoostNodes() { + if rb.ID().ChainID() == rbID.ChainID() { + return true + } + } + oprbID := stack.OPRBuilderNodeID(engine) + for _, oprb := range elem.OPRBuilderNodes() { + if oprb.ID() == oprbID { + return true + } + } return false }) } @@ -20,6 +34,18 @@ func EngineFor(cl stack.L2CLNode) stack.Matcher[stack.L2ELNodeID, stack.L2ELNode return true } } + rbID := stack.RollupBoostNodeID(elem.ID()) + for _, rb := range cl.RollupBoostNodes() { + if rb.ID().ChainID() == rbID.ChainID() { + return true + } + } + oprbID := stack.OPRBuilderNodeID(elem.ID()) + for _, oprb := range cl.OPRBuilderNodes() { + if oprb.ID() == oprbID { + return true + } + } return false }) } diff --git a/op-devstack/stack/match/first.go b/op-devstack/stack/match/first.go index 572b49e335c..6d1a53ab6f9 100644 --- a/op-devstack/stack/match/first.go +++ b/op-devstack/stack/match/first.go @@ -21,3 +21,6 @@ var FirstCluster = First[stack.ClusterID, stack.Cluster]() var FirstFaucet = First[stack.FaucetID, stack.Faucet]() var FirstSyncTester = First[stack.SyncTesterID, stack.SyncTester]() + +var FirstOPRBuilderNode = First[stack.OPRBuilderNodeID, stack.OPRBuilderNode]() +var FirstRollupBoostNode = First[stack.RollupBoostNodeID, stack.RollupBoostNode]() diff --git a/op-devstack/stack/match/labels.go b/op-devstack/stack/match/labels.go index 9224f6c8550..be32276b337 100644 --- a/op-devstack/stack/match/labels.go +++ b/op-devstack/stack/match/labels.go @@ -20,13 +20,13 @@ const ( type Vendor string const ( - Geth Vendor = "geth" - OpReth Vendor = "op-reth" - OpGeth Vendor = "op-geth" - Proxyd Vendor = "proxyd" - FlashblocksWebsocketProxy Vendor = "flashblocks-websocket-proxy" - OpNode Vendor = "op-node" - KonaNode Vendor = "kona-node" + Geth Vendor = "geth" + OpReth Vendor = "op-reth" + OpGeth Vendor = "op-geth" + Proxyd Vendor = "proxyd" + FlashblocksWSClient Vendor = "flashblocks-websocket-proxy" + OpNode Vendor = "op-node" + KonaNode Vendor = "kona-node" ) func (v Vendor) Match(elems []stack.L2ELNode) []stack.L2ELNode { diff --git a/op-devstack/stack/matcher.go b/op-devstack/stack/matcher.go index afe97cfeff9..8053c5e80b6 100644 --- a/op-devstack/stack/matcher.go +++ b/op-devstack/stack/matcher.go @@ -55,10 +55,12 @@ type TestSequencerMatcher = Matcher[TestSequencerID, TestSequencer] type ConductorMatcher = Matcher[ConductorID, Conductor] -type FlashblocksBuilderMatcher = Matcher[FlashblocksBuilderID, FlashblocksBuilderNode] - type L2ELMatcher = Matcher[L2ELNodeID, L2ELNode] type FaucetMatcher = Matcher[FaucetID, Faucet] type SyncTesterMatcher = Matcher[SyncTesterID, SyncTester] + +type RollupBoostNodeMatcher = Matcher[RollupBoostNodeID, RollupBoostNode] + +type OPRBuilderNodeMatcher = Matcher[OPRBuilderNodeID, OPRBuilderNode] diff --git a/op-devstack/stack/op_rbuilder.go b/op-devstack/stack/op_rbuilder.go new file mode 100644 index 00000000000..7c655e32237 --- /dev/null +++ b/op-devstack/stack/op_rbuilder.go @@ -0,0 +1,78 @@ +package stack + +import ( + "log/slog" + + "github.com/ethereum-optimism/optimism/op-service/apis" + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +// OPRBuilderNodeID identifies a L2ELNode by name and chainID, is type-safe, and can be value-copied and used as map key. +type OPRBuilderNodeID idWithChain + +var _ IDWithChain = (*OPRBuilderNodeID)(nil) + +const OPRBuilderNodeKind Kind = "OPRBuilderNode" + +func NewOPRBuilderNodeID(key string, chainID eth.ChainID) OPRBuilderNodeID { + return OPRBuilderNodeID{ + key: key, + chainID: chainID, + } +} + +func (id OPRBuilderNodeID) String() string { + return idWithChain(id).string(OPRBuilderNodeKind) +} + +func (id OPRBuilderNodeID) ChainID() eth.ChainID { + return id.chainID +} + +func (id OPRBuilderNodeID) Kind() Kind { + return OPRBuilderNodeKind +} + +func (id OPRBuilderNodeID) Key() string { + return id.key +} + +func (id OPRBuilderNodeID) LogValue() slog.Value { + return slog.StringValue(id.String()) +} + +func (id OPRBuilderNodeID) MarshalText() ([]byte, error) { + return idWithChain(id).marshalText(OPRBuilderNodeKind) +} + +func (id *OPRBuilderNodeID) UnmarshalText(data []byte) error { + return (*idWithChain)(id).unmarshalText(OPRBuilderNodeKind, data) +} + +func SortOPRBuilderIDs(ids []OPRBuilderNodeID) []OPRBuilderNodeID { + return copyAndSort(ids, func(a, b OPRBuilderNodeID) bool { + return lessIDWithChain(idWithChain(a), idWithChain(b)) + }) +} + +func SortOPRBuilderNodes(elems []OPRBuilderNode) []OPRBuilderNode { + return copyAndSort(elems, func(a, b OPRBuilderNode) bool { + return lessIDWithChain(idWithChain(a.ID()), idWithChain(b.ID())) + }) +} + +var _ OPRBuilderNodeMatcher = OPRBuilderNodeID{} + +func (id OPRBuilderNodeID) Match(elems []OPRBuilderNode) []OPRBuilderNode { + return findByID(id, elems) +} + +// L2ELNode is a L2 ethereum execution-layer node +type OPRBuilderNode interface { + ID() OPRBuilderNodeID + L2EthClient() apis.L2EthClient + L2EngineClient() apis.EngineClient + FlashblocksClient() FlashblocksWSClient + + ELNode +} diff --git a/op-devstack/stack/orchestrator.go b/op-devstack/stack/orchestrator.go index d9dab5e48ff..a1e636fe26b 100644 --- a/op-devstack/stack/orchestrator.go +++ b/op-devstack/stack/orchestrator.go @@ -24,6 +24,8 @@ type ControlPlane interface { L2CLNodeState(id L2CLNodeID, action ControlAction) L2ELNodeState(id L2ELNodeID, action ControlAction) FakePoSState(id L1CLNodeID, action ControlAction) + RollupBoostNodeState(id RollupBoostNodeID, action ControlAction) + OPRBuilderNodeState(id OPRBuilderNodeID, action ControlAction) } // Orchestrator is the base interface for all system orchestrators. diff --git a/op-devstack/stack/rollup_boost.go b/op-devstack/stack/rollup_boost.go new file mode 100644 index 00000000000..92fdd6e760f --- /dev/null +++ b/op-devstack/stack/rollup_boost.go @@ -0,0 +1,78 @@ +package stack + +import ( + "log/slog" + + "github.com/ethereum-optimism/optimism/op-service/apis" + "github.com/ethereum-optimism/optimism/op-service/eth" +) + +// RollupBoostNodeID identifies a RollupBoost node by name and chainID, is type-safe, and can be value-copied and used as map key. +type RollupBoostNodeID L2ELNodeID + +var _ IDWithChain = (*RollupBoostNodeID)(nil) + +const RollupBoostNodeKind Kind = "RollupBoostNode" + +func NewRollupBoostNodeID(key string, chainID eth.ChainID) RollupBoostNodeID { + return RollupBoostNodeID{ + key: key, + chainID: chainID, + } +} + +func (id RollupBoostNodeID) String() string { + return idWithChain(id).string(RollupBoostNodeKind) +} + +func (id RollupBoostNodeID) ChainID() eth.ChainID { + return id.chainID +} + +func (id RollupBoostNodeID) Kind() Kind { + return RollupBoostNodeKind +} + +func (id RollupBoostNodeID) Key() string { + return id.key +} + +func (id RollupBoostNodeID) LogValue() slog.Value { + return slog.StringValue(id.String()) +} + +func (id RollupBoostNodeID) MarshalText() ([]byte, error) { + return idWithChain(id).marshalText(RollupBoostNodeKind) +} + +func (id *RollupBoostNodeID) UnmarshalText(data []byte) error { + return (*idWithChain)(id).unmarshalText(RollupBoostNodeKind, data) +} + +func SortRollupBoostIDs(ids []RollupBoostNodeID) []RollupBoostNodeID { + return copyAndSort(ids, func(a, b RollupBoostNodeID) bool { + return lessIDWithChain(idWithChain(a), idWithChain(b)) + }) +} + +func SortRollupBoostNodes(elems []RollupBoostNode) []RollupBoostNode { + return copyAndSort(elems, func(a, b RollupBoostNode) bool { + return lessIDWithChain(idWithChain(a.ID()), idWithChain(b.ID())) + }) +} + +var _ RollupBoostNodeMatcher = RollupBoostNodeID{} + +func (id RollupBoostNodeID) Match(elems []RollupBoostNode) []RollupBoostNode { + return findByID(id, elems) +} + +// RollupBoostNode is a shim service between an L2 consensus-layer node and an L2 ethereum execution-layer node +type RollupBoostNode interface { + ID() RollupBoostNodeID + L2EthClient() apis.L2EthClient + L2EngineClient() apis.EngineClient + FlashblocksClient() FlashblocksWSClient + + ELNode +} diff --git a/op-devstack/sysext/control_plane.go b/op-devstack/sysext/control_plane.go index ecdcdd63d01..65eb37d9d52 100644 --- a/op-devstack/sysext/control_plane.go +++ b/op-devstack/sysext/control_plane.go @@ -42,4 +42,12 @@ func (c *ControlPlane) FakePoSState(id stack.L1CLNodeID, mode stack.ControlActio panic("not implemented: plug in kurtosis wrapper, or gate for the test that uses this method to not run in kurtosis") } +func (c *ControlPlane) RollupBoostNodeState(id stack.RollupBoostNodeID, mode stack.ControlAction) { + c.setLifecycleState(id.Key(), mode) +} + +func (c *ControlPlane) OPRBuilderNodeState(id stack.OPRBuilderNodeID, mode stack.ControlAction) { + c.setLifecycleState(id.Key(), mode) +} + var _ stack.ControlPlane = (*ControlPlane)(nil) diff --git a/op-devstack/sysext/helpers.go b/op-devstack/sysext/helpers.go index cd36fb192d2..c6160a43802 100644 --- a/op-devstack/sysext/helpers.go +++ b/op-devstack/sysext/helpers.go @@ -14,10 +14,11 @@ import ( ) const ( - ELServiceName = "el" - CLServiceName = "cl" - RBuilderServiceName = "rbuilder" - ConductorServiceName = "conductor" + ELServiceName = "el" + CLServiceName = "cl" + OPRBuilderServiceName = "op-rbuilder" + RollupBoostServiceName = "rollup-boost" + ConductorServiceName = "conductor" HTTPProtocol = "http" RPCProtocol = "rpc" diff --git a/op-devstack/sysext/l2.go b/op-devstack/sysext/l2.go index 5ea7c198420..05a12fb0bd7 100644 --- a/op-devstack/sysext/l2.go +++ b/op-devstack/sysext/l2.go @@ -65,13 +65,13 @@ func (o *Orchestrator) hydrateL2(net *descriptors.L2Chain, system stack.Extensib for _, node := range net.Nodes { o.hydrateL2ELCL(&node, l2, opts) o.hydrateConductors(&node, l2) - o.hydrateFlashblocksBuilderIfPresent(&node, l2, opts) + o.hydrateRollupBoostNodeMaybe(&node, l2, opts) + o.hydrateOPRBuilderMaybe(&node, l2, opts) } o.hydrateBatcherMaybe(net, l2) o.hydrateProposerMaybe(net, l2) o.hydrateChallengerMaybe(net, l2) o.hydrateL2ProxydMaybe(net, l2) - o.hydrateFlashblocksWebsocketProxyMaybe(net, l2) if faucet, ok := net.Services["faucet"]; ok { for _, instance := range faucet { @@ -193,35 +193,36 @@ func (o *Orchestrator) hydrateConductors(node *descriptors.Node, l2Net stack.Ext l2Net.AddConductor(conductor) } -func (o *Orchestrator) hydrateFlashblocksBuilderIfPresent(node *descriptors.Node, l2Net stack.ExtensibleL2Network, opts []client.RPCOption) { +func (o *Orchestrator) hydrateRollupBoostNodeMaybe(node *descriptors.Node, l2Net stack.ExtensibleL2Network, opts []client.RPCOption) { require := l2Net.T().Require() l2ID := l2Net.ID() - rbuilderService, ok := node.Services[RBuilderServiceName] + rollupBoostService, ok := node.Services[RollupBoostServiceName] if !ok { - l2Net.Logger().Debug("L2 net node is missing the flashblocksBuilder service", "node", node.Name, "l2", l2ID) + l2Net.Logger().Debug("L2 net node does not have a rollup-boost service", "node", node.Name, "l2", l2ID) return } - associatedConductorService, ok := node.Services[ConductorServiceName] - require.True(ok, "L2 rbuilder service must have an associated conductor service", l2ID) + flashblocksWsUrl, flashblocksWsHeaders, err := o.findProtocolService(rollupBoostService, WebsocketFlashblocksProtocol) + require.NoError(err, "failed to find websocket service for rollup-boost") - flashblocksWsUrl, flashblocksWsHeaders, err := o.findProtocolService(rbuilderService, WebsocketFlashblocksProtocol) - require.NoError(err, "failed to find websocket service for rbuilder") - - flashblocksBuilder := shim.NewFlashblocksBuilderNode(shim.FlashblocksBuilderNodeConfig{ - ID: stack.NewFlashblocksBuilderID(rbuilderService.Name, l2ID.ChainID()), + rollupBoost := shim.NewRollupBoostNode(shim.RollupBoostNodeConfig{ + ID: stack.NewRollupBoostNodeID(rollupBoostService.Name, l2ID.ChainID()), ELNodeConfig: shim.ELNodeConfig{ CommonConfig: shim.NewCommonConfig(l2Net.T()), - Client: o.rpcClient(l2Net.T(), rbuilderService, RPCProtocol, "/", opts...), + Client: o.rpcClient(l2Net.T(), rollupBoostService, RPCProtocol, "/", opts...), ChainID: l2ID.ChainID(), }, - Conductor: l2Net.Conductor(stack.ConductorID(associatedConductorService.Name)), - FlashblocksWsUrl: flashblocksWsUrl, - FlashblocksWsHeaders: flashblocksWsHeaders, + RollupCfg: l2Net.RollupConfig(), + FlashblocksWsClient: shim.NewFlashblocksWSClient(shim.FlashblocksWSClientConfig{ + CommonConfig: shim.NewCommonConfig(l2Net.T()), + ID: stack.NewFlashblocksWSClientID(rollupBoostService.Name, l2ID.ChainID()), + WsUrl: flashblocksWsUrl, + WsHeaders: flashblocksWsHeaders, + }), }) - l2Net.AddFlashblocksBuilder(flashblocksBuilder) + l2Net.AddRollupBoostNode(rollupBoost) } func (o *Orchestrator) hydrateL2ProxydMaybe(net *descriptors.L2Chain, l2Net stack.ExtensibleL2Network) { @@ -250,29 +251,35 @@ func (o *Orchestrator) hydrateL2ProxydMaybe(net *descriptors.L2Chain, l2Net stac } } -func (o *Orchestrator) hydrateFlashblocksWebsocketProxyMaybe(net *descriptors.L2Chain, l2Net stack.ExtensibleL2Network) { +func (o *Orchestrator) hydrateOPRBuilderMaybe(node *descriptors.Node, l2Net stack.ExtensibleL2Network, opts []client.RPCOption) { require := l2Net.T().Require() - l2ID := getL2ID(net) - require.Equal(l2ID, l2Net.ID(), "must match L2 chain descriptor and target L2 net") + l2ID := l2Net.ID() - fbWsProxyService, ok := net.Services["flashblocks-websocket-proxy"] + rbuilderService, ok := node.Services[OPRBuilderServiceName] if !ok { + l2Net.Logger().Debug("L2 net node does not have a oprbuilder service", "node", node.Name, "l2", l2ID) return } - for _, instance := range fbWsProxyService { - wsUrl, wsHeaders, err := o.findProtocolService(instance, WebsocketFlashblocksProtocol) - require.NoError(err, "failed to get the websocket url for the flashblocks websocket proxy", "service", instance.Name) + flashblocksWsUrl, flashblocksWsHeaders, err := o.findProtocolService(rbuilderService, WebsocketFlashblocksProtocol) + require.NoError(err, "failed to find websocket service for rbuilder") - fbWsProxyShim := shim.NewFlashblocksWebsocketProxy(shim.FlashblocksWebsocketProxyConfig{ + flashblocksBuilder := shim.NewOPRBuilderNode(shim.OPRBuilderNodeConfig{ + ID: stack.NewOPRBuilderNodeID(rbuilderService.Name, l2ID.ChainID()), + ELNodeConfig: shim.ELNodeConfig{ CommonConfig: shim.NewCommonConfig(l2Net.T()), - ID: stack.NewFlashblocksWebsocketProxyID(instance.Name, l2ID.ChainID()), - WsUrl: wsUrl, - WsHeaders: wsHeaders, - }) - fbWsProxyShim.SetLabel(match.LabelVendor, string(match.FlashblocksWebsocketProxy)) - l2Net.AddFlashblocksWebsocketProxy(fbWsProxyShim) - } + Client: o.rpcClient(l2Net.T(), rbuilderService, RPCProtocol, "/", opts...), + ChainID: l2ID.ChainID(), + }, + FlashblocksWsClient: shim.NewFlashblocksWSClient(shim.FlashblocksWSClientConfig{ + CommonConfig: shim.NewCommonConfig(l2Net.T()), + ID: stack.NewFlashblocksWSClientID(rbuilderService.Name, l2ID.ChainID()), + WsUrl: flashblocksWsUrl, + WsHeaders: flashblocksWsHeaders, + }), + }) + + l2Net.AddOPRBuilderNode(flashblocksBuilder) } func (o *Orchestrator) hydrateBatcherMaybe(net *descriptors.L2Chain, l2Net stack.ExtensibleL2Network) { diff --git a/op-devstack/sysgo/control_plane.go b/op-devstack/sysgo/control_plane.go index 09aa358c02f..a2817b6a11a 100644 --- a/op-devstack/sysgo/control_plane.go +++ b/op-devstack/sysgo/control_plane.go @@ -42,4 +42,16 @@ func (c *ControlPlane) FakePoSState(id stack.L1CLNodeID, mode stack.ControlActio control(s.fakepos, mode) } +func (c *ControlPlane) OPRBuilderNodeState(id stack.OPRBuilderNodeID, mode stack.ControlAction) { + s, ok := c.o.oprbuilderNodes.Get(id) + c.o.P().Require().True(ok, "need oprbuilder node to change state") + control(s, mode) +} + +func (c *ControlPlane) RollupBoostNodeState(id stack.RollupBoostNodeID, mode stack.ControlAction) { + s, ok := c.o.rollupBoosts.Get(id) + c.o.P().Require().True(ok, "need rollup boost node to change state") + control(s, mode) +} + var _ stack.ControlPlane = (*ControlPlane)(nil) diff --git a/op-devstack/sysgo/el_node_identity.go b/op-devstack/sysgo/el_node_identity.go new file mode 100644 index 00000000000..bd21dfa48d8 --- /dev/null +++ b/op-devstack/sysgo/el_node_identity.go @@ -0,0 +1,47 @@ +package sysgo + +import ( + "crypto/ecdsa" + "encoding/hex" + "net" + "strconv" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +type ELNodeIdentity struct { + Key *ecdsa.PrivateKey + Port int + Enode string +} + +func NewELNodeIdentity(addr string, port int) *ELNodeIdentity { + key, err := crypto.GenerateKey() + if err != nil { + panic(err) + } + if port <= 0 { + portStr, err := getAvailableLocalPort() + if err != nil { + panic(err) + } + port, err = strconv.Atoi(portStr) + if err != nil { + panic(err) + } + } + ip := net.ParseIP(addr) + if ip == nil { + panic("invalid ip for ELNodeIdentity: " + addr) + } + return &ELNodeIdentity{ + Key: key, + Port: port, + Enode: enode.NewV4(&key.PublicKey, ip, port, port).String(), + } +} + +func (id *ELNodeIdentity) KeyHex() string { + return hex.EncodeToString(crypto.FromECDSA(id.Key)) +} diff --git a/op-devstack/sysgo/l1_nodes_subprocess.go b/op-devstack/sysgo/l1_nodes_subprocess.go index ee88a2d9e73..4ebc01ac373 100644 --- a/op-devstack/sysgo/l1_nodes_subprocess.go +++ b/op-devstack/sysgo/l1_nodes_subprocess.go @@ -109,12 +109,12 @@ func (n *ExternalL1Geth) Start() { } } } - stdOutLogs := logpipe.LogProcessor(func(line []byte) { + stdOutLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseGoStructuredLogs(line) logOut(e) onLogEntry(e) }) - stdErrLogs := logpipe.LogProcessor(func(line []byte) { + stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseGoStructuredLogs(line) logErr(e) onLogEntry(e) diff --git a/op-devstack/sysgo/l2_cl_kona.go b/op-devstack/sysgo/l2_cl_kona.go index 43bc28f5fa8..b9dfe61c327 100644 --- a/op-devstack/sysgo/l2_cl_kona.go +++ b/op-devstack/sysgo/l2_cl_kona.go @@ -111,12 +111,12 @@ func (k *KonaNode) Start() { metricsTargetChan <- NewPrometheusMetricsTarget(parsedUrl.Hostname(), parsedUrl.Port(), false) } } - stdOutLogs := logpipe.LogProcessor(func(line []byte) { + stdOutLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logOut(e) onLogEntry(e) }) - stdErrLogs := logpipe.LogProcessor(func(line []byte) { + stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logErr(e) }) @@ -181,7 +181,7 @@ func WithKonaNode(l2CLID stack.L2CLNodeID, l1CLID stack.L1CLNodeID, l1ELID stack l1CL, ok := orch.l1CLs.Get(l1CLID) require.True(ok, "l1 CL node required") - l2EL, ok := orch.l2ELs.Get(l2ELID) + l2EL, ok := orch.GetL2EL(l2ELID) require.True(ok, "l2 EL node required") cfg := DefaultL2CLConfig() diff --git a/op-devstack/sysgo/l2_cl_opnode.go b/op-devstack/sysgo/l2_cl_opnode.go index 6f867b20313..75e2fd5ffc7 100644 --- a/op-devstack/sysgo/l2_cl_opnode.go +++ b/op-devstack/sysgo/l2_cl_opnode.go @@ -73,7 +73,26 @@ func (n *OpNode) hydrate(system stack.ExtensibleSystem) { l2Net := system.L2Network(stack.L2NetworkID(n.id.ChainID())) l2Net.(stack.ExtensibleL2Network).AddL2CLNode(sysL2CL) if n.el != nil { - sysL2CL.(stack.LinkableL2CLNode).LinkEL(l2Net.L2ELNode(n.el)) + for _, el := range l2Net.L2ELNodes() { + if el.ID() == *n.el { + sysL2CL.(stack.LinkableL2CLNode).LinkEL(el) + return + } + } + rbID := stack.RollupBoostNodeID(*n.el) + for _, rb := range l2Net.RollupBoostNodes() { + if rb.ID() == rbID { + sysL2CL.(stack.LinkableL2CLNode).LinkRollupBoostNode(rb) + return + } + } + oprbID := stack.OPRBuilderNodeID(*n.el) + for _, oprb := range l2Net.OPRBuilderNodes() { + if oprb.ID() == oprbID { + sysL2CL.(stack.LinkableL2CLNode).LinkOPRBuilderNode(oprb) + return + } + } } } @@ -160,7 +179,7 @@ func WithOpNode(l2CLID stack.L2CLNodeID, l1CLID stack.L1CLNodeID, l1ELID stack.L require.True(ok, "l1 CL node required") // Get the L2EL node (which can be a regular EL node or a SyncTesterEL) - l2EL, ok := orch.l2ELs.Get(l2ELID) + l2EL, ok := orch.GetL2EL(l2ELID) require.True(ok, "l2 EL node required") // Get dependency set from cluster if available @@ -245,9 +264,6 @@ func WithOpNode(l2CLID stack.L2CLNodeID, l1CLID stack.L1CLNodeID, l1ELID stack.L // Set the req-resp sync flag as per config p2pConfig.EnableReqRespSync = cfg.EnableReqRespSync - // Get the L2 engine address from the EL node (which can be a regular EL node or a SyncTesterEL) - l2EngineAddr := l2EL.EngineRPC() - nodeCfg := &config.Config{ L1: &config.L1EndpointConfig{ L1NodeAddr: l1EL.UserRPC(), @@ -261,7 +277,7 @@ func WithOpNode(l2CLID stack.L2CLNodeID, l1CLID stack.L1CLNodeID, l1ELID stack.L }, L1ChainConfig: l1Net.genesis.Config, L2: &config.L2EndpointConfig{ - L2EngineAddr: l2EngineAddr, + L2EngineAddr: l2EL.EngineRPC(), L2EngineJWTSecret: jwtSecret, }, Beacon: &config.L1BeaconEndpointConfig{ diff --git a/op-devstack/sysgo/l2_el.go b/op-devstack/sysgo/l2_el.go index 19501a32766..8efb7d390aa 100644 --- a/op-devstack/sysgo/l2_el.go +++ b/op-devstack/sysgo/l2_el.go @@ -16,7 +16,12 @@ type L2ELNode interface { } type L2ELConfig struct { - SupervisorID *stack.SupervisorID + SupervisorID *stack.SupervisorID + P2PAddr string + P2PPort int + P2PNodeKeyHex string + StaticPeers []string + TrustedPeers []string } func L2ELWithSupervisor(supervisorID stack.SupervisorID) L2ELOption { @@ -25,9 +30,25 @@ func L2ELWithSupervisor(supervisorID stack.SupervisorID) L2ELOption { }) } +// L2ELWithP2PConfig sets deterministic P2P identity and static peers for the L2 EL. +func L2ELWithP2PConfig(addr string, port int, nodeKeyHex string, staticPeers, trustedPeers []string) L2ELOption { + return L2ELOptionFn(func(p devtest.P, id stack.L2ELNodeID, cfg *L2ELConfig) { + cfg.P2PAddr = addr + cfg.P2PPort = port + cfg.P2PNodeKeyHex = nodeKeyHex + cfg.StaticPeers = staticPeers + cfg.TrustedPeers = trustedPeers + }) +} + func DefaultL2ELConfig() *L2ELConfig { return &L2ELConfig{ - SupervisorID: nil, + SupervisorID: nil, + P2PAddr: "127.0.0.1", + P2PPort: 0, + P2PNodeKeyHex: "", + StaticPeers: nil, + TrustedPeers: nil, } } diff --git a/op-devstack/sysgo/l2_el_opgeth.go b/op-devstack/sysgo/l2_el_opgeth.go index e53f07ff3ad..34f8175fe10 100644 --- a/op-devstack/sysgo/l2_el_opgeth.go +++ b/op-devstack/sysgo/l2_el_opgeth.go @@ -1,12 +1,16 @@ package sysgo import ( + "strconv" + "strings" "sync" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" gn "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum-optimism/optimism/op-devstack/devtest" @@ -30,6 +34,7 @@ type OpGeth struct { supervisorRPC string l2Geth *geth.GethInstance readOnly bool + cfg *L2ELConfig authRPC string userRPC string @@ -112,11 +117,46 @@ func (n *OpGeth) Start() { func(ethCfg *ethconfig.Config, nodeCfg *gn.Config) error { ethCfg.InteropMessageRPC = n.supervisorRPC ethCfg.InteropMempoolFiltering = true + + listenAddr := n.cfg.P2PAddr + port := n.cfg.P2PPort + listenAddr = listenAddr + ":" + strconv.Itoa(port) + nodeCfg.P2P = p2p.Config{ NoDiscovery: true, - ListenAddr: "127.0.0.1:0", + ListenAddr: listenAddr, MaxPeers: 10, } + + if n.cfg.P2PNodeKeyHex != "" { + priv, err := crypto.HexToECDSA(strings.TrimPrefix(n.cfg.P2PNodeKeyHex, "0x")) + if err != nil { + return err + } + nodeCfg.P2P.PrivateKey = priv + } + if len(n.cfg.StaticPeers) > 0 { + nodes := make([]*enode.Node, 0, len(n.cfg.StaticPeers)) + for _, p := range n.cfg.StaticPeers { + nn, err := enode.Parse(enode.ValidSchemes, p) + if err != nil { + return err + } + nodes = append(nodes, nn) + } + nodeCfg.P2P.StaticNodes = nodes + } + if len(n.cfg.TrustedPeers) > 0 { + nodes := make([]*enode.Node, 0, len(n.cfg.TrustedPeers)) + for _, p := range n.cfg.TrustedPeers { + nn, err := enode.Parse(enode.ValidSchemes, p) + if err != nil { + return err + } + nodes = append(nodes, nn) + } + nodeCfg.P2P.TrustedNodes = nodes + } return nil }) require.NoError(err) @@ -173,6 +213,7 @@ func WithOpGeth(id stack.L2ELNodeID, opts ...L2ELOption) stack.Option[*Orchestra jwtPath: jwtPath, jwtSecret: jwtSecret, supervisorRPC: supervisorRPC, + cfg: cfg, } l2EL.Start() p.Cleanup(func() { diff --git a/op-devstack/sysgo/l2_el_opreth.go b/op-devstack/sysgo/l2_el_opreth.go index c625b7290aa..3377ba09209 100644 --- a/op-devstack/sysgo/l2_el_opreth.go +++ b/op-devstack/sysgo/l2_el_opreth.go @@ -25,7 +25,6 @@ type OpReth struct { mu sync.Mutex id stack.L2ELNodeID - l2Net *L2Network jwtPath string jwtSecret [32]byte authRPC string @@ -134,12 +133,12 @@ func (n *OpReth) Start() { metricsTargetChan <- NewPrometheusMetricsTarget(parsedUrl.Hostname(), parsedUrl.Port(), false) } } - stdOutLogs := logpipe.LogProcessor(func(line []byte) { + stdOutLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logOut(e) onLogEntry(e) }) - stdErrLogs := logpipe.LogProcessor(func(line []byte) { + stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logErr(e) }) @@ -280,7 +279,6 @@ func WithOpReth(id stack.L2ELNodeID, opts ...L2ELOption) stack.Option[*Orchestra l2EL := &OpReth{ id: id, - l2Net: l2Net, jwtPath: jwtPath, jwtSecret: jwtSecret, authRPC: "", diff --git a/op-devstack/sysgo/l2_metrics_dashboard.go b/op-devstack/sysgo/l2_metrics_dashboard.go index ba7bbc2bf6f..645c51f00f8 100644 --- a/op-devstack/sysgo/l2_metrics_dashboard.go +++ b/op-devstack/sysgo/l2_metrics_dashboard.go @@ -93,11 +93,11 @@ func (g *L2MetricsDashboard) startPrometheus() { logOut := logpipe.ToLogger(g.p.Logger().New("component", "prometheus", "src", "stdout")) logErr := logpipe.ToLogger(g.p.Logger().New("component", "prometheus", "src", "stderr")) - stdOutLogs := logpipe.LogProcessor(func(line []byte) { + stdOutLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logOut(e) }) - stdErrLogs := logpipe.LogProcessor(func(line []byte) { + stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logErr(e) }) @@ -116,11 +116,11 @@ func (g *L2MetricsDashboard) startGrafana() { logOut := logpipe.ToLogger(g.p.Logger().New("component", "grafana", "src", "stdout")) logErr := logpipe.ToLogger(g.p.Logger().New("component", "grafana", "src", "stderr")) - stdOutLogs := logpipe.LogProcessor(func(line []byte) { + stdOutLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logOut(e) }) - stdErrLogs := logpipe.LogProcessor(func(line []byte) { + stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logErr(e) }) diff --git a/op-devstack/sysgo/op_rbuilder.go b/op-devstack/sysgo/op_rbuilder.go new file mode 100644 index 00000000000..4bab3d0002e --- /dev/null +++ b/op-devstack/sysgo/op_rbuilder.go @@ -0,0 +1,470 @@ +// Moved from fb_OPRbuilderNode_real.go +package sysgo + +import ( + "encoding/hex" + "encoding/json" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/shim" + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-node/rollup" + "github.com/ethereum-optimism/optimism/op-service/client" + "github.com/ethereum-optimism/optimism/op-service/logpipe" + "github.com/ethereum-optimism/optimism/op-service/testutils/tcpproxy" +) + +type OPRBuilderNode struct { + mu sync.Mutex + + id stack.OPRBuilderNodeID + rollupCfg *rollup.Config + + wsProxyURL string + wsProxy *tcpproxy.Proxy + + rpcProxyURL string + rpcProxy *tcpproxy.Proxy + + authProxyURL string + authProxy *tcpproxy.Proxy + + logger log.Logger + p devtest.P + + sub *SubProcess + cfg *OPRBuilderNodeConfig //nolint:unused,structcheck // configuration retained for restarts and JWT lookups +} + +var _ hydrator = (*OPRBuilderNode)(nil) +var _ stack.Lifecycle = (*OPRBuilderNode)(nil) +var _ L2ELNode = (*OPRBuilderNode)(nil) + +// OPRBuilderNodeConfig contains configuration used to generate the op-OPRbuilderNode CLI. +// Callers can modify the defaults via OPRbuilderNodeOption functions. +type OPRBuilderNodeConfig struct { + // Chain selector (defaults to "dev" to avoid mainnet imports during tests) + Chain string + + // DataDir for op-OPRbuilderNode. If empty, a temp dir is created and cleaned up. + DataDir string + + // Logging formats + LogStdoutFormat string // e.g. "json" + LogFileFormat string // e.g. "json" + + // Flashblocks websocket bind address (host) + FlashblocksAddr string + // Flashblocks websocket port. 0 means auto-allocate an available local port. + FlashblocksPort int + // EnableFlashblocks enables the flashblocks feature. + EnableFlashblocks bool + + // --http + EnableRPC bool + RPCAPI string + RPCAddr string + RPCPort int + RPCJWTPath string + + AuthRPCJWTPath string + AuthRPCAddr string + AuthRPCPort int + + // P2P + P2PPort int + P2PAddr string + P2PNodeKeyHex string + StaticPeers []string + TrustedPeers []string + + // Misc process toggles + WithUnusedPorts bool // choose unused ports for subsystems + DisableDiscovery bool // avoid discv5 UDP socket collisions + + Full bool + + // ExtraArgs are appended to the generated CLI allowing callers to override defaults + // if the binary respects "last flag wins". + ExtraArgs []string + // Env is passed to the subprocess environment. + Env []string +} + +func DefaultOPRbuilderNodeConfig() *OPRBuilderNodeConfig { + return &OPRBuilderNodeConfig{ + EnableFlashblocks: true, + FlashblocksAddr: "127.0.0.1", + FlashblocksPort: 0, + EnableRPC: true, + RPCAPI: "admin,web3,debug,eth,txpool,net,miner", + RPCAddr: "127.0.0.1", + RPCPort: 0, + RPCJWTPath: "", + AuthRPCAddr: "127.0.0.1", + AuthRPCPort: 0, + AuthRPCJWTPath: "", + P2PAddr: "127.0.0.1", + P2PPort: 0, + P2PNodeKeyHex: "", + StaticPeers: nil, + TrustedPeers: nil, + Full: true, + LogStdoutFormat: "json", + LogFileFormat: "json", + Chain: "dev", + WithUnusedPorts: false, + DisableDiscovery: true, + DataDir: "", + ExtraArgs: nil, + Env: nil, + } +} + +func (cfg *OPRBuilderNodeConfig) LaunchSpec(p devtest.P) (args []string, env []string) { + p.Require().NotNil(cfg, "nil OPRbuilderNodeConfig") + + env = append([]string(nil), cfg.Env...) + args = make([]string, 0, len(cfg.ExtraArgs)+8) + + args = append(args, "node") + + if cfg.EnableFlashblocks { + if cfg.FlashblocksAddr == "" { + cfg.FlashblocksAddr = "127.0.0.1" + } + if cfg.FlashblocksPort <= 0 { + portStr, err := getAvailableLocalPort() + p.Require().NoError(err, "allocate flashblocks port") + portVal, err := strconv.Atoi(portStr) + p.Require().NoError(err, "parse flashblocks port") + cfg.FlashblocksPort = portVal + } + fbPortStr := strconv.Itoa(cfg.FlashblocksPort) + args = append(args, "--flashblocks.enabled") + args = append(args, "--flashblocks.addr="+cfg.FlashblocksAddr, "--flashblocks.port="+fbPortStr) + } + + // P2P configuration: enforce deterministic identity and static peering to the sequencer EL. + if cfg.P2PNodeKeyHex != "" { + key := strings.TrimPrefix(cfg.P2PNodeKeyHex, "0x") + _, err := hex.DecodeString(key) + p.Require().NoError(err, "decode p2p node key") + keyPath := filepath.Join(p.TempDir(), "oprbuilder-nodekey") + p.Require().NoError(os.WriteFile(keyPath, []byte(key), 0o600), "write p2p node key") + args = append(args, "--p2p-secret-key", keyPath) + } + if cfg.P2PAddr != "" { + args = append(args, "--addr", cfg.P2PAddr) + } + if len(cfg.StaticPeers) > 0 { + args = append(args, "--bootnodes", strings.Join(cfg.StaticPeers, ",")) + } + if len(cfg.TrustedPeers) > 0 { + args = append(args, "--trusted-peers", strings.Join(cfg.TrustedPeers, ",")) + } + + if cfg.EnableRPC { + args = append(args, "--http") + args = append(args, "--http.addr="+cfg.RPCAddr) + if cfg.RPCPort <= 0 { + portStr, err := getAvailableLocalPort() + p.Require().NoError(err, "allocate rpc port") + portVal, err := strconv.Atoi(portStr) + p.Require().NoError(err, "parse rpc port") + cfg.RPCPort = portVal + } + rpcPortStr := strconv.Itoa(cfg.RPCPort) + args = append(args, "--http.port="+rpcPortStr) + args = append(args, "--http.api="+cfg.RPCAPI) + + } + + if cfg.AuthRPCAddr != "" { + args = append(args, "--authrpc.addr="+cfg.AuthRPCAddr) + } + if cfg.AuthRPCPort <= 0 { + portStr, err := getAvailableLocalPort() + p.Require().NoError(err, "allocate auth rpc port") + portVal, err := strconv.Atoi(portStr) + p.Require().NoError(err, "parse auth rpc port") + cfg.AuthRPCPort = portVal + } + args = append(args, "--authrpc.port="+strconv.Itoa(cfg.AuthRPCPort)) + if cfg.AuthRPCJWTPath != "" { + args = append(args, "--authrpc.jwtsecret="+cfg.AuthRPCJWTPath) + } + + if cfg.Full { + args = append(args, "--full") + } + + if cfg.LogStdoutFormat != "" { + args = append(args, "--log.stdout.format="+cfg.LogStdoutFormat) + } + if cfg.LogFileFormat != "" { + args = append(args, "--log.file.format="+cfg.LogFileFormat) + } + if cfg.Chain != "" { + args = append(args, "--chain="+cfg.Chain) + } + if cfg.WithUnusedPorts { + args = append(args, "--with-unused-ports") + } + if cfg.DisableDiscovery { + args = append(args, "--disable-discovery") + } + + if !cfg.WithUnusedPorts { + if cfg.P2PPort <= 0 { + portStr, err := getAvailableLocalPort() + p.Require().NoError(err, "allocate p2p port") + portVal, err := strconv.Atoi(portStr) + p.Require().NoError(err, "parse p2p port") + cfg.P2PPort = portVal + } + args = append(args, "--port="+strconv.Itoa(cfg.P2PPort)) + } + + if cfg.DataDir == "" { + tmpDir, err := os.MkdirTemp("", "op-OPRBuilderNode-datadir-*") + p.Require().NoError(err, "create temp datadir for op-OPRBuilderNode") + args = append(args, "--datadir="+tmpDir) + p.Cleanup(func() { _ = os.RemoveAll(tmpDir) }) + } else { + args = append(args, "--datadir="+cfg.DataDir) + } + + args = append(args, cfg.ExtraArgs...) + + return args, env +} + +type OPRBuilderNodeOption interface { + Apply(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) +} + +type OPRBuilderNodeOptionFn func(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) + +var _ OPRBuilderNodeOption = OPRBuilderNodeOptionFn(nil) + +func (fn OPRBuilderNodeOptionFn) Apply(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) { + fn(p, id, cfg) +} + +// OPRBuilderNodeOptionBundle applies multiple OPRBuilderNodeOptions in order. +type OPRBuilderNodeOptionBundle []OPRBuilderNodeOption + +var _ OPRBuilderNodeOption = OPRBuilderNodeOptionBundle(nil) + +func (b OPRBuilderNodeOptionBundle) Apply(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) { + for _, opt := range b { + p.Require().NotNil(opt, "cannot Apply nil OPRBuilderNodeOption") + opt.Apply(p, id, cfg) + } +} + +// OPRBuilderWithP2PConfig sets deterministic P2P identity and static peers for the builder EL. +func OPRBuilderWithP2PConfig(addr string, port int, nodeKeyHex string, staticPeers, trustedPeers []string) OPRBuilderNodeOption { + return OPRBuilderNodeOptionFn(func(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) { + cfg.P2PAddr = addr + cfg.P2PPort = port + cfg.P2PNodeKeyHex = nodeKeyHex + cfg.StaticPeers = staticPeers + cfg.TrustedPeers = trustedPeers + }) +} + +// OPRBuilderWithNodeIdentity applies an ELNodeIdentity directly to the builder EL. +func OPRBuilderWithNodeIdentity(identity *ELNodeIdentity, addr string, staticPeers, trustedPeers []string) OPRBuilderNodeOption { + return OPRBuilderNodeOptionFn(func(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) { + cfg.P2PAddr = addr + cfg.P2PPort = identity.Port + cfg.P2PNodeKeyHex = identity.KeyHex() + cfg.StaticPeers = staticPeers + cfg.TrustedPeers = trustedPeers + }) +} + +func OPRBuilderNodeWithExtraArgs(args ...string) OPRBuilderNodeOption { + return OPRBuilderNodeOptionFn(func(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) { + cfg.ExtraArgs = append(cfg.ExtraArgs, args...) + }) +} + +func OPRBuilderNodeWithEnv(env ...string) OPRBuilderNodeOption { + return OPRBuilderNodeOptionFn(func(p devtest.P, id stack.OPRBuilderNodeID, cfg *OPRBuilderNodeConfig) { + cfg.Env = append(cfg.Env, env...) + }) +} + +func (b *OPRBuilderNode) hydrate(system stack.ExtensibleSystem) { + elRPC, err := client.NewRPC(system.T().Ctx(), system.Logger(), b.rpcProxyURL, client.WithLazyDial()) + system.T().Require().NoError(err) + system.T().Cleanup(elRPC.Close) + + node := shim.NewOPRBuilderNode(shim.OPRBuilderNodeConfig{ + ID: b.id, + ELNodeConfig: shim.ELNodeConfig{ + CommonConfig: shim.NewCommonConfig(system.T()), + Client: elRPC, + ChainID: b.id.ChainID(), + }, + RollupCfg: b.rollupCfg, + FlashblocksWsClient: shim.NewFlashblocksWSClient(shim.FlashblocksWSClientConfig{ + CommonConfig: shim.NewCommonConfig(system.T()), + ID: stack.NewFlashblocksWSClientID(b.id.Key(), b.id.ChainID()), + WsUrl: b.wsProxyURL, + WsHeaders: nil, + }), + }) + system.L2Network(stack.L2NetworkID(b.id.ChainID())).(stack.ExtensibleL2Network).AddOPRBuilderNode(node) +} + +func (b *OPRBuilderNode) Start() { + b.mu.Lock() + defer b.mu.Unlock() + if b.sub != nil { + b.logger.Warn("OPRbuilderNode already started") + return + } + cfg := b.cfg + b.p.Require().NotNil(cfg, "OPRbuilderNode config not initialized") + + if b.wsProxy == nil { + b.wsProxy = tcpproxy.New(b.p.Logger()) + b.p.Require().NoError(b.wsProxy.Start()) + b.wsProxyURL = "ws://" + b.wsProxy.Addr() + b.p.Cleanup(func() { b.wsProxy.Close() }) + } + + if b.rpcProxy == nil { + b.rpcProxy = tcpproxy.New(b.p.Logger()) + b.p.Require().NoError(b.rpcProxy.Start()) + b.rpcProxyURL = "http://" + b.rpcProxy.Addr() + b.p.Cleanup(func() { b.rpcProxy.Close() }) + } + + if cfg.EnableRPC && b.authProxy == nil { + b.authProxy = tcpproxy.New(b.p.Logger()) + b.p.Require().NoError(b.authProxy.Start()) + b.authProxyURL = "http://" + b.authProxy.Addr() + b.p.Cleanup(func() { b.authProxy.Close() }) + } + + args, env := cfg.LaunchSpec(b.p) + + // Forward structured logs to Go logger + logOut := logpipe.ToLogger(b.logger.New("component", "op-OPRbuilderNode", "src", "stdout")) + logErr := logpipe.ToLogger(b.logger.New("component", "op-OPRbuilderNode", "src", "stderr")) + + stdOut := logpipe.LogCallback(func(line []byte) { + logOut(logpipe.ParseRustStructuredLogs(line)) + }) + stdErr := logpipe.LogCallback(func(line []byte) { + logErr(logpipe.ParseRustStructuredLogs(line)) + }) + + b.sub = NewSubProcess(b.p, stdOut, stdErr) + + exec := os.Getenv("OP_RBUILDER_EXEC_PATH") + b.p.Require().NotEmpty(exec, "OP_RBUILDER_EXEC_PATH must be set") + + err := b.sub.Start(exec, args, env) + b.p.Require().NoError(err, "start OPRBuilderNode") + + const readinessTimeout = 15 * time.Second + + if cfg.EnableRPC { + rpcUpstreamHostport := net.JoinHostPort(cfg.RPCAddr, strconv.Itoa(cfg.RPCPort)) + rpcUpstreamURL := "http://" + rpcUpstreamHostport + waitTCPReady(b.p, rpcUpstreamURL, readinessTimeout) + b.logger.Info("OPRBuilderNode upstream RPC ready", "rpc", rpcUpstreamURL) + b.rpcProxy.SetUpstream(ProxyAddr(b.p.Require(), rpcUpstreamURL)) + waitTCPReady(b.p, b.rpcProxyURL, readinessTimeout) + b.logger.Info("OPRBuilderNode proxy RPC ready", "proxy_rpc", b.rpcProxyURL) + + authUpstreamHostport := net.JoinHostPort(cfg.RPCAddr, strconv.Itoa(cfg.AuthRPCPort)) + authUpstreamURL := "http://" + authUpstreamHostport + waitTCPReady(b.p, authUpstreamURL, readinessTimeout) + b.logger.Info("OPRBuilderNode upstream auth RPC ready", "auth_rpc", authUpstreamURL) + b.authProxy.SetUpstream(ProxyAddr(b.p.Require(), authUpstreamURL)) + waitTCPReady(b.p, b.authProxyURL, readinessTimeout) + b.logger.Info("OPRBuilderNode proxy auth RPC ready", "proxy_auth_rpc", b.authProxyURL) + } + + if cfg.EnableFlashblocks { + wsUpstreamHostport := net.JoinHostPort(cfg.FlashblocksAddr, strconv.Itoa(cfg.FlashblocksPort)) + wsUpstreamURL := "ws://" + wsUpstreamHostport + waitWSReady(b.p, wsUpstreamURL, readinessTimeout) + b.logger.Info("OPRBuilderNode upstream WS ready", "ws", wsUpstreamURL) + b.wsProxy.SetUpstream(ProxyAddr(b.p.Require(), wsUpstreamURL)) + waitWSReady(b.p, b.wsProxyURL, readinessTimeout) + b.logger.Info("OPRBuilderNode proxy WS ready", "proxy_ws", b.wsProxyURL) + } +} + +func (b *OPRBuilderNode) Stop() { + b.mu.Lock() + defer b.mu.Unlock() + if b.sub == nil { + b.logger.Warn("OPRbuilderNode already stopped") + return + } + b.p.Require().NoError(b.sub.Stop(true)) + b.sub = nil +} + +// WithOPRBuilderNode constructs and starts an OPRbuilderNode using the provided options. +func WithOPRBuilderNode(id stack.OPRBuilderNodeID, opts ...OPRBuilderNodeOption) stack.Option[*Orchestrator] { + return stack.AfterDeploy(func(orch *Orchestrator) { + p := orch.P().WithCtx(stack.ContextWithID(orch.P().Ctx(), id)) + l2Net, ok := orch.l2Nets.Get(id.ChainID()) + p.Require().True(ok, "l2 network required") + + tempDir := p.TempDir() + data, err := json.Marshal(l2Net.genesis) + p.Require().NoError(err, "must json-encode genesis") + chainConfigPath := filepath.Join(tempDir, "genesis.json") + p.Require().NoError(os.WriteFile(chainConfigPath, data, 0o644), "must write genesis file") + + // Build config from options + cfg := DefaultOPRbuilderNodeConfig() + cfg.AuthRPCJWTPath, _ = orch.writeDefaultJWT() + cfg.Chain = chainConfigPath + OPRBuilderNodeOptionBundle(opts).Apply(orch.P(), id, cfg) + + rb := &OPRBuilderNode{ + id: id, + logger: p.Logger(), + p: p, + rollupCfg: l2Net.rollupCfg, + cfg: cfg, + } + p.Logger().Info("Starting OPRbuilderNode") + rb.Start() + p.Cleanup(rb.Stop) + orch.oprbuilderNodes.Set(id, rb) + }) +} + +func (b *OPRBuilderNode) EngineRPC() string { + return b.authProxyURL +} + +func (b *OPRBuilderNode) JWTPath() string { + return b.cfg.AuthRPCJWTPath +} + +func (b *OPRBuilderNode) UserRPC() string { + return b.rpcProxyURL +} diff --git a/op-devstack/sysgo/orchestrator.go b/op-devstack/sysgo/orchestrator.go index 90e0b261ebf..574af454c9e 100644 --- a/op-devstack/sysgo/orchestrator.go +++ b/op-devstack/sysgo/orchestrator.go @@ -37,19 +37,21 @@ type Orchestrator struct { SyncTesterELOptions SyncTesterELOptionBundle deployerPipelineOptions []DeployerPipelineOption - superchains locks.RWMap[stack.SuperchainID, *Superchain] - clusters locks.RWMap[stack.ClusterID, *Cluster] - l1Nets locks.RWMap[eth.ChainID, *L1Network] - l2Nets locks.RWMap[eth.ChainID, *L2Network] - l1ELs locks.RWMap[stack.L1ELNodeID, L1ELNode] - l1CLs locks.RWMap[stack.L1CLNodeID, *L1CLNode] - l2ELs locks.RWMap[stack.L2ELNodeID, L2ELNode] - l2CLs locks.RWMap[stack.L2CLNodeID, L2CLNode] - supervisors locks.RWMap[stack.SupervisorID, Supervisor] - testSequencers locks.RWMap[stack.TestSequencerID, *TestSequencer] - batchers locks.RWMap[stack.L2BatcherID, *L2Batcher] - challengers locks.RWMap[stack.L2ChallengerID, *L2Challenger] - proposers locks.RWMap[stack.L2ProposerID, *L2Proposer] + superchains locks.RWMap[stack.SuperchainID, *Superchain] + clusters locks.RWMap[stack.ClusterID, *Cluster] + l1Nets locks.RWMap[eth.ChainID, *L1Network] + l2Nets locks.RWMap[eth.ChainID, *L2Network] + l1ELs locks.RWMap[stack.L1ELNodeID, L1ELNode] + l1CLs locks.RWMap[stack.L1CLNodeID, *L1CLNode] + l2ELs locks.RWMap[stack.L2ELNodeID, L2ELNode] + l2CLs locks.RWMap[stack.L2CLNodeID, L2CLNode] + supervisors locks.RWMap[stack.SupervisorID, Supervisor] + testSequencers locks.RWMap[stack.TestSequencerID, *TestSequencer] + batchers locks.RWMap[stack.L2BatcherID, *L2Batcher] + challengers locks.RWMap[stack.L2ChallengerID, *L2Challenger] + proposers locks.RWMap[stack.L2ProposerID, *L2Proposer] + rollupBoosts locks.RWMap[stack.RollupBoostNodeID, *RollupBoostNode] + oprbuilderNodes locks.RWMap[stack.OPRBuilderNodeID, *OPRBuilderNode] // service name => prometheus endpoints to scrape l2MetricsEndpoints locks.RWMap[string, []PrometheusMetricsTarget] @@ -91,6 +93,29 @@ func (o *Orchestrator) EnableTimeTravel() { } } +// GetL2EL attempts to find an L2 EL node by checking various collections of EL-like nodes. +// It returns the L2ELNode interface if found in the standard L2ELs collection, +// or the raw node object if found in other collections (e.g. RollupBoostNode). +func (o *Orchestrator) GetL2EL(id stack.L2ELNodeID) (L2ELNode, bool) { + if el, ok := o.l2ELs.Get(id); ok { + return el, true + } + + // Check RollupBoost + rbID := stack.NewRollupBoostNodeID(id.Key(), id.ChainID()) + if rb, ok := o.rollupBoosts.Get(rbID); ok { + return rb, true + } + + // Check op-rbuilder + oprbID := stack.NewOPRBuilderNodeID(id.Key(), id.ChainID()) + if oprbuilder, ok := o.oprbuilderNodes.Get(oprbID); ok { + return oprbuilder, true + } + + return nil, false +} + var _ stack.Orchestrator = (*Orchestrator)(nil) func NewOrchestrator(p devtest.P, hook stack.SystemHook) *Orchestrator { @@ -129,6 +154,8 @@ func (o *Orchestrator) Hydrate(sys stack.ExtensibleSystem) { o.l1ELs.Range(rangeHydrateFn[stack.L1ELNodeID, L1ELNode](sys)) o.l1CLs.Range(rangeHydrateFn[stack.L1CLNodeID, *L1CLNode](sys)) o.l2ELs.Range(rangeHydrateFn[stack.L2ELNodeID, L2ELNode](sys)) + o.oprbuilderNodes.Range(rangeHydrateFn[stack.OPRBuilderNodeID, *OPRBuilderNode](sys)) + o.rollupBoosts.Range(rangeHydrateFn[stack.RollupBoostNodeID, *RollupBoostNode](sys)) o.l2CLs.Range(rangeHydrateFn[stack.L2CLNodeID, L2CLNode](sys)) o.supervisors.Range(rangeHydrateFn[stack.SupervisorID, Supervisor](sys)) o.testSequencers.Range(rangeHydrateFn[stack.TestSequencerID, *TestSequencer](sys)) diff --git a/op-devstack/sysgo/rollup_boost.go b/op-devstack/sysgo/rollup_boost.go new file mode 100644 index 00000000000..90258e40a21 --- /dev/null +++ b/op-devstack/sysgo/rollup_boost.go @@ -0,0 +1,406 @@ +package sysgo + +import ( + "net" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/shim" + "github.com/ethereum-optimism/optimism/op-devstack/stack" + "github.com/ethereum-optimism/optimism/op-service/client" + "github.com/ethereum-optimism/optimism/op-service/logpipe" + "github.com/ethereum-optimism/optimism/op-service/testutils/tcpproxy" +) + +// RollupBoostNode is a lightweight sysgo-managed process wrapper around a rollup-boost +// WebSocket stream source. It exposes a stable proxied ws URL and hydrates the L2 +// network with a FlashblocksWSClient shim that points at it. +type RollupBoostNode struct { + mu sync.Mutex + + id stack.RollupBoostNodeID + wsProxyURL string + wsProxy *tcpproxy.Proxy + + rpcProxyURL string + rpcProxy *tcpproxy.Proxy + + header http.Header + + logger log.Logger + p devtest.P + + sub *SubProcess + + cfg *RollupBoostConfig +} + +var _ hydrator = (*RollupBoostNode)(nil) +var _ stack.Lifecycle = (*RollupBoostNode)(nil) +var _ L2ELNode = (*RollupBoostNode)(nil) + +func (r *RollupBoostNode) hydrate(system stack.ExtensibleSystem) { + elRPC, err := client.NewRPC(system.T().Ctx(), system.Logger(), r.rpcProxyURL, client.WithLazyDial()) + system.T().Require().NoError(err) + system.T().Cleanup(elRPC.Close) + + node := shim.NewRollupBoostNode(shim.RollupBoostNodeConfig{ + ID: r.id, + ELNodeConfig: shim.ELNodeConfig{ + CommonConfig: shim.NewCommonConfig(system.T()), + Client: elRPC, + ChainID: r.id.ChainID(), + }, + RollupCfg: system.L2Network(stack.L2NetworkID(r.id.ChainID())).RollupConfig(), + FlashblocksWsClient: shim.NewFlashblocksWSClient(shim.FlashblocksWSClientConfig{ + CommonConfig: shim.NewCommonConfig(system.T()), + ID: stack.NewFlashblocksWSClientID(r.id.Key(), r.id.ChainID()), + WsUrl: r.wsProxyURL, + WsHeaders: r.header, + }), + }) + system.L2Network(stack.L2NetworkID(r.id.ChainID())).(stack.ExtensibleL2Network).AddRollupBoostNode(node) +} + +func (r *RollupBoostNode) Start() { + r.mu.Lock() + defer r.mu.Unlock() + if r.sub != nil { + r.logger.Warn("rollup-boost already started") + return + } + + cfg := r.cfg + r.p.Require().NotNil(cfg, "rollup-boost config not initialized") + + args, env := cfg.LaunchSpec(r.p) + + if r.wsProxy == nil { + r.wsProxy = tcpproxy.New(r.p.Logger()) + r.p.Require().NoError(r.wsProxy.Start()) + r.wsProxyURL = "ws://" + r.wsProxy.Addr() + r.p.Cleanup(func() { r.wsProxy.Close() }) + } + + if r.rpcProxy == nil { + r.rpcProxy = tcpproxy.New(r.p.Logger()) + r.p.Require().NoError(r.rpcProxy.Start()) + r.rpcProxyURL = "http://" + r.rpcProxy.Addr() + r.p.Cleanup(func() { r.rpcProxy.Close() }) + } + + // Parse Rust-structured logs and forward into Go logger with attributes + logOut := logpipe.ToLogger(r.logger.New("stream", "stdout")) + logErr := logpipe.ToLogger(r.logger.New("stream", "stderr")) + + stdOut := logpipe.LogCallback(func(line []byte) { + logOut(logpipe.ParseRustStructuredLogs(line)) + }) + stdErr := logpipe.LogCallback(func(line []byte) { + logErr(logpipe.ParseRustStructuredLogs(line)) + }) + + r.sub = NewSubProcess(r.p, stdOut, stdErr) + + exec := os.Getenv("ROLLUP_BOOST_EXEC_PATH") + r.p.Require().NotEmpty(exec, "ROLLUP_BOOST_EXEC_PATH must be set") + + err := r.sub.Start(exec, args, env) + r.p.Require().NoError(err, "start rollup-boost") + + rpcUpstreamURL := "http://" + cfg.RPCHost + ":" + strconv.Itoa(int(cfg.RPCPort)) + waitTCPReady(r.p, rpcUpstreamURL, 5*time.Second) + r.logger.Info("rollup-boost upstream RPC ready", "rpc", rpcUpstreamURL) + r.rpcProxy.SetUpstream(ProxyAddr(r.p.Require(), rpcUpstreamURL)) + waitTCPReady(r.p, r.rpcProxyURL, 10*time.Second) + r.logger.Info("rollup-boost proxy RPC ready", "proxy_rpc", r.rpcProxyURL) + + // WS: wait for upstream first, then configure and test proxy + if cfg.EnableFlashblocks { + wsUpstreamHostport := net.JoinHostPort(cfg.FlashblocksHost, strconv.Itoa(cfg.FlashblocksPort)) + wsUpstreamURL := "ws://" + wsUpstreamHostport + + // Wait for upstream WS TCP endpoint + waitTCPReady(r.p, wsUpstreamURL, 5*time.Second) + r.logger.Info("rollup-boost upstream WS ready", "upstream_ws", wsUpstreamURL) + + r.wsProxy.SetUpstream(ProxyAddr(r.p.Require(), wsUpstreamURL)) + waitWSReady(r.p, r.wsProxyURL, 10*time.Second) + r.logger.Info("rollup-boost proxy WS ready", "proxy_ws", r.wsProxyURL) + } +} + +func (r *RollupBoostNode) Stop() { + r.mu.Lock() + defer r.mu.Unlock() + if r.sub == nil { + r.logger.Warn("rollup-boost already stopped") + return + } + r.p.Require().NoError(r.sub.Stop(true)) + r.sub = nil +} + +// WithRollupBoost starts a rollup-boost process using the provided options +// and registers a FlashblocksWSClient on the target L2 chain. +// l2ELID is required to link the proxy to the L2 EL it serves. +func WithRollupBoost(id stack.RollupBoostNodeID, l2ELID stack.L2ELNodeID, opts ...RollupBoostOption) stack.Option[*Orchestrator] { + return stack.AfterDeploy(func(orch *Orchestrator) { + p := orch.P().WithCtx(stack.ContextWithID(orch.P().Ctx(), id)) + logger := p.Logger() + + // Build config from options and derive sensible defaults + cfg := DefaultRollupBoostConfig() + RollupBoostOptionBundle(opts).Apply(orch, id, cfg) + // Source L2 engine/JWT from the L2 EL object (mandatory) + if l2EL, ok := orch.l2ELs.Get(l2ELID); ok { + engineRPC := l2EL.EngineRPC() + switch { + case strings.HasPrefix(engineRPC, "ws://"): + engineRPC = "http://" + strings.TrimPrefix(engineRPC, "ws://") + case strings.HasPrefix(engineRPC, "wss://"): + engineRPC = "https://" + strings.TrimPrefix(engineRPC, "wss://") + } + cfg.L2EngineURL = engineRPC + cfg.L2JWTPath = l2EL.JWTPath() + } + // Normalize builder URL and fallback JWT will be handled after builder link options are applied below. + + r := &RollupBoostNode{ + id: id, + logger: logger, + p: p, + cfg: cfg, + header: cfg.Headers, + } + // Apply any node-level link options + for _, opt := range opts { + if linkOpt, ok := opt.(interface { + applyNode(p devtest.P, id stack.RollupBoostNodeID, r *RollupBoostNode) + }); ok { + linkOpt.applyNode(p, id, r) + } + } + logger.Info("Starting rollup-boost") + r.Start() + p.Cleanup(r.Stop) + // Register for hydration + orch.rollupBoosts.Set(id, r) + }) +} + +// RollupBoostConfig configures the rollup-boost process CLI and environment. +type RollupBoostConfig struct { + // RPC endpoint for rollup-boost itself + RPCHost string + RPCPort uint16 + + // Flashblocks proxy WebSocket exposure + EnableFlashblocks bool + FlashblocksHost string + FlashblocksPort int + + // L2 engine connection details (HTTP(S)) + L2EngineURL string + L2JWTPath string + + // Builder engine connection details (HTTP(S)) + BuilderURL string + BuilderJWTPath string + FlashblocksBuilderURL string // upstream builder WS url (e.g. op-rbuilder ws) + + // Other settings + ExecutionMode string // e.g. "enabled" + LogFormat string // e.g. "json" + + // Debug server + DebugHost string + DebugPort int + + // Optional WS headers to expose to clients through the proxy + Headers http.Header + + // Env variables for the subprocess + Env []string + // ExtraArgs appended to the generated CLI (last-flag-wins semantics) + ExtraArgs []string +} + +func DefaultRollupBoostConfig() *RollupBoostConfig { + return &RollupBoostConfig{ + RPCHost: "127.0.0.1", + RPCPort: 0, + EnableFlashblocks: true, + FlashblocksHost: "127.0.0.1", + FlashblocksPort: 0, + FlashblocksBuilderURL: "", + L2EngineURL: "", + L2JWTPath: "", + BuilderURL: "127.0.0.1:8551", // normalized to http:// later + BuilderJWTPath: "", + ExecutionMode: "enabled", + LogFormat: "json", + DebugHost: "127.0.0.1", + DebugPort: 0, + Headers: http.Header{}, + Env: nil, + ExtraArgs: nil, + } +} + +func (cfg *RollupBoostConfig) LaunchSpec(p devtest.P) (args []string, env []string) { + p.Require().NotNil(cfg, "nil RollupBoostConfig") + + env = append([]string(nil), cfg.Env...) + args = make([]string, 0, len(cfg.ExtraArgs)+16) + + if cfg.EnableFlashblocks { + if cfg.FlashblocksHost == "" { + cfg.FlashblocksHost = "127.0.0.1" + } + if cfg.FlashblocksPort <= 0 { + portStr, err := getAvailableLocalPort() + p.Require().NoError(err, "allocate flashblocks port") + portVal, err := strconv.Atoi(portStr) + p.Require().NoError(err, "parse flashblocks port") + cfg.FlashblocksPort = portVal + } + fbPortStr := strconv.Itoa(cfg.FlashblocksPort) + args = append(args, "--flashblocks", "--flashblocks-host="+cfg.FlashblocksHost, "--flashblocks-port="+fbPortStr) + if cfg.FlashblocksBuilderURL != "" { + args = append(args, "--flashblocks-builder-url="+cfg.FlashblocksBuilderURL) + } + } + + if cfg.RPCPort <= 0 { + portStr, err := getAvailableLocalPort() + p.Require().NoError(err, "allocate rollup-boost rpc port") + portVal, err := strconv.ParseUint(portStr, 10, 16) + p.Require().NoError(err, "parse rollup-boost rpc port") + cfg.RPCPort = uint16(portVal) + } + p.Require().True(cfg.RPCPort > 0, "RPCPort must be > 0") + args = append(args, "--rpc-host="+cfg.RPCHost, "--rpc-port="+strconv.Itoa(int(cfg.RPCPort))) + + if cfg.L2EngineURL != "" { + args = append(args, "--l2-url="+ensureHTTPURL(cfg.L2EngineURL)) + } + if cfg.L2JWTPath != "" { + args = append(args, "--l2-jwt-path="+cfg.L2JWTPath) + } + if cfg.BuilderURL != "" { + args = append(args, "--builder-url="+ensureHTTPURL(cfg.BuilderURL)) + } + if cfg.BuilderJWTPath != "" { + args = append(args, "--builder-jwt-path="+cfg.BuilderJWTPath) + } + + if cfg.ExecutionMode != "" { + args = append(args, "--execution-mode="+cfg.ExecutionMode) + } + if cfg.LogFormat != "" { + args = append(args, "--log-format="+cfg.LogFormat) + } + + if cfg.DebugHost == "" { + cfg.DebugHost = "127.0.0.1" + } + if cfg.DebugPort <= 0 { + portStr, err := getAvailableLocalPort() + p.Require().NoError(err, "allocate rollup-boost debug port") + portVal, err := strconv.Atoi(portStr) + p.Require().NoError(err, "parse rollup-boost debug port") + cfg.DebugPort = portVal + } + args = append(args, "--debug-host="+cfg.DebugHost, "--debug-server-port="+strconv.Itoa(cfg.DebugPort)) + + args = append(args, cfg.ExtraArgs...) + + return args, env +} + +type RollupBoostOption interface { + Apply(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) +} + +type RollupBoostOptionFn func(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) + +var _ RollupBoostOption = RollupBoostOptionFn(nil) + +func (fn RollupBoostOptionFn) Apply(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) { + fn(orch, id, cfg) +} + +type RollupBoostOptionBundle []RollupBoostOption + +var _ RollupBoostOption = RollupBoostOptionBundle(nil) + +func (b RollupBoostOptionBundle) Apply(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) { + for _, opt := range b { + orch.P().Require().NotNil(opt, "cannot Apply nil RollupBoostOption") + opt.Apply(orch, id, cfg) + } +} + +// Convenience options +func RollupBoostWithExecutionMode(mode string) RollupBoostOption { + return RollupBoostOptionFn(func(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) { + cfg.ExecutionMode = mode + }) +} + +func RollupBoostWithEnv(env ...string) RollupBoostOption { + return RollupBoostOptionFn(func(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) { + cfg.Env = append(cfg.Env, env...) + }) +} + +func RollupBoostWithExtraArgs(args ...string) RollupBoostOption { + return RollupBoostOptionFn(func(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) { + cfg.ExtraArgs = append(cfg.ExtraArgs, args...) + }) +} + +func RollupBoostWithBuilderNode(id stack.OPRBuilderNodeID) RollupBoostOption { + return RollupBoostOptionFn(func(orch *Orchestrator, rbID stack.RollupBoostNodeID, cfg *RollupBoostConfig) { + builderNode, ok := orch.oprbuilderNodes.Get(id) + if !ok { + orch.P().Require().FailNow("builder node not found") + } + cfg.BuilderURL = ensureHTTPURL(builderNode.authProxyURL) + cfg.BuilderJWTPath = builderNode.cfg.AuthRPCJWTPath + cfg.FlashblocksBuilderURL = builderNode.wsProxyURL + }) +} + +func RollupBoostWithFlashblocksDisabled() RollupBoostOption { + return RollupBoostOptionFn(func(orch *Orchestrator, id stack.RollupBoostNodeID, cfg *RollupBoostConfig) { + cfg.EnableFlashblocks = false + }) +} + +func ensureHTTPURL(u string) string { + if strings.Contains(u, "://") { + return u + } + return "http://" + u +} + +func (r *RollupBoostNode) EngineRPC() string { + return r.rpcProxyURL +} + +func (r *RollupBoostNode) JWTPath() string { + return r.cfg.L2JWTPath +} + +func (r *RollupBoostNode) UserRPC() string { + return r.rpcProxyURL +} diff --git a/op-devstack/sysgo/subproc.go b/op-devstack/sysgo/subproc.go index d2f5cb202ae..cb60e013444 100644 --- a/op-devstack/sysgo/subproc.go +++ b/op-devstack/sysgo/subproc.go @@ -15,17 +15,20 @@ type SubProcess struct { p devtest.P cmd *exec.Cmd - stdOutLogs logpipe.LogProcessor - stdErrLogs logpipe.LogProcessor + stdOutCallback logpipe.LogCallback + stdErrCallback logpipe.LogCallback + + stdOutProc *logpipe.LineBuffer + stdErrProc *logpipe.LineBuffer mu sync.Mutex } -func NewSubProcess(p devtest.P, stdOutLogs, stdErrLogs logpipe.LogProcessor) *SubProcess { +func NewSubProcess(p devtest.P, stdOutCallback, stdErrCallback logpipe.LogCallback) *SubProcess { return &SubProcess{ - p: p, - stdOutLogs: stdOutLogs, - stdErrLogs: stdErrLogs, + p: p, + stdOutCallback: stdOutCallback, + stdErrCallback: stdErrCallback, } } @@ -35,14 +38,21 @@ func (sp *SubProcess) Start(cmdPath string, args []string, env []string) error { if sp.cmd != nil { return fmt.Errorf("process is still running (PID: %d)", sp.cmd.Process.Pid) } + sp.p.Logger().Info("Starting subprocess", "cmd", cmdPath, "args", args) + + stdOutProc := logpipe.NewLineBuffer(sp.stdOutCallback) + stdErrProc := logpipe.NewLineBuffer(sp.stdErrCallback) + cmd := exec.Command(cmdPath, args...) cmd.Env = append(os.Environ(), env...) - cmd.Stdout = sp.stdOutLogs - cmd.Stderr = sp.stdErrLogs + cmd.Stdout = stdOutProc + cmd.Stderr = stdErrProc if err := cmd.Start(); err != nil { return err } sp.cmd = cmd + sp.stdOutProc = stdOutProc + sp.stdErrProc = stdErrProc sp.p.Cleanup(func() { err := sp.Stop(true) if err != nil { @@ -75,6 +85,14 @@ func (sp *SubProcess) Stop(interrupt bool) error { sp.p.Logger().Info("Sub-process gracefully exited") } + if sp.stdOutProc != nil { + _ = sp.stdOutProc.Close() + sp.stdOutProc = nil + } + if sp.stdErrProc != nil { + _ = sp.stdErrProc.Close() + sp.stdErrProc = nil + } sp.cmd = nil return nil } diff --git a/op-devstack/sysgo/subproc_test.go b/op-devstack/sysgo/subproc_test.go index e44e039aa9c..8e5a3c37fdf 100644 --- a/op-devstack/sysgo/subproc_test.go +++ b/op-devstack/sysgo/subproc_test.go @@ -26,11 +26,11 @@ func TestSubProcess(gt *testing.T) { p := devtest.NewP(context.Background(), logger, onFailNow, onSkipNow) gt.Cleanup(p.Close) - logProc := logpipe.LogProcessor(func(line []byte) { + logCallback := logpipe.LogCallback(func(line []byte) { logger.Info(string(line)) tLog.Info("Sub-process logged message", "line", string(line)) }) - sp := NewSubProcess(p, logProc, logProc) + sp := NewSubProcess(p, logCallback, logCallback) gt.Log("Running first sub-process") testSleep(gt, capt, sp) diff --git a/op-devstack/sysgo/supervisor_kona.go b/op-devstack/sysgo/supervisor_kona.go index b4473d9e73d..8d4b925b570 100644 --- a/op-devstack/sysgo/supervisor_kona.go +++ b/op-devstack/sysgo/supervisor_kona.go @@ -82,12 +82,12 @@ func (s *KonaSupervisor) Start() { userRPC <- "http://" + e.FieldValue("addr").(string) } } - stdOutLogs := logpipe.LogProcessor(func(line []byte) { + stdOutLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logOut(e) onLogEntry(e) }) - stdErrLogs := logpipe.LogProcessor(func(line []byte) { + stdErrLogs := logpipe.LogCallback(func(line []byte) { e := logpipe.ParseRustStructuredLogs(line) logErr(e) }) diff --git a/op-devstack/sysgo/system.go b/op-devstack/sysgo/system.go index 1fa516fe381..ef5e17a8a91 100644 --- a/op-devstack/sysgo/system.go +++ b/op-devstack/sysgo/system.go @@ -1,6 +1,9 @@ package sysgo import ( + "os" + "strings" + "github.com/ethereum-optimism/optimism/op-chain-ops/devkeys" "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer" "github.com/ethereum-optimism/optimism/op-devstack/stack" @@ -574,3 +577,108 @@ func ProofSystem(dest *DefaultMinimalSystemIDs) stack.Option[*Orchestrator] { opt.Add(WithCannonGameTypeAdded(ids.L1EL, ids.L2.ChainID())) return opt } + +type SingleChainSystemWithFlashblocksIDs struct { + L1 stack.L1NetworkID + L1EL stack.L1ELNodeID + L1CL stack.L1CLNodeID + + L2 stack.L2NetworkID + L2CL stack.L2CLNodeID + L2EL stack.L2ELNodeID + L2Builder stack.OPRBuilderNodeID + L2RollupBoost stack.RollupBoostNodeID + + L2Batcher stack.L2BatcherID + L2Proposer stack.L2ProposerID + L2Challenger stack.L2ChallengerID + + TestSequencer stack.TestSequencerID +} + +func NewDefaultSingleChainSystemWithFlashblocksIDs(l1ID, l2ID eth.ChainID) SingleChainSystemWithFlashblocksIDs { + ids := SingleChainSystemWithFlashblocksIDs{ + L1: stack.L1NetworkID(l1ID), + L1EL: stack.NewL1ELNodeID("l1", l1ID), + L1CL: stack.NewL1CLNodeID("l1", l1ID), + L2: stack.L2NetworkID(l2ID), + L2CL: stack.NewL2CLNodeID("sequencer", l2ID), + L2EL: stack.NewL2ELNodeID("sequencer", l2ID), + L2Builder: stack.NewOPRBuilderNodeID("sequencer", l2ID), + L2RollupBoost: stack.NewRollupBoostNodeID("rollup-boost", l2ID), + L2Batcher: stack.NewL2BatcherID("main", l2ID), + L2Proposer: stack.NewL2ProposerID("main", l2ID), + L2Challenger: stack.NewL2ChallengerID("main", l2ID), + TestSequencer: "test-sequencer", + } + return ids +} + +func DefaultSingleChainSystemWithFlashblocks(dest *SingleChainSystemWithFlashblocksIDs) stack.Option[*Orchestrator] { + ids := NewDefaultSingleChainSystemWithFlashblocksIDs(DefaultL1ID, DefaultL2AID) + return singleChainSystemWithFlashblocksOpts(&ids, dest) +} + +func singleChainSystemWithFlashblocksOpts(ids *SingleChainSystemWithFlashblocksIDs, dest *SingleChainSystemWithFlashblocksIDs) stack.CombinedOption[*Orchestrator] { + opt := stack.Combine[*Orchestrator]() + // Precompute deterministic P2P identity and peering between sequencer EL and op-rbuilder EL. + seqID := NewELNodeIdentity("127.0.0.1", 0) + builderID := NewELNodeIdentity("127.0.0.1", 30303) // use default reth p2p port + + var missingEnv []string + if os.Getenv("OP_RBUILDER_EXEC_PATH") == "" { + missingEnv = append(missingEnv, "OP_RBUILDER_EXEC_PATH") + } + if os.Getenv("ROLLUP_BOOST_EXEC_PATH") == "" { + missingEnv = append(missingEnv, "ROLLUP_BOOST_EXEC_PATH") + } + if len(missingEnv) > 0 { + missing := strings.Join(missingEnv, ", ") + opt.Add(stack.BeforeDeploy(func(o *Orchestrator) { + o.P().Logger().Warn("Skipping single-chain flashblocks system; missing executables", "missing_env", missing) + o.P().SkipNow() + })) + return opt + } + + opt.Add(stack.BeforeDeploy(func(o *Orchestrator) { + o.P().Logger().Info("Setting up") + })) + + opt.Add(WithMnemonicKeys(devkeys.TestMnemonic)) + + opt.Add(WithDeployer(), + WithDeployerOptions( + WithLocalContractSources(), + WithCommons(ids.L1.ChainID()), + WithPrefundedL2(ids.L1.ChainID(), ids.L2.ChainID()), + ), + ) + + opt.Add(WithL1Nodes(ids.L1EL, ids.L1CL)) + + opt.Add(WithL2ELNode(ids.L2EL, L2ELWithP2PConfig("127.0.0.1", seqID.Port, seqID.KeyHex(), []string{builderID.Enode}, nil))) + opt.Add(WithOPRBuilderNode(ids.L2Builder, OPRBuilderWithNodeIdentity(builderID, "127.0.0.1", []string{seqID.Enode}, []string{seqID.Enode}))) + opt.Add(WithRollupBoost(ids.L2RollupBoost, ids.L2EL, RollupBoostWithBuilderNode(ids.L2Builder))) + + opt.Add(WithL2CLNode(ids.L2CL, ids.L1CL, ids.L1EL, stack.L2ELNodeID(ids.L2RollupBoost), L2CLSequencer())) + + opt.Add(WithBatcher(ids.L2Batcher, ids.L1EL, ids.L2CL, ids.L2EL)) + opt.Add(WithProposer(ids.L2Proposer, ids.L1EL, &ids.L2CL, nil)) + + opt.Add(WithFaucets([]stack.L1ELNodeID{ids.L1EL}, []stack.L2ELNodeID{ids.L2EL})) + + opt.Add(WithTestSequencer(ids.TestSequencer, ids.L1CL, ids.L2CL, ids.L1EL, ids.L2EL)) + + opt.Add(WithL2Challenger(ids.L2Challenger, ids.L1EL, ids.L1CL, nil, nil, &ids.L2CL, []stack.L2ELNodeID{ + ids.L2EL, + })) + + opt.Add(WithL2MetricsDashboard()) + + opt.Add(stack.Finally(func(orch *Orchestrator) { + *dest = *ids + })) + + return opt +} diff --git a/op-devstack/sysgo/test_sequencer.go b/op-devstack/sysgo/test_sequencer.go index 22f9cf011cc..9937dc8cb56 100644 --- a/op-devstack/sysgo/test_sequencer.go +++ b/op-devstack/sysgo/test_sequencer.go @@ -210,6 +210,8 @@ func WithTestSequencer(testSequencerID stack.TestSequencerID, l1CLID stack.L1CLN }, } + logger.Info("Configuring test sequencer", "l1EL", l1EL.UserRPC(), "l2EL", l2EL.UserRPC(), "l2CL", l2CL.UserRPC()) + jobs := work.NewJobRegistry() ensemble, err := v.Start(context.Background(), &work.StartOpts{ Log: logger, diff --git a/op-devstack/sysgo/util.go b/op-devstack/sysgo/util.go index 33ac3aa600f..68fb9d0eedd 100644 --- a/op-devstack/sysgo/util.go +++ b/op-devstack/sysgo/util.go @@ -1,11 +1,19 @@ package sysgo import ( + "context" "errors" "fmt" "net" + "net/url" "os" + "strconv" "sync" + "time" + + "github.com/coder/websocket" + "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/stretchr/testify/assert" ) // getEnvVarOrDefault returns the value of the provided env var or the provided default value if unset. @@ -28,8 +36,6 @@ func propagateEnvVarOrDefault(envVarName string, defaultValue string) string { } } -// NB: arbitrary start port with a low probability of conflict -var availableLocalPortStart = 20_000 var availableLocalPortMutex sync.Mutex // getAvailableLocalPort searches for and returns a currently unused local port. @@ -38,15 +44,49 @@ func getAvailableLocalPort() (string, error) { availableLocalPortMutex.Lock() defer availableLocalPortMutex.Unlock() - for port := availableLocalPortStart; port < 65_535; port++ { - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) - if err != nil { - continue - } - _ = ln.Close() - availableLocalPortStart = port + 1 - return fmt.Sprintf("%d", port), nil + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", fmt.Errorf("could not listen on ephemeral port: %w", err) } + defer ln.Close() + + addr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + return "", errors.New("listener did not return a TCP addr") + } + return strconv.Itoa(addr.Port), nil +} - return "", errors.New("could not find open port") +// waitTCPReady parses a URL and waits for its TCP endpoint to become ready using EventuallyWithT. +func waitTCPReady(p devtest.P, rawURL string, timeout time.Duration) { + p.Helper() + u, err := url.Parse(rawURL) + p.Require().NoError(err, "parse URL: %s", rawURL) + p.Require().NotEmpty(u.Host, "URL has no host: %s", rawURL) + waitMsg := fmt.Sprintf("TCP endpoint %s not ready within %v", u.Host, timeout) + p.Require().EventuallyWithT(func(c *assert.CollectT) { + conn, err := net.DialTimeout("tcp", u.Host, 300*time.Millisecond) + if err == nil { + _ = conn.Close() + } + assert.NoError(c, err, "TCP connection to %s should succeed", u.Host) + }, timeout, 100*time.Millisecond, waitMsg) +} + +// waitWSReady attempts an actual WebSocket handshake to confirm readiness using EventuallyWithT. +func waitWSReady(p devtest.P, rawURL string, timeout time.Duration) { + p.Helper() + waitWSMsg := fmt.Sprintf("WebSocket endpoint %s not ready within %v", rawURL, timeout) + p.Require().EventuallyWithT(func(c *assert.CollectT) { + ctx, cancel := context.WithTimeout(context.Background(), 750*time.Millisecond) + conn, resp, err := websocket.Dial(ctx, rawURL, nil) + cancel() + if resp != nil && resp.Body != nil { + _ = resp.Body.Close() + } + if conn != nil { + _ = conn.Close(websocket.StatusNormalClosure, "") + } + assert.NoError(c, err, "WebSocket handshake to %s should succeed", rawURL) + }, timeout, 100*time.Millisecond, waitWSMsg) } diff --git a/op-service/logpipe/line_buffer.go b/op-service/logpipe/line_buffer.go new file mode 100644 index 00000000000..0a6395822e9 --- /dev/null +++ b/op-service/logpipe/line_buffer.go @@ -0,0 +1,62 @@ +package logpipe + +import ( + "io" + "sync" +) + +// LogCallback is the function signature for processing a complete log line. +type LogCallback func(line []byte) + +// LineBuffer is an io.WriteCloser that buffers data across writes and emits complete lines +// to the provided callback, preserving log entries that may span multiple writes. +// Any trailing partial line is flushed when Close is called. +type LineBuffer struct { + mu sync.Mutex + buf []byte + callback LogCallback +} + +// NewLineBuffer creates a new LineBuffer that calls the given callback for each complete line. +func NewLineBuffer(callback LogCallback) *LineBuffer { + return &LineBuffer{callback: callback} +} + +// Write appends data, emitting full lines to the callback. +// Empty lines are ignored. +func (lp *LineBuffer) Write(p []byte) (int, error) { + lp.mu.Lock() + defer lp.mu.Unlock() + + lp.buf = append(lp.buf, p...) + + start := 0 + for i := 0; i < len(lp.buf); i++ { + if lp.buf[i] == '\n' { + line := lp.buf[start:i] + if len(line) > 0 { + lp.callback(line) + } + start = i + 1 + } + } + // Keep any partial trailing line + if start > 0 { + lp.buf = append([]byte(nil), lp.buf[start:]...) + } + return len(p), nil +} + +// Close flushes any buffered partial line. +func (lp *LineBuffer) Close() error { + lp.mu.Lock() + defer lp.mu.Unlock() + + if len(lp.buf) > 0 { + lp.callback(lp.buf) + lp.buf = nil + } + return nil +} + +var _ io.WriteCloser = (*LineBuffer)(nil) diff --git a/op-service/logpipe/line_buffer_test.go b/op-service/logpipe/line_buffer_test.go new file mode 100644 index 00000000000..b28dad1f90d --- /dev/null +++ b/op-service/logpipe/line_buffer_test.go @@ -0,0 +1,45 @@ +package logpipe + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLogProcessorSplitsAcrossWrites(t *testing.T) { + var lines [][]byte + proc := NewLineBuffer(func(line []byte) { + // Copy to avoid aliasing the buffer + lines = append(lines, append([]byte(nil), line...)) + }) + + _, err := proc.Write([]byte("hello ")) + require.NoError(t, err) + _, err = proc.Write([]byte("world\nsecond")) + require.NoError(t, err) + _, err = proc.Write([]byte(" line\nthird line\n")) + require.NoError(t, err) + + require.NoError(t, proc.Close()) + + require.Equal(t, [][]byte{ + []byte("hello world"), + []byte("second line"), + []byte("third line"), + }, lines) +} + +func TestLogProcessorFlushesTrailingPartialLine(t *testing.T) { + var lines [][]byte + proc := NewLineBuffer(func(line []byte) { + lines = append(lines, append([]byte(nil), line...)) + }) + + _, err := proc.Write([]byte("partial line without newline")) + require.NoError(t, err) + require.NoError(t, proc.Close()) + + require.Equal(t, [][]byte{ + []byte("partial line without newline"), + }, lines) +} diff --git a/op-service/logpipe/pipe.go b/op-service/logpipe/pipe.go index 1ee6eeb2520..abc22c09ec7 100644 --- a/op-service/logpipe/pipe.go +++ b/op-service/logpipe/pipe.go @@ -1,7 +1,6 @@ package logpipe import ( - "bufio" "bytes" "encoding/json" "log/slog" @@ -78,22 +77,6 @@ type LogEntry interface { FieldValue(key string) any } -type LogProcessor func(line []byte) - -func (lo LogProcessor) Write(data []byte) (int, error) { - startingLength := len(data) - buf := bytes.NewBuffer(data) - scanner := bufio.NewScanner(buf) - for scanner.Scan() { - lineBytes := scanner.Bytes() - if len(lineBytes) == 0 { - continue // Skip empty lines - } - lo(lineBytes) - } - return startingLength - buf.Len(), scanner.Err() -} - type LogParser func(line []byte) LogEntry func ToLogger(logger log.Logger) func(e LogEntry) { diff --git a/op-service/logpipe/pipe_test.go b/op-service/logpipe/pipe_test.go index df6b7ed9c5a..475c7971d10 100644 --- a/op-service/logpipe/pipe_test.go +++ b/op-service/logpipe/pipe_test.go @@ -15,14 +15,14 @@ import ( func TestWriteToLogProcessor(t *testing.T) { logger, capt := testlog.CaptureLogger(t, log.LevelTrace) - logProc := LogProcessor(func(line []byte) { + proc := NewLineBuffer(func(line []byte) { ToLogger(logger)(ParseRustStructuredLogs(line)) }) - _, err := io.Copy(logProc, strings.NewReader(`{"level": "DEBUG", "fields": {"message": "hello", "foo": 1}}`+"\n")) + _, err := io.Copy(proc, strings.NewReader(`{"level": "DEBUG", "fields": {"message": "hello", "foo": 1}}`+"\n")) require.NoError(t, err) - _, err = io.Copy(logProc, strings.NewReader(`test invalid JSON`+"\n")) + _, err = io.Copy(proc, strings.NewReader(`test invalid JSON`+"\n")) require.NoError(t, err) - _, err = io.Copy(logProc, strings.NewReader(`{"fields": {"message": "world", "bar": "sunny"}, "level": "INFO"}`+"\n")) + _, err = io.Copy(proc, strings.NewReader(`{"fields": {"message": "world", "bar": "sunny"}, "level": "INFO"}`+"\n")) require.NoError(t, err) entry1 := capt.FindLog( diff --git a/op-test-sequencer/sequencer/backend/work/committers/standardcommitter/committer.go b/op-test-sequencer/sequencer/backend/work/committers/standardcommitter/committer.go index 317e854b963..10663488f49 100644 --- a/op-test-sequencer/sequencer/backend/work/committers/standardcommitter/committer.go +++ b/op-test-sequencer/sequencer/backend/work/committers/standardcommitter/committer.go @@ -49,6 +49,8 @@ func (n *Committer) Commit(ctx context.Context, block work.SignedBlock) error { err := n.api.CommitBlock(ctx, bl) if err != nil { n.log.Error("Failed to publish block", "block", block, "err", err) + return err } - return err + n.log.Info("Committed block to op-stack", "block", bl.ID()) + return nil } diff --git a/op-test-sequencer/sequencer/backend/work/publishers/standardpublisher/publisher.go b/op-test-sequencer/sequencer/backend/work/publishers/standardpublisher/publisher.go index a26ff2f6260..dfedd9225b3 100644 --- a/op-test-sequencer/sequencer/backend/work/publishers/standardpublisher/publisher.go +++ b/op-test-sequencer/sequencer/backend/work/publishers/standardpublisher/publisher.go @@ -47,6 +47,8 @@ func (n *Publisher) Publish(ctx context.Context, block work.SignedBlock) error { err := n.api.PublishBlock(ctx, bl) if err != nil { n.log.Error("Failed to publish block", "block", block, "err", err) + return err } - return err + n.log.Info("Published block to op-stack", "block", bl.ID()) + return nil }