diff --git a/.gitignore b/.gitignore index 069e8ea53af..f5abc09574b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ config/jwt.txt *.pem packages/contracts-bedrock/lib/automate/ + +# AI tools +.claude diff --git a/espresso/cli.go b/espresso/cli.go index 3e7b81dba63..30589a00d1e 100644 --- a/espresso/cli.go +++ b/espresso/cli.go @@ -27,19 +27,31 @@ func espressoEnvs(envprefix, v string) []string { return []string{envprefix + "_ESPRESSO_" + v} } +// Default values for batch submission receipt verification tuning. +// Defined here so that both the CLI flag defaults and the batcher logic +// can reference a single source of truth. +const ( + DefaultVerifyReceiptMaxBlocks uint64 = 5 + DefaultVerifyReceiptSafetyTimeout time.Duration = 5 * time.Minute + DefaultVerifyReceiptRetryDelay time.Duration = 100 * time.Millisecond +) + var ( - EnabledFlagName = espressoFlags("enabled") - PollIntervalFlagName = espressoFlags("poll-interval") - QueryServiceUrlsFlagName = espressoFlags("urls") - LightClientAddrFlagName = espressoFlags("light-client-addr") - L1UrlFlagName = espressoFlags("l1-url") - TestingBatcherPrivateKeyFlagName = espressoFlags("testing-batcher-private-key") - CaffeinationHeightEspresso = espressoFlags("origin-height-espresso") - CaffeinationHeightL2 = espressoFlags("origin-height-l2") - NamespaceFlagName = espressoFlags("namespace") - RollupL1UrlFlagName = espressoFlags("rollup-l1-url") - AttestationServiceFlagName = espressoFlags("espresso-attestation-service") - BatchAuthenticatorAddrFlagName = espressoFlags("batch-authenticator-addr") + EnabledFlagName = espressoFlags("enabled") + PollIntervalFlagName = espressoFlags("poll-interval") + QueryServiceUrlsFlagName = espressoFlags("urls") + LightClientAddrFlagName = espressoFlags("light-client-addr") + L1UrlFlagName = espressoFlags("l1-url") + TestingBatcherPrivateKeyFlagName = espressoFlags("testing-batcher-private-key") + CaffeinationHeightEspresso = espressoFlags("origin-height-espresso") + CaffeinationHeightL2 = espressoFlags("origin-height-l2") + NamespaceFlagName = espressoFlags("namespace") + RollupL1UrlFlagName = espressoFlags("rollup-l1-url") + AttestationServiceFlagName = espressoFlags("espresso-attestation-service") + BatchAuthenticatorAddrFlagName = espressoFlags("batch-authenticator-addr") + VerifyReceiptMaxBlocksFlagName = espressoFlags("verify-receipt-max-blocks") + VerifyReceiptSafetyTimeoutFlagName = espressoFlags("verify-receipt-safety-timeout") + VerifyReceiptRetryDelayFlagName = espressoFlags("verify-receipt-retry-delay") ) func CLIFlags(envPrefix string, category string) []cli.Flag { @@ -119,6 +131,27 @@ func CLIFlags(envPrefix string, category string) []cli.Flag { EnvVars: espressoEnvs(envPrefix, "BATCH_AUTHENTICATOR_ADDR"), Category: category, }, + &cli.Uint64Flag{ + Name: VerifyReceiptMaxBlocksFlagName, + Usage: "Number of HotShot blocks to wait for a submitted transaction to become queryable before re-submitting", + Value: DefaultVerifyReceiptMaxBlocks, + EnvVars: espressoEnvs(envPrefix, "VERIFY_RECEIPT_MAX_BLOCKS"), + Category: category, + }, + &cli.DurationFlag{ + Name: VerifyReceiptSafetyTimeoutFlagName, + Usage: "Wall-clock backstop for receipt verification; re-submits the transaction if this duration is exceeded", + Value: DefaultVerifyReceiptSafetyTimeout, + EnvVars: espressoEnvs(envPrefix, "VERIFY_RECEIPT_SAFETY_TIMEOUT"), + Category: category, + }, + &cli.DurationFlag{ + Name: VerifyReceiptRetryDelayFlagName, + Usage: "Delay between receipt verification retries", + Value: DefaultVerifyReceiptRetryDelay, + EnvVars: espressoEnvs(envPrefix, "VERIFY_RECEIPT_RETRY_DELAY"), + Category: category, + }, } } @@ -136,6 +169,11 @@ type CLIConfig struct { CaffeinationHeightL2 uint64 EspressoAttestationService string + // Batch submission receipt verification tuning + VerifyReceiptMaxBlocks uint64 + VerifyReceiptSafetyTimeout time.Duration + VerifyReceiptRetryDelay time.Duration + // Non directly configurable option allowEmptyAttestationService bool `json:"-"` } @@ -169,6 +207,15 @@ func (c CLIConfig) Check() error { if !c.allowEmptyAttestationService && c.EspressoAttestationService == "" { return fmt.Errorf("attestation service URL is required when Espresso is enabled") } + if c.VerifyReceiptMaxBlocks == 0 { + return fmt.Errorf("verify-receipt-max-blocks must be > 0") + } + if c.VerifyReceiptSafetyTimeout <= 0 { + return fmt.Errorf("verify-receipt-safety-timeout must be > 0") + } + if c.VerifyReceiptRetryDelay <= 0 { + return fmt.Errorf("verify-receipt-retry-delay must be > 0") + } } return nil } @@ -183,6 +230,9 @@ func ReadCLIConfig(c *cli.Context) CLIConfig { CaffeinationHeightEspresso: c.Uint64(CaffeinationHeightEspresso), CaffeinationHeightL2: c.Uint64(CaffeinationHeightL2), EspressoAttestationService: c.String(AttestationServiceFlagName), + VerifyReceiptMaxBlocks: c.Uint64(VerifyReceiptMaxBlocksFlagName), + VerifyReceiptSafetyTimeout: c.Duration(VerifyReceiptSafetyTimeoutFlagName), + VerifyReceiptRetryDelay: c.Duration(VerifyReceiptRetryDelayFlagName), } config.QueryServiceURLs = c.StringSlice(QueryServiceUrlsFlagName) diff --git a/espresso/environment/optitmism_espresso_test_helpers.go b/espresso/environment/optitmism_espresso_test_helpers.go index ff4900e1b72..a79c088ddb6 100644 --- a/espresso/environment/optitmism_espresso_test_helpers.go +++ b/espresso/environment/optitmism_espresso_test_helpers.go @@ -825,7 +825,7 @@ func launchEspressoDevNodeStartOption(ct *E2eDevnetLauncherContext) e2esys.Start l1EthRpcURLPtr, err := url.Parse(c.L1EthRpc) if err != nil { - ct.Error = FailedToDetermineL1RPCURL{Cause: err} + ct.T.Fatalf("failed to parse L1 RPC URL %q: %v", c.L1EthRpc, err) return } @@ -834,13 +834,13 @@ func launchEspressoDevNodeStartOption(ct *E2eDevnetLauncherContext) e2esys.Start // Let's spin up the espresso-dev-node l1EthRpcURL, err := translateContainerToNodeURL(*l1EthRpcURLPtr, network) if err != nil { - ct.Error = err + ct.T.Fatalf("failed to translate L1 RPC URL for Docker: %v", err) return } dockerConfig, portRemapping, err := determineEspressoDevNodeDockerContainerConfig(l1EthRpcURL, network) if err != nil { - ct.Error = err + ct.T.Fatalf("failed to build espresso dev node Docker config: %v", err) return } @@ -848,7 +848,7 @@ func launchEspressoDevNodeStartOption(ct *E2eDevnetLauncherContext) e2esys.Start espressoDevNodeContainerInfo, err := containerCli.LaunchContainer(ct.Ctx, dockerConfig) if err != nil { - ct.Error = FailedToLaunchDockerContainer{Cause: err} + ct.T.Fatalf("failed to launch espresso dev node container: %v", err) return } @@ -856,7 +856,7 @@ func launchEspressoDevNodeStartOption(ct *E2eDevnetLauncherContext) e2esys.Start // Wait for Espresso to be ready if err := waitForEspressoToFinishSpinningUp(ct, espressoDevNodeContainerInfo); err != nil { - ct.Error = err + ct.T.Fatalf("espresso dev node failed to become ready: %v", err) return } diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index 3f8ecb1a985..2d1c51fd009 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -284,6 +284,9 @@ func (l *BatchSubmitter) StartBatchSubmitting() error { WithContext(l.shutdownCtx), WithWaitGroup(l.wg), WithEspressoClient(l.Espresso), + WithVerifyReceiptMaxBlocks(l.Config.VerifyReceiptMaxBlocks), + WithVerifyReceiptSafetyTimeout(l.Config.VerifyReceiptSafetyTimeout), + WithVerifyReceiptRetryDelay(l.Config.VerifyReceiptRetryDelay), ) l.espressoSubmitter.SpawnWorkers(4, 4) l.espressoSubmitter.Start() diff --git a/op-batcher/batcher/espresso.go b/op-batcher/batcher/espresso.go index d7f489d3690..6079bbcbfd6 100644 --- a/op-batcher/batcher/espresso.go +++ b/op-batcher/batcher/espresso.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/signer/core/apitypes" + "github.com/ethereum-optimism/optimism/espresso" "github.com/ethereum-optimism/optimism/espresso/bindings" "github.com/ethereum-optimism/optimism/espresso/logmodule" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" @@ -109,16 +110,19 @@ type espressoVerifyReceiptJobAttempt struct { // the worker queue processing details for submitting transactions to Espresso // without spawning arbitrarily many goroutines. type espressoTransactionSubmitter struct { - ctx context.Context - wg *sync.WaitGroup - submitJobQueue chan espressoSubmitTransactionJob - submitRespQueue chan espressoSubmitTransactionJobResponse - submitWorkerQueue chan chan espressoTransactionJobAttempt - verifyReceiptJobQueue chan espressoVerifyReceiptJob - verifyReceiptRespQueue chan espressoVerifyReceiptJobResponse - verifyReceiptWorkerQueue chan chan espressoVerifyReceiptJobAttempt - espresso espressoClient.EspressoClient - latestBlockHeight atomic.Uint64 // shared HotShot block height, updated by trackBlockHeight + ctx context.Context + wg *sync.WaitGroup + submitJobQueue chan espressoSubmitTransactionJob + submitRespQueue chan espressoSubmitTransactionJobResponse + submitWorkerQueue chan chan espressoTransactionJobAttempt + verifyReceiptJobQueue chan espressoVerifyReceiptJob + verifyReceiptRespQueue chan espressoVerifyReceiptJobResponse + verifyReceiptWorkerQueue chan chan espressoVerifyReceiptJobAttempt + espresso espressoClient.EspressoClient + latestBlockHeight atomic.Uint64 // shared HotShot block height, updated by trackBlockHeight + verifyReceiptMaxBlocks uint64 + verifyReceiptSafetyTimeout time.Duration + verifyReceiptRetryDelay time.Duration } // EspressoTransactionSubmitterConfig is a configuration struct for the @@ -132,6 +136,9 @@ type EspressoTransactionSubmitterConfig struct { SubmitResponseQueueCapacity int VerifyReceiptJobQueueCapacity int VerifyReceiptResponseQueueCapacity int + VerifyReceiptMaxBlocks uint64 + VerifyReceiptSafetyTimeout time.Duration + VerifyReceiptRetryDelay time.Duration } // EspressoTransactionSubmitterOption is a function that can be used to @@ -162,6 +169,30 @@ func WithWaitGroup(wg *sync.WaitGroup) EspressoTransactionSubmitterOption { } } +// WithVerifyReceiptMaxBlocks sets the number of HotShot blocks to wait for a +// submitted transaction to become queryable before re-submitting. +func WithVerifyReceiptMaxBlocks(n uint64) EspressoTransactionSubmitterOption { + return func(config *EspressoTransactionSubmitterConfig) { + config.VerifyReceiptMaxBlocks = n + } +} + +// WithVerifyReceiptSafetyTimeout sets the wall-clock backstop for receipt +// verification. If the block height tracker is stale or broken, re-submission +// is triggered after this duration. +func WithVerifyReceiptSafetyTimeout(d time.Duration) EspressoTransactionSubmitterOption { + return func(config *EspressoTransactionSubmitterConfig) { + config.VerifyReceiptSafetyTimeout = d + } +} + +// WithVerifyReceiptRetryDelay sets the delay between receipt verification retries. +func WithVerifyReceiptRetryDelay(d time.Duration) EspressoTransactionSubmitterOption { + return func(config *EspressoTransactionSubmitterConfig) { + config.VerifyReceiptRetryDelay = d + } +} + // NewEspressoTransactionSubmitter creates a new EspressoTransactionSubmitter // with the given context and espresso client. It will create a new transaction // submitter with some default options, and apply those options to the @@ -180,6 +211,9 @@ func NewEspressoTransactionSubmitter(options ...EspressoTransactionSubmitterOpti SubmitResponseQueueCapacity: 10, VerifyReceiptJobQueueCapacity: 1024, VerifyReceiptResponseQueueCapacity: 10, + VerifyReceiptMaxBlocks: espresso.DefaultVerifyReceiptMaxBlocks, + VerifyReceiptSafetyTimeout: espresso.DefaultVerifyReceiptSafetyTimeout, + VerifyReceiptRetryDelay: espresso.DefaultVerifyReceiptRetryDelay, } for _, option := range options { @@ -191,15 +225,18 @@ func NewEspressoTransactionSubmitter(options ...EspressoTransactionSubmitterOpti } return &espressoTransactionSubmitter{ - ctx: config.Ctx, - wg: config.Wg, - submitJobQueue: make(chan espressoSubmitTransactionJob, config.SubmitJobQueueCapacity), - submitRespQueue: make(chan espressoSubmitTransactionJobResponse, config.SubmitResponseQueueCapacity), - submitWorkerQueue: make(chan chan espressoTransactionJobAttempt), - verifyReceiptJobQueue: make(chan espressoVerifyReceiptJob, config.VerifyReceiptJobQueueCapacity), - verifyReceiptRespQueue: make(chan espressoVerifyReceiptJobResponse, config.VerifyReceiptResponseQueueCapacity), - verifyReceiptWorkerQueue: make(chan chan espressoVerifyReceiptJobAttempt), - espresso: config.EspressoClient, + ctx: config.Ctx, + wg: config.Wg, + submitJobQueue: make(chan espressoSubmitTransactionJob, config.SubmitJobQueueCapacity), + submitRespQueue: make(chan espressoSubmitTransactionJobResponse, config.SubmitResponseQueueCapacity), + submitWorkerQueue: make(chan chan espressoTransactionJobAttempt), + verifyReceiptJobQueue: make(chan espressoVerifyReceiptJob, config.VerifyReceiptJobQueueCapacity), + verifyReceiptRespQueue: make(chan espressoVerifyReceiptJobResponse, config.VerifyReceiptResponseQueueCapacity), + verifyReceiptWorkerQueue: make(chan chan espressoVerifyReceiptJobAttempt), + espresso: config.EspressoClient, + verifyReceiptMaxBlocks: config.VerifyReceiptMaxBlocks, + verifyReceiptSafetyTimeout: config.VerifyReceiptSafetyTimeout, + verifyReceiptRetryDelay: config.VerifyReceiptRetryDelay, } } @@ -315,22 +352,11 @@ func (s *espressoTransactionSubmitter) handleTransactionSubmitJobResponse() { } } -// VERIFY_RECEIPT_MAX_BLOCKS is the number of HotShot blocks we will wait -// for a submitted transaction to become queryable before re-submitting. -// Using block count instead of wall-clock time makes us resilient to -// variable block times across different Espresso networks. -const VERIFY_RECEIPT_MAX_BLOCKS uint64 = 5 - -// VERIFY_RECEIPT_SAFETY_TIMEOUT is a wall-clock backstop for receipt -// verification. If the block height tracker is stale or broken, we fall -// back to this generous timeout before re-submitting. -const VERIFY_RECEIPT_SAFETY_TIMEOUT = 5 * time.Minute - -// VERIFY_RECEIPT_RETRY_DELAY is the amount of time we will wait before -// retrying a job that failed to verify the receipt. -const VERIFY_RECEIPT_RETRY_DELAY = 100 * time.Millisecond +// Default values for receipt verification tuning are defined as exported +// constants in the espresso package (espresso.DefaultVerifyReceipt*) so that +// the CLI flag defaults and this batcher logic share a single source of truth. -// Evaluate the verification job. +// evaluateVerification evaluates the verification job response. // // # Returns // @@ -343,7 +369,7 @@ const VERIFY_RECEIPT_RETRY_DELAY = 100 * time.Millisecond // * If the wall-clock safety timeout is exceeded: RetrySubmission. // // * Otherwise: RetryVerification. -func evaluateVerification(jobResp espressoVerifyReceiptJobResponse) JobEvaluation { +func (s *espressoTransactionSubmitter) evaluateVerification(jobResp espressoVerifyReceiptJobResponse) JobEvaluation { err := jobResp.err // If there's no error, continue handling the verification. @@ -363,20 +389,20 @@ func evaluateVerification(jobResp espressoVerifyReceiptJobResponse) JobEvaluatio // Block-count-based timeout: re-submit if enough HotShot blocks have // passed since verification started. The startHeight guard handles the // edge case where the height tracker hasn't fetched its first value yet. - if jobResp.job.startHeight > 0 && jobResp.currentHeight >= jobResp.job.startHeight+VERIFY_RECEIPT_MAX_BLOCKS { + if jobResp.job.startHeight > 0 && jobResp.currentHeight >= jobResp.job.startHeight+s.verifyReceiptMaxBlocks { log.Info("Verification timed out by block count, re-submitting", "startHeight", jobResp.job.startHeight, "currentHeight", jobResp.currentHeight, - "maxBlocks", VERIFY_RECEIPT_MAX_BLOCKS) + "maxBlocks", s.verifyReceiptMaxBlocks) return RetrySubmission } // Wall-clock safety backstop in case the block height tracker is stale // or broken (e.g., query service returning old data). - if elapsed := time.Since(jobResp.job.startTime); elapsed > VERIFY_RECEIPT_SAFETY_TIMEOUT { + if elapsed := time.Since(jobResp.job.startTime); elapsed > s.verifyReceiptSafetyTimeout { log.Warn("Verification timed out by safety timeout, re-submitting", "elapsed", elapsed, - "safetyTimeout", VERIFY_RECEIPT_SAFETY_TIMEOUT) + "safetyTimeout", s.verifyReceiptSafetyTimeout) return RetrySubmission } @@ -412,7 +438,7 @@ func (s *espressoTransactionSubmitter) handleVerifyReceiptJobResponse() { } } - switch evaluation := evaluateVerification(jobResp); evaluation { + switch evaluation := s.evaluateVerification(jobResp); evaluation { case Skip: continue case RetrySubmission: @@ -601,6 +627,7 @@ func espressoVerifyTransactionWorker( cli espressoClient.EspressoClient, workerQueue chan<- chan espressoVerifyReceiptJobAttempt, latestHeight *atomic.Uint64, + retryDelay time.Duration, ) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -640,7 +667,7 @@ func espressoVerifyTransactionWorker( // We have already attempted this job, so we will wait a bit // NOTE: this prevents this worker from being able to process // other jobs while we wait for this delay. - time.Sleep(VERIFY_RECEIPT_RETRY_DELAY) + time.Sleep(retryDelay) } _, err := cli.FetchTransactionByHash(ctx, jobAttempt.job.hash) @@ -673,7 +700,7 @@ func (s *espressoTransactionSubmitter) SpawnWorkers(numSubmitTransactionWorkers, for i := 0; i < numVerifyReceiptWorkers; i++ { s.wg.Add(1) - go espressoVerifyTransactionWorker(workersCtx, s.wg, s.espresso, s.verifyReceiptWorkerQueue, &s.latestBlockHeight) + go espressoVerifyTransactionWorker(workersCtx, s.wg, s.espresso, s.verifyReceiptWorkerQueue, &s.latestBlockHeight, s.verifyReceiptRetryDelay) } } @@ -691,7 +718,7 @@ func (s *espressoTransactionSubmitter) trackBlockHeight() { // Wait for the next interval or until context is done. select { - case <-time.After(VERIFY_RECEIPT_RETRY_DELAY): + case <-time.After(s.verifyReceiptRetryDelay): case <-s.ctx.Done(): return } diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index c3740ead5f0..78f556902e3 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -73,6 +73,11 @@ type BatcherConfig struct { // Starting position for the Espresso streamer. CaffeinationHeightEspresso uint64 CaffeinationHeightL2 uint64 + + // Receipt verification tuning for the Espresso transaction submitter. + VerifyReceiptMaxBlocks uint64 + VerifyReceiptSafetyTimeout time.Duration + VerifyReceiptRetryDelay time.Duration } // BatcherService represents a full batch-submitter instance and its resources, @@ -755,6 +760,9 @@ func (bs *BatcherService) initEspresso(cfg *CLIConfig) error { bs.EspressoAttestationService = cfg.Espresso.EspressoAttestationService bs.CaffeinationHeightEspresso = cfg.Espresso.CaffeinationHeightEspresso bs.CaffeinationHeightL2 = cfg.Espresso.CaffeinationHeightL2 + bs.VerifyReceiptMaxBlocks = cfg.Espresso.VerifyReceiptMaxBlocks + bs.VerifyReceiptSafetyTimeout = cfg.Espresso.VerifyReceiptSafetyTimeout + bs.VerifyReceiptRetryDelay = cfg.Espresso.VerifyReceiptRetryDelay client, err := espressoClient.NewMultipleNodesClient(cfg.Espresso.QueryServiceURLs) if err != nil { diff --git a/op-e2e/system/e2esys/setup.go b/op-e2e/system/e2esys/setup.go index d2472a4995c..dd66bba40b4 100644 --- a/op-e2e/system/e2esys/setup.go +++ b/op-e2e/system/e2esys/setup.go @@ -1015,11 +1015,14 @@ func (cfg SystemConfig) Start(t *testing.T, startOpts ...StartOption) (*System, return nil, fmt.Errorf("failed to parse pre-approved batcher private key: %w", err) } espressoCfg := espresso.CLIConfig{ - Enabled: cfg.AllocType.IsEspresso(), - PollInterval: 250 * time.Millisecond, - L1URL: sys.EthInstances[RoleL1].UserRPC().RPC(), - RollupL1URL: sys.EthInstances[RoleL1].UserRPC().RPC(), - TestingBatcherPrivateKey: testingBatcherPk, + Enabled: cfg.AllocType.IsEspresso(), + PollInterval: 250 * time.Millisecond, + L1URL: sys.EthInstances[RoleL1].UserRPC().RPC(), + RollupL1URL: sys.EthInstances[RoleL1].UserRPC().RPC(), + TestingBatcherPrivateKey: testingBatcherPk, + VerifyReceiptMaxBlocks: espresso.DefaultVerifyReceiptMaxBlocks, + VerifyReceiptSafetyTimeout: espresso.DefaultVerifyReceiptSafetyTimeout, + VerifyReceiptRetryDelay: espresso.DefaultVerifyReceiptRetryDelay, } // When Espresso is enabled, the primary batcher is the Espresso batcher which uses