feat(rollup-relayer): add a tool to analyze chunk/batch/bundle proposing

common/utils/flags.go

@@ -22,6 +22,10 @@ var (
 	RollupRelayerFlags = []cli.Flag{
 		&MinCodecVersionFlag,
 	}
+	// ProposerToolFlags contains flags only used in proposer tool
+	ProposerToolFlags = []cli.Flag{
+		&StartL2BlockFlag,
+	}
 	// ConfigFileFlag load json type config file.
 	ConfigFileFlag = cli.StringFlag{
 		Name:  "config",
@@ -90,4 +94,10 @@ var (
 		Usage:    "Minimum required codec version for the chunk/batch/bundle proposers",
 		Required: true,
 	}
+	// StartL2BlockFlag indicates the start L2 block number for proposer tool
+	StartL2BlockFlag = cli.Uint64Flag{
+		Name:  "start-l2-block",
+		Usage: "Start L2 block number for proposer tool",
+		Value: 0,
+	}
 )

rollup/README.md

@@ -33,3 +33,45 @@ make rollup_bins
 ./build/bin/gas_oracle --config ./conf/config.json
 ./build/bin/rollup_relayer --config ./conf/config.json
 ```
+
+## Proposer Tool
+
+The Proposer Tool replays historical blocks with custom configurations (e.g., future hardfork configs, customized chunk/batch/bundle proposer configs) to generate chunks/batches/bundles, helping test parameter changes before protocol upgrade.
+
+You can:
+
+1. Enable different hardforks in the genesis configuration.
+2. Set custom chunk-proposer, batch-proposer, and bundle-proposer parameters.
+3. Analyze resulting metrics (blob size, block count, transaction count, gas usage).
+
+## How to run the proposer tool?
+
+### Set the configs
+
+1. Set genesis config to enable desired hardforks in [`proposer-tool-genesis.json`](./proposer-tool-genesis.json).
+2. Set proposer config in [`proposer-tool-config.json`](./proposer-tool-config.json) for data analysis.
+3. Set `start-l2-block` in the launch command of proposer-tool in [`docker-compose-proposer-tool.yml`](./docker-compose-proposer-tool.yml) to the block number you want to start from. The default is `0`, which means starting from the genesis block.
+
+### Start the proposer tool using docker-compose
+
+Prerequisite: an RPC URL to an archive L2 node. The default url in [`proposer-tool-config.json`](./proposer-tool-config.json) is `https://rpc.scroll.io`.
+
+```
+cd rollup
+DOCKER_BUILDKIT=1 docker-compose -f docker-compose-proposer-tool.yml up -d
+```
+
+> Note: The port 5432 of database is mapped to the host machine. You can use `psql` or any db clients to connect to the database.
+
+> The DSN for the database is `postgres://postgres:postgres@db:5432/scroll?sslmode=disable`.
+
+
+### Reset env
+```
+docker-compose -f docker-compose-proposer-tool.yml down -v
+```
+
+If you need to rebuild the images, removing the old images is necessary. You can do this by running the following command:
+```
+docker images | grep rollup | awk '{print $3}' | xargs docker rmi -f
+```

rollup/cmd/proposer_tool/app/app.go

@@ -0,0 +1,185 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/scroll-tech/da-codec/encoding"
+	"github.com/scroll-tech/go-ethereum/common"
+	gethTypes "github.com/scroll-tech/go-ethereum/core/types"
+	"github.com/scroll-tech/go-ethereum/ethclient"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/utils"
+	"scroll-tech/common/version"
+	"scroll-tech/database/migrate"
+
+	"scroll-tech/rollup/internal/config"
+	"scroll-tech/rollup/internal/controller/watcher"
+	"scroll-tech/rollup/internal/orm"
+	rutils "scroll-tech/rollup/internal/utils"
+)
+
+var app *cli.App
+
+func init() {
+	// Set up proposer-tool app info.
+	app = cli.NewApp()
+	app.Action = action
+	app.Name = "proposer-tool"
+	app.Usage = "The Scroll Proposer Tool"
+	app.Version = version.Version
+	app.Flags = append(app.Flags, utils.CommonFlags...)
+	app.Flags = append(app.Flags, utils.RollupRelayerFlags...)
+	app.Flags = append(app.Flags, utils.ProposerToolFlags...)
+	app.Commands = []*cli.Command{}
+	app.Before = func(ctx *cli.Context) error {
+		return utils.LogSetup(ctx)
+	}
+}
+
+func action(ctx *cli.Context) error {
+	// Load config file.
+	cfgFile := ctx.String(utils.ConfigFileFlag.Name)
+	cfg, err := config.NewConfig(cfgFile)
+	if err != nil {
+		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
+	}
+
+	subCtx, cancel := context.WithCancel(ctx.Context)
+	// Init db connection
+	db, err := database.InitDB(cfg.DBConfig)
+	if err != nil {
+		log.Crit("failed to init db connection", "err", err)
+	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		log.Crit("failed to get db connection", "error", err)
+	}
+	if err = migrate.ResetDB(sqlDB); err != nil {
+		log.Crit("failed to reset db", "error", err)
+	}
+	log.Info("successfully reset db")
+	defer func() {
+		cancel()
+		if err = database.CloseDB(db); err != nil {
+			log.Crit("failed to close db connection", "error", err)
+		}
+	}()
+
+	// Init dbForReplay connection
+	dbForReplay, err := database.InitDB(cfg.DBConfigForReplay)
+	if err != nil {
+		log.Crit("failed to init dbForReplay connection", "err", err)
+	}
+	defer func() {
+		cancel()
+		if err = database.CloseDB(dbForReplay); err != nil {
+			log.Crit("failed to close dbForReplay connection", "error", err)
+		}
+	}()
+
+	// Init l2geth connection
+	l2Client, err := ethclient.Dial(cfg.L2Config.Endpoint)
+	if err != nil {
+		log.Crit("failed to connect l2 geth", "config file", cfgFile, "error", err)
+	}
+
+	startL2BlockHeight := ctx.Uint64(utils.StartL2BlockFlag.Name)
+	startL2Block, err := l2Client.BlockByNumber(context.Background(), big.NewInt(int64(startL2BlockHeight)))
+	if err != nil {
+		log.Crit("failed to get start l2 block", "startL2BlockHeight", startL2BlockHeight, "error", err)
+	}
+
+	chunk := &encoding.Chunk{Blocks: []*encoding.Block{{Header: startL2Block.Header()}}}
+
+	prevChunk, err := orm.NewChunk(dbForReplay).GetParentChunkByBlockNumber(subCtx, startL2BlockHeight)
+	if err != nil {
+		log.Crit("failed to get previous chunk", "error", err)
+	}
+
+	var startQueueIndex uint64
+	if prevChunk != nil {
+		startQueueIndex = prevChunk.TotalL1MessagesPoppedBefore + prevChunk.TotalL1MessagesPoppedInChunk
+	}
+
+	for _, tx := range startL2Block.Transactions() {
+		if tx.Type() == gethTypes.L1MessageTxType {
+			startQueueIndex++
+		}
+	}
+
+	_, err = orm.NewChunk(db).InsertTestChunkForProposerTool(subCtx, chunk, encoding.CodecV0, startQueueIndex)
+	if err != nil {
+		log.Crit("failed to insert chunk", "error", err)
+	}
+
+	batch := &encoding.Batch{
+		Index:                      0,
+		TotalL1MessagePoppedBefore: 0,
+		ParentBatchHash:            common.Hash{},
+		Chunks:                     []*encoding.Chunk{chunk},
+	}
+
+	var dbBatch *orm.Batch
+	dbBatch, err = orm.NewBatch(db).InsertBatch(subCtx, batch, encoding.CodecV0, rutils.BatchMetrics{})
+	if err != nil {
+		log.Crit("failed to insert batch", "error", err)
+	}
+
+	if err = orm.NewChunk(db).UpdateBatchHashInRange(subCtx, 0, 0, dbBatch.Hash); err != nil {
+		log.Crit("failed to update batch hash for chunks", "error", err)
+	}
+
+	registry := prometheus.DefaultRegisterer
+
+	genesisPath := ctx.String(utils.Genesis.Name)
+	genesis, err := utils.ReadGenesis(genesisPath)
+	if err != nil {
+		log.Crit("failed to read genesis", "genesis file", genesisPath, "error", err)
+	}
+
+	// sanity check config
+	if cfg.L2Config.BatchProposerConfig.MaxChunksPerBatch <= 0 {
+		log.Crit("cfg.L2Config.BatchProposerConfig.MaxChunksPerBatch must be greater than 0")
+	}
+	if cfg.L2Config.ChunkProposerConfig.MaxL2GasPerChunk <= 0 {
+		log.Crit("cfg.L2Config.ChunkProposerConfig.MaxL2GasPerChunk must be greater than 0")
+	}
+
+	minCodecVersion := encoding.CodecVersion(ctx.Uint(utils.MinCodecVersionFlag.Name))
+	chunkProposer := watcher.NewChunkProposer(subCtx, cfg.L2Config.ChunkProposerConfig, minCodecVersion, genesis.Config, dbForReplay, db, registry)
+	batchProposer := watcher.NewBatchProposer(subCtx, cfg.L2Config.BatchProposerConfig, minCodecVersion, genesis.Config, dbForReplay, db, registry)
+	bundleProposer := watcher.NewBundleProposer(subCtx, cfg.L2Config.BundleProposerConfig, minCodecVersion, genesis.Config, db, registry)
+
+	go utils.Loop(subCtx, 100*time.Millisecond, chunkProposer.TryProposeChunk)
+	go utils.Loop(subCtx, 100*time.Millisecond, batchProposer.TryProposeBatch)
+	go utils.Loop(subCtx, 100*time.Millisecond, bundleProposer.TryProposeBundle)
+
+	// Finish start all proposer tool functions.
+	log.Info("Start proposer-tool successfully", "version", version.Version)
+
+	// Catch CTRL-C to ensure a graceful shutdown.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	// Wait until the interrupt signal is received from an OS signal.
+	<-interrupt
+
+	return nil
+}
+
+// Run proposer tool cmd instance.
+func Run() {
+	if err := app.Run(os.Args); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

rollup/cmd/proposer_tool/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "scroll-tech/rollup/cmd/proposer_tool/app"
+
+func main() {
+	app.Run()
+}

rollup/cmd/rollup_relayer/app/app.go

@@ -102,8 +102,8 @@ func action(ctx *cli.Context) error {
 	}
 
 	minCodecVersion := encoding.CodecVersion(ctx.Uint(utils.MinCodecVersionFlag.Name))
-	chunkProposer := watcher.NewChunkProposer(subCtx, cfg.L2Config.ChunkProposerConfig, minCodecVersion, genesis.Config, db, registry)
-	batchProposer := watcher.NewBatchProposer(subCtx, cfg.L2Config.BatchProposerConfig, minCodecVersion, genesis.Config, db, registry)
+	chunkProposer := watcher.NewChunkProposer(subCtx, cfg.L2Config.ChunkProposerConfig, minCodecVersion, genesis.Config, db, db, registry)
+	batchProposer := watcher.NewBatchProposer(subCtx, cfg.L2Config.BatchProposerConfig, minCodecVersion, genesis.Config, db, db, registry)
 	bundleProposer := watcher.NewBundleProposer(subCtx, cfg.L2Config.BundleProposerConfig, minCodecVersion, genesis.Config, db, registry)
 
 	l2watcher := watcher.NewL2WatcherClient(subCtx, l2client, cfg.L2Config.Confirmations, cfg.L2Config.L2MessageQueueAddress, cfg.L2Config.WithdrawTrieRootSlot, genesis.Config, db, registry)

rollup/docker-compose-proposer-tool.yml

@@ -0,0 +1,40 @@
+version: '3'
+
+services:
+  db:
+    image: postgres:14
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=scroll
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  proposer-tool:
+    build:
+      context: ..
+      dockerfile: ./rollup/proposer_tool.Dockerfile
+    depends_on:
+      db:
+        condition: service_healthy
+    command: [
+      "--config", "/app/conf/proposer-tool-config.json",
+      "--genesis", "/app/conf/proposer-tool-genesis.json",
+      "--min-codec-version", "4",
+      "--start-l2-block", "10000",
+      "--log.debug", "--verbosity", "3"
+    ]
+    volumes:
+      - ./proposer-tool-config.json:/app/conf/proposer-tool-config.json
+      - ./proposer-tool-genesis.json:/app/conf/proposer-tool-genesis.json
+    restart: unless-stopped
+
+volumes:
+  postgres_data:

rollup/internal/config/config.go

@@ -15,9 +15,10 @@ import (
 
 // Config load configuration items.
 type Config struct {
-	L1Config *L1Config        `json:"l1_config"`
-	L2Config *L2Config        `json:"l2_config"`
-	DBConfig *database.Config `json:"db_config"`
+	L1Config          *L1Config        `json:"l1_config"`
+	L2Config          *L2Config        `json:"l2_config"`
+	DBConfig          *database.Config `json:"db_config"`
+	DBConfigForReplay *database.Config `json:"db_config_for_replay"`
 }
 
 // NewConfig returns a new instance of Config.

rollup/internal/controller/watcher/batch_proposer.go

@@ -13,6 +13,8 @@ import (
 	"github.com/scroll-tech/go-ethereum/params"
 	"gorm.io/gorm"
 
+	"scroll-tech/common/types"
+
 	"scroll-tech/rollup/internal/config"
 	"scroll-tech/rollup/internal/orm"
 	"scroll-tech/rollup/internal/utils"
@@ -34,6 +36,7 @@ type BatchProposer struct {
 	maxUncompressedBatchBytesSize   uint64
 	maxChunksPerBatch               int
 
+	replayMode      bool
 	minCodecVersion encoding.CodecVersion
 	chainCfg        *params.ChainConfig
 
@@ -59,7 +62,7 @@ type BatchProposer struct {
 }
 
 // NewBatchProposer creates a new BatchProposer instance.
-func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minCodecVersion encoding.CodecVersion, chainCfg *params.ChainConfig, db *gorm.DB, reg prometheus.Registerer) *BatchProposer {
+func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minCodecVersion encoding.CodecVersion, chainCfg *params.ChainConfig, l2BlockDB, db *gorm.DB, reg prometheus.Registerer) *BatchProposer {
 	log.Info("new batch proposer",
 		"maxL1CommitGasPerBatch", cfg.MaxL1CommitGasPerBatch,
 		"maxL1CommitCalldataSizePerBatch", cfg.MaxL1CommitCalldataSizePerBatch,
@@ -73,13 +76,14 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minC
 		db:                              db,
 		batchOrm:                        orm.NewBatch(db),
 		chunkOrm:                        orm.NewChunk(db),
-		l2BlockOrm:                      orm.NewL2Block(db),
+		l2BlockOrm:                      orm.NewL2Block(l2BlockDB),
 		maxL1CommitGasPerBatch:          cfg.MaxL1CommitGasPerBatch,
 		maxL1CommitCalldataSizePerBatch: cfg.MaxL1CommitCalldataSizePerBatch,
 		batchTimeoutSec:                 cfg.BatchTimeoutSec,
 		gasCostIncreaseMultiplier:       cfg.GasCostIncreaseMultiplier,
 		maxUncompressedBatchBytesSize:   cfg.MaxUncompressedBatchBytesSize,
 		maxChunksPerBatch:               cfg.MaxChunksPerBatch,
+		replayMode:                      db != l2BlockDB,
 		minCodecVersion:                 minCodecVersion,
 		chainCfg:                        chainCfg,
 
@@ -226,6 +230,15 @@ func (p *BatchProposer) updateDBBatchInfo(batch *encoding.Batch, codecVersion en
 			log.Warn("BatchProposer.UpdateBatchHashInRange update the chunk's batch hash failure", "hash", dbBatch.Hash, "error", dbErr)
 			return dbErr
 		}
+		if p.replayMode {
+			// If replayMode is true, meaning the batch was proposed by the proposer tool,
+			// set batch status to types.RollupCommitted and assign a unique commit tx hash to enable new bundle proposals.
+			if dbErr = p.batchOrm.UpdateCommitTxHashAndRollupStatus(p.ctx, dbBatch.Hash, dbBatch.Hash, types.RollupCommitted, dbTX); dbErr != nil {
+				log.Warn("BatchProposer.UpdateCommitTxHashAndRollupStatus update the batch's commit tx hash failure", "hash", dbBatch.Hash, "error", dbErr)
+				return dbErr
+			}
+		}
+
 		return nil
 	})
 	if err != nil {