feat(rollup-relayer): add a tool to analyze chunk/batch/bundle proposing

common/utils/flags.go

@@ -22,6 +22,10 @@ var (
 	RollupRelayerFlags = []cli.Flag{
 		&MinCodecVersionFlag,
 	}
+	// ProposerToolFlags contains flags only used in proposer tool
+	ProposerToolFlags = []cli.Flag{
+		&StartL2BlockFlag,
+	}
 	// ConfigFileFlag load json type config file.
 	ConfigFileFlag = cli.StringFlag{
 		Name:  "config",
@@ -90,4 +94,10 @@ var (
 		Usage:    "Minimum required codec version for the chunk/batch/bundle proposers",
 		Required: true,
 	}
+	// StartL2BlockFlag indicates the start L2 block number for proposer tool
+	StartL2BlockFlag = cli.Uint64Flag{
+		Name:  "start-l2-block",
+		Usage: "Start L2 block number for proposer tool",
+		Value: 0,
+	}
 )

database/cmd/app/client.go

@@ -42,7 +42,7 @@ func resetDB(ctx *cli.Context) error {
 	}
 
 	var version int64
-	err = migrate.ResetDB(db.DB)
+	err = migrate.Rollback(db.DB, &version)
 	if err != nil {
 		return err
 	}

rollup/README.md

@@ -34,15 +34,28 @@ make rollup_bins
 ./build/bin/rollup_relayer --config ./conf/config.json
 ```
 
+## Proposer Tool
+
+The Proposer Tool replays historical blocks with custom configurations (e.g., future hardfork configs, customized chunk/batch/bundle proposer configs) to generate chunks/batches/bundles, helping test parameter changes before protocol upgrade.
+
+You can:
+
+1. Enable different hardforks in the genesis configuration.
+2. Set custom chunk-proposer, batch-proposer, and bundle-proposer parameters.
+3. Analyze resulting metrics (blob size, block count, transaction count, gas usage).
+
 ## How to run the proposer tool?
 
 ### Set the configs
 
 1. Set genesis config to enable desired hardforks in [`proposer-tool-genesis.json`](./proposer-tool-genesis.json).
 2. Set proposer config in [`proposer-tool-config.json`](./proposer-tool-config.json) for data analysis.
+3. Set `start-l2-block` in the launch command of proposer-tool in [`docker-compose-proposer-tool.yml`](./docker-compose-proposer-tool.yml) to the block number you want to start from. The default is `0`, which means starting from the genesis block.
 
 ### Start the proposer tool using docker-compose
 
+Prerequisite: an RPC URL to an archive L2 node. The default url in [`proposer-tool-config.json`](./proposer-tool-config.json) is `https://rpc.scroll.io`.
+
 ```
 cd rollup
 DOCKER_BUILDKIT=1 docker-compose -f docker-compose-proposer-tool.yml up -d

rollup/cmd/proposer_tool/app/app.go

@@ -17,9 +17,9 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"scroll-tech/common/database"
-	"scroll-tech/common/types"
 	"scroll-tech/common/utils"
 	"scroll-tech/common/version"
+	"scroll-tech/database/migrate"
 
 	"scroll-tech/rollup/internal/config"
 	"scroll-tech/rollup/internal/controller/watcher"
@@ -38,6 +38,7 @@ func init() {
 	app.Version = version.Version
 	app.Flags = append(app.Flags, utils.CommonFlags...)
 	app.Flags = append(app.Flags, utils.RollupRelayerFlags...)
+	app.Flags = append(app.Flags, utils.ProposerToolFlags...)
 	app.Commands = []*cli.Command{}
 	app.Before = func(ctx *cli.Context) error {
 		return utils.LogSetup(ctx)
@@ -58,6 +59,14 @@ func action(ctx *cli.Context) error {
 	if err != nil {
 		log.Crit("failed to init db connection", "err", err)
 	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		log.Crit("failed to get db connection", "error", err)
+	}
+	if err = migrate.ResetDB(sqlDB); err != nil {
+		log.Crit("failed to reset db", "error", err)
+	}
+	log.Info("successfully reset db")
 	defer func() {
 		cancel()
 		if err = database.CloseDB(db); err != nil {
@@ -83,28 +92,33 @@ func action(ctx *cli.Context) error {
 		log.Crit("failed to connect l2 geth", "config file", cfgFile, "error", err)
 	}
 
-	genesisHeader, err := l2Client.HeaderByNumber(subCtx, big.NewInt(0))
+	startL2BlockHeight := ctx.Uint64(utils.StartL2BlockFlag.Name)
+	startL2Block, err := l2Client.BlockByNumber(context.Background(), big.NewInt(int64(startL2BlockHeight)))
 	if err != nil {
-		return fmt.Errorf("failed to retrieve L2 genesis header: %v", err)
+		log.Crit("failed to get start l2 block", "startL2BlockHeight", startL2BlockHeight, "error", err)
 	}
 
-	chunk := &encoding.Chunk{
-		Blocks: []*encoding.Block{{
-			Header:         genesisHeader,
-			Transactions:   nil,
-			WithdrawRoot:   common.Hash{},
-			RowConsumption: &gethTypes.RowConsumption{},
-		}},
-	}
+	chunk := &encoding.Chunk{Blocks: []*encoding.Block{{Header: startL2Block.Header()}}}
 
-	var dbChunk *orm.Chunk
-	dbChunk, err = orm.NewChunk(db).InsertChunk(subCtx, chunk, encoding.CodecV0, rutils.ChunkMetrics{})
+	prevChunk, err := orm.NewChunk(dbForReplay).GetParentChunkByBlockNumber(subCtx, startL2BlockHeight)
 	if err != nil {
-		log.Crit("failed to insert chunk", "error", err)
+		log.Crit("failed to get previous chunk", "error", err)
+	}
+
+	var startQueueIndex uint64
+	if prevChunk != nil {
+		startQueueIndex = prevChunk.TotalL1MessagesPoppedBefore + prevChunk.TotalL1MessagesPoppedInChunk
+	}
+
+	for _, tx := range startL2Block.Transactions() {
+		if tx.Type() == gethTypes.L1MessageTxType {
+			startQueueIndex++
+		}
 	}
 
-	if err = orm.NewChunk(db).UpdateProvingStatus(subCtx, dbChunk.Hash, types.ProvingTaskVerified); err != nil {
-		log.Crit("failed to update genesis chunk proving status", "error", err)
+	_, err = orm.NewChunk(db).InsertTestChunkForProposerTool(subCtx, chunk, encoding.CodecV0, startQueueIndex)
+	if err != nil {
+		log.Crit("failed to insert chunk", "error", err)
 	}
 
 	batch := &encoding.Batch{

rollup/docker-compose-proposer-tool.yml

@@ -17,28 +17,18 @@ services:
       timeout: 5s
       retries: 5
 
-  db-client:
-    build:
-      context: ..
-      dockerfile: ./build/dockerfiles/db_cli.Dockerfile
-    depends_on:
-      db:
-        condition: service_healthy
-    command: ["reset", "--config", "/app/conf/proposer-tool-db-config.json"]
-    volumes:
-      - ./proposer-tool-db-config.json:/app/conf/proposer-tool-db-config.json
-
   proposer-tool:
     build:
       context: ..
       dockerfile: ./rollup/proposer_tool.Dockerfile
     depends_on:
-      db-client:
-        condition: service_completed_successfully
+      db:
+        condition: service_healthy
     command: [
       "--config", "/app/conf/proposer-tool-config.json",
       "--genesis", "/app/conf/proposer-tool-genesis.json",
       "--min-codec-version", "4",
+      "--start-l2-block", "10000",
       "--log.debug", "--verbosity", "3"
     ]
     volumes:

rollup/internal/orm/chunk.go

@@ -179,6 +179,25 @@ func (o *Chunk) GetChunksByBatchHash(ctx context.Context, batchHash string) ([]*
 	return chunks, nil
 }
 
+// GetParentChunkByBlockNumber retrieves the parent chunk by block number
+// only used by proposer tool for analysis usage
+func (o *Chunk) GetParentChunkByBlockNumber(ctx context.Context, blockNumber uint64) (*Chunk, error) {
+	db := o.db.WithContext(ctx)
+	db = db.Model(&Chunk{})
+	db = db.Where("end_block_number < ?", blockNumber)
+	db = db.Order("end_block_number DESC")
+	db = db.Limit(1)
+
+	var chunk Chunk
+	if err := db.First(&chunk).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("Chunk.GetParentChunkByBlockNumber error: %w", err)
+	}
+	return &chunk, nil
+}
+
 // InsertChunk inserts a new chunk into the database.
 func (o *Chunk) InsertChunk(ctx context.Context, chunk *encoding.Chunk, codecVersion encoding.CodecVersion, metrics rutils.ChunkMetrics, dbTX ...*gorm.DB) (*Chunk, error) {
 	if chunk == nil || len(chunk.Blocks) == 0 {
@@ -259,6 +278,49 @@ func (o *Chunk) InsertChunk(ctx context.Context, chunk *encoding.Chunk, codecVer
 	return &newChunk, nil
 }
 
+// InsertTestChunkForProposerTool inserts a new chunk into the database only for analysis usage by proposer tool.
+func (o *Chunk) InsertTestChunkForProposerTool(ctx context.Context, chunk *encoding.Chunk, codecVersion encoding.CodecVersion, totalL1MessagePoppedBefore uint64, dbTX ...*gorm.DB) (*Chunk, error) {
+	if chunk == nil || len(chunk.Blocks) == 0 {
+		return nil, errors.New("invalid args")
+	}
+
+	chunkHash, err := rutils.GetChunkHash(chunk, totalL1MessagePoppedBefore, codecVersion)
+	if err != nil {
+		log.Error("failed to get chunk hash", "err", err)
+		return nil, fmt.Errorf("Chunk.InsertChunk error: %w", err)
+	}
+
+	numBlocks := len(chunk.Blocks)
+	newChunk := Chunk{
+		Index:                       0,
+		Hash:                        chunkHash.Hex(),
+		StartBlockNumber:            chunk.Blocks[0].Header.Number.Uint64(),
+		StartBlockHash:              chunk.Blocks[0].Header.Hash().Hex(),
+		EndBlockNumber:              chunk.Blocks[numBlocks-1].Header.Number.Uint64(),
+		EndBlockHash:                chunk.Blocks[numBlocks-1].Header.Hash().Hex(),
+		TotalL2TxGas:                chunk.TotalGasUsed(),
+		TotalL2TxNum:                chunk.NumL2Transactions(),
+		StartBlockTime:              chunk.Blocks[0].Header.Time,
+		TotalL1MessagesPoppedBefore: totalL1MessagePoppedBefore,
+		StateRoot:                   chunk.Blocks[numBlocks-1].Header.Root.Hex(),
+		WithdrawRoot:                chunk.Blocks[numBlocks-1].WithdrawRoot.Hex(),
+		CodecVersion:                int16(codecVersion),
+	}
+
+	db := o.db
+	if len(dbTX) > 0 && dbTX[0] != nil {
+		db = dbTX[0]
+	}
+	db = db.WithContext(ctx)
+	db = db.Model(&Chunk{})
+
+	if err := db.Create(&newChunk).Error; err != nil {
+		return nil, fmt.Errorf("Chunk.InsertChunk error: %w, chunk hash: %v", err, newChunk.Hash)
+	}
+
+	return &newChunk, nil
+}
+
 // UpdateProvingStatus updates the proving status of a chunk.
 func (o *Chunk) UpdateProvingStatus(ctx context.Context, hash string, status types.ProvingStatus, dbTX ...*gorm.DB) error {
 	updateFields := make(map[string]interface{})