feat(rollup-relayer): add a tool to analyze chunk/batch/bundle proposing

common/version/version.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.5.4"
+var tag = "v4.5.5"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

database/cmd/app/client.go

@@ -42,7 +42,7 @@ func resetDB(ctx *cli.Context) error {
 	}
 
 	var version int64
-	err = migrate.Rollback(db.DB, &version)
+	err = migrate.ResetDB(db.DB)
 	if err != nil {
 		return err
 	}

rollup/README.md

@@ -33,3 +33,32 @@ make rollup_bins
 ./build/bin/gas_oracle --config ./conf/config.json
 ./build/bin/rollup_relayer --config ./conf/config.json
 ```
+
+## How to run the proposer tool?
+
+### Set the configs
+
+1. Set genesis config to enable desired hardforks in [`proposer-tool-genesis.json`](./proposer-tool-genesis.json).
+2. Set proposer config in [`proposer-tool-config.json`](./proposer-tool-config.json) for data analysis.
+
+### Start the proposer tool using docker-compose.
+
+```
+cd rollup
+DOCKER_BUILDKIT=1 docker-compose -f docker-compose-proposer-tool.yml up -d
+```
+
+> Note: The port 5432 of database is mapped to the host machine. You can use `psql` or any db clients to connect to the database.
+
+> The DSN for the database is `postgres://postgres:postgres@db:5432/scroll?sslmode=disable`.
+
+
+### Reset env
+```
+docker-compose -f docker-compose-proposer-tool.yml down -v
+```
+
+If you need to rebuild the images, removing the old images is necessary. You can do this by running the following command:
+```
+docker images | grep rollup | awk '{print $3}' | xargs docker rmi -f
+```

rollup/cmd/proposer_tool/app/app.go

@@ -0,0 +1,179 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/scroll-tech/da-codec/encoding"
+	"github.com/scroll-tech/go-ethereum/common"
+	gethTypes "github.com/scroll-tech/go-ethereum/core/types"
+	"github.com/scroll-tech/go-ethereum/ethclient"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/types"
+	"scroll-tech/common/utils"
+	"scroll-tech/common/version"
+
+	"scroll-tech/rollup/internal/config"
+	"scroll-tech/rollup/internal/controller/watcher"
+	"scroll-tech/rollup/internal/orm"
+	rutils "scroll-tech/rollup/internal/utils"
+)
+
+var app *cli.App
+
+func init() {
+	// Set up proposer-tool app info.
+	app = cli.NewApp()
+	app.Action = action
+	app.Name = "proposer-tool"
+	app.Usage = "The Scroll Proposer Tool"
+	app.Version = version.Version
+	app.Flags = append(app.Flags, utils.CommonFlags...)
+	app.Flags = append(app.Flags, utils.RollupRelayerFlags...)
+	app.Commands = []*cli.Command{}
+	app.Before = func(ctx *cli.Context) error {
+		return utils.LogSetup(ctx)
+	}
+}
+
+func action(ctx *cli.Context) error {
+	// Load config file.
+	cfgFile := ctx.String(utils.ConfigFileFlag.Name)
+	cfg, err := config.NewConfig(cfgFile)
+	if err != nil {
+		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
+	}
+
+	subCtx, cancel := context.WithCancel(ctx.Context)
+	// Init db connection
+	db, err := database.InitDB(cfg.DBConfig)
+	if err != nil {
+		log.Crit("failed to init db connection", "err", err)
+	}
+	defer func() {
+		cancel()
+		if err = database.CloseDB(db); err != nil {
+			log.Crit("failed to close db connection", "error", err)
+		}
+	}()
+
+	// Init l2BlockOrm connection
+	dbForReplay, err := database.InitDB(cfg.DBConfigForReplay)
+	if err != nil {
+		log.Crit("failed to init l2BlockOrm connection", "err", err)
+	}
+	defer func() {
+		cancel()
+		if err = database.CloseDB(dbForReplay); err != nil {
+			log.Crit("failed to close l2BlockOrm connection", "error", err)
+		}
+	}()
+
+	// Init l2geth connection
+	l2Client, err := ethclient.Dial(cfg.L2Config.Endpoint)
+	if err != nil {
+		log.Crit("failed to connect l2 geth", "config file", cfgFile, "error", err)
+	}
+
+	genesisHeader, err := l2Client.HeaderByNumber(subCtx, big.NewInt(0))
+	if err != nil {
+		return fmt.Errorf("failed to retrieve L2 genesis header: %v", err)
+	}
+
+	genesisTime := genesisHeader.Time
+	currentTime := uint64(time.Now().Unix())
+	timeDrift := currentTime - genesisTime
+
+	cfg.L2Config.ChunkProposerConfig.ChunkTimeoutSec += timeDrift
+	cfg.L2Config.BatchProposerConfig.BatchTimeoutSec += timeDrift
+	cfg.L2Config.BundleProposerConfig.BundleTimeoutSec += timeDrift
+
+	chunk := &encoding.Chunk{
+		Blocks: []*encoding.Block{{
+			Header:         genesisHeader,
+			Transactions:   nil,
+			WithdrawRoot:   common.Hash{},
+			RowConsumption: &gethTypes.RowConsumption{},
+		}},
+	}
+
+	var dbChunk *orm.Chunk
+	dbChunk, err = orm.NewChunk(db).InsertChunk(subCtx, chunk, encoding.CodecV0, rutils.ChunkMetrics{})
+	if err != nil {
+		log.Crit("failed to insert chunk", "error", err)
+	}
+
+	if err = orm.NewChunk(db).UpdateProvingStatus(subCtx, dbChunk.Hash, types.ProvingTaskVerified); err != nil {
+		log.Crit("failed to update genesis chunk proving status", "error", err)
+	}
+
+	batch := &encoding.Batch{
+		Index:                      0,
+		TotalL1MessagePoppedBefore: 0,
+		ParentBatchHash:            common.Hash{},
+		Chunks:                     []*encoding.Chunk{chunk},
+	}
+
+	var dbBatch *orm.Batch
+	dbBatch, err = orm.NewBatch(db).InsertBatch(subCtx, batch, encoding.CodecV0, rutils.BatchMetrics{})
+	if err != nil {
+		log.Crit("failed to insert batch", "error", err)
+	}
+
+	if err = orm.NewChunk(db).UpdateBatchHashInRange(subCtx, 0, 0, dbBatch.Hash); err != nil {
+		log.Crit("failed to update batch hash for chunks", "error", err)
+	}
+
+	registry := prometheus.DefaultRegisterer
+
+	genesisPath := ctx.String(utils.Genesis.Name)
+	genesis, err := utils.ReadGenesis(genesisPath)
+	if err != nil {
+		log.Crit("failed to read genesis", "genesis file", genesisPath, "error", err)
+	}
+
+	// sanity check config
+	if cfg.L2Config.BatchProposerConfig.MaxChunksPerBatch <= 0 {
+		log.Crit("cfg.L2Config.BatchProposerConfig.MaxChunksPerBatch must be greater than 0")
+	}
+	if cfg.L2Config.ChunkProposerConfig.MaxL2GasPerChunk <= 0 {
+		log.Crit("cfg.L2Config.ChunkProposerConfig.MaxL2GasPerChunk must be greater than 0")
+	}
+
+	minCodecVersion := encoding.CodecVersion(ctx.Uint(utils.MinCodecVersionFlag.Name))
+	chunkProposer := watcher.NewChunkProposer(subCtx, cfg.L2Config.ChunkProposerConfig, minCodecVersion, genesis.Config, dbForReplay, db, registry, true /* used by tool */)
+	batchProposer := watcher.NewBatchProposer(subCtx, cfg.L2Config.BatchProposerConfig, minCodecVersion, genesis.Config, dbForReplay, db, registry)
+	bundleProposer := watcher.NewBundleProposer(subCtx, cfg.L2Config.BundleProposerConfig, minCodecVersion, genesis.Config, db, registry)
+
+	go utils.Loop(subCtx, 100*time.Millisecond, chunkProposer.TryProposeChunk)
+	go utils.Loop(subCtx, 100*time.Millisecond, batchProposer.TryProposeBatch)
+	go utils.Loop(subCtx, 100*time.Millisecond, bundleProposer.TryProposeBundle)
+
+	// Finish start all proposer tool functions.
+	log.Info("Start proposer-tool successfully", "version", version.Version)
+
+	// Catch CTRL-C to ensure a graceful shutdown.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	// Wait until the interrupt signal is received from an OS signal.
+	<-interrupt
+
+	return nil
+}
+
+// Run proposer tool cmd instance.
+func Run() {
+	if err := app.Run(os.Args); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

rollup/cmd/proposer_tool/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "scroll-tech/rollup/cmd/proposer_tool/app"
+
+func main() {
+	app.Run()
+}

rollup/cmd/rollup_relayer/app/app.go

@@ -102,8 +102,8 @@ func action(ctx *cli.Context) error {
 	}
 
 	minCodecVersion := encoding.CodecVersion(ctx.Uint(utils.MinCodecVersionFlag.Name))
-	chunkProposer := watcher.NewChunkProposer(subCtx, cfg.L2Config.ChunkProposerConfig, minCodecVersion, genesis.Config, db, registry)
-	batchProposer := watcher.NewBatchProposer(subCtx, cfg.L2Config.BatchProposerConfig, minCodecVersion, genesis.Config, db, registry)
+	chunkProposer := watcher.NewChunkProposer(subCtx, cfg.L2Config.ChunkProposerConfig, minCodecVersion, genesis.Config, db, db, registry, false /* not used by tool */)
+	batchProposer := watcher.NewBatchProposer(subCtx, cfg.L2Config.BatchProposerConfig, minCodecVersion, genesis.Config, db, db, registry)
 	bundleProposer := watcher.NewBundleProposer(subCtx, cfg.L2Config.BundleProposerConfig, minCodecVersion, genesis.Config, db, registry)
 
 	l2watcher := watcher.NewL2WatcherClient(subCtx, l2client, cfg.L2Config.Confirmations, cfg.L2Config.L2MessageQueueAddress, cfg.L2Config.WithdrawTrieRootSlot, genesis.Config, db, registry)

rollup/docker-compose-proposer-tool.yml

@@ -0,0 +1,50 @@
+version: '3'
+
+services:
+  db:
+    image: postgres:14
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=scroll
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  db-client:
+    build:
+      context: ..
+      dockerfile: ./build/dockerfiles/db_cli.Dockerfile
+    depends_on:
+      db:
+        condition: service_healthy
+    command: ["reset", "--config", "/app/conf/proposer-tool-db-config.json"]
+    volumes:
+      - ./proposer-tool-db-config.json:/app/conf/proposer-tool-db-config.json
+
+  proposer-tool:
+    build:
+      context: ..
+      dockerfile: ./rollup/proposer_tool.Dockerfile
+    depends_on:
+      db-client:
+        condition: service_completed_successfully
+    command: [
+      "--config", "/app/conf/proposer-tool-config.json",
+      "--genesis", "/app/conf/proposer-tool-genesis.json",
+      "--min-codec-version", "4",
+      "--log.debug", "--verbosity", "3"
+    ]
+    volumes:
+      - ./proposer-tool-config.json:/app/conf/proposer-tool-config.json
+      - ./proposer-tool-genesis.json:/app/conf/proposer-tool-genesis.json
+    restart: unless-stopped
+
+volumes:
+  postgres_data:

rollup/internal/config/config.go

@@ -15,9 +15,10 @@ import (
 
 // Config load configuration items.
 type Config struct {
-	L1Config *L1Config        `json:"l1_config"`
-	L2Config *L2Config        `json:"l2_config"`
-	DBConfig *database.Config `json:"db_config"`
+	L1Config          *L1Config        `json:"l1_config"`
+	L2Config          *L2Config        `json:"l2_config"`
+	DBConfig          *database.Config `json:"db_config"`
+	DBConfigForReplay *database.Config `json:"db_config_for_replay"`
 }
 
 // NewConfig returns a new instance of Config.

rollup/internal/controller/watcher/batch_proposer.go

@@ -59,7 +59,7 @@ type BatchProposer struct {
 }
 
 // NewBatchProposer creates a new BatchProposer instance.
-func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minCodecVersion encoding.CodecVersion, chainCfg *params.ChainConfig, db *gorm.DB, reg prometheus.Registerer) *BatchProposer {
+func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minCodecVersion encoding.CodecVersion, chainCfg *params.ChainConfig, l2BlockDB, db *gorm.DB, reg prometheus.Registerer) *BatchProposer {
 	log.Info("new batch proposer",
 		"maxL1CommitGasPerBatch", cfg.MaxL1CommitGasPerBatch,
 		"maxL1CommitCalldataSizePerBatch", cfg.MaxL1CommitCalldataSizePerBatch,
@@ -73,7 +73,7 @@ func NewBatchProposer(ctx context.Context, cfg *config.BatchProposerConfig, minC
 		db:                              db,
 		batchOrm:                        orm.NewBatch(db),
 		chunkOrm:                        orm.NewChunk(db),
-		l2BlockOrm:                      orm.NewL2Block(db),
+		l2BlockOrm:                      orm.NewL2Block(l2BlockDB),
 		maxL1CommitGasPerBatch:          cfg.MaxL1CommitGasPerBatch,
 		maxL1CommitCalldataSizePerBatch: cfg.MaxL1CommitCalldataSizePerBatch,
 		batchTimeoutSec:                 cfg.BatchTimeoutSec,