feat: add blob storage service

build/dockerfiles/blob_uploader.Dockerfile

@@ -0,0 +1,30 @@
+# Download Go dependencies
+FROM scrolltech/go-rust-builder:go-1.22-rust-nightly-2023-12-03 as base
+
+WORKDIR /src
+COPY go.work* ./
+COPY ./rollup/go.* ./rollup/
+COPY ./common/go.* ./common/
+COPY ./coordinator/go.* ./coordinator/
+COPY ./database/go.* ./database/
+COPY ./tests/integration-test/go.* ./tests/integration-test/
+COPY ./bridge-history-api/go.* ./bridge-history-api/
+RUN go mod download -x
+
+# Build blob_uploader
+FROM base as builder
+
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    cd /src/rollup/cmd/blob_uploader/ && CGO_LDFLAGS="-ldl" go build -v -p 4 -o /bin/blob_uploader
+
+# Pull blob_uploader into a second stage deploy ubuntu container
+FROM ubuntu:20.04
+
+RUN apt update && apt install vim netcat-openbsd net-tools curl ca-certificates -y
+
+ENV CGO_LDFLAGS="-ldl"
+
+COPY --from=builder /bin/blob_uploader /bin/
+WORKDIR /app
+ENTRYPOINT ["blob_uploader"]

build/dockerfiles/blob_uploader.Dockerfile.dockerignore

@@ -0,0 +1,5 @@
+assets/
+docs/
+l2geth/
+rpc-gateway/
+*target/*

common/types/db.go

@@ -326,3 +326,53 @@ func (s TxStatus) String() string {
 		return fmt.Sprintf("Unknown TxStatus (%d)", int32(s))
 	}
 }
+
+// BlobUploadStatus represents the status of a blob upload
+type BlobUploadStatus int
+
+const (
+	// BlobUploadStatusUndefined indicates an undefined status
+	BlobUploadStatusUndefined BlobUploadStatus = iota
+	// BlobUploadStatusPending indicates a pending upload status
+	BlobUploadStatusPending
+	// BlobUploadStatusUploaded indicates a successful upload status
+	BlobUploadStatusUploaded
+	// BlobUploadStatusFailed indicates a failed upload status
+	BlobUploadStatusFailed
+)
+
+func (s BlobUploadStatus) String() string {
+	switch s {
+	case BlobUploadStatusPending:
+		return "BlobUploadStatusPending"
+	case BlobUploadStatusUploaded:
+		return "BlobUploadStatusUploaded"
+	case BlobUploadStatusFailed:
+		return "BlobUploadStatusFailed"
+	default:
+		return fmt.Sprintf("Unknown BlobUploadStatus (%d)", int32(s))
+	}
+}
+
+// BlobStoragePlatform represents the platform a blob upload to
+type BlobStoragePlatform int
+
+const (
+	// BlobStoragePlatformUndefined indicates an undefined platform
+	BlobStoragePlatformUndefined BlobStoragePlatform = iota
+	// BlobStoragePlatformS3 represents AWS S3
+	BlobStoragePlatformS3
+	// BlobStoragePlatformArweave represents storage blockchain Arweave
+	BlobStoragePlatformArweave
+)
+
+func (s BlobStoragePlatform) String() string {
+	switch s {
+	case BlobStoragePlatformS3:
+		return "BlobStoragePlatformS3"
+	case BlobStoragePlatformArweave:
+		return "BlobStoragePlatformArweave"
+	default:
+		return fmt.Sprintf("Unknown BlobStoragePlatform (%d)", int32(s))
+	}
+}

common/utils/ethereum.go

@@ -0,0 +1,23 @@
+package utils
+
+import (
+	"crypto/sha256"
+	"fmt"
+
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+)
+
+// CalculateVersionedBlobHash calculate the kzg4844 versioned blob hash from a blob
+func CalculateVersionedBlobHash(blob kzg4844.Blob) ([32]byte, error) {
+	// calculate kzg4844 commitment from blob
+	commit, err := kzg4844.BlobToCommitment(&blob)
+	if err != nil {
+		return [32]byte{}, fmt.Errorf("failed to get blob commitment, err: %w", err)
+	}
+
+	// calculate kzg4844 versioned blob hash from blob commitment
+	hasher := sha256.New()
+	vh := kzg4844.CalcBlobHashV1(hasher, &commit)
+
+	return vh, nil
+}

common/utils/ethereum_test.go

@@ -0,0 +1,53 @@
+package utils
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+)
+
+type BlobData struct {
+	VersionedBlobHash string `json:"versionedBlobHash"`
+	BlobData          string `json:"blobData"`
+}
+
+// TestCalculateVersionedBlobHash tests the CalculateVersionedBlobHash function
+func TestCalculateVersionedBlobHash(t *testing.T) {
+	// Read the test data
+	data, err := os.ReadFile("../testdata/blobdata.json")
+	if err != nil {
+		t.Fatalf("Failed to read blobdata.json: %v", err)
+	}
+
+	var blobData BlobData
+	if err := json.Unmarshal(data, &blobData); err != nil {
+		t.Fatalf("Failed to parse blobdata.json: %v", err)
+	}
+
+	fmt.Println(blobData.BlobData)
+	blobBytes, err := hex.DecodeString(blobData.BlobData)
+	if err != nil {
+		t.Fatalf("Failed to decode blob data: %v", err)
+	}
+
+	// Convert []byte to kzg4844.Blob
+	var blob kzg4844.Blob
+	copy(blob[:], blobBytes)
+
+	// Calculate the hash
+	calculatedHashBytes, err := CalculateVersionedBlobHash(blob)
+	if err != nil {
+		t.Fatalf("Failed to calculate versioned blob hash: %v", err)
+	}
+
+	calculatedHash := hex.EncodeToString(calculatedHashBytes[:])
+
+	if calculatedHash != blobData.VersionedBlobHash {
+		t.Fatalf("Hash mismatch: got %s, want %s", calculatedHash, blobData.VersionedBlobHash)
+	}
+
+}

coordinator/internal/types/prover.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"fmt"
+
 	"scroll-tech/common/types/message"
 )
 

database/migrate/migrate_test.go

@@ -59,20 +59,20 @@ func testResetDB(t *testing.T) {
 	cur, err := Current(pgDB)
 	assert.NoError(t, err)
 	// total number of tables.
-	assert.Equal(t, int64(26), cur)
+	assert.Equal(t, int64(28), cur)
 }
 
 func testMigrate(t *testing.T) {
 	assert.NoError(t, Migrate(pgDB))
 	cur, err := Current(pgDB)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(26), cur)
+	assert.Equal(t, int64(28), cur)
 }
 
 func testRollback(t *testing.T) {
 	version, err := Current(pgDB)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(26), version)
+	assert.Equal(t, int64(28), version)
 
 	assert.NoError(t, Rollback(pgDB, nil))
 

database/migrate/migrations/00027_alter_batch_add_primary_key_index.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE batch ADD PRIMARY KEY (index);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE batch DROP CONSTRAINT batch_pkey;
+-- +goose StatementEnd

database/migrate/migrations/00028_ blob_upload.sql

@@ -0,0 +1,37 @@
+-- +goose Up
+-- +goose StatementBegin
+
+CREATE TABLE blob_upload (
+    batch_index     BIGINT          NOT NULL,
+
+    platform        SMALLINT        NOT NULL,
+    status          SMALLINT        NOT NULL,
+
+-- metadata
+    updated_at      TIMESTAMP       NOT NULL DEFAULT now(),
+    deleted_at      TIMESTAMP(0)    DEFAULT NULL,
+
+    PRIMARY KEY (batch_index, platform),
+    FOREIGN KEY (batch_index) REFERENCES batch(index)
+);
+
+COMMENT ON COLUMN blob_upload.status IS 'undefined, pending, uploaded, failed';
+
+CREATE INDEX IF NOT EXISTS idx_blob_upload_batch_index ON blob_upload(batch_index) WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_blob_upload_platform ON blob_upload(platform) WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_blob_upload_status ON blob_upload(status) WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_blob_upload_updated_at ON blob_upload(updated_at) WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_blob_upload_status_platform ON blob_upload(status, platform) WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_blob_upload_batch_index_status_platform ON blob_upload(batch_index, status, platform) WHERE deleted_at IS NULL;
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP TABLE blob_upload;
+-- +goose StatementEnd

go.work.sum

@@ -646,6 +646,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6/go.mod h1:PudwVKUTApfm0nYaPutOXa
 github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM=
 github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw=
+github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
+github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -1357,7 +1359,6 @@ github.com/scroll-tech/da-codec v0.1.1-0.20241014152913-2703f226fb0b h1:5H6V6yba
 github.com/scroll-tech/da-codec v0.1.1-0.20241014152913-2703f226fb0b/go.mod h1:48uxaqVgpD8ulH8p+nrBtfeLHZ9tX82bVVdPNkW3rPE=
 github.com/scroll-tech/da-codec v0.1.3-0.20250227072756-a1482833595f h1:YYbhuUwjowqI4oyXtECRofck7Fyj18e1tcRjuQlZpJE=
 github.com/scroll-tech/da-codec v0.1.3-0.20250227072756-a1482833595f/go.mod h1:xECEHZLVzbdUn+tNbRJhRIjLGTOTmnFQuTgUTeVLX58=
-github.com/scroll-tech/da-codec v0.1.3-0.20250519114140-bfa7133d4ad1/go.mod h1:yhTS9OVC0xQGhg7DN5iV5KZJvnSIlFWAxDdp+6jxQtY=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240607130425-e2becce6a1a4/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240821074444-b3fa00861e5e/go.mod h1:swB5NSp8pKNDuYsTxfR08bHS6L56i119PBx8fxvV8Cs=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20241010064814-3d88e870ae22/go.mod h1:r9FwtxCtybMkTbWYCyBuevT9TW3zHmOTHqD082Uh+Oo=

rollup/Makefile

@@ -11,13 +11,17 @@ mock_abi:
 rollup_bins: ## Builds the Rollup bins.
 	go build -o $(PWD)/build/bin/gas_oracle ./cmd/gas_oracle/
 	go build -o $(PWD)/build/bin/rollup_relayer ./cmd/rollup_relayer/
+	go build -o $(PWD)/build/bin/blob_uploader ./cmd/blob_uploader/
 
 gas_oracle: ## Builds the gas_oracle bin
 	go build -o $(PWD)/build/bin/gas_oracle ./cmd/gas_oracle/
 
 rollup_relayer: ## Builds the rollup_relayer bin
 	go build -o $(PWD)/build/bin/rollup_relayer ./cmd/rollup_relayer/
 
+blob_uploader: ## Builds the blob_uploader bin
+	go build -o $(PWD)/build/bin/blob_uploader ./cmd/blob_uploader/
+
 test:
 	go test -v -race -coverprofile=coverage.txt -covermode=atomic -p 1 $(PWD)/...
 

rollup/cmd/blob_uploader/app/app.go

@@ -0,0 +1,103 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/scroll-tech/da-codec/encoding"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/observability"
+	"scroll-tech/common/utils"
+	"scroll-tech/common/version"
+
+	"scroll-tech/rollup/internal/config"
+	"scroll-tech/rollup/internal/controller/blob_uploader"
+)
+
+var app *cli.App
+
+func init() {
+	// Set up blob-uploader app info.
+	app = cli.NewApp()
+	app.Action = action
+	app.Name = "blob-uploader"
+	app.Usage = "The Scroll Blob Uploader"
+	app.Version = version.Version
+	app.Flags = append(app.Flags, utils.CommonFlags...)
+	app.Flags = append(app.Flags, utils.RollupRelayerFlags...)
+	app.Commands = []*cli.Command{}
+	app.Before = func(ctx *cli.Context) error {
+		return utils.LogSetup(ctx)
+	}
+	// Register `rollup-relayer-test` app for integration-test.
+	utils.RegisterSimulation(app, utils.RollupRelayerApp)
+}
+
+func action(ctx *cli.Context) error {
+	// Load config file.
+	cfgFile := ctx.String(utils.ConfigFileFlag.Name)
+	cfg, err := config.NewConfig(cfgFile)
+	if err != nil {
+		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
+	}
+
+	subCtx, cancel := context.WithCancel(ctx.Context)
+	// Init db connection
+	db, err := database.InitDB(cfg.DBConfig)
+	if err != nil {
+		log.Crit("failed to init db connection", "err", err)
+	}
+	defer func() {
+		cancel()
+		if err = database.CloseDB(db); err != nil {
+			log.Crit("failed to close db connection", "error", err)
+		}
+	}()
+
+	registry := prometheus.DefaultRegisterer
+	observability.Server(ctx, db)
+
+	// sanity check config
+	if cfg.L2Config.BlobUploaderConfig == nil {
+		log.Crit("cfg.L2Config.BlobUploaderConfig must not be nil")
+	}
+
+	blobUploader, err := blob_uploader.NewBlobUploader(ctx.Context, db, cfg.L2Config.BlobUploaderConfig, registry)
+	if err != nil {
+		log.Crit("failed to create l2 relayer", "config file", cfgFile, "error", err)
+	}
+
+	minCodecVersion := encoding.CodecVersion(ctx.Uint(utils.MinCodecVersionFlag.Name))
+	if minCodecVersion < encoding.CodecV7 {
+		log.Crit("min codec version must be greater than or equal to CodecV7", "minCodecVersion", minCodecVersion)
+	}
+
+	go utils.Loop(subCtx, 2*time.Second, blobUploader.UploadBlobToS3)
+
+	// Finish start all blob-uploader functions.
+	log.Info("Start blob-uploader successfully", "version", version.Version)
+
+	// Catch CTRL-C to ensure a graceful shutdown.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	// Wait until the interrupt signal is received from an OS signal.
+	<-interrupt
+
+	return nil
+}
+
+// Run blob uploader cmd instance.
+func Run() {
+	if err := app.Run(os.Args); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

rollup/cmd/blob_uploader/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "scroll-tech/rollup/cmd/blob_uploader/app"
+
+func main() {
+	app.Run()
+}