From fa721499815bbe0cdb469e470cd3572ff48f3fea Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Sun, 28 Apr 2024 23:48:12 -0300 Subject: [PATCH 01/30] Initial script to play with celo DB history migration --- op-chain-ops/cmd/celo-dbplay/main.go | 229 +++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 op-chain-ops/cmd/celo-dbplay/main.go diff --git a/op-chain-ops/cmd/celo-dbplay/main.go b/op-chain-ops/cmd/celo-dbplay/main.go new file mode 100644 index 0000000000000..e40d1457dac1b --- /dev/null +++ b/op-chain-ops/cmd/celo-dbplay/main.go @@ -0,0 +1,229 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/mattn/go-isatty" + + "github.com/urfave/cli/v2" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + dbPathFlag = &cli.StringFlag{ + Name: "db-path", + Usage: "Path to database", + Required: true, + } + dbCacheFlag = &cli.IntFlag{ + Name: "db-cache", + Usage: "LevelDB cache size in mb", + Value: 1024, + } + dbHandlesFlag = &cli.IntFlag{ + Name: "db-handles", + Usage: "LevelDB number of handles", + Value: 60, + } + dryRunFlag = &cli.BoolFlag{ + Name: "dry-run", + Usage: "Dry run the upgrade by not committing the database", + } + + flags = []cli.Flag{ + dbPathFlag, + dbCacheFlag, + dbHandlesFlag, + dryRunFlag, + } + + // from `packages/contracts-bedrock/deploy-config/internal-devnet.json` + EIP1559Denominator = uint64(50) // TODO(pl): select values + EIP1559Elasticity = uint64(10) +) + +var app = &cli.App{ + Name: "migrate", + Usage: "Migrate Celo state to a CeL2 DB", + Flags: flags, + Action: appMain, +} + +func main() { + log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd())))) + if err := app.Run(os.Args); err != nil { + log.Crit("error", "err", err) + } +} + +func appMain(ctx *cli.Context) error { + // Write changes to state to actual state database + dbPath := ctx.String("db-path") + if dbPath == "" { + return fmt.Errorf("must specify --db-path") + } + dbCache := ctx.Int("db-cache") + dbHandles := ctx.Int("db-handles") + // dryRun := ctx.Bool("dry-run") + + log.Info("Opening database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) + ldb, err := openCeloDb(dbPath, dbCache, dbHandles) + if err != nil { + return fmt.Errorf("cannot open DB: %w", err) + } + log.Info("Loaded Celo L1 DB", "db", ldb) + + printStats(ldb) + + findFirstCorruptedHeader(ldb) + + // // Read last block before gingerbread (alfajores) + // header, err := ReadCanonicalHeader(ldb, 19814000-1) + // if err != nil { + // return fmt.Errorf("cannot read header: %w", err) + // } + // log.Info("Read header", "header", header) + + return nil +} + +// Opens a Celo database, stored in the `celo` subfolder +func openCeloDb(path string, cache int, handles int) (ethdb.Database, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return nil, err + } + + ancientPath := filepath.Join(path, "ancient") + ldb, err := rawdb.Open(rawdb.OpenOptions{ + Type: "leveldb", + Directory: path, + AncientsDirectory: ancientPath, + Namespace: "", + Cache: cache, + Handles: handles, + ReadOnly: false, + }) + if err != nil { + return nil, err + } + return ldb, nil +} + +// print stats about the database +func printStats(ldb ethdb.Database) { + // Print some stats about the database + chainMetaData := rawdb.ReadChainMetadata(ldb) + for _, v := range chainMetaData { + if len(v) == 2 { + log.Info("Database Metadata", v[0], v[1]) + } else { + log.Info("Database Metadata", v[0], v[1:]) + } + } +} + +func canLoadHeader(ldb ethdb.Database, number uint64) bool { + // log.Trace("Checking if header can be loaded", "number", number) + _, err := ReadCanonicalHeader(ldb, number) + // if err != nil { + // log.Trace("failed to load header", "number", number, "error", err) + // } + return err == nil +} + +// does a binary search to find the first header that fails to load +func findFirstCorruptedHeader(ldb ethdb.Database) { + // Grab the hash of the tip of the legacy chain. + hash := rawdb.ReadHeadHeaderHash(ldb) + lastBlockNumber := *rawdb.ReadHeaderNumber(ldb, hash) + + log.Info("Starting from HEAD of then chain", "number", lastBlockNumber) + + if !canLoadHeader(ldb, lastBlockNumber) { + log.Error("Can't fetch the last block header, something is wrong") + return + } + + // Binary search from 1 to LastBlockNumber + low := uint64(1) + high := lastBlockNumber + + for low <= high { + mid := (low + high) / 2 + + // Call the test condition function to check if the header can be loaded + if !canLoadHeader(ldb, mid) { + low = mid + 1 + } else { + high = mid - 1 + } + } + + log.Info("Search Finished", "lastBlockThatLoads", high+1, "firstBlockThatFails", high) +} + +// prints the hash of the last x blocks +func printLastBlocks(ldb ethdb.Database, x uint64) { + + hash := rawdb.ReadHeadHeaderHash(ldb) + lastBlockNumber := *rawdb.ReadHeaderNumber(ldb, hash) + toBlockNumber := lastBlockNumber - x + log.Debug("Iterating over blocks", "from", lastBlockNumber, "to", toBlockNumber) + log.Info("Block", "number", lastBlockNumber, "hash", hash) + + for i := lastBlockNumber; i > toBlockNumber; i-- { + header := rawdb.ReadHeader(ldb, hash, i) + log.Info("Block", "number", header.Number, "hash", header.Hash()) + hash = header.ParentHash + } +} + +func ReadCanonicalHeader(db ethdb.Reader, number uint64) (*types.Header, error) { + hash := rawdb.ReadCanonicalHash(db, number) + checkNumber := rawdb.ReadHeaderNumber(db, hash) + if checkNumber == nil || *checkNumber != number { + return nil, errors.New("something is bad") + } + return ReadHeader(db, hash, number) +} + +// ReadHeader retrieves the block header corresponding to the hash. +func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header, error) { + data := rawdb.ReadHeaderRLP(db, hash, number) + if len(data) == 0 { + return nil, errors.New("header not found") + } + header := new(types.Header) + if err := rlp.DecodeBytes(data, header); err != nil { + return nil, err + } + return header, nil +} + +// func debugReadAncientHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header, error) { +// db.ReadAncients(func(reader ethdb.AncientReaderOp) error { +// data, err := reader.Ancient(rawdb.ChainFreezerHeaderTable, number) +// if err != nil { +// log.Error("Failed to read from ancient database", "err", err) +// return err +// } +// if len(data) > 0 && crypto.Keccak256Hash(data) == hash { +// return nil +// } +// data, err = db.Get(headerKey(number, hash)) +// if err != nil { +// log.Error("Failed to read from leveldb", "err", err) +// return err +// } +// return nil +// }) +// } From e9ba2bd6f4cb1783ef1fb5140432f77e34685014 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Mon, 29 Apr 2024 17:20:36 -0300 Subject: [PATCH 02/30] Can Read All the headers Co-authored-by: Alec Schaefer --- go.mod | 4 +- go.sum | 2 - op-chain-ops/celo1/block.go | 377 ++++++++++++++++++++++ op-chain-ops/celo1/block_compatibility.go | 129 ++++++++ op-chain-ops/celo1/hashing.go | 113 +++++++ op-chain-ops/cmd/celo-dbplay/main.go | 117 ++++--- 6 files changed, 701 insertions(+), 41 deletions(-) create mode 100644 op-chain-ops/celo1/block.go create mode 100644 op-chain-ops/celo1/block_compatibility.go create mode 100644 op-chain-ops/celo1/hashing.go diff --git a/go.mod b/go.mod index 65cb929249bae..2ec9ebbe3586d 100644 --- a/go.mod +++ b/go.mod @@ -219,7 +219,7 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571 +//replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571 //replace github.com/ethereum-optimism/superchain-registry/superchain => ../superchain-registry/superchain -//replace github.com/ethereum/go-ethereum v1.13.5 => ../go-ethereum +replace github.com/ethereum/go-ethereum => ../op-geth diff --git a/go.sum b/go.sum index 5d3bc1ef7f642..3570de8274cb0 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,6 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571 h1:U+6eOYz2hEun/zm+bGCF9ntKx6ZzxBW9lojOJ4ufybI= -github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571/go.mod h1:dQVCa+D5zi0oJJyrP3nwWciio0BJLZ2urH8OU7RJ2qs= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/op-chain-ops/celo1/block.go b/op-chain-ops/celo1/block.go new file mode 100644 index 0000000000000..3d4d0d5f27134 --- /dev/null +++ b/op-chain-ops/celo1/block.go @@ -0,0 +1,377 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package types contains data types related to Ethereum consensus. +package celo1 + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "reflect" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + EmptyUncleHash = rlpHash([]*Header(nil)) // 1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347 + EmptyMixDigest = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000000") +) + +// A BlockNonce is a 64-bit hash which proves (combined with the +// mix-hash) that a sufficient amount of computation has been carried +// out on a block. +type BlockNonce [8]byte + +// EncodeNonce converts the given integer to a block nonce. +func EncodeNonce(i uint64) BlockNonce { + var n BlockNonce + binary.BigEndian.PutUint64(n[:], i) + return n +} + +// Uint64 returns the integer value of a block nonce. +func (n BlockNonce) Uint64() uint64 { + return binary.BigEndian.Uint64(n[:]) +} + +// MarshalText encodes n as a hex string with 0x prefix. +func (n BlockNonce) MarshalText() ([]byte, error) { + return hexutil.Bytes(n[:]).MarshalText() +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (n *BlockNonce) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) +} + +//go:generate gencodec -type Header -field-override headerMarshaling -out gen_header_json.go + +// Header represents a block header in the Ethereum blockchain. +type Header struct { + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + UncleHash common.Hash `json:"sha3Uncles"` + Coinbase common.Address `json:"miner" gencodec:"required"` + Root common.Hash `json:"stateRoot" gencodec:"required"` + TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` + ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` + Bloom types.Bloom `json:"logsBloom" gencodec:"required"` + Difficulty *big.Int `json:"difficulty"` + Number *big.Int `json:"number" gencodec:"required"` + GasLimit uint64 `json:"gasLimit"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Time uint64 `json:"timestamp" gencodec:"required"` + Extra []byte `json:"extraData" gencodec:"required"` + MixDigest common.Hash `json:"mixHash"` + Nonce BlockNonce `json:"nonce"` + + // BaseFee was added by EIP-1559 and is ignored in legacy headers. + BaseFee *big.Int `json:"baseFeePerGas" rlp:"optional"` + + // Used to cache deserialized istanbul extra data + extraLock sync.Mutex + extraError error +} + +// field type overrides for gencodec +type headerMarshaling struct { + Difficulty *hexutil.Big + Number *hexutil.Big + GasLimit hexutil.Uint64 + GasUsed hexutil.Uint64 + Time hexutil.Uint64 + Extra hexutil.Bytes + BaseFee *hexutil.Big + Hash common.Hash `json:"hash"` // adds call to Hash() in MarshalJSON +} + +// Hash returns the block hash of the header, which is simply the keccak256 hash of its +// RLP encoding. +func (h *Header) Hash() common.Hash { + // // Seal is reserved in extra-data. To prove block is signed by the proposer. + // if len(h.Extra) >= IstanbulExtraVanity { + // // This branch is always used during normal Celo operation, but not in all tests. + // if istanbulHeader := IstanbulFilteredHeader(h, true); istanbulHeader != nil { + // return rlpHash(istanbulHeader) + // } + // } + return rlpHash(h) +} + +var headerSize = common.StorageSize(reflect.TypeOf(Header{}).Size()) + +// Size returns the approximate memory used by all internal contents. It is used +// to approximate and limit the memory consumption of various caches. +func (h *Header) Size() common.StorageSize { + return headerSize + common.StorageSize(len(h.Extra)+(h.Number.BitLen()/8)) +} + +// SanityCheck checks a few basic things -- these checks are way beyond what +// any 'sane' production values should hold, and can mainly be used to prevent +// that the unbounded fields are stuffed with junk data to add processing +// overhead +func (h *Header) SanityCheck() error { + if h.Number != nil && !h.Number.IsUint64() { + return fmt.Errorf("too large block number: bitlen %d", h.Number.BitLen()) + } + if h.Difficulty != nil { + if diffLen := h.Difficulty.BitLen(); diffLen > 80 { + return fmt.Errorf("too large block difficulty: bitlen %d", diffLen) + } + } + if eLen := len(h.Extra); eLen > 100*1024 { + return fmt.Errorf("too large block extradata: size %d", eLen) + } + if h.BaseFee != nil { + if bfLen := h.BaseFee.BitLen(); bfLen > 256 { + return fmt.Errorf("too large base fee: bitlen %d", bfLen) + } + } + return nil +} + +// Body is a simple (mutable, non-safe) data container for storing and moving +// a block's data contents (transactions and uncles) together. +type Body struct { + Transactions []*types.Transaction + Randomness []byte + EpochSnarkData []byte +} + +// Block represents an entire block in the Ethereum blockchain. +type Block struct { + header *Header + transactions types.Transactions + randomness []byte + epochSnarkData []byte + + // caches + hash atomic.Value + size atomic.Value + + // Td is used by package core to store the total difficulty + // of the chain up to and including the block. + td *big.Int + + // These fields are used by package eth to track + // inter-peer block relay. + ReceivedAt time.Time + ReceivedFrom interface{} +} + +// "external" block encoding. used for eth protocol, etc. +type extblock struct { + Header *Header + Txs []*types.Transaction + Randomness []byte + EpochSnarkData []byte +} + +// NewBlock creates a new block. The input data is copied, +// changes to header and to the field values will not affect the +// block. +// +// The values of TxHash, ReceiptHash and Bloom in header +// are ignored and set to values derived from the given txs and receipts. +func NewBlock(header *Header, txs []*types.Transaction, receipts []*types.Receipt, hasher TrieHasher) *Block { + b := &Block{header: CopyHeader(header), td: new(big.Int)} + + // TODO: panic if len(txs) != len(receipts) + if len(txs) == 0 { + b.header.TxHash = EmptyRootHash + } else { + b.header.TxHash = DeriveSha(types.Transactions(txs), hasher) + b.transactions = make(types.Transactions, len(txs)) + copy(b.transactions, txs) + } + + if len(receipts) == 0 { + b.header.ReceiptHash = EmptyRootHash + } else { + b.header.ReceiptHash = DeriveSha(types.Receipts(receipts), hasher) + b.header.Bloom = types.CreateBloom(receipts) + } + + // if randomness == nil { + // b.randomness = &EmptyRandomness + // } + + return b +} + +// NewBlockWithHeader creates a block with the given header data. The +// header data is copied, changes to header and to the field values +// will not affect the block. +func NewBlockWithHeader(header *Header) *Block { + return &Block{header: CopyHeader(header), randomness: nil, epochSnarkData: nil} +} + +// CopyHeader creates a deep copy of a block header to prevent side effects from +// modifying a header variable. +func CopyHeader(h *Header) *Header { + cpy := Header{ + ParentHash: h.ParentHash, + UncleHash: h.UncleHash, + Coinbase: h.Coinbase, + Root: h.Root, + TxHash: h.TxHash, + ReceiptHash: h.ReceiptHash, + Bloom: h.Bloom, + Difficulty: h.Difficulty, + Number: new(big.Int), + GasLimit: h.GasLimit, + GasUsed: h.GasUsed, + Time: h.Time, + MixDigest: h.MixDigest, + Nonce: h.Nonce, + } + + if h.Number != nil { + cpy.Number.Set(h.Number) + } + if h.BaseFee != nil { + cpy.BaseFee = new(big.Int).Set(h.BaseFee) + } + if len(h.Extra) > 0 { + cpy.Extra = make([]byte, len(h.Extra)) + copy(cpy.Extra, h.Extra) + } + return &cpy +} + +// DecodeRLP decodes the Ethereum +func (b *Block) DecodeRLP(s *rlp.Stream) error { + var eb extblock + _, size, _ := s.Kind() + if err := s.Decode(&eb); err != nil { + return err + } + b.header, b.transactions, b.randomness, b.epochSnarkData = eb.Header, eb.Txs, eb.Randomness, eb.EpochSnarkData + b.size.Store(common.StorageSize(rlp.ListSize(size))) + return nil +} + +// EncodeRLP serializes b into the Ethereum RLP block format. +func (b *Block) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, extblock{ + Header: b.header, + Txs: b.transactions, + Randomness: b.randomness, + EpochSnarkData: b.epochSnarkData, + }) +} + +// TODO: copies + +func (b *Block) Transactions() types.Transactions { return b.transactions } + +func (b *Block) Transaction(hash common.Hash) *types.Transaction { + for _, transaction := range b.transactions { + if transaction.Hash() == hash { + return transaction + } + } + return nil +} + +func (b *Block) Number() *big.Int { return new(big.Int).Set(b.header.Number) } +func (b *Block) GasLimit() uint64 { return b.header.GasLimit } +func (b *Block) GasUsed() uint64 { return b.header.GasUsed } +func (b *Block) Difficulty() *big.Int { return new(big.Int).Set(b.header.Difficulty) } +func (b *Block) Time() uint64 { return b.header.Time } +func (b *Block) TotalDifficulty() *big.Int { return new(big.Int).Add(b.header.Number, big.NewInt(1)) } + +func (b *Block) NumberU64() uint64 { return b.header.Number.Uint64() } +func (b *Block) MixDigest() common.Hash { return b.header.MixDigest } +func (b *Block) Nonce() uint64 { return binary.BigEndian.Uint64(b.header.Nonce[:]) } +func (b *Block) Bloom() types.Bloom { return b.header.Bloom } +func (b *Block) Coinbase() common.Address { return b.header.Coinbase } +func (b *Block) Root() common.Hash { return b.header.Root } +func (b *Block) ParentHash() common.Hash { return b.header.ParentHash } +func (b *Block) TxHash() common.Hash { return b.header.TxHash } +func (b *Block) ReceiptHash() common.Hash { return b.header.ReceiptHash } +func (b *Block) UncleHash() common.Hash { return b.header.UncleHash } +func (b *Block) Extra() []byte { return common.CopyBytes(b.header.Extra) } + +func (b *Block) BaseFee() *big.Int { + if b.header.BaseFee == nil { + return nil + } + return new(big.Int).Set(b.header.BaseFee) +} + +func (b *Block) Header() *Header { return CopyHeader(b.header) } +func (b *Block) MutableHeader() *Header { return b.header } + +// Body returns the non-header content of the block. +func (b *Block) Body() *Body { return &Body{b.transactions, b.randomness, b.epochSnarkData} } + +// Size returns the true RLP encoded storage size of the block, either by encoding +// and returning it, or returning a previsouly cached value. +func (b *Block) Size() common.StorageSize { + if size := b.size.Load(); size != nil { + return size.(common.StorageSize) + } + c := writeCounter(0) + rlp.Encode(&c, b) + b.size.Store(common.StorageSize(c)) + return common.StorageSize(c) +} + +// SanityCheck can be used to prevent that unbounded fields are +// stuffed with junk data to add processing overhead +func (b *Block) SanityCheck() error { + return b.header.SanityCheck() +} + +type writeCounter common.StorageSize + +func (c *writeCounter) Write(b []byte) (int, error) { + *c += writeCounter(len(b)) + return len(b), nil +} + +// WithHeader returns a new block with the data from b but the header replaced with +// the sealed one. +func (b *Block) WithHeader(header *Header) *Block { + cpy := CopyHeader(header) + + return &Block{ + header: cpy, + transactions: b.transactions, + randomness: b.randomness, + epochSnarkData: b.epochSnarkData, + } +} + +// Hash returns the keccak256 hash of b's header. +// The hash is computed on the first call and cached thereafter. +func (b *Block) Hash() common.Hash { + if hash := b.hash.Load(); hash != nil { + return hash.(common.Hash) + } + v := b.header.Hash() + b.hash.Store(v) + return v +} diff --git a/op-chain-ops/celo1/block_compatibility.go b/op-chain-ops/celo1/block_compatibility.go new file mode 100644 index 0000000000000..80dfb89bdcf47 --- /dev/null +++ b/op-chain-ops/celo1/block_compatibility.go @@ -0,0 +1,129 @@ +package celo1 + +import ( + "io" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// This file takes care of supporting older block header formats from before +// the gingerbread fork. + +type beforeGingerbreadHeader struct { + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + Coinbase common.Address `json:"miner" gencodec:"required"` + Root common.Hash `json:"stateRoot" gencodec:"required"` + TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` + ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` + Bloom types.Bloom `json:"logsBloom" gencodec:"required"` + Number *big.Int `json:"number" gencodec:"required"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Time uint64 `json:"timestamp" gencodec:"required"` + Extra []byte `json:"extraData" gencodec:"required"` + + // Used to cache deserialized istanbul extra data + extraLock sync.Mutex + extraError error +} + +type afterGingerbreadHeader Header + +func (h *Header) DecodeRLP(s *rlp.Stream) error { + _, size, _ := s.Kind() + var raw rlp.RawValue + err := s.Decode(&raw) + if err != nil { + return err + } + headerSize := len(raw) - int(size) + numElems, err := rlp.CountValues(raw[headerSize:]) + if err != nil { + return err + } + if numElems == 10 { + // Before gingerbread + decodedHeader := beforeGingerbreadHeader{} + err = rlp.DecodeBytes(raw, &decodedHeader) + + h.ParentHash = decodedHeader.ParentHash + h.Coinbase = decodedHeader.Coinbase + h.Root = decodedHeader.Root + h.TxHash = decodedHeader.TxHash + h.ReceiptHash = decodedHeader.ReceiptHash + h.Bloom = decodedHeader.Bloom + h.Number = decodedHeader.Number + h.GasUsed = decodedHeader.GasUsed + h.Time = decodedHeader.Time + h.Extra = decodedHeader.Extra + } else { + // After gingerbread + decodedHeader := afterGingerbreadHeader{} + err = rlp.DecodeBytes(raw, &decodedHeader) + + h.ParentHash = decodedHeader.ParentHash + h.UncleHash = decodedHeader.UncleHash + h.Coinbase = decodedHeader.Coinbase + h.Root = decodedHeader.Root + h.TxHash = decodedHeader.TxHash + h.ReceiptHash = decodedHeader.ReceiptHash + h.Bloom = decodedHeader.Bloom + h.Difficulty = decodedHeader.Difficulty + h.Number = decodedHeader.Number + h.GasLimit = decodedHeader.GasLimit + h.GasUsed = decodedHeader.GasUsed + h.Time = decodedHeader.Time + h.Extra = decodedHeader.Extra + h.MixDigest = decodedHeader.MixDigest + h.Nonce = decodedHeader.Nonce + h.BaseFee = decodedHeader.BaseFee + } + + return err +} + +func (h *Header) EncodeRLP(w io.Writer) error { + if (h.UncleHash == common.Hash{}) { + // Before gingerbread hardfork Celo did not include all of + // Ethereum's header fields. In that case we must omit the new + // fields from the header when encoding as RLP to maintain the same encoding and hashes. + // `UncleHash` is a safe way to check, since it is the zero hash before + // gingerbread and non-zero after. + rlpFields := []interface{}{ + h.ParentHash, + h.Coinbase, + h.Root, + h.TxHash, + h.ReceiptHash, + h.Bloom, + h.Number, + h.GasUsed, + h.Time, + h.Extra, + } + return rlp.Encode(w, rlpFields) + } else { + rlpFields := []interface{}{ + h.ParentHash, + h.UncleHash, + h.Coinbase, + h.Root, + h.TxHash, + h.ReceiptHash, + h.Bloom, + h.Difficulty, + h.Number, + h.GasLimit, + h.GasUsed, + h.Time, + h.Extra, + h.MixDigest, + h.Nonce, + h.BaseFee, + } + return rlp.Encode(w, rlpFields) + } +} diff --git a/op-chain-ops/celo1/hashing.go b/op-chain-ops/celo1/hashing.go new file mode 100644 index 0000000000000..153f0f56add6a --- /dev/null +++ b/op-chain-ops/celo1/hashing.go @@ -0,0 +1,113 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package celo1 + +import ( + "bytes" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/crypto/sha3" +) + +// hasherPool holds LegacyKeccak256 hashers for rlpHash. +var hasherPool = sync.Pool{ + New: func() interface{} { return sha3.NewLegacyKeccak256() }, +} + +// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. +var encodeBufferPool = sync.Pool{ + New: func() interface{} { return new(bytes.Buffer) }, +} + +// rlpHash encodes x and hashes the encoded bytes. +func rlpHash(x interface{}) (h common.Hash) { + sha := hasherPool.Get().(crypto.KeccakState) + defer hasherPool.Put(sha) + sha.Reset() + rlp.Encode(sha, x) + sha.Read(h[:]) + return h +} + +// prefixedRlpHash writes the prefix into the hasher before rlp-encoding x. +// It's used for typed transactions. +func prefixedRlpHash(prefix byte, x interface{}) (h common.Hash) { + sha := hasherPool.Get().(crypto.KeccakState) + defer hasherPool.Put(sha) + sha.Reset() + sha.Write([]byte{prefix}) + rlp.Encode(sha, x) + sha.Read(h[:]) + return h +} + +// TrieHasher is the tool used to calculate the hash of derivable list. +// This is internal, do not use. +type TrieHasher interface { + Reset() + Update([]byte, []byte) + Hash() common.Hash +} + +// DerivableList is the input to DeriveSha. +// It is implemented by the 'Transactions' and 'Receipts' types. +// This is internal, do not use these methods. +type DerivableList interface { + Len() int + EncodeIndex(int, *bytes.Buffer) +} + +func encodeForDerive(list DerivableList, i int, buf *bytes.Buffer) []byte { + buf.Reset() + list.EncodeIndex(i, buf) + // It's really unfortunate that we need to do perform this copy. + // StackTrie holds onto the values until Hash is called, so the values + // written to it must not alias. + return common.CopyBytes(buf.Bytes()) +} + +// DeriveSha creates the tree hashes of transactions and receipts in a block header. +func DeriveSha(list DerivableList, hasher TrieHasher) common.Hash { + hasher.Reset() + + valueBuf := encodeBufferPool.Get().(*bytes.Buffer) + defer encodeBufferPool.Put(valueBuf) + + // StackTrie requires values to be inserted in increasing hash order, which is not the + // order that `list` provides hashes in. This insertion sequence ensures that the + // order is correct. + var indexBuf []byte + for i := 1; i < list.Len() && i <= 0x7f; i++ { + indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i)) + value := encodeForDerive(list, i, valueBuf) + hasher.Update(indexBuf, value) + } + if list.Len() > 0 { + indexBuf = rlp.AppendUint64(indexBuf[:0], 0) + value := encodeForDerive(list, 0, valueBuf) + hasher.Update(indexBuf, value) + } + for i := 0x80; i < list.Len(); i++ { + indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i)) + value := encodeForDerive(list, i, valueBuf) + hasher.Update(indexBuf, value) + } + return hasher.Hash() +} diff --git a/op-chain-ops/cmd/celo-dbplay/main.go b/op-chain-ops/cmd/celo-dbplay/main.go index e40d1457dac1b..b7e54141a5529 100644 --- a/op-chain-ops/cmd/celo-dbplay/main.go +++ b/op-chain-ops/cmd/celo-dbplay/main.go @@ -1,11 +1,13 @@ package main import ( + "encoding/binary" "errors" "fmt" "os" "path/filepath" + "github.com/ethereum-optimism/optimism/op-chain-ops/celo1" "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" @@ -82,20 +84,30 @@ func appMain(ctx *cli.Context) error { } log.Info("Loaded Celo L1 DB", "db", ldb) - printStats(ldb) + // printStats(ldb) findFirstCorruptedHeader(ldb) - // // Read last block before gingerbread (alfajores) - // header, err := ReadCanonicalHeader(ldb, 19814000-1) - // if err != nil { - // return fmt.Errorf("cannot read header: %w", err) - // } - // log.Info("Read header", "header", header) + // tryHeader(ldb, 19814000-1) //just before gingerbread activation (alfajores) + // tryHeader(ldb, 19814000) // just after gingerbread activation (alfajores) + // tryHeader(ldb, 4960000-1) // before churrito block + // tryHeader(ldb, 4960000) // churrito block + // tryHeader(ldb, 4960000) // donut block + // tryHeader(ldb, 9472000) // espresso block return nil } +func tryHeader(ldb ethdb.Database, number uint64) { + header, err := ReadCeloCanonicalHeader(ldb, number) + if err != nil { + log.Error("failed to load header", "number", number, "error", err) + } else { + log.Info("loaded header", "number", number, "hash", header.Hash()) + log.Info("Read header", "header", header) + } +} + // Opens a Celo database, stored in the `celo` subfolder func openCeloDb(path string, cache int, handles int) (ethdb.Database, error) { if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { @@ -133,10 +145,10 @@ func printStats(ldb ethdb.Database) { func canLoadHeader(ldb ethdb.Database, number uint64) bool { // log.Trace("Checking if header can be loaded", "number", number) - _, err := ReadCanonicalHeader(ldb, number) - // if err != nil { - // log.Trace("failed to load header", "number", number, "error", err) - // } + _, err := ReadCeloCanonicalHeader(ldb, number) + if err != nil { + log.Debug("failed to load header", "number", number, "error", err) + } return err == nil } @@ -187,18 +199,56 @@ func printLastBlocks(ldb ethdb.Database, x uint64) { } } -func ReadCanonicalHeader(db ethdb.Reader, number uint64) (*types.Header, error) { - hash := rawdb.ReadCanonicalHash(db, number) - checkNumber := rawdb.ReadHeaderNumber(db, hash) - if checkNumber == nil || *checkNumber != number { - return nil, errors.New("something is bad") +// ReadHeaderRLP retrieves a block header in its raw RLP database encoding. (copied from rawdb) +func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + var data []byte + db.ReadAncients(func(reader ethdb.AncientReaderOp) error { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. + data, _ = reader.Ancient(rawdb.ChainFreezerHeaderTable, number) + if len(data) != 0 { + return nil + } + // If not, try reading from leveldb + data, _ = db.Get(headerKey(number, hash)) + return nil + }) + return data +} + +var ( + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header +) + +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} + +// headerKey = headerPrefix + num (uint64 big endian) + hash +func headerKey(number uint64, hash common.Hash) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +func ReadCeloHeader(db ethdb.Reader, hash common.Hash, number uint64) (*celo1.Header, error) { + data := ReadHeaderRLP(db, hash, number) + if len(data) == 0 { + return nil, errors.New("header not found") } - return ReadHeader(db, hash, number) + + header := new(celo1.Header) + if err := rlp.DecodeBytes(data, header); err != nil { + return nil, err + } + return header, nil } -// ReadHeader retrieves the block header corresponding to the hash. +// ReadHeader retrieves the block header corresponding to the hash. (copied from rawdb) func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header, error) { - data := rawdb.ReadHeaderRLP(db, hash, number) + data := ReadHeaderRLP(db, hash, number) if len(data) == 0 { return nil, errors.New("header not found") } @@ -209,21 +259,14 @@ func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header return header, nil } -// func debugReadAncientHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header, error) { -// db.ReadAncients(func(reader ethdb.AncientReaderOp) error { -// data, err := reader.Ancient(rawdb.ChainFreezerHeaderTable, number) -// if err != nil { -// log.Error("Failed to read from ancient database", "err", err) -// return err -// } -// if len(data) > 0 && crypto.Keccak256Hash(data) == hash { -// return nil -// } -// data, err = db.Get(headerKey(number, hash)) -// if err != nil { -// log.Error("Failed to read from leveldb", "err", err) -// return err -// } -// return nil -// }) -// } +// ReadCanonicalHeader retrieves the cannoical block header at the given number. +func ReadCanonicalHeader(db ethdb.Reader, number uint64) (*types.Header, error) { + hash := rawdb.ReadCanonicalHash(db, number) + return ReadHeader(db, hash, number) +} + +// ReadCanonicalHeader retrieves the cannoical block header at the given number. +func ReadCeloCanonicalHeader(db ethdb.Reader, number uint64) (*celo1.Header, error) { + hash := rawdb.ReadCanonicalHash(db, number) + return ReadCeloHeader(db, hash, number) +} From ca8512e03e060c35793694c16f640829c77fbb53 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 30 Apr 2024 10:06:55 -0300 Subject: [PATCH 03/30] Adds new command to migrate ancients db --- op-chain-ops/celo1/db.go | 84 ++++++++++++ op-chain-ops/cmd/celo-dbmigrate/main.go | 164 ++++++++++++++++++++++++ op-chain-ops/cmd/celo-dbplay/main.go | 83 +----------- 3 files changed, 252 insertions(+), 79 deletions(-) create mode 100644 op-chain-ops/celo1/db.go create mode 100644 op-chain-ops/cmd/celo-dbmigrate/main.go diff --git a/op-chain-ops/celo1/db.go b/op-chain-ops/celo1/db.go new file mode 100644 index 0000000000000..dc5943efd37f5 --- /dev/null +++ b/op-chain-ops/celo1/db.go @@ -0,0 +1,84 @@ +package celo1 + +import ( + "encoding/binary" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" +) + +// ReadHeaderRLP retrieves a block header in its raw RLP database encoding. (copied from rawdb) +func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + var data []byte + db.ReadAncients(func(reader ethdb.AncientReaderOp) error { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. + data, _ = reader.Ancient(rawdb.ChainFreezerHeaderTable, number) + if len(data) != 0 { + return nil + } + // If not, try reading from leveldb + data, _ = db.Get(headerKey(number, hash)) + return nil + }) + return data +} + +var ( + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header +) + +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} + +// headerKey = headerPrefix + num (uint64 big endian) + hash +func headerKey(number uint64, hash common.Hash) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +func ReadCeloHeader(db ethdb.Reader, hash common.Hash, number uint64) (*Header, error) { + data := ReadHeaderRLP(db, hash, number) + if len(data) == 0 { + return nil, errors.New("header not found") + } + + header := new(Header) + if err := rlp.DecodeBytes(data, header); err != nil { + return nil, err + } + return header, nil +} + +// ReadHeader retrieves the block header corresponding to the hash. (copied from rawdb) +func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header, error) { + data := ReadHeaderRLP(db, hash, number) + if len(data) == 0 { + return nil, errors.New("header not found") + } + header := new(types.Header) + if err := rlp.DecodeBytes(data, header); err != nil { + return nil, err + } + return header, nil +} + +// ReadCanonicalHeader retrieves the cannoical block header at the given number. +func ReadCanonicalHeader(db ethdb.Reader, number uint64) (*types.Header, error) { + hash := rawdb.ReadCanonicalHash(db, number) + return ReadHeader(db, hash, number) +} + +// ReadCanonicalHeader retrieves the cannoical block header at the given number. +func ReadCeloCanonicalHeader(db ethdb.Reader, number uint64) (*Header, error) { + hash := rawdb.ReadCanonicalHash(db, number) + return ReadCeloHeader(db, hash, number) +} diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go new file mode 100644 index 0000000000000..6b77303b70201 --- /dev/null +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -0,0 +1,164 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/mattn/go-isatty" +) + +const ( + dbCache = 1024 // size of the cache in MB + dbHandles = 60 // number of handles + ancientHashSize = 38 // size of a hash entry on ancient database (empiric value) +) + +func main() { + log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd())))) + + oldDBPath := flag.String("oldDB", "", "Path to the old database") + newDBPath := flag.String("newDB", "", "Path to the new database") + resetDB := flag.Bool("resetDB", false, "Use to reset the new database before migrating data") + batchSize := flag.Uint64("batchSize", 1000, "Number of ancient records to migrate in one batch") + + flag.Parse() + + if *oldDBPath == "" || *newDBPath == "" { + log.Info("Please provide both oldDB and newDB flags") + return + } + + if *resetDB { + if err := os.RemoveAll(*newDBPath); err != nil { + log.Crit("Failed to remove new database", "err", err) + } + } + + // Open the existing database in read-only mode + oldDB, err := openCeloDb(*oldDBPath, dbCache, dbHandles, true) + if err != nil { + log.Crit("Failed to open old database", "err", err) + } + + // Create a new database + newDB, err := openCeloDb(*newDBPath, dbCache, dbHandles, false) + if err != nil { + log.Crit("Failed to create new database", "err", err) + } + + // Close the databases + defer oldDB.Close() + defer newDB.Close() + + toMigrateRecords := MustAncientLength(oldDB) + migratedRecords := MustAncientLength(newDB) + log.Info("Ancient Migration Initial Status", "migrated", migratedRecords, "total", toMigrateRecords) + + if err = freezeRange(oldDB, newDB, migratedRecords, *batchSize); err != nil { + log.Crit("Failed to freeze range", "err", err) + } + + log.Info("Ancient Migration End Status", "migrated", MustAncientLength(newDB), "total", toMigrateRecords) + +} + +// Opens a Celo database, stored in the `celo` subfolder +func openCeloDb(path string, cache int, handles int, readonly bool) (ethdb.Database, error) { + ancientPath := filepath.Join(path, "ancient") + ldb, err := rawdb.Open(rawdb.OpenOptions{ + Type: "leveldb", + Directory: path, + AncientsDirectory: ancientPath, + Namespace: "", + Cache: cache, + Handles: handles, + ReadOnly: readonly, + }) + if err != nil { + return nil, err + } + return ldb, nil +} + +func freezeRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { + _, err := newDb.ModifyAncients(func(op ethdb.AncientWriteOp) error { + hashes, err := oldDb.AncientRange(rawdb.ChainFreezerHashTable, number, count, 0) + if err != nil { + return fmt.Errorf("can't read hashes from old freezer: %v", err) + } + headers, err := oldDb.AncientRange(rawdb.ChainFreezerHeaderTable, number, count, 0) + if err != nil { + return fmt.Errorf("can't read headers from old freezer: %v", err) + } + bodies, err := oldDb.AncientRange(rawdb.ChainFreezerBodiesTable, number, count, 0) + if err != nil { + return fmt.Errorf("can't read bodies from old freezer: %v", err) + } + receipts, err := oldDb.AncientRange(rawdb.ChainFreezerReceiptTable, number, count, 0) + if err != nil { + return fmt.Errorf("can't read receipts from old freezer: %v", err) + } + tds, err := oldDb.AncientRange(rawdb.ChainFreezerDifficultyTable, number, count, 0) + if err != nil { + return fmt.Errorf("can't read tds from old freezer: %v", err) + } + + if len(hashes) != len(headers) || len(headers) != len(bodies) || len(bodies) != len(receipts) || len(receipts) != len(tds) { + return fmt.Errorf("inconsistent data in ancient tables") + } + + for i := 0; i < len(hashes); i++ { + log.Debug("Migrating ancient data", "number", number+uint64(i)) + + // TODO: perform encoding migration here + + if err := op.AppendRaw(rawdb.ChainFreezerHashTable, number+uint64(i), hashes[i]); err != nil { + return fmt.Errorf("can't write hash to Freezer: %v", err) + } + if err := op.AppendRaw(rawdb.ChainFreezerHeaderTable, number+uint64(i), headers[i]); err != nil { + return fmt.Errorf("can't write header to Freezer: %v", err) + } + if err := op.AppendRaw(rawdb.ChainFreezerBodiesTable, number+uint64(i), bodies[i]); err != nil { + return fmt.Errorf("can't write body to Freezer: %v", err) + } + if err := op.AppendRaw(rawdb.ChainFreezerReceiptTable, number+uint64(i), receipts[i]); err != nil { + return fmt.Errorf("can't write receipts to Freezer: %v", err) + } + if err := op.AppendRaw(rawdb.ChainFreezerDifficultyTable, number+uint64(i), tds[i]); err != nil { + return fmt.Errorf("can't write td to Freezer: %v", err) + } + } + return nil + }) + + return err +} + +func MustAncientLength(db ethdb.Database) uint64 { + byteSize, err := db.AncientSize(rawdb.ChainFreezerHashTable) + if err != nil { + log.Crit("Failed to get ancient size", "error", err) + } + return byteSize / ancientHashSize +} + +// finds number of items in the ancients database +func findAncientsSize(ldb ethdb.Database, high uint64) uint64 { + // note: AncientSize returns a byte size, not number of items. Maybe for a well known size we can compute the number of items + // runs a binary search using Ancient.HasAncient to find the first hash it can't find + low := uint64(0) + for low < high { + mid := (low + high) / 2 + if ok, err := ldb.HasAncient(rawdb.ChainFreezerHashTable, mid); ok && err == nil { + low = mid + 1 + } else { + high = mid + } + } + return low +} diff --git a/op-chain-ops/cmd/celo-dbplay/main.go b/op-chain-ops/cmd/celo-dbplay/main.go index b7e54141a5529..8943b77a30941 100644 --- a/op-chain-ops/cmd/celo-dbplay/main.go +++ b/op-chain-ops/cmd/celo-dbplay/main.go @@ -1,7 +1,6 @@ package main import ( - "encoding/binary" "errors" "fmt" "os" @@ -12,12 +11,9 @@ import ( "github.com/urfave/cli/v2" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -84,7 +80,7 @@ func appMain(ctx *cli.Context) error { } log.Info("Loaded Celo L1 DB", "db", ldb) - // printStats(ldb) + printStats(ldb) findFirstCorruptedHeader(ldb) @@ -94,12 +90,13 @@ func appMain(ctx *cli.Context) error { // tryHeader(ldb, 4960000) // churrito block // tryHeader(ldb, 4960000) // donut block // tryHeader(ldb, 9472000) // espresso block + // tryHeader(ldb, 1) // espresso block return nil } func tryHeader(ldb ethdb.Database, number uint64) { - header, err := ReadCeloCanonicalHeader(ldb, number) + header, err := celo1.ReadCeloCanonicalHeader(ldb, number) if err != nil { log.Error("failed to load header", "number", number, "error", err) } else { @@ -145,7 +142,7 @@ func printStats(ldb ethdb.Database) { func canLoadHeader(ldb ethdb.Database, number uint64) bool { // log.Trace("Checking if header can be loaded", "number", number) - _, err := ReadCeloCanonicalHeader(ldb, number) + _, err := celo1.ReadCeloCanonicalHeader(ldb, number) if err != nil { log.Debug("failed to load header", "number", number, "error", err) } @@ -198,75 +195,3 @@ func printLastBlocks(ldb ethdb.Database, x uint64) { hash = header.ParentHash } } - -// ReadHeaderRLP retrieves a block header in its raw RLP database encoding. (copied from rawdb) -func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { - var data []byte - db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - // First try to look up the data in ancient database. Extra hash - // comparison is necessary since ancient database only maintains - // the canonical data. - data, _ = reader.Ancient(rawdb.ChainFreezerHeaderTable, number) - if len(data) != 0 { - return nil - } - // If not, try reading from leveldb - data, _ = db.Get(headerKey(number, hash)) - return nil - }) - return data -} - -var ( - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header -) - -// encodeBlockNumber encodes a block number as big endian uint64 -func encodeBlockNumber(number uint64) []byte { - enc := make([]byte, 8) - binary.BigEndian.PutUint64(enc, number) - return enc -} - -// headerKey = headerPrefix + num (uint64 big endian) + hash -func headerKey(number uint64, hash common.Hash) []byte { - return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) -} - -func ReadCeloHeader(db ethdb.Reader, hash common.Hash, number uint64) (*celo1.Header, error) { - data := ReadHeaderRLP(db, hash, number) - if len(data) == 0 { - return nil, errors.New("header not found") - } - - header := new(celo1.Header) - if err := rlp.DecodeBytes(data, header); err != nil { - return nil, err - } - return header, nil -} - -// ReadHeader retrieves the block header corresponding to the hash. (copied from rawdb) -func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header, error) { - data := ReadHeaderRLP(db, hash, number) - if len(data) == 0 { - return nil, errors.New("header not found") - } - header := new(types.Header) - if err := rlp.DecodeBytes(data, header); err != nil { - return nil, err - } - return header, nil -} - -// ReadCanonicalHeader retrieves the cannoical block header at the given number. -func ReadCanonicalHeader(db ethdb.Reader, number uint64) (*types.Header, error) { - hash := rawdb.ReadCanonicalHash(db, number) - return ReadHeader(db, hash, number) -} - -// ReadCanonicalHeader retrieves the cannoical block header at the given number. -func ReadCeloCanonicalHeader(db ethdb.Reader, number uint64) (*celo1.Header, error) { - hash := rawdb.ReadCanonicalHash(db, number) - return ReadCeloHeader(db, hash, number) -} From 37b781627c896d68aa2db71ab9f1e23d02616e32 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 30 Apr 2024 10:09:54 -0300 Subject: [PATCH 04/30] Adds comment --- op-chain-ops/cmd/celo-dbmigrate/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 6b77303b70201..2008a1ed88854 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -12,6 +12,14 @@ import ( "github.com/mattn/go-isatty" ) +// How to run: +// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-resetDB] [-batchSize 1000] +// +// This script will migrate ancient data from the old database to the new database +// The new database will be reset if the -resetDB flag is provided +// The number of ancient records to migrate in one batch can be set using the -batchSize flag +// The default batch size is 1000 + const ( dbCache = 1024 // size of the cache in MB dbHandles = 60 // number of handles From 430b875a4ef2f85cc27a64f6eacedf0560ce1352 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 30 Apr 2024 14:04:01 -0300 Subject: [PATCH 05/30] Adds extension methods for transformation --- op-chain-ops/cmd/celo-dbmigrate/main.go | 28 +++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 2008a1ed88854..0780a47202925 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -123,15 +123,22 @@ func freezeRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint6 for i := 0; i < len(hashes); i++ { log.Debug("Migrating ancient data", "number", number+uint64(i)) - // TODO: perform encoding migration here + newHeader, err := transformHeader(headers[i]) + if err != nil { + return fmt.Errorf("can't transform header: %v", err) + } + newBody, err := transformBlockBody(bodies[i]) + if err != nil { + return fmt.Errorf("can't transform body: %v", err) + } if err := op.AppendRaw(rawdb.ChainFreezerHashTable, number+uint64(i), hashes[i]); err != nil { return fmt.Errorf("can't write hash to Freezer: %v", err) } - if err := op.AppendRaw(rawdb.ChainFreezerHeaderTable, number+uint64(i), headers[i]); err != nil { + if err := op.AppendRaw(rawdb.ChainFreezerHeaderTable, number+uint64(i), newHeader); err != nil { return fmt.Errorf("can't write header to Freezer: %v", err) } - if err := op.AppendRaw(rawdb.ChainFreezerBodiesTable, number+uint64(i), bodies[i]); err != nil { + if err := op.AppendRaw(rawdb.ChainFreezerBodiesTable, number+uint64(i), newBody); err != nil { return fmt.Errorf("can't write body to Freezer: %v", err) } if err := op.AppendRaw(rawdb.ChainFreezerReceiptTable, number+uint64(i), receipts[i]); err != nil { @@ -147,6 +154,19 @@ func freezeRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint6 return err } +// transformHeader migrates the header from the old format to the new format (works with []byte input output) +func transformHeader(oldHeader []byte) ([]byte, error) { + // TODO: implement the transformation (remove only validator bls Signature) + return oldHeader, nil +} + +// transformBlockBody migrates the block body from the old format to the new format (works with []byte input output) +func transformBlockBody(oldBody []byte) ([]byte, error) { + // TODO: implement the transformation (remove epochSnarkData and randomness data from the body) + return oldBody, nil +} + +// MustAncientLength returns the number of items in the ancients database func MustAncientLength(db ethdb.Database) uint64 { byteSize, err := db.AncientSize(rawdb.ChainFreezerHashTable) if err != nil { @@ -157,7 +177,6 @@ func MustAncientLength(db ethdb.Database) uint64 { // finds number of items in the ancients database func findAncientsSize(ldb ethdb.Database, high uint64) uint64 { - // note: AncientSize returns a byte size, not number of items. Maybe for a well known size we can compute the number of items // runs a binary search using Ancient.HasAncient to find the first hash it can't find low := uint64(0) for low < high { @@ -169,4 +188,5 @@ func findAncientsSize(ldb ethdb.Database, high uint64) uint64 { } } return low + } From 587ccdbd49115cd555d142d25980c0a5aca89c2c Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 30 Apr 2024 14:30:14 -0300 Subject: [PATCH 06/30] Implements Transform CeloBody --- op-chain-ops/cmd/celo-dbmigrate/main.go | 32 +++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 0780a47202925..84b11c4beedc8 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" "github.com/mattn/go-isatty" ) @@ -161,9 +162,36 @@ func transformHeader(oldHeader []byte) ([]byte, error) { } // transformBlockBody migrates the block body from the old format to the new format (works with []byte input output) -func transformBlockBody(oldBody []byte) ([]byte, error) { +func transformBlockBody(oldBodyData []byte) ([]byte, error) { // TODO: implement the transformation (remove epochSnarkData and randomness data from the body) - return oldBody, nil + + // decode body into celo-blockchain Body structure + // remove epochSnarkData and randomness data + celoBody := new(CeloBody) + if err := rlp.DecodeBytes(oldBodyData, celoBody); err != nil { + log.Error("Invalid block body RLP", "err", err) + return nil, err + } + + // Alternatively, decode into op-geth types.Body structure + // body := new(types.Body) + // newBodyData, err := rlp.EncodeToBytes(body) + + // transform into op-geth types.Body structure + // since Body is a slice of types.Transactions, we can just remove the randomness and epochSnarkData and add empty array for UnclesHashes + newBodyData, err := rlp.EncodeToBytes([]interface{}{celoBody.Transactions, nil}) + if err != nil { + log.Crit("Failed to RLP encode body", "err", err) + } + // encode the new structure into []byte + return newBodyData, nil +} + +// CeloBody is the body of a celo block +type CeloBody struct { + Transactions rlp.RawValue + Randomness rlp.RawValue + EpochSnarkData rlp.RawValue } // MustAncientLength returns the number of items in the ancients database From aaca3c58ede598eefc25b61bf37cad5be9222671 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 30 Apr 2024 18:38:57 -0300 Subject: [PATCH 07/30] Adds impl that runs steps in a concurrent pipeline --- op-chain-ops/cmd/celo-dbmigrate/main.go | 155 +++++++++++++++++++++++- 1 file changed, 151 insertions(+), 4 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 84b11c4beedc8..5b9c23f0469f3 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "flag" "fmt" "os" @@ -11,6 +12,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/mattn/go-isatty" + + "golang.org/x/sync/errgroup" ) // How to run: @@ -68,7 +71,7 @@ func main() { migratedRecords := MustAncientLength(newDB) log.Info("Ancient Migration Initial Status", "migrated", migratedRecords, "total", toMigrateRecords) - if err = freezeRange(oldDB, newDB, migratedRecords, *batchSize); err != nil { + if err = parMigrateRange(oldDB, newDB, migratedRecords, *batchSize); err != nil { log.Crit("Failed to freeze range", "err", err) } @@ -94,7 +97,7 @@ func openCeloDb(path string, cache int, handles int, readonly bool) (ethdb.Datab return ldb, nil } -func freezeRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { +func seqMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { _, err := newDb.ModifyAncients(func(op ethdb.AncientWriteOp) error { hashes, err := oldDb.AncientRange(rawdb.ChainFreezerHashTable, number, count, 0) if err != nil { @@ -155,6 +158,152 @@ func freezeRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint6 return err } +type RLPBlockRange struct { + start uint64 + hashes [][]byte + headers [][]byte + bodies [][]byte + receipts [][]byte + tds [][]byte +} + +// parMigrateRange migrates ancient data from the old database to the new database in parallel +func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { + g, ctx := errgroup.WithContext(context.Background()) + readChan := make(chan RLPBlockRange, 10) + transformChan := make(chan RLPBlockRange, 10) + + g.Go(func() error { return readBlocks(ctx, oldDb, number, number+3*count, count, readChan) }) + g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) + g.Go(func() error { return writeAncients(ctx, newDb, transformChan) }) + + return g.Wait() +} + +func readBlocks(ctx context.Context, oldDb ethdb.Database, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange) error { + defer close(out) + // Read blocks and send them to the out channel + // This could be reading from a database, a file, etc. + for i := startBlock; i < endBlock; i += batchSize { + select { + case <-ctx.Done(): + return ctx.Err() + default: + var block RLPBlockRange + var err error + + count := min(batchSize, endBlock-i+1) + log.Info("Reading blocks", "start", i, "end", i+count-1, "count", count) + block.start = i + block.hashes, err = oldDb.AncientRange(rawdb.ChainFreezerHashTable, i, count, 0) + if err != nil { + log.Error("can't read hashes from old freezer", "err", err) + return err + } + + block.headers, err = oldDb.AncientRange(rawdb.ChainFreezerHeaderTable, i, count, 0) + if err != nil { + log.Error("can't read headers from old freezer", "err", err) + return err + } + + block.bodies, err = oldDb.AncientRange(rawdb.ChainFreezerBodiesTable, i, count, 0) + if err != nil { + log.Error("can't read bodies from old freezer", "err", err) + return err + } + + block.receipts, err = oldDb.AncientRange(rawdb.ChainFreezerReceiptTable, i, count, 0) + if err != nil { + log.Error("can't read receipts from old freezer", "err", err) + return err + } + + block.tds, err = oldDb.AncientRange(rawdb.ChainFreezerDifficultyTable, i, count, 0) + if err != nil { + log.Error("can't read tds from old freezer", "err", err) + return err + } + + out <- block + } + } + return nil +} + +func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RLPBlockRange) error { + // Transform blocks from the in channel and send them to the out channel + defer close(out) + for block := range in { + select { + case <-ctx.Done(): + return ctx.Err() + default: + // Transform headers + for i, header := range block.headers { + newHeader, err := transformHeader(header) + if err != nil { + log.Error("Failed to transform header", "err", err) + return err + } + block.headers[i] = newHeader + } + // Transform bodies + for i, body := range block.bodies { + newBody, err := transformBlockBody(body) + if err != nil { + log.Error("Failed to transform body", "err", err) + return err + } + block.bodies[i] = newBody + } + out <- block + } + } + return nil +} + +func writeAncients(ctx context.Context, newDb ethdb.Database, in <-chan RLPBlockRange) error { + // Write blocks from the in channel to the newDb + for block := range in { + select { + case <-ctx.Done(): + return ctx.Err() + default: + _, err := newDb.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { + for i := range block.hashes { + if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, block.start+uint64(i), block.hashes[i]); err != nil { + log.Error("can't write hash to Freezer", "err", err) + return err + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, block.start+uint64(i), block.headers[i]); err != nil { + log.Error("can't write header to Freezer", "err", err) + return err + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, block.start+uint64(i), block.bodies[i]); err != nil { + log.Error("can't write body to Freezer", "err", err) + return err + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, block.start+uint64(i), block.receipts[i]); err != nil { + log.Error("can't write receipts to Freezer", "err", err) + return err + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, block.start+uint64(i), block.tds[i]); err != nil { + log.Error("can't write td to Freezer", "err", err) + return err + } + } + return nil + }) + if err != nil { + log.Error("Failed to write ancients", "err", err) + return err + } + } + } + return nil +} + // transformHeader migrates the header from the old format to the new format (works with []byte input output) func transformHeader(oldHeader []byte) ([]byte, error) { // TODO: implement the transformation (remove only validator bls Signature) @@ -163,8 +312,6 @@ func transformHeader(oldHeader []byte) ([]byte, error) { // transformBlockBody migrates the block body from the old format to the new format (works with []byte input output) func transformBlockBody(oldBodyData []byte) ([]byte, error) { - // TODO: implement the transformation (remove epochSnarkData and randomness data from the body) - // decode body into celo-blockchain Body structure // remove epochSnarkData and randomness data celoBody := new(CeloBody) From 56a52c56d0f70e7f212773767c09900c03022608 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 30 Apr 2024 19:03:46 -0300 Subject: [PATCH 08/30] Adds transformHead, verify hashing works cleanup --- op-chain-ops/celo1/istanbul.go | 66 +++++++++++++++++++++++++ op-chain-ops/cmd/celo-dbmigrate/main.go | 22 +++++++-- 2 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 op-chain-ops/celo1/istanbul.go diff --git a/op-chain-ops/celo1/istanbul.go b/op-chain-ops/celo1/istanbul.go new file mode 100644 index 0000000000000..012814bd8dd81 --- /dev/null +++ b/op-chain-ops/celo1/istanbul.go @@ -0,0 +1,66 @@ +package celo1 + +import ( + "errors" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + IstanbulExtraVanity = 32 // Fixed number of extra-data bytes reserved for validator vanity +) + +type IstanbulAggregatedSeal struct { + // Bitmap is a bitmap having an active bit for each validator that signed this block + Bitmap *big.Int + // Signature is an aggregated BLS signature resulting from signatures by each validator that signed this block + Signature []byte + // Round is the round in which the signature was created. + Round *big.Int +} + +type IstanbulExtra struct { + // AddedValidators are the validators that have been added in the block + AddedValidators []common.Address + // AddedValidatorsPublicKeys are the BLS public keys for the validators added in the block + AddedValidatorsPublicKeys [][96]byte + // RemovedValidators is a bitmap having an active bit for each removed validator in the block + RemovedValidators *big.Int + // Seal is an ECDSA signature by the proposer + Seal []byte + // AggregatedSeal contains the aggregated BLS signature created via IBFT consensus. + AggregatedSeal IstanbulAggregatedSeal + // ParentAggregatedSeal contains and aggregated BLS signature for the previous block. + ParentAggregatedSeal IstanbulAggregatedSeal +} + +func RemoveIstanbulAggregatedSeal(header []byte) ([]byte, error) { + newHeader := new(Header) + err := rlp.DecodeBytes(header, newHeader) + if err != nil { + return nil, err + } + + if len(newHeader.Extra) < IstanbulExtraVanity { + return nil, errors.New("invalid istanbul header extra-data") + } + + istanbulExtra := IstanbulExtra{} + err = rlp.DecodeBytes(newHeader.Extra[IstanbulExtraVanity:], &istanbulExtra) + if err != nil { + return nil, err + } + + istanbulExtra.AggregatedSeal = IstanbulAggregatedSeal{} + + payload, err := rlp.EncodeToBytes(&istanbulExtra) + if err != nil { + return nil, err + } + + newHeader.Extra = append(newHeader.Extra[:IstanbulExtraVanity], payload...) + + return rlp.EncodeToBytes(newHeader) +} diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 5b9c23f0469f3..d76ec0f98a67b 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -1,13 +1,18 @@ package main import ( + "bytes" "context" "flag" "fmt" "os" "path/filepath" + "github.com/ethereum-optimism/optimism/op-chain-ops/celo1" + + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -71,7 +76,7 @@ func main() { migratedRecords := MustAncientLength(newDB) log.Info("Ancient Migration Initial Status", "migrated", migratedRecords, "total", toMigrateRecords) - if err = parMigrateRange(oldDB, newDB, migratedRecords, *batchSize); err != nil { + if err = parMigrateRange(oldDB, newDB, migratedRecords, toMigrateRecords, *batchSize); err != nil { log.Crit("Failed to freeze range", "err", err) } @@ -97,6 +102,7 @@ func openCeloDb(path string, cache int, handles int, readonly bool) (ethdb.Datab return ldb, nil } +// seqMigrateRange migrates ancient data from the old database to the new database sequentially func seqMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { _, err := newDb.ModifyAncients(func(op ethdb.AncientWriteOp) error { hashes, err := oldDb.AncientRange(rawdb.ChainFreezerHashTable, number, count, 0) @@ -168,12 +174,12 @@ type RLPBlockRange struct { } // parMigrateRange migrates ancient data from the old database to the new database in parallel -func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { +func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, end, step uint64) error { g, ctx := errgroup.WithContext(context.Background()) readChan := make(chan RLPBlockRange, 10) transformChan := make(chan RLPBlockRange, 10) - g.Go(func() error { return readBlocks(ctx, oldDb, number, number+3*count, count, readChan) }) + g.Go(func() error { return readBlocks(ctx, oldDb, start, end, step, readChan) }) g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) g.Go(func() error { return writeAncients(ctx, newDb, transformChan) }) @@ -246,6 +252,13 @@ func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RL log.Error("Failed to transform header", "err", err) return err } + + // Check that hashing the new header gives the same hash as the saved hash + newHash := crypto.Keccak256Hash(newHeader) + if !bytes.Equal(block.hashes[i], newHash.Bytes()) { + log.Error("Hash mismatch", "block", block.start+uint64(i), "oldHash", common.BytesToHash(block.hashes[i]), "newHash", newHash) + return fmt.Errorf("hash mismatch at block %d", block.start+uint64(i)) + } block.headers[i] = newHeader } // Transform bodies @@ -306,8 +319,7 @@ func writeAncients(ctx context.Context, newDb ethdb.Database, in <-chan RLPBlock // transformHeader migrates the header from the old format to the new format (works with []byte input output) func transformHeader(oldHeader []byte) ([]byte, error) { - // TODO: implement the transformation (remove only validator bls Signature) - return oldHeader, nil + return celo1.RemoveIstanbulAggregatedSeal(oldHeader) } // transformBlockBody migrates the block body from the old format to the new format (works with []byte input output) From d8d9ecf752b765063d56478f71889e97b8290617 Mon Sep 17 00:00:00 2001 From: alecps Date: Wed, 1 May 2024 21:35:18 -0400 Subject: [PATCH 09/30] add migration for non-frozen blocks --- op-chain-ops/celo1/db.go | 50 +++- op-chain-ops/celo1/istanbul.go | 4 +- op-chain-ops/cmd/celo-dbmigrate/main.go | 351 ++++++++++++++++-------- op-chain-ops/cmd/celo-dbplay/main.go | 35 ++- 4 files changed, 323 insertions(+), 117 deletions(-) diff --git a/op-chain-ops/celo1/db.go b/op-chain-ops/celo1/db.go index dc5943efd37f5..38c392271ad63 100644 --- a/op-chain-ops/celo1/db.go +++ b/op-chain-ops/celo1/db.go @@ -23,14 +23,23 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu return nil } // If not, try reading from leveldb - data, _ = db.Get(headerKey(number, hash)) + data, _ = db.Get(HeaderKey(number, hash)) return nil }) return data } var ( - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td + headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash + headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian) + + blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + + txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata ) // encodeBlockNumber encodes a block number as big endian uint64 @@ -40,11 +49,46 @@ func encodeBlockNumber(number uint64) []byte { return enc } +// headerKeyPrefix = headerPrefix + num (uint64 big endian) +func headerKeyPrefix(number uint64) []byte { + return append(headerPrefix, encodeBlockNumber(number)...) +} + // headerKey = headerPrefix + num (uint64 big endian) + hash -func headerKey(number uint64, hash common.Hash) []byte { +func HeaderKey(number uint64, hash common.Hash) []byte { return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) } +// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix +func HeaderTDKey(number uint64, hash common.Hash) []byte { + return append(HeaderKey(number, hash), headerTDSuffix...) +} + +// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix +func HeaderHashKey(number uint64) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) +} + +// headerNumberKey = headerNumberPrefix + hash +func HeaderNumberKey(hash common.Hash) []byte { + return append(headerNumberPrefix, hash.Bytes()...) +} + +// blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash +func BlockBodyKey(number uint64, hash common.Hash) []byte { + return append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// blockReceiptsKey = blockReceiptsPrefix + num (uint64 big endian) + hash +func BlockReceiptsKey(number uint64, hash common.Hash) []byte { + return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// txLookupKey = txLookupPrefix + hash +func txLookupKey(hash common.Hash) []byte { + return append(txLookupPrefix, hash.Bytes()...) +} + func ReadCeloHeader(db ethdb.Reader, hash common.Hash, number uint64) (*Header, error) { data := ReadHeaderRLP(db, hash, number) if len(data) == 0 { diff --git a/op-chain-ops/celo1/istanbul.go b/op-chain-ops/celo1/istanbul.go index 012814bd8dd81..c5291002b3f2d 100644 --- a/op-chain-ops/celo1/istanbul.go +++ b/op-chain-ops/celo1/istanbul.go @@ -5,6 +5,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" ) @@ -36,8 +37,9 @@ type IstanbulExtra struct { ParentAggregatedSeal IstanbulAggregatedSeal } +// RemoveIstanbulAggregatedSeal removes the aggregated seal from the header func RemoveIstanbulAggregatedSeal(header []byte) ([]byte, error) { - newHeader := new(Header) + newHeader := new(types.Header) // TODO double check on decoding type err := rlp.DecodeBytes(header, newHeader) if err != nil { return nil, err diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index d76ec0f98a67b..0d78eeabfacda 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -24,7 +24,7 @@ import ( // How to run: // go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-resetDB] [-batchSize 1000] // -// This script will migrate ancient data from the old database to the new database +// This script will migrate block data from the old database to the new database // The new database will be reset if the -resetDB flag is provided // The number of ancient records to migrate in one batch can be set using the -batchSize flag // The default batch size is 1000 @@ -41,7 +41,7 @@ func main() { oldDBPath := flag.String("oldDB", "", "Path to the old database") newDBPath := flag.String("newDB", "", "Path to the new database") resetDB := flag.Bool("resetDB", false, "Use to reset the new database before migrating data") - batchSize := flag.Uint64("batchSize", 1000, "Number of ancient records to migrate in one batch") + batchSize := flag.Uint64("batchSize", 1000, "Number of records to migrate in one batch") flag.Parse() @@ -72,16 +72,30 @@ func main() { defer oldDB.Close() defer newDB.Close() - toMigrateRecords := MustAncientLength(oldDB) - migratedRecords := MustAncientLength(newDB) - log.Info("Ancient Migration Initial Status", "migrated", migratedRecords, "total", toMigrateRecords) + numAncients := MustAncientLength(oldDB) + numAncientsMigrated := MustAncientLength(newDB) + log.Info("Ancient Migration Initial Status", "migrated", numAncientsMigrated, "total", numAncients) - if err = parMigrateRange(oldDB, newDB, migratedRecords, toMigrateRecords, *batchSize); err != nil { - log.Crit("Failed to freeze range", "err", err) + if err = parMigrateRange(oldDB, newDB, numAncientsMigrated, numAncients, *batchSize, readAncientBlockRange, writeAncientBlockRange); err != nil { + log.Crit("Failed to migrate ancient range", "err", err) } - log.Info("Ancient Migration End Status", "migrated", MustAncientLength(newDB), "total", toMigrateRecords) + log.Info("Ancient Migration End Status", "migrated", MustAncientLength(newDB), "total", numAncients) + numBlocks := GetLastBlockNumber(oldDB) + 1 + numBlocksMigrated := GetLastBlockNumber(newDB) + 1 + log.Info("Migration Initial Status", "migrated", numBlocksMigrated, "total", numBlocks) + + if err := parMigrateRange(oldDB, newDB, numBlocksMigrated, numBlocks, *batchSize, readBlockRange, writeBlockRange); err != nil { + log.Crit("Failed to migrate range", "err", err) + } + + // TODO there are other misc things like this that need to be written + rawdb.WriteHeadHeaderHash(newDB, rawdb.ReadHeadHeaderHash(oldDB)) + + log.Info("Migration End Status", "migrated", GetLastBlockNumber(newDB), "total", numBlocks) + + log.Info("Migration complete") } // Opens a Celo database, stored in the `celo` subfolder @@ -102,8 +116,18 @@ func openCeloDb(path string, cache int, handles int, readonly bool) (ethdb.Datab return ldb, nil } -// seqMigrateRange migrates ancient data from the old database to the new database sequentially -func seqMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { +// RLPBlockRange is a range of blocks in RLP format +type RLPBlockRange struct { + start uint64 + hashes [][]byte + headers [][]byte + bodies [][]byte + receipts [][]byte + tds [][]byte +} + +// seqMigrateAncientRange migrates ancient data from the old database to the new database sequentially +func seqMigrateAncientRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { _, err := newDb.ModifyAncients(func(op ethdb.AncientWriteOp) error { hashes, err := oldDb.AncientRange(rawdb.ChainFreezerHashTable, number, count, 0) if err != nil { @@ -164,29 +188,37 @@ func seqMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, number, count u return err } -type RLPBlockRange struct { - start uint64 - hashes [][]byte - headers [][]byte - bodies [][]byte - receipts [][]byte - tds [][]byte +// seqMigrateRange migrates data from the old database to the new database sequentially +func seqMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, count uint64) error { + blockRange, err := readBlockRange(oldDb, start, count) + if err != nil { + return fmt.Errorf("can't read block range: %v", err) + } + err = transformBlockRange(blockRange) + if err != nil { + return fmt.Errorf("can't transform block range: %v", err) + } + err = writeBlockRange(newDb, blockRange) + if err != nil { + return fmt.Errorf("can't write block range: %v", err) + } + return nil } // parMigrateRange migrates ancient data from the old database to the new database in parallel -func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, end, step uint64) error { +func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, end, step uint64, reader func(ethdb.Database, uint64, uint64) (*RLPBlockRange, error), writer func(ethdb.Database, *RLPBlockRange) error) error { g, ctx := errgroup.WithContext(context.Background()) readChan := make(chan RLPBlockRange, 10) transformChan := make(chan RLPBlockRange, 10) - g.Go(func() error { return readBlocks(ctx, oldDb, start, end, step, readChan) }) + g.Go(func() error { return readBlocks(ctx, oldDb, start, end, step, readChan, reader) }) g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) - g.Go(func() error { return writeAncients(ctx, newDb, transformChan) }) + g.Go(func() error { return writeBlocks(ctx, newDb, transformChan, writer) }) return g.Wait() } -func readBlocks(ctx context.Context, oldDb ethdb.Database, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange) error { +func readBlocks(ctx context.Context, oldDb ethdb.Database, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange, readBlockRange func(ethdb.Database, uint64, uint64) (*RLPBlockRange, error)) error { defer close(out) // Read blocks and send them to the out channel // This could be reading from a database, a file, etc. @@ -195,43 +227,12 @@ func readBlocks(ctx context.Context, oldDb ethdb.Database, startBlock, endBlock, case <-ctx.Done(): return ctx.Err() default: - var block RLPBlockRange - var err error - count := min(batchSize, endBlock-i+1) - log.Info("Reading blocks", "start", i, "end", i+count-1, "count", count) - block.start = i - block.hashes, err = oldDb.AncientRange(rawdb.ChainFreezerHashTable, i, count, 0) - if err != nil { - log.Error("can't read hashes from old freezer", "err", err) - return err - } - - block.headers, err = oldDb.AncientRange(rawdb.ChainFreezerHeaderTable, i, count, 0) + blockRange, err := readBlockRange(oldDb, i, count) if err != nil { - log.Error("can't read headers from old freezer", "err", err) - return err + return fmt.Errorf("Failed to read block range: %v", err) } - - block.bodies, err = oldDb.AncientRange(rawdb.ChainFreezerBodiesTable, i, count, 0) - if err != nil { - log.Error("can't read bodies from old freezer", "err", err) - return err - } - - block.receipts, err = oldDb.AncientRange(rawdb.ChainFreezerReceiptTable, i, count, 0) - if err != nil { - log.Error("can't read receipts from old freezer", "err", err) - return err - } - - block.tds, err = oldDb.AncientRange(rawdb.ChainFreezerDifficultyTable, i, count, 0) - if err != nil { - log.Error("can't read tds from old freezer", "err", err) - return err - } - - out <- block + out <- *blockRange } } return nil @@ -240,83 +241,199 @@ func readBlocks(ctx context.Context, oldDb ethdb.Database, startBlock, endBlock, func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RLPBlockRange) error { // Transform blocks from the in channel and send them to the out channel defer close(out) - for block := range in { + for blockRange := range in { select { case <-ctx.Done(): return ctx.Err() default: - // Transform headers - for i, header := range block.headers { - newHeader, err := transformHeader(header) - if err != nil { - log.Error("Failed to transform header", "err", err) - return err - } - - // Check that hashing the new header gives the same hash as the saved hash - newHash := crypto.Keccak256Hash(newHeader) - if !bytes.Equal(block.hashes[i], newHash.Bytes()) { - log.Error("Hash mismatch", "block", block.start+uint64(i), "oldHash", common.BytesToHash(block.hashes[i]), "newHash", newHash) - return fmt.Errorf("hash mismatch at block %d", block.start+uint64(i)) - } - block.headers[i] = newHeader - } - // Transform bodies - for i, body := range block.bodies { - newBody, err := transformBlockBody(body) - if err != nil { - log.Error("Failed to transform body", "err", err) - return err - } - block.bodies[i] = newBody + err := transformBlockRange(&blockRange) + if err != nil { + return fmt.Errorf("Failed to transform block range: %v", err) } - out <- block + out <- blockRange } } return nil } -func writeAncients(ctx context.Context, newDb ethdb.Database, in <-chan RLPBlockRange) error { +func writeBlocks(ctx context.Context, newDb ethdb.Database, in <-chan RLPBlockRange, writeBlockRange func(ethdb.Database, *RLPBlockRange) error) error { // Write blocks from the in channel to the newDb - for block := range in { + for blockRange := range in { select { case <-ctx.Done(): return ctx.Err() default: - _, err := newDb.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { - for i := range block.hashes { - if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, block.start+uint64(i), block.hashes[i]); err != nil { - log.Error("can't write hash to Freezer", "err", err) - return err - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, block.start+uint64(i), block.headers[i]); err != nil { - log.Error("can't write header to Freezer", "err", err) - return err - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, block.start+uint64(i), block.bodies[i]); err != nil { - log.Error("can't write body to Freezer", "err", err) - return err - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, block.start+uint64(i), block.receipts[i]); err != nil { - log.Error("can't write receipts to Freezer", "err", err) - return err - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, block.start+uint64(i), block.tds[i]); err != nil { - log.Error("can't write td to Freezer", "err", err) - return err - } - } - return nil - }) + err := writeBlockRange(newDb, &blockRange) if err != nil { - log.Error("Failed to write ancients", "err", err) - return err + return fmt.Errorf("Failed to write block range: %v", err) } } } return nil } +func readAncientBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRange, error) { + blockRange := RLPBlockRange{ + start: start, + hashes: make([][]byte, count), + headers: make([][]byte, count), + bodies: make([][]byte, count), + receipts: make([][]byte, count), + tds: make([][]byte, count), + } + var err error + + log.Info("Reading ancient blocks", "start", start, "end", start+count-1, "count", count) + + blockRange.hashes, err = db.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) + if err != nil { + return nil, fmt.Errorf("Failed to read hashes from old freezer: %v", err) + } + blockRange.headers, err = db.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0) + if err != nil { + return nil, fmt.Errorf("Failed to read headers from old freezer: %v", err) + } + blockRange.bodies, err = db.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0) + if err != nil { + return nil, fmt.Errorf("Failed to read bodies from old freezer: %v", err) + } + blockRange.receipts, err = db.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0) + if err != nil { + return nil, fmt.Errorf("Failed to read receipts from old freezer: %v", err) + } + blockRange.tds, err = db.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0) + if err != nil { + return nil, fmt.Errorf("Failed to read tds from old freezer: %v", err) + } + + return &blockRange, nil +} + +func readBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRange, error) { + blockRange := RLPBlockRange{ + start: start, + hashes: make([][]byte, count), + headers: make([][]byte, count), + bodies: make([][]byte, count), + receipts: make([][]byte, count), + tds: make([][]byte, count), + } + var err error + + log.Info("Reading blocks", "start", start, "end", start+count-1, "count", count) + + for i := start; i < start+count; i++ { + log.Debug("Reading old data", "number", i) + + blockRange.hashes[i-start], err = db.Get(celo1.HeaderHashKey(i)) + if err != nil { + return nil, fmt.Errorf("Failed to load hash, number: %d, err: %v", i, err) + } + hash := common.BytesToHash(blockRange.hashes[i-start]) + blockRange.headers[i-start], err = db.Get(celo1.HeaderKey(i, hash)) + if err != nil { + return nil, fmt.Errorf("Failed to load header, number: %d, err: %v", i, err) + } + blockRange.bodies[i-start], err = db.Get(celo1.BlockBodyKey(i, hash)) + if err != nil { + return nil, fmt.Errorf("Failed to load body, number: %d, err: %v", i, err) + } + blockRange.receipts[i-start], err = db.Get(celo1.BlockReceiptsKey(i, hash)) + if err != nil { + return nil, fmt.Errorf("Failed to load receipts, number: %d, err: %v", i, err) + } + blockRange.tds[i-start], err = db.Get(celo1.HeaderTDKey(i, hash)) + if err != nil { + return nil, fmt.Errorf("Failed to load td, number: %d, err: %v", i, err) + } + } + + return &blockRange, nil +} + +func transformBlockRange(blockRange *RLPBlockRange) error { + for i := range blockRange.hashes { + blockNumber := blockRange.start + uint64(i) + log.Debug("Migrating data", "number", blockNumber) + + newHeader, err := transformHeader(blockRange.headers[i]) + if err != nil { + return fmt.Errorf("can't transform header: %v", err) + } + newBody, err := transformBlockBody(blockRange.bodies[i]) + if err != nil { + return fmt.Errorf("can't transform body: %v", err) + } + + // Check that hashing the new header gives the same hash as the saved hash + newHash := crypto.Keccak256Hash(newHeader) + if !bytes.Equal(blockRange.hashes[i], newHash.Bytes()) { + log.Error("Hash mismatch", "block", blockNumber, "oldHash", common.BytesToHash(blockRange.hashes[i]), "newHash", newHash) + return fmt.Errorf("hash mismatch at block %d", blockNumber) + } + + blockRange.headers[i] = newHeader + blockRange.bodies[i] = newBody + } + + return nil +} + +func writeAncientBlockRange(db ethdb.Database, blockRange *RLPBlockRange) error { + _, err := db.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { + for i := range blockRange.hashes { + blockNumber := blockRange.start + uint64(i) + if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, blockNumber, blockRange.hashes[i]); err != nil { + return fmt.Errorf("can't write hash to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, blockNumber, blockRange.headers[i]); err != nil { + return fmt.Errorf("can't write header to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, blockNumber, blockRange.bodies[i]); err != nil { + return fmt.Errorf("can't write body to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, blockNumber, blockRange.receipts[i]); err != nil { + return fmt.Errorf("can't write receipts to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, blockNumber, blockRange.tds[i]); err != nil { + return fmt.Errorf("can't write td to Freezer: %v", err) + } + } + return nil + }) + + return err +} + +func writeBlockRange(db ethdb.Database, blockRange *RLPBlockRange) error { + for i, hashBytes := range blockRange.hashes { + hash := common.BytesToHash(hashBytes) + blockNumber := blockRange.start + uint64(i) + + log.Debug("Writing data", "number", blockNumber) + + if err := db.Put(celo1.HeaderHashKey(blockNumber), hashBytes); err != nil { + return fmt.Errorf("can't write hash to new database: %v", err) + } + if err := db.Put(celo1.HeaderKey(blockNumber, hash), blockRange.headers[i]); err != nil { + return fmt.Errorf("can't write header to new database: %v", err) + } + if err := db.Put(celo1.BlockBodyKey(blockNumber, hash), blockRange.bodies[i]); err != nil { + return fmt.Errorf("can't write body to new database: %v", err) + } + if err := db.Put(celo1.BlockReceiptsKey(blockNumber, hash), blockRange.receipts[i]); err != nil { + return fmt.Errorf("can't write receipts to new database: %v", err) + } + if err := db.Put(celo1.HeaderTDKey(blockNumber, hash), blockRange.tds[i]); err != nil { + return fmt.Errorf("can't write td to new database: %v", err) + } + // TODO there are other misc things like this that need to be written + rawdb.WriteHeaderNumber(db, hash, blockNumber) + } + + return nil +} + // transformHeader migrates the header from the old format to the new format (works with []byte input output) func transformHeader(oldHeader []byte) ([]byte, error) { return celo1.RemoveIstanbulAggregatedSeal(oldHeader) @@ -328,10 +445,11 @@ func transformBlockBody(oldBodyData []byte) ([]byte, error) { // remove epochSnarkData and randomness data celoBody := new(CeloBody) if err := rlp.DecodeBytes(oldBodyData, celoBody); err != nil { - log.Error("Invalid block body RLP", "err", err) - return nil, err + return nil, fmt.Errorf("Failed to RLP decode body: %v", err) } + // TODO(Alec) is this doing everything its supposed to + // Alternatively, decode into op-geth types.Body structure // body := new(types.Body) // newBodyData, err := rlp.EncodeToBytes(body) @@ -362,6 +480,15 @@ func MustAncientLength(db ethdb.Database) uint64 { return byteSize / ancientHashSize } +// GetLastBlockNumber returns the number of the last block in the database +func GetLastBlockNumber(db ethdb.Database) uint64 { + hash := rawdb.ReadHeadHeaderHash(db) + if hash == (common.Hash{}) { + return max(MustAncientLength(db)-1, 0) + } + return *rawdb.ReadHeaderNumber(db, hash) +} + // finds number of items in the ancients database func findAncientsSize(ldb ethdb.Database, high uint64) uint64 { // runs a binary search using Ancient.HasAncient to find the first hash it can't find diff --git a/op-chain-ops/cmd/celo-dbplay/main.go b/op-chain-ops/cmd/celo-dbplay/main.go index 8943b77a30941..d76915d801333 100644 --- a/op-chain-ops/cmd/celo-dbplay/main.go +++ b/op-chain-ops/cmd/celo-dbplay/main.go @@ -12,8 +12,10 @@ import ( "github.com/urfave/cli/v2" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -82,7 +84,9 @@ func appMain(ctx *cli.Context) error { printStats(ldb) - findFirstCorruptedHeader(ldb) + // findFirstCorruptedHeader(ldb) + + migrateHeaders(ldb, ldb, 1, 2) // tryHeader(ldb, 19814000-1) //just before gingerbread activation (alfajores) // tryHeader(ldb, 19814000) // just after gingerbread activation (alfajores) @@ -95,6 +99,35 @@ func appMain(ctx *cli.Context) error { return nil } +func migrateHeaders(oldDb, newDb ethdb.Database, start, end uint64) { + // Grab the hash of the tip of the legacy chain. + hash := rawdb.ReadHeadHeaderHash(oldDb) + lastBlockNumber := *rawdb.ReadHeaderNumber(oldDb, hash) + log.Info("Starting from HEAD of the chain", "number", lastBlockNumber) + + // migrate headers from 1 to lastBlockNumber + for i := start; i <= end; i++ { + hash := rawdb.ReadCanonicalHash(oldDb, i) + data := celo1.ReadHeaderRLP(oldDb, hash, i) // skips hash comparison + if len(data) == 0 { + log.Error("failed to load header", "number", i) + } + transformedData, err := celo1.RemoveIstanbulAggregatedSeal(data) + if err != nil { + log.Error("failed to remove istanbul aggregated seal", "number", i, "error", err) + } + + // Write celo2Header to celo2 db + celo2Header := new(types.Header) + err = rlp.DecodeBytes(transformedData, &celo2Header) + if err != nil { + log.Error("failed to decode header", "number", i, "error", err) + } + // rawdb.WriteHeader(newDB, celo2Header) + log.Info("Wrote header", "number", i, "hash", celo2Header.Hash(), "header", celo2Header) + } +} + func tryHeader(ldb ethdb.Database, number uint64) { header, err := celo1.ReadCeloCanonicalHeader(ldb, number) if err != nil { From 389aad2bf0ed4405ec91741b377b83c1164f0501 Mon Sep 17 00:00:00 2001 From: alecps Date: Thu, 2 May 2024 20:18:12 -0400 Subject: [PATCH 10/30] copy over entire db and modify in place, works with op-geth at piersy/minimal-data-migration --- op-chain-ops/cmd/celo-dbmigrate/main.go | 76 ++++++++++++++++++++----- 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 0d78eeabfacda..9704f13583f9e 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "os" + "os/exec" "path/filepath" "github.com/ethereum-optimism/optimism/op-chain-ops/celo1" @@ -22,7 +23,7 @@ import ( ) // How to run: -// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-resetDB] [-batchSize 1000] +// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-batchSize 1000] // // This script will migrate block data from the old database to the new database // The new database will be reset if the -resetDB flag is provided @@ -36,11 +37,11 @@ const ( ) func main() { - log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd())))) + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))) oldDBPath := flag.String("oldDB", "", "Path to the old database") newDBPath := flag.String("newDB", "", "Path to the new database") - resetDB := flag.Bool("resetDB", false, "Use to reset the new database before migrating data") + resetDB := flag.Bool("resetDB", false, "Use to reset the new database before migrating data (recommended)") batchSize := flag.Uint64("batchSize", 1000, "Number of records to migrate in one batch") flag.Parse() @@ -51,23 +52,40 @@ func main() { } if *resetDB { + // Reset the new database if err := os.RemoveAll(*newDBPath); err != nil { log.Crit("Failed to remove new database", "err", err) } } + // if err := os.MkdirAll(filepath.Join(*newDBPath, "geth"), 0755); err != nil { + // log.Crit("Failed to make new datadir", "err", err) + // } + + // Copy the old database to the new database + // cmd := exec.Command("cp", "-r", filepath.Join(*oldDBPath, "celo", "chaindata"), filepath.Join(*newDBPath, "geth")) + // err := cmd.Run() + // if err != nil { + // log.Crit("Failed to copy old database to new database", "err", err) + // } // Open the existing database in read-only mode - oldDB, err := openCeloDb(*oldDBPath, dbCache, dbHandles, true) + oldDB, err := openDB(filepath.Join(*oldDBPath, "celo"), dbCache, dbHandles, true) if err != nil { log.Crit("Failed to open old database", "err", err) } // Create a new database - newDB, err := openCeloDb(*newDBPath, dbCache, dbHandles, false) + newDB, err := openDB(filepath.Join(*newDBPath, "geth"), dbCache, dbHandles, false) if err != nil { log.Crit("Failed to create new database", "err", err) } + // Ancients is append only, so we need to remove and recreate it to transform the data + // newAncientPath := filepath.Join(*newDBPath, "geth", "chaindata", "ancient") + // if err := os.RemoveAll(newAncientPath); err != nil { + // log.Crit("Failed to remove copied ancient database", "err", err) + // } + // Close the databases defer oldDB.Close() defer newDB.Close() @@ -82,6 +100,37 @@ func main() { log.Info("Ancient Migration End Status", "migrated", MustAncientLength(newDB), "total", numAncients) + // Move the ancient directory up one level, delete everything else and move it back + + // Move the ancient directory up one level + cmd := exec.Command("rsync", "-av", filepath.Join(*newDBPath, "geth", "chaindata", "ancient"), filepath.Join(*newDBPath, "geth")) + err = cmd.Run() + if err != nil { + log.Crit("Failed to move ancient directory up one level", "err", err) + } + // Delete everything in chaindata directory + err = os.RemoveAll(filepath.Join(*newDBPath, "geth", "chaindata", "ancient")) + if err != nil { + log.Crit("Failed to clean chaindata directory", "err", err) + } + // Copy the old database to the new database, excluding the ancient folder + cmd = exec.Command("rsync", "-av", filepath.Join(*oldDBPath, "celo", "chaindata"), filepath.Join(*newDBPath, "geth"), "--exclude", "ancient") + err = cmd.Run() + if err != nil { + log.Crit("Failed to copy old database to new database", "err", err) + } + // Move the ancient directory back into the chaindata directory + cmd = exec.Command("rsync", "-av", filepath.Join(*newDBPath, "geth", "ancient"), filepath.Join(*newDBPath, "geth", "chaindata")) + err = cmd.Run() + if err != nil { + log.Crit("Failed to move ancient directory up one level", "err", err) + } + // Delete the extra ancient directory + err = os.RemoveAll(filepath.Join(*newDBPath, "geth", "ancient")) + if err != nil { + log.Crit("Failed to remove extra ancient directory", "err", err) + } + numBlocks := GetLastBlockNumber(oldDB) + 1 numBlocksMigrated := GetLastBlockNumber(newDB) + 1 log.Info("Migration Initial Status", "migrated", numBlocksMigrated, "total", numBlocks) @@ -90,20 +139,21 @@ func main() { log.Crit("Failed to migrate range", "err", err) } - // TODO there are other misc things like this that need to be written + // TODO do we still need to do this now that we've copied everything over? rawdb.WriteHeadHeaderHash(newDB, rawdb.ReadHeadHeaderHash(oldDB)) - log.Info("Migration End Status", "migrated", GetLastBlockNumber(newDB), "total", numBlocks) + log.Info("Migration End Status", "migrated", GetLastBlockNumber(newDB)+1, "total", numBlocks) log.Info("Migration complete") } -// Opens a Celo database, stored in the `celo` subfolder -func openCeloDb(path string, cache int, handles int, readonly bool) (ethdb.Database, error) { - ancientPath := filepath.Join(path, "ancient") +// Opens a database +func openDB(path string, cache int, handles int, readonly bool) (ethdb.Database, error) { + chaindataPath := filepath.Join(path, "chaindata") + ancientPath := filepath.Join(chaindataPath, "ancient") ldb, err := rawdb.Open(rawdb.OpenOptions{ Type: "leveldb", - Directory: path, + Directory: chaindataPath, AncientsDirectory: ancientPath, Namespace: "", Cache: cache, @@ -283,7 +333,7 @@ func readAncientBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRan } var err error - log.Info("Reading ancient blocks", "start", start, "end", start+count-1, "count", count) + log.Debug("Reading ancient blocks", "start", start, "end", start+count-1, "count", count) blockRange.hashes, err = db.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) if err != nil { @@ -320,7 +370,7 @@ func readBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRange, err } var err error - log.Info("Reading blocks", "start", start, "end", start+count-1, "count", count) + log.Debug("Reading blocks", "start", start, "end", start+count-1, "count", count) for i := start; i < start+count; i++ { log.Debug("Reading old data", "number", i) From 3d73ab368ed2f42e8d054498d2df8da0f0e82cf2 Mon Sep 17 00:00:00 2001 From: alecps Date: Thu, 2 May 2024 23:41:34 -0400 Subject: [PATCH 11/30] remove unecessary copying, cleanup code --- op-chain-ops/cmd/celo-dbmigrate/main.go | 124 +++++++++--------------- 1 file changed, 46 insertions(+), 78 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 9704f13583f9e..bf6b274794db2 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -23,10 +23,11 @@ import ( ) // How to run: -// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-batchSize 1000] +// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-resetDB] [-batchSize 1000] [-verbosity 3] // // This script will migrate block data from the old database to the new database // The new database will be reset if the -resetDB flag is provided +// You can set the log level using the -verbosity flag // The number of ancient records to migrate in one batch can be set using the -batchSize flag // The default batch size is 1000 @@ -37,15 +38,16 @@ const ( ) func main() { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))) - oldDBPath := flag.String("oldDB", "", "Path to the old database") newDBPath := flag.String("newDB", "", "Path to the new database") resetDB := flag.Bool("resetDB", false, "Use to reset the new database before migrating data (recommended)") batchSize := flag.Uint64("batchSize", 1000, "Number of records to migrate in one batch") + verbosity := flag.Uint64("verbosity", 3, "Log level (0:crit, 1:err, 2:warn, 3:info, 4:debug, 5:trace)") flag.Parse() + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*verbosity), log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))) + if *oldDBPath == "" || *newDBPath == "" { log.Info("Please provide both oldDB and newDB flags") return @@ -57,16 +59,6 @@ func main() { log.Crit("Failed to remove new database", "err", err) } } - // if err := os.MkdirAll(filepath.Join(*newDBPath, "geth"), 0755); err != nil { - // log.Crit("Failed to make new datadir", "err", err) - // } - - // Copy the old database to the new database - // cmd := exec.Command("cp", "-r", filepath.Join(*oldDBPath, "celo", "chaindata"), filepath.Join(*newDBPath, "geth")) - // err := cmd.Run() - // if err != nil { - // log.Crit("Failed to copy old database to new database", "err", err) - // } // Open the existing database in read-only mode oldDB, err := openDB(filepath.Join(*oldDBPath, "celo"), dbCache, dbHandles, true) @@ -74,18 +66,12 @@ func main() { log.Crit("Failed to open old database", "err", err) } - // Create a new database + // Open the new database newDB, err := openDB(filepath.Join(*newDBPath, "geth"), dbCache, dbHandles, false) if err != nil { log.Crit("Failed to create new database", "err", err) } - // Ancients is append only, so we need to remove and recreate it to transform the data - // newAncientPath := filepath.Join(*newDBPath, "geth", "chaindata", "ancient") - // if err := os.RemoveAll(newAncientPath); err != nil { - // log.Crit("Failed to remove copied ancient database", "err", err) - // } - // Close the databases defer oldDB.Close() defer newDB.Close() @@ -100,35 +86,8 @@ func main() { log.Info("Ancient Migration End Status", "migrated", MustAncientLength(newDB), "total", numAncients) - // Move the ancient directory up one level, delete everything else and move it back - - // Move the ancient directory up one level - cmd := exec.Command("rsync", "-av", filepath.Join(*newDBPath, "geth", "chaindata", "ancient"), filepath.Join(*newDBPath, "geth")) - err = cmd.Run() - if err != nil { - log.Crit("Failed to move ancient directory up one level", "err", err) - } - // Delete everything in chaindata directory - err = os.RemoveAll(filepath.Join(*newDBPath, "geth", "chaindata", "ancient")) - if err != nil { - log.Crit("Failed to clean chaindata directory", "err", err) - } - // Copy the old database to the new database, excluding the ancient folder - cmd = exec.Command("rsync", "-av", filepath.Join(*oldDBPath, "celo", "chaindata"), filepath.Join(*newDBPath, "geth"), "--exclude", "ancient") - err = cmd.Run() - if err != nil { - log.Crit("Failed to copy old database to new database", "err", err) - } - // Move the ancient directory back into the chaindata directory - cmd = exec.Command("rsync", "-av", filepath.Join(*newDBPath, "geth", "ancient"), filepath.Join(*newDBPath, "geth", "chaindata")) - err = cmd.Run() - if err != nil { - log.Crit("Failed to move ancient directory up one level", "err", err) - } - // Delete the extra ancient directory - err = os.RemoveAll(filepath.Join(*newDBPath, "geth", "ancient")) - if err != nil { - log.Crit("Failed to remove extra ancient directory", "err", err) + if err = copyAllDataExceptAncients(*oldDBPath, *newDBPath); err != nil { + log.Crit("Failed to copy all data except ancients", "err", err) } numBlocks := GetLastBlockNumber(oldDB) + 1 @@ -139,14 +98,44 @@ func main() { log.Crit("Failed to migrate range", "err", err) } - // TODO do we still need to do this now that we've copied everything over? - rawdb.WriteHeadHeaderHash(newDB, rawdb.ReadHeadHeaderHash(oldDB)) - log.Info("Migration End Status", "migrated", GetLastBlockNumber(newDB)+1, "total", numBlocks) log.Info("Migration complete") } +// copyAllDataExceptAncients copies all data from the old database to the new database except the ancient data which has already been migrated +func copyAllDataExceptAncients(oldDBPath, newDBPath string) error { + // Move the ancient directory up one level, delete everything else, copy over the old data and move the ancient directory back + + log.Info("Copying over all data except ancients") + + // Move the ancient directory up one level + cmd := exec.Command("rsync", "-av", filepath.Join(newDBPath, "geth", "chaindata", "ancient"), filepath.Join(newDBPath, "geth")) + if err := cmd.Run(); err != nil { + return fmt.Errorf("Failed to move ancient directory up one level: %v", err) + } + // Delete everything in chaindata directory that was created when the database was opened + if err := os.RemoveAll(filepath.Join(newDBPath, "geth", "chaindata")); err != nil { + return fmt.Errorf("Failed to clean chaindata directory: %v", err) + } + // Copy everything in the old database to the new database, excluding the ancient directory + cmd = exec.Command("rsync", "-av", filepath.Join(oldDBPath, "celo", "chaindata"), filepath.Join(newDBPath, "geth"), "--exclude", "ancient") + if err := cmd.Run(); err != nil { + return fmt.Errorf("Failed to copy old database to new database: %v", err) + } + // Move the ancient directory back down into the chaindata directory + cmd = exec.Command("rsync", "-av", filepath.Join(newDBPath, "geth", "ancient"), filepath.Join(newDBPath, "geth", "chaindata")) + if err := cmd.Run(); err != nil { + return fmt.Errorf("Failed to move ancient directory back down: %v", err) + } + // Delete the extra ancient directory that was up one level + if err := os.RemoveAll(filepath.Join(newDBPath, "geth", "ancient")); err != nil { + return fmt.Errorf("Failed to remove extra ancient directory copy: %v", err) + } + + return nil +} + // Opens a database func openDB(path string, cache int, handles int, readonly bool) (ethdb.Database, error) { chaindataPath := filepath.Join(path, "chaindata") @@ -361,12 +350,10 @@ func readAncientBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRan func readBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRange, error) { blockRange := RLPBlockRange{ - start: start, - hashes: make([][]byte, count), - headers: make([][]byte, count), - bodies: make([][]byte, count), - receipts: make([][]byte, count), - tds: make([][]byte, count), + start: start, + hashes: make([][]byte, count), + headers: make([][]byte, count), + bodies: make([][]byte, count), } var err error @@ -388,14 +375,6 @@ func readBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRange, err if err != nil { return nil, fmt.Errorf("Failed to load body, number: %d, err: %v", i, err) } - blockRange.receipts[i-start], err = db.Get(celo1.BlockReceiptsKey(i, hash)) - if err != nil { - return nil, fmt.Errorf("Failed to load receipts, number: %d, err: %v", i, err) - } - blockRange.tds[i-start], err = db.Get(celo1.HeaderTDKey(i, hash)) - if err != nil { - return nil, fmt.Errorf("Failed to load td, number: %d, err: %v", i, err) - } } return &blockRange, nil @@ -462,23 +441,12 @@ func writeBlockRange(db ethdb.Database, blockRange *RLPBlockRange) error { log.Debug("Writing data", "number", blockNumber) - if err := db.Put(celo1.HeaderHashKey(blockNumber), hashBytes); err != nil { - return fmt.Errorf("can't write hash to new database: %v", err) - } if err := db.Put(celo1.HeaderKey(blockNumber, hash), blockRange.headers[i]); err != nil { return fmt.Errorf("can't write header to new database: %v", err) } if err := db.Put(celo1.BlockBodyKey(blockNumber, hash), blockRange.bodies[i]); err != nil { return fmt.Errorf("can't write body to new database: %v", err) } - if err := db.Put(celo1.BlockReceiptsKey(blockNumber, hash), blockRange.receipts[i]); err != nil { - return fmt.Errorf("can't write receipts to new database: %v", err) - } - if err := db.Put(celo1.HeaderTDKey(blockNumber, hash), blockRange.tds[i]); err != nil { - return fmt.Errorf("can't write td to new database: %v", err) - } - // TODO there are other misc things like this that need to be written - rawdb.WriteHeaderNumber(db, hash, blockNumber) } return nil @@ -498,7 +466,7 @@ func transformBlockBody(oldBodyData []byte) ([]byte, error) { return nil, fmt.Errorf("Failed to RLP decode body: %v", err) } - // TODO(Alec) is this doing everything its supposed to + // TODO is this doing everything its supposed to // Alternatively, decode into op-geth types.Body structure // body := new(types.Body) From 212e4c7b0f616ad5baf9dae5b7a1ac4edfb8e8ca Mon Sep 17 00:00:00 2001 From: alecps Date: Mon, 6 May 2024 12:51:28 -0400 Subject: [PATCH 12/30] close and reopen DBs --- op-chain-ops/cmd/celo-dbmigrate/main.go | 37 +++++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index bf6b274794db2..36d97e13975d9 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -60,22 +60,16 @@ func main() { } } - // Open the existing database in read-only mode - oldDB, err := openDB(filepath.Join(*oldDBPath, "celo"), dbCache, dbHandles, true) + // Open the databases + oldDB, err := openDB(filepath.Join(*oldDBPath, "celo"), dbCache, dbHandles, false) if err != nil { log.Crit("Failed to open old database", "err", err) } - - // Open the new database newDB, err := openDB(filepath.Join(*newDBPath, "geth"), dbCache, dbHandles, false) if err != nil { - log.Crit("Failed to create new database", "err", err) + log.Crit("Failed to open new database", "err", err) } - // Close the databases - defer oldDB.Close() - defer newDB.Close() - numAncients := MustAncientLength(oldDB) numAncientsMigrated := MustAncientLength(newDB) log.Info("Ancient Migration Initial Status", "migrated", numAncientsMigrated, "total", numAncients) @@ -86,13 +80,30 @@ func main() { log.Info("Ancient Migration End Status", "migrated", MustAncientLength(newDB), "total", numAncients) + numBlocks := GetLastBlockNumber(oldDB) + 1 + numBlocksMigrated := GetLastBlockNumber(newDB) + 1 + log.Info("Non-Ancient Migration Initial Status", "migrated", numBlocksMigrated, "total", numBlocks) + + // Close the databases + oldDB.Close() + newDB.Close() + if err = copyAllDataExceptAncients(*oldDBPath, *newDBPath); err != nil { log.Crit("Failed to copy all data except ancients", "err", err) } - numBlocks := GetLastBlockNumber(oldDB) + 1 - numBlocksMigrated := GetLastBlockNumber(newDB) + 1 - log.Info("Migration Initial Status", "migrated", numBlocksMigrated, "total", numBlocks) + // Reopen the databases + oldDB, err = openDB(filepath.Join(*oldDBPath, "celo"), dbCache, dbHandles, false) + if err != nil { + log.Crit("Failed to open old database", "err", err) + } + newDB, err = openDB(filepath.Join(*newDBPath, "geth"), dbCache, dbHandles, false) + if err != nil { + log.Crit("Failed to open new database", "err", err) + } + + defer oldDB.Close() + defer newDB.Close() if err := parMigrateRange(oldDB, newDB, numBlocksMigrated, numBlocks, *batchSize, readBlockRange, writeBlockRange); err != nil { log.Crit("Failed to migrate range", "err", err) @@ -250,6 +261,8 @@ func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, end, ste readChan := make(chan RLPBlockRange, 10) transformChan := make(chan RLPBlockRange, 10) + log.Info("Migrating data", "start", start, "end", end, "step", step) + g.Go(func() error { return readBlocks(ctx, oldDb, start, end, step, readChan, reader) }) g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) g.Go(func() error { return writeBlocks(ctx, newDb, transformChan, writer) }) From bbf552a053a2698d36187a4aafef55d940aad766 Mon Sep 17 00:00:00 2001 From: alecps Date: Mon, 6 May 2024 12:53:30 -0400 Subject: [PATCH 13/30] migrate newdb in place --- op-chain-ops/cmd/celo-dbmigrate/main.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 36d97e13975d9..83e8c1894321d 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -93,19 +93,14 @@ func main() { } // Reopen the databases - oldDB, err = openDB(filepath.Join(*oldDBPath, "celo"), dbCache, dbHandles, false) - if err != nil { - log.Crit("Failed to open old database", "err", err) - } newDB, err = openDB(filepath.Join(*newDBPath, "geth"), dbCache, dbHandles, false) if err != nil { log.Crit("Failed to open new database", "err", err) } - defer oldDB.Close() defer newDB.Close() - if err := parMigrateRange(oldDB, newDB, numBlocksMigrated, numBlocks, *batchSize, readBlockRange, writeBlockRange); err != nil { + if err := parMigrateRange(newDB, newDB, numBlocksMigrated, numBlocks, *batchSize, readBlockRange, writeBlockRange); err != nil { log.Crit("Failed to migrate range", "err", err) } From c6695bb80cd45d5bbfcd59340dfbff1c5416d8db Mon Sep 17 00:00:00 2001 From: alecps Date: Mon, 6 May 2024 15:32:14 -0400 Subject: [PATCH 14/30] saving progress Co-authored-by: Mariano Cortesi --- op-chain-ops/cmd/celo-dbmigrate/main.go | 192 ++++++++++++++++++++---- 1 file changed, 160 insertions(+), 32 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 83e8c1894321d..ee32e8882a721 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -60,46 +60,114 @@ func main() { } } - // Open the databases - oldDB, err := openDB(filepath.Join(*oldDBPath, "celo"), dbCache, dbHandles, false) + // Script Steps: + // 1. Copy everything but ancients over to new datadir + // 2. Open up Ancients(freezer)DBs OLD and NEW + // 3. Trasnfer and transform ancients from OLD to NEW + // 4. Close DBs + // 5. Cleanup directories (one directory with new datadir + new ancients) + // 6. Open up a new database + // 7. Transform block bodies and headers in place in the new database + + // Script Steps: + // 1. Create empty new db directory. (chaindata) + // 2. Open up Ancients(freezer)DBs OLD and NEW. New opens in `newdb/chaindata/ancients`. Old open from `oldDb/chaindata/ancients` Old opens readonly + // 3. Trasnfer and transform ancients from OLD to NEW + // 4. Close DBs + // status: + // oldDb - same as before (everything read only) + // newDb - chainddata with ancients directory, the rest is empty + // 5. Copy everything else from oldDb to newDb (don't copy ancients) + // 6. Open up a new database + // 7. Transform block bodies and headers in place in the new database + + if err := createEmptyNewDb(*newDBPath); err != nil { + + } + + if err := migrateAncientsDb(*oldDBPath, *newDBPath); err != nil { + + } + + if err := migrateNonAncientsDb(*oldDBPath, *newDBPath); err != nil { + + } + + + + + // Copy everything from the old database to the new database + if err := os.MkdirAll(filepath.Join(*newDBPath, "geth"), 0755); err != nil { + log.Crit("Failed to create new geth db directory", "err", err) + } + // TODO not copy ancient + cmd := exec.Command("cp", "-a", "-R", filepath.Join(*oldDBPath, "celo", "chaindata"), filepath.Join(*newDBPath, "geth")) + if err := cmd.Run(); err != nil { + log.Crit("Failed to copy old database to new database", "err", err) + } + + chaindataPath := filepath.Join(*newDBPath, "geth", "chaindata") + ancientPath := filepath.Join(chaindataPath, "ancient") + oldFreezer, err := rawdb.NewChainFreezer(ancientPath, "", true) if err != nil { - log.Crit("Failed to open old database", "err", err) + log.Crit("Failed to open old freezer", "err", err) } - newDB, err := openDB(filepath.Join(*newDBPath, "geth"), dbCache, dbHandles, false) + ancientPathTmp := filepath.Join(chaindataPath, "ancient_new") + newFreezer, err := rawdb.NewChainFreezer(ancientPathTmp, "", false) if err != nil { - log.Crit("Failed to open new database", "err", err) + log.Crit("Failed to open new freezer", "err", err) } - numAncients := MustAncientLength(oldDB) - numAncientsMigrated := MustAncientLength(newDB) - log.Info("Ancient Migration Initial Status", "migrated", numAncientsMigrated, "total", numAncients) + numAncientsOld, err := oldFreezer.Ancients() + if err != nil { + log.Crit("Failed to get number of ancients in old freezer", "err", err) + } + numAncientsNew, err := newFreezer.Ancients() + if err != nil { + log.Crit("Failed to get number of ancients in new freezer", "err", err) + } - if err = parMigrateRange(oldDB, newDB, numAncientsMigrated, numAncients, *batchSize, readAncientBlockRange, writeAncientBlockRange); err != nil { + log.Info("Ancient Migration Initial Status", "migrated", numAncientsNew, "total", numAncientsOld) + + if err = parMigrateAncientRange(oldFreezer, newFreezer, numAncientsNew, numAncientsOld, *batchSize, readAncientBlockRange, writeAncientBlockRange); err != nil { log.Crit("Failed to migrate ancient range", "err", err) } - log.Info("Ancient Migration End Status", "migrated", MustAncientLength(newDB), "total", numAncients) + numAncientsOld, err = oldFreezer.Ancients() + if err != nil { + log.Crit("Failed to get number of ancients in old freezer", "err", err) + } + numAncientsNew, err = newFreezer.Ancients() + if err != nil { + log.Crit("Failed to get number of ancients in new freezer", "err", err) + } - numBlocks := GetLastBlockNumber(oldDB) + 1 - numBlocksMigrated := GetLastBlockNumber(newDB) + 1 - log.Info("Non-Ancient Migration Initial Status", "migrated", numBlocksMigrated, "total", numBlocks) + log.Info("Ancient Migration End Status", "migrated", numAncientsNew, "total", numAncientsOld) - // Close the databases - oldDB.Close() - newDB.Close() + oldFreezer.Close() + newFreezer.Close() - if err = copyAllDataExceptAncients(*oldDBPath, *newDBPath); err != nil { - log.Crit("Failed to copy all data except ancients", "err", err) + // Remove old ancient directory and rename new ancient directory + if err := os.RemoveAll(ancientPath); err != nil { + log.Crit("Failed to remove old ancient directory", "err", err) + } + if err := os.Rename(ancientPathTmp, ancientPath); err != nil { + log.Crit("Failed to rename new ancient directory", "err", err) } - // Reopen the databases - newDB, err = openDB(filepath.Join(*newDBPath, "geth"), dbCache, dbHandles, false) + // Reopen the new database + newDB, err := openDB(chaindataPath, ancientPath, dbCache, dbHandles, false) if err != nil { log.Crit("Failed to open new database", "err", err) } defer newDB.Close() + numBlocks := GetLastBlockNumber(newDB) + 1 + // numBlocksMigrated := GetLastBlockNumber(newDB) + 1 + numBlocksMigrated := MustAncientLength(newDB) + log.Info("Non-Ancient Migration Initial Status", "migrated", numBlocksMigrated, "total", numBlocks) + if err := parMigrateRange(newDB, newDB, numBlocksMigrated, numBlocks, *batchSize, readBlockRange, writeBlockRange); err != nil { log.Crit("Failed to migrate range", "err", err) } @@ -109,6 +177,9 @@ func main() { log.Info("Migration complete") } +func createEmptyNewDb(newDBPath string) error { + + // copyAllDataExceptAncients copies all data from the old database to the new database except the ancient data which has already been migrated func copyAllDataExceptAncients(oldDBPath, newDBPath string) error { // Move the ancient directory up one level, delete everything else, copy over the old data and move the ancient directory back @@ -143,9 +214,7 @@ func copyAllDataExceptAncients(oldDBPath, newDBPath string) error { } // Opens a database -func openDB(path string, cache int, handles int, readonly bool) (ethdb.Database, error) { - chaindataPath := filepath.Join(path, "chaindata") - ancientPath := filepath.Join(chaindataPath, "ancient") +func openDB(chaindataPath, ancientPath string, cache int, handles int, readonly bool) (ethdb.Database, error) { ldb, err := rawdb.Open(rawdb.OpenOptions{ Type: "leveldb", Directory: chaindataPath, @@ -250,6 +319,21 @@ func seqMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, count ui return nil } +// parMigrateRange migrates ancient data from the old database to the new database in parallel +func parMigrateAncientRange(oldFreezer, newFreezer *rawdb.Freezer, start, end, step uint64, reader func(*rawdb.Freezer, uint64, uint64) (*RLPBlockRange, error), writer func(*rawdb.Freezer, *RLPBlockRange) error) error { + g, ctx := errgroup.WithContext(context.Background()) + readChan := make(chan RLPBlockRange, 10) + transformChan := make(chan RLPBlockRange, 10) + + log.Info("Migrating data", "start", start, "end", end, "step", step) + + g.Go(func() error { return readAncientBlocks(ctx, oldFreezer, start, end, step, readChan, reader) }) + g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) + g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan, writer) }) + + return g.Wait() +} + // parMigrateRange migrates ancient data from the old database to the new database in parallel func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, end, step uint64, reader func(ethdb.Database, uint64, uint64) (*RLPBlockRange, error), writer func(ethdb.Database, *RLPBlockRange) error) error { g, ctx := errgroup.WithContext(context.Background()) @@ -265,6 +349,26 @@ func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, end, ste return g.Wait() } +func readAncientBlocks(ctx context.Context, oldFreezer *rawdb.Freezer, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange, readBlockRange func(*rawdb.Freezer, uint64, uint64) (*RLPBlockRange, error)) error { + defer close(out) + // Read blocks and send them to the out channel + // This could be reading from a database, a file, etc. + for i := startBlock; i < endBlock; i += batchSize { + select { + case <-ctx.Done(): + return ctx.Err() + default: + count := min(batchSize, endBlock-i+1) + blockRange, err := readAncientBlockRange(oldFreezer, i, count) + if err != nil { + return fmt.Errorf("Failed to read block range: %v", err) + } + out <- *blockRange + } + } + return nil +} + func readBlocks(ctx context.Context, oldDb ethdb.Database, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange, readBlockRange func(ethdb.Database, uint64, uint64) (*RLPBlockRange, error)) error { defer close(out) // Read blocks and send them to the out channel @@ -303,6 +407,22 @@ func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RL return nil } +func writeAncientBlocks(ctx context.Context, newFreezer *rawdb.Freezer, in <-chan RLPBlockRange, writeBlockRange func(*rawdb.Freezer, *RLPBlockRange) error) error { + // Write blocks from the in channel to the newDb + for blockRange := range in { + select { + case <-ctx.Done(): + return ctx.Err() + default: + err := writeAncientBlockRange(newFreezer, &blockRange) + if err != nil { + return fmt.Errorf("Failed to write block range: %v", err) + } + } + } + return nil +} + func writeBlocks(ctx context.Context, newDb ethdb.Database, in <-chan RLPBlockRange, writeBlockRange func(ethdb.Database, *RLPBlockRange) error) error { // Write blocks from the in channel to the newDb for blockRange := range in { @@ -319,7 +439,7 @@ func writeBlocks(ctx context.Context, newDb ethdb.Database, in <-chan RLPBlockRa return nil } -func readAncientBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRange, error) { +func readAncientBlockRange(freezer *rawdb.Freezer, start, count uint64) (*RLPBlockRange, error) { blockRange := RLPBlockRange{ start: start, hashes: make([][]byte, count), @@ -332,23 +452,23 @@ func readAncientBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRan log.Debug("Reading ancient blocks", "start", start, "end", start+count-1, "count", count) - blockRange.hashes, err = db.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) + blockRange.hashes, err = freezer.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) if err != nil { return nil, fmt.Errorf("Failed to read hashes from old freezer: %v", err) } - blockRange.headers, err = db.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0) + blockRange.headers, err = freezer.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0) if err != nil { return nil, fmt.Errorf("Failed to read headers from old freezer: %v", err) } - blockRange.bodies, err = db.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0) + blockRange.bodies, err = freezer.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0) if err != nil { return nil, fmt.Errorf("Failed to read bodies from old freezer: %v", err) } - blockRange.receipts, err = db.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0) + blockRange.receipts, err = freezer.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0) if err != nil { return nil, fmt.Errorf("Failed to read receipts from old freezer: %v", err) } - blockRange.tds, err = db.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0) + blockRange.tds, err = freezer.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0) if err != nil { return nil, fmt.Errorf("Failed to read tds from old freezer: %v", err) } @@ -416,8 +536,8 @@ func transformBlockRange(blockRange *RLPBlockRange) error { return nil } -func writeAncientBlockRange(db ethdb.Database, blockRange *RLPBlockRange) error { - _, err := db.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { +func writeAncientBlockRange(freezer *rawdb.Freezer, blockRange *RLPBlockRange) error { + _, err := freezer.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { for i := range blockRange.hashes { blockNumber := blockRange.start + uint64(i) if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, blockNumber, blockRange.hashes[i]); err != nil { @@ -499,13 +619,21 @@ type CeloBody struct { // MustAncientLength returns the number of items in the ancients database func MustAncientLength(db ethdb.Database) uint64 { - byteSize, err := db.AncientSize(rawdb.ChainFreezerHashTable) + byteSize, err := db.AncientSize(rawdb.ChainFreezerHashTable) // TODO why not just use .Ancients()? if err != nil { log.Crit("Failed to get ancient size", "error", err) } return byteSize / ancientHashSize } +// func MustAncientLength(freezer rawdb.Freezer) uint64 { +// // byteSize, err := freezer.AncientSize(rawdb.ChainFreezerHashTable) +// // if err != nil { +// // log.Crit("Failed to get ancient size", "error", err) +// // } +// // return byteSize / ancientHashSize +// } + // GetLastBlockNumber returns the number of the last block in the database func GetLastBlockNumber(db ethdb.Database) uint64 { hash := rawdb.ReadHeadHeaderHash(db) From fd245bea69ad4adc1cdc91a34e7f6d775a76e188 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 7 May 2024 00:09:57 -0300 Subject: [PATCH 15/30] Refactor code to improve database migration process --- op-chain-ops/cmd/celo-dbmigrate/main.go | 683 ++++++++---------------- 1 file changed, 225 insertions(+), 458 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index ee32e8882a721..19232b73387fe 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/binary" "flag" "fmt" "os" @@ -23,211 +24,186 @@ import ( ) // How to run: -// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-resetDB] [-batchSize 1000] [-verbosity 3] +// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-resetDB] [-batchSize 1000] [-verbosity 3] [-clear-all] [-clear-nonAncients] // // This script will migrate block data from the old database to the new database // The new database will be reset if the -resetDB flag is provided // You can set the log level using the -verbosity flag // The number of ancient records to migrate in one batch can be set using the -batchSize flag // The default batch size is 1000 +// Use -clear-all to start with a fresh new database +// Use -clear-nonAncients to keep migrated ancients, but not non-ancients const ( - dbCache = 1024 // size of the cache in MB - dbHandles = 60 // number of handles - ancientHashSize = 38 // size of a hash entry on ancient database (empiric value) + DB_CACHE = 1024 // size of the cache in MB + DB_HANDLES = 60 // number of handles + LAST_MIGRATED_BLOCK_KEY = "celoLastMigratedBlock" ) func main() { - oldDBPath := flag.String("oldDB", "", "Path to the old database") + oldDBPath := flag.String("oldDB", "", "Path to the old database chaindata directory (read-only)") newDBPath := flag.String("newDB", "", "Path to the new database") - resetDB := flag.Bool("resetDB", false, "Use to reset the new database before migrating data (recommended)") - batchSize := flag.Uint64("batchSize", 1000, "Number of records to migrate in one batch") + batchSize := flag.Uint64("batchSize", 10000, "Number of records to migrate in one batch") verbosity := flag.Uint64("verbosity", 3, "Log level (0:crit, 1:err, 2:warn, 3:info, 4:debug, 5:trace)") + clearAll := flag.Bool("clear-all", false, "Use this to start with a fresh new database") + clearNonAncients := flag.Bool("clear-nonAncients", false, "Use to keep migrated ancients, but not non-ancients") + // flag.Usage = usage flag.Parse() log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*verbosity), log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))) + var err error + + // check that `rsync` command is available + if _, err := exec.LookPath("rsync"); err != nil { + log.Info("Please install `rsync` to use this script") + return + } + if *oldDBPath == "" || *newDBPath == "" { log.Info("Please provide both oldDB and newDB flags") + flag.Usage() return } - if *resetDB { - // Reset the new database + if *clearAll { if err := os.RemoveAll(*newDBPath); err != nil { log.Crit("Failed to remove new database", "err", err) } } - - // Script Steps: - // 1. Copy everything but ancients over to new datadir - // 2. Open up Ancients(freezer)DBs OLD and NEW - // 3. Trasnfer and transform ancients from OLD to NEW - // 4. Close DBs - // 5. Cleanup directories (one directory with new datadir + new ancients) - // 6. Open up a new database - // 7. Transform block bodies and headers in place in the new database - - // Script Steps: - // 1. Create empty new db directory. (chaindata) - // 2. Open up Ancients(freezer)DBs OLD and NEW. New opens in `newdb/chaindata/ancients`. Old open from `oldDb/chaindata/ancients` Old opens readonly - // 3. Trasnfer and transform ancients from OLD to NEW - // 4. Close DBs - // status: - // oldDb - same as before (everything read only) - // newDb - chainddata with ancients directory, the rest is empty - // 5. Copy everything else from oldDb to newDb (don't copy ancients) - // 6. Open up a new database - // 7. Transform block bodies and headers in place in the new database + if *clearNonAncients { + if err := cleanupNonAncientDb(*newDBPath); err != nil { + log.Crit("Failed to cleanup non-ancient database", "err", err) + } + } if err := createEmptyNewDb(*newDBPath); err != nil { - + log.Crit("Failed to create new database", "err", err) } - if err := migrateAncientsDb(*oldDBPath, *newDBPath); err != nil { - + var numAncientsNew uint64 + if numAncientsNew, err = migrateAncientsDb(*oldDBPath, *newDBPath, *batchSize); err != nil { + log.Crit("Failed to migrate ancients database", "err", err) } - if err := migrateNonAncientsDb(*oldDBPath, *newDBPath); err != nil { - + var numAncientsNonAncients uint64 + if numAncientsNonAncients, err = migrateNonAncientsDb(*oldDBPath, *newDBPath, numAncientsNew-1, *batchSize); err != nil { + log.Crit("Failed to migrate non-ancients database", "err", err) } + log.Info("Migration Completed", "migratedAncients", numAncientsNew, "migratedNonAncients", numAncientsNonAncients) +} - - - // Copy everything from the old database to the new database - if err := os.MkdirAll(filepath.Join(*newDBPath, "geth"), 0755); err != nil { - log.Crit("Failed to create new geth db directory", "err", err) - } - // TODO not copy ancient - cmd := exec.Command("cp", "-a", "-R", filepath.Join(*oldDBPath, "celo", "chaindata"), filepath.Join(*newDBPath, "geth")) - if err := cmd.Run(); err != nil { - log.Crit("Failed to copy old database to new database", "err", err) - } - - chaindataPath := filepath.Join(*newDBPath, "geth", "chaindata") - ancientPath := filepath.Join(chaindataPath, "ancient") - oldFreezer, err := rawdb.NewChainFreezer(ancientPath, "", true) +func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, error) { + oldFreezer, err := rawdb.NewChainFreezer(filepath.Join(oldDBPath, "ancient"), "", true) if err != nil { - log.Crit("Failed to open old freezer", "err", err) + return 0, fmt.Errorf("failed to open old freezer: %v", err) } - ancientPathTmp := filepath.Join(chaindataPath, "ancient_new") - newFreezer, err := rawdb.NewChainFreezer(ancientPathTmp, "", false) + defer oldFreezer.Close() + + newFreezer, err := rawdb.NewChainFreezer(filepath.Join(newDBPath, "ancient"), "", false) if err != nil { - log.Crit("Failed to open new freezer", "err", err) + return 0, fmt.Errorf("failed to open new freezer: %v", err) } + defer newFreezer.Close() numAncientsOld, err := oldFreezer.Ancients() if err != nil { - log.Crit("Failed to get number of ancients in old freezer", "err", err) + return 0, fmt.Errorf("failed to get number of ancients in old freezer: %v", err) } + numAncientsNew, err := newFreezer.Ancients() if err != nil { - log.Crit("Failed to get number of ancients in new freezer", "err", err) + return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) } log.Info("Ancient Migration Initial Status", "migrated", numAncientsNew, "total", numAncientsOld) - if err = parMigrateAncientRange(oldFreezer, newFreezer, numAncientsNew, numAncientsOld, *batchSize, readAncientBlockRange, writeAncientBlockRange); err != nil { - log.Crit("Failed to migrate ancient range", "err", err) + if err = parMigrateAncientRange(oldFreezer, newFreezer, numAncientsNew, numAncientsOld, batchSize); err != nil { + return 0, fmt.Errorf("failed to migrate ancient range: %v", err) } - numAncientsOld, err = oldFreezer.Ancients() - if err != nil { - log.Crit("Failed to get number of ancients in old freezer", "err", err) - } numAncientsNew, err = newFreezer.Ancients() if err != nil { - log.Crit("Failed to get number of ancients in new freezer", "err", err) + return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) } log.Info("Ancient Migration End Status", "migrated", numAncientsNew, "total", numAncientsOld) + return numAncientsNew, nil +} - oldFreezer.Close() - newFreezer.Close() - - // Remove old ancient directory and rename new ancient directory - if err := os.RemoveAll(ancientPath); err != nil { - log.Crit("Failed to remove old ancient directory", "err", err) - } - if err := os.Rename(ancientPathTmp, ancientPath); err != nil { - log.Crit("Failed to rename new ancient directory", "err", err) +func migrateNonAncientsDb(oldDbPath, newDbPath string, fromBlock, batchSize uint64) (uint64, error) { + // First copy files from old database to new database + cmd := exec.Command("rsync", "-v", "-a", "--exclude=ancient", oldDbPath+"/", newDbPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return 0, fmt.Errorf("failed to copy old database to new database: %v", err) } - // Reopen the new database - newDB, err := openDB(chaindataPath, ancientPath, dbCache, dbHandles, false) + // Open the new database without access to AncientsDb + newDB, err := rawdb.NewLevelDBDatabase(newDbPath, DB_CACHE, DB_HANDLES, "", false) if err != nil { - log.Crit("Failed to open new database", "err", err) + return 0, fmt.Errorf("failed to open new database: %v", err) } - defer newDB.Close() - numBlocks := GetLastBlockNumber(newDB) + 1 - // numBlocksMigrated := GetLastBlockNumber(newDB) + 1 - numBlocksMigrated := MustAncientLength(newDB) - log.Info("Non-Ancient Migration Initial Status", "migrated", numBlocksMigrated, "total", numBlocks) + // get the last block number + lastBlock := GetLastBlockNumber(newDB) + lastMigratedBlock := readLastMigratedBlock(newDB) - if err := parMigrateRange(newDB, newDB, numBlocksMigrated, numBlocks, *batchSize, readBlockRange, writeBlockRange); err != nil { - log.Crit("Failed to migrate range", "err", err) - } + // if migration was interrupted, start from the last migrated block + fromBlock = max(fromBlock, lastMigratedBlock+1) - log.Info("Migration End Status", "migrated", GetLastBlockNumber(newDB)+1, "total", numBlocks) + log.Info("Non Ancient Migration", "from", fromBlock, "to", lastBlock, "count", lastBlock-fromBlock) - log.Info("Migration complete") -} + for i := fromBlock; i <= lastBlock; i += batchSize { + numbersHash := rawdb.ReadAllHashesInRange(newDB, i, i+batchSize-1) -func createEmptyNewDb(newDBPath string) error { + log.Info("Processing Range", "from", i, "to(inclusve)", i+batchSize-1, "count", len(numbersHash)) + for _, numberHash := range numbersHash { + // read header and body + header := rawdb.ReadHeaderRLP(newDB, numberHash.Hash, numberHash.Number) + body := rawdb.ReadBodyRLP(newDB, numberHash.Hash, numberHash.Number) + // transform header and body + newHeader, err := transformHeader(header) + if err != nil { + return 0, fmt.Errorf("failed to transform header: block %d - %x: %w", numberHash.Number, numberHash.Hash, err) + } + newBody, err := transformBlockBody(body) + if err != nil { + return 0, fmt.Errorf("failed to transform body: block %d - %x: %w", numberHash.Number, numberHash.Hash, err) + } -// copyAllDataExceptAncients copies all data from the old database to the new database except the ancient data which has already been migrated -func copyAllDataExceptAncients(oldDBPath, newDBPath string) error { - // Move the ancient directory up one level, delete everything else, copy over the old data and move the ancient directory back + // write header and body + rawdb.WriteBodyRLP(newDB, numberHash.Hash, numberHash.Number, newBody) - log.Info("Copying over all data except ancients") + if err := newDB.Put(celo1.HeaderKey(numberHash.Number, numberHash.Hash), newHeader); err != nil { + return 0, fmt.Errorf("can't write header to new database: %v", err) + } - // Move the ancient directory up one level - cmd := exec.Command("rsync", "-av", filepath.Join(newDBPath, "geth", "chaindata", "ancient"), filepath.Join(newDBPath, "geth")) - if err := cmd.Run(); err != nil { - return fmt.Errorf("Failed to move ancient directory up one level: %v", err) - } - // Delete everything in chaindata directory that was created when the database was opened - if err := os.RemoveAll(filepath.Join(newDBPath, "geth", "chaindata")); err != nil { - return fmt.Errorf("Failed to clean chaindata directory: %v", err) - } - // Copy everything in the old database to the new database, excluding the ancient directory - cmd = exec.Command("rsync", "-av", filepath.Join(oldDBPath, "celo", "chaindata"), filepath.Join(newDBPath, "geth"), "--exclude", "ancient") - if err := cmd.Run(); err != nil { - return fmt.Errorf("Failed to copy old database to new database: %v", err) - } - // Move the ancient directory back down into the chaindata directory - cmd = exec.Command("rsync", "-av", filepath.Join(newDBPath, "geth", "ancient"), filepath.Join(newDBPath, "geth", "chaindata")) - if err := cmd.Run(); err != nil { - return fmt.Errorf("Failed to move ancient directory back down: %v", err) + if err = writeLastMigratedBlock(newDB, numberHash.Number); err != nil { + return 0, fmt.Errorf("failed to write last migration number: %v", err) + } + + } } - // Delete the extra ancient directory that was up one level - if err := os.RemoveAll(filepath.Join(newDBPath, "geth", "ancient")); err != nil { - return fmt.Errorf("Failed to remove extra ancient directory copy: %v", err) + + // if migration finished, remove the last migration number + if err := deleteLastMigratedBlock(newDB); err != nil { + return 0, fmt.Errorf("failed to delete last migration number: %v", err) } - return nil + return lastBlock - fromBlock + 1, nil } -// Opens a database -func openDB(chaindataPath, ancientPath string, cache int, handles int, readonly bool) (ethdb.Database, error) { - ldb, err := rawdb.Open(rawdb.OpenOptions{ - Type: "leveldb", - Directory: chaindataPath, - AncientsDirectory: ancientPath, - Namespace: "", - Cache: cache, - Handles: handles, - ReadOnly: readonly, - }) - if err != nil { - return nil, err +func createEmptyNewDb(newDBPath string) error { + if err := os.MkdirAll(newDBPath, 0755); err != nil { + return fmt.Errorf("failed to create new database directory: %v", err) } - return ldb, nil + return nil } // RLPBlockRange is a range of blocks in RLP format @@ -240,150 +216,64 @@ type RLPBlockRange struct { tds [][]byte } -// seqMigrateAncientRange migrates ancient data from the old database to the new database sequentially -func seqMigrateAncientRange(oldDb ethdb.Database, newDb ethdb.Database, number, count uint64) error { - _, err := newDb.ModifyAncients(func(op ethdb.AncientWriteOp) error { - hashes, err := oldDb.AncientRange(rawdb.ChainFreezerHashTable, number, count, 0) - if err != nil { - return fmt.Errorf("can't read hashes from old freezer: %v", err) - } - headers, err := oldDb.AncientRange(rawdb.ChainFreezerHeaderTable, number, count, 0) - if err != nil { - return fmt.Errorf("can't read headers from old freezer: %v", err) - } - bodies, err := oldDb.AncientRange(rawdb.ChainFreezerBodiesTable, number, count, 0) - if err != nil { - return fmt.Errorf("can't read bodies from old freezer: %v", err) - } - receipts, err := oldDb.AncientRange(rawdb.ChainFreezerReceiptTable, number, count, 0) - if err != nil { - return fmt.Errorf("can't read receipts from old freezer: %v", err) - } - tds, err := oldDb.AncientRange(rawdb.ChainFreezerDifficultyTable, number, count, 0) - if err != nil { - return fmt.Errorf("can't read tds from old freezer: %v", err) - } - - if len(hashes) != len(headers) || len(headers) != len(bodies) || len(bodies) != len(receipts) || len(receipts) != len(tds) { - return fmt.Errorf("inconsistent data in ancient tables") - } - - for i := 0; i < len(hashes); i++ { - log.Debug("Migrating ancient data", "number", number+uint64(i)) - - newHeader, err := transformHeader(headers[i]) - if err != nil { - return fmt.Errorf("can't transform header: %v", err) - } - newBody, err := transformBlockBody(bodies[i]) - if err != nil { - return fmt.Errorf("can't transform body: %v", err) - } - - if err := op.AppendRaw(rawdb.ChainFreezerHashTable, number+uint64(i), hashes[i]); err != nil { - return fmt.Errorf("can't write hash to Freezer: %v", err) - } - if err := op.AppendRaw(rawdb.ChainFreezerHeaderTable, number+uint64(i), newHeader); err != nil { - return fmt.Errorf("can't write header to Freezer: %v", err) - } - if err := op.AppendRaw(rawdb.ChainFreezerBodiesTable, number+uint64(i), newBody); err != nil { - return fmt.Errorf("can't write body to Freezer: %v", err) - } - if err := op.AppendRaw(rawdb.ChainFreezerReceiptTable, number+uint64(i), receipts[i]); err != nil { - return fmt.Errorf("can't write receipts to Freezer: %v", err) - } - if err := op.AppendRaw(rawdb.ChainFreezerDifficultyTable, number+uint64(i), tds[i]); err != nil { - return fmt.Errorf("can't write td to Freezer: %v", err) - } - } - return nil - }) - - return err -} - -// seqMigrateRange migrates data from the old database to the new database sequentially -func seqMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, count uint64) error { - blockRange, err := readBlockRange(oldDb, start, count) - if err != nil { - return fmt.Errorf("can't read block range: %v", err) - } - err = transformBlockRange(blockRange) - if err != nil { - return fmt.Errorf("can't transform block range: %v", err) - } - err = writeBlockRange(newDb, blockRange) - if err != nil { - return fmt.Errorf("can't write block range: %v", err) - } - return nil -} - -// parMigrateRange migrates ancient data from the old database to the new database in parallel -func parMigrateAncientRange(oldFreezer, newFreezer *rawdb.Freezer, start, end, step uint64, reader func(*rawdb.Freezer, uint64, uint64) (*RLPBlockRange, error), writer func(*rawdb.Freezer, *RLPBlockRange) error) error { - g, ctx := errgroup.WithContext(context.Background()) - readChan := make(chan RLPBlockRange, 10) - transformChan := make(chan RLPBlockRange, 10) - - log.Info("Migrating data", "start", start, "end", end, "step", step) - - g.Go(func() error { return readAncientBlocks(ctx, oldFreezer, start, end, step, readChan, reader) }) - g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) - g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan, writer) }) - - return g.Wait() -} - // parMigrateRange migrates ancient data from the old database to the new database in parallel -func parMigrateRange(oldDb ethdb.Database, newDb ethdb.Database, start, end, step uint64, reader func(ethdb.Database, uint64, uint64) (*RLPBlockRange, error), writer func(ethdb.Database, *RLPBlockRange) error) error { +func parMigrateAncientRange(oldFreezer, newFreezer *rawdb.Freezer, start, end, step uint64) error { g, ctx := errgroup.WithContext(context.Background()) readChan := make(chan RLPBlockRange, 10) transformChan := make(chan RLPBlockRange, 10) log.Info("Migrating data", "start", start, "end", end, "step", step) - g.Go(func() error { return readBlocks(ctx, oldDb, start, end, step, readChan, reader) }) + g.Go(func() error { return readAncientBlocks(ctx, oldFreezer, start, end, step, readChan) }) g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) - g.Go(func() error { return writeBlocks(ctx, newDb, transformChan, writer) }) + g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan) }) return g.Wait() } -func readAncientBlocks(ctx context.Context, oldFreezer *rawdb.Freezer, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange, readBlockRange func(*rawdb.Freezer, uint64, uint64) (*RLPBlockRange, error)) error { +func readAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange) error { defer close(out) - // Read blocks and send them to the out channel - // This could be reading from a database, a file, etc. + for i := startBlock; i < endBlock; i += batchSize { select { case <-ctx.Done(): return ctx.Err() default: count := min(batchSize, endBlock-i+1) - blockRange, err := readAncientBlockRange(oldFreezer, i, count) - if err != nil { - return fmt.Errorf("Failed to read block range: %v", err) + start := i + + blockRange := RLPBlockRange{ + start: start, + hashes: make([][]byte, count), + headers: make([][]byte, count), + bodies: make([][]byte, count), + receipts: make([][]byte, count), + tds: make([][]byte, count), } - out <- *blockRange - } - } - return nil -} + var err error -func readBlocks(ctx context.Context, oldDb ethdb.Database, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange, readBlockRange func(ethdb.Database, uint64, uint64) (*RLPBlockRange, error)) error { - defer close(out) - // Read blocks and send them to the out channel - // This could be reading from a database, a file, etc. - for i := startBlock; i < endBlock; i += batchSize { - select { - case <-ctx.Done(): - return ctx.Err() - default: - count := min(batchSize, endBlock-i+1) - blockRange, err := readBlockRange(oldDb, i, count) + blockRange.hashes, err = freezer.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read hashes from old freezer: %v", err) + } + blockRange.headers, err = freezer.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read headers from old freezer: %v", err) + } + blockRange.bodies, err = freezer.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read bodies from old freezer: %v", err) + } + blockRange.receipts, err = freezer.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0) if err != nil { - return fmt.Errorf("Failed to read block range: %v", err) + return fmt.Errorf("failed to read receipts from old freezer: %v", err) } - out <- *blockRange + blockRange.tds, err = freezer.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read tds from old freezer: %v", err) + } + + out <- blockRange } } return nil @@ -397,9 +287,27 @@ func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RL case <-ctx.Done(): return ctx.Err() default: - err := transformBlockRange(&blockRange) - if err != nil { - return fmt.Errorf("Failed to transform block range: %v", err) + for i := range blockRange.hashes { + blockNumber := blockRange.start + uint64(i) + + newHeader, err := transformHeader(blockRange.headers[i]) + if err != nil { + return fmt.Errorf("can't transform header: %v", err) + } + newBody, err := transformBlockBody(blockRange.bodies[i]) + if err != nil { + return fmt.Errorf("can't transform body: %v", err) + } + + // Check that hashing the new header gives the same hash as the saved hash + newHash := crypto.Keccak256Hash(newHeader) + if !bytes.Equal(blockRange.hashes[i], newHash.Bytes()) { + log.Error("Hash mismatch", "block", blockNumber, "oldHash", common.BytesToHash(blockRange.hashes[i]), "newHash", newHash) + return fmt.Errorf("hash mismatch at block %d", blockNumber) + } + + blockRange.headers[i] = newHeader + blockRange.bodies[i] = newBody } out <- blockRange } @@ -407,176 +315,40 @@ func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RL return nil } -func writeAncientBlocks(ctx context.Context, newFreezer *rawdb.Freezer, in <-chan RLPBlockRange, writeBlockRange func(*rawdb.Freezer, *RLPBlockRange) error) error { - // Write blocks from the in channel to the newDb - for blockRange := range in { - select { - case <-ctx.Done(): - return ctx.Err() - default: - err := writeAncientBlockRange(newFreezer, &blockRange) - if err != nil { - return fmt.Errorf("Failed to write block range: %v", err) - } - } - } - return nil -} - -func writeBlocks(ctx context.Context, newDb ethdb.Database, in <-chan RLPBlockRange, writeBlockRange func(ethdb.Database, *RLPBlockRange) error) error { +func writeAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, in <-chan RLPBlockRange) error { // Write blocks from the in channel to the newDb for blockRange := range in { select { case <-ctx.Done(): return ctx.Err() default: - err := writeBlockRange(newDb, &blockRange) + _, err := freezer.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { + for i := range blockRange.hashes { + blockNumber := blockRange.start + uint64(i) + if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, blockNumber, blockRange.hashes[i]); err != nil { + return fmt.Errorf("can't write hash to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, blockNumber, blockRange.headers[i]); err != nil { + return fmt.Errorf("can't write header to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, blockNumber, blockRange.bodies[i]); err != nil { + return fmt.Errorf("can't write body to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, blockNumber, blockRange.receipts[i]); err != nil { + return fmt.Errorf("can't write receipts to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, blockNumber, blockRange.tds[i]); err != nil { + return fmt.Errorf("can't write td to Freezer: %v", err) + } + } + return nil + }) if err != nil { - return fmt.Errorf("Failed to write block range: %v", err) - } - } - } - return nil -} - -func readAncientBlockRange(freezer *rawdb.Freezer, start, count uint64) (*RLPBlockRange, error) { - blockRange := RLPBlockRange{ - start: start, - hashes: make([][]byte, count), - headers: make([][]byte, count), - bodies: make([][]byte, count), - receipts: make([][]byte, count), - tds: make([][]byte, count), - } - var err error - - log.Debug("Reading ancient blocks", "start", start, "end", start+count-1, "count", count) - - blockRange.hashes, err = freezer.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) - if err != nil { - return nil, fmt.Errorf("Failed to read hashes from old freezer: %v", err) - } - blockRange.headers, err = freezer.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0) - if err != nil { - return nil, fmt.Errorf("Failed to read headers from old freezer: %v", err) - } - blockRange.bodies, err = freezer.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0) - if err != nil { - return nil, fmt.Errorf("Failed to read bodies from old freezer: %v", err) - } - blockRange.receipts, err = freezer.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0) - if err != nil { - return nil, fmt.Errorf("Failed to read receipts from old freezer: %v", err) - } - blockRange.tds, err = freezer.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0) - if err != nil { - return nil, fmt.Errorf("Failed to read tds from old freezer: %v", err) - } - - return &blockRange, nil -} - -func readBlockRange(db ethdb.Database, start, count uint64) (*RLPBlockRange, error) { - blockRange := RLPBlockRange{ - start: start, - hashes: make([][]byte, count), - headers: make([][]byte, count), - bodies: make([][]byte, count), - } - var err error - - log.Debug("Reading blocks", "start", start, "end", start+count-1, "count", count) - - for i := start; i < start+count; i++ { - log.Debug("Reading old data", "number", i) - - blockRange.hashes[i-start], err = db.Get(celo1.HeaderHashKey(i)) - if err != nil { - return nil, fmt.Errorf("Failed to load hash, number: %d, err: %v", i, err) - } - hash := common.BytesToHash(blockRange.hashes[i-start]) - blockRange.headers[i-start], err = db.Get(celo1.HeaderKey(i, hash)) - if err != nil { - return nil, fmt.Errorf("Failed to load header, number: %d, err: %v", i, err) - } - blockRange.bodies[i-start], err = db.Get(celo1.BlockBodyKey(i, hash)) - if err != nil { - return nil, fmt.Errorf("Failed to load body, number: %d, err: %v", i, err) - } - } - - return &blockRange, nil -} - -func transformBlockRange(blockRange *RLPBlockRange) error { - for i := range blockRange.hashes { - blockNumber := blockRange.start + uint64(i) - log.Debug("Migrating data", "number", blockNumber) - - newHeader, err := transformHeader(blockRange.headers[i]) - if err != nil { - return fmt.Errorf("can't transform header: %v", err) - } - newBody, err := transformBlockBody(blockRange.bodies[i]) - if err != nil { - return fmt.Errorf("can't transform body: %v", err) - } - - // Check that hashing the new header gives the same hash as the saved hash - newHash := crypto.Keccak256Hash(newHeader) - if !bytes.Equal(blockRange.hashes[i], newHash.Bytes()) { - log.Error("Hash mismatch", "block", blockNumber, "oldHash", common.BytesToHash(blockRange.hashes[i]), "newHash", newHash) - return fmt.Errorf("hash mismatch at block %d", blockNumber) - } - - blockRange.headers[i] = newHeader - blockRange.bodies[i] = newBody - } - - return nil -} - -func writeAncientBlockRange(freezer *rawdb.Freezer, blockRange *RLPBlockRange) error { - _, err := freezer.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { - for i := range blockRange.hashes { - blockNumber := blockRange.start + uint64(i) - if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, blockNumber, blockRange.hashes[i]); err != nil { - return fmt.Errorf("can't write hash to Freezer: %v", err) - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, blockNumber, blockRange.headers[i]); err != nil { - return fmt.Errorf("can't write header to Freezer: %v", err) + return fmt.Errorf("failed to write block range: %v", err) } - if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, blockNumber, blockRange.bodies[i]); err != nil { - return fmt.Errorf("can't write body to Freezer: %v", err) - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, blockNumber, blockRange.receipts[i]); err != nil { - return fmt.Errorf("can't write receipts to Freezer: %v", err) - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, blockNumber, blockRange.tds[i]); err != nil { - return fmt.Errorf("can't write td to Freezer: %v", err) - } - } - return nil - }) - - return err -} - -func writeBlockRange(db ethdb.Database, blockRange *RLPBlockRange) error { - for i, hashBytes := range blockRange.hashes { - hash := common.BytesToHash(hashBytes) - blockNumber := blockRange.start + uint64(i) - - log.Debug("Writing data", "number", blockNumber) - - if err := db.Put(celo1.HeaderKey(blockNumber, hash), blockRange.headers[i]); err != nil { - return fmt.Errorf("can't write header to new database: %v", err) - } - if err := db.Put(celo1.BlockBodyKey(blockNumber, hash), blockRange.bodies[i]); err != nil { - return fmt.Errorf("can't write body to new database: %v", err) + log.Info("Wrote ancient blocks", "start", blockRange.start, "end", blockRange.start+uint64(len(blockRange.hashes)-1), "count", len(blockRange.hashes)) } } - return nil } @@ -589,72 +361,67 @@ func transformHeader(oldHeader []byte) ([]byte, error) { func transformBlockBody(oldBodyData []byte) ([]byte, error) { // decode body into celo-blockchain Body structure // remove epochSnarkData and randomness data - celoBody := new(CeloBody) - if err := rlp.DecodeBytes(oldBodyData, celoBody); err != nil { - return nil, fmt.Errorf("Failed to RLP decode body: %v", err) + var celoBody struct { + Transactions rlp.RawValue // TODO use types.Transactions to make sure all tx are deserializable + Randomness rlp.RawValue + EpochSnarkData rlp.RawValue + } + if err := rlp.DecodeBytes(oldBodyData, &celoBody); err != nil { + return nil, fmt.Errorf("failed to RLP decode body: %w", err) } - // TODO is this doing everything its supposed to - - // Alternatively, decode into op-geth types.Body structure - // body := new(types.Body) - // newBodyData, err := rlp.EncodeToBytes(body) + // TODO create a types.BlockBody structure and encode it back to []byte // transform into op-geth types.Body structure // since Body is a slice of types.Transactions, we can just remove the randomness and epochSnarkData and add empty array for UnclesHashes newBodyData, err := rlp.EncodeToBytes([]interface{}{celoBody.Transactions, nil}) if err != nil { - log.Crit("Failed to RLP encode body", "err", err) + return nil, fmt.Errorf("failed to RLP encode body: %w", err) } - // encode the new structure into []byte + return newBodyData, nil } -// CeloBody is the body of a celo block -type CeloBody struct { - Transactions rlp.RawValue - Randomness rlp.RawValue - EpochSnarkData rlp.RawValue +// GetLastBlockNumber returns the number of the last block in the database +func GetLastBlockNumber(db ethdb.Database) uint64 { + hash := rawdb.ReadHeadHeaderHash(db) + return *rawdb.ReadHeaderNumber(db, hash) } -// MustAncientLength returns the number of items in the ancients database -func MustAncientLength(db ethdb.Database) uint64 { - byteSize, err := db.AncientSize(rawdb.ChainFreezerHashTable) // TODO why not just use .Ancients()? +func cleanupNonAncientDb(dir string) error { + files, err := os.ReadDir(dir) if err != nil { - log.Crit("Failed to get ancient size", "error", err) + return fmt.Errorf("failed to read directory: %v", err) + } + for _, file := range files { + if file.Name() != "ancient" { + err := os.RemoveAll(filepath.Join(dir, file.Name())) + if err != nil { + return fmt.Errorf("failed to remove file: %v", err) + } + } } - return byteSize / ancientHashSize + return nil } -// func MustAncientLength(freezer rawdb.Freezer) uint64 { -// // byteSize, err := freezer.AncientSize(rawdb.ChainFreezerHashTable) -// // if err != nil { -// // log.Crit("Failed to get ancient size", "error", err) -// // } -// // return byteSize / ancientHashSize -// } - -// GetLastBlockNumber returns the number of the last block in the database -func GetLastBlockNumber(db ethdb.Database) uint64 { - hash := rawdb.ReadHeadHeaderHash(db) - if hash == (common.Hash{}) { - return max(MustAncientLength(db)-1, 0) +// readLastMigratedBlock returns the last migration number. +func readLastMigratedBlock(db ethdb.KeyValueReader) uint64 { + data, err := db.Get([]byte(LAST_MIGRATED_BLOCK_KEY)) + if err != nil { + return 0 } - return *rawdb.ReadHeaderNumber(db, hash) + number := binary.BigEndian.Uint64(data) + return number } -// finds number of items in the ancients database -func findAncientsSize(ldb ethdb.Database, high uint64) uint64 { - // runs a binary search using Ancient.HasAncient to find the first hash it can't find - low := uint64(0) - for low < high { - mid := (low + high) / 2 - if ok, err := ldb.HasAncient(rawdb.ChainFreezerHashTable, mid); ok && err == nil { - low = mid + 1 - } else { - high = mid - } - } - return low +// writeLastMigratedBlock stores the last migration number. +func writeLastMigratedBlock(db ethdb.KeyValueWriter, number uint64) error { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return db.Put([]byte(LAST_MIGRATED_BLOCK_KEY), enc) +} +// deleteLastMigratedBlock removes the last migration number. +func deleteLastMigratedBlock(db ethdb.KeyValueWriter) error { + return db.Delete([]byte(LAST_MIGRATED_BLOCK_KEY)) } From 1050eb931c3d3b030e8adde103fe9bafc4029431 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 7 May 2024 00:28:10 -0300 Subject: [PATCH 16/30] better logging --- op-chain-ops/cmd/celo-dbmigrate/main.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 19232b73387fe..8047fe28736bc 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -48,7 +48,6 @@ func main() { clearAll := flag.Bool("clear-all", false, "Use this to start with a fresh new database") clearNonAncients := flag.Bool("clear-nonAncients", false, "Use to keep migrated ancients, but not non-ancients") - // flag.Usage = usage flag.Parse() log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*verbosity), log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))) @@ -118,7 +117,7 @@ func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, e return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) } - log.Info("Ancient Migration Initial Status", "migrated", numAncientsNew, "total", numAncientsOld) + log.Info("Migration Started", "process", "ancients migration", "startBlock", numAncientsNew, "endBlock", numAncientsOld, "count", numAncientsOld-numAncientsNew+1) if err = parMigrateAncientRange(oldFreezer, newFreezer, numAncientsNew, numAncientsOld, batchSize); err != nil { return 0, fmt.Errorf("failed to migrate ancient range: %v", err) @@ -129,12 +128,13 @@ func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, e return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) } - log.Info("Ancient Migration End Status", "migrated", numAncientsNew, "total", numAncientsOld) + log.Info("Migration End", "process", "ancients migration", "totalBlocks", numAncientsNew) return numAncientsNew, nil } func migrateNonAncientsDb(oldDbPath, newDbPath string, fromBlock, batchSize uint64) (uint64, error) { // First copy files from old database to new database + log.Info("Copy files from old database", "process", "db migration") cmd := exec.Command("rsync", "-v", "-a", "--exclude=ancient", oldDbPath+"/", newDbPath) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -156,12 +156,12 @@ func migrateNonAncientsDb(oldDbPath, newDbPath string, fromBlock, batchSize uint // if migration was interrupted, start from the last migrated block fromBlock = max(fromBlock, lastMigratedBlock+1) - log.Info("Non Ancient Migration", "from", fromBlock, "to", lastBlock, "count", lastBlock-fromBlock) + log.Info("Migration started", "process", "db migration", "startBlock", fromBlock, "endBlock", lastBlock, "count", lastBlock-fromBlock) for i := fromBlock; i <= lastBlock; i += batchSize { numbersHash := rawdb.ReadAllHashesInRange(newDB, i, i+batchSize-1) - log.Info("Processing Range", "from", i, "to(inclusve)", i+batchSize-1, "count", len(numbersHash)) + log.Info("Processing Range", "process", "db migration", "from", i, "to(inclusve)", i+batchSize-1, "count", len(numbersHash)) for _, numberHash := range numbersHash { // read header and body header := rawdb.ReadHeaderRLP(newDB, numberHash.Hash, numberHash.Number) @@ -196,6 +196,7 @@ func migrateNonAncientsDb(oldDbPath, newDbPath string, fromBlock, batchSize uint return 0, fmt.Errorf("failed to delete last migration number: %v", err) } + log.Info("Migration ended", "process", "db migration", "migratedBlocks", lastBlock-fromBlock+1) return lastBlock - fromBlock + 1, nil } From 7d357072b8173568d57559239f3e27df04c60636 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 7 May 2024 16:51:49 -0300 Subject: [PATCH 17/30] refactor: inline parMigrateAncientRange --- op-chain-ops/cmd/celo-dbmigrate/main.go | 30 +++++++++++-------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 8047fe28736bc..b5595efd61f84 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -118,9 +118,20 @@ func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, e } log.Info("Migration Started", "process", "ancients migration", "startBlock", numAncientsNew, "endBlock", numAncientsOld, "count", numAncientsOld-numAncientsNew+1) + g, ctx := errgroup.WithContext(context.Background()) + readChan := make(chan RLPBlockRange, 10) + transformChan := make(chan RLPBlockRange, 10) - if err = parMigrateAncientRange(oldFreezer, newFreezer, numAncientsNew, numAncientsOld, batchSize); err != nil { - return 0, fmt.Errorf("failed to migrate ancient range: %v", err) + log.Info("Migrating data", "start", numAncientsNew, "end", numAncientsOld, "step", batchSize) + + g.Go(func() error { + return readAncientBlocks(ctx, oldFreezer, numAncientsNew, numAncientsOld, batchSize, readChan) + }) + g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) + g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan) }) + + if err = g.Wait(); err != nil { + return 0, fmt.Errorf("failed to migrate ancients: %v", err) } numAncientsNew, err = newFreezer.Ancients() @@ -217,21 +228,6 @@ type RLPBlockRange struct { tds [][]byte } -// parMigrateRange migrates ancient data from the old database to the new database in parallel -func parMigrateAncientRange(oldFreezer, newFreezer *rawdb.Freezer, start, end, step uint64) error { - g, ctx := errgroup.WithContext(context.Background()) - readChan := make(chan RLPBlockRange, 10) - transformChan := make(chan RLPBlockRange, 10) - - log.Info("Migrating data", "start", start, "end", end, "step", step) - - g.Go(func() error { return readAncientBlocks(ctx, oldFreezer, start, end, step, readChan) }) - g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) - g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan) }) - - return g.Wait() -} - func readAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange) error { defer close(out) From 3999ba4fa1e9f05790ce9ddd0b4e6695f2c2d5f6 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 7 May 2024 16:56:31 -0300 Subject: [PATCH 18/30] Remove frozen blocks from nonAncient DB --- op-chain-ops/cmd/celo-dbmigrate/main.go | 32 +++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index b5595efd61f84..2792f938db7ee 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -143,7 +143,7 @@ func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, e return numAncientsNew, nil } -func migrateNonAncientsDb(oldDbPath, newDbPath string, fromBlock, batchSize uint64) (uint64, error) { +func migrateNonAncientsDb(oldDbPath, newDbPath string, lastAncientBlock, batchSize uint64) (uint64, error) { // First copy files from old database to new database log.Info("Copy files from old database", "process", "db migration") cmd := exec.Command("rsync", "-v", "-a", "--exclude=ancient", oldDbPath+"/", newDbPath) @@ -165,7 +165,7 @@ func migrateNonAncientsDb(oldDbPath, newDbPath string, fromBlock, batchSize uint lastMigratedBlock := readLastMigratedBlock(newDB) // if migration was interrupted, start from the last migrated block - fromBlock = max(fromBlock, lastMigratedBlock+1) + fromBlock := max(lastAncientBlock, lastMigratedBlock) + 1 log.Info("Migration started", "process", "db migration", "startBlock", fromBlock, "endBlock", lastBlock, "count", lastBlock-fromBlock) @@ -189,25 +189,33 @@ func migrateNonAncientsDb(oldDbPath, newDbPath string, fromBlock, batchSize uint } // write header and body - rawdb.WriteBodyRLP(newDB, numberHash.Hash, numberHash.Number, newBody) - - if err := newDB.Put(celo1.HeaderKey(numberHash.Number, numberHash.Hash), newHeader); err != nil { - return 0, fmt.Errorf("can't write header to new database: %v", err) - } - - if err = writeLastMigratedBlock(newDB, numberHash.Number); err != nil { - return 0, fmt.Errorf("failed to write last migration number: %v", err) + batch := newDB.NewBatch() + rawdb.WriteBodyRLP(batch, numberHash.Hash, numberHash.Number, newBody) + _ = batch.Put(celo1.HeaderKey(numberHash.Number, numberHash.Hash), newHeader) + _ = writeLastMigratedBlock(batch, numberHash.Number) + if err := batch.Write(); err != nil { + return 0, fmt.Errorf("failed to write header and body: block %d - %x: %w", numberHash.Number, numberHash.Hash, err) } - } } + toBeRemoved := rawdb.ReadAllHashesInRange(newDB, 1, lastAncientBlock) + log.Info("Removing frozen blocks", "process", "db migration", "count", len(toBeRemoved)) + batch := newDB.NewBatch() + for _, numberHash := range toBeRemoved { + rawdb.DeleteBlockWithoutNumber(batch, numberHash.Hash, numberHash.Number) + rawdb.DeleteCanonicalHash(batch, numberHash.Number) + } + if err := batch.Write(); err != nil { + return 0, fmt.Errorf("failed to delete frozen blocks: %w", err) + } + // if migration finished, remove the last migration number if err := deleteLastMigratedBlock(newDB); err != nil { return 0, fmt.Errorf("failed to delete last migration number: %v", err) } + log.Info("Migration ended", "process", "db migration", "migratedBlocks", lastBlock-fromBlock+1, "removedBlocks", len(toBeRemoved)) - log.Info("Migration ended", "process", "db migration", "migratedBlocks", lastBlock-fromBlock+1) return lastBlock - fromBlock + 1, nil } From 49ae16ab52e22df2d54b7c0960b51713f4fc8211 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 7 May 2024 17:11:01 -0300 Subject: [PATCH 19/30] check hash matches on nonAncients migration --- op-chain-ops/cmd/celo-dbmigrate/main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 2792f938db7ee..a80e69c1625d2 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -188,6 +188,11 @@ func migrateNonAncientsDb(oldDbPath, newDbPath string, lastAncientBlock, batchSi return 0, fmt.Errorf("failed to transform body: block %d - %x: %w", numberHash.Number, numberHash.Hash, err) } + if yes, newHash := hasSameHash(newHeader, numberHash.Hash[:]); !yes { + log.Error("Hash mismatch", "block", numberHash.Number, "oldHash", numberHash.Hash, "newHash", newHash) + return 0, fmt.Errorf("hash mismatch at block %d - %x", numberHash.Number, numberHash.Hash) + } + // write header and body batch := newDB.NewBatch() rawdb.WriteBodyRLP(batch, numberHash.Hash, numberHash.Number, newBody) @@ -304,9 +309,7 @@ func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RL return fmt.Errorf("can't transform body: %v", err) } - // Check that hashing the new header gives the same hash as the saved hash - newHash := crypto.Keccak256Hash(newHeader) - if !bytes.Equal(blockRange.hashes[i], newHash.Bytes()) { + if yes, newHash := hasSameHash(newHeader, blockRange.hashes[i]); !yes { log.Error("Hash mismatch", "block", blockNumber, "oldHash", common.BytesToHash(blockRange.hashes[i]), "newHash", newHash) return fmt.Errorf("hash mismatch at block %d", blockNumber) } @@ -320,6 +323,11 @@ func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RL return nil } +func hasSameHash(newHeader, oldHash []byte) (bool, common.Hash) { + newHash := crypto.Keccak256Hash(newHeader) + return bytes.Equal(oldHash, newHash.Bytes()), newHash +} + func writeAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, in <-chan RLPBlockRange) error { // Write blocks from the in channel to the newDb for blockRange := range in { From 577f6ad2499efb95130689bc952dd2cf5b346766 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Tue, 7 May 2024 17:33:12 -0300 Subject: [PATCH 20/30] clean up branch Removes unused code, move code for better separation of concerns. --- op-chain-ops/celo1/block.go | 377 ------------------ op-chain-ops/celo1/block_compatibility.go | 129 ------ op-chain-ops/celo1/db.go | 128 ------ op-chain-ops/celo1/hashing.go | 113 ------ op-chain-ops/cmd/celo-dbmigrate/ancients.go | 191 +++++++++ op-chain-ops/cmd/celo-dbmigrate/db.go | 52 +++ op-chain-ops/cmd/celo-dbmigrate/main.go | 263 +----------- .../celo-dbmigrate/transform.go} | 39 +- op-chain-ops/cmd/celo-dbplay/main.go | 230 ----------- 9 files changed, 282 insertions(+), 1240 deletions(-) delete mode 100644 op-chain-ops/celo1/block.go delete mode 100644 op-chain-ops/celo1/block_compatibility.go delete mode 100644 op-chain-ops/celo1/db.go delete mode 100644 op-chain-ops/celo1/hashing.go create mode 100644 op-chain-ops/cmd/celo-dbmigrate/ancients.go create mode 100644 op-chain-ops/cmd/celo-dbmigrate/db.go rename op-chain-ops/{celo1/istanbul.go => cmd/celo-dbmigrate/transform.go} (58%) delete mode 100644 op-chain-ops/cmd/celo-dbplay/main.go diff --git a/op-chain-ops/celo1/block.go b/op-chain-ops/celo1/block.go deleted file mode 100644 index 3d4d0d5f27134..0000000000000 --- a/op-chain-ops/celo1/block.go +++ /dev/null @@ -1,377 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package types contains data types related to Ethereum consensus. -package celo1 - -import ( - "encoding/binary" - "fmt" - "io" - "math/big" - "reflect" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" -) - -var ( - EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - EmptyUncleHash = rlpHash([]*Header(nil)) // 1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347 - EmptyMixDigest = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000000") -) - -// A BlockNonce is a 64-bit hash which proves (combined with the -// mix-hash) that a sufficient amount of computation has been carried -// out on a block. -type BlockNonce [8]byte - -// EncodeNonce converts the given integer to a block nonce. -func EncodeNonce(i uint64) BlockNonce { - var n BlockNonce - binary.BigEndian.PutUint64(n[:], i) - return n -} - -// Uint64 returns the integer value of a block nonce. -func (n BlockNonce) Uint64() uint64 { - return binary.BigEndian.Uint64(n[:]) -} - -// MarshalText encodes n as a hex string with 0x prefix. -func (n BlockNonce) MarshalText() ([]byte, error) { - return hexutil.Bytes(n[:]).MarshalText() -} - -// UnmarshalText implements encoding.TextUnmarshaler. -func (n *BlockNonce) UnmarshalText(input []byte) error { - return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) -} - -//go:generate gencodec -type Header -field-override headerMarshaling -out gen_header_json.go - -// Header represents a block header in the Ethereum blockchain. -type Header struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - UncleHash common.Hash `json:"sha3Uncles"` - Coinbase common.Address `json:"miner" gencodec:"required"` - Root common.Hash `json:"stateRoot" gencodec:"required"` - TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` - ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` - Bloom types.Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *big.Int `json:"difficulty"` - Number *big.Int `json:"number" gencodec:"required"` - GasLimit uint64 `json:"gasLimit"` - GasUsed uint64 `json:"gasUsed" gencodec:"required"` - Time uint64 `json:"timestamp" gencodec:"required"` - Extra []byte `json:"extraData" gencodec:"required"` - MixDigest common.Hash `json:"mixHash"` - Nonce BlockNonce `json:"nonce"` - - // BaseFee was added by EIP-1559 and is ignored in legacy headers. - BaseFee *big.Int `json:"baseFeePerGas" rlp:"optional"` - - // Used to cache deserialized istanbul extra data - extraLock sync.Mutex - extraError error -} - -// field type overrides for gencodec -type headerMarshaling struct { - Difficulty *hexutil.Big - Number *hexutil.Big - GasLimit hexutil.Uint64 - GasUsed hexutil.Uint64 - Time hexutil.Uint64 - Extra hexutil.Bytes - BaseFee *hexutil.Big - Hash common.Hash `json:"hash"` // adds call to Hash() in MarshalJSON -} - -// Hash returns the block hash of the header, which is simply the keccak256 hash of its -// RLP encoding. -func (h *Header) Hash() common.Hash { - // // Seal is reserved in extra-data. To prove block is signed by the proposer. - // if len(h.Extra) >= IstanbulExtraVanity { - // // This branch is always used during normal Celo operation, but not in all tests. - // if istanbulHeader := IstanbulFilteredHeader(h, true); istanbulHeader != nil { - // return rlpHash(istanbulHeader) - // } - // } - return rlpHash(h) -} - -var headerSize = common.StorageSize(reflect.TypeOf(Header{}).Size()) - -// Size returns the approximate memory used by all internal contents. It is used -// to approximate and limit the memory consumption of various caches. -func (h *Header) Size() common.StorageSize { - return headerSize + common.StorageSize(len(h.Extra)+(h.Number.BitLen()/8)) -} - -// SanityCheck checks a few basic things -- these checks are way beyond what -// any 'sane' production values should hold, and can mainly be used to prevent -// that the unbounded fields are stuffed with junk data to add processing -// overhead -func (h *Header) SanityCheck() error { - if h.Number != nil && !h.Number.IsUint64() { - return fmt.Errorf("too large block number: bitlen %d", h.Number.BitLen()) - } - if h.Difficulty != nil { - if diffLen := h.Difficulty.BitLen(); diffLen > 80 { - return fmt.Errorf("too large block difficulty: bitlen %d", diffLen) - } - } - if eLen := len(h.Extra); eLen > 100*1024 { - return fmt.Errorf("too large block extradata: size %d", eLen) - } - if h.BaseFee != nil { - if bfLen := h.BaseFee.BitLen(); bfLen > 256 { - return fmt.Errorf("too large base fee: bitlen %d", bfLen) - } - } - return nil -} - -// Body is a simple (mutable, non-safe) data container for storing and moving -// a block's data contents (transactions and uncles) together. -type Body struct { - Transactions []*types.Transaction - Randomness []byte - EpochSnarkData []byte -} - -// Block represents an entire block in the Ethereum blockchain. -type Block struct { - header *Header - transactions types.Transactions - randomness []byte - epochSnarkData []byte - - // caches - hash atomic.Value - size atomic.Value - - // Td is used by package core to store the total difficulty - // of the chain up to and including the block. - td *big.Int - - // These fields are used by package eth to track - // inter-peer block relay. - ReceivedAt time.Time - ReceivedFrom interface{} -} - -// "external" block encoding. used for eth protocol, etc. -type extblock struct { - Header *Header - Txs []*types.Transaction - Randomness []byte - EpochSnarkData []byte -} - -// NewBlock creates a new block. The input data is copied, -// changes to header and to the field values will not affect the -// block. -// -// The values of TxHash, ReceiptHash and Bloom in header -// are ignored and set to values derived from the given txs and receipts. -func NewBlock(header *Header, txs []*types.Transaction, receipts []*types.Receipt, hasher TrieHasher) *Block { - b := &Block{header: CopyHeader(header), td: new(big.Int)} - - // TODO: panic if len(txs) != len(receipts) - if len(txs) == 0 { - b.header.TxHash = EmptyRootHash - } else { - b.header.TxHash = DeriveSha(types.Transactions(txs), hasher) - b.transactions = make(types.Transactions, len(txs)) - copy(b.transactions, txs) - } - - if len(receipts) == 0 { - b.header.ReceiptHash = EmptyRootHash - } else { - b.header.ReceiptHash = DeriveSha(types.Receipts(receipts), hasher) - b.header.Bloom = types.CreateBloom(receipts) - } - - // if randomness == nil { - // b.randomness = &EmptyRandomness - // } - - return b -} - -// NewBlockWithHeader creates a block with the given header data. The -// header data is copied, changes to header and to the field values -// will not affect the block. -func NewBlockWithHeader(header *Header) *Block { - return &Block{header: CopyHeader(header), randomness: nil, epochSnarkData: nil} -} - -// CopyHeader creates a deep copy of a block header to prevent side effects from -// modifying a header variable. -func CopyHeader(h *Header) *Header { - cpy := Header{ - ParentHash: h.ParentHash, - UncleHash: h.UncleHash, - Coinbase: h.Coinbase, - Root: h.Root, - TxHash: h.TxHash, - ReceiptHash: h.ReceiptHash, - Bloom: h.Bloom, - Difficulty: h.Difficulty, - Number: new(big.Int), - GasLimit: h.GasLimit, - GasUsed: h.GasUsed, - Time: h.Time, - MixDigest: h.MixDigest, - Nonce: h.Nonce, - } - - if h.Number != nil { - cpy.Number.Set(h.Number) - } - if h.BaseFee != nil { - cpy.BaseFee = new(big.Int).Set(h.BaseFee) - } - if len(h.Extra) > 0 { - cpy.Extra = make([]byte, len(h.Extra)) - copy(cpy.Extra, h.Extra) - } - return &cpy -} - -// DecodeRLP decodes the Ethereum -func (b *Block) DecodeRLP(s *rlp.Stream) error { - var eb extblock - _, size, _ := s.Kind() - if err := s.Decode(&eb); err != nil { - return err - } - b.header, b.transactions, b.randomness, b.epochSnarkData = eb.Header, eb.Txs, eb.Randomness, eb.EpochSnarkData - b.size.Store(common.StorageSize(rlp.ListSize(size))) - return nil -} - -// EncodeRLP serializes b into the Ethereum RLP block format. -func (b *Block) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, extblock{ - Header: b.header, - Txs: b.transactions, - Randomness: b.randomness, - EpochSnarkData: b.epochSnarkData, - }) -} - -// TODO: copies - -func (b *Block) Transactions() types.Transactions { return b.transactions } - -func (b *Block) Transaction(hash common.Hash) *types.Transaction { - for _, transaction := range b.transactions { - if transaction.Hash() == hash { - return transaction - } - } - return nil -} - -func (b *Block) Number() *big.Int { return new(big.Int).Set(b.header.Number) } -func (b *Block) GasLimit() uint64 { return b.header.GasLimit } -func (b *Block) GasUsed() uint64 { return b.header.GasUsed } -func (b *Block) Difficulty() *big.Int { return new(big.Int).Set(b.header.Difficulty) } -func (b *Block) Time() uint64 { return b.header.Time } -func (b *Block) TotalDifficulty() *big.Int { return new(big.Int).Add(b.header.Number, big.NewInt(1)) } - -func (b *Block) NumberU64() uint64 { return b.header.Number.Uint64() } -func (b *Block) MixDigest() common.Hash { return b.header.MixDigest } -func (b *Block) Nonce() uint64 { return binary.BigEndian.Uint64(b.header.Nonce[:]) } -func (b *Block) Bloom() types.Bloom { return b.header.Bloom } -func (b *Block) Coinbase() common.Address { return b.header.Coinbase } -func (b *Block) Root() common.Hash { return b.header.Root } -func (b *Block) ParentHash() common.Hash { return b.header.ParentHash } -func (b *Block) TxHash() common.Hash { return b.header.TxHash } -func (b *Block) ReceiptHash() common.Hash { return b.header.ReceiptHash } -func (b *Block) UncleHash() common.Hash { return b.header.UncleHash } -func (b *Block) Extra() []byte { return common.CopyBytes(b.header.Extra) } - -func (b *Block) BaseFee() *big.Int { - if b.header.BaseFee == nil { - return nil - } - return new(big.Int).Set(b.header.BaseFee) -} - -func (b *Block) Header() *Header { return CopyHeader(b.header) } -func (b *Block) MutableHeader() *Header { return b.header } - -// Body returns the non-header content of the block. -func (b *Block) Body() *Body { return &Body{b.transactions, b.randomness, b.epochSnarkData} } - -// Size returns the true RLP encoded storage size of the block, either by encoding -// and returning it, or returning a previsouly cached value. -func (b *Block) Size() common.StorageSize { - if size := b.size.Load(); size != nil { - return size.(common.StorageSize) - } - c := writeCounter(0) - rlp.Encode(&c, b) - b.size.Store(common.StorageSize(c)) - return common.StorageSize(c) -} - -// SanityCheck can be used to prevent that unbounded fields are -// stuffed with junk data to add processing overhead -func (b *Block) SanityCheck() error { - return b.header.SanityCheck() -} - -type writeCounter common.StorageSize - -func (c *writeCounter) Write(b []byte) (int, error) { - *c += writeCounter(len(b)) - return len(b), nil -} - -// WithHeader returns a new block with the data from b but the header replaced with -// the sealed one. -func (b *Block) WithHeader(header *Header) *Block { - cpy := CopyHeader(header) - - return &Block{ - header: cpy, - transactions: b.transactions, - randomness: b.randomness, - epochSnarkData: b.epochSnarkData, - } -} - -// Hash returns the keccak256 hash of b's header. -// The hash is computed on the first call and cached thereafter. -func (b *Block) Hash() common.Hash { - if hash := b.hash.Load(); hash != nil { - return hash.(common.Hash) - } - v := b.header.Hash() - b.hash.Store(v) - return v -} diff --git a/op-chain-ops/celo1/block_compatibility.go b/op-chain-ops/celo1/block_compatibility.go deleted file mode 100644 index 80dfb89bdcf47..0000000000000 --- a/op-chain-ops/celo1/block_compatibility.go +++ /dev/null @@ -1,129 +0,0 @@ -package celo1 - -import ( - "io" - "math/big" - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" -) - -// This file takes care of supporting older block header formats from before -// the gingerbread fork. - -type beforeGingerbreadHeader struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - Coinbase common.Address `json:"miner" gencodec:"required"` - Root common.Hash `json:"stateRoot" gencodec:"required"` - TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` - ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` - Bloom types.Bloom `json:"logsBloom" gencodec:"required"` - Number *big.Int `json:"number" gencodec:"required"` - GasUsed uint64 `json:"gasUsed" gencodec:"required"` - Time uint64 `json:"timestamp" gencodec:"required"` - Extra []byte `json:"extraData" gencodec:"required"` - - // Used to cache deserialized istanbul extra data - extraLock sync.Mutex - extraError error -} - -type afterGingerbreadHeader Header - -func (h *Header) DecodeRLP(s *rlp.Stream) error { - _, size, _ := s.Kind() - var raw rlp.RawValue - err := s.Decode(&raw) - if err != nil { - return err - } - headerSize := len(raw) - int(size) - numElems, err := rlp.CountValues(raw[headerSize:]) - if err != nil { - return err - } - if numElems == 10 { - // Before gingerbread - decodedHeader := beforeGingerbreadHeader{} - err = rlp.DecodeBytes(raw, &decodedHeader) - - h.ParentHash = decodedHeader.ParentHash - h.Coinbase = decodedHeader.Coinbase - h.Root = decodedHeader.Root - h.TxHash = decodedHeader.TxHash - h.ReceiptHash = decodedHeader.ReceiptHash - h.Bloom = decodedHeader.Bloom - h.Number = decodedHeader.Number - h.GasUsed = decodedHeader.GasUsed - h.Time = decodedHeader.Time - h.Extra = decodedHeader.Extra - } else { - // After gingerbread - decodedHeader := afterGingerbreadHeader{} - err = rlp.DecodeBytes(raw, &decodedHeader) - - h.ParentHash = decodedHeader.ParentHash - h.UncleHash = decodedHeader.UncleHash - h.Coinbase = decodedHeader.Coinbase - h.Root = decodedHeader.Root - h.TxHash = decodedHeader.TxHash - h.ReceiptHash = decodedHeader.ReceiptHash - h.Bloom = decodedHeader.Bloom - h.Difficulty = decodedHeader.Difficulty - h.Number = decodedHeader.Number - h.GasLimit = decodedHeader.GasLimit - h.GasUsed = decodedHeader.GasUsed - h.Time = decodedHeader.Time - h.Extra = decodedHeader.Extra - h.MixDigest = decodedHeader.MixDigest - h.Nonce = decodedHeader.Nonce - h.BaseFee = decodedHeader.BaseFee - } - - return err -} - -func (h *Header) EncodeRLP(w io.Writer) error { - if (h.UncleHash == common.Hash{}) { - // Before gingerbread hardfork Celo did not include all of - // Ethereum's header fields. In that case we must omit the new - // fields from the header when encoding as RLP to maintain the same encoding and hashes. - // `UncleHash` is a safe way to check, since it is the zero hash before - // gingerbread and non-zero after. - rlpFields := []interface{}{ - h.ParentHash, - h.Coinbase, - h.Root, - h.TxHash, - h.ReceiptHash, - h.Bloom, - h.Number, - h.GasUsed, - h.Time, - h.Extra, - } - return rlp.Encode(w, rlpFields) - } else { - rlpFields := []interface{}{ - h.ParentHash, - h.UncleHash, - h.Coinbase, - h.Root, - h.TxHash, - h.ReceiptHash, - h.Bloom, - h.Difficulty, - h.Number, - h.GasLimit, - h.GasUsed, - h.Time, - h.Extra, - h.MixDigest, - h.Nonce, - h.BaseFee, - } - return rlp.Encode(w, rlpFields) - } -} diff --git a/op-chain-ops/celo1/db.go b/op-chain-ops/celo1/db.go deleted file mode 100644 index 38c392271ad63..0000000000000 --- a/op-chain-ops/celo1/db.go +++ /dev/null @@ -1,128 +0,0 @@ -package celo1 - -import ( - "encoding/binary" - "errors" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/rlp" -) - -// ReadHeaderRLP retrieves a block header in its raw RLP database encoding. (copied from rawdb) -func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { - var data []byte - db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - // First try to look up the data in ancient database. Extra hash - // comparison is necessary since ancient database only maintains - // the canonical data. - data, _ = reader.Ancient(rawdb.ChainFreezerHeaderTable, number) - if len(data) != 0 { - return nil - } - // If not, try reading from leveldb - data, _ = db.Get(HeaderKey(number, hash)) - return nil - }) - return data -} - -var ( - // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header - headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td - headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash - headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian) - - blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body - blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - - txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata -) - -// encodeBlockNumber encodes a block number as big endian uint64 -func encodeBlockNumber(number uint64) []byte { - enc := make([]byte, 8) - binary.BigEndian.PutUint64(enc, number) - return enc -} - -// headerKeyPrefix = headerPrefix + num (uint64 big endian) -func headerKeyPrefix(number uint64) []byte { - return append(headerPrefix, encodeBlockNumber(number)...) -} - -// headerKey = headerPrefix + num (uint64 big endian) + hash -func HeaderKey(number uint64, hash common.Hash) []byte { - return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) -} - -// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -func HeaderTDKey(number uint64, hash common.Hash) []byte { - return append(HeaderKey(number, hash), headerTDSuffix...) -} - -// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix -func HeaderHashKey(number uint64) []byte { - return append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) -} - -// headerNumberKey = headerNumberPrefix + hash -func HeaderNumberKey(hash common.Hash) []byte { - return append(headerNumberPrefix, hash.Bytes()...) -} - -// blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash -func BlockBodyKey(number uint64, hash common.Hash) []byte { - return append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) -} - -// blockReceiptsKey = blockReceiptsPrefix + num (uint64 big endian) + hash -func BlockReceiptsKey(number uint64, hash common.Hash) []byte { - return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) -} - -// txLookupKey = txLookupPrefix + hash -func txLookupKey(hash common.Hash) []byte { - return append(txLookupPrefix, hash.Bytes()...) -} - -func ReadCeloHeader(db ethdb.Reader, hash common.Hash, number uint64) (*Header, error) { - data := ReadHeaderRLP(db, hash, number) - if len(data) == 0 { - return nil, errors.New("header not found") - } - - header := new(Header) - if err := rlp.DecodeBytes(data, header); err != nil { - return nil, err - } - return header, nil -} - -// ReadHeader retrieves the block header corresponding to the hash. (copied from rawdb) -func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) (*types.Header, error) { - data := ReadHeaderRLP(db, hash, number) - if len(data) == 0 { - return nil, errors.New("header not found") - } - header := new(types.Header) - if err := rlp.DecodeBytes(data, header); err != nil { - return nil, err - } - return header, nil -} - -// ReadCanonicalHeader retrieves the cannoical block header at the given number. -func ReadCanonicalHeader(db ethdb.Reader, number uint64) (*types.Header, error) { - hash := rawdb.ReadCanonicalHash(db, number) - return ReadHeader(db, hash, number) -} - -// ReadCanonicalHeader retrieves the cannoical block header at the given number. -func ReadCeloCanonicalHeader(db ethdb.Reader, number uint64) (*Header, error) { - hash := rawdb.ReadCanonicalHash(db, number) - return ReadCeloHeader(db, hash, number) -} diff --git a/op-chain-ops/celo1/hashing.go b/op-chain-ops/celo1/hashing.go deleted file mode 100644 index 153f0f56add6a..0000000000000 --- a/op-chain-ops/celo1/hashing.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package celo1 - -import ( - "bytes" - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" -) - -// hasherPool holds LegacyKeccak256 hashers for rlpHash. -var hasherPool = sync.Pool{ - New: func() interface{} { return sha3.NewLegacyKeccak256() }, -} - -// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. -var encodeBufferPool = sync.Pool{ - New: func() interface{} { return new(bytes.Buffer) }, -} - -// rlpHash encodes x and hashes the encoded bytes. -func rlpHash(x interface{}) (h common.Hash) { - sha := hasherPool.Get().(crypto.KeccakState) - defer hasherPool.Put(sha) - sha.Reset() - rlp.Encode(sha, x) - sha.Read(h[:]) - return h -} - -// prefixedRlpHash writes the prefix into the hasher before rlp-encoding x. -// It's used for typed transactions. -func prefixedRlpHash(prefix byte, x interface{}) (h common.Hash) { - sha := hasherPool.Get().(crypto.KeccakState) - defer hasherPool.Put(sha) - sha.Reset() - sha.Write([]byte{prefix}) - rlp.Encode(sha, x) - sha.Read(h[:]) - return h -} - -// TrieHasher is the tool used to calculate the hash of derivable list. -// This is internal, do not use. -type TrieHasher interface { - Reset() - Update([]byte, []byte) - Hash() common.Hash -} - -// DerivableList is the input to DeriveSha. -// It is implemented by the 'Transactions' and 'Receipts' types. -// This is internal, do not use these methods. -type DerivableList interface { - Len() int - EncodeIndex(int, *bytes.Buffer) -} - -func encodeForDerive(list DerivableList, i int, buf *bytes.Buffer) []byte { - buf.Reset() - list.EncodeIndex(i, buf) - // It's really unfortunate that we need to do perform this copy. - // StackTrie holds onto the values until Hash is called, so the values - // written to it must not alias. - return common.CopyBytes(buf.Bytes()) -} - -// DeriveSha creates the tree hashes of transactions and receipts in a block header. -func DeriveSha(list DerivableList, hasher TrieHasher) common.Hash { - hasher.Reset() - - valueBuf := encodeBufferPool.Get().(*bytes.Buffer) - defer encodeBufferPool.Put(valueBuf) - - // StackTrie requires values to be inserted in increasing hash order, which is not the - // order that `list` provides hashes in. This insertion sequence ensures that the - // order is correct. - var indexBuf []byte - for i := 1; i < list.Len() && i <= 0x7f; i++ { - indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i)) - value := encodeForDerive(list, i, valueBuf) - hasher.Update(indexBuf, value) - } - if list.Len() > 0 { - indexBuf = rlp.AppendUint64(indexBuf[:0], 0) - value := encodeForDerive(list, 0, valueBuf) - hasher.Update(indexBuf, value) - } - for i := 0x80; i < list.Len(); i++ { - indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i)) - value := encodeForDerive(list, i, valueBuf) - hasher.Update(indexBuf, value) - } - return hasher.Hash() -} diff --git a/op-chain-ops/cmd/celo-dbmigrate/ancients.go b/op-chain-ops/cmd/celo-dbmigrate/ancients.go new file mode 100644 index 0000000000000..47b5333335a90 --- /dev/null +++ b/op-chain-ops/cmd/celo-dbmigrate/ancients.go @@ -0,0 +1,191 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "golang.org/x/sync/errgroup" +) + +// RLPBlockRange is a range of blocks in RLP format +type RLPBlockRange struct { + start uint64 + hashes [][]byte + headers [][]byte + bodies [][]byte + receipts [][]byte + tds [][]byte +} + +func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, error) { + oldFreezer, err := rawdb.NewChainFreezer(filepath.Join(oldDBPath, "ancient"), "", true) + if err != nil { + return 0, fmt.Errorf("failed to open old freezer: %v", err) + } + defer oldFreezer.Close() + + newFreezer, err := rawdb.NewChainFreezer(filepath.Join(newDBPath, "ancient"), "", false) + if err != nil { + return 0, fmt.Errorf("failed to open new freezer: %v", err) + } + defer newFreezer.Close() + + numAncientsOld, err := oldFreezer.Ancients() + if err != nil { + return 0, fmt.Errorf("failed to get number of ancients in old freezer: %v", err) + } + + numAncientsNew, err := newFreezer.Ancients() + if err != nil { + return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) + } + + log.Info("Migration Started", "process", "ancients migration", "startBlock", numAncientsNew, "endBlock", numAncientsOld, "count", numAncientsOld-numAncientsNew+1) + g, ctx := errgroup.WithContext(context.Background()) + readChan := make(chan RLPBlockRange, 10) + transformChan := make(chan RLPBlockRange, 10) + + log.Info("Migrating data", "start", numAncientsNew, "end", numAncientsOld, "step", batchSize) + + g.Go(func() error { + return readAncientBlocks(ctx, oldFreezer, numAncientsNew, numAncientsOld, batchSize, readChan) + }) + g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) + g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan) }) + + if err = g.Wait(); err != nil { + return 0, fmt.Errorf("failed to migrate ancients: %v", err) + } + + numAncientsNew, err = newFreezer.Ancients() + if err != nil { + return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) + } + + log.Info("Migration End", "process", "ancients migration", "totalBlocks", numAncientsNew) + return numAncientsNew, nil +} + +func readAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange) error { + defer close(out) + + for i := startBlock; i < endBlock; i += batchSize { + select { + case <-ctx.Done(): + return ctx.Err() + default: + count := min(batchSize, endBlock-i+1) + start := i + + blockRange := RLPBlockRange{ + start: start, + hashes: make([][]byte, count), + headers: make([][]byte, count), + bodies: make([][]byte, count), + receipts: make([][]byte, count), + tds: make([][]byte, count), + } + var err error + + blockRange.hashes, err = freezer.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read hashes from old freezer: %v", err) + } + blockRange.headers, err = freezer.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read headers from old freezer: %v", err) + } + blockRange.bodies, err = freezer.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read bodies from old freezer: %v", err) + } + blockRange.receipts, err = freezer.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read receipts from old freezer: %v", err) + } + blockRange.tds, err = freezer.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0) + if err != nil { + return fmt.Errorf("failed to read tds from old freezer: %v", err) + } + + out <- blockRange + } + } + return nil +} + +func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RLPBlockRange) error { + // Transform blocks from the in channel and send them to the out channel + defer close(out) + for blockRange := range in { + select { + case <-ctx.Done(): + return ctx.Err() + default: + for i := range blockRange.hashes { + blockNumber := blockRange.start + uint64(i) + + newHeader, err := transformHeader(blockRange.headers[i]) + if err != nil { + return fmt.Errorf("can't transform header: %v", err) + } + newBody, err := transformBlockBody(blockRange.bodies[i]) + if err != nil { + return fmt.Errorf("can't transform body: %v", err) + } + + if yes, newHash := hasSameHash(newHeader, blockRange.hashes[i]); !yes { + log.Error("Hash mismatch", "block", blockNumber, "oldHash", common.BytesToHash(blockRange.hashes[i]), "newHash", newHash) + return fmt.Errorf("hash mismatch at block %d", blockNumber) + } + + blockRange.headers[i] = newHeader + blockRange.bodies[i] = newBody + } + out <- blockRange + } + } + return nil +} + +func writeAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, in <-chan RLPBlockRange) error { + // Write blocks from the in channel to the newDb + for blockRange := range in { + select { + case <-ctx.Done(): + return ctx.Err() + default: + _, err := freezer.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { + for i := range blockRange.hashes { + blockNumber := blockRange.start + uint64(i) + if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, blockNumber, blockRange.hashes[i]); err != nil { + return fmt.Errorf("can't write hash to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, blockNumber, blockRange.headers[i]); err != nil { + return fmt.Errorf("can't write header to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, blockNumber, blockRange.bodies[i]); err != nil { + return fmt.Errorf("can't write body to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, blockNumber, blockRange.receipts[i]); err != nil { + return fmt.Errorf("can't write receipts to Freezer: %v", err) + } + if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, blockNumber, blockRange.tds[i]); err != nil { + return fmt.Errorf("can't write td to Freezer: %v", err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to write block range: %v", err) + } + log.Info("Wrote ancient blocks", "start", blockRange.start, "end", blockRange.start+uint64(len(blockRange.hashes)-1), "count", len(blockRange.hashes)) + } + } + return nil +} diff --git a/op-chain-ops/cmd/celo-dbmigrate/db.go b/op-chain-ops/cmd/celo-dbmigrate/db.go new file mode 100644 index 0000000000000..37675c07ca232 --- /dev/null +++ b/op-chain-ops/cmd/celo-dbmigrate/db.go @@ -0,0 +1,52 @@ +package main + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +const ( + DB_CACHE = 1024 // size of the cache in MB + DB_HANDLES = 60 // number of handles + LAST_MIGRATED_BLOCK_KEY = "celoLastMigratedBlock" +) + +var ( + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header +) + +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} + +// headerKey = headerPrefix + num (uint64 big endian) + hash +func headerKey(number uint64, hash common.Hash) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// readLastMigratedBlock returns the last migration number. +func readLastMigratedBlock(db ethdb.KeyValueReader) uint64 { + data, err := db.Get([]byte(LAST_MIGRATED_BLOCK_KEY)) + if err != nil { + return 0 + } + number := binary.BigEndian.Uint64(data) + return number +} + +// writeLastMigratedBlock stores the last migration number. +func writeLastMigratedBlock(db ethdb.KeyValueWriter, number uint64) error { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return db.Put([]byte(LAST_MIGRATED_BLOCK_KEY), enc) +} + +// deleteLastMigratedBlock removes the last migration number. +func deleteLastMigratedBlock(db ethdb.KeyValueWriter) error { + return db.Delete([]byte(LAST_MIGRATED_BLOCK_KEY)) +} diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index a80e69c1625d2..9149d21abbcf8 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -1,26 +1,15 @@ package main import ( - "bytes" - "context" - "encoding/binary" "flag" "fmt" "os" "os/exec" "path/filepath" - "github.com/ethereum-optimism/optimism/op-chain-ops/celo1" - - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" "github.com/mattn/go-isatty" - - "golang.org/x/sync/errgroup" ) // How to run: @@ -34,12 +23,6 @@ import ( // Use -clear-all to start with a fresh new database // Use -clear-nonAncients to keep migrated ancients, but not non-ancients -const ( - DB_CACHE = 1024 // size of the cache in MB - DB_HANDLES = 60 // number of handles - LAST_MIGRATED_BLOCK_KEY = "celoLastMigratedBlock" -) - func main() { oldDBPath := flag.String("oldDB", "", "Path to the old database chaindata directory (read-only)") newDBPath := flag.String("newDB", "", "Path to the new database") @@ -94,55 +77,6 @@ func main() { log.Info("Migration Completed", "migratedAncients", numAncientsNew, "migratedNonAncients", numAncientsNonAncients) } -func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, error) { - oldFreezer, err := rawdb.NewChainFreezer(filepath.Join(oldDBPath, "ancient"), "", true) - if err != nil { - return 0, fmt.Errorf("failed to open old freezer: %v", err) - } - defer oldFreezer.Close() - - newFreezer, err := rawdb.NewChainFreezer(filepath.Join(newDBPath, "ancient"), "", false) - if err != nil { - return 0, fmt.Errorf("failed to open new freezer: %v", err) - } - defer newFreezer.Close() - - numAncientsOld, err := oldFreezer.Ancients() - if err != nil { - return 0, fmt.Errorf("failed to get number of ancients in old freezer: %v", err) - } - - numAncientsNew, err := newFreezer.Ancients() - if err != nil { - return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) - } - - log.Info("Migration Started", "process", "ancients migration", "startBlock", numAncientsNew, "endBlock", numAncientsOld, "count", numAncientsOld-numAncientsNew+1) - g, ctx := errgroup.WithContext(context.Background()) - readChan := make(chan RLPBlockRange, 10) - transformChan := make(chan RLPBlockRange, 10) - - log.Info("Migrating data", "start", numAncientsNew, "end", numAncientsOld, "step", batchSize) - - g.Go(func() error { - return readAncientBlocks(ctx, oldFreezer, numAncientsNew, numAncientsOld, batchSize, readChan) - }) - g.Go(func() error { return transformBlocks(ctx, readChan, transformChan) }) - g.Go(func() error { return writeAncientBlocks(ctx, newFreezer, transformChan) }) - - if err = g.Wait(); err != nil { - return 0, fmt.Errorf("failed to migrate ancients: %v", err) - } - - numAncientsNew, err = newFreezer.Ancients() - if err != nil { - return 0, fmt.Errorf("failed to get number of ancients in new freezer: %v", err) - } - - log.Info("Migration End", "process", "ancients migration", "totalBlocks", numAncientsNew) - return numAncientsNew, nil -} - func migrateNonAncientsDb(oldDbPath, newDbPath string, lastAncientBlock, batchSize uint64) (uint64, error) { // First copy files from old database to new database log.Info("Copy files from old database", "process", "db migration") @@ -161,7 +95,8 @@ func migrateNonAncientsDb(oldDbPath, newDbPath string, lastAncientBlock, batchSi defer newDB.Close() // get the last block number - lastBlock := GetLastBlockNumber(newDB) + hash := rawdb.ReadHeadHeaderHash(newDB) + lastBlock := *rawdb.ReadHeaderNumber(newDB, hash) lastMigratedBlock := readLastMigratedBlock(newDB) // if migration was interrupted, start from the last migrated block @@ -196,7 +131,7 @@ func migrateNonAncientsDb(oldDbPath, newDbPath string, lastAncientBlock, batchSi // write header and body batch := newDB.NewBatch() rawdb.WriteBodyRLP(batch, numberHash.Hash, numberHash.Number, newBody) - _ = batch.Put(celo1.HeaderKey(numberHash.Number, numberHash.Hash), newHeader) + _ = batch.Put(headerKey(numberHash.Number, numberHash.Hash), newHeader) _ = writeLastMigratedBlock(batch, numberHash.Number) if err := batch.Write(); err != nil { return 0, fmt.Errorf("failed to write header and body: block %d - %x: %w", numberHash.Number, numberHash.Hash, err) @@ -231,176 +166,6 @@ func createEmptyNewDb(newDBPath string) error { return nil } -// RLPBlockRange is a range of blocks in RLP format -type RLPBlockRange struct { - start uint64 - hashes [][]byte - headers [][]byte - bodies [][]byte - receipts [][]byte - tds [][]byte -} - -func readAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, startBlock, endBlock, batchSize uint64, out chan<- RLPBlockRange) error { - defer close(out) - - for i := startBlock; i < endBlock; i += batchSize { - select { - case <-ctx.Done(): - return ctx.Err() - default: - count := min(batchSize, endBlock-i+1) - start := i - - blockRange := RLPBlockRange{ - start: start, - hashes: make([][]byte, count), - headers: make([][]byte, count), - bodies: make([][]byte, count), - receipts: make([][]byte, count), - tds: make([][]byte, count), - } - var err error - - blockRange.hashes, err = freezer.AncientRange(rawdb.ChainFreezerHashTable, start, count, 0) - if err != nil { - return fmt.Errorf("failed to read hashes from old freezer: %v", err) - } - blockRange.headers, err = freezer.AncientRange(rawdb.ChainFreezerHeaderTable, start, count, 0) - if err != nil { - return fmt.Errorf("failed to read headers from old freezer: %v", err) - } - blockRange.bodies, err = freezer.AncientRange(rawdb.ChainFreezerBodiesTable, start, count, 0) - if err != nil { - return fmt.Errorf("failed to read bodies from old freezer: %v", err) - } - blockRange.receipts, err = freezer.AncientRange(rawdb.ChainFreezerReceiptTable, start, count, 0) - if err != nil { - return fmt.Errorf("failed to read receipts from old freezer: %v", err) - } - blockRange.tds, err = freezer.AncientRange(rawdb.ChainFreezerDifficultyTable, start, count, 0) - if err != nil { - return fmt.Errorf("failed to read tds from old freezer: %v", err) - } - - out <- blockRange - } - } - return nil -} - -func transformBlocks(ctx context.Context, in <-chan RLPBlockRange, out chan<- RLPBlockRange) error { - // Transform blocks from the in channel and send them to the out channel - defer close(out) - for blockRange := range in { - select { - case <-ctx.Done(): - return ctx.Err() - default: - for i := range blockRange.hashes { - blockNumber := blockRange.start + uint64(i) - - newHeader, err := transformHeader(blockRange.headers[i]) - if err != nil { - return fmt.Errorf("can't transform header: %v", err) - } - newBody, err := transformBlockBody(blockRange.bodies[i]) - if err != nil { - return fmt.Errorf("can't transform body: %v", err) - } - - if yes, newHash := hasSameHash(newHeader, blockRange.hashes[i]); !yes { - log.Error("Hash mismatch", "block", blockNumber, "oldHash", common.BytesToHash(blockRange.hashes[i]), "newHash", newHash) - return fmt.Errorf("hash mismatch at block %d", blockNumber) - } - - blockRange.headers[i] = newHeader - blockRange.bodies[i] = newBody - } - out <- blockRange - } - } - return nil -} - -func hasSameHash(newHeader, oldHash []byte) (bool, common.Hash) { - newHash := crypto.Keccak256Hash(newHeader) - return bytes.Equal(oldHash, newHash.Bytes()), newHash -} - -func writeAncientBlocks(ctx context.Context, freezer *rawdb.Freezer, in <-chan RLPBlockRange) error { - // Write blocks from the in channel to the newDb - for blockRange := range in { - select { - case <-ctx.Done(): - return ctx.Err() - default: - _, err := freezer.ModifyAncients(func(aWriter ethdb.AncientWriteOp) error { - for i := range blockRange.hashes { - blockNumber := blockRange.start + uint64(i) - if err := aWriter.AppendRaw(rawdb.ChainFreezerHashTable, blockNumber, blockRange.hashes[i]); err != nil { - return fmt.Errorf("can't write hash to Freezer: %v", err) - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerHeaderTable, blockNumber, blockRange.headers[i]); err != nil { - return fmt.Errorf("can't write header to Freezer: %v", err) - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerBodiesTable, blockNumber, blockRange.bodies[i]); err != nil { - return fmt.Errorf("can't write body to Freezer: %v", err) - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerReceiptTable, blockNumber, blockRange.receipts[i]); err != nil { - return fmt.Errorf("can't write receipts to Freezer: %v", err) - } - if err := aWriter.AppendRaw(rawdb.ChainFreezerDifficultyTable, blockNumber, blockRange.tds[i]); err != nil { - return fmt.Errorf("can't write td to Freezer: %v", err) - } - } - return nil - }) - if err != nil { - return fmt.Errorf("failed to write block range: %v", err) - } - log.Info("Wrote ancient blocks", "start", blockRange.start, "end", blockRange.start+uint64(len(blockRange.hashes)-1), "count", len(blockRange.hashes)) - } - } - return nil -} - -// transformHeader migrates the header from the old format to the new format (works with []byte input output) -func transformHeader(oldHeader []byte) ([]byte, error) { - return celo1.RemoveIstanbulAggregatedSeal(oldHeader) -} - -// transformBlockBody migrates the block body from the old format to the new format (works with []byte input output) -func transformBlockBody(oldBodyData []byte) ([]byte, error) { - // decode body into celo-blockchain Body structure - // remove epochSnarkData and randomness data - var celoBody struct { - Transactions rlp.RawValue // TODO use types.Transactions to make sure all tx are deserializable - Randomness rlp.RawValue - EpochSnarkData rlp.RawValue - } - if err := rlp.DecodeBytes(oldBodyData, &celoBody); err != nil { - return nil, fmt.Errorf("failed to RLP decode body: %w", err) - } - - // TODO create a types.BlockBody structure and encode it back to []byte - - // transform into op-geth types.Body structure - // since Body is a slice of types.Transactions, we can just remove the randomness and epochSnarkData and add empty array for UnclesHashes - newBodyData, err := rlp.EncodeToBytes([]interface{}{celoBody.Transactions, nil}) - if err != nil { - return nil, fmt.Errorf("failed to RLP encode body: %w", err) - } - - return newBodyData, nil -} - -// GetLastBlockNumber returns the number of the last block in the database -func GetLastBlockNumber(db ethdb.Database) uint64 { - hash := rawdb.ReadHeadHeaderHash(db) - return *rawdb.ReadHeaderNumber(db, hash) -} - func cleanupNonAncientDb(dir string) error { files, err := os.ReadDir(dir) if err != nil { @@ -416,25 +181,3 @@ func cleanupNonAncientDb(dir string) error { } return nil } - -// readLastMigratedBlock returns the last migration number. -func readLastMigratedBlock(db ethdb.KeyValueReader) uint64 { - data, err := db.Get([]byte(LAST_MIGRATED_BLOCK_KEY)) - if err != nil { - return 0 - } - number := binary.BigEndian.Uint64(data) - return number -} - -// writeLastMigratedBlock stores the last migration number. -func writeLastMigratedBlock(db ethdb.KeyValueWriter, number uint64) error { - enc := make([]byte, 8) - binary.BigEndian.PutUint64(enc, number) - return db.Put([]byte(LAST_MIGRATED_BLOCK_KEY), enc) -} - -// deleteLastMigratedBlock removes the last migration number. -func deleteLastMigratedBlock(db ethdb.KeyValueWriter) error { - return db.Delete([]byte(LAST_MIGRATED_BLOCK_KEY)) -} diff --git a/op-chain-ops/celo1/istanbul.go b/op-chain-ops/cmd/celo-dbmigrate/transform.go similarity index 58% rename from op-chain-ops/celo1/istanbul.go rename to op-chain-ops/cmd/celo-dbmigrate/transform.go index c5291002b3f2d..9af747d90d890 100644 --- a/op-chain-ops/celo1/istanbul.go +++ b/op-chain-ops/cmd/celo-dbmigrate/transform.go @@ -1,11 +1,14 @@ -package celo1 +package main import ( + "bytes" "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -37,8 +40,8 @@ type IstanbulExtra struct { ParentAggregatedSeal IstanbulAggregatedSeal } -// RemoveIstanbulAggregatedSeal removes the aggregated seal from the header -func RemoveIstanbulAggregatedSeal(header []byte) ([]byte, error) { +// transformHeader removes the aggregated seal from the header +func transformHeader(header []byte) ([]byte, error) { newHeader := new(types.Header) // TODO double check on decoding type err := rlp.DecodeBytes(header, newHeader) if err != nil { @@ -66,3 +69,33 @@ func RemoveIstanbulAggregatedSeal(header []byte) ([]byte, error) { return rlp.EncodeToBytes(newHeader) } + +func hasSameHash(newHeader, oldHash []byte) (bool, common.Hash) { + newHash := crypto.Keccak256Hash(newHeader) + return bytes.Equal(oldHash, newHash.Bytes()), newHash +} + +// transformBlockBody migrates the block body from the old format to the new format (works with []byte input output) +func transformBlockBody(oldBodyData []byte) ([]byte, error) { + // decode body into celo-blockchain Body structure + // remove epochSnarkData and randomness data + var celoBody struct { + Transactions rlp.RawValue // TODO use types.Transactions to make sure all tx are deserializable + Randomness rlp.RawValue + EpochSnarkData rlp.RawValue + } + if err := rlp.DecodeBytes(oldBodyData, &celoBody); err != nil { + return nil, fmt.Errorf("failed to RLP decode body: %w", err) + } + + // TODO create a types.BlockBody structure and encode it back to []byte + + // transform into op-geth types.Body structure + // since Body is a slice of types.Transactions, we can just remove the randomness and epochSnarkData and add empty array for UnclesHashes + newBodyData, err := rlp.EncodeToBytes([]interface{}{celoBody.Transactions, nil}) + if err != nil { + return nil, fmt.Errorf("failed to RLP encode body: %w", err) + } + + return newBodyData, nil +} diff --git a/op-chain-ops/cmd/celo-dbplay/main.go b/op-chain-ops/cmd/celo-dbplay/main.go deleted file mode 100644 index d76915d801333..0000000000000 --- a/op-chain-ops/cmd/celo-dbplay/main.go +++ /dev/null @@ -1,230 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/ethereum-optimism/optimism/op-chain-ops/celo1" - "github.com/mattn/go-isatty" - - "github.com/urfave/cli/v2" - - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" -) - -var ( - dbPathFlag = &cli.StringFlag{ - Name: "db-path", - Usage: "Path to database", - Required: true, - } - dbCacheFlag = &cli.IntFlag{ - Name: "db-cache", - Usage: "LevelDB cache size in mb", - Value: 1024, - } - dbHandlesFlag = &cli.IntFlag{ - Name: "db-handles", - Usage: "LevelDB number of handles", - Value: 60, - } - dryRunFlag = &cli.BoolFlag{ - Name: "dry-run", - Usage: "Dry run the upgrade by not committing the database", - } - - flags = []cli.Flag{ - dbPathFlag, - dbCacheFlag, - dbHandlesFlag, - dryRunFlag, - } - - // from `packages/contracts-bedrock/deploy-config/internal-devnet.json` - EIP1559Denominator = uint64(50) // TODO(pl): select values - EIP1559Elasticity = uint64(10) -) - -var app = &cli.App{ - Name: "migrate", - Usage: "Migrate Celo state to a CeL2 DB", - Flags: flags, - Action: appMain, -} - -func main() { - log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd())))) - if err := app.Run(os.Args); err != nil { - log.Crit("error", "err", err) - } -} - -func appMain(ctx *cli.Context) error { - // Write changes to state to actual state database - dbPath := ctx.String("db-path") - if dbPath == "" { - return fmt.Errorf("must specify --db-path") - } - dbCache := ctx.Int("db-cache") - dbHandles := ctx.Int("db-handles") - // dryRun := ctx.Bool("dry-run") - - log.Info("Opening database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) - ldb, err := openCeloDb(dbPath, dbCache, dbHandles) - if err != nil { - return fmt.Errorf("cannot open DB: %w", err) - } - log.Info("Loaded Celo L1 DB", "db", ldb) - - printStats(ldb) - - // findFirstCorruptedHeader(ldb) - - migrateHeaders(ldb, ldb, 1, 2) - - // tryHeader(ldb, 19814000-1) //just before gingerbread activation (alfajores) - // tryHeader(ldb, 19814000) // just after gingerbread activation (alfajores) - // tryHeader(ldb, 4960000-1) // before churrito block - // tryHeader(ldb, 4960000) // churrito block - // tryHeader(ldb, 4960000) // donut block - // tryHeader(ldb, 9472000) // espresso block - // tryHeader(ldb, 1) // espresso block - - return nil -} - -func migrateHeaders(oldDb, newDb ethdb.Database, start, end uint64) { - // Grab the hash of the tip of the legacy chain. - hash := rawdb.ReadHeadHeaderHash(oldDb) - lastBlockNumber := *rawdb.ReadHeaderNumber(oldDb, hash) - log.Info("Starting from HEAD of the chain", "number", lastBlockNumber) - - // migrate headers from 1 to lastBlockNumber - for i := start; i <= end; i++ { - hash := rawdb.ReadCanonicalHash(oldDb, i) - data := celo1.ReadHeaderRLP(oldDb, hash, i) // skips hash comparison - if len(data) == 0 { - log.Error("failed to load header", "number", i) - } - transformedData, err := celo1.RemoveIstanbulAggregatedSeal(data) - if err != nil { - log.Error("failed to remove istanbul aggregated seal", "number", i, "error", err) - } - - // Write celo2Header to celo2 db - celo2Header := new(types.Header) - err = rlp.DecodeBytes(transformedData, &celo2Header) - if err != nil { - log.Error("failed to decode header", "number", i, "error", err) - } - // rawdb.WriteHeader(newDB, celo2Header) - log.Info("Wrote header", "number", i, "hash", celo2Header.Hash(), "header", celo2Header) - } -} - -func tryHeader(ldb ethdb.Database, number uint64) { - header, err := celo1.ReadCeloCanonicalHeader(ldb, number) - if err != nil { - log.Error("failed to load header", "number", number, "error", err) - } else { - log.Info("loaded header", "number", number, "hash", header.Hash()) - log.Info("Read header", "header", header) - } -} - -// Opens a Celo database, stored in the `celo` subfolder -func openCeloDb(path string, cache int, handles int) (ethdb.Database, error) { - if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { - return nil, err - } - - ancientPath := filepath.Join(path, "ancient") - ldb, err := rawdb.Open(rawdb.OpenOptions{ - Type: "leveldb", - Directory: path, - AncientsDirectory: ancientPath, - Namespace: "", - Cache: cache, - Handles: handles, - ReadOnly: false, - }) - if err != nil { - return nil, err - } - return ldb, nil -} - -// print stats about the database -func printStats(ldb ethdb.Database) { - // Print some stats about the database - chainMetaData := rawdb.ReadChainMetadata(ldb) - for _, v := range chainMetaData { - if len(v) == 2 { - log.Info("Database Metadata", v[0], v[1]) - } else { - log.Info("Database Metadata", v[0], v[1:]) - } - } -} - -func canLoadHeader(ldb ethdb.Database, number uint64) bool { - // log.Trace("Checking if header can be loaded", "number", number) - _, err := celo1.ReadCeloCanonicalHeader(ldb, number) - if err != nil { - log.Debug("failed to load header", "number", number, "error", err) - } - return err == nil -} - -// does a binary search to find the first header that fails to load -func findFirstCorruptedHeader(ldb ethdb.Database) { - // Grab the hash of the tip of the legacy chain. - hash := rawdb.ReadHeadHeaderHash(ldb) - lastBlockNumber := *rawdb.ReadHeaderNumber(ldb, hash) - - log.Info("Starting from HEAD of then chain", "number", lastBlockNumber) - - if !canLoadHeader(ldb, lastBlockNumber) { - log.Error("Can't fetch the last block header, something is wrong") - return - } - - // Binary search from 1 to LastBlockNumber - low := uint64(1) - high := lastBlockNumber - - for low <= high { - mid := (low + high) / 2 - - // Call the test condition function to check if the header can be loaded - if !canLoadHeader(ldb, mid) { - low = mid + 1 - } else { - high = mid - 1 - } - } - - log.Info("Search Finished", "lastBlockThatLoads", high+1, "firstBlockThatFails", high) -} - -// prints the hash of the last x blocks -func printLastBlocks(ldb ethdb.Database, x uint64) { - - hash := rawdb.ReadHeadHeaderHash(ldb) - lastBlockNumber := *rawdb.ReadHeaderNumber(ldb, hash) - toBlockNumber := lastBlockNumber - x - log.Debug("Iterating over blocks", "from", lastBlockNumber, "to", toBlockNumber) - log.Info("Block", "number", lastBlockNumber, "hash", hash) - - for i := lastBlockNumber; i > toBlockNumber; i-- { - header := rawdb.ReadHeader(ldb, hash, i) - log.Info("Block", "number", header.Number, "hash", header.Hash()) - hash = header.ParentHash - } -} From 4a27e32427b1dab1fc37676a2a1cdd28b68864f0 Mon Sep 17 00:00:00 2001 From: alecps Date: Tue, 7 May 2024 23:16:06 -0400 Subject: [PATCH 21/30] decode into new types --- op-chain-ops/cmd/celo-dbmigrate/main.go | 3 +-- op-chain-ops/cmd/celo-dbmigrate/transform.go | 15 +++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index 9149d21abbcf8..c5372b2f5a1b4 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -13,10 +13,9 @@ import ( ) // How to run: -// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-resetDB] [-batchSize 1000] [-verbosity 3] [-clear-all] [-clear-nonAncients] +// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-batchSize 1000] [-verbosity 3] [-clear-all] [-clear-nonAncients] // // This script will migrate block data from the old database to the new database -// The new database will be reset if the -resetDB flag is provided // You can set the log level using the -verbosity flag // The number of ancient records to migrate in one batch can be set using the -batchSize flag // The default batch size is 1000 diff --git a/op-chain-ops/cmd/celo-dbmigrate/transform.go b/op-chain-ops/cmd/celo-dbmigrate/transform.go index 9af747d90d890..e5e500596a2a6 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/transform.go +++ b/op-chain-ops/cmd/celo-dbmigrate/transform.go @@ -16,6 +16,7 @@ var ( IstanbulExtraVanity = 32 // Fixed number of extra-data bytes reserved for validator vanity ) +// IstanbulAggregatedSeal is the aggregated seal for Istanbul blocks type IstanbulAggregatedSeal struct { // Bitmap is a bitmap having an active bit for each validator that signed this block Bitmap *big.Int @@ -25,6 +26,7 @@ type IstanbulAggregatedSeal struct { Round *big.Int } +// IstanbulExtra is the extra-data for Istanbul blocks type IstanbulExtra struct { // AddedValidators are the validators that have been added in the block AddedValidators []common.Address @@ -42,7 +44,7 @@ type IstanbulExtra struct { // transformHeader removes the aggregated seal from the header func transformHeader(header []byte) ([]byte, error) { - newHeader := new(types.Header) // TODO double check on decoding type + newHeader := types.Header{} err := rlp.DecodeBytes(header, newHeader) if err != nil { return nil, err @@ -80,7 +82,7 @@ func transformBlockBody(oldBodyData []byte) ([]byte, error) { // decode body into celo-blockchain Body structure // remove epochSnarkData and randomness data var celoBody struct { - Transactions rlp.RawValue // TODO use types.Transactions to make sure all tx are deserializable + Transactions types.Transactions Randomness rlp.RawValue EpochSnarkData rlp.RawValue } @@ -88,11 +90,12 @@ func transformBlockBody(oldBodyData []byte) ([]byte, error) { return nil, fmt.Errorf("failed to RLP decode body: %w", err) } - // TODO create a types.BlockBody structure and encode it back to []byte - // transform into op-geth types.Body structure - // since Body is a slice of types.Transactions, we can just remove the randomness and epochSnarkData and add empty array for UnclesHashes - newBodyData, err := rlp.EncodeToBytes([]interface{}{celoBody.Transactions, nil}) + newBody := types.Body{ + Transactions: celoBody.Transactions, + Uncles: []*types.Header{}, + } + newBodyData, err := rlp.EncodeToBytes(newBody) if err != nil { return nil, fmt.Errorf("failed to RLP encode body: %w", err) } From df6cf15da2124c239be130ae4a11ca33084001a6 Mon Sep 17 00:00:00 2001 From: Mariano Cortesi Date: Wed, 8 May 2024 11:37:14 -0300 Subject: [PATCH 22/30] fix transformHeader --- op-chain-ops/cmd/celo-dbmigrate/transform.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/transform.go b/op-chain-ops/cmd/celo-dbmigrate/transform.go index e5e500596a2a6..650c64b72230a 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/transform.go +++ b/op-chain-ops/cmd/celo-dbmigrate/transform.go @@ -44,8 +44,8 @@ type IstanbulExtra struct { // transformHeader removes the aggregated seal from the header func transformHeader(header []byte) ([]byte, error) { - newHeader := types.Header{} - err := rlp.DecodeBytes(header, newHeader) + newHeader := new(types.Header) + err := rlp.DecodeBytes(header, &newHeader) if err != nil { return nil, err } From c0883593724fe4c64ecb3e9877f9a8443ec94ab1 Mon Sep 17 00:00:00 2001 From: alecps Date: Wed, 8 May 2024 18:21:24 -0400 Subject: [PATCH 23/30] make old freezer not readonly so that .meta files are created --- op-chain-ops/cmd/celo-dbmigrate/ancients.go | 2 +- op-chain-ops/cmd/celo-dbmigrate/main.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/ancients.go b/op-chain-ops/cmd/celo-dbmigrate/ancients.go index 47b5333335a90..10f3ade5dfd17 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/ancients.go +++ b/op-chain-ops/cmd/celo-dbmigrate/ancients.go @@ -23,7 +23,7 @@ type RLPBlockRange struct { } func migrateAncientsDb(oldDBPath, newDBPath string, batchSize uint64) (uint64, error) { - oldFreezer, err := rawdb.NewChainFreezer(filepath.Join(oldDBPath, "ancient"), "", true) + oldFreezer, err := rawdb.NewChainFreezer(filepath.Join(oldDBPath, "ancient"), "", false) // TODO can't be readonly because we need the .meta files to be created if err != nil { return 0, fmt.Errorf("failed to open old freezer: %v", err) } diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index c5372b2f5a1b4..d52f4e7649322 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -13,7 +13,7 @@ import ( ) // How to run: -// go run main.go -oldDB /path/to/oldDB -newDB /path/to/newDB [-batchSize 1000] [-verbosity 3] [-clear-all] [-clear-nonAncients] +// go run ./op-chain-ops/cmd/celo-dbmigrate -oldDB /path/to/oldDB -newDB /path/to/newDB [-batchSize 1000] [-verbosity 3] [-clear-all] [-clear-nonAncients] // // This script will migrate block data from the old database to the new database // You can set the log level using the -verbosity flag @@ -68,12 +68,12 @@ func main() { log.Crit("Failed to migrate ancients database", "err", err) } - var numAncientsNonAncients uint64 - if numAncientsNonAncients, err = migrateNonAncientsDb(*oldDBPath, *newDBPath, numAncientsNew-1, *batchSize); err != nil { + var numNonAncients uint64 + if numNonAncients, err = migrateNonAncientsDb(*oldDBPath, *newDBPath, numAncientsNew-1, *batchSize); err != nil { log.Crit("Failed to migrate non-ancients database", "err", err) } - log.Info("Migration Completed", "migratedAncients", numAncientsNew, "migratedNonAncients", numAncientsNonAncients) + log.Info("Migration Completed", "migratedAncients", numAncientsNew, "migratedNonAncients", numNonAncients) } func migrateNonAncientsDb(oldDbPath, newDbPath string, lastAncientBlock, batchSize uint64) (uint64, error) { From 47d9e9295bd4fb1d504640a90cd347df5748a651 Mon Sep 17 00:00:00 2001 From: alecps Date: Fri, 10 May 2024 12:08:47 -0400 Subject: [PATCH 24/30] update go.mod to use migration op-geth branch --- go.mod | 4 +++- go.sum | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2ec9ebbe3586d..ab7d15dc7bb8d 100644 --- a/go.mod +++ b/go.mod @@ -222,4 +222,6 @@ require ( //replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240220150757-6c346a984571 //replace github.com/ethereum-optimism/superchain-registry/superchain => ../superchain-registry/superchain -replace github.com/ethereum/go-ethereum => ../op-geth +replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240509214009-57109af77703 + +//replace github.com/ethereum/go-ethereum => ../op-geth diff --git a/go.sum b/go.sum index 3570de8274cb0..acf0dbdc287d1 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/celo-org/op-geth v0.0.0-20240509214009-57109af77703 h1:BU8PvKoZ4alcn9SjQVc3nkrF+sUGThQ1VfWkhZig5IM= +github.com/celo-org/op-geth v0.0.0-20240509214009-57109af77703/go.mod h1:dQVCa+D5zi0oJJyrP3nwWciio0BJLZ2urH8OU7RJ2qs= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= From 7452e6c7172905188b0fdb572aabfae74653cf0c Mon Sep 17 00:00:00 2001 From: alecps Date: Tue, 14 May 2024 14:54:06 -0400 Subject: [PATCH 25/30] add configurable memory limit --- op-chain-ops/cmd/celo-dbmigrate/main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index d52f4e7649322..fa70be4d1d4a6 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime/debug" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/log" @@ -13,12 +14,13 @@ import ( ) // How to run: -// go run ./op-chain-ops/cmd/celo-dbmigrate -oldDB /path/to/oldDB -newDB /path/to/newDB [-batchSize 1000] [-verbosity 3] [-clear-all] [-clear-nonAncients] +// go run ./op-chain-ops/cmd/celo-dbmigrate -oldDB /path/to/oldDB -newDB /path/to/newDB [-batchSize 1000] [-verbosity 3] [-memoryLimit 7500] [-clear-all] [-clear-nonAncients] // // This script will migrate block data from the old database to the new database // You can set the log level using the -verbosity flag // The number of ancient records to migrate in one batch can be set using the -batchSize flag // The default batch size is 1000 +// You can set a memory limit in MB using the -memoryLimit flag. Defaults to 7500 MB // Use -clear-all to start with a fresh new database // Use -clear-nonAncients to keep migrated ancients, but not non-ancients @@ -27,6 +29,7 @@ func main() { newDBPath := flag.String("newDB", "", "Path to the new database") batchSize := flag.Uint64("batchSize", 10000, "Number of records to migrate in one batch") verbosity := flag.Uint64("verbosity", 3, "Log level (0:crit, 1:err, 2:warn, 3:info, 4:debug, 5:trace)") + memoryLimit := flag.Int64("memoryLimit", 7500, "Memory limit in MB") clearAll := flag.Bool("clear-all", false, "Use this to start with a fresh new database") clearNonAncients := flag.Bool("clear-nonAncients", false, "Use to keep migrated ancients, but not non-ancients") @@ -34,6 +37,8 @@ func main() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*verbosity), log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))) + debug.SetMemoryLimit(*memoryLimit * 1 << 20) // Set memory limit, converting from MB to bytes + var err error // check that `rsync` command is available From 5f092c884ef4566a7257dcb88efd75a05990f9db Mon Sep 17 00:00:00 2001 From: alecps Date: Tue, 14 May 2024 14:57:20 -0400 Subject: [PATCH 26/30] add comment about memory --- op-chain-ops/cmd/celo-dbmigrate/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-chain-ops/cmd/celo-dbmigrate/main.go b/op-chain-ops/cmd/celo-dbmigrate/main.go index fa70be4d1d4a6..6998ef5e4fedd 100644 --- a/op-chain-ops/cmd/celo-dbmigrate/main.go +++ b/op-chain-ops/cmd/celo-dbmigrate/main.go @@ -20,7 +20,7 @@ import ( // You can set the log level using the -verbosity flag // The number of ancient records to migrate in one batch can be set using the -batchSize flag // The default batch size is 1000 -// You can set a memory limit in MB using the -memoryLimit flag. Defaults to 7500 MB +// You can set a memory limit in MB using the -memoryLimit flag. Defaults to 7500 MB. Make sure to set a limit that is less than your machine's available memory. // Use -clear-all to start with a fresh new database // Use -clear-nonAncients to keep migrated ancients, but not non-ancients From 02de03846263fd984bdd5ecceff45fe0bc83d094 Mon Sep 17 00:00:00 2001 From: Javier Cortejoso Date: Wed, 15 May 2024 11:36:37 +0200 Subject: [PATCH 27/30] Added celo-dbmigrate Makefile target --- op-chain-ops/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/op-chain-ops/Makefile b/op-chain-ops/Makefile index 69fba75999c6c..29d2612cb1935 100644 --- a/op-chain-ops/Makefile +++ b/op-chain-ops/Makefile @@ -7,6 +7,9 @@ ecotone-scalar: receipt-reference-builder: go build -o ./bin/receipt-reference-builder ./cmd/receipt-reference-builder/*.go +celo-dbmigrate: + go build -o ./bin/celo-dbmigrate ./cmd/celo-dbmigrate/*.go + celo-migrate: go build -o ./bin/celo-migrate ./cmd/celo-migrate/*.go From 0208fb3274a2eae50fe76c20c72c8e8d1ebd3f6d Mon Sep 17 00:00:00 2001 From: Javier Cortejoso Date: Wed, 15 May 2024 11:37:56 +0200 Subject: [PATCH 28/30] Added dockerfile for celo-dbmigrate and celo-migrate tools --- op-chain-ops/Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 op-chain-ops/Dockerfile diff --git a/op-chain-ops/Dockerfile b/op-chain-ops/Dockerfile new file mode 100644 index 0000000000000..7ab849471454d --- /dev/null +++ b/op-chain-ops/Dockerfile @@ -0,0 +1,30 @@ +FROM golang:1.21.1-alpine3.18 as builder + +RUN apk --no-cache add make + +COPY ./go.mod /app/go.mod +COPY ./go.sum /app/go.sum + +WORKDIR /app + +RUN go mod download + +COPY ./op-bindings /app/op-bindings +COPY ./op-service /app/op-service +COPY ./op-node /app/op-node +COPY ./op-chain-ops /app/op-chain-ops +WORKDIR /app/op-chain-ops +RUN make celo-dbmigrate celo-migrate + +FROM alpine:3.18 +RUN apk --no-cache add ca-certificates bash rsync + +# RUN addgroup -S app && adduser -S app -G app +# USER app +WORKDIR /app + +COPY --from=builder /app/op-chain-ops/bin/celo-dbmigrate /app +COPY --from=builder /app/op-chain-ops/bin/celo-migrate /app +ENV PATH="/app:${PATH}" + +ENTRYPOINT ["/app/celo-dbmigrate"] From 85a8aa263b9c65d09d68d23966846c108ac485bd Mon Sep 17 00:00:00 2001 From: Javier Cortejoso Date: Mon, 20 May 2024 15:14:52 +0200 Subject: [PATCH 29/30] Workflow for running cel2-migration-tool --- .github/workflows/docker-build-scan.yaml | 132 ++++++++--------------- 1 file changed, 45 insertions(+), 87 deletions(-) diff --git a/.github/workflows/docker-build-scan.yaml b/.github/workflows/docker-build-scan.yaml index 30ad5196c7fec..4ee1c584f5dcf 100644 --- a/.github/workflows/docker-build-scan.yaml +++ b/.github/workflows/docker-build-scan.yaml @@ -1,92 +1,50 @@ name: Docker Build Scan on: + pull_request: + branches: + - "master" + - "celo*" workflow_dispatch: jobs: - Build-Scan-Container-op-ufm: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-ufm/Dockerfile - - Build-Scan-Container-ops-bedrock-l1: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: ops-bedrock/Dockerfile.l1 - context: ops-bedrock - - Build-Scan-Container-ops-bedrock-l2: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: ops-bedrock/Dockerfile.l2 - context: ops-bedrock - - Build-Scan-Container-indexer: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: indexer/Dockerfile - - Build-Scan-Container-op-heartbeat: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-heartbeat/Dockerfile - - Build-Scan-Container-op-exporter: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-exporter/Dockerfile - - Build-Scan-Container-op-program: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-program/Dockerfile - - Build-Scan-Container-ops-bedrock: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: ops-bedrock/Dockerfile.stateviz - - Build-Scan-Container-ci-builder: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: ops/docker/ci-builder/Dockerfile - - Build-Scan-Container-proxyd: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: proxyd/Dockerfile - - Build-Scan-Container-op-node: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-node/Dockerfile - - Build-Scan-Container-op-batcher: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-batcher/Dockerfile - - Build-Scan-Container-indexer-ui: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: indexer/ui/Dockerfile - - Build-Scan-Container-op-proposer: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-proposer/Dockerfile - - Build-Scan-Container-op-challenger: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-challenger/Dockerfile - - Build-Scan-Container-endpoint-monitor: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: endpoint-monitor/Dockerfile - - Build-Scan-Container-opwheel: - uses: celo-org/reusable-workflows/.github/workflows/container-cicd-local.yaml@v1.11.2 - with: - dockerfile: op-wheel/Dockerfile - + detect-files-changed: + runs-on: ubuntu-latest + outputs: + files-changed: ${{ steps.detect-files-changed.outputs.all_changed_files }} + steps: + - uses: actions/checkout@v4 + - name: Detect files changed + id: detect-files-changed + uses: tj-actions/changed-files@v44 + with: + separator: ',' + + build-cel2-migration-tool: + runs-on: ubuntu-latest + needs: detect-files-changed + if: | + contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/cmd/celo-dbmigrate') || + contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/cmd/celo-migrate') || + contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/Dockerfile') + permissions: + contents: read + id-token: write + security-events: write + steps: + - uses: actions/checkout@v4 + - name: Login at GCP Artifact Registry + uses: celo-org/reusable-workflows/.github/actions/auth-gcp-artifact-registry@v2.0 + with: + workload-id-provider: 'projects/1094498259535/locations/global/workloadIdentityPools/gh-optimism/providers/github-by-repos' + service-account: 'celo-optimism-gh@devopsre.iam.gserviceaccount.com' + docker-gcp-registries: us-west1-docker.pkg.dev + - name: Build and push container + uses: celo-org/reusable-workflows/.github/actions/build-container@v2.0 + with: + platforms: linux/amd64 + registry: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/cel2-migration-tool + tags: ${{ github.sha }} + context: ./ + dockerfile: ./op-chain-ops/Dockerfile + push: true + trivy: false From 1aa71672059fc02f941409ac454c563be149aef1 Mon Sep 17 00:00:00 2001 From: Javier Cortejoso Date: Mon, 20 May 2024 15:21:34 +0200 Subject: [PATCH 30/30] Update cel2-migration-tool image registry --- .github/workflows/docker-build-scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-scan.yaml b/.github/workflows/docker-build-scan.yaml index 4ee1c584f5dcf..631cdb37b0c87 100644 --- a/.github/workflows/docker-build-scan.yaml +++ b/.github/workflows/docker-build-scan.yaml @@ -42,7 +42,7 @@ jobs: uses: celo-org/reusable-workflows/.github/actions/build-container@v2.0 with: platforms: linux/amd64 - registry: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/cel2-migration-tool + registry: us-west1-docker.pkg.dev/devopsre/dev-images/cel2-migration-tool tags: ${{ github.sha }} context: ./ dockerfile: ./op-chain-ops/Dockerfile