From dbaab7c87f164322ee767694232c980973c4a427 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Wed, 10 Apr 2024 13:06:21 +0200 Subject: [PATCH 01/10] Add `celo-migrate` tool --- op-chain-ops/Makefile | 3 + .../{op-migrate => celo-migrate}/README.md | 2 +- op-chain-ops/cmd/celo-migrate/main.go | 252 ++++++++++++++ .../cmd/celo-migrate/test_config.json | 74 +++++ op-chain-ops/cmd/op-migrate/main.go | 312 ------------------ 5 files changed, 330 insertions(+), 313 deletions(-) rename op-chain-ops/cmd/{op-migrate => celo-migrate}/README.md (96%) create mode 100644 op-chain-ops/cmd/celo-migrate/main.go create mode 100644 op-chain-ops/cmd/celo-migrate/test_config.json delete mode 100644 op-chain-ops/cmd/op-migrate/main.go diff --git a/op-chain-ops/Makefile b/op-chain-ops/Makefile index 1e39aca416949..69fba75999c6c 100644 --- a/op-chain-ops/Makefile +++ b/op-chain-ops/Makefile @@ -7,6 +7,9 @@ ecotone-scalar: receipt-reference-builder: go build -o ./bin/receipt-reference-builder ./cmd/receipt-reference-builder/*.go +celo-migrate: + go build -o ./bin/celo-migrate ./cmd/celo-migrate/*.go + test: go test ./... diff --git a/op-chain-ops/cmd/op-migrate/README.md b/op-chain-ops/cmd/celo-migrate/README.md similarity index 96% rename from op-chain-ops/cmd/op-migrate/README.md rename to op-chain-ops/cmd/celo-migrate/README.md index 700aab909a92f..b8f50ead12b0f 100644 --- a/op-chain-ops/cmd/op-migrate/README.md +++ b/op-chain-ops/cmd/celo-migrate/README.md @@ -1,4 +1,4 @@ -# State migrator +# Celo L1 -> Cel2 migrator This tool allows migrating the state of a Celo chain to a genesis block for a CeL2 chain. diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go new file mode 100644 index 0000000000000..2c4379ec0f4db --- /dev/null +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "fmt" + "math/big" + "os" + "path/filepath" + + "github.com/ethereum-optimism/optimism/op-chain-ops/genesis" + "github.com/ethereum-optimism/optimism/op-service/jsonutil" + "github.com/mattn/go-isatty" + + "github.com/urfave/cli/v2" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +var ( + deployConfigFlag = &cli.PathFlag{ + Name: "deploy-config", + Usage: "Path to deploy config file", + Required: true, + } + l1DeploymentsFlag = &cli.PathFlag{ + Name: "l1-deployments", + Usage: "Path to L1 deployments JSON file. Cannot be used with --deployment-dir", + } + l1RPCFlag = &cli.StringFlag{ + Name: "l1-rpc", + Usage: "RPC URL for an Ethereum L1 node. Cannot be used with --l1-starting-block", + } + outfileL2Flag = &cli.PathFlag{ + Name: "outfile.l2", + Usage: "Path to L2 genesis output file", + } + outfileRollupFlag = &cli.PathFlag{ + Name: "outfile.rollup", + Usage: "Path to rollup output file", + } + + dbPathFlag = &cli.StringFlag{ + Name: "db-path", + Usage: "Path to database", + Required: true, + } + dbCacheFlag = &cli.IntFlag{ + Name: "db-cache", + Usage: "LevelDB cache size in mb", + Value: 1024, + } + dbHandlesFlag = &cli.IntFlag{ + Name: "db-handles", + Usage: "LevelDB number of handles", + Value: 60, + } + dryRunFlag = &cli.BoolFlag{ + Name: "dry-run", + Usage: "Dry run the upgrade by not committing the database", + } + + flags = []cli.Flag{ + deployConfigFlag, + l1DeploymentsFlag, + l1RPCFlag, + outfileL2Flag, + outfileRollupFlag, + dbPathFlag, + dbCacheFlag, + dbHandlesFlag, + dryRunFlag, + } + + // from `packages/contracts-bedrock/deploy-config/internal-devnet.json` + EIP1559Denominator = uint64(50) // TODO(pl): select values + EIP1559Elasticity = uint64(10) +) + +func main() { + log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd())))) + + app := &cli.App{ + Name: "migrate", + Usage: "Migrate Celo state to a CeL2 DB", + Flags: flags, + Action: func(ctx *cli.Context) error { + deployConfig := ctx.Path("deploy-config") + if deployConfig == "" { + return fmt.Errorf("must specify --deploy-config") + } + log.Info("Deploy config", "path", deployConfig) + config, err := genesis.NewDeployConfig(deployConfig) + if err != nil { + return err + } + + // Try reading the L1 deployment information + l1Deployments := ctx.Path("l1-deployments") + if l1Deployments == "" { + return fmt.Errorf("must specify --l1-deployments") + } + deployments, err := genesis.NewL1Deployments(l1Deployments) + if err != nil { + return fmt.Errorf("cannot read L1 deployments at %s: %w", l1Deployments, err) + } + config.SetDeployments(deployments) + + // Get latest block information from L1 + l1RPC := ctx.String("l1-rpc") + if l1RPC == "" { + return fmt.Errorf("must specify --l1-rpc") + } + + var l1StartBlock *types.Block + client, err := ethclient.Dial(l1RPC) + if err != nil { + return fmt.Errorf("cannot dial %s: %w", l1RPC, err) + } + + if config.L1StartingBlockTag == nil { + l1StartBlock, err = client.BlockByNumber(context.Background(), nil) + if err != nil { + return fmt.Errorf("cannot fetch latest block: %w", err) + } + tag := rpc.BlockNumberOrHashWithHash(l1StartBlock.Hash(), true) + config.L1StartingBlockTag = (*genesis.MarshalableRPCBlockNumberOrHash)(&tag) + } else if config.L1StartingBlockTag.BlockHash != nil { + l1StartBlock, err = client.BlockByHash(context.Background(), *config.L1StartingBlockTag.BlockHash) + if err != nil { + return fmt.Errorf("cannot fetch block by hash: %w", err) + } + } else if config.L1StartingBlockTag.BlockNumber != nil { + l1StartBlock, err = client.BlockByNumber(context.Background(), big.NewInt(config.L1StartingBlockTag.BlockNumber.Int64())) + if err != nil { + return fmt.Errorf("cannot fetch block by number: %w", err) + } + } + + // Ensure that there is a starting L1 block + if l1StartBlock == nil { + return fmt.Errorf("no starting L1 block") + } + + // Sanity check the config. Do this after filling in the L1StartingBlockTag + // if it is not defined. + if err := config.Check(); err != nil { + return err + } + + log.Info("Using L1 Start Block", "number", l1StartBlock.Number(), "hash", l1StartBlock.Hash().Hex()) + + // Build the L2 genesis block + l2Genesis, err := genesis.BuildL2Genesis(config, l1StartBlock) + if err != nil { + return fmt.Errorf("error creating l2 genesis: %w", err) + } + + l2GenesisBlock := l2Genesis.ToBlock() + rollupConfig, err := config.RollupConfig(l1StartBlock, l2GenesisBlock.Hash(), l2GenesisBlock.Number().Uint64()) + if err != nil { + return err + } + if err := rollupConfig.Check(); err != nil { + return fmt.Errorf("generated rollup config does not pass validation: %w", err) + } + + outfileL2 := ctx.Path("outfile.l2") + if outfileL2 == "" { + return fmt.Errorf("must specify --outfile.l2") + } + + outfileRollup := ctx.Path("outfile.rollup") + if outfileRollup == "" { + return fmt.Errorf("must specify --outfile.rollup") + } + + if err := jsonutil.WriteJSON(outfileL2, l2Genesis); err != nil { + return err + } + if err := jsonutil.WriteJSON(outfileRollup, rollupConfig); err != nil { + return err + } + + // So far we applied changes in the memory VM and collected changes in the genesis struct + // No we iterate through all accounts that have been written there and set them inside the statedb. + // This will change the state root + // TODO(pl): We should have some checks that check we don't write accounts that aren't necessary, e.g. dev account + // Another property is that the total balance changes must be 0 + + // Write changes to state to actual state database + dbPath := ctx.String("db-path") + if dbPath == "" { + return fmt.Errorf("must specify --db-path") + } + dbCache := ctx.Int("db-cache") + dbHandles := ctx.Int("db-handles") + // dryRun := ctx.Bool("dry-run") + // TODO(pl): Move this into the function + log.Info("Opening database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) + ldb, err := Open(dbPath, dbCache, dbHandles) + if err != nil { + return fmt.Errorf("cannot open DB: %w", err) + } + + if err := ApplyMigrationChangesToDB(ldb, l2Genesis); err != nil { + return err + } + + // Close the database handle + if err := ldb.Close(); err != nil { + return err + } + + log.Info("Loaded Celo L1 DB", "db", ldb) + + return nil + }, + } + + if err := app.Run(os.Args); err != nil { + log.Crit("error in migration", "err", err) + } + log.Info("Finished migration successfully!") +} + +func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis) error { + panic("unimplemented") +} + +func Open(path string, cache int, handles int) (ethdb.Database, error) { + chaindataPath := filepath.Join(path, "celo", "chaindata") + ancientPath := filepath.Join(chaindataPath, "ancient") + ldb, err := rawdb.Open(rawdb.OpenOptions{ + Type: "leveldb", + Directory: chaindataPath, + AncientsDirectory: ancientPath, + Namespace: "", + Cache: cache, + Handles: handles, + ReadOnly: false, + }) + if err != nil { + return nil, err + } + return ldb, nil +} diff --git a/op-chain-ops/cmd/celo-migrate/test_config.json b/op-chain-ops/cmd/celo-migrate/test_config.json new file mode 100644 index 0000000000000..3ad28a45875b6 --- /dev/null +++ b/op-chain-ops/cmd/celo-migrate/test_config.json @@ -0,0 +1,74 @@ +{ + "l1StartingBlockTag": "$blockhash", + + "l1ChainID": 11155111, + "l2ChainID": 42069, + "l2BlockTime": 2, + "l1BlockTime": 12, + + "maxSequencerDrift": 600, + "sequencerWindowSize": 3600, + "channelTimeout": 300, + + "p2pSequencerAddress": "$GS_SEQUENCER_ADDRESS", + "batchInboxAddress": "0xff00000000000000000000000000000000042069", + "batchSenderAddress": "$GS_BATCHER_ADDRESS", + + "l2OutputOracleSubmissionInterval": 120, + "l2OutputOracleStartingBlockNumber": 0, + "l2OutputOracleStartingTimestamp": 123456789, + + "l2OutputOracleProposer": "$GS_PROPOSER_ADDRESS", + "l2OutputOracleChallenger": "$GS_ADMIN_ADDRESS", + + "finalizationPeriodSeconds": 12, + + "proxyAdminOwner": "$GS_ADMIN_ADDRESS", + "baseFeeVaultRecipient": "$GS_ADMIN_ADDRESS", + "l1FeeVaultRecipient": "$GS_ADMIN_ADDRESS", + "sequencerFeeVaultRecipient": "$GS_ADMIN_ADDRESS", + "finalSystemOwner": "$GS_ADMIN_ADDRESS", + "superchainConfigGuardian": "$GS_ADMIN_ADDRESS", + + "baseFeeVaultMinimumWithdrawalAmount": "0x8ac7230489e80000", + "l1FeeVaultMinimumWithdrawalAmount": "0x8ac7230489e80000", + "sequencerFeeVaultMinimumWithdrawalAmount": "0x8ac7230489e80000", + "baseFeeVaultWithdrawalNetwork": 0, + "l1FeeVaultWithdrawalNetwork": 0, + "sequencerFeeVaultWithdrawalNetwork": 0, + + "gasPriceOracleOverhead": 2100, + "gasPriceOracleScalar": 1000000, + + "enableGovernance": true, + "governanceTokenSymbol": "OP", + "governanceTokenName": "Optimism", + "governanceTokenOwner": "$GS_ADMIN_ADDRESS", + + "l2GenesisBlockGasLimit": "0x1c9c380", + "l2GenesisBlockBaseFeePerGas": "0x3b9aca00", + "l2GenesisRegolithTimeOffset": "0x0", + + "eip1559Denominator": 50, + "eip1559DenominatorCanyon": 250, + "eip1559Elasticity": 6, + + "l2GenesisDeltaTimeOffset": null, + "l2GenesisCanyonTimeOffset": "0x0", + + "systemConfigStartBlock": 0, + + "requiredProtocolVersion": "0x0000000000000000000000000000000000000000000000000000000000000000", + "recommendedProtocolVersion": "0x0000000000000000000000000000000000000000000000000000000000000000", + + "faultGameAbsolutePrestate": "0x03c7ae758795765c6664a5d39bf63841c71ff191e9189522bad8ebff5d4eca98", + "faultGameMaxDepth": 44, + "faultGameMaxDuration": 1200, + "faultGameGenesisBlock": 0, + "faultGameGenesisOutputRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "faultGameSplitDepth": 14, + + "preimageOracleMinProposalSize": 1800000, + "preimageOracleChallengePeriod": 86400, + "preimageOracleCancunActivationTimestamp": 0 +} diff --git a/op-chain-ops/cmd/op-migrate/main.go b/op-chain-ops/cmd/op-migrate/main.go deleted file mode 100644 index b1e189023802a..0000000000000 --- a/op-chain-ops/cmd/op-migrate/main.go +++ /dev/null @@ -1,312 +0,0 @@ -package main - -import ( - "fmt" - "math/big" - "os" - "path/filepath" - - "github.com/ethereum-optimism/optimism/op-bindings/predeploys" - "github.com/mattn/go-isatty" - - "github.com/urfave/cli/v2" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -// from `packages/contracts-bedrock/deploy-config/internal-devnet.json` -var ( - EIP1559Denominator = uint64(50) // TODO: what values - EIP1559Elasticity = uint64(10) -) - -func main() { - log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd())))) - - app := &cli.App{ - Name: "migrate", - Usage: "Migrate Celo state to a CeL2 genesis DB", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "db-path", - Usage: "Path to database", - Required: true, - }, - &cli.BoolFlag{ - Name: "dry-run", - Usage: "Dry run the upgrade by not committing the database", - }, - &cli.BoolFlag{ - Name: "no-check", - Usage: "Do not perform sanity checks. This should only be used for testing", - }, - &cli.IntFlag{ - Name: "db-cache", - Usage: "LevelDB cache size in mb", - Value: 1024, - }, - &cli.IntFlag{ - Name: "db-handles", - Usage: "LevelDB number of handles", - Value: 60, - }, - }, - Action: func(ctx *cli.Context) error { - dbCache := ctx.Int("db-cache") - dbHandles := ctx.Int("db-handles") - dbPath := ctx.String("db-path") - log.Info("Opening database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) - ldb, err := Open(dbPath, dbCache, dbHandles) - if err != nil { - return fmt.Errorf("cannot open DB: %w", err) - } - - dryRun := ctx.Bool("dry-run") - noCheck := ctx.Bool("no-check") - if noCheck { - panic("must run with check on") - } - - // Perform the migration - _, err = MigrateDB(ldb, !dryRun, noCheck) - if err != nil { - return err - } - - // Close the database handle - if err := ldb.Close(); err != nil { - return err - } - - log.Info("Finished migration successfully!") - - return nil - }, - } - - if err := app.Run(os.Args); err != nil { - log.Crit("error in migration", "err", err) - } -} - -type MigrationResult struct { - TransitionHeight uint64 - TransitionTimestamp uint64 - TransitionBlockHash common.Hash -} - -// MigrateDB will migrate a celo database to a new OP genesis block -func MigrateDB(ldb ethdb.Database, commit, noCheck bool) (*MigrationResult, error) { - log.Info("Migrating DB") - - // Grab the hash of the tip of the legacy chain. - hash := rawdb.ReadHeadHeaderHash(ldb) - log.Info("Reading chain tip from database", "hash", hash) - - // Grab the header number. - num := rawdb.ReadHeaderNumber(ldb, hash) - if num == nil { - return nil, fmt.Errorf("cannot find header number for %s", hash) - } - log.Info("Reading chain tip num from database", "number", num) - - // Grab the full header. - header := rawdb.ReadHeader(ldb, hash, *num) - trieRoot := header.Root - log.Info("Read header from database", "number", header) - - // We need to update the chain config to set the correct hardforks. - genesisHash := rawdb.ReadCanonicalHash(ldb, 0) - cfg := rawdb.ReadChainConfig(ldb, genesisHash) - if cfg == nil { - log.Crit("chain config not found") - } - log.Info("Read config from database", "config", cfg) - - // dbFactory := func() (*state.StateDB, error) { - // // Set up the backing store. - // underlyingDB := state.NewDatabaseWithConfig(ldb, &trie.Config{ - // Preimages: true, - // Cache: 1024, - // }) - - // // Open up the state database. - // db, err := state.New(header.Root, underlyingDB, nil) - // if err != nil { - // return nil, fmt.Errorf("cannot open StateDB: %w", err) - // } - - // return db, nil - // } - - // db, err := dbFactory() - // if err != nil { - // return nil, fmt.Errorf("cannot create StateDB: %w", err) - // } - - // Remove old blocks, so that we start with a fresh genesis block - currentHash := header.ParentHash - for { - // There are no uncles in Celo - num = rawdb.ReadHeaderNumber(ldb, currentHash) - hash = rawdb.ReadCanonicalHash(ldb, *num) - - log.Info("Deleting block", "hash", currentHash, "c", hash, "number", *num) - if commit { - rawdb.DeleteBlock(ldb, currentHash, *num) - } - if *num == 0 { - break - } - - header = rawdb.ReadHeader(ldb, currentHash, *num) - if header == nil { - return nil, fmt.Errorf("couldn't find header") - } - currentHash = header.ParentHash - } - - log.Info("Successfully cleaned old blocks") - - // We're done messing around with the database, so we can now commit the changes to the DB. - // Note that this doesn't actually write the changes to disk. - // log.Info("Committing state DB") - // newRoot, err := db.Commit(true) - // if err != nil { - // return nil, err - // } - - log.Info("Creating new Genesis block") - // Create the header for the Bedrock transition block. - cel2Header := &types.Header{ - ParentHash: common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - UncleHash: types.EmptyUncleHash, - Coinbase: predeploys.SequencerFeeVaultAddr, // TODO - Root: trieRoot, - TxHash: types.EmptyRootHash, - ReceiptHash: types.EmptyRootHash, - Bloom: types.Bloom{}, - Difficulty: common.Big0, - Number: common.Big0, - GasLimit: (uint64)(20_000_000), - GasUsed: 0, - Time: uint64(12345), - Extra: []byte("CeL2"), - MixDigest: common.Hash{}, - Nonce: types.BlockNonce{}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - - // Create the Bedrock transition block from the header. Note that there are no transactions, - // uncle blocks, or receipts in the Bedrock transition block. - cel2Block := types.NewBlock(cel2Header, nil, nil, nil, trie.NewStackTrie(nil)) - - // We did it! - log.Info( - "Built Bedrock transition", - "hash", cel2Block.Hash(), - "root", cel2Block.Root(), - "number", cel2Block.NumberU64(), - "gas-used", cel2Block.GasUsed(), - "gas-limit", cel2Block.GasLimit(), - ) - - log.Info("Header", "header", cel2Header) - log.Info("Body", "Body", cel2Block) - - // Create the result of the migration. - res := &MigrationResult{ - TransitionHeight: cel2Block.NumberU64(), - TransitionTimestamp: cel2Block.Time(), - TransitionBlockHash: cel2Block.Hash(), - } - - // If we're not actually writing this to disk, then we're done. - if !commit { - log.Info("Dry run complete") - return res, nil - } - - // Otherwise we need to write the changes to disk. First we commit the state changes. - // log.Info("Committing trie DB") - // if err := db.Database().TrieDB().Commit(newRoot, true); err != nil { - // return nil, err - // } - - // Next we write the Cel2 genesis block to the database. - rawdb.WriteTd(ldb, cel2Block.Hash(), cel2Block.NumberU64(), cel2Block.Difficulty()) - rawdb.WriteBlock(ldb, cel2Block) - rawdb.WriteReceipts(ldb, cel2Block.Hash(), cel2Block.NumberU64(), nil) - rawdb.WriteCanonicalHash(ldb, cel2Block.Hash(), cel2Block.NumberU64()) - rawdb.WriteHeadBlockHash(ldb, cel2Block.Hash()) - rawdb.WriteHeadFastBlockHash(ldb, cel2Block.Hash()) - rawdb.WriteHeadHeaderHash(ldb, cel2Block.Hash()) - - // TODO - // Make the first CeL2 block a finalized block. - rawdb.WriteFinalizedBlockHash(ldb, cel2Block.Hash()) - - // Set the standard options. - // TODO: What about earlier hardforks, e.g. does berlin have to be enabled as it never was on Celo? - cfg.LondonBlock = cel2Block.Number() - cfg.ArrowGlacierBlock = cel2Block.Number() - cfg.GrayGlacierBlock = cel2Block.Number() - cfg.MergeNetsplitBlock = cel2Block.Number() - cfg.TerminalTotalDifficulty = big.NewInt(0) - cfg.TerminalTotalDifficultyPassed = true - - // Set the Optimism options. - cfg.BedrockBlock = cel2Block.Number() - // Enable Regolith from the start of Bedrock - cfg.RegolithTime = new(uint64) // what are those? do we need those? - cfg.Optimism = ¶ms.OptimismConfig{ - EIP1559Denominator: EIP1559Denominator, - EIP1559Elasticity: EIP1559Elasticity, - } - - // Write the chain config to disk. - rawdb.WriteChainConfig(ldb, cel2Block.Hash(), cfg) - - // Yay! - log.Info( - "Wrote chain config", - "1559-denominator", EIP1559Denominator, - "1559-elasticity", EIP1559Elasticity, - ) - - // We're done! - log.Info( - "Wrote CeL2 transition block", - "height", cel2Header.Number, - "root", cel2Header.Root.String(), - "hash", cel2Header.Hash().String(), - "timestamp", cel2Header.Time, - ) - - // Return the result and have a nice day. - return res, nil -} - -func Open(path string, cache int, handles int) (ethdb.Database, error) { - chaindataPath := filepath.Join(path, "celo", "chaindata") - ancientPath := filepath.Join(chaindataPath, "ancient") - ldb, err := rawdb.Open(rawdb.OpenOptions{ - Type: "leveldb", - Directory: chaindataPath, - AncientsDirectory: ancientPath, - Namespace: "", - Cache: cache, - Handles: handles, - ReadOnly: false, - }) - if err != nil { - return nil, err - } - return ldb, nil -} From 944aacdab762c94151971664726ff7010526998a Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Wed, 10 Apr 2024 13:49:10 +0200 Subject: [PATCH 02/10] Add logic to write changes to state DB --- op-chain-ops/cmd/celo-migrate/README.md | 8 +- op-chain-ops/cmd/celo-migrate/main.go | 190 +++++++++++++++++++++++- 2 files changed, 193 insertions(+), 5 deletions(-) diff --git a/op-chain-ops/cmd/celo-migrate/README.md b/op-chain-ops/cmd/celo-migrate/README.md index b8f50ead12b0f..9c0777cf47811 100644 --- a/op-chain-ops/cmd/celo-migrate/README.md +++ b/op-chain-ops/cmd/celo-migrate/README.md @@ -19,6 +19,12 @@ build/bin/mycelo load-bot tmp/testenv ``` To run the migration, run in `op-chain-ops` (set `CELO_DATADIR` if the `celo-blockchain` repo is not located at `~/celo-blockchain`): + ```sh make && ./migrate.sh -``` \ No newline at end of file +``` + +## Tasks + +- Import and change `BuildL2Genesis` + - Don't set balances for predeploys/precompiles diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index 2c4379ec0f4db..316df641d80f7 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -7,19 +7,24 @@ import ( "os" "path/filepath" + "github.com/ethereum-optimism/optimism/op-bindings/predeploys" "github.com/ethereum-optimism/optimism/op-chain-ops/genesis" "github.com/ethereum-optimism/optimism/op-service/jsonutil" "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -200,7 +205,7 @@ func main() { } dbCache := ctx.Int("db-cache") dbHandles := ctx.Int("db-handles") - // dryRun := ctx.Bool("dry-run") + dryRun := ctx.Bool("dry-run") // TODO(pl): Move this into the function log.Info("Opening database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) ldb, err := Open(dbPath, dbCache, dbHandles) @@ -208,7 +213,7 @@ func main() { return fmt.Errorf("cannot open DB: %w", err) } - if err := ApplyMigrationChangesToDB(ldb, l2Genesis); err != nil { + if err := ApplyMigrationChangesToDB(ldb, l2Genesis, !dryRun); err != nil { return err } @@ -229,8 +234,185 @@ func main() { log.Info("Finished migration successfully!") } -func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis) error { - panic("unimplemented") +func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit bool) error { + log.Info("Migrating DB") + + // Grab the hash of the tip of the legacy chain. + hash := rawdb.ReadHeadHeaderHash(ldb) + log.Info("Reading chain tip from database", "hash", hash) + + // Grab the header number. + num := rawdb.ReadHeaderNumber(ldb, hash) + if num == nil { + return fmt.Errorf("cannot find header number for %s", hash) + } + log.Info("Reading chain tip num from database", "number", num) + + // Grab the full header. + header := rawdb.ReadHeader(ldb, hash, *num) + // trieRoot := header.Root + log.Info("Read header from database", "number", header) + + // We need to update the chain config to set the correct hardforks. + genesisHash := rawdb.ReadCanonicalHash(ldb, 0) + cfg := rawdb.ReadChainConfig(ldb, genesisHash) + if cfg == nil { + log.Crit("chain config not found") + } + log.Info("Read config from database", "config", cfg) + + dbFactory := func() (*state.StateDB, error) { + // Set up the backing store. + underlyingDB := state.NewDatabaseWithConfig(ldb, &trie.Config{ + Preimages: true, + }) + + // Open up the state database. + db, err := state.New(header.Root, underlyingDB, nil) + if err != nil { + return nil, fmt.Errorf("cannot open StateDB: %w", err) + } + + return db, nil + } + + db, err := dbFactory() + if err != nil { + return fmt.Errorf("cannot create StateDB: %w", err) + } + + for k, v := range genesis.Alloc { + if db.Exist(k) { + log.Warn("Operating on existing state", "account", k) + } + // TODO(pl): decide what to do with existing accounts. + db.CreateAccount(k) + + db.SetNonce(k, v.Nonce) + db.SetBalance(k, v.Balance) + db.SetCode(k, v.Code) + db.SetStorage(k, v.Storage) + log.Info("Moved account", "address", k) + } + + // We're done messing around with the database, so we can now commit the changes to the DB. + // Note that this doesn't actually write the changes to disk. + log.Info("Committing state DB") + // TODO(pl): What block info to put here? + newRoot, err := db.Commit(1234, true) + if err != nil { + return err + } + + log.Info("Creating new Genesis block") + // Create the header for the Bedrock transition block. + cel2Header := &types.Header{ + ParentHash: header.Hash(), + UncleHash: types.EmptyUncleHash, + Coinbase: predeploys.SequencerFeeVaultAddr, // TODO(pl) + Root: newRoot, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + Bloom: types.Bloom{}, + Difficulty: common.Big0, + Number: common.Big0, + GasLimit: (uint64)(20_000_000), + GasUsed: 0, + Time: uint64(12345), + Extra: []byte("CeL2"), + MixDigest: common.Hash{}, + Nonce: types.BlockNonce{}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + + // Create the Bedrock transition block from the header. Note that there are no transactions, + // uncle blocks, or receipts in the Bedrock transition block. + cel2Block := types.NewBlock(cel2Header, nil, nil, nil, trie.NewStackTrie(nil)) + + // We did it! + log.Info( + "Built Celo migration block", + "hash", cel2Block.Hash(), + "root", cel2Block.Root(), + "number", cel2Block.NumberU64(), + "gas-used", cel2Block.GasUsed(), + "gas-limit", cel2Block.GasLimit(), + ) + + log.Info("Header", "header", cel2Header) + log.Info("Body", "Body", cel2Block) + + // Create the result of the migration. + // res := &MigrationResult{ + // TransitionHeight: cel2Block.NumberU64(), + // TransitionTimestamp: cel2Block.Time(), + // TransitionBlockHash: cel2Block.Hash(), + // } + + // If we're not actually writing this to disk, then we're done. + if !commit { + log.Info("Dry run complete") + return nil + } + + // Otherwise we need to write the changes to disk. First we commit the state changes. + log.Info("Committing trie DB") + if err := db.Database().TrieDB().Commit(newRoot, true); err != nil { + return err + } + + // Next we write the Cel2 genesis block to the database. + rawdb.WriteTd(ldb, cel2Block.Hash(), cel2Block.NumberU64(), cel2Block.Difficulty()) + rawdb.WriteBlock(ldb, cel2Block) + rawdb.WriteReceipts(ldb, cel2Block.Hash(), cel2Block.NumberU64(), nil) + rawdb.WriteCanonicalHash(ldb, cel2Block.Hash(), cel2Block.NumberU64()) + rawdb.WriteHeadBlockHash(ldb, cel2Block.Hash()) + rawdb.WriteHeadFastBlockHash(ldb, cel2Block.Hash()) + rawdb.WriteHeadHeaderHash(ldb, cel2Block.Hash()) + + // TODO(pl): What does this mean? + // Make the first CeL2 block a finalized block. + rawdb.WriteFinalizedBlockHash(ldb, cel2Block.Hash()) + + // Set the standard options. + // TODO: What about earlier hardforks, e.g. does berlin have to be enabled as it never was on Celo? + cfg.LondonBlock = cel2Block.Number() + cfg.ArrowGlacierBlock = cel2Block.Number() + cfg.GrayGlacierBlock = cel2Block.Number() + cfg.MergeNetsplitBlock = cel2Block.Number() + cfg.TerminalTotalDifficulty = big.NewInt(0) + cfg.TerminalTotalDifficultyPassed = true + + // Set the Optimism options. + cfg.BedrockBlock = cel2Block.Number() + // Enable Regolith from the start of Bedrock + cfg.RegolithTime = new(uint64) // what are those? do we need those? + cfg.Optimism = ¶ms.OptimismConfig{ + EIP1559Denominator: EIP1559Denominator, + EIP1559Elasticity: EIP1559Elasticity, + } + // TODO(pl) Add Ecotone and other hardforks + + // Write the chain config to disk. + rawdb.WriteChainConfig(ldb, cel2Block.Hash(), cfg) + + // Yay! + log.Info( + "Wrote chain config", + "1559-denominator", EIP1559Denominator, + "1559-elasticity", EIP1559Elasticity, + ) + + // We're done! + log.Info( + "Wrote CeL2 transition block", + "height", cel2Header.Number, + "root", cel2Header.Root.String(), + "hash", cel2Header.Hash().String(), + "timestamp", cel2Header.Time, + ) + + return nil } func Open(path string, cache int, handles int) (ethdb.Database, error) { From b4fff1b8fc2ac2ff411c622991daf82355510379 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Wed, 17 Apr 2024 12:23:09 +0200 Subject: [PATCH 03/10] Minor changes --- op-chain-ops/cmd/celo-migrate/main.go | 7 +- .../cmd/celo-migrate/test_config.json | 74 ------------------- op-chain-ops/genesis/layer_two.go | 2 + op-chain-ops/squash/sim.go | 2 +- 4 files changed, 9 insertions(+), 76 deletions(-) delete mode 100644 op-chain-ops/cmd/celo-migrate/test_config.json diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index 316df641d80f7..62dc416c5797e 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -391,7 +391,12 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit EIP1559Denominator: EIP1559Denominator, EIP1559Elasticity: EIP1559Elasticity, } - // TODO(pl) Add Ecotone and other hardforks + cfg.CanyonTime = &cel2Header.Time + cfg.EcotoneTime = &cel2Header.Time + + // TODO(pl): What about Ethereum hardforks + + log.Info("Write new config to database", "config", cfg) // Write the chain config to disk. rawdb.WriteChainConfig(ldb, cel2Block.Hash(), cfg) diff --git a/op-chain-ops/cmd/celo-migrate/test_config.json b/op-chain-ops/cmd/celo-migrate/test_config.json deleted file mode 100644 index 3ad28a45875b6..0000000000000 --- a/op-chain-ops/cmd/celo-migrate/test_config.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "l1StartingBlockTag": "$blockhash", - - "l1ChainID": 11155111, - "l2ChainID": 42069, - "l2BlockTime": 2, - "l1BlockTime": 12, - - "maxSequencerDrift": 600, - "sequencerWindowSize": 3600, - "channelTimeout": 300, - - "p2pSequencerAddress": "$GS_SEQUENCER_ADDRESS", - "batchInboxAddress": "0xff00000000000000000000000000000000042069", - "batchSenderAddress": "$GS_BATCHER_ADDRESS", - - "l2OutputOracleSubmissionInterval": 120, - "l2OutputOracleStartingBlockNumber": 0, - "l2OutputOracleStartingTimestamp": 123456789, - - "l2OutputOracleProposer": "$GS_PROPOSER_ADDRESS", - "l2OutputOracleChallenger": "$GS_ADMIN_ADDRESS", - - "finalizationPeriodSeconds": 12, - - "proxyAdminOwner": "$GS_ADMIN_ADDRESS", - "baseFeeVaultRecipient": "$GS_ADMIN_ADDRESS", - "l1FeeVaultRecipient": "$GS_ADMIN_ADDRESS", - "sequencerFeeVaultRecipient": "$GS_ADMIN_ADDRESS", - "finalSystemOwner": "$GS_ADMIN_ADDRESS", - "superchainConfigGuardian": "$GS_ADMIN_ADDRESS", - - "baseFeeVaultMinimumWithdrawalAmount": "0x8ac7230489e80000", - "l1FeeVaultMinimumWithdrawalAmount": "0x8ac7230489e80000", - "sequencerFeeVaultMinimumWithdrawalAmount": "0x8ac7230489e80000", - "baseFeeVaultWithdrawalNetwork": 0, - "l1FeeVaultWithdrawalNetwork": 0, - "sequencerFeeVaultWithdrawalNetwork": 0, - - "gasPriceOracleOverhead": 2100, - "gasPriceOracleScalar": 1000000, - - "enableGovernance": true, - "governanceTokenSymbol": "OP", - "governanceTokenName": "Optimism", - "governanceTokenOwner": "$GS_ADMIN_ADDRESS", - - "l2GenesisBlockGasLimit": "0x1c9c380", - "l2GenesisBlockBaseFeePerGas": "0x3b9aca00", - "l2GenesisRegolithTimeOffset": "0x0", - - "eip1559Denominator": 50, - "eip1559DenominatorCanyon": 250, - "eip1559Elasticity": 6, - - "l2GenesisDeltaTimeOffset": null, - "l2GenesisCanyonTimeOffset": "0x0", - - "systemConfigStartBlock": 0, - - "requiredProtocolVersion": "0x0000000000000000000000000000000000000000000000000000000000000000", - "recommendedProtocolVersion": "0x0000000000000000000000000000000000000000000000000000000000000000", - - "faultGameAbsolutePrestate": "0x03c7ae758795765c6664a5d39bf63841c71ff191e9189522bad8ebff5d4eca98", - "faultGameMaxDepth": 44, - "faultGameMaxDuration": 1200, - "faultGameGenesisBlock": 0, - "faultGameGenesisOutputRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "faultGameSplitDepth": 14, - - "preimageOracleMinProposalSize": 1800000, - "preimageOracleChallengePeriod": 86400, - "preimageOracleCancunActivationTimestamp": 0 -} diff --git a/op-chain-ops/genesis/layer_two.go b/op-chain-ops/genesis/layer_two.go index 3b2bf85e1a3ba..9692e3a2e51e8 100644 --- a/op-chain-ops/genesis/layer_two.go +++ b/op-chain-ops/genesis/layer_two.go @@ -122,6 +122,8 @@ func BuildL2Genesis(config *DeployConfig, l1StartBlock *types.Block) (*core.Gene } func PerformUpgradeTxs(db *state.MemoryStateDB) error { + log.Info("Running Ecotone upgrade transactions") + // Only the Ecotone upgrade is performed with upgrade-txs. if !db.Genesis().Config.IsEcotone(db.Genesis().Timestamp) { return nil diff --git a/op-chain-ops/squash/sim.go b/op-chain-ops/squash/sim.go index 1d66bc135e06a..808ab83d1c10f 100644 --- a/op-chain-ops/squash/sim.go +++ b/op-chain-ops/squash/sim.go @@ -165,7 +165,7 @@ func (sim *SquashSim) AddUpgradeTxs(txs []hexutil.Bytes) error { return fmt.Errorf("failed to turn upgrade tx %d into message: %w", i, err) } if !msg.IsDepositTx { - return fmt.Errorf("upgrade tx %d is not a depost", i) + return fmt.Errorf("upgrade tx %d is not a deposit", i) } if res, err := sim.AddMessage(msg); err != nil { return fmt.Errorf("invalid upgrade tx %d, EVM invocation failed: %w", i, err) From de413d766b0f4744b49a8fb942d835d8b9fa4caa Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Wed, 17 Apr 2024 14:35:56 +0200 Subject: [PATCH 04/10] Check that datadir exists --- op-chain-ops/cmd/celo-migrate/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index 62dc416c5797e..da12e6a4f478b 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "math/big" "os" @@ -421,6 +422,10 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit } func Open(path string, cache int, handles int) (ethdb.Database, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return nil, err + } + chaindataPath := filepath.Join(path, "celo", "chaindata") ancientPath := filepath.Join(chaindataPath, "ancient") ldb, err := rawdb.Open(rawdb.OpenOptions{ From 75c2279b614229142dfa81f56c20d39852aaaed1 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Wed, 17 Apr 2024 15:24:01 +0200 Subject: [PATCH 05/10] Use cel2 migration header for rollup config --- op-chain-ops/cmd/celo-migrate/main.go | 97 +++++++++++++-------------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index da12e6a4f478b..389143ca0c00b 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -123,6 +123,16 @@ func main() { return fmt.Errorf("must specify --l1-rpc") } + outfileL2 := ctx.Path("outfile.l2") + if outfileL2 == "" { + return fmt.Errorf("must specify --outfile.l2") + } + + outfileRollup := ctx.Path("outfile.rollup") + if outfileRollup == "" { + return fmt.Errorf("must specify --outfile.rollup") + } + var l1StartBlock *types.Block client, err := ethclient.Dial(l1RPC) if err != nil { @@ -167,32 +177,6 @@ func main() { return fmt.Errorf("error creating l2 genesis: %w", err) } - l2GenesisBlock := l2Genesis.ToBlock() - rollupConfig, err := config.RollupConfig(l1StartBlock, l2GenesisBlock.Hash(), l2GenesisBlock.Number().Uint64()) - if err != nil { - return err - } - if err := rollupConfig.Check(); err != nil { - return fmt.Errorf("generated rollup config does not pass validation: %w", err) - } - - outfileL2 := ctx.Path("outfile.l2") - if outfileL2 == "" { - return fmt.Errorf("must specify --outfile.l2") - } - - outfileRollup := ctx.Path("outfile.rollup") - if outfileRollup == "" { - return fmt.Errorf("must specify --outfile.rollup") - } - - if err := jsonutil.WriteJSON(outfileL2, l2Genesis); err != nil { - return err - } - if err := jsonutil.WriteJSON(outfileRollup, rollupConfig); err != nil { - return err - } - // So far we applied changes in the memory VM and collected changes in the genesis struct // No we iterate through all accounts that have been written there and set them inside the statedb. // This will change the state root @@ -213,8 +197,10 @@ func main() { if err != nil { return fmt.Errorf("cannot open DB: %w", err) } + log.Info("Loaded Celo L1 DB", "db", ldb) - if err := ApplyMigrationChangesToDB(ldb, l2Genesis, !dryRun); err != nil { + cel2Header, err := ApplyMigrationChangesToDB(ldb, l2Genesis, !dryRun) + if err != nil { return err } @@ -223,7 +209,26 @@ func main() { return err } - log.Info("Loaded Celo L1 DB", "db", ldb) + log.Info("Updated Cel2 state") + + log.Info("Writing state diff", "file", outfileL2) + // Write genesis file to check created state + if err := jsonutil.WriteJSON(outfileL2, l2Genesis); err != nil { + return err + } + + rollupConfig, err := config.RollupConfig(l1StartBlock, cel2Header.Hash(), cel2Header.Number.Uint64()) + if err != nil { + return err + } + if err := rollupConfig.Check(); err != nil { + return fmt.Errorf("generated rollup config does not pass validation: %w", err) + } + + log.Info("Writing rollup config", "file", outfileRollup) + if err := jsonutil.WriteJSON(outfileRollup, rollupConfig); err != nil { + return err + } return nil }, @@ -235,7 +240,7 @@ func main() { log.Info("Finished migration successfully!") } -func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit bool) error { +func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit bool) (*types.Header, error) { log.Info("Migrating DB") // Grab the hash of the tip of the legacy chain. @@ -245,7 +250,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit // Grab the header number. num := rawdb.ReadHeaderNumber(ldb, hash) if num == nil { - return fmt.Errorf("cannot find header number for %s", hash) + return nil, fmt.Errorf("cannot find header number for %s", hash) } log.Info("Reading chain tip num from database", "number", num) @@ -279,7 +284,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit db, err := dbFactory() if err != nil { - return fmt.Errorf("cannot create StateDB: %w", err) + return nil, fmt.Errorf("cannot create StateDB: %w", err) } for k, v := range genesis.Alloc { @@ -302,7 +307,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit // TODO(pl): What block info to put here? newRoot, err := db.Commit(1234, true) if err != nil { - return err + return nil, err } log.Info("Creating new Genesis block") @@ -310,20 +315,20 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit cel2Header := &types.Header{ ParentHash: header.Hash(), UncleHash: types.EmptyUncleHash, - Coinbase: predeploys.SequencerFeeVaultAddr, // TODO(pl) + Coinbase: predeploys.SequencerFeeVaultAddr, Root: newRoot, TxHash: types.EmptyRootHash, ReceiptHash: types.EmptyRootHash, Bloom: types.Bloom{}, - Difficulty: common.Big0, - Number: common.Big0, - GasLimit: (uint64)(20_000_000), + Difficulty: new(big.Int).Set(common.Big0), + Number: new(big.Int).Add(header.Number, common.Big1), + GasLimit: header.GasLimit, GasUsed: 0, - Time: uint64(12345), - Extra: []byte("CeL2"), + Time: header.Time, + Extra: []byte("CeL2 migration"), MixDigest: common.Hash{}, Nonce: types.BlockNonce{}, - BaseFee: big.NewInt(params.InitialBaseFee), + BaseFee: new(big.Int).Set(header.BaseFee), } // Create the Bedrock transition block from the header. Note that there are no transactions, @@ -341,25 +346,17 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit ) log.Info("Header", "header", cel2Header) - log.Info("Body", "Body", cel2Block) - - // Create the result of the migration. - // res := &MigrationResult{ - // TransitionHeight: cel2Block.NumberU64(), - // TransitionTimestamp: cel2Block.Time(), - // TransitionBlockHash: cel2Block.Hash(), - // } // If we're not actually writing this to disk, then we're done. if !commit { log.Info("Dry run complete") - return nil + return nil, nil } // Otherwise we need to write the changes to disk. First we commit the state changes. log.Info("Committing trie DB") if err := db.Database().TrieDB().Commit(newRoot, true); err != nil { - return err + return nil, err } // Next we write the Cel2 genesis block to the database. @@ -418,7 +415,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit "timestamp", cel2Header.Time, ) - return nil + return cel2Header, nil } func Open(path string, cache int, handles int) (ethdb.Database, error) { From a31c8ea4320792b4e9dc93808a94cf8f634a8736 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Thu, 25 Apr 2024 10:56:13 +0200 Subject: [PATCH 06/10] Fixes from testing --- op-chain-ops/cmd/celo-migrate/main.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index 389143ca0c00b..594daf6692e1e 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -7,6 +7,7 @@ import ( "math/big" "os" "path/filepath" + "time" "github.com/ethereum-optimism/optimism/op-bindings/predeploys" "github.com/ethereum-optimism/optimism/op-chain-ops/genesis" @@ -193,7 +194,7 @@ func main() { dryRun := ctx.Bool("dry-run") // TODO(pl): Move this into the function log.Info("Opening database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) - ldb, err := Open(dbPath, dbCache, dbHandles) + ldb, err := openCeloDb(dbPath, dbCache, dbHandles) if err != nil { return fmt.Errorf("cannot open DB: %w", err) } @@ -324,7 +325,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit Number: new(big.Int).Add(header.Number, common.Big1), GasLimit: header.GasLimit, GasUsed: 0, - Time: header.Time, + Time: uint64(time.Now().Unix()), // TODO(pl): Needed to avoid L1-L2 time mismatches Extra: []byte("CeL2 migration"), MixDigest: common.Hash{}, Nonce: types.BlockNonce{}, @@ -368,13 +369,13 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit rawdb.WriteHeadFastBlockHash(ldb, cel2Block.Hash()) rawdb.WriteHeadHeaderHash(ldb, cel2Block.Hash()) - // TODO(pl): What does this mean? + // TODO(pl): What does finalized mean here? // Make the first CeL2 block a finalized block. rawdb.WriteFinalizedBlockHash(ldb, cel2Block.Hash()) // Set the standard options. - // TODO: What about earlier hardforks, e.g. does berlin have to be enabled as it never was on Celo? cfg.LondonBlock = cel2Block.Number() + cfg.BerlinBlock = cel2Block.Number() cfg.ArrowGlacierBlock = cel2Block.Number() cfg.GrayGlacierBlock = cel2Block.Number() cfg.MergeNetsplitBlock = cel2Block.Number() @@ -391,13 +392,14 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit } cfg.CanyonTime = &cel2Header.Time cfg.EcotoneTime = &cel2Header.Time - - // TODO(pl): What about Ethereum hardforks + cfg.ShanghaiTime = &cel2Header.Time + cfg.Cel2Time = &cel2Header.Time log.Info("Write new config to database", "config", cfg) // Write the chain config to disk. - rawdb.WriteChainConfig(ldb, cel2Block.Hash(), cfg) + // TODO(pl): Why do we need to write this with the genesis hash, not `cel2Block.Hash()`?` + rawdb.WriteChainConfig(ldb, genesisHash, cfg) // Yay! log.Info( @@ -418,7 +420,8 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit return cel2Header, nil } -func Open(path string, cache int, handles int) (ethdb.Database, error) { +// Opens a Celo database, stored in the `celo` subfolder +func openCeloDb(path string, cache int, handles int) (ethdb.Database, error) { if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { return nil, err } From effffb17060079de0eb514d42415f2592fc32081 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Thu, 25 Apr 2024 11:05:26 +0200 Subject: [PATCH 07/10] Add checks when copying deployed contracts into state db --- op-chain-ops/cmd/celo-migrate/main.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index 594daf6692e1e..65073f43c10b3 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "errors" "fmt" @@ -178,12 +179,6 @@ func main() { return fmt.Errorf("error creating l2 genesis: %w", err) } - // So far we applied changes in the memory VM and collected changes in the genesis struct - // No we iterate through all accounts that have been written there and set them inside the statedb. - // This will change the state root - // TODO(pl): We should have some checks that check we don't write accounts that aren't necessary, e.g. dev account - // Another property is that the total balance changes must be 0 - // Write changes to state to actual state database dbPath := ctx.String("db-path") if dbPath == "" { @@ -288,9 +283,20 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit return nil, fmt.Errorf("cannot create StateDB: %w", err) } + // So far we applied changes in the memory VM and collected changes in the genesis struct + // No we iterate through all accounts that have been written there and set them inside the statedb. + // This will change the state root + // Another property is that the total balance changes must be 0 + accountCounter := 0 + overwriteCounter := 0 + balanceDiff := big.NewInt(0) for k, v := range genesis.Alloc { + accountCounter++ if db.Exist(k) { - log.Warn("Operating on existing state", "account", k) + equal := bytes.Equal(db.GetCode(k), v.Code) + + log.Warn("Operating on existing state", "account", k, "equalCode", equal) + overwriteCounter++ } // TODO(pl): decide what to do with existing accounts. db.CreateAccount(k) @@ -299,7 +305,13 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit db.SetBalance(k, v.Balance) db.SetCode(k, v.Code) db.SetStorage(k, v.Storage) + log.Info("Moved account", "address", k) + balanceDiff = balanceDiff.Add(balanceDiff, v.Balance) + } + log.Info("Migrated OP contracts into state DB", "copiedAccounts", accountCounter, "overwrittenAccounts", overwriteCounter) + if balanceDiff.Sign() != 0 { + log.Warn("Deploying OP contracts changed native balance", "diff", balanceDiff) } // We're done messing around with the database, so we can now commit the changes to the DB. From 0900d6e230b8625a559d94ce4450545db9418699 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Fri, 3 May 2024 17:09:27 +0200 Subject: [PATCH 08/10] Review changes --- op-chain-ops/cmd/celo-migrate/README.md | 31 +------ op-chain-ops/cmd/celo-migrate/main.go | 104 ++++++++++-------------- 2 files changed, 46 insertions(+), 89 deletions(-) diff --git a/op-chain-ops/cmd/celo-migrate/README.md b/op-chain-ops/cmd/celo-migrate/README.md index 9c0777cf47811..2f4cf2652e3a6 100644 --- a/op-chain-ops/cmd/celo-migrate/README.md +++ b/op-chain-ops/cmd/celo-migrate/README.md @@ -1,30 +1,3 @@ -# Celo L1 -> Cel2 migrator +# Celo L1 -> Cel2 migration tool -This tool allows migrating the state of a Celo chain to a genesis block for a CeL2 chain. - -## Test Setup - -Create a local chain (assuming this in run in the `celo-blockchain` repository): - -```sh -build/bin/mycelo genesis --buildpath compiled-system-contracts --dev.accounts 2 --newenv tmp/testenv --mnemonic "miss fire behind decide egg buyer honey seven advance uniform profit renew" -build/bin/mycelo validator-init tmp/testenv/ -build/bin/mycelo validator-run tmp/testenv/ -``` - -Create some data - -```sh -build/bin/mycelo load-bot tmp/testenv -``` - -To run the migration, run in `op-chain-ops` (set `CELO_DATADIR` if the `celo-blockchain` repo is not located at `~/celo-blockchain`): - -```sh -make && ./migrate.sh -``` - -## Tasks - -- Import and change `BuildL2Genesis` - - Don't set balances for predeploys/precompiles +This tool allows migrating the state of a Celo chain to a state usable in op-geth. diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index 65073f43c10b3..8f24d4cbeeefc 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -38,11 +38,11 @@ var ( } l1DeploymentsFlag = &cli.PathFlag{ Name: "l1-deployments", - Usage: "Path to L1 deployments JSON file. Cannot be used with --deployment-dir", + Usage: "Path to L1 deployments JSON file", } l1RPCFlag = &cli.StringFlag{ Name: "l1-rpc", - Usage: "RPC URL for an Ethereum L1 node. Cannot be used with --l1-starting-block", + Usage: "RPC URL for an Ethereum L1 node", } outfileL2Flag = &cli.PathFlag{ Name: "outfile.l2", @@ -52,7 +52,6 @@ var ( Name: "outfile.rollup", Usage: "Path to rollup output file", } - dbPathFlag = &cli.StringFlag{ Name: "db-path", Usage: "Path to database", @@ -85,8 +84,8 @@ var ( dryRunFlag, } - // from `packages/contracts-bedrock/deploy-config/internal-devnet.json` - EIP1559Denominator = uint64(50) // TODO(pl): select values + // TODO: read those form the deploy config + EIP1559Denominator = uint64(50) EIP1559Elasticity = uint64(10) ) @@ -102,39 +101,45 @@ func main() { if deployConfig == "" { return fmt.Errorf("must specify --deploy-config") } - log.Info("Deploy config", "path", deployConfig) - config, err := genesis.NewDeployConfig(deployConfig) - if err != nil { - return err - } - - // Try reading the L1 deployment information l1Deployments := ctx.Path("l1-deployments") if l1Deployments == "" { return fmt.Errorf("must specify --l1-deployments") } - deployments, err := genesis.NewL1Deployments(l1Deployments) - if err != nil { - return fmt.Errorf("cannot read L1 deployments at %s: %w", l1Deployments, err) - } - config.SetDeployments(deployments) - - // Get latest block information from L1 l1RPC := ctx.String("l1-rpc") if l1RPC == "" { return fmt.Errorf("must specify --l1-rpc") } - outfileL2 := ctx.Path("outfile.l2") if outfileL2 == "" { return fmt.Errorf("must specify --outfile.l2") } - outfileRollup := ctx.Path("outfile.rollup") if outfileRollup == "" { return fmt.Errorf("must specify --outfile.rollup") } + dbPath := ctx.String("db-path") + if dbPath == "" { + return fmt.Errorf("must specify --db-path") + } + dbCache := ctx.Int("db-cache") + dbHandles := ctx.Int("db-handles") + dryRun := ctx.Bool("dry-run") + + // Read deployment configuration + log.Info("Deploy config", "path", deployConfig) + config, err := genesis.NewDeployConfig(deployConfig) + if err != nil { + return err + } + + // Try reading the L1 deployment information + deployments, err := genesis.NewL1Deployments(l1Deployments) + if err != nil { + return fmt.Errorf("cannot read L1 deployments at %s: %w", l1Deployments, err) + } + config.SetDeployments(deployments) + // Get latest block information from L1 var l1StartBlock *types.Block client, err := ethclient.Dial(l1RPC) if err != nil { @@ -180,31 +185,10 @@ func main() { } // Write changes to state to actual state database - dbPath := ctx.String("db-path") - if dbPath == "" { - return fmt.Errorf("must specify --db-path") - } - dbCache := ctx.Int("db-cache") - dbHandles := ctx.Int("db-handles") - dryRun := ctx.Bool("dry-run") - // TODO(pl): Move this into the function - log.Info("Opening database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) - ldb, err := openCeloDb(dbPath, dbCache, dbHandles) + cel2Header, err := ApplyMigrationChangesToDB(l2Genesis, dbPath, dbCache, dbHandles, !dryRun) if err != nil { - return fmt.Errorf("cannot open DB: %w", err) - } - log.Info("Loaded Celo L1 DB", "db", ldb) - - cel2Header, err := ApplyMigrationChangesToDB(ldb, l2Genesis, !dryRun) - if err != nil { - return err - } - - // Close the database handle - if err := ldb.Close(); err != nil { return err } - log.Info("Updated Cel2 state") log.Info("Writing state diff", "file", outfileL2) @@ -236,8 +220,13 @@ func main() { log.Info("Finished migration successfully!") } -func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit bool) (*types.Header, error) { - log.Info("Migrating DB") +func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int, dbHandles int, commit bool) (*types.Header, error) { + log.Info("Opening celo database", "dbCache", dbCache, "dbHandles", dbHandles, "dbPath", dbPath) + ldb, err := openCeloDb(dbPath, dbCache, dbHandles) + if err != nil { + return nil, fmt.Errorf("cannot open DB: %w", err) + } + log.Info("Loaded Celo L1 DB", "db", ldb) // Grab the hash of the tip of the legacy chain. hash := rawdb.ReadHeadHeaderHash(ldb) @@ -261,7 +250,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit if cfg == nil { log.Crit("chain config not found") } - log.Info("Read config from database", "config", cfg) + log.Info("Read chain config from database", "config", cfg) dbFactory := func() (*state.StateDB, error) { // Set up the backing store. @@ -323,7 +312,6 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit return nil, err } - log.Info("Creating new Genesis block") // Create the header for the Bedrock transition block. cel2Header := &types.Header{ ParentHash: header.Hash(), @@ -343,6 +331,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit Nonce: types.BlockNonce{}, BaseFee: new(big.Int).Set(header.BaseFee), } + log.Info("Build Cel2 migration header", "header", cel2Header) // Create the Bedrock transition block from the header. Note that there are no transactions, // uncle blocks, or receipts in the Bedrock transition block. @@ -350,7 +339,7 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit // We did it! log.Info( - "Built Celo migration block", + "Built Cel2 migration block", "hash", cel2Block.Hash(), "root", cel2Block.Root(), "number", cel2Block.NumberU64(), @@ -358,8 +347,6 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit "gas-limit", cel2Block.GasLimit(), ) - log.Info("Header", "header", cel2Header) - // If we're not actually writing this to disk, then we're done. if !commit { log.Info("Dry run complete") @@ -407,28 +394,25 @@ func ApplyMigrationChangesToDB(ldb ethdb.Database, genesis *core.Genesis, commit cfg.ShanghaiTime = &cel2Header.Time cfg.Cel2Time = &cel2Header.Time - log.Info("Write new config to database", "config", cfg) - // Write the chain config to disk. // TODO(pl): Why do we need to write this with the genesis hash, not `cel2Block.Hash()`?` rawdb.WriteChainConfig(ldb, genesisHash, cfg) - - // Yay! - log.Info( - "Wrote chain config", - "1559-denominator", EIP1559Denominator, - "1559-elasticity", EIP1559Elasticity, - ) + log.Info("Wrote updated chain config", "config", cfg) // We're done! log.Info( - "Wrote CeL2 transition block", + "Wrote CeL2 migration block", "height", cel2Header.Number, "root", cel2Header.Root.String(), "hash", cel2Header.Hash().String(), "timestamp", cel2Header.Time, ) + // Close the database handle + if err := ldb.Close(); err != nil { + return nil, err + } + return cel2Header, nil } From bf9af3b1d9e289f6f8fb2b6cdbe53460d688662a Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Tue, 7 May 2024 11:45:42 +0200 Subject: [PATCH 09/10] Review changes II --- op-chain-ops/cmd/celo-migrate/main.go | 33 ++++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index 8f24d4cbeeefc..a65ef96b5fa9a 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -241,8 +241,7 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int // Grab the full header. header := rawdb.ReadHeader(ldb, hash, *num) - // trieRoot := header.Root - log.Info("Read header from database", "number", header) + log.Info("Read header from database", "header", header) // We need to update the chain config to set the correct hardforks. genesisHash := rawdb.ReadCanonicalHash(ldb, 0) @@ -273,12 +272,11 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int } // So far we applied changes in the memory VM and collected changes in the genesis struct - // No we iterate through all accounts that have been written there and set them inside the statedb. + // Now we iterate through all accounts that have been written there and set them inside the statedb. // This will change the state root // Another property is that the total balance changes must be 0 accountCounter := 0 overwriteCounter := 0 - balanceDiff := big.NewInt(0) for k, v := range genesis.Alloc { accountCounter++ if db.Exist(k) { @@ -290,24 +288,27 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int // TODO(pl): decide what to do with existing accounts. db.CreateAccount(k) + // CreateAccount above copied the balance, check if we change it + if db.GetBalance(k).Cmp(v.Balance) != 0 { + // TODO(pl): make this a hard error once the migration has been tested more + log.Warn("Moving account changed native balance", "address", k, "oldBalance", db.GetBalance(k), "newBalance", v.Balance) + } + db.SetNonce(k, v.Nonce) db.SetBalance(k, v.Balance) db.SetCode(k, v.Code) db.SetStorage(k, v.Storage) log.Info("Moved account", "address", k) - balanceDiff = balanceDiff.Add(balanceDiff, v.Balance) } log.Info("Migrated OP contracts into state DB", "copiedAccounts", accountCounter, "overwrittenAccounts", overwriteCounter) - if balanceDiff.Sign() != 0 { - log.Warn("Deploying OP contracts changed native balance", "diff", balanceDiff) - } + + migrationBlock := new(big.Int).Add(header.Number, common.Big1) // We're done messing around with the database, so we can now commit the changes to the DB. // Note that this doesn't actually write the changes to disk. log.Info("Committing state DB") - // TODO(pl): What block info to put here? - newRoot, err := db.Commit(1234, true) + newRoot, err := db.Commit(migrationBlock.Uint64(), true) if err != nil { return nil, err } @@ -318,11 +319,11 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int UncleHash: types.EmptyUncleHash, Coinbase: predeploys.SequencerFeeVaultAddr, Root: newRoot, - TxHash: types.EmptyRootHash, - ReceiptHash: types.EmptyRootHash, + TxHash: types.EmptyTxsHash, + ReceiptHash: types.EmptyReceiptsHash, Bloom: types.Bloom{}, Difficulty: new(big.Int).Set(common.Big0), - Number: new(big.Int).Add(header.Number, common.Big1), + Number: migrationBlock, GasLimit: header.GasLimit, GasUsed: 0, Time: uint64(time.Now().Unix()), // TODO(pl): Needed to avoid L1-L2 time mismatches @@ -368,8 +369,7 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int rawdb.WriteHeadFastBlockHash(ldb, cel2Block.Hash()) rawdb.WriteHeadHeaderHash(ldb, cel2Block.Hash()) - // TODO(pl): What does finalized mean here? - // Make the first CeL2 block a finalized block. + // Mark the first CeL2 block as finalized rawdb.WriteFinalizedBlockHash(ldb, cel2Block.Hash()) // Set the standard options. @@ -380,6 +380,8 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int cfg.MergeNetsplitBlock = cel2Block.Number() cfg.TerminalTotalDifficulty = big.NewInt(0) cfg.TerminalTotalDifficultyPassed = true + cfg.ShanghaiTime = &cel2Header.Time + cfg.CancunTime = &cel2Header.Time // Set the Optimism options. cfg.BedrockBlock = cel2Block.Number() @@ -391,7 +393,6 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int } cfg.CanyonTime = &cel2Header.Time cfg.EcotoneTime = &cel2Header.Time - cfg.ShanghaiTime = &cel2Header.Time cfg.Cel2Time = &cel2Header.Time // Write the chain config to disk. From d2245a3f438743d04be3fff7117ed081584b38d2 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Tue, 7 May 2024 11:48:49 +0200 Subject: [PATCH 10/10] Update op-chain-ops/cmd/celo-migrate/main.go Co-authored-by: piersy --- op-chain-ops/cmd/celo-migrate/main.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/op-chain-ops/cmd/celo-migrate/main.go b/op-chain-ops/cmd/celo-migrate/main.go index a65ef96b5fa9a..d6536084dd11a 100644 --- a/op-chain-ops/cmd/celo-migrate/main.go +++ b/op-chain-ops/cmd/celo-migrate/main.go @@ -251,24 +251,15 @@ func ApplyMigrationChangesToDB(genesis *core.Genesis, dbPath string, dbCache int } log.Info("Read chain config from database", "config", cfg) - dbFactory := func() (*state.StateDB, error) { - // Set up the backing store. - underlyingDB := state.NewDatabaseWithConfig(ldb, &trie.Config{ - Preimages: true, - }) - - // Open up the state database. - db, err := state.New(header.Root, underlyingDB, nil) - if err != nil { - return nil, fmt.Errorf("cannot open StateDB: %w", err) - } - - return db, nil - } + // Set up the backing store. + underlyingDB := state.NewDatabaseWithConfig(ldb, &trie.Config{ + Preimages: true, + }) - db, err := dbFactory() + // Open up the state database. + db, err := state.New(header.Root, underlyingDB, nil) if err != nil { - return nil, fmt.Errorf("cannot create StateDB: %w", err) + return nil, fmt.Errorf("cannot open StateDB: %w", err) } // So far we applied changes in the memory VM and collected changes in the genesis struct