diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a96ce3ec..7b8c8168bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,26 @@ # Changelog +## v1.4.9 +### FEATURE +* [\#2463](https://github.com/bnb-chain/bsc/pull/2463) utils: add check_blobtx.js +* [\#2470](https://github.com/bnb-chain/bsc/pull/2470) jsutils: faucet successful requests within blocks +* [\#2467](https://github.com/bnb-chain/bsc/pull/2467) internal/ethapi: add optional parameter for blobSidecars + +### IMPROVEMENT +* [\#2462](https://github.com/bnb-chain/bsc/pull/2462) cmd/utils: add a flag to change breathe block interval for testing +* [\#2497](https://github.com/bnb-chain/bsc/pull/2497) params/config: add Bohr hardfork +* [\#2479](https://github.com/bnb-chain/bsc/pull/2479) dev: ensure consistency in BPS bundle result + +### BUGFIX +* [\#2461](https://github.com/bnb-chain/bsc/pull/2461) eth/handler: check lists in body before broadcast blocks +* [\#2455](https://github.com/bnb-chain/bsc/pull/2455) cmd: fix memory leak when big dataset +* [\#2466](https://github.com/bnb-chain/bsc/pull/2466) sync: fix some sync issues caused by prune-block. +* [\#2475](https://github.com/bnb-chain/bsc/pull/2475) fix: move mev op to MinerAPI & add command to console +* [\#2473](https://github.com/bnb-chain/bsc/pull/2473) fix: limit the gas price of the mev bid +* [\#2484](https://github.com/bnb-chain/bsc/pull/2484) fix: fix inspect database error +* [\#2481](https://github.com/bnb-chain/bsc/pull/2481) fix: keep 9W blocks in ancient db when prune block +* [\#2495](https://github.com/bnb-chain/bsc/pull/2495) fix: add an empty freeze db +* [\#2507](https://github.com/bnb-chain/bsc/pull/2507) fix: waiting for the last simulation before pick best bid + ## v1.4.8 ### FEATURE * [\#2483](https://github.com/bnb-chain/bsc/pull/2483) core/vm: add secp256r1 into PrecompiledContractsHaber diff --git a/README.md b/README.md index d489e6aa40..9e38330f44 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ To combine DPoS and PoA for consensus, BNB Smart Chain implement a novel consens 2. Validators take turns to produce blocks in a PoA manner, similar to Ethereum's Clique consensus engine. 3. Validator set are elected in and out based on a staking based governance on BNB Beacon Chain. 4. The validator set change is relayed via a cross-chain communication mechanism. -5. Parlia consensus engine will interact with a set of [system contracts](https://docs.bnbchain.org/docs/learn/system-contract) to achieve liveness slash, revenue distributing and validator set renewing func. +5. Parlia consensus engine will interact with a set of [system contracts](https://docs.bnbchain.org/bnb-smart-chain/staking/overview/#system-contracts) to achieve liveness slash, revenue distributing and validator set renewing func. ### Light Client of BNB Beacon Chain @@ -183,7 +183,7 @@ This tool is optional and if you leave it out you can always attach to an alread #### 7. More -More details about [running a node](https://docs.bnbchain.org/docs/validator/fullnode) and [becoming a validator](https://docs.bnbchain.org/docs/validator/create-val) +More details about [running a node](https://docs.bnbchain.org/bnb-smart-chain/developers/node_operators/full_node/) and [becoming a validator](https://docs.bnbchain.org/bnb-smart-chain/validator/create-val/) *Note: Although some internal protective measures prevent transactions from crossing over between the main network and test network, you should always diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 7f810dbe20..6ffc150de1 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -64,6 +64,7 @@ var ( utils.CachePreimagesFlag, utils.OverrideCancun, utils.OverrideHaber, + utils.OverrideBohr, utils.OverrideVerkle, }, utils.DatabaseFlags), Description: ` @@ -261,6 +262,10 @@ func initGenesis(ctx *cli.Context) error { v := ctx.Uint64(utils.OverrideHaber.Name) overrides.OverrideHaber = &v } + if ctx.IsSet(utils.OverrideBohr.Name) { + v := ctx.Uint64(utils.OverrideBohr.Name) + overrides.OverrideBohr = &v + } if ctx.IsSet(utils.OverrideVerkle.Name) { v := ctx.Uint64(utils.OverrideVerkle.Name) overrides.OverrideVerkle = &v diff --git a/cmd/geth/config.go b/cmd/geth/config.go index d35e088ff2..e535dc8155 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -193,6 +193,10 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { v := ctx.Uint64(utils.OverrideHaber.Name) cfg.Eth.OverrideHaber = &v } + if ctx.IsSet(utils.OverrideBohr.Name) { + v := ctx.Uint64(utils.OverrideBohr.Name) + cfg.Eth.OverrideBohr = &v + } if ctx.IsSet(utils.OverrideVerkle.Name) { v := ctx.Uint64(utils.OverrideVerkle.Name) cfg.Eth.OverrideVerkle = &v diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 5a0aaf011a..0b2017eac4 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -106,12 +106,12 @@ Remove blockchain and state databases`, dbInspectTrieCmd = &cli.Command{ Action: inspectTrie, Name: "inspect-trie", - ArgsUsage: " ", + ArgsUsage: " ", Flags: []cli.Flag{ utils.DataDirFlag, utils.SyncModeFlag, }, - Usage: "Inspect the MPT tree of the account and contract.", + Usage: "Inspect the MPT tree of the account and contract. 'blocknum' can be latest/snapshot/number. 'topn' means output the top N storage tries info ranked by the total number of TrieNodes", Description: `This commands iterates the entrie WorldState.`, } dbCheckStateContentCmd = &cli.Command{ @@ -386,6 +386,7 @@ func inspectTrie(ctx *cli.Context) error { blockNumber uint64 trieRootHash common.Hash jobnum uint64 + topN uint64 ) stack, _ := makeConfigNode(ctx) @@ -405,24 +406,37 @@ func inspectTrie(ctx *cli.Context) error { var err error blockNumber, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) if err != nil { - return fmt.Errorf("failed to Parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err) + return fmt.Errorf("failed to parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err) } } if ctx.NArg() == 1 { jobnum = 1000 + topN = 10 + } else if ctx.NArg() == 2 { + var err error + jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + } + topN = 10 } else { var err error jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64) if err != nil { - return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + return fmt.Errorf("failed to parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + } + + topN, err = strconv.ParseUint(ctx.Args().Get(2), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse topn, Args[1]: %v, err: %v", ctx.Args().Get(1), err) } } if blockNumber != math.MaxUint64 { headerBlockHash = rawdb.ReadCanonicalHash(db, blockNumber) if headerBlockHash == (common.Hash{}) { - return errors.New("ReadHeadBlockHash empry hash") + return errors.New("ReadHeadBlockHash empty hash") } blockHeader := rawdb.ReadHeader(db, headerBlockHash, blockNumber) trieRootHash = blockHeader.Root @@ -437,6 +451,7 @@ func inspectTrie(ctx *cli.Context) error { if dbScheme == rawdb.PathScheme { config = &triedb.Config{ PathDB: utils.PathDBConfigAddJournalFilePath(stack, pathdb.ReadOnly), + Cache: 0, } } else if dbScheme == rawdb.HashScheme { config = triedb.HashDefaults @@ -448,7 +463,7 @@ func inspectTrie(ctx *cli.Context) error { fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, trieRootHash.String()) return err } - theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum) + theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum, int(topN)) if err != nil { return err } @@ -493,7 +508,7 @@ func ancientInspect(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() return rawdb.AncientInspect(db) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4689290d6b..9763794f7e 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -74,6 +74,7 @@ var ( utils.RialtoHash, utils.OverrideCancun, utils.OverrideHaber, + utils.OverrideBohr, utils.OverrideVerkle, utils.OverrideFullImmutabilityThreshold, utils.OverrideMinBlocksForBlobRequests, diff --git a/cmd/geth/pruneblock_test.go b/cmd/geth/pruneblock_test.go index 141f64f3ba..b2a93f65aa 100644 --- a/cmd/geth/pruneblock_test.go +++ b/cmd/geth/pruneblock_test.go @@ -75,7 +75,7 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient if err != nil { return nil, err } - frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly, disableFreeze, isLastOffset, pruneAncientData) + frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly, disableFreeze, isLastOffset, pruneAncientData, false) if err != nil { kvdb.Close() return nil, err @@ -178,11 +178,10 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai // Force run a freeze cycle type freezer interface { - Freeze() error + Freeze(threshold uint64) error Ancients() (uint64, error) } - blockchain.SetFinalized(blocks[len(blocks)-1].Header()) - db.(freezer).Freeze() + db.(freezer).Freeze(10) frozen, err := db.Ancients() //make sure there're frozen items diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 4409d21977..040736a0d7 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -43,9 +43,11 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" cli "github.com/urfave/cli/v2" ) @@ -245,7 +247,16 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) { NoBuild: true, AsyncBuild: false, } - snaptree, err := snapshot.New(snapconfig, chaindb, triedb.NewDatabase(chaindb, nil), headBlock.Root(), TriesInMemory, false) + dbScheme := rawdb.ReadStateScheme(chaindb) + var config *triedb.Config + if dbScheme == rawdb.PathScheme { + config = &triedb.Config{ + PathDB: utils.PathDBConfigAddJournalFilePath(stack, pathdb.ReadOnly), + } + } else if dbScheme == rawdb.HashScheme { + config = triedb.HashDefaults + } + snaptree, err := snapshot.New(snapconfig, chaindb, triedb.NewDatabase(chaindb, config), headBlock.Root(), TriesInMemory, false) if err != nil { log.Error("snaptree error", "err", err) return nil, err // The relevant snapshot(s) might not exist @@ -333,6 +344,9 @@ func pruneBlock(ctx *cli.Context) error { stack, config = makeConfigNode(ctx) defer stack.Close() blockAmountReserved = ctx.Uint64(utils.BlockAmountReserved.Name) + if blockAmountReserved < params.FullImmutabilityThreshold { + return fmt.Errorf("block-amount-reserved must be greater than or equal to %d", params.FullImmutabilityThreshold) + } chaindb, err = accessDb(ctx, stack) if err != nil { return err diff --git a/cmd/jsutils/check_blobtx.js b/cmd/jsutils/check_blobtx.js new file mode 100644 index 0000000000..2e8fe1ed1e --- /dev/null +++ b/cmd/jsutils/check_blobtx.js @@ -0,0 +1,51 @@ +import { ethers } from "ethers"; +import program from "commander"; + +// depends on ethjs v6.11.0+ for 4844, https://github.com/ethers-io/ethers.js/releases/tag/v6.11.0 +// BSC testnet enabled 4844 on block: 39539137 +// Usage: +// nvm use 20 +// node check_blobtx.js --rpc https://data-seed-prebsc-1-s1.binance.org:8545 --startNum 39539137 +// node check_blobtx.js --rpc https://data-seed-prebsc-1-s1.binance.org:8545 --startNum 39539137 --endNum 40345994 +program.option("--rpc ", "Rpc Server URL"); +program.option("--startNum ", "start block", 0); +program.option("--endNum ", "end block", 0); +program.parse(process.argv); + +const provider = new ethers.JsonRpcProvider(program.rpc); +const main = async () => { + var startBlock = parseInt(program.startNum) + var endBlock = parseInt(program.endNum) + if (isNaN(endBlock) || isNaN(startBlock) || startBlock == 0) { + console.error("invalid input, --startNum", program.startNum, "--end", program.endNum) + return + } + // if --endNum is not specified, set it to the latest block number. + if (endBlock == 0) { + endBlock = await provider.getBlockNumber(); + } + if (startBlock > endBlock) { + console.error("invalid input, startBlock:",startBlock, " endBlock:", endBlock); + return + } + + for (let i = startBlock; i <= endBlock; i++) { + let blockData = await provider.getBlock(i); + console.log("startBlock:",startBlock, "endBlock:", endBlock, "curBlock", i, "blobGasUsed", blockData.blobGasUsed); + if (blockData.blobGasUsed == 0) { + continue + } + for (let txIndex = 0; txIndex<= blockData.transactions.length - 1; txIndex++) { + let txHash = blockData.transactions[txIndex] + let txData = await provider.getTransaction(txHash); + if (txData.type == 3) { + console.log("BlobTx in block:",i, " txIndex:", txIndex, " txHash:", txHash); + } + } + } +}; +main().then(() => process.exit(0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); \ No newline at end of file diff --git a/cmd/jsutils/faucet_request.js b/cmd/jsutils/faucet_request.js new file mode 100644 index 0000000000..bd4179747b --- /dev/null +++ b/cmd/jsutils/faucet_request.js @@ -0,0 +1,49 @@ +import { ethers } from "ethers"; +import program from "commander"; + +// Usage: +// node faucet_request.js --rpc localhost:8545 --startNum 39539137 +// node faucet_request.js --rpc localhost:8545 --startNum 39539137 --endNum 40345994 + +// node faucet_request.js --rpc https://data-seed-prebsc-1-s1.bnbchain.org:8545 --startNum 39539137 --endNum 40345994 +program.option("--rpc ", "Rpc Server URL"); +program.option("--startNum ", "start block", 0); +program.option("--endNum ", "end block", 0); +program.parse(process.argv); + +const provider = new ethers.JsonRpcProvider(program.rpc); +const main = async () => { + var startBlock = parseInt(program.startNum) + var endBlock = parseInt(program.endNum) + if (isNaN(endBlock) || isNaN(startBlock) || startBlock == 0) { + console.error("invalid input, --startNum", program.startNum, "--end", program.endNum) + return + } + // if --endNum is not specified, set it to the latest block number. + if (endBlock == 0) { + endBlock = await provider.getBlockNumber(); + } + if (startBlock > endBlock) { + console.error("invalid input, startBlock:",startBlock, " endBlock:", endBlock); + return + } + + let startBalance = await provider.getBalance("0xaa25Aa7a19f9c426E07dee59b12f944f4d9f1DD3", startBlock) + let endBalance = await provider.getBalance("0xaa25Aa7a19f9c426E07dee59b12f944f4d9f1DD3", endBlock) + const faucetAmount = BigInt(0.3 * 10**18); // Convert 0.3 ether to wei as a BigInt + const numFaucetRequest = (startBalance - endBalance) / faucetAmount; + + // Convert BigInt to ether + const startBalanceEth = Number(startBalance) / 10**18; + const endBalanceEth = Number(endBalance) / 10**18; + + console.log(`Start Balance: ${startBalanceEth} ETH`); + console.log(`End Balance: ${endBalanceEth} ETH`); + + console.log("successful faucet request: ",numFaucetRequest); +}; +main().then(() => process.exit(0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e6faa8d0f8..8707840692 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -315,6 +315,11 @@ var ( Usage: "Manually specify the Haber fork timestamp, overriding the bundled setting", Category: flags.EthCategory, } + OverrideBohr = &cli.Uint64Flag{ + Name: "override.bohr", + Usage: "Manually specify the Bohr fork timestamp, overriding the bundled setting", + Category: flags.EthCategory, + } OverrideVerkle = &cli.Uint64Flag{ Name: "override.verkle", Usage: "Manually specify the Verkle fork timestamp, overriding the bundled setting", @@ -1082,6 +1087,7 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Name: "block-amount-reserved", Usage: "Sets the expected remained amount of blocks for offline block prune", Category: flags.BlockHistoryCategory, + Value: params.FullImmutabilityThreshold, } CheckSnapshotWithMPT = &cli.BoolFlag{ diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 45a9da6265..ba206e9823 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -163,7 +163,7 @@ func TestHistoryImportAndExport(t *testing.T) { // Now import Era. freezer := t.TempDir() - db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false) + db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false, false) if err != nil { panic(err) } diff --git a/core/block_validator.go b/core/block_validator.go index d15e2cd786..b82965a99d 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -66,6 +66,31 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin return validator } +// ValidateListsInBody validates that UncleHash, WithdrawalsHash, and WithdrawalsHash correspond to the lists in the block body, respectively. +func ValidateListsInBody(block *types.Block) error { + header := block.Header() + if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash { + return fmt.Errorf("uncle root hash mismatch (header value %x, calculated %x)", header.UncleHash, hash) + } + if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { + return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) + } + // Withdrawals are present after the Shanghai fork. + if header.WithdrawalsHash != nil { + // Withdrawals list must be present in body after Shanghai. + if block.Withdrawals() == nil { + return errors.New("missing withdrawals in block body") + } + if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash { + return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash) + } + } else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars + // Withdrawals are not allowed prior to shanghai fork + return errors.New("withdrawals present in block body") + } + return nil +} + // ValidateBody validates the given block's uncles and verifies the block // header's transaction and uncle roots. The headers are assumed to be already // validated at this point. @@ -83,31 +108,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if err := v.engine.VerifyUncles(v.bc, block); err != nil { return err } - if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash { - return fmt.Errorf("uncle root hash mismatch (header value %x, calculated %x)", header.UncleHash, hash) - } validateFuns := []func() error{ func() error { - if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { - return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) - } - return nil + return ValidateListsInBody(block) }, func() error { - // Withdrawals are present after the Shanghai fork. - if header.WithdrawalsHash != nil { - // Withdrawals list must be present in body after Shanghai. - if block.Withdrawals() == nil { - return errors.New("missing withdrawals in block body") - } - if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash { - return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash) - } - } else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars - // Withdrawals are not allowed prior to shanghai fork - return errors.New("withdrawals present in block body") - } // Blob transactions may be present after the Cancun fork. var blobs int for i, tx := range block.Transactions() { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 8c22d66d11..d440590b8b 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -511,3 +511,12 @@ func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscr func (bc *BlockChain) SubscribeFinalizedHeaderEvent(ch chan<- FinalizedHeaderEvent) event.Subscription { return bc.scope.Track(bc.finalizedHeaderFeed.Subscribe(ch)) } + +// AncientTail retrieves the tail the ancients blocks +func (bc *BlockChain) AncientTail() (uint64, error) { + tail, err := bc.db.Tail() + if err != nil { + return 0, err + } + return tail, nil +} diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 820d4a1f64..3cfcdafe4a 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1832,14 +1832,10 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s } // Force run a freeze cycle type freezer interface { - Freeze() error + Freeze(threshold uint64) error Ancients() (uint64, error) } - if tt.freezeThreshold < uint64(tt.canonicalBlocks) { - final := uint64(tt.canonicalBlocks) - tt.freezeThreshold - chain.SetFinalized(canonblocks[int(final)-1].Header()) - } - db.(freezer).Freeze() + db.(freezer).Freeze(tt.freezeThreshold) // Set the simulated pivot block if tt.pivotBlock != nil { diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index cf067312e2..217610c33a 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -2045,14 +2045,10 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme // Force run a freeze cycle type freezer interface { - Freeze() error + Freeze(threshold uint64) error Ancients() (uint64, error) } - if tt.freezeThreshold < uint64(tt.canonicalBlocks) { - final := uint64(tt.canonicalBlocks) - tt.freezeThreshold - chain.SetFinalized(canonblocks[int(final)-1].Header()) - } - db.(freezer).Freeze() + db.(freezer).Freeze(tt.freezeThreshold) // Set the simulated pivot block if tt.pivotBlock != nil { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 43357713ef..3917117b91 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -974,7 +974,7 @@ func testFastVsFullChains(t *testing.T, scheme string) { t.Fatalf("failed to insert receipt %d: %v", n, err) } // Freezer style fast import the chain. - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -1069,7 +1069,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // makeDb creates a db instance for testing. makeDb := func() ethdb.Database { - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -1957,7 +1957,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) { competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) defer db.Close() chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) @@ -2026,7 +2026,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) { _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil) // Import the chain as a ancient-first node and ensure all pointers are updated - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2097,7 +2097,7 @@ func testInsertReceiptChainRollback(t *testing.T, scheme string) { } // Set up a BlockChain that uses the ancient store. - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2167,7 +2167,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) { }) // Import the canonical chain - diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) defer diskdb.Close() chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) @@ -2384,7 +2384,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) { b.OffsetTime(-9) // A higher difficulty }) // Import the shared chain and the original canonical one - chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2555,7 +2555,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i } }) // Import the shared chain and the original canonical one - chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -3858,7 +3858,7 @@ func testSetCanonical(t *testing.T, scheme string) { } gen.AddTx(tx) }) - diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) defer diskdb.Close() chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) @@ -4483,7 +4483,7 @@ func (c *mockParlia) CalcDifficulty(chain consensus.ChainHeaderReader, time uint func TestParliaBlobFeeReward(t *testing.T) { // Have N headers in the freezer frdir := t.TempDir() - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false) + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create database with ancient backend") } diff --git a/core/genesis.go b/core/genesis.go index b4833ac8a6..986f2c79d8 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -218,6 +218,7 @@ func (e *GenesisMismatchError) Error() string { type ChainOverrides struct { OverrideCancun *uint64 OverrideHaber *uint64 + OverrideBohr *uint64 OverrideVerkle *uint64 } @@ -250,6 +251,9 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g if overrides != nil && overrides.OverrideHaber != nil { config.HaberTime = overrides.OverrideHaber } + if overrides != nil && overrides.OverrideBohr != nil { + config.BohrTime = overrides.OverrideBohr + } if overrides != nil && overrides.OverrideVerkle != nil { config.VerkleTime = overrides.OverrideVerkle } diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 1b3e45f3db..8c76141369 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -518,7 +518,7 @@ func checkBlobSidecarsRLP(have, want types.BlobSidecars) error { func TestAncientStorage(t *testing.T) { // Freezer style fast import the chain. frdir := t.TempDir() - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false) + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create database with ancient backend") } @@ -657,7 +657,7 @@ func TestHashesInRange(t *testing.T) { func BenchmarkWriteAncientBlocks(b *testing.B) { // Open freezer database. frdir := b.TempDir() - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false) + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false, false) if err != nil { b.Fatalf("failed to create database with ancient backend") } @@ -1001,7 +1001,7 @@ func TestHeadersRLPStorage(t *testing.T) { // Have N headers in the freezer frdir := t.TempDir() - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false) + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create database with ancient backend") } diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 5d9e1d0394..31dd80c320 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -24,12 +24,12 @@ import ( "sync/atomic" "time" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" ) const ( @@ -51,25 +51,31 @@ var ( // The background thread will keep moving ancient chain segments from key-value // database to flat files for saving space on live database. type chainFreezer struct { + threshold atomic.Uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) + *Freezer quit chan struct{} wg sync.WaitGroup trigger chan chan struct{} // Manual blocking freeze trigger, test determinism freezeEnv atomic.Value + + multiDatabase bool } // newChainFreezer initializes the freezer for ancient chain data. -func newChainFreezer(datadir string, namespace string, readonly bool, offset uint64) (*chainFreezer, error) { +func newChainFreezer(datadir string, namespace string, readonly bool, offset uint64, multiDatabase bool) (*chainFreezer, error) { freezer, err := NewChainFreezer(datadir, namespace, readonly, offset) if err != nil { return nil, err } - return &chainFreezer{ + cf := chainFreezer{ Freezer: freezer, quit: make(chan struct{}), trigger: make(chan chan struct{}), - }, nil + } + cf.threshold.Store(params.FullImmutabilityThreshold) + return &cf, nil } // Close closes the chain freezer instance and terminates the background thread. @@ -185,29 +191,101 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { continue } - threshold, err := f.freezeThreshold(nfdb) - if err != nil { - backoff = true - log.Debug("Current full block not old enough to freeze", "err", err) - continue - } - frozen := f.frozen.Load() + var ( + frozen uint64 + threshold uint64 + first uint64 // the first block to freeze + last uint64 // the last block to freeze + + hash common.Hash + number *uint64 + head *types.Header + ) - // Short circuit if the blocks below threshold are already frozen. - if frozen != 0 && frozen-1 >= threshold { - backoff = true - log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen) - continue + // use finalized block as the chain freeze indicator was used for multiDatabase feature, if multiDatabase is false, keep 9W blocks in db + if f.multiDatabase { + threshold, err = f.freezeThreshold(nfdb) + if err != nil { + backoff = true + log.Debug("Current full block not old enough to freeze", "err", err) + continue + } + frozen = f.frozen.Load() + + // Short circuit if the blocks below threshold are already frozen. + if frozen != 0 && frozen-1 >= threshold { + backoff = true + log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen) + continue + } + + hash = ReadHeadBlockHash(nfdb) + if hash == (common.Hash{}) { + log.Debug("Current full block hash unavailable") // new chain, empty database + backoff = true + continue + } + number = ReadHeaderNumber(nfdb, hash) + if number == nil { + log.Error("Current full block number unavailable", "hash", hash) + backoff = true + continue + } + head = ReadHeader(nfdb, hash, *number) + if head == nil { + log.Error("Current full block unavailable", "number", *number, "hash", hash) + backoff = true + continue + } + + first = frozen + last = threshold + if last-first+1 > freezerBatchLimit { + last = freezerBatchLimit + first - 1 + } + } else { + // Retrieve the freezing threshold. + hash = ReadHeadBlockHash(nfdb) + if hash == (common.Hash{}) { + log.Debug("Current full block hash unavailable") // new chain, empty database + backoff = true + continue + } + number = ReadHeaderNumber(nfdb, hash) + threshold = f.threshold.Load() + frozen = f.frozen.Load() + switch { + case number == nil: + log.Error("Current full block number unavailable", "hash", hash) + backoff = true + continue + + case *number < threshold: + log.Debug("Current full block not old enough to freeze", "number", *number, "hash", hash, "delay", threshold) + backoff = true + continue + + case *number-threshold <= frozen: + log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", frozen) + backoff = true + continue + } + head = ReadHeader(nfdb, hash, *number) + if head == nil { + log.Error("Current full block unavailable", "number", *number, "hash", hash) + backoff = true + continue + } + first, _ = f.Ancients() + last = *number - threshold + if last-first > freezerBatchLimit { + last = first + freezerBatchLimit + } } // Seems we have data ready to be frozen, process in usable batches var ( start = time.Now() - first = frozen // the first block to freeze - last = threshold // the last block to freeze ) - if last-first+1 > freezerBatchLimit { - last = freezerBatchLimit + first - 1 - } ancients, err := f.freezeRangeWithBlobs(nfdb, first, last) if err != nil { @@ -295,24 +373,6 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { log.Debug("Deep froze chain segment", context...) env, _ := f.freezeEnv.Load().(*ethdb.FreezerEnv) - hash := ReadHeadBlockHash(nfdb) - if hash == (common.Hash{}) { - log.Debug("Current full block hash unavailable") // new chain, empty database - backoff = true - continue - } - number := ReadHeaderNumber(nfdb, hash) - if number == nil { - log.Error("Current full block number unavailable", "hash", hash) - backoff = true - continue - } - head := ReadHeader(nfdb, hash, *number) - if head == nil { - log.Error("Current full block unavailable", "number", *number, "hash", hash) - backoff = true - continue - } // try prune blob data after cancun fork if isCancun(env, head.Number, head.Time) { f.tryPruneBlobAncientTable(env, *number) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index a0e3147f2a..39dfcc5118 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -61,7 +61,7 @@ func (frdb *freezerdb) BlockStoreReader() ethdb.Reader { } func (frdb *freezerdb) BlockStoreWriter() ethdb.Writer { - //TODO implement me + // TODO implement me panic("implement me") } @@ -138,13 +138,22 @@ func (frdb *freezerdb) SetBlockStore(block ethdb.Database) { frdb.blockStore = block } +func (frdb *freezerdb) HasSeparateBlockStore() bool { + return frdb.blockStore != nil +} + // Freeze is a helper method used for external testing to trigger and block until // a freeze cycle completes, without having to sleep for a minute to trigger the // automatic background run. -func (frdb *freezerdb) Freeze() error { +func (frdb *freezerdb) Freeze(threshold uint64) error { if frdb.AncientStore.(*chainFreezer).readonly { return errReadOnly } + // Set the freezer threshold to a temporary value + defer func(old uint64) { + frdb.AncientStore.(*chainFreezer).threshold.Store(old) + }(frdb.AncientStore.(*chainFreezer).threshold.Load()) + frdb.AncientStore.(*chainFreezer).threshold.Store(threshold) // Trigger a freeze cycle and block until it's done trigger := make(chan struct{}, 1) frdb.AncientStore.(*chainFreezer).trigger <- trigger @@ -184,7 +193,7 @@ func (db *nofreezedb) Ancients() (uint64, error) { return 0, errNotSupported } -// Ancients returns an error as we don't have a backing chain freezer. +// ItemAmountInAncient returns an error as we don't have a backing chain freezer. func (db *nofreezedb) ItemAmountInAncient() (uint64, error) { return 0, errNotSupported } @@ -263,6 +272,10 @@ func (db *nofreezedb) SetBlockStore(block ethdb.Database) { db.blockStore = block } +func (db *nofreezedb) HasSeparateBlockStore() bool { + return db.blockStore != nil +} + func (db *nofreezedb) BlockStoreReader() ethdb.Reader { if db.blockStore != nil { return db.blockStore @@ -318,6 +331,110 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { return &nofreezedb{KeyValueStore: db} } +type emptyfreezedb struct { + ethdb.KeyValueStore +} + +// HasAncient returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) HasAncient(kind string, number uint64) (bool, error) { + return false, nil +} + +// Ancient returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) Ancient(kind string, number uint64) ([]byte, error) { + return nil, nil +} + +// AncientRange returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) { + return nil, nil +} + +// Ancients returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) Ancients() (uint64, error) { + return 0, nil +} + +// ItemAmountInAncient returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) ItemAmountInAncient() (uint64, error) { + return 0, nil +} + +// Tail returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) Tail() (uint64, error) { + return 0, nil +} + +// AncientSize returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) AncientSize(kind string) (uint64, error) { + return 0, nil +} + +// ModifyAncients returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) ModifyAncients(func(ethdb.AncientWriteOp) error) (int64, error) { + return 0, nil +} + +// TruncateHead returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) TruncateHead(items uint64) (uint64, error) { + return 0, nil +} + +// TruncateTail returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) TruncateTail(items uint64) (uint64, error) { + return 0, nil +} + +// TruncateTableTail returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) TruncateTableTail(kind string, tail uint64) (uint64, error) { + return 0, nil +} + +// ResetTable returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) ResetTable(kind string, startAt uint64, onlyEmpty bool) error { + return nil +} + +// Sync returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) Sync() error { + return nil +} + +func (db *emptyfreezedb) DiffStore() ethdb.KeyValueStore { return db } +func (db *emptyfreezedb) SetDiffStore(diff ethdb.KeyValueStore) {} +func (db *emptyfreezedb) StateStore() ethdb.Database { return db } +func (db *emptyfreezedb) SetStateStore(state ethdb.Database) {} +func (db *emptyfreezedb) StateStoreReader() ethdb.Reader { return db } +func (db *emptyfreezedb) BlockStore() ethdb.Database { return db } +func (db *emptyfreezedb) SetBlockStore(block ethdb.Database) {} +func (db *emptyfreezedb) HasSeparateBlockStore() bool { return false } +func (db *emptyfreezedb) BlockStoreReader() ethdb.Reader { return db } +func (db *emptyfreezedb) BlockStoreWriter() ethdb.Writer { return db } +func (db *emptyfreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { + return nil +} +func (db *emptyfreezedb) AncientOffSet() uint64 { return 0 } + +// MigrateTable returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) MigrateTable(kind string, convert convertLegacyFn) error { + return nil +} + +// AncientDatadir returns nil for pruned db that we don't have a backing chain freezer. +func (db *emptyfreezedb) AncientDatadir() (string, error) { + return "", nil +} +func (db *emptyfreezedb) SetupFreezerEnv(env *ethdb.FreezerEnv) error { + return nil +} + +// NewEmptyFreezeDB is used for CLI such as `geth db inspect` in pruned db that we don't +// have a backing chain freezer. +// WARNING: it must be only used in the above case. +func NewEmptyFreezeDB(db ethdb.KeyValueStore) ethdb.Database { + return &emptyfreezedb{KeyValueStore: db} +} + // NewFreezerDb only create a freezer without statedb. func NewFreezerDb(db ethdb.KeyValueStore, frz, namespace string, readonly bool, newOffSet uint64) (*Freezer, error) { // Create the idle freezer instance, this operation should be atomic to avoid mismatch between offset and acientDB. @@ -358,7 +475,7 @@ func resolveChainFreezerDir(ancient string) string { // value data store with a freezer moving immutable chain segments into cold // storage. The passed ancient indicates the path of root ancient directory // where the chain freezer can be opened. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly, disableFreeze, isLastOffset, pruneAncientData bool) (ethdb.Database, error) { +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly, disableFreeze, isLastOffset, pruneAncientData, multiDatabase bool) (ethdb.Database, error) { var offset uint64 // The offset of ancientDB should be handled differently in different scenarios. if isLastOffset { @@ -367,6 +484,12 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st offset = ReadOffSetOfCurrentAncientFreezer(db) } + // This case is used for someone who wants to execute geth db inspect CLI in a pruned db + if !disableFreeze && readonly && ReadAncientType(db) == PruneFreezerType { + log.Warn("Disk db is pruned, using an empty freezer db for CLI") + return NewEmptyFreezeDB(db), nil + } + if pruneAncientData && !disableFreeze && !readonly { frdb, err := newPrunedFreezer(resolveChainFreezerDir(ancient), db, offset) if err != nil { @@ -394,7 +517,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st } // Create the idle freezer instance - frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, offset) + frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, offset, multiDatabase) if err != nil { printChainMetadata(db) return nil, err @@ -575,6 +698,8 @@ type OpenOptions struct { // Ephemeral means that filesystem sync operations should be avoided: data integrity in the face of // a crash is not important. This option should typically be used in tests. Ephemeral bool + + MultiDataBase bool } // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. @@ -619,13 +744,13 @@ func Open(o OpenOptions) (ethdb.Database, error) { } if ReadAncientType(kvdb) == PruneFreezerType { if !o.PruneAncientData { - log.Warn("Disk db is pruned") + log.Warn("NOTICE: You're opening a pruned disk db!") } } if len(o.AncientsDirectory) == 0 { return kvdb, nil } - frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly, o.DisableFreeze, o.IsLastOffset, o.PruneAncientData) + frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly, o.DisableFreeze, o.IsLastOffset, o.PruneAncientData, o.MultiDataBase) if err != nil { kvdb.Close() return nil, err @@ -769,7 +894,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { trieIter = db.StateStore().NewIterator(keyPrefix, nil) defer trieIter.Release() } - if db.BlockStore() != db { + if db.HasSeparateBlockStore() { blockIter = db.BlockStore().NewIterator(keyPrefix, nil) defer blockIter.Release() } diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index ba1542294d..e1dc46bde3 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -239,7 +239,7 @@ func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) { // - if maxBytes is not specified, 'count' items will be returned if they are present. func (f *Freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { if table := f.tables[kind]; table != nil { - return table.RetrieveItems(start, count, maxBytes) + return table.RetrieveItems(start-f.offset, count, maxBytes) } return nil, errUnknownTable } @@ -252,7 +252,7 @@ func (f *Freezer) Ancients() (uint64, error) { func (f *Freezer) TableAncients(kind string) (uint64, error) { f.writeLock.RLock() defer f.writeLock.RUnlock() - return f.tables[kind].items.Load(), nil + return f.tables[kind].items.Load() + f.offset, nil } // ItemAmountInAncient returns the actual length of current ancientDB. diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 2dab09f869..c4a029bcad 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -43,6 +43,10 @@ func (t *table) SetBlockStore(block ethdb.Database) { panic("not implement") } +func (t *table) HasSeparateBlockStore() bool { + panic("not implement") +} + // NewTable returns a database object that prefixes all keys with a given string. func NewTable(db ethdb.Database, prefix string) ethdb.Database { return &table{ diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index e65c1b8929..f4820c06b2 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -382,7 +382,7 @@ func (p *BlockPruner) backUpOldDb(name string, cache, handles int, namespace str log.Info("chainDB opened successfully") // Get the number of items in old ancient db. - itemsOfAncient, err := chainDb.ItemAmountInAncient() + itemsOfAncient, err := chainDb.BlockStore().ItemAmountInAncient() log.Info("the number of items in ancientDB is ", "itemsOfAncient", itemsOfAncient) // If we can't access the freezer or it's empty, abort. diff --git a/core/txindexer_test.go b/core/txindexer_test.go index b59d2a841a..d078046355 100644 --- a/core/txindexer_test.go +++ b/core/txindexer_test.go @@ -212,7 +212,7 @@ func TestTxIndexer(t *testing.T) { } for _, c := range cases { frdir := t.TempDir() - db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false) + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false, false) rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) // Index the initial blocks from ancient store diff --git a/core/types/bid.go b/core/types/bid.go index a0d50239a8..e65ab0f003 100644 --- a/core/types/bid.go +++ b/core/types/bid.go @@ -6,6 +6,8 @@ import ( "sync/atomic" "time" + mapset "github.com/deckarep/golang-set/v2" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" @@ -40,6 +42,12 @@ func (b *BidArgs) ToBid(builder common.Address, signer Signer) (*Bid, error) { return nil, err } + if len(b.RawBid.UnRevertible) > len(txs) { + return nil, fmt.Errorf("expect NonRevertible no more than %d", len(txs)) + } + unRevertibleHashes := mapset.NewThreadUnsafeSetWithSize[common.Hash](len(b.RawBid.UnRevertible)) + unRevertibleHashes.Append(b.RawBid.UnRevertible...) + if len(b.PayBidTx) != 0 { var payBidTx = new(Transaction) err = payBidTx.UnmarshalBinary(b.PayBidTx) @@ -51,14 +59,15 @@ func (b *BidArgs) ToBid(builder common.Address, signer Signer) (*Bid, error) { } bid := &Bid{ - Builder: builder, - BlockNumber: b.RawBid.BlockNumber, - ParentHash: b.RawBid.ParentHash, - Txs: txs, - GasUsed: b.RawBid.GasUsed + b.PayBidTxGasUsed, - GasFee: b.RawBid.GasFee, - BuilderFee: b.RawBid.BuilderFee, - rawBid: *b.RawBid, + Builder: builder, + BlockNumber: b.RawBid.BlockNumber, + ParentHash: b.RawBid.ParentHash, + Txs: txs, + UnRevertible: unRevertibleHashes, + GasUsed: b.RawBid.GasUsed + b.PayBidTxGasUsed, + GasFee: b.RawBid.GasFee, + BuilderFee: b.RawBid.BuilderFee, + rawBid: *b.RawBid, } if bid.BuilderFee == nil { @@ -70,12 +79,13 @@ func (b *BidArgs) ToBid(builder common.Address, signer Signer) (*Bid, error) { // RawBid represents a raw bid from builder directly. type RawBid struct { - BlockNumber uint64 `json:"blockNumber"` - ParentHash common.Hash `json:"parentHash"` - Txs []hexutil.Bytes `json:"txs"` - GasUsed uint64 `json:"gasUsed"` - GasFee *big.Int `json:"gasFee"` - BuilderFee *big.Int `json:"builderFee"` + BlockNumber uint64 `json:"blockNumber"` + ParentHash common.Hash `json:"parentHash"` + Txs []hexutil.Bytes `json:"txs"` + UnRevertible []common.Hash `json:"unRevertible"` + GasUsed uint64 `json:"gasUsed"` + GasFee *big.Int `json:"gasFee"` + BuilderFee *big.Int `json:"builderFee"` hash atomic.Value } @@ -154,13 +164,14 @@ func (b *RawBid) Hash() common.Hash { // Bid represents a bid. type Bid struct { - Builder common.Address - BlockNumber uint64 - ParentHash common.Hash - Txs Transactions - GasUsed uint64 - GasFee *big.Int - BuilderFee *big.Int + Builder common.Address + BlockNumber uint64 + ParentHash common.Hash + Txs Transactions + UnRevertible mapset.Set[common.Hash] + GasUsed uint64 + GasFee *big.Int + BuilderFee *big.Int rawBid RawBid } diff --git a/eth/api_admin.go b/eth/api_admin.go index 76a0d087bb..4a3ccb84e8 100644 --- a/eth/api_admin.go +++ b/eth/api_admin.go @@ -24,7 +24,6 @@ import ( "os" "strings" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" @@ -142,31 +141,3 @@ func (api *AdminAPI) ImportChain(file string) (bool, error) { } return true, nil } - -// MevRunning returns true if the validator accept bids from builder -func (api *AdminAPI) MevRunning() bool { - return api.eth.APIBackend.MevRunning() -} - -// StartMev starts mev. It notifies the miner to start to receive bids. -func (api *AdminAPI) StartMev() { - api.eth.APIBackend.StartMev() -} - -// StopMev stops mev. It notifies the miner to stop receiving bids from this moment, -// but the bids before this moment would still been taken into consideration by mev. -func (api *AdminAPI) StopMev() { - api.eth.APIBackend.StopMev() -} - -// AddBuilder adds a builder to the bid simulator. -// url is the endpoint of the builder, for example, "https://mev-builder.amazonaws.com", -// if validator is equipped with sentry, ignore the url. -func (api *AdminAPI) AddBuilder(builder common.Address, url string) error { - return api.eth.APIBackend.AddBuilder(builder, url) -} - -// RemoveBuilder removes a builder from the bid simulator. -func (api *AdminAPI) RemoveBuilder(builder common.Address) error { - return api.eth.APIBackend.RemoveBuilder(builder) -} diff --git a/eth/api_miner.go b/eth/api_miner.go index 76398435c5..b8d571a475 100644 --- a/eth/api_miner.go +++ b/eth/api_miner.go @@ -89,3 +89,31 @@ func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool { func (api *MinerAPI) SetRecommitInterval(interval int) { api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) } + +// MevRunning returns true if the validator accept bids from builder +func (api *MinerAPI) MevRunning() bool { + return api.e.APIBackend.MevRunning() +} + +// StartMev starts mev. It notifies the miner to start to receive bids. +func (api *MinerAPI) StartMev() { + api.e.APIBackend.StartMev() +} + +// StopMev stops mev. It notifies the miner to stop receiving bids from this moment, +// but the bids before this moment would still been taken into consideration by mev. +func (api *MinerAPI) StopMev() { + api.e.APIBackend.StopMev() +} + +// AddBuilder adds a builder to the bid simulator. +// url is the endpoint of the builder, for example, "https://mev-builder.amazonaws.com", +// if validator is equipped with sentry, ignore the url. +func (api *MinerAPI) AddBuilder(builder common.Address, url string) error { + return api.e.APIBackend.AddBuilder(builder, url) +} + +// RemoveBuilder removes a builder from the bid simulator. +func (api *MinerAPI) RemoveBuilder(builder common.Address) error { + return api.e.APIBackend.RemoveBuilder(builder) +} diff --git a/eth/backend.go b/eth/backend.go index 89ce49ad1d..23136d719c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -187,6 +187,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { chainConfig.HaberTime = config.OverrideHaber overrides.OverrideHaber = config.OverrideHaber } + if config.OverrideBohr != nil { + chainConfig.BohrTime = config.OverrideBohr + overrides.OverrideBohr = config.OverrideBohr + } if config.OverrideVerkle != nil { chainConfig.VerkleTime = config.OverrideVerkle overrides.OverrideVerkle = config.OverrideVerkle diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 7e57153c28..664f775052 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -209,6 +209,9 @@ type BlockChain interface { // UpdateChasingHead update remote best chain head, used by DA check now. UpdateChasingHead(head *types.Header) + + // AncientTail retrieves the tail the ancients blocks + AncientTail() (uint64, error) } type DownloadOption func(downloader *Downloader) *Downloader @@ -797,6 +800,11 @@ func (d *Downloader) findAncestor(p *peerConnection, localHeight uint64, remoteH // We're above the max reorg threshold, find the earliest fork point floor = int64(localHeight - maxForkAncestry) } + // if we have pruned too much history, reset the floor + if tail, err := d.blockchain.AncientTail(); err == nil && tail > uint64(floor) { + floor = int64(tail) + } + // If we're doing a light sync, ensure the floor doesn't go below the CHT, as // all headers before that point will be missing. if mode == LightSync { diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 27fb95e74d..3c113b9134 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -60,7 +60,7 @@ func newTester(t *testing.T) *downloadTester { // newTester creates a new downloader test mocker. func newTesterWithNotification(t *testing.T, success func()) *downloadTester { freezer := t.TempDir() - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false) + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false, false) if err != nil { panic(err) } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 0b69b62219..a389a52fde 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -194,6 +194,9 @@ type Config struct { // OverrideHaber (TODO: remove after the fork) OverrideHaber *uint64 `toml:",omitempty"` + // OverrideBohr (TODO: remove after the fork) + OverrideBohr *uint64 `toml:",omitempty"` + // OverrideVerkle (TODO: remove after the fork) OverrideVerkle *uint64 `toml:",omitempty"` diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 4ed328d632..26c4587fe4 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -72,6 +72,7 @@ func (c Config) MarshalTOML() (interface{}, error) { RPCTxFeeCap float64 OverrideCancun *uint64 `toml:",omitempty"` OverrideHaber *uint64 `toml:",omitempty"` + OverrideBohr *uint64 `toml:",omitempty"` OverrideVerkle *uint64 `toml:",omitempty"` BlobExtraReserve uint64 } @@ -131,6 +132,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.OverrideCancun = c.OverrideCancun enc.OverrideHaber = c.OverrideHaber + enc.OverrideBohr = c.OverrideBohr enc.OverrideVerkle = c.OverrideVerkle enc.BlobExtraReserve = c.BlobExtraReserve return &enc, nil @@ -194,6 +196,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { RPCTxFeeCap *float64 OverrideCancun *uint64 `toml:",omitempty"` OverrideHaber *uint64 `toml:",omitempty"` + OverrideBohr *uint64 `toml:",omitempty"` OverrideVerkle *uint64 `toml:",omitempty"` BlobExtraReserve *uint64 } @@ -366,6 +369,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.OverrideHaber != nil { c.OverrideHaber = dec.OverrideHaber } + if dec.OverrideBohr != nil { + c.OverrideBohr = dec.OverrideBohr + } if dec.OverrideVerkle != nil { c.OverrideVerkle = dec.OverrideVerkle } diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index ea810c727d..fa39b1bbbd 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -158,7 +158,7 @@ func (f *fetcherTester) chainFinalizedHeight() uint64 { return f.blocks[f.hashes[len(f.hashes)-3]].NumberU64() } -// insertChain injects a new headers into the simulated chain. +// insertHeaders injects a new headers into the simulated chain. func (f *fetcherTester) insertHeaders(headers []*types.Header) (int, error) { f.lock.Lock() defer f.lock.Unlock() diff --git a/eth/handler.go b/eth/handler.go index 4a41276ed1..23dba9e14d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -320,26 +320,22 @@ func newHandler(config *handlerConfig) (*handler, error) { } broadcastBlockWithCheck := func(block *types.Block, propagate bool) { - // All the block fetcher activities should be disabled - // after the transition. Print the warning log. - if h.merger.PoSFinalized() { - log.Warn("Unexpected validation activity", "hash", block.Hash(), "number", block.Number()) - return - } - // Reject all the PoS style headers in the first place. No matter - // the chain has finished the transition or not, the PoS headers - // should only come from the trusted consensus layer instead of - // p2p network. - if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok { - if beacon.IsPoSHeader(block.Header()) { - log.Warn("unexpected post-merge header") - return - } - } if propagate { - if err := core.IsDataAvailable(h.chain, block); err != nil { - log.Error("Propagating block with invalid sidecars", "number", block.Number(), "hash", block.Hash(), "err", err) - return + checkErrs := make(chan error, 2) + + go func() { + checkErrs <- core.ValidateListsInBody(block) + }() + go func() { + checkErrs <- core.IsDataAvailable(h.chain, block) + }() + + for i := 0; i < cap(checkErrs); i++ { + err := <-checkErrs + if err != nil { + log.Error("Propagating invalid block", "number", block.Number(), "hash", block.Hash(), "err", err) + return + } } } h.BroadcastBlock(block, propagate) diff --git a/eth/handler_test.go b/eth/handler_test.go index 6db7bad6f0..5584afe857 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -258,7 +258,7 @@ func (c *mockParlia) CalcDifficulty(chain consensus.ChainHeaderReader, time uint func newTestParliaHandlerAfterCancun(t *testing.T, config *params.ChainConfig, mode downloader.SyncMode, preCancunBlks, postCancunBlks uint64) *testHandler { // Have N headers in the freezer frdir := t.TempDir() - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false) + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false, false) if err != nil { t.Fatalf("failed to create database with ancient backend") } diff --git a/ethdb/database.go b/ethdb/database.go index daff198e32..e4bfbc95bc 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -183,6 +183,7 @@ type StateStoreReader interface { type BlockStore interface { BlockStore() Database SetBlockStore(block Database) + HasSeparateBlockStore() bool } type BlockStoreReader interface { diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 18890f2227..2ba5807a32 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -44,6 +44,10 @@ func (db *Database) BlockStore() ethdb.Database { return db } +func (db *Database) HasSeparateBlockStore() bool { + return false +} + func (db *Database) SetBlockStore(block ethdb.Database) { panic("not supported") } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 793d31d315..361c46f9c0 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -659,6 +659,30 @@ web3._extend({ name: 'stop', call: 'miner_stop' }), + new web3._extend.Method({ + name: 'mevRunning', + call: 'miner_mevRunning' + }), + new web3._extend.Method({ + name: 'startMev', + call: 'miner_startMev' + }), + new web3._extend.Method({ + name: 'stopMev', + call: 'miner_stopMev' + }), + new web3._extend.Method({ + name: 'addBuilder', + call: 'miner_addBuilder', + params: 2, + inputFormatter: [web3._extend.formatters.inputAddressFormatter, null] + }), + new web3._extend.Method({ + name: 'removeBuilder', + call: 'miner_removeBuilder', + params: 1, + inputFormatter: [web3._extend.formatters.inputAddressFormatter] + }), new web3._extend.Method({ name: 'setEtherbase', call: 'miner_setEtherbase', diff --git a/miner/bid_simulator.go b/miner/bid_simulator.go index fd4c912257..e80dc90b13 100644 --- a/miner/bid_simulator.go +++ b/miner/bid_simulator.go @@ -29,11 +29,6 @@ import ( const ( // maxBidPerBuilderPerBlock is the max bid number per builder maxBidPerBuilderPerBlock = 3 - - // leftOverTimeRate is the rate of left over time to simulate a bid - leftOverTimeRate = 11 - // leftOverTimeScale is the scale of left over time to simulate a bid - leftOverTimeScale = 10 ) var ( @@ -78,6 +73,7 @@ type simBidReq struct { type bidSimulator struct { config *MevConfig delayLeftOver time.Duration + minGasPrice *big.Int chain *core.BlockChain chainConfig *params.ChainConfig engine consensus.Engine @@ -114,6 +110,7 @@ type bidSimulator struct { func newBidSimulator( config *MevConfig, delayLeftOver time.Duration, + minGasPrice *big.Int, chain *core.BlockChain, chainConfig *params.ChainConfig, engine consensus.Engine, @@ -122,6 +119,7 @@ func newBidSimulator( b := &bidSimulator{ config: config, delayLeftOver: delayLeftOver, + minGasPrice: minGasPrice, chain: chain, chainConfig: chainConfig, engine: engine, @@ -315,18 +313,6 @@ func (b *bidSimulator) newBidLoop() { // commit aborts in-flight bid execution with given signal and resubmits a new one. commit := func(reason int32, bidRuntime *BidRuntime) { - // if the left time is not enough to do simulation, return - var simDuration time.Duration - if lastBid := b.GetBestBid(bidRuntime.bid.ParentHash); lastBid != nil && lastBid.duration != 0 { - simDuration = lastBid.duration - } - - if time.Until(b.bidMustBefore(bidRuntime.bid.ParentHash)) <= simDuration*leftOverTimeRate/leftOverTimeScale { - log.Debug("BidSimulator: abort commit, not enough time to simulate", - "builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().Hex()) - return - } - if interruptCh != nil { // each commit work will have its own interruptCh to stop work with a reason interruptCh <- reason @@ -367,6 +353,7 @@ func (b *bidSimulator) newBidLoop() { expectedValidatorReward: expectedValidatorReward, packedBlockReward: big.NewInt(0), packedValidatorReward: big.NewInt(0), + finished: make(chan struct{}), } simulatingBid := b.GetSimulatingBid(newBid.ParentHash) @@ -407,11 +394,6 @@ func (b *bidSimulator) newBidLoop() { } } -func (b *bidSimulator) bidMustBefore(parentHash common.Hash) time.Time { - parentHeader := b.chain.GetHeaderByHash(parentHash) - return bidutil.BidMustBefore(parentHeader, b.chainConfig.Parlia.Period, b.delayLeftOver) -} - func (b *bidSimulator) bidBetterBefore(parentHash common.Hash) time.Time { parentHeader := b.chain.GetHeaderByHash(parentHash) return bidutil.BidBetterBefore(parentHeader, b.chainConfig.Parlia.Period, b.delayLeftOver, b.config.BidSimulationLeftOver) @@ -527,7 +509,6 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) { // ensure simulation exited then start next simulation b.SetSimulatingBid(parentHash, bidRuntime) - start := time.Now() defer func(simStart time.Time) { logCtx := []any{ @@ -553,10 +534,11 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) { } b.RemoveSimulatingBid(parentHash) - bidSimTimer.UpdateSince(start) + close(bidRuntime.finished) if success { bidRuntime.duration = time.Since(simStart) + bidSimTimer.UpdateSince(simStart) // only recommit self bid when newBidCh is empty if len(b.newBidCh) > 0 { @@ -580,6 +562,14 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) { return } + // if the left time is not enough to do simulation, return + delay := b.engine.Delay(b.chain, bidRuntime.env.header, &b.delayLeftOver) + if delay == nil || *delay <= 0 { + log.Info("BidSimulator: abort commit, not enough time to simulate", + "builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().Hex()) + return + } + gasLimit := bidRuntime.env.header.GasLimit if bidRuntime.env.gasPool == nil { bidRuntime.env.gasPool = new(core.GasPool).AddGas(gasLimit) @@ -592,6 +582,7 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) { return } + // commit transactions in bid for _, tx := range bidRuntime.bid.Txs { select { case <-interruptCh: @@ -609,7 +600,7 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) { break } - err = bidRuntime.commitTransaction(b.chain, b.chainConfig, tx) + err = bidRuntime.commitTransaction(b.chain, b.chainConfig, tx, bidRuntime.bid.UnRevertible.Contains(tx.Hash())) if err != nil { log.Error("BidSimulator: failed to commit tx", "bidHash", bidRuntime.bid.Hash(), "tx", tx.Hash(), "err", err) err = fmt.Errorf("invalid tx in bid, %v", err) @@ -617,26 +608,41 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) { } } - bidRuntime.packReward(b.config.ValidatorCommission) + // check if bid reward is valid + { + bidRuntime.packReward(b.config.ValidatorCommission) + if !bidRuntime.validReward() { + err = errors.New("reward does not achieve the expectation") + return + } + } + + // check if bid gas price is lower than min gas price + { + bidGasUsed := uint64(0) + bidGasFee := bidRuntime.env.state.GetBalance(consensus.SystemAddress) - // return if bid is invalid, reportIssue issue to mev-sentry/builder if simulation is fully done - if !bidRuntime.validReward() { - err = errors.New("reward does not achieve the expectation") - return + for _, receipt := range bidRuntime.env.receipts { + bidGasUsed += receipt.GasUsed + } + + bidGasPrice := new(big.Int).Div(bidGasFee.ToBig(), new(big.Int).SetUint64(bidGasUsed)) + if bidGasPrice.Cmp(b.minGasPrice) < 0 { + err = errors.New("bid gas price is lower than min gas price") + return + } } - // fill transactions from mempool + // if enable greedy merge, fill bid env with transactions from mempool if b.config.GreedyMergeTx { delay := b.engine.Delay(b.chain, bidRuntime.env.header, &b.delayLeftOver) if delay != nil && *delay > 0 { - stopTimer := time.NewTimer(*delay) - bidTxsSet := mapset.NewSet[common.Hash]() for _, tx := range bidRuntime.bid.Txs { bidTxsSet.Add(tx.Hash()) } - fillErr := b.bidWorker.fillTransactions(interruptCh, bidRuntime.env, stopTimer, bidTxsSet) + fillErr := b.bidWorker.fillTransactions(interruptCh, bidRuntime.env, nil, bidTxsSet) log.Trace("BidSimulator: greedy merge stopped", "block", bidRuntime.env.header.Number, "builder", bidRuntime.bid.Builder, "tx count", bidRuntime.env.tcount-bidTxLen+1, "err", fillErr) @@ -645,8 +651,9 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) { } } + // commit payBidTx at the end of the block bidRuntime.env.gasPool.AddGas(params.PayBidTxGasLimit) - err = bidRuntime.commitTransaction(b.chain, b.chainConfig, payBidTx) + err = bidRuntime.commitTransaction(b.chain, b.chainConfig, payBidTx, true) if err != nil { log.Error("BidSimulator: failed to commit tx", "builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash(), "tx", payBidTx.Hash(), "err", err) @@ -711,6 +718,7 @@ type BidRuntime struct { packedBlockReward *big.Int packedValidatorReward *big.Int + finished chan struct{} duration time.Duration } @@ -727,12 +735,10 @@ func (r *BidRuntime) packReward(validatorCommission uint64) { r.packedValidatorReward.Sub(r.packedValidatorReward, r.bid.BuilderFee) } -func (r *BidRuntime) commitTransaction(chain *core.BlockChain, chainConfig *params.ChainConfig, tx *types.Transaction) error { +func (r *BidRuntime) commitTransaction(chain *core.BlockChain, chainConfig *params.ChainConfig, tx *types.Transaction, unRevertible bool) error { var ( - env = r.env - snap = env.state.Snapshot() - gp = env.gasPool.Gas() - sc *types.BlobSidecar + env = r.env + sc *types.BlobSidecar ) // Start executing the transaction @@ -755,9 +761,9 @@ func (r *BidRuntime) commitTransaction(chain *core.BlockChain, chainConfig *para receipt, err := core.ApplyTransaction(chainConfig, chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, *chain.GetVMConfig(), core.NewReceiptBloomGenerator()) if err != nil { - env.state.RevertToSnapshot(snap) - env.gasPool.SetGas(gp) return err + } else if unRevertible && receipt.Status == types.ReceiptStatusFailed { + return errors.New("no revertible transaction failed") } if tx.Type() == types.BlobTxType { diff --git a/miner/miner.go b/miner/miner.go index 40503eb473..41f93ea388 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -102,7 +102,7 @@ func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *even worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, false), } - miner.bidSimulator = newBidSimulator(&config.Mev, config.DelayLeftOver, eth.BlockChain(), chainConfig, engine, miner.worker) + miner.bidSimulator = newBidSimulator(&config.Mev, config.DelayLeftOver, config.GasPrice, eth.BlockChain(), chainConfig, engine, miner.worker) miner.worker.setBestBidFetcher(miner.bidSimulator) miner.wg.Add(1) diff --git a/miner/worker.go b/miner/worker.go index 1677dc12b4..424a449355 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -67,6 +67,9 @@ const ( // the current 4 mining loops could have asynchronous risk of mining block with // save height, keep recently mined blocks to avoid double sign for safety, recentMinedCacheLimit = 20 + + // the default to wait for the mev miner to finish + waitMEVMinerEndTimeLimit = 50 * time.Millisecond ) var ( @@ -171,6 +174,7 @@ type getWorkReq struct { type bidFetcher interface { GetBestBid(parentHash common.Hash) *BidRuntime + GetSimulatingBid(prevBlockHash common.Hash) *BidRuntime } // worker is the main object which takes care of submitting new work to consensus engine @@ -1336,6 +1340,15 @@ LOOP: // when in-turn, compare with remote work. from := bestWork.coinbase if w.bidFetcher != nil && bestWork.header.Difficulty.Cmp(diffInTurn) == 0 { + if pendingBid := w.bidFetcher.GetSimulatingBid(bestWork.header.ParentHash); pendingBid != nil { + waitBidTimer := time.NewTimer(waitMEVMinerEndTimeLimit) + defer waitBidTimer.Stop() + select { + case <-waitBidTimer.C: + case <-pendingBid.finished: + } + } + bestBid := w.bidFetcher.GetBestBid(bestWork.header.ParentHash) if bestBid != nil { diff --git a/node/node.go b/node/node.go index 1dfc255535..7d07e576df 100644 --- a/node/node.go +++ b/node/node.go @@ -773,12 +773,13 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r db = rawdb.NewMemoryDatabase() } else { db, err = rawdb.Open(rawdb.OpenOptions{ - Type: n.config.DBEngine, - Directory: n.ResolvePath(name), - Namespace: namespace, - Cache: cache, - Handles: handles, - ReadOnly: readonly, + Type: n.config.DBEngine, + Directory: n.ResolvePath(name), + Namespace: namespace, + Cache: cache, + Handles: handles, + ReadOnly: readonly, + MultiDataBase: n.CheckIfMultiDataBase(), }) } diff --git a/params/config.go b/params/config.go index 610afc470f..e8d7049c36 100644 --- a/params/config.go +++ b/params/config.go @@ -153,6 +153,7 @@ var ( FeynmanFixTime: newUint64(1713419340), // 2024-04-18 05:49:00 AM UTC CancunTime: newUint64(1718863500), // 2024-06-20 06:05:00 AM UTC HaberTime: newUint64(1718863500), // 2024-06-20 06:05:00 AM UTC + BohrTime: nil, Parlia: &ParliaConfig{ Period: 3, @@ -192,6 +193,7 @@ var ( FeynmanFixTime: newUint64(1711342800), // 2024-03-25 5:00:00 AM UTC CancunTime: newUint64(1713330442), // 2024-04-17 05:07:22 AM UTC HaberTime: newUint64(1716962820), // 2024-05-29 06:07:00 AM UTC + BohrTime: nil, Parlia: &ParliaConfig{ Period: 3, @@ -232,6 +234,7 @@ var ( FeynmanFixTime: newUint64(0), CancunTime: newUint64(0), HaberTime: newUint64(0), + BohrTime: newUint64(0), Parlia: &ParliaConfig{ Period: 3, @@ -509,6 +512,7 @@ type ChainConfig struct { FeynmanFixTime *uint64 `json:"feynmanFixTime,omitempty"` // FeynmanFix switch time (nil = no fork, 0 = already activated) CancunTime *uint64 `json:"cancunTime,omitempty"` // Cancun switch time (nil = no fork, 0 = already on cancun) HaberTime *uint64 `json:"haberTime,omitempty"` // Haber switch time (nil = no fork, 0 = already on haber) + BohrTime *uint64 `json:"bohrTime,omitempty"` // Bohr switch time (nil = no fork, 0 = already on bohr) PragueTime *uint64 `json:"pragueTime,omitempty"` // Prague switch time (nil = no fork, 0 = already on prague) VerkleTime *uint64 `json:"verkleTime,omitempty"` // Verkle switch time (nil = no fork, 0 = already on verkle) @@ -619,7 +623,12 @@ func (c *ChainConfig) String() string { HaberTime = big.NewInt(0).SetUint64(*c.HaberTime) } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Ramanujan: %v, Niels: %v, MirrorSync: %v, Bruno: %v, Berlin: %v, YOLO v3: %v, CatalystBlock: %v, London: %v, ArrowGlacier: %v, MergeFork:%v, Euler: %v, Gibbs: %v, Nano: %v, Moran: %v, Planck: %v,Luban: %v, Plato: %v, Hertz: %v, Hertzfix: %v, ShanghaiTime: %v, KeplerTime: %v, FeynmanTime: %v, FeynmanFixTime: %v, CancunTime: %v, HaberTime: %v, Engine: %v}", + var BohrTime *big.Int + if c.BohrTime != nil { + BohrTime = big.NewInt(0).SetUint64(*c.BohrTime) + } + + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Ramanujan: %v, Niels: %v, MirrorSync: %v, Bruno: %v, Berlin: %v, YOLO v3: %v, CatalystBlock: %v, London: %v, ArrowGlacier: %v, MergeFork:%v, Euler: %v, Gibbs: %v, Nano: %v, Moran: %v, Planck: %v,Luban: %v, Plato: %v, Hertz: %v, Hertzfix: %v, ShanghaiTime: %v, KeplerTime: %v, FeynmanTime: %v, FeynmanFixTime: %v, CancunTime: %v, HaberTime: %v, BohrTime: %v, Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -657,6 +666,7 @@ func (c *ChainConfig) String() string { FeynmanFixTime, CancunTime, HaberTime, + BohrTime, engine, ) } @@ -929,6 +939,20 @@ func (c *ChainConfig) IsHaber(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.HaberTime, time) } +// IsBohr returns whether time is either equal to the Bohr fork time or greater. +func (c *ChainConfig) IsBohr(num *big.Int, time uint64) bool { + return c.IsLondon(num) && isTimestampForked(c.BohrTime, time) +} + +// IsOnBohr returns whether currentBlockTime is either equal to the Bohr fork time or greater firstly. +func (c *ChainConfig) IsOnBohr(currentBlockNumber *big.Int, lastBlockTime uint64, currentBlockTime uint64) bool { + lastBlockNumber := new(big.Int) + if currentBlockNumber.Cmp(big.NewInt(1)) >= 0 { + lastBlockNumber.Sub(currentBlockNumber, big.NewInt(1)) + } + return !c.IsBohr(lastBlockNumber, lastBlockTime) && c.IsBohr(currentBlockNumber, currentBlockTime) +} + // IsPrague returns whether num is either equal to the Prague fork time or greater. func (c *ChainConfig) IsPrague(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.PragueTime, time) @@ -993,6 +1017,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "feynmanFixTime", timestamp: c.FeynmanFixTime}, {name: "cancunTime", timestamp: c.CancunTime}, {name: "haberTime", timestamp: c.HaberTime}, + {name: "bohrTime", timestamp: c.BohrTime}, {name: "pragueTime", timestamp: c.PragueTime, optional: true}, {name: "verkleTime", timestamp: c.VerkleTime, optional: true}, } { @@ -1323,7 +1348,7 @@ type Rules struct { IsHertz bool IsHertzfix bool IsShanghai, IsKepler, IsFeynman, IsCancun, IsHaber bool - IsPrague, IsVerkle bool + IsBohr, IsPrague, IsVerkle bool } // Rules ensures c's ChainID is not nil. @@ -1359,6 +1384,7 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules IsFeynman: c.IsFeynman(num, timestamp), IsCancun: c.IsCancun(num, timestamp), IsHaber: c.IsHaber(num, timestamp), + IsBohr: c.IsBohr(num, timestamp), IsPrague: c.IsPrague(num, timestamp), IsVerkle: c.IsVerkle(num, timestamp), } diff --git a/params/version.go b/params/version.go index 69fff6015a..1b8117a5b8 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 4 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release + VersionPatch = 9 // Patch version component of the current release VersionMeta = "" // Version metadata to append to the version string ) diff --git a/trie/inspect_trie.go b/trie/inspect_trie.go index 885d3c7454..89f691a36d 100644 --- a/trie/inspect_trie.go +++ b/trie/inspect_trie.go @@ -4,17 +4,15 @@ import ( "bytes" "errors" "fmt" - "math/big" - "os" "runtime" - "sort" - "strconv" + "strings" "sync" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -26,63 +24,113 @@ import ( "golang.org/x/sync/semaphore" ) -type Account struct { - Nonce uint64 - Balance *big.Int - Root common.Hash // merkle root of the storage trie - CodeHash []byte -} - type Database interface { database.Database Scheme() string Cap(limit common.StorageSize) error DiskDB() ethdb.Database } + +const TopN = 3 + type Inspector struct { trie *Trie // traverse trie db Database stateRootHash common.Hash - blocknum uint64 + blockNum uint64 root node // root of triedb - totalNum uint64 - wg sync.WaitGroup - statLock sync.RWMutex - result map[string]*TrieTreeStat sem *semaphore.Weighted eoaAccountNums uint64 + + wg sync.WaitGroup + + results stat + topN int + + totalAccountNum atomic.Uint64 + totalStorageNum atomic.Uint64 + lastTime mclock.AbsTime +} + +type stat struct { + lock sync.RWMutex + account *trieStat + storageTopN []*trieStat + storageTopNTotal []uint64 + storageTotal nodeStat + storageTrieNum uint64 } -type TrieTreeStat struct { - is_account_trie bool - theNodeStatByLevel [15]NodeStat - totalNodeStat NodeStat +type trieStat struct { + owner common.Hash + totalNodeStat nodeStat + nodeStatByLevel [16]nodeStat } -type NodeStat struct { - ShortNodeCnt uint64 - FullNodeCnt uint64 - ValueNodeCnt uint64 +type nodeStat struct { + ShortNodeCnt atomic.Uint64 + FullNodeCnt atomic.Uint64 + ValueNodeCnt atomic.Uint64 +} + +func (ns *nodeStat) IsEmpty() bool { + if ns.FullNodeCnt.Load() == 0 && ns.ShortNodeCnt.Load() == 0 && ns.ValueNodeCnt.Load() == 0 { + return true + } + return false +} + +func (s *stat) add(ts *trieStat, topN int) { + s.lock.Lock() + defer s.lock.Unlock() + if ts.owner == (common.Hash{}) { + s.account = ts + return + } + + total := ts.totalNodeStat.ValueNodeCnt.Load() + ts.totalNodeStat.FullNodeCnt.Load() + ts.totalNodeStat.ShortNodeCnt.Load() + if len(s.storageTopNTotal) == 0 || total > s.storageTopNTotal[len(s.storageTopNTotal)-1] { + var ( + i int + t uint64 + ) + for i, t = range s.storageTopNTotal { + if total < t { + continue + } + break + } + s.storageTopNTotal = append(s.storageTopNTotal[:i], append([]uint64{total}, s.storageTopNTotal[i:]...)...) + s.storageTopN = append(s.storageTopN[:i], append([]*trieStat{ts}, s.storageTopN[i:]...)...) + if len(s.storageTopN) > topN { + s.storageTopNTotal = s.storageTopNTotal[:topN] + s.storageTopN = s.storageTopN[:topN] + } + } + + s.storageTotal.ShortNodeCnt.Add(ts.totalNodeStat.ShortNodeCnt.Load()) + s.storageTotal.ValueNodeCnt.Add(ts.totalNodeStat.ValueNodeCnt.Load()) + s.storageTotal.FullNodeCnt.Add(ts.totalNodeStat.FullNodeCnt.Load()) + s.storageTrieNum++ } -func (trieStat *TrieTreeStat) AtomicAdd(theNode node, height uint32) { +func (trieStat *trieStat) add(theNode node, height int) { switch (theNode).(type) { case *shortNode: - atomic.AddUint64(&trieStat.totalNodeStat.ShortNodeCnt, 1) - atomic.AddUint64(&(trieStat.theNodeStatByLevel[height].ShortNodeCnt), 1) + trieStat.totalNodeStat.ShortNodeCnt.Add(1) + trieStat.nodeStatByLevel[height].ShortNodeCnt.Add(1) case *fullNode: - atomic.AddUint64(&trieStat.totalNodeStat.FullNodeCnt, 1) - atomic.AddUint64(&trieStat.theNodeStatByLevel[height].FullNodeCnt, 1) + trieStat.totalNodeStat.FullNodeCnt.Add(1) + trieStat.nodeStatByLevel[height].FullNodeCnt.Add(1) case valueNode: - atomic.AddUint64(&trieStat.totalNodeStat.ValueNodeCnt, 1) - atomic.AddUint64(&((trieStat.theNodeStatByLevel[height]).ValueNodeCnt), 1) - default: - panic(errors.New("Invalid node type to statistics")) + trieStat.totalNodeStat.ValueNodeCnt.Add(1) + trieStat.nodeStatByLevel[height].ValueNodeCnt.Add(1) } } -func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) { - table := tablewriter.NewWriter(os.Stdout) +func (trieStat *trieStat) Display(ownerAddress string, treeType string) string { + sw := new(strings.Builder) + table := tablewriter.NewWriter(sw) table.SetHeader([]string{"-", "Level", "ShortNodeCnt", "FullNodeCnt", "ValueNodeCnt"}) if ownerAddress == "" { table.SetCaption(true, fmt.Sprintf("%v", treeType)) @@ -90,38 +138,27 @@ func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) { table.SetCaption(true, fmt.Sprintf("%v-%v", treeType, ownerAddress)) } table.SetAlignment(1) - for i := 0; i < len(trieStat.theNodeStatByLevel); i++ { - nodeStat := trieStat.theNodeStatByLevel[i] - if nodeStat.FullNodeCnt == 0 && nodeStat.ShortNodeCnt == 0 && nodeStat.ValueNodeCnt == 0 { - break + + for i := range trieStat.nodeStatByLevel { + if trieStat.nodeStatByLevel[i].IsEmpty() { + continue } table.AppendBulk([][]string{ - {"-", strconv.Itoa(i), nodeStat.ShortNodeCount(), nodeStat.FullNodeCount(), nodeStat.ValueNodeCount()}, + {"-", fmt.Sprintf("%d", i), + fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].ShortNodeCnt.Load()), + fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].FullNodeCnt.Load()), + fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].ValueNodeCnt.Load())}, }) } table.AppendBulk([][]string{ - {"Total", "-", trieStat.totalNodeStat.ShortNodeCount(), trieStat.totalNodeStat.FullNodeCount(), trieStat.totalNodeStat.ValueNodeCount()}, + {"Total", "-", fmt.Sprintf("%d", trieStat.totalNodeStat.ShortNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.FullNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.ValueNodeCnt.Load())}, }) table.Render() -} - -func Uint64ToString(cnt uint64) string { - return fmt.Sprintf("%v", cnt) -} - -func (nodeStat *NodeStat) ShortNodeCount() string { - return Uint64ToString(nodeStat.ShortNodeCnt) -} - -func (nodeStat *NodeStat) FullNodeCount() string { - return Uint64ToString(nodeStat.FullNodeCnt) -} -func (nodeStat *NodeStat) ValueNodeCount() string { - return Uint64ToString(nodeStat.ValueNodeCnt) + return sw.String() } // NewInspector return a inspector obj -func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Inspector, error) { +func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blockNum uint64, jobNum uint64, topN int) (*Inspector, error) { if tr == nil { return nil, errors.New("trie is nil") } @@ -131,15 +168,20 @@ func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uin } ins := &Inspector{ - trie: tr, - db: db, - stateRootHash: stateRootHash, - blocknum: blocknum, - root: tr.root, - result: make(map[string]*TrieTreeStat), - totalNum: (uint64)(0), - wg: sync.WaitGroup{}, - sem: semaphore.NewWeighted(int64(jobnum)), + trie: tr, + db: db, + stateRootHash: stateRootHash, + blockNum: blockNum, + root: tr.root, + results: stat{}, + topN: topN, + totalAccountNum: atomic.Uint64{}, + totalStorageNum: atomic.Uint64{}, + lastTime: mclock.Now(), + sem: semaphore.NewWeighted(int64(jobNum)), + + wg: sync.WaitGroup{}, + eoaAccountNums: 0, } @@ -147,155 +189,123 @@ func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uin } // Run statistics, external call -func (inspect *Inspector) Run() { - accountTrieStat := &TrieTreeStat{ - is_account_trie: true, - } - if inspect.db.Scheme() == rawdb.HashScheme { - ticker := time.NewTicker(30 * time.Second) - go func() { - defer ticker.Stop() - for range ticker.C { - inspect.db.Cap(DEFAULT_TRIEDBCACHE_SIZE) +func (s *Inspector) Run() { + ticker := time.NewTicker(30 * time.Second) + go func() { + defer ticker.Stop() + for range ticker.C { + if s.db.Scheme() == rawdb.HashScheme { + s.db.Cap(DEFAULT_TRIEDBCACHE_SIZE) } - }() - } - - if _, ok := inspect.result[""]; !ok { - inspect.result[""] = accountTrieStat - } - log.Info("Find Account Trie Tree", "rootHash: ", inspect.trie.Hash().String(), "BlockNum: ", inspect.blocknum) - - inspect.ConcurrentTraversal(inspect.trie, accountTrieStat, inspect.root, 0, []byte{}) - inspect.wg.Wait() -} + runtime.GC() + } + }() -func (inspect *Inspector) SubConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, theNode, height, path) - inspect.wg.Done() -} + log.Info("Find Account Trie Tree", "rootHash: ", s.trie.Hash().String(), "BlockNum: ", s.blockNum) -func (inspect *Inspector) ConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { - // print process progress - total_num := atomic.AddUint64(&inspect.totalNum, 1) - if total_num%100000 == 0 { - fmt.Printf("Complete progress: %v, go routines Num: %v\n", total_num, runtime.NumGoroutine()) + ts := &trieStat{ + owner: common.Hash{}, } + s.traversal(s.trie, ts, s.root, 0, []byte{}) + s.results.add(ts, s.topN) + s.wg.Wait() +} +func (s *Inspector) traversal(trie *Trie, ts *trieStat, n node, height int, path []byte) { // nil node - if theNode == nil { + if n == nil { return } - switch current := (theNode).(type) { + ts.add(n, height) + + switch current := (n).(type) { case *shortNode: - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, current.Val, height, append(path, current.Key...)) + s.traversal(trie, ts, current.Val, height, append(path, current.Key...)) case *fullNode: for idx, child := range current.Children { if child == nil { continue } - childPath := append(path, byte(idx)) - if inspect.sem.TryAcquire(1) { - inspect.wg.Add(1) - dst := make([]byte, len(childPath)) - copy(dst, childPath) - go inspect.SubConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, dst) - } else { - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, childPath) - } + p := common.CopyBytes(append(path, byte(idx))) + s.traversal(trie, ts, child, height+1, p) } case hashNode: - n, err := theTrie.resloveWithoutTrack(current, path) + tn, err := trie.resloveWithoutTrack(current, path) if err != nil { - fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, theTrie.Hash().String(), height+1, path) + fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, trie.Hash().String(), height+1, path) return } - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, n, height, path) - return + s.PrintProgress(trie) + s.traversal(trie, ts, tn, height, path) case valueNode: if !hasTerm(path) { break } - var account Account + var account types.StateAccount if err := rlp.Decode(bytes.NewReader(current), &account); err != nil { break } if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash { - inspect.eoaAccountNums++ + s.eoaAccountNums++ } if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { break } ownerAddress := common.BytesToHash(hexToCompact(path)) - contractTrie, err := New(StorageTrieID(inspect.stateRootHash, ownerAddress, account.Root), inspect.db) + contractTrie, err := New(StorageTrieID(s.stateRootHash, ownerAddress, account.Root), s.db) if err != nil { - fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", theNode, err, height, path) - break + panic(err) } contractTrie.tracer.reset() - trieStat := &TrieTreeStat{ - is_account_trie: false, - } - inspect.statLock.Lock() - if _, ok := inspect.result[ownerAddress.String()]; !ok { - inspect.result[ownerAddress.String()] = trieStat + if s.sem.TryAcquire(1) { + s.wg.Add(1) + go func() { + t := &trieStat{ + owner: ownerAddress, + } + s.traversal(contractTrie, t, contractTrie.root, 0, []byte{}) + s.results.add(t, s.topN) + s.sem.Release(1) + s.wg.Done() + }() + } else { + t := &trieStat{ + owner: ownerAddress, + } + s.traversal(contractTrie, t, contractTrie.root, 0, []byte{}) + s.results.add(t, s.topN) } - inspect.statLock.Unlock() - - // log.Info("Find Contract Trie Tree, rootHash: ", contractTrie.Hash().String(), "") - inspect.wg.Add(1) - go inspect.SubConcurrentTraversal(contractTrie, trieStat, contractTrie.root, 0, []byte{}) default: - panic(errors.New("Invalid node type to traverse.")) + panic(errors.New("invalid node type to traverse")) } - theTrieTreeStat.AtomicAdd(theNode, height) } -func (inspect *Inspector) DisplayResult() { - // display root hash - if _, ok := inspect.result[""]; !ok { - log.Info("Display result error", "missing account trie") - return +func (s *Inspector) PrintProgress(t *Trie) { + var ( + elapsed = mclock.Now().Sub(s.lastTime) + ) + if t.owner == (common.Hash{}) { + s.totalAccountNum.Add(1) + } else { + s.totalStorageNum.Add(1) } - inspect.result[""].Display("", "AccountTrie") - - type SortedTrie struct { - totalNum uint64 - ownerAddress string + if elapsed > 4*time.Second { + log.Info("traversal progress", "TotalAccountNum", s.totalAccountNum.Load(), "TotalStorageNum", s.totalStorageNum.Load(), "Goroutine", runtime.NumGoroutine()) + s.lastTime = mclock.Now() } - // display contract trie - var sortedTriesByNums []SortedTrie - var totalContactsNodeStat NodeStat - var contractTrieCnt uint64 = 0 +} - for ownerAddress, stat := range inspect.result { - if ownerAddress == "" { - continue - } - contractTrieCnt++ - totalContactsNodeStat.ShortNodeCnt += stat.totalNodeStat.ShortNodeCnt - totalContactsNodeStat.FullNodeCnt += stat.totalNodeStat.FullNodeCnt - totalContactsNodeStat.ValueNodeCnt += stat.totalNodeStat.ValueNodeCnt - totalNodeCnt := stat.totalNodeStat.ShortNodeCnt + stat.totalNodeStat.ValueNodeCnt + stat.totalNodeStat.FullNodeCnt - sortedTriesByNums = append(sortedTriesByNums, SortedTrie{totalNum: totalNodeCnt, ownerAddress: ownerAddress}) - } - sort.Slice(sortedTriesByNums, func(i, j int) bool { - return sortedTriesByNums[i].totalNum > sortedTriesByNums[j].totalNum - }) - fmt.Println("EOA accounts num: ", inspect.eoaAccountNums) - // only display top 5 - for i, t := range sortedTriesByNums { - if i > 5 { - break - } - if stat, ok := inspect.result[t.ownerAddress]; !ok { - log.Error("Storage trie stat not found", "ownerAddress", t.ownerAddress) - } else { - stat.Display(t.ownerAddress, "ContractTrie") - } +func (s *Inspector) DisplayResult() { + // display root hash + fmt.Println(s.results.account.Display("", "AccountTrie")) + fmt.Println("EOA accounts num: ", s.eoaAccountNums) + + // display contract trie + for _, st := range s.results.storageTopN { + fmt.Println(st.Display(st.owner.String(), "StorageTrie")) } fmt.Printf("Contract Trie, total trie num: %v, ShortNodeCnt: %v, FullNodeCnt: %v, ValueNodeCnt: %v\n", - contractTrieCnt, totalContactsNodeStat.ShortNodeCnt, totalContactsNodeStat.FullNodeCnt, totalContactsNodeStat.ValueNodeCnt) + s.results.storageTrieNum, s.results.storageTotal.ShortNodeCnt.Load(), s.results.storageTotal.FullNodeCnt.Load(), s.results.storageTotal.ValueNodeCnt.Load()) } diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 123b4e9ebf..368f1e5a8f 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -98,7 +98,7 @@ type tester struct { func newTester(t *testing.T, historyLimit uint64) *tester { var ( - disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) + disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false) db = New(disk, &Config{ StateHistory: historyLimit, CleanCacheSize: 256 * 1024, diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index 4fdcdd131e..3150ed1795 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -152,12 +152,13 @@ func (kr *JournalKVReader) Close() { } func newJournalWriter(file string, db ethdb.Database, journalType JournalType) JournalWriter { - log.Info("New journal writer", "path", file, "journalType", journalType) if journalType == JournalKVType { + log.Info("New journal writer for journal kv") return &JournalKVWriter{ diskdb: db, } } else { + log.Info("New journal writer for journal file", "path", file) fd, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { return nil @@ -169,8 +170,8 @@ func newJournalWriter(file string, db ethdb.Database, journalType JournalType) J } func newJournalReader(file string, db ethdb.Database, journalType JournalType) (JournalReader, error) { - log.Info("New journal reader", "path", file, "journalType", journalType) if journalType == JournalKVType { + log.Info("New journal reader for journal kv") journal := rawdb.ReadTrieJournal(db) if len(journal) == 0 { return nil, errMissJournal @@ -179,6 +180,7 @@ func newJournalReader(file string, db ethdb.Database, journalType JournalType) ( journalBuf: bytes.NewBuffer(journal), }, nil } else { + log.Info("New journal reader for journal file", "path", file) fd, err := os.Open(file) if errors.Is(err, fs.ErrNotExist) { return nil, errMissJournal