Skip to content

Commit

Permalink
core: separated databases for block data (#2227)
Browse files Browse the repository at this point in the history 
* core: use finalized block as the chain freeze indicator (#28683)
* core/rawdb: use max(finality, head-90k) as chain freezing threshold
* core: impl multi database for block data
* core: fix db inspect total size bug
* core: add tips for user who use multi-database
* core: adapt some cmd for multi-database
* core: adapter blockBlobSidecars
* core: fix freezer readHeader bug

---------
Co-authored-by: rjl493456442 <[email protected]>
  • Loading branch information
@jingjunLi
jingjunLi authored Apr 18, 2024
1 parent fb435eb commit c0afdc9
Show file tree
Hide file tree
Showing 27 changed files with 693 additions and 227 deletions.
18 changes: 16 additions & 2 deletions cmd/geth/chaincmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,12 +268,18 @@ func initGenesis(ctx *cli.Context) error {
defer chaindb.Close()

// if the trie data dir has been set, new trie db with a new state database
if ctx.IsSet(utils.SeparateDBFlag.Name) {
if ctx.IsSet(utils.MultiDataBaseFlag.Name) {
statediskdb, dbErr := stack.OpenDatabaseWithFreezer(name+"/state", 0, 0, "", "", false, false, false, false)
if dbErr != nil {
utils.Fatalf("Failed to open separate trie database: %v", dbErr)
}
chaindb.SetStateStore(statediskdb)
blockdb, err := stack.OpenDatabaseWithFreezer(name+"/block", 0, 0, "", "", false, false, false, false)
if err != nil {
utils.Fatalf("Failed to open separate block database: %v", err)
}
chaindb.SetBlockStore(blockdb)
log.Warn("Multi-database is an experimental feature")
}

triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
Expand Down Expand Up @@ -474,6 +480,13 @@ func dumpGenesis(ctx *cli.Context) error {
}
continue
}
// set the separate state & block database
if stack.CheckIfMultiDataBase() && err == nil {
stateDiskDb := utils.MakeStateDataBase(ctx, stack, true, false)
db.SetStateStore(stateDiskDb)
blockDb := utils.MakeBlockDatabase(ctx, stack, true, false)
db.SetBlockStore(blockDb)
}
genesis, err := core.ReadGenesis(db)
if err != nil {
utils.Fatalf("failed to read genesis: %s", err)
Expand Down Expand Up @@ -746,7 +759,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
arg := ctx.Args().First()
if hashish(arg) {
hash := common.HexToHash(arg)
if number := rawdb.ReadHeaderNumber(db, hash); number != nil {
if number := rawdb.ReadHeaderNumber(db.BlockStore(), hash); number != nil {
header = rawdb.ReadHeader(db, hash, *number)
} else {
return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
Expand Down Expand Up @@ -815,6 +828,7 @@ func dump(ctx *cli.Context) error {
if err != nil {
return err
}
defer db.Close()
triedb := utils.MakeTrieDatabase(ctx, db, true, true, false) // always enable preimage lookup
defer triedb.Close()

Expand Down
4 changes: 1 addition & 3 deletions cmd/geth/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
if ctx.IsSet(utils.OverrideDefaultExtraReserveForBlobRequests.Name) {
params.DefaultExtraReserveForBlobRequests = ctx.Uint64(utils.OverrideDefaultExtraReserveForBlobRequests.Name)
}
if ctx.IsSet(utils.SeparateDBFlag.Name) && !stack.IsSeparatedDB() {
utils.Fatalf("Failed to locate separate database subdirectory when separatedb parameter has been set")
}

backend, eth := utils.RegisterEthService(stack, &cfg.Eth)

// Create gauge with geth system and build information
Expand Down
96 changes: 65 additions & 31 deletions cmd/geth/dbcmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,8 +396,8 @@ func inspectTrie(ctx *cli.Context) error {
var headerBlockHash common.Hash
if ctx.NArg() >= 1 {
if ctx.Args().Get(0) == "latest" {
headerHash := rawdb.ReadHeadHeaderHash(db)
blockNumber = *(rawdb.ReadHeaderNumber(db, headerHash))
headerHash := rawdb.ReadHeadHeaderHash(db.BlockStore())
blockNumber = *(rawdb.ReadHeaderNumber(db.BlockStore(), headerHash))
} else if ctx.Args().Get(0) == "snapshot" {
trieRootHash = rawdb.ReadSnapshotRoot(db)
blockNumber = math.MaxUint64
Expand Down Expand Up @@ -519,14 +519,19 @@ func checkStateContent(ctx *cli.Context) error {
db := utils.MakeChainDatabase(ctx, stack, true, false)
defer db.Close()
var (
it = rawdb.NewKeyLengthIterator(db.NewIterator(prefix, start), 32)
it ethdb.Iterator
hasher = crypto.NewKeccakState()
got = make([]byte, 32)
errs int
count int
startTime = time.Now()
lastLog = time.Now()
)
if stack.CheckIfMultiDataBase() {
it = rawdb.NewKeyLengthIterator(db.StateStore().NewIterator(prefix, start), 32)
} else {
it = rawdb.NewKeyLengthIterator(db.NewIterator(prefix, start), 32)
}
for it.Next() {
count++
k := it.Key()
Expand Down Expand Up @@ -573,9 +578,11 @@ func dbStats(ctx *cli.Context) error {
defer db.Close()

showLeveldbStats(db)
if db.StateStore() != nil {
if stack.CheckIfMultiDataBase() {
fmt.Println("show stats of state store")
showLeveldbStats(db.StateStore())
fmt.Println("show stats of block store")
showLeveldbStats(db.BlockStore())
}

return nil
Expand All @@ -591,10 +598,11 @@ func dbCompact(ctx *cli.Context) error {
log.Info("Stats before compaction")
showLeveldbStats(db)

statediskdb := db.StateStore()
if statediskdb != nil {
if stack.CheckIfMultiDataBase() {
fmt.Println("show stats of state store")
showLeveldbStats(statediskdb)
showLeveldbStats(db.StateStore())
fmt.Println("show stats of block store")
showLeveldbStats(db.BlockStore())
}

log.Info("Triggering compaction")
Expand All @@ -603,18 +611,24 @@ func dbCompact(ctx *cli.Context) error {
return err
}

if statediskdb != nil {
if err := statediskdb.Compact(nil, nil); err != nil {
if stack.CheckIfMultiDataBase() {
if err := db.StateStore().Compact(nil, nil); err != nil {
log.Error("Compact err", "error", err)
return err
}
if err := db.BlockStore().Compact(nil, nil); err != nil {
log.Error("Compact err", "error", err)
return err
}
}

log.Info("Stats after compaction")
showLeveldbStats(db)
if statediskdb != nil {
if stack.CheckIfMultiDataBase() {
fmt.Println("show stats of state store after compaction")
showLeveldbStats(statediskdb)
showLeveldbStats(db.StateStore())
fmt.Println("show stats of block store after compaction")
showLeveldbStats(db.BlockStore())
}
return nil
}
Expand All @@ -635,18 +649,18 @@ func dbGet(ctx *cli.Context) error {
log.Info("Could not decode the key", "error", err)
return err
}
opDb := db
if stack.CheckIfMultiDataBase() {
keyType := rawdb.DataTypeByKey(key)
if keyType == rawdb.StateDataType {
opDb = db.StateStore()
} else if keyType == rawdb.BlockDataType {
opDb = db.BlockStore()
}
}

statediskdb := db.StateStore()
data, err := db.Get(key)
data, err := opDb.Get(key)
if err != nil {
// if separate trie db exist, try to get it from separate db
if statediskdb != nil {
statedata, dberr := statediskdb.Get(key)
if dberr == nil {
fmt.Printf("key %#x: %#x\n", key, statedata)
return nil
}
}
log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err)
return err
}
Expand Down Expand Up @@ -809,11 +823,21 @@ func dbDelete(ctx *cli.Context) error {
log.Info("Could not decode the key", "error", err)
return err
}
data, err := db.Get(key)
opDb := db
if stack.CheckIfMultiDataBase() {
keyType := rawdb.DataTypeByKey(key)
if keyType == rawdb.StateDataType {
opDb = db.StateStore()
} else if keyType == rawdb.BlockDataType {
opDb = db.BlockStore()
}
}

data, err := opDb.Get(key)
if err == nil {
fmt.Printf("Previous value: %#x\n", data)
}
if err = db.Delete(key); err != nil {
if err = opDb.Delete(key); err != nil {
log.Info("Delete operation returned an error", "key", fmt.Sprintf("%#x", key), "error", err)
return err
}
Expand Down Expand Up @@ -923,11 +947,22 @@ func dbPut(ctx *cli.Context) error {
log.Info("Could not decode the value", "error", err)
return err
}
data, err = db.Get(key)

opDb := db
if stack.CheckIfMultiDataBase() {
keyType := rawdb.DataTypeByKey(key)
if keyType == rawdb.StateDataType {
opDb = db.StateStore()
} else if keyType == rawdb.BlockDataType {
opDb = db.BlockStore()
}
}

data, err = opDb.Get(key)
if err == nil {
fmt.Printf("Previous value: %#x\n", data)
}
return db.Put(key, value)
return opDb.Put(key, value)
}

// dbDumpTrie shows the key-value slots of a given storage trie
Expand All @@ -940,7 +975,6 @@ func dbDumpTrie(ctx *cli.Context) error {

db := utils.MakeChainDatabase(ctx, stack, true, false)
defer db.Close()

triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
defer triedb.Close()

Expand Down Expand Up @@ -1019,7 +1053,7 @@ func freezerInspect(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
ancient := stack.ResolveAncient("chaindata", ctx.String(utils.AncientFlag.Name))
stack.Close()
return rawdb.InspectFreezerTable(ancient, freezer, table, start, end)
return rawdb.InspectFreezerTable(ancient, freezer, table, start, end, stack.CheckIfMultiDataBase())
}

func importLDBdata(ctx *cli.Context) error {
Expand Down Expand Up @@ -1159,11 +1193,11 @@ func showMetaData(ctx *cli.Context) error {
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack, true, false)
defer db.Close()
ancients, err := db.Ancients()
ancients, err := db.BlockStore().Ancients()
if err != nil {
fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err)
}
data := rawdb.ReadChainMetadata(db)
data := rawdb.ReadChainMetadataFromMultiDatabase(db)
data = append(data, []string{"frozen", fmt.Sprintf("%d items", ancients)})
data = append(data, []string{"snapshotGenerator", snapshot.ParseGeneratorStatus(rawdb.ReadSnapshotGenerator(db))})
if b := rawdb.ReadHeadBlock(db); b != nil {
Expand Down Expand Up @@ -1224,8 +1258,8 @@ func hbss2pbss(ctx *cli.Context) error {
log.Info("hbss2pbss triedb", "scheme", triedb.Scheme())
defer triedb.Close()

headerHash := rawdb.ReadHeadHeaderHash(db)
blockNumber := rawdb.ReadHeaderNumber(db, headerHash)
headerHash := rawdb.ReadHeadHeaderHash(db.BlockStore())
blockNumber := rawdb.ReadHeaderNumber(db.BlockStore(), headerHash)
if blockNumber == nil {
log.Error("read header number failed.")
return fmt.Errorf("read header number failed")
Expand Down
5 changes: 3 additions & 2 deletions cmd/geth/pruneblock_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,10 +178,11 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai

// Force run a freeze cycle
type freezer interface {
Freeze(threshold uint64) error
Freeze() error
Ancients() (uint64, error)
}
db.(freezer).Freeze(10)
blockchain.SetFinalized(blocks[len(blocks)-1].Header())
db.(freezer).Freeze()

frozen, err := db.Ancients()
//make sure there're frozen items
Expand Down
11 changes: 10 additions & 1 deletion cmd/geth/snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,15 @@ func pruneBlock(ctx *cli.Context) error {
if !ctx.IsSet(utils.AncientFlag.Name) {
return errors.New("datadir.ancient must be set")
} else {
oldAncientPath = ctx.String(utils.AncientFlag.Name)
if stack.CheckIfMultiDataBase() {
ancientPath := ctx.String(utils.AncientFlag.Name)
index := strings.LastIndex(ancientPath, "/ancient/chain")
if index != -1 {
oldAncientPath = ancientPath[:index] + "/block/ancient/chain"
}
} else {
oldAncientPath = ctx.String(utils.AncientFlag.Name)
}
if !filepath.IsAbs(oldAncientPath) {
// force absolute paths, which often fail due to the splicing of relative paths
return errors.New("datadir.ancient not abs path")
Expand Down Expand Up @@ -852,6 +860,7 @@ func dumpState(ctx *cli.Context) error {
if err != nil {
return err
}
defer db.Close()
triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
defer triedb.Close()

Expand Down
27 changes: 20 additions & 7 deletions cmd/utils/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,10 @@ var (
Value: flags.DirectoryString(node.DefaultDataDir()),
Category: flags.EthCategory,
}
SeparateDBFlag = &cli.BoolFlag{
Name: "separatedb",
Usage: "Enable a separated trie database, it will be created within a subdirectory called state, " +
"Users can copy this state directory to another directory or disk, and then create a symbolic link to the state directory under the chaindata",
MultiDataBaseFlag = &cli.BoolFlag{
Name: "multidatabase",
Usage: "Enable a separated state and block database, it will be created within two subdirectory called state and block, " +
"Users can copy this state or block directory to another directory or disk, and then create a symbolic link to the state directory under the chaindata",
Category: flags.EthCategory,
}
DirectBroadcastFlag = &cli.BoolFlag{
Expand Down Expand Up @@ -1150,7 +1150,7 @@ var (
DBEngineFlag,
StateSchemeFlag,
HttpHeaderFlag,
SeparateDBFlag,
MultiDataBaseFlag,
}
)

Expand Down Expand Up @@ -2387,9 +2387,11 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFree
default:
chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly, disableFreeze, false, false)
// set the separate state database
if stack.IsSeparatedDB() && err == nil {
if stack.CheckIfMultiDataBase() && err == nil {
stateDiskDb := MakeStateDataBase(ctx, stack, readonly, false)
chainDb.SetStateStore(stateDiskDb)
blockDb := MakeBlockDatabase(ctx, stack, readonly, false)
chainDb.SetBlockStore(blockDb)
}
}
if err != nil {
Expand All @@ -2401,14 +2403,25 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFree
// MakeStateDataBase open a separate state database using the flags passed to the client and will hard crash if it fails.
func MakeStateDataBase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database {
cache := ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100
handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) / 2
handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) * 90 / 100
statediskdb, err := stack.OpenDatabaseWithFreezer("chaindata/state", cache, handles, "", "", readonly, disableFreeze, false, false)
if err != nil {
Fatalf("Failed to open separate trie database: %v", err)
}
return statediskdb
}

// MakeBlockDatabase open a separate block database using the flags passed to the client and will hard crash if it fails.
func MakeBlockDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database {
cache := ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100
handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) / 10
blockDb, err := stack.OpenDatabaseWithFreezer("chaindata/block", cache, handles, "", "", readonly, disableFreeze, false, false)
if err != nil {
Fatalf("Failed to open separate block database: %v", err)
}
return blockDb
}

// tryMakeReadOnlyDatabase try to open the chain database in read-only mode,
// or fallback to write mode if the database is not initialized.
//
Expand Down
Loading

0 comments on commit c0afdc9

Please sign in to comment.