From 45dd6c5e2c770b85b6f213432f80cec7f449be86 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 28 Aug 2025 06:38:41 +0000 Subject: [PATCH 01/13] core/rawdb: inspect database in parallel Signed-off-by: jsvisa --- core/rawdb/database.go | 260 +++++++++++++++++++++++------------------ 1 file changed, 144 insertions(+), 116 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 6a1b7170664d..f39710d730de 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -25,6 +25,8 @@ import ( "path/filepath" "slices" "strings" + "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -32,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" + "golang.org/x/sync/errgroup" ) var ErrDeleteRangeInterrupted = errors.New("safe delete range operation interrupted") @@ -362,38 +365,40 @@ func (c counter) Percentage(current uint64) string { return fmt.Sprintf("%d", current*100/uint64(c)) } -// stat stores sizes and count for a parameter +// stat provides lock-free statistics aggregation using atomic operations type stat struct { - size common.StorageSize - count counter + size uint64 // stored as uint64 for atomic operations + count uint64 } -// Add size to the stat and increase the counter by 1 func (s *stat) Add(size common.StorageSize) { - s.size += size - s.count++ + atomic.AddUint64(&s.size, uint64(size)) + atomic.AddUint64(&s.count, 1) } func (s *stat) Size() string { - return s.size.String() + return common.StorageSize(atomic.LoadUint64(&s.size)).String() } func (s *stat) Count() string { - return s.count.String() + return counter(atomic.LoadUint64(&s.count)).String() +} + +func (s *stat) GetSize() common.StorageSize { + return common.StorageSize(atomic.LoadUint64(&s.size)) +} + +func (s *stat) GetCount() counter { + return counter(atomic.LoadUint64(&s.count)) } // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { - it := db.NewIterator(keyPrefix, keyStart) - defer it.Release() - var ( - count int64 - start = time.Now() - logged = time.Now() + start = time.Now() - // Key-value store statistics + // Key-value store statistics (lock-free with atomic operations) headers stat bodies stat receipts stat @@ -428,115 +433,128 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { unaccounted stat // Totals - total common.StorageSize + totalSize uint64 + totalCount int64 - // This map tracks example keys for unaccounted data. - // For each unique two-byte prefix, the first unaccounted key encountered - // by the iterator will be stored. unaccountedKeys = make(map[[2]byte][]byte) + unaccountedMu sync.Mutex ) - // Inspect key-value database first. - for it.Next() { - var ( - key = it.Key() - size = common.StorageSize(len(key) + len(it.Value())) - ) - total += size - switch { - case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): - headers.Add(size) - case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): - bodies.Add(size) - case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): - receipts.Add(size) - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): - tds.Add(size) - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): - numHashPairings.Add(size) - case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): - hashNumPairings.Add(size) - case IsLegacyTrieNode(key, it.Value()): - legacyTries.Add(size) - case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: - stateLookups.Add(size) - case IsAccountTrieNode(key): - accountTries.Add(size) - case IsStorageTrieNode(key): - storageTries.Add(size) - case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: - codes.Add(size) - case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): - txLookups.Add(size) - case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): - accountSnaps.Add(size) - case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): - storageSnaps.Add(size) - case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): - preimages.Add(size) - case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): - metadata.Add(size) - case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): - metadata.Add(size) - case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): - beaconHeaders.Add(size) - case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: - cliqueSnaps.Add(size) - - // new log index - case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9: - filterMapRows.Add(size) - case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4: - filterMapLastBlock.Add(size) - case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8: - filterMapBlockLV.Add(size) - - // old log index (deprecated) - case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): - bloomBits.Add(size) - case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: - bloomBits.Add(size) - - // Path-based historic state indexes - case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: - stateIndex.Add(size) - - // Verkle trie data is detected, determine the sub-category - case bytes.HasPrefix(key, VerklePrefix): - remain := key[len(VerklePrefix):] + + // Create worker function to process key ranges + processRange := func(rangePrefix []byte) error { + it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), keyStart) + defer it.Release() + + for it.Next() { + var ( + key = it.Key() + size = common.StorageSize(len(key) + len(it.Value())) + ) + atomic.AddUint64(&totalSize, uint64(size)) + atomic.AddInt64(&totalCount, 1) + switch { - case IsAccountTrieNode(remain): - verkleTries.Add(size) - case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength: - verkleStateLookups.Add(size) - case bytes.Equal(remain, persistentStateIDKey): + case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): + headers.Add(size) + case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): + bodies.Add(size) + case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): + receipts.Add(size) + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): + tds.Add(size) + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): + numHashPairings.Add(size) + case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): + hashNumPairings.Add(size) + case IsLegacyTrieNode(key, it.Value()): + legacyTries.Add(size) + case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: + stateLookups.Add(size) + case IsAccountTrieNode(key): + accountTries.Add(size) + case IsStorageTrieNode(key): + storageTries.Add(size) + case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: + codes.Add(size) + case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): + txLookups.Add(size) + case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): + accountSnaps.Add(size) + case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): + storageSnaps.Add(size) + case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): + preimages.Add(size) + case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): metadata.Add(size) - case bytes.Equal(remain, trieJournalKey): + case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): metadata.Add(size) - case bytes.Equal(remain, snapSyncStatusFlagKey): + case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): + beaconHeaders.Add(size) + case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: + cliqueSnaps.Add(size) + case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9: + filterMapRows.Add(size) + case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4: + filterMapLastBlock.Add(size) + case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8: + filterMapBlockLV.Add(size) + case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): + bloomBits.Add(size) + case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: + bloomBits.Add(size) + case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: + stateIndex.Add(size) + case bytes.HasPrefix(key, VerklePrefix): + remain := key[len(VerklePrefix):] + switch { + case IsAccountTrieNode(remain): + verkleTries.Add(size) + case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength: + verkleStateLookups.Add(size) + case bytes.Equal(remain, persistentStateIDKey): + metadata.Add(size) + case bytes.Equal(remain, trieJournalKey): + metadata.Add(size) + case bytes.Equal(remain, snapSyncStatusFlagKey): + metadata.Add(size) + default: + unaccounted.Add(size) + } + case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }): metadata.Add(size) default: unaccounted.Add(size) - } - - // Metadata keys - case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }): - metadata.Add(size) - - default: - unaccounted.Add(size) - if len(key) >= 2 { - prefix := [2]byte(key[:2]) - if _, ok := unaccountedKeys[prefix]; !ok { - unaccountedKeys[prefix] = bytes.Clone(key) + if len(key) >= 2 { + prefix := [2]byte(key[:2]) + unaccountedMu.Lock() + if _, ok := unaccountedKeys[prefix]; !ok { + unaccountedKeys[prefix] = bytes.Clone(key) + } + unaccountedMu.Unlock() } } } - count++ - if count%1000 == 0 && time.Since(logged) > 8*time.Second { - log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) - logged = time.Now() - } + + return it.Error() + } + + // Create error group for parallel processing + var g errgroup.Group + + // Split key space into 256 ranges (0x00 to 0xFF) + log.Info("Starting parallel database inspection", "workers", 256) + for i := 0; i < 256; i++ { + rangePrefix := []byte{byte(i)} + g.Go(func() error { + return processRange(rangePrefix) + }) + } + + // Wait for all workers to complete + if err := g.Wait(); err != nil { + return err } + // Display the database statistic of key-value store. stats := [][]string{ {"Key-Value store", "Headers", headers.Size(), headers.Count()}, @@ -565,10 +583,17 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, } + // Inspect all registered append-only file store then. ancients, err := inspectFreezers(db) if err != nil { - return err + // If freezer inspection is not supported, continue without ancient data + if err.Error() == "this operation is not supported" { + log.Warn("Freezer inspection not supported, skipping ancient data") + ancients = nil + } else { + return err + } } for _, ancient := range ancients { for _, table := range ancient.sizes { @@ -579,16 +604,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { fmt.Sprintf("%d", ancient.count()), }) } - total += ancient.size() + atomic.AddUint64(&totalSize, uint64(ancient.size())) } + table := newTableWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) - table.SetFooter([]string{"", "Total", total.String(), " "}) + table.SetFooter([]string{"", "Total", common.StorageSize(atomic.LoadUint64(&totalSize)).String(), fmt.Sprintf("%d", atomic.LoadInt64(&totalCount))}) table.AppendBulk(stats) table.Render() - if unaccounted.size > 0 { - log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count) + log.Info("Parallel database inspection completed", "total_entries", atomic.LoadInt64(&totalCount), "elapsed", common.PrettyDuration(time.Since(start))) + + if unaccounted.GetSize() > 0 { + log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount()) for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) { log.Error(fmt.Sprintf(" example key: %x", e)) } From fb2c4c3c5256ea291fb36c9fa5f6d937bfb289d1 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 28 Aug 2025 06:43:45 +0000 Subject: [PATCH 02/13] use atomic.Uint64 Signed-off-by: jsvisa --- core/rawdb/database.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index f39710d730de..b2d776a01d93 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -433,8 +433,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { unaccounted stat // Totals - totalSize uint64 - totalCount int64 + totalSize atomic.Uint64 + totalCount atomic.Int64 unaccountedKeys = make(map[[2]byte][]byte) unaccountedMu sync.Mutex @@ -450,8 +450,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { key = it.Key() size = common.StorageSize(len(key) + len(it.Value())) ) - atomic.AddUint64(&totalSize, uint64(size)) - atomic.AddInt64(&totalCount, 1) + totalSize.Add(uint64(size)) + totalCount.Add(1) switch { case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): @@ -604,16 +604,16 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { fmt.Sprintf("%d", ancient.count()), }) } - atomic.AddUint64(&totalSize, uint64(ancient.size())) + totalSize.Add(uint64(ancient.size())) } table := newTableWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) - table.SetFooter([]string{"", "Total", common.StorageSize(atomic.LoadUint64(&totalSize)).String(), fmt.Sprintf("%d", atomic.LoadInt64(&totalCount))}) + table.SetFooter([]string{"", "Total", common.StorageSize(totalSize.Load()).String(), fmt.Sprintf("%d", totalCount.Load())}) table.AppendBulk(stats) table.Render() - log.Info("Parallel database inspection completed", "total_entries", atomic.LoadInt64(&totalCount), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Parallel database inspection completed", "total_entries", totalCount.Load(), "elapsed", common.PrettyDuration(time.Since(start))) if unaccounted.GetSize() > 0 { log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount()) From 1b0289c6c6bf21c1d4c75bef19a68c381a36cbaa Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 28 Aug 2025 06:56:32 +0000 Subject: [PATCH 03/13] inspect --workers Signed-off-by: jsvisa --- cmd/geth/dbcmd.go | 12 ++++++++++-- core/rawdb/database.go | 19 ++++++++----------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 44a52521f04a..adf89188b233 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -22,6 +22,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -89,7 +90,13 @@ Remove blockchain and state databases`, Action: inspect, Name: "inspect", ArgsUsage: " ", - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags, []cli.Flag{ + &cli.IntFlag{ + Name: "workers", + Usage: "Number of parallel workers for database inspection", + Value: runtime.NumCPU(), + }, + }), Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } @@ -329,7 +336,8 @@ func inspect(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() - return rawdb.InspectDatabase(db, prefix, start) + workers := ctx.Int("workers") + return rawdb.InspectDatabase(db, prefix, start, workers) } func checkStateContent(ctx *cli.Context) error { diff --git a/core/rawdb/database.go b/core/rawdb/database.go index b2d776a01d93..82219d96b900 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -394,7 +394,7 @@ func (s *stat) GetCount() counter { // InspectDatabase traverses the entire database and checks the size // of all different categories of data. -func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { +func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) error { var ( start = time.Now() @@ -538,20 +538,17 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { return it.Error() } - // Create error group for parallel processing - var g errgroup.Group + var eg errgroup.Group + eg.SetLimit(workers) + + log.Info("Starting parallel database inspection", "workers", workers) - // Split key space into 256 ranges (0x00 to 0xFF) - log.Info("Starting parallel database inspection", "workers", 256) for i := 0; i < 256; i++ { rangePrefix := []byte{byte(i)} - g.Go(func() error { - return processRange(rangePrefix) - }) + eg.Go(func() error { return processRange(rangePrefix) }) } - // Wait for all workers to complete - if err := g.Wait(); err != nil { + if err := eg.Wait(); err != nil { return err } @@ -613,7 +610,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { table.AppendBulk(stats) table.Render() - log.Info("Parallel database inspection completed", "total_entries", totalCount.Load(), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Database inspection completed", "count", totalCount.Load(), "elapsed", common.PrettyDuration(time.Since(start))) if unaccounted.GetSize() > 0 { log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount()) From f10e78bece03fffbe56f487b5f6a85a071e76a9c Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 28 Aug 2025 07:00:27 +0000 Subject: [PATCH 04/13] timer print Signed-off-by: jsvisa --- core/rawdb/database.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 82219d96b900..c6165d582c90 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -541,6 +541,22 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) var eg errgroup.Group eg.SetLimit(workers) + // Start progress reporting goroutine + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(8 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + log.Info("Inspecting database progress", "count", totalCount.Load(), "size", common.StorageSize(totalSize.Load()), "elapsed", common.PrettyDuration(time.Since(start))) + case <-done: + return + } + } + }() + log.Info("Starting parallel database inspection", "workers", workers) for i := 0; i < 256; i++ { @@ -549,8 +565,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) } if err := eg.Wait(); err != nil { + close(done) return err } + close(done) // Display the database statistic of key-value store. stats := [][]string{ From 07515cf07346f97c6bb27407d241678f671f40c3 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 28 Aug 2025 07:16:30 +0000 Subject: [PATCH 05/13] refine Signed-off-by: jsvisa --- core/rawdb/database.go | 53 +++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index c6165d582c90..4c7e1f442ca5 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -367,10 +367,11 @@ func (c counter) Percentage(current uint64) string { // stat provides lock-free statistics aggregation using atomic operations type stat struct { - size uint64 // stored as uint64 for atomic operations + size uint64 count uint64 } +// Add size to the stat and increase the counter by 1 func (s *stat) Add(size common.StorageSize) { atomic.AddUint64(&s.size, uint64(size)) atomic.AddUint64(&s.count, 1) @@ -397,8 +398,10 @@ func (s *stat) GetCount() counter { func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) error { var ( start = time.Now() + count atomic.Int64 + total atomic.Uint64 - // Key-value store statistics (lock-free with atomic operations) + // Key-value store statistics headers stat bodies stat receipts stat @@ -432,15 +435,13 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) metadata stat unaccounted stat - // Totals - totalSize atomic.Uint64 - totalCount atomic.Int64 - + // This map tracks example keys for unaccounted data. + // For each unique two-byte prefix, the first unaccounted key encountered + // by the iterator will be stored. unaccountedKeys = make(map[[2]byte][]byte) unaccountedMu sync.Mutex ) - // Create worker function to process key ranges processRange := func(rangePrefix []byte) error { it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), keyStart) defer it.Release() @@ -450,8 +451,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) key = it.Key() size = common.StorageSize(len(key) + len(it.Value())) ) - totalSize.Add(uint64(size)) - totalCount.Add(1) + total.Add(uint64(size)) + count.Add(1) switch { case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): @@ -492,18 +493,26 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) beaconHeaders.Add(size) case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: cliqueSnaps.Add(size) + + // new log index case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9: filterMapRows.Add(size) case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4: filterMapLastBlock.Add(size) case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8: filterMapBlockLV.Add(size) + + // old log index (deprecated) case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): bloomBits.Add(size) case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: bloomBits.Add(size) + + // Path-based historic state indexes case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: stateIndex.Add(size) + + // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): remain := key[len(VerklePrefix):] switch { @@ -520,8 +529,11 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) default: unaccounted.Add(size) } + + // Metadata keys case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }): metadata.Add(size) + default: unaccounted.Add(size) if len(key) >= 2 { @@ -541,7 +553,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) var eg errgroup.Group eg.SetLimit(workers) - // Start progress reporting goroutine + log.Info("Starting parallel database inspection", "workers", workers) + done := make(chan struct{}) go func() { ticker := time.NewTicker(8 * time.Second) @@ -550,15 +563,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) for { select { case <-ticker.C: - log.Info("Inspecting database progress", "count", totalCount.Load(), "size", common.StorageSize(totalSize.Load()), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Inspecting database progress", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) case <-done: return } } }() - log.Info("Starting parallel database inspection", "workers", workers) - + // Inspect key-value database in parallel. for i := 0; i < 256; i++ { rangePrefix := []byte{byte(i)} eg.Go(func() error { return processRange(rangePrefix) }) @@ -602,13 +614,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) // Inspect all registered append-only file store then. ancients, err := inspectFreezers(db) if err != nil { - // If freezer inspection is not supported, continue without ancient data - if err.Error() == "this operation is not supported" { - log.Warn("Freezer inspection not supported, skipping ancient data") - ancients = nil - } else { - return err - } + return err } for _, ancient := range ancients { for _, table := range ancient.sizes { @@ -619,23 +625,22 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) fmt.Sprintf("%d", ancient.count()), }) } - totalSize.Add(uint64(ancient.size())) + total.Add(uint64(ancient.size())) } table := newTableWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) - table.SetFooter([]string{"", "Total", common.StorageSize(totalSize.Load()).String(), fmt.Sprintf("%d", totalCount.Load())}) + table.SetFooter([]string{"", "Total", common.StorageSize(total.Load()).String(), fmt.Sprintf("%d", count.Load())}) table.AppendBulk(stats) table.Render() - log.Info("Database inspection completed", "count", totalCount.Load(), "elapsed", common.PrettyDuration(time.Since(start))) - if unaccounted.GetSize() > 0 { log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount()) for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) { log.Error(fmt.Sprintf(" example key: %x", e)) } } + log.Info("Database inspection completed", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) return nil } From 11cf3a677b4932e6219d72fbf9f3c87e367ad547 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 28 Aug 2025 07:54:47 +0000 Subject: [PATCH 06/13] fmt Signed-off-by: jsvisa --- cmd/geth/dbcmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index adf89188b233..8c6f9532e824 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -87,9 +87,9 @@ Remove blockchain and state databases`, }, } dbInspectCmd = &cli.Command{ - Action: inspect, - Name: "inspect", - ArgsUsage: " ", + Action: inspect, + Name: "inspect", + ArgsUsage: " ", Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags, []cli.Flag{ &cli.IntFlag{ Name: "workers", From 27d9dc796a18fb6d78b9b439823d7b24cf1c81ae Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 28 Aug 2025 08:10:07 +0000 Subject: [PATCH 07/13] no need cli Signed-off-by: jsvisa --- cmd/geth/dbcmd.go | 18 +++++------------- core/rawdb/database.go | 8 ++++++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 8c6f9532e824..44a52521f04a 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" "path/filepath" - "runtime" "slices" "strconv" "strings" @@ -87,16 +86,10 @@ Remove blockchain and state databases`, }, } dbInspectCmd = &cli.Command{ - Action: inspect, - Name: "inspect", - ArgsUsage: " ", - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags, []cli.Flag{ - &cli.IntFlag{ - Name: "workers", - Usage: "Number of parallel workers for database inspection", - Value: runtime.NumCPU(), - }, - }), + Action: inspect, + Name: "inspect", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } @@ -336,8 +329,7 @@ func inspect(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() - workers := ctx.Int("workers") - return rawdb.InspectDatabase(db, prefix, start, workers) + return rawdb.InspectDatabase(db, prefix, start) } func checkStateContent(ctx *cli.Context) error { diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 4c7e1f442ca5..7b972e50e198 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -23,6 +23,7 @@ import ( "maps" "os" "path/filepath" + "runtime" "slices" "strings" "sync" @@ -395,7 +396,7 @@ func (s *stat) GetCount() counter { // InspectDatabase traverses the entire database and checks the size // of all different categories of data. -func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) error { +func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { var ( start = time.Now() count atomic.Int64 @@ -550,7 +551,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, workers int) return it.Error() } - var eg errgroup.Group + var ( + eg errgroup.Group + workers = runtime.NumCPU() + ) eg.SetLimit(workers) log.Info("Starting parallel database inspection", "workers", workers) From a63160d4c9926356e2ec2b38374e1e1c038e8246 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Fri, 29 Aug 2025 01:17:43 +0800 Subject: [PATCH 08/13] nil as the later ranges --- core/rawdb/database.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 7b972e50e198..93fb453f9939 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -444,7 +444,21 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { ) processRange := func(rangePrefix []byte) error { - it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), keyStart) + // Skip ranges that are entirely before keyStart + if len(keyStart) > 0 && len(rangePrefix) > 0 && rangePrefix[0] < keyStart[0] { + return nil + } + + var ( + prefix = slices.Concat(keyPrefix, rangePrefix) + start = []byte(nil) + ) + // Only apply keyStart to the range that contains it + if len(keyStart) > 0 && len(rangePrefix) > 0 && rangePrefix[0] == keyStart[0] { + start = keyStart[1:] // skip the first byte, as it's in rangePrefix + } + + it := db.NewIterator(prefix, start) defer it.Release() for it.Next() { From 57fec99aac1d85c1e13ec7084ec819da5f4a7a6f Mon Sep 17 00:00:00 2001 From: jsvisa Date: Sun, 31 Aug 2025 09:22:35 +0800 Subject: [PATCH 09/13] fix: with context to quick exit --- core/rawdb/database.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 93fb453f9939..de7efd3db3f3 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -18,6 +18,7 @@ package rawdb import ( "bytes" + "context" "errors" "fmt" "maps" @@ -443,7 +444,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { unaccountedMu sync.Mutex ) - processRange := func(rangePrefix []byte) error { + processRange := func(ctx context.Context, rangePrefix []byte) error { // Skip ranges that are entirely before keyStart if len(keyStart) > 0 && len(rangePrefix) > 0 && rangePrefix[0] < keyStart[0] { return nil @@ -560,13 +561,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { unaccountedMu.Unlock() } } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } } return it.Error() } var ( - eg errgroup.Group + eg, ctx = errgroup.WithContext(context.Background()) workers = runtime.NumCPU() ) eg.SetLimit(workers) @@ -591,7 +598,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Inspect key-value database in parallel. for i := 0; i < 256; i++ { rangePrefix := []byte{byte(i)} - eg.Go(func() error { return processRange(rangePrefix) }) + eg.Go(func() error { return processRange(ctx, rangePrefix) }) } if err := eg.Wait(); err != nil { From 9e80e2fd1de3fee396ca2a010a60c399ba276ddd Mon Sep 17 00:00:00 2001 From: jsvisa Date: Sun, 31 Aug 2025 09:47:56 +0800 Subject: [PATCH 10/13] fix: with context --- core/rawdb/database.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index de7efd3db3f3..824a53ad1cc7 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -445,21 +445,20 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { ) processRange := func(ctx context.Context, rangePrefix []byte) error { - // Skip ranges that are entirely before keyStart - if len(keyStart) > 0 && len(rangePrefix) > 0 && rangePrefix[0] < keyStart[0] { - return nil - } + var start = []byte(nil) - var ( - prefix = slices.Concat(keyPrefix, rangePrefix) - start = []byte(nil) - ) - // Only apply keyStart to the range that contains it - if len(keyStart) > 0 && len(rangePrefix) > 0 && rangePrefix[0] == keyStart[0] { - start = keyStart[1:] // skip the first byte, as it's in rangePrefix + if len(keyStart) > 0 && len(rangePrefix) > 0 { + r0, s0 := rangePrefix[0], keyStart[0] + if r0 < s0 { + // Skip ranges that are entirely before keyStart + return nil + } else if r0 == s0 { + // Only apply keyStart to the range that contains it + start = keyStart[1:] // skip the first byte, as it's in rangePrefix + } } - it := db.NewIterator(prefix, start) + it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), start) defer it.Release() for it.Next() { From d778ea25e46145fba395fe0ca7d1a5a6b38fc472 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Sun, 31 Aug 2025 09:50:16 +0800 Subject: [PATCH 11/13] typo --- core/rawdb/database.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 824a53ad1cc7..edbe067b3305 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -445,7 +445,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { ) processRange := func(ctx context.Context, rangePrefix []byte) error { - var start = []byte(nil) + var start []byte if len(keyStart) > 0 && len(rangePrefix) > 0 { r0, s0 := rangePrefix[0], keyStart[0] @@ -453,8 +453,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Skip ranges that are entirely before keyStart return nil } else if r0 == s0 { - // Only apply keyStart to the range that contains it - start = keyStart[1:] // skip the first byte, as it's in rangePrefix + // Only apply keyStart to the range that contains it, + // skip the first byte, as it's already in rangePrefix + start = keyStart[1:] } } From 9c92f0ec37eec46829140cb7261379e46a464a5c Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 1 Sep 2025 10:36:46 +0800 Subject: [PATCH 12/13] core/rawdb: polish --- core/rawdb/database.go | 172 +++++++++++++++++++---------------------- 1 file changed, 81 insertions(+), 91 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index edbe067b3305..5a6a8a377334 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -373,28 +373,23 @@ type stat struct { count uint64 } -// Add size to the stat and increase the counter by 1 -func (s *stat) Add(size common.StorageSize) { +func (s *stat) empty() bool { + return atomic.LoadUint64(&s.count) == 0 +} + +func (s *stat) add(size common.StorageSize) { atomic.AddUint64(&s.size, uint64(size)) atomic.AddUint64(&s.count, 1) } -func (s *stat) Size() string { +func (s *stat) sizeString() string { return common.StorageSize(atomic.LoadUint64(&s.size)).String() } -func (s *stat) Count() string { +func (s *stat) countString() string { return counter(atomic.LoadUint64(&s.count)).String() } -func (s *stat) GetSize() common.StorageSize { - return common.StorageSize(atomic.LoadUint64(&s.size)) -} - -func (s *stat) GetCount() counter { - return counter(atomic.LoadUint64(&s.count)) -} - // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { @@ -444,22 +439,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { unaccountedMu sync.Mutex ) - processRange := func(ctx context.Context, rangePrefix []byte) error { - var start []byte - - if len(keyStart) > 0 && len(rangePrefix) > 0 { - r0, s0 := rangePrefix[0], keyStart[0] - if r0 < s0 { - // Skip ranges that are entirely before keyStart + inspectRange := func(ctx context.Context, r byte) error { + var s []byte + if len(keyStart) > 0 { + switch { + case r < keyStart[0]: return nil - } else if r0 == s0 { - // Only apply keyStart to the range that contains it, - // skip the first byte, as it's already in rangePrefix - start = keyStart[1:] + case r == keyStart[0]: + s = keyStart[1:] + default: + // entire key range is included for inspection } } - - it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), start) + it := db.NewIterator(append(keyPrefix, r), s) defer it.Release() for it.Next() { @@ -472,86 +464,86 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { switch { case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): - headers.Add(size) + headers.add(size) case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): - bodies.Add(size) + bodies.add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): - receipts.Add(size) + receipts.add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): - tds.Add(size) + tds.add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): - numHashPairings.Add(size) + numHashPairings.add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): - hashNumPairings.Add(size) + hashNumPairings.add(size) case IsLegacyTrieNode(key, it.Value()): - legacyTries.Add(size) + legacyTries.add(size) case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: - stateLookups.Add(size) + stateLookups.add(size) case IsAccountTrieNode(key): - accountTries.Add(size) + accountTries.add(size) case IsStorageTrieNode(key): - storageTries.Add(size) + storageTries.add(size) case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: - codes.Add(size) + codes.add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): - txLookups.Add(size) + txLookups.add(size) case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): - accountSnaps.Add(size) + accountSnaps.add(size) case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): - storageSnaps.Add(size) + storageSnaps.add(size) case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): - preimages.Add(size) + preimages.add(size) case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): - metadata.Add(size) + metadata.add(size) case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): - metadata.Add(size) + metadata.add(size) case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): - beaconHeaders.Add(size) + beaconHeaders.add(size) case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: - cliqueSnaps.Add(size) + cliqueSnaps.add(size) // new log index case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9: - filterMapRows.Add(size) + filterMapRows.add(size) case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4: - filterMapLastBlock.Add(size) + filterMapLastBlock.add(size) case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8: - filterMapBlockLV.Add(size) + filterMapBlockLV.add(size) // old log index (deprecated) case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): - bloomBits.Add(size) + bloomBits.add(size) case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: - bloomBits.Add(size) + bloomBits.add(size) // Path-based historic state indexes case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: - stateIndex.Add(size) + stateIndex.add(size) // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): remain := key[len(VerklePrefix):] switch { case IsAccountTrieNode(remain): - verkleTries.Add(size) + verkleTries.add(size) case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength: - verkleStateLookups.Add(size) + verkleStateLookups.add(size) case bytes.Equal(remain, persistentStateIDKey): - metadata.Add(size) + metadata.add(size) case bytes.Equal(remain, trieJournalKey): - metadata.Add(size) + metadata.add(size) case bytes.Equal(remain, snapSyncStatusFlagKey): - metadata.Add(size) + metadata.add(size) default: - unaccounted.Add(size) + unaccounted.add(size) } // Metadata keys case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }): - metadata.Add(size) + metadata.add(size) default: - unaccounted.Add(size) + unaccounted.add(size) if len(key) >= 2 { prefix := [2]byte(key[:2]) unaccountedMu.Lock() @@ -576,10 +568,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { eg, ctx = errgroup.WithContext(context.Background()) workers = runtime.NumCPU() ) + ctx.Done() eg.SetLimit(workers) - log.Info("Starting parallel database inspection", "workers", workers) - + // Progress reporter done := make(chan struct{}) go func() { ticker := time.NewTicker(8 * time.Second) @@ -588,7 +580,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { for { select { case <-ticker.C: - log.Info("Inspecting database progress", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Inspecting database", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) case <-done: return } @@ -597,8 +589,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Inspect key-value database in parallel. for i := 0; i < 256; i++ { - rangePrefix := []byte{byte(i)} - eg.Go(func() error { return processRange(ctx, rangePrefix) }) + eg.Go(func() error { return inspectRange(ctx, byte(i)) }) } if err := eg.Wait(); err != nil { @@ -609,31 +600,31 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Display the database statistic of key-value store. stats := [][]string{ - {"Key-Value store", "Headers", headers.Size(), headers.Count()}, - {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, - {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, - {"Key-Value store", "Difficulties (deprecated)", tds.Size(), tds.Count()}, - {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, - {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, - {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, - {"Key-Value store", "Log index filter-map rows", filterMapRows.Size(), filterMapRows.Count()}, - {"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.Size(), filterMapLastBlock.Count()}, - {"Key-Value store", "Log index block-lv", filterMapBlockLV.Size(), filterMapBlockLV.Count()}, - {"Key-Value store", "Log bloombits (deprecated)", bloomBits.Size(), bloomBits.Count()}, - {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, - {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, - {"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, - {"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()}, - {"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()}, - {"Key-Value store", "Path state history indexes", stateIndex.Size(), stateIndex.Count()}, - {"Key-Value store", "Verkle trie nodes", verkleTries.Size(), verkleTries.Count()}, - {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.Size(), verkleStateLookups.Count()}, - {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, - {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, - {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, - {"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, - {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, - {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, + {"Key-Value store", "Headers", headers.sizeString(), headers.countString()}, + {"Key-Value store", "Bodies", bodies.sizeString(), bodies.countString()}, + {"Key-Value store", "Receipt lists", receipts.sizeString(), receipts.countString()}, + {"Key-Value store", "Difficulties (deprecated)", tds.sizeString(), tds.countString()}, + {"Key-Value store", "Block number->hash", numHashPairings.sizeString(), numHashPairings.countString()}, + {"Key-Value store", "Block hash->number", hashNumPairings.sizeString(), hashNumPairings.countString()}, + {"Key-Value store", "Transaction index", txLookups.sizeString(), txLookups.countString()}, + {"Key-Value store", "Log index filter-map rows", filterMapRows.sizeString(), filterMapRows.countString()}, + {"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.sizeString(), filterMapLastBlock.countString()}, + {"Key-Value store", "Log index block-lv", filterMapBlockLV.sizeString(), filterMapBlockLV.countString()}, + {"Key-Value store", "Log bloombits (deprecated)", bloomBits.sizeString(), bloomBits.countString()}, + {"Key-Value store", "Contract codes", codes.sizeString(), codes.countString()}, + {"Key-Value store", "Hash trie nodes", legacyTries.sizeString(), legacyTries.countString()}, + {"Key-Value store", "Path trie state lookups", stateLookups.sizeString(), stateLookups.countString()}, + {"Key-Value store", "Path trie account nodes", accountTries.sizeString(), accountTries.countString()}, + {"Key-Value store", "Path trie storage nodes", storageTries.sizeString(), storageTries.countString()}, + {"Key-Value store", "Path state history indexes", stateIndex.sizeString(), stateIndex.countString()}, + {"Key-Value store", "Verkle trie nodes", verkleTries.sizeString(), verkleTries.countString()}, + {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.sizeString(), verkleStateLookups.countString()}, + {"Key-Value store", "Trie preimages", preimages.sizeString(), preimages.countString()}, + {"Key-Value store", "Account snapshot", accountSnaps.sizeString(), accountSnaps.countString()}, + {"Key-Value store", "Storage snapshot", storageSnaps.sizeString(), storageSnaps.countString()}, + {"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()}, + {"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()}, + {"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()}, } // Inspect all registered append-only file store then. @@ -659,13 +650,12 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { table.AppendBulk(stats) table.Render() - if unaccounted.GetSize() > 0 { - log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount()) + if !unaccounted.empty() { + log.Error("Database contains unaccounted data", "size", unaccounted.sizeString(), "count", unaccounted.countString()) for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) { log.Error(fmt.Sprintf(" example key: %x", e)) } } - log.Info("Database inspection completed", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) return nil } From 9aa6aaa1dfd18128a50d3356757cd7d5d1e0dabd Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 1 Sep 2025 11:20:52 +0800 Subject: [PATCH 13/13] core/rawdb: remove typo --- core/rawdb/database.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 5a6a8a377334..626d390c0d35 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -568,7 +568,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { eg, ctx = errgroup.WithContext(context.Background()) workers = runtime.NumCPU() ) - ctx.Done() eg.SetLimit(workers) // Progress reporter