Skip to content

core/rawdb: rewind KV to Ancient in case chain gap detected #20745

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
meowsbits wants to merge 1 commit into from
+138 −42
Closed

core/rawdb: rewind KV to Ancient in case chain gap detected #20745

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 138 additions & 42 deletions core/rawdb/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"os"
"strings"
"time"

"github.com/ethereum/go-ethereum/common"
Expand Down Expand Up @@ -104,9 +105,9 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database {
// NewDatabaseWithFreezer creates a high level database on top of a given key-
// value data store with a freezer moving immutable chain segments into cold
// storage.
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) {
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezerStr string, namespace string) (ethdb.Database, error) {
// Create the idle freezer instance
frdb, err := newFreezer(freezer, namespace)
frdb, err := newFreezer(freezerStr, namespace)
if err != nil {
return nil, err
}
Expand All @@ -128,49 +129,39 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
// upgrading to the freezer release, or we might have had a small chain and
// not frozen anything yet. Ensure that no blocks are missing yet from the
// key-value store, since that would mean we already had an old freezer.
validateErr := validateFreezerVsKV(frdb, db)
if validateErr != nil {

// If the genesis hash is empty, we have a new key-value store, so nothing to
// validate in this method. If, however, the genesis hash is not nil, compare
// it to the freezer content.
if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
if frozen, _ := frdb.Ancients(); frozen > 0 {
// If the freezer already contains something, ensure that the genesis blocks
// match, otherwise we might mix up freezers across chains and destroy both
// the freezer and the key-value store.
if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) {
return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
}
// Key-value store and freezer belong to the same network. Ensure that they
// are contiguous, otherwise we might end up with a non-functional freezer.
if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
// Subsequent header after the freezer limit is missing from the database.
// Reject startup is the database has a more recent head.
if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 {
return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen)
}
// Database contains only older data than the freezer, this happens if the
// state was wiped and reinited from an existing freezer.
}
// Otherwise, key-value store continues where the freezer left off, all is fine.
// We might have duplicate blocks (crash after freezer write but before key-value
// store deletion, but that's fine).
} else {
// If the freezer is empty, ensure nothing was moved yet from the key-value
// store, otherwise we'll end up missing data. We check block #1 to decide
// if we froze anything previously or not, but do take care of databases with
// only the genesis block.
if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
// Key-value store contains more data than the genesis block, make sure we
// didn't freeze anything yet.
if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
}
// Block #1 is still in the database, we're allowed to init a new feezer
}
// Otherwise, the head header is still the genesis, we're allowed to init a new
// feezer.
log.Warn("Freezer/KV validation error, attempting freezer repair", "error", validateErr)
if reperr := frdb.repair(); reperr != nil {
log.Warn("Freezer repair errored", "error", reperr)

// Repair did error, AND the validation errored, so return both together because that's double bad.
return nil, fmt.Errorf("freezer/kv error=%v freezer repair error=%v", validateErr, reperr)
}

log.Warn("Freezer repair OK")

truncateKVtoFreezer(frdb, db)

// Re-validate the ancient/kv dbs.
// If still a gap, try removing the kv data back to the ancient level.
validateErr = validateFreezerVsKV(frdb, db)
}

if validateErr != nil && strings.Contains(validateErr.Error(), "gap") {
// Re-validate again.
validateErr = validateFreezerVsKV(frdb, db)
} else if validateErr != nil {
return nil, validateErr
}

if validateErr != nil {
// If this fails, there's nothing left for us to do.
log.Warn("KV truncation failed to resuscitate Freezer/KV db gap.")
return nil, validateErr
}

// Freezer is consistent with the key-value database, permit combining the two
go frdb.freeze(db)

Expand Down Expand Up @@ -352,3 +343,108 @@ func InspectDatabase(db ethdb.Database) error {
}
return nil
}

func validateFreezerVsKV(freezerdb *freezer, db ethdb.KeyValueStore) error {
// If the genesis hash is empty, we have a new key-value store, so nothing to
// validate in this method. If, however, the genesis hash is not nil, compare
// it to the freezer content.
if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
if frozen, _ := freezerdb.Ancients(); frozen > 0 {
// If the freezer already contains something, ensure that the genesis blocks
// match, otherwise we might mix up freezers across chains and destroy both
// the freezer and the key-value store.
if frgenesis, _ := freezerdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) {
return fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
}
// Key-value store and freezer belong to the same network. Ensure that they
// are contiguous, otherwise we might end up with a non-functional freezer.
if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
// Subsequent header after the freezer limit is missing from the database.
// Reject startup is the database has a more recent head.
if headHeaderN := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); headHeaderN > frozen-1 {
return fmt.Errorf("gap (chaindb=#%d frozen=#%d) in the chain between ancients and leveldb", headHeaderN, frozen)
}
// Database contains only older data than the freezer, this happens if the
// state was wiped and reinited from an existing freezer.
}
// Otherwise, key-value store continues where the freezer left off, all is fine.
// We might have duplicate blocks (crash after freezer write but before key-value
// store deletion, but that's fine).
} else {
// If the freezer is empty, ensure nothing was moved yet from the key-value
// store, otherwise we'll end up missing data. We check block #1 to decide
// if we froze anything previously or not, but do take care of databases with
// only the genesis block.
if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
// Key-value store contains more data than the genesis block, make sure we
// didn't freeze anything yet.
if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
return errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
}
// Block #1 is still in the database, we're allowed to init a new freezer.
}
// Otherwise, the head header is still the genesis, we're allowed to init a new
// freezer.
}
}
return nil
}

func truncateKVtoFreezer(freezerdb *freezer, db ethdb.KeyValueStore) {
hhh := ReadHeadHeaderHash(db)
n := *ReadHeaderNumber(db, hhh)
frozen, _ := freezerdb.Ancients()

normalize := func(n *uint64) uint64 {
if n == nil {
return 0
}
return *n
}

headHeaderHash := ReadHeadHeaderHash(db)
headHeader := normalize(ReadHeaderNumber(db, headHeaderHash))

headFastHash := ReadHeadFastBlockHash(db)
headFast := normalize(ReadHeaderNumber(db, headFastHash))

headFullHash := ReadHeadBlockHash(db)
headFull := normalize(ReadHeaderNumber(db, headFullHash))

log.Warn("Head header", "number", headHeader, "hash", headHeaderHash)
log.Warn("Head fast", "number", headFast, "hash", headFastHash)
log.Warn("Head full", "number", headFull, "hash", headFullHash)

log.Warn("Persistent Freezer/KV gap: Truncating KV database to freezer height", "ancients", frozen, "kv.head_header_number", n, "kv.head_header_hash", hhh)

for ; n > frozen-1 && n != 0; n-- {
for _, hash := range ReadAllHashes(db, n) {
if n%10000 == 0 {
log.Warn("Removing KV block data", "n", n, "hash", hash.String())
}
DeleteBlock(db, hash, n)
DeleteCanonicalHash(db, n)
}
}
log.Warn("Finished KV truncation")

data, _ := freezerdb.Ancient(freezerHashTable, n)
h := common.BytesToHash(data)

// If h is the empty common hash, then when the headHeaderHash gets read, whoever's reading it isn't going to like that.
// This logic doesn't check for that because there's really nothing that can be sensibly done in this scope,
// and it seems reasonable to think that when a higher level function like `loadLastState` finds an empty hash in the
// headHeaderHash value, it's going to bark pretty loudly and probably just roll the whole thing (database(s)) back since the
// ancient database would appear to be screwy beyond repair since it lied about what frozen headers it had.
// So we're just gonna write this sucker.
log.Warn("Writing KV head header", "hash", h.String())
WriteHeadHeaderHash(db, h)

// If we had nonzero values for full and/or fast blocks, infer that preceding states will still be valid.
if headFast != 0 {
WriteHeadFastBlockHash(db, h)
}
if headFull != 0 {
WriteHeadBlockHash(db, h)
}
}