diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 838c084359ac..1279ff0acc78 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "strings" "time" "github.com/ethereum/go-ethereum/common" @@ -104,9 +105,9 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold // storage. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) { +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezerStr string, namespace string) (ethdb.Database, error) { // Create the idle freezer instance - frdb, err := newFreezer(freezer, namespace) + frdb, err := newFreezer(freezerStr, namespace) if err != nil { return nil, err } @@ -128,49 +129,39 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // upgrading to the freezer release, or we might have had a small chain and // not frozen anything yet. Ensure that no blocks are missing yet from the // key-value store, since that would mean we already had an old freezer. + validateErr := validateFreezerVsKV(frdb, db) + if validateErr != nil { - // If the genesis hash is empty, we have a new key-value store, so nothing to - // validate in this method. If, however, the genesis hash is not nil, compare - // it to the freezer content. - if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { - if frozen, _ := frdb.Ancients(); frozen > 0 { - // If the freezer already contains something, ensure that the genesis blocks - // match, otherwise we might mix up freezers across chains and destroy both - // the freezer and the key-value store. - if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) { - return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) - } - // Key-value store and freezer belong to the same network. Ensure that they - // are contiguous, otherwise we might end up with a non-functional freezer. - if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { - // Subsequent header after the freezer limit is missing from the database. - // Reject startup is the database has a more recent head. - if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { - return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) - } - // Database contains only older data than the freezer, this happens if the - // state was wiped and reinited from an existing freezer. - } - // Otherwise, key-value store continues where the freezer left off, all is fine. - // We might have duplicate blocks (crash after freezer write but before key-value - // store deletion, but that's fine). - } else { - // If the freezer is empty, ensure nothing was moved yet from the key-value - // store, otherwise we'll end up missing data. We check block #1 to decide - // if we froze anything previously or not, but do take care of databases with - // only the genesis block. - if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { - // Key-value store contains more data than the genesis block, make sure we - // didn't freeze anything yet. - if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { - return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") - } - // Block #1 is still in the database, we're allowed to init a new feezer - } - // Otherwise, the head header is still the genesis, we're allowed to init a new - // feezer. + log.Warn("Freezer/KV validation error, attempting freezer repair", "error", validateErr) + if reperr := frdb.repair(); reperr != nil { + log.Warn("Freezer repair errored", "error", reperr) + + // Repair did error, AND the validation errored, so return both together because that's double bad. + return nil, fmt.Errorf("freezer/kv error=%v freezer repair error=%v", validateErr, reperr) } + + log.Warn("Freezer repair OK") + + truncateKVtoFreezer(frdb, db) + + // Re-validate the ancient/kv dbs. + // If still a gap, try removing the kv data back to the ancient level. + validateErr = validateFreezerVsKV(frdb, db) + } + + if validateErr != nil && strings.Contains(validateErr.Error(), "gap") { + // Re-validate again. + validateErr = validateFreezerVsKV(frdb, db) + } else if validateErr != nil { + return nil, validateErr + } + + if validateErr != nil { + // If this fails, there's nothing left for us to do. + log.Warn("KV truncation failed to resuscitate Freezer/KV db gap.") + return nil, validateErr } + // Freezer is consistent with the key-value database, permit combining the two go frdb.freeze(db) @@ -352,3 +343,108 @@ func InspectDatabase(db ethdb.Database) error { } return nil } + +func validateFreezerVsKV(freezerdb *freezer, db ethdb.KeyValueStore) error { + // If the genesis hash is empty, we have a new key-value store, so nothing to + // validate in this method. If, however, the genesis hash is not nil, compare + // it to the freezer content. + if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { + if frozen, _ := freezerdb.Ancients(); frozen > 0 { + // If the freezer already contains something, ensure that the genesis blocks + // match, otherwise we might mix up freezers across chains and destroy both + // the freezer and the key-value store. + if frgenesis, _ := freezerdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) { + return fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) + } + // Key-value store and freezer belong to the same network. Ensure that they + // are contiguous, otherwise we might end up with a non-functional freezer. + if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { + // Subsequent header after the freezer limit is missing from the database. + // Reject startup is the database has a more recent head. + if headHeaderN := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); headHeaderN > frozen-1 { + return fmt.Errorf("gap (chaindb=#%d frozen=#%d) in the chain between ancients and leveldb", headHeaderN, frozen) + } + // Database contains only older data than the freezer, this happens if the + // state was wiped and reinited from an existing freezer. + } + // Otherwise, key-value store continues where the freezer left off, all is fine. + // We might have duplicate blocks (crash after freezer write but before key-value + // store deletion, but that's fine). + } else { + // If the freezer is empty, ensure nothing was moved yet from the key-value + // store, otherwise we'll end up missing data. We check block #1 to decide + // if we froze anything previously or not, but do take care of databases with + // only the genesis block. + if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { + // Key-value store contains more data than the genesis block, make sure we + // didn't freeze anything yet. + if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { + return errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") + } + // Block #1 is still in the database, we're allowed to init a new freezer. + } + // Otherwise, the head header is still the genesis, we're allowed to init a new + // freezer. + } + } + return nil +} + +func truncateKVtoFreezer(freezerdb *freezer, db ethdb.KeyValueStore) { + hhh := ReadHeadHeaderHash(db) + n := *ReadHeaderNumber(db, hhh) + frozen, _ := freezerdb.Ancients() + + normalize := func(n *uint64) uint64 { + if n == nil { + return 0 + } + return *n + } + + headHeaderHash := ReadHeadHeaderHash(db) + headHeader := normalize(ReadHeaderNumber(db, headHeaderHash)) + + headFastHash := ReadHeadFastBlockHash(db) + headFast := normalize(ReadHeaderNumber(db, headFastHash)) + + headFullHash := ReadHeadBlockHash(db) + headFull := normalize(ReadHeaderNumber(db, headFullHash)) + + log.Warn("Head header", "number", headHeader, "hash", headHeaderHash) + log.Warn("Head fast", "number", headFast, "hash", headFastHash) + log.Warn("Head full", "number", headFull, "hash", headFullHash) + + log.Warn("Persistent Freezer/KV gap: Truncating KV database to freezer height", "ancients", frozen, "kv.head_header_number", n, "kv.head_header_hash", hhh) + + for ; n > frozen-1 && n != 0; n-- { + for _, hash := range ReadAllHashes(db, n) { + if n%10000 == 0 { + log.Warn("Removing KV block data", "n", n, "hash", hash.String()) + } + DeleteBlock(db, hash, n) + DeleteCanonicalHash(db, n) + } + } + log.Warn("Finished KV truncation") + + data, _ := freezerdb.Ancient(freezerHashTable, n) + h := common.BytesToHash(data) + + // If h is the empty common hash, then when the headHeaderHash gets read, whoever's reading it isn't going to like that. + // This logic doesn't check for that because there's really nothing that can be sensibly done in this scope, + // and it seems reasonable to think that when a higher level function like `loadLastState` finds an empty hash in the + // headHeaderHash value, it's going to bark pretty loudly and probably just roll the whole thing (database(s)) back since the + // ancient database would appear to be screwy beyond repair since it lied about what frozen headers it had. + // So we're just gonna write this sucker. + log.Warn("Writing KV head header", "hash", h.String()) + WriteHeadHeaderHash(db, h) + + // If we had nonzero values for full and/or fast blocks, infer that preceding states will still be valid. + if headFast != 0 { + WriteHeadFastBlockHash(db, h) + } + if headFull != 0 { + WriteHeadBlockHash(db, h) + } +}