diff --git a/src/Makefile.am b/src/Makefile.am index 2f3472e904df..fe3589195ac8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -312,6 +312,7 @@ BITCOIN_CORE_H = \ versionbits.h \ versionbitsinfo.h \ walletinitinterface.h \ + wallet/bdb.h \ wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ @@ -319,6 +320,7 @@ BITCOIN_CORE_H = \ wallet/load.h \ wallet/psbtwallet.h \ wallet/rpcwallet.h \ + wallet/salvage.h \ wallet/wallet.h \ wallet/walletdb.h \ wallet/wallettool.h \ @@ -469,6 +471,7 @@ libdash_wallet_a_SOURCES = \ coinjoin/options.cpp \ coinjoin/util.cpp \ interfaces/wallet.cpp \ + wallet/bdb.cpp \ wallet/coincontrol.cpp \ wallet/crypter.cpp \ wallet/db.cpp \ @@ -477,6 +480,7 @@ libdash_wallet_a_SOURCES = \ wallet/psbtwallet.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ + wallet/salvage.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ wallet/walletutil.cpp \ diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 43d6b42fbed2..f787e9e2c814 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -29,7 +29,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector> wtxs; LOCK(wallet.cs_wallet); @@ -60,7 +60,7 @@ static void CoinSelection(benchmark::Bench& bench) typedef std::set CoinSet; static auto testChain = interfaces::MakeChain(); -static const CWallet testWallet(*testChain, WalletLocation(), WalletDatabase::CreateDummy()); +static const CWallet testWallet(*testChain, WalletLocation(), CreateDummyWalletDatabase()); std::vector> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index edb790f735d8..fa04cb86919a 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -15,7 +15,7 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b const auto& ADDRESS_WATCHONLY = ADDRESS_B58T_UNSPENDABLE; std::unique_ptr chain = interfaces::MakeChain(); - CWallet wallet{*chain.get(), WalletLocation(), WalletDatabase::CreateMock()}; + CWallet wallet{*chain.get(), WalletLocation(), CreateMockWalletDatabase()}; { bool first_run; if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); diff --git a/src/dash-wallet.cpp b/src/dash-wallet.cpp index 7f94a32c5de7..e62c0623a42f 100644 --- a/src/dash-wallet.cpp +++ b/src/dash-wallet.cpp @@ -31,6 +31,7 @@ static void SetupWalletToolArgs() // Hidden gArgs.AddArg("-h", "", ArgsManager::ALLOW_ANY, OptionsCategory::HIDDEN); gArgs.AddArg("-help", "", ArgsManager::ALLOW_ANY, OptionsCategory::HIDDEN); + gArgs.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); } static bool WalletAppInit(int argc, char* argv[]) diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 24cd41ee5379..af9718e562fb 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -147,39 +147,13 @@ - - - Using BerkeleyDB version - - - 10 - - - - - - - IBeamCursor - - - N/A - - - Qt::PlainText - - - Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse - - - - Datadir - + IBeamCursor @@ -201,14 +175,14 @@ - + Blocksdir - + IBeamCursor @@ -230,14 +204,14 @@ - + Startup time - + IBeamCursor @@ -253,21 +227,21 @@ - + Network - + Name - + IBeamCursor @@ -283,14 +257,14 @@ - + Number of connections - + IBeamCursor @@ -306,14 +280,14 @@ - + Number of Masternodes - + IBeamCursor @@ -321,29 +295,26 @@ N/A - - Qt::PlainText - Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse - + Block chain - + Current block height - + IBeamCursor @@ -359,14 +330,14 @@ - + Last block time - + IBeamCursor @@ -382,14 +353,14 @@ - + Last block hash - + IBeamCursor @@ -405,14 +376,14 @@ - + Latest ChainLocked block hash - + IBeamCursor @@ -428,14 +399,14 @@ - + Latest ChainLocked block height - + IBeamCursor @@ -451,21 +422,21 @@ - + Memory Pool - + Current number of transactions - + IBeamCursor @@ -481,14 +452,14 @@ - + Memory usage - + IBeamCursor @@ -504,7 +475,7 @@ - + 3 @@ -544,14 +515,14 @@ - + InstantSend locks - + IBeamCursor @@ -567,7 +538,7 @@ - + Qt::Vertical @@ -1419,30 +1390,11 @@ - - - - 10 - 100 - 301 - 23 - - - - - 180 - 23 - - - - Salvage wallet - - 10 - 150 + 100 301 23 @@ -1461,7 +1413,7 @@ 10 - 200 + 150 301 23 @@ -1480,7 +1432,7 @@ 10 - 250 + 200 301 23 @@ -1499,7 +1451,7 @@ 10 - 300 + 250 301 23 @@ -1518,7 +1470,7 @@ 10 - 350 + 300 301 23 @@ -1549,27 +1501,11 @@ true - - - - 330 - 90 - 411 - 41 - - - - -salvagewallet: Attempt to recover private keys from a corrupt wallet.dat. - - - true - - 330 - 140 + 90 411 41 @@ -1585,7 +1521,7 @@ 330 - 190 + 140 411 41 @@ -1601,7 +1537,7 @@ 330 - 240 + 190 411 41 @@ -1617,7 +1553,7 @@ 330 - 290 + 240 411 41 @@ -1633,7 +1569,7 @@ 330 - 340 + 290 411 41 @@ -1665,7 +1601,7 @@ 10 - 400 + 350 301 23 @@ -1678,7 +1614,7 @@ 330 - 390 + 340 411 41 diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 54195c2ee1cf..a20783eaee5c 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -54,7 +54,6 @@ const char fontSizeSettingsKey[] = "consoleFontSize"; const TrafficGraphData::GraphRange INITIAL_TRAFFIC_GRAPH_SETTING = TrafficGraphData::Range_30m; // Repair parameters -const QString SALVAGEWALLET("-salvagewallet"); const QString RESCAN1("-rescan=1"); const QString RESCAN2("-rescan=2"); const QString ZAPTXES1("-zapwallettxes=1 -persistmempool=0"); @@ -491,9 +490,6 @@ RPCConsole::RPCConsole(interfaces::Node& node, QWidget* parent, Qt::WindowFlags ui->WalletSelectorLabel->setVisible(false); // Wallet Repair Buttons - // connect(ui->btn_salvagewallet, &QPushButton::clicked, this, &RPCConsole::walletSalvage); - // Disable salvage option in GUI, it's way too powerful and can lead to funds loss - ui->btn_salvagewallet->setEnabled(false); // Disable wallet repair options that require a wallet (enable them later when a wallet is added) ui->btn_rescan1->setEnabled(false); ui->btn_rescan2->setEnabled(false); @@ -507,16 +503,12 @@ RPCConsole::RPCConsole(interfaces::Node& node, QWidget* parent, Qt::WindowFlags connect(ui->btn_upgradewallet, &QPushButton::clicked, this, &RPCConsole::walletUpgrade); connect(ui->btn_reindex, &QPushButton::clicked, this, &RPCConsole::walletReindex); - // set library version labels #ifdef ENABLE_WALLET - ui->berkeleyDBVersion->setText(QString::fromStdString(BerkeleyDatabaseVersion())); std::string walletPath = GetDataDir().string(); walletPath += QDir::separator().toLatin1() + gArgs.GetArg("-wallet", "wallet.dat"); ui->wallet_path->setText(QString::fromStdString(walletPath)); -#else - ui->label_berkeleyDBVersion->hide(); - ui->berkeleyDBVersion->hide(); #endif + // Register RPC timer interface rpcTimerInterface = new QtRPCTimerInterface(); // avoid accidentally overwriting an existing, non QTThread @@ -810,12 +802,6 @@ void RPCConsole::setFontSize(int newSize) ui->messagesWidget->verticalScrollBar()->setValue(oldPosFactor * ui->messagesWidget->verticalScrollBar()->maximum()); } -/** Restart wallet with "-salvagewallet" */ -void RPCConsole::walletSalvage() -{ - buildParameterlist(SALVAGEWALLET); -} - /** Restart wallet with "-rescan=1" */ void RPCConsole::walletRescan1() { @@ -860,7 +846,6 @@ void RPCConsole::buildParameterlist(QString arg) args.removeFirst(); // Remove existing repair-options - args.removeAll(SALVAGEWALLET); args.removeAll(RESCAN1); args.removeAll(RESCAN2); args.removeAll(ZAPTXES1); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index ccc4e9128ed2..4fbfc328a2c9 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -104,7 +104,6 @@ public Q_SLOTS: void setFontSize(int newSize); /** Wallet repair options */ - void walletSalvage(); void walletRescan1(); void walletRescan2(); void walletZaptxes1(); diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 6a9a28c9f197..a4c513f9bf77 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -54,7 +54,7 @@ void TestAddAddressesToSendBook() { TestChain100Setup test; auto chain = interfaces::MakeChain(); - std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index fc73cf164d54..24b230fb10a9 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -105,7 +105,7 @@ void TestGUI() test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); } auto chain = interfaces::MakeChain(); - std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), CreateMockWalletDatabase()); AddWallet(wallet); bool firstRun; wallet->LoadWallet(firstRun); diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp new file mode 100644 index 000000000000..712c931ea77e --- /dev/null +++ b/src/wallet/bdb.cpp @@ -0,0 +1,858 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +#include + +#include + +#ifndef WIN32 +#include +#endif + +namespace { + +//! Make sure database has a unique fileid within the environment. If it +//! doesn't, throw an error. BDB caches do not work properly when more than one +//! open database has the same fileid (values written to one database may show +//! up in reads to other databases). +//! +//! BerkeleyDB generates unique fileids by default +//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), +//! so bitcoin should never create different databases with the same fileid, but +//! this error can be triggered if users manually copy database files. +void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid) +{ + if (env.IsMock()) return; + + int ret = db.get_mpf()->get_fileid(fileid.value); + if (ret != 0) { + throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret)); + } + + for (const auto& item : env.m_fileids) { + if (fileid == item.second && &fileid != &item.second) { + throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (duplicates fileid %s from %s)", filename, + HexStr(item.second.value), item.first)); + } + } +} + +CCriticalSection cs_db; +std::map> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. +} // namespace + +bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const +{ + return memcmp(value, &rhs.value, sizeof(value)) == 0; +} + +bool IsBDBWalletLoaded(const fs::path& wallet_path) +{ + fs::path env_directory; + std::string database_filename; + SplitWalletPath(wallet_path, env_directory, database_filename); + LOCK(cs_db); + auto env = g_dbenvs.find(env_directory.string()); + if (env == g_dbenvs.end()) return false; + auto database = env->second.lock(); + return database && database->IsDatabaseLoaded(database_filename); +} + +/** + * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. + * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. + * @return A shared pointer to the BerkeleyEnvironment object for the wallet directory, never empty because ~BerkeleyEnvironment + * erases the weak pointer from the g_dbenvs map. + * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map. + */ +std::shared_ptr GetWalletEnv(const fs::path& wallet_path, std::string& database_filename) +{ + fs::path env_directory; + SplitWalletPath(wallet_path, env_directory, database_filename); + LOCK(cs_db); + auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr()); + if (inserted.second) { + auto env = std::make_shared(env_directory.string()); + inserted.first->second = env; + return env; + } + return inserted.first->second.lock(); +} + +// +// BerkeleyBatch +// + +void BerkeleyEnvironment::Close() +{ + if (!fDbEnvInit) + return; + + fDbEnvInit = false; + + for (auto& db : m_databases) { + auto count = mapFileUseCount.find(db.first); + assert(count == mapFileUseCount.end() || count->second == 0); + BerkeleyDatabase& database = db.second.get(); + if (database.m_db) { + database.m_db->close(0); + database.m_db.reset(); + } + } + + FILE* error_file = nullptr; + dbenv->get_errfile(&error_file); + + int ret = dbenv->close(0); + if (ret != 0) + LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret)); + if (!fMockDb) + DbEnv((u_int32_t)0).remove(strPath.c_str(), 0); + + if (error_file) fclose(error_file); + + UnlockDirectory(strPath, ".walletlock"); +} + +void BerkeleyEnvironment::Reset() +{ + dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); + fDbEnvInit = false; + fMockDb = false; +} + +BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string()) +{ + Reset(); +} + +BerkeleyEnvironment::~BerkeleyEnvironment() +{ + LOCK(cs_db); + g_dbenvs.erase(strPath); + Close(); +} + +bool BerkeleyEnvironment::Open(bool retry) +{ + if (fDbEnvInit) { + return true; + } + + fs::path pathIn = strPath; + TryCreateDirectories(pathIn); + if (!LockDirectory(pathIn, ".walletlock")) { + LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); + return false; + } + + fs::path pathLogDir = pathIn / "database"; + TryCreateDirectories(pathLogDir); + fs::path pathErrorFile = pathIn / "db.log"; + LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); + + unsigned int nEnvFlags = 0; + if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) + nEnvFlags |= DB_PRIVATE; + + dbenv->set_lg_dir(pathLogDir.string().c_str()); + dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet + dbenv->set_lg_bsize(0x10000); + dbenv->set_lg_max(1048576); + dbenv->set_lk_max_locks(40000); + dbenv->set_lk_max_objects(40000); + dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug + dbenv->set_flags(DB_AUTO_COMMIT, 1); + dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); + dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); + int ret = dbenv->open(strPath.c_str(), + DB_CREATE | + DB_INIT_LOCK | + DB_INIT_LOG | + DB_INIT_MPOOL | + DB_INIT_TXN | + DB_THREAD | + DB_RECOVER | + nEnvFlags, + S_IRUSR | S_IWUSR); + if (ret != 0) { + LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret)); + int ret2 = dbenv->close(0); + if (ret2 != 0) { + LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2)); + } + Reset(); + if (retry) { + // try moving the database env out of the way + fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime()); + try { + fs::rename(pathLogDir, pathDatabaseBak); + LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string()); + } catch (const fs::filesystem_error&) { + // failure is ok (well, not really, but it's not worse than what we started with) + } + // try opening it again one more time + if (!Open(false /* retry */)) { + // if it still fails, it probably means we can't even create the database env + return false; + } + } else { + return false; + } + } + + fDbEnvInit = true; + fMockDb = false; + return true; +} + +//! Construct an in-memory mock Berkeley environment for testing +BerkeleyEnvironment::BerkeleyEnvironment() +{ + Reset(); + + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); + + dbenv->set_cachesize(1, 0, 1); + dbenv->set_lg_bsize(10485760 * 4); + dbenv->set_lg_max(10485760); + dbenv->set_lk_max_locks(10000); + dbenv->set_lk_max_objects(10000); + dbenv->set_flags(DB_AUTO_COMMIT, 1); + dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); + int ret = dbenv->open(nullptr, + DB_CREATE | + DB_INIT_LOCK | + DB_INIT_LOG | + DB_INIT_MPOOL | + DB_INIT_TXN | + DB_THREAD | + DB_PRIVATE, + S_IRUSR | S_IWUSR); + if (ret > 0) { + throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret)); + } + + fDbEnvInit = true; + fMockDb = true; +} + +bool BerkeleyEnvironment::Verify(const std::string& strFile) +{ + LOCK(cs_db); + assert(mapFileUseCount.count(strFile) == 0); + + Db db(dbenv.get(), 0); + int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); + return result == 0; +} + +BerkeleyBatch::SafeDbt::SafeDbt() +{ + m_dbt.set_flags(DB_DBT_MALLOC); +} + +BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size) + : m_dbt(data, size) +{ +} + +BerkeleyBatch::SafeDbt::~SafeDbt() +{ + if (m_dbt.get_data() != nullptr) { + // Clear memory, e.g. in case it was a private key + memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); + // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be + // freed by the caller. + // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html + if (m_dbt.get_flags() & DB_DBT_MALLOC) { + free(m_dbt.get_data()); + } + } +} + +const void* BerkeleyBatch::SafeDbt::get_data() const +{ + return m_dbt.get_data(); +} + +u_int32_t BerkeleyBatch::SafeDbt::get_size() const +{ + return m_dbt.get_size(); +} + +BerkeleyBatch::SafeDbt::operator Dbt*() +{ + return &m_dbt; +} + +bool BerkeleyDatabase::Verify(std::string& errorStr) +{ + fs::path walletDir = env->Directory(); + fs::path file_path = walletDir / strFile; + + LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); + LogPrintf("Using wallet %s\n", file_path.string()); + + if (!env->Open(true /* retry */)) { + errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); + return false; + } + + if (fs::exists(file_path)) + { + if (!env->Verify(strFile)) { + errorStr = strprintf(_("%s corrupt. Try using the wallet tool dash-wallet to salvage or restoring a backup."), file_path); + return false; + } + } + // also return true if files does not exists + return true; +} + +void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) +{ + dbenv->txn_checkpoint(0, 0, 0); + if (fMockDb) + return; + dbenv->lsn_reset(strFile.c_str(), 0); +} + +BerkeleyDatabase::~BerkeleyDatabase() +{ + if (env) { + LOCK(cs_db); + size_t erased = env->m_databases.erase(strFile); + assert(erased == 1); + env->m_fileids.erase(strFile); + } +} + +BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr), m_cursor(nullptr), m_database(database) +{ + fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); + fFlushOnClose = fFlushOnCloseIn; + env = database.env.get(); + const std::string &strFilename = database.strFile; + + bool fCreate = strchr(pszMode, 'c') != nullptr; + unsigned int nFlags = DB_THREAD; + if (fCreate) + nFlags |= DB_CREATE; + + { + LOCK(cs_db); + if (!env->Open(false /* retry */)) + throw std::runtime_error("BerkeleyBatch: Failed to open database environment."); + + pdb = database.m_db.get(); + if (pdb == nullptr) { + int ret; + std::unique_ptr pdb_temp = MakeUnique(env->dbenv.get(), 0); + + bool fMockDb = env->IsMock(); + if (fMockDb) { + DbMpoolFile* mpf = pdb_temp->get_mpf(); + ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); + if (ret != 0) { + throw std::runtime_error(strprintf("BerkeleyBatch: Failed to configure for no temp file backing for database %s", strFilename)); + } + } + + ret = pdb_temp->open(nullptr, // Txn pointer + fMockDb ? nullptr : strFilename.c_str(), // Filename + fMockDb ? strFilename.c_str() : "main", // Logical db name + DB_BTREE, // Database type + nFlags, // Flags + 0); + + if (ret != 0) { + throw std::runtime_error(strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename)); + } + + // Call CheckUniqueFileid on the containing BDB environment to + // avoid BDB data consistency bugs that happen when different data + // files in the same environment have the same fileid. + // + // Also call CheckUniqueFileid on all the other g_dbenvs to prevent + // bitcoin from opening the same data file through another + // environment when the file is referenced through equivalent but + // not obviously identical symlinked or hard linked or bind mounted + // paths. In the future a more relaxed check for equal inode and + // device ids could be done instead, which would allow opening + // different backup copies of a wallet at the same time. Maybe even + // more ideally, an exclusive lock for accessing the database could + // be implemented, so no equality checks are needed at all. (Newer + // versions of BDB have an set_lk_exclusive method for this + // purpose, but the older version we use does not.) + for (const auto& env : g_dbenvs) { + CheckUniqueFileid(*env.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]); + } + + pdb = pdb_temp.release(); + database.m_db.reset(pdb); + + if (fCreate && !Exists(std::string("version"))) { + bool fTmp = fReadOnly; + fReadOnly = false; + Write(std::string("version"), CLIENT_VERSION); + fReadOnly = fTmp; + } + } + database.AddRef(); + strFile = strFilename; + } +} + +void BerkeleyDatabase::Open(const char* mode) +{ + throw std::logic_error("BerkeleyDatabase does not implement Open. This function should not be called."); +} + +void BerkeleyBatch::Flush() +{ + if (activeTxn) + return; + + // Flush database activity from memory pool to disk log + unsigned int nMinutes = 0; + if (fReadOnly) + nMinutes = 1; + + if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault + env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + } +} + +void BerkeleyDatabase::IncrementUpdateCounter() +{ + ++nUpdateCounter; +} + +void BerkeleyBatch::Close() +{ + if (!pdb) + return; + if (activeTxn) + activeTxn->abort(); + activeTxn = nullptr; + pdb = nullptr; + CloseCursor(); + + if (fFlushOnClose) + Flush(); + + m_database.RemoveRef(); +} + +void BerkeleyEnvironment::CloseDb(const std::string& strFile) +{ + { + LOCK(cs_db); + auto it = m_databases.find(strFile); + assert(it != m_databases.end()); + BerkeleyDatabase& database = it->second.get(); + if (database.m_db) { + // Close the database handle + database.m_db->close(0); + database.m_db.reset(); + } + } +} + +void BerkeleyEnvironment::ReloadDbEnv() +{ + // Make sure that no Db's are in use + AssertLockNotHeld(cs_db); + std::unique_lock lock(cs_db); + m_db_in_use.wait(lock, [this](){ + for (auto& count : mapFileUseCount) { + if (count.second > 0) return false; + } + return true; + }); + + std::vector filenames; + for (auto it : m_databases) { + filenames.push_back(it.first); + } + // Close the individual Db's + for (const std::string& filename : filenames) { + CloseDb(filename); + } + // Reset the environment + Flush(true); // This will flush and close the environment + Reset(); + Open(true); +} + +bool BerkeleyDatabase::Rewrite(const char* pszSkip) +{ + while (true) { + { + LOCK(cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); + + bool fSuccess = true; + LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); + std::string strFileRes = strFile + ".rewrite"; + { // surround usage of db with extra {} + BerkeleyBatch db(*this, "r"); + std::unique_ptr pdbCopy = MakeUnique(env->dbenv.get(), 0); + + int ret = pdbCopy->open(nullptr, // Txn pointer + strFileRes.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) { + LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes); + fSuccess = false; + } + + if (db.StartCursor()) { + while (fSuccess) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + bool complete; + bool ret1 = db.ReadAtCursor(ssKey, ssValue, complete); + if (complete) { + break; + } else if (!ret1) { + fSuccess = false; + break; + } + if (pszSkip && + strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) + continue; + if (strncmp(ssKey.data(), "\x07version", 8) == 0) { + // Update version: + ssValue.clear(); + ssValue << CLIENT_VERSION; + } + Dbt datKey(ssKey.data(), ssKey.size()); + Dbt datValue(ssValue.data(), ssValue.size()); + int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + db.CloseCursor(); + } + if (fSuccess) { + db.Close(); + env->CloseDb(strFile); + if (pdbCopy->close(0)) + fSuccess = false; + } else { + pdbCopy->close(0); + } + } + if (fSuccess) { + Db dbA(env->dbenv.get(), 0); + if (dbA.remove(strFile.c_str(), nullptr, 0)) + fSuccess = false; + Db dbB(env->dbenv.get(), 0); + if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) + fSuccess = false; + } + if (!fSuccess) + LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes); + return fSuccess; + } + } + UninterruptibleSleep(std::chrono::milliseconds{100}); + } +} + + +void BerkeleyEnvironment::Flush(bool fShutdown) +{ + int64_t nStart = GetTimeMillis(); + // Flush log data to the actual data file on all files that are not in use + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); + if (!fDbEnvInit) + return; + { + LOCK(cs_db); + std::map::iterator mi = mapFileUseCount.begin(); + while (mi != mapFileUseCount.end()) { + std::string strFile = (*mi).first; + int nRefCount = (*mi).second; + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); + if (nRefCount == 0) { + // Move log data to the dat file + CloseDb(strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); + dbenv->txn_checkpoint(0, 0, 0); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); + if (!fMockDb) + dbenv->lsn_reset(strFile.c_str(), 0); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); + mapFileUseCount.erase(mi++); + } else + mi++; + } + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); + if (fShutdown) { + char** listp; + if (mapFileUseCount.empty()) { + dbenv->log_archive(&listp, DB_ARCH_REMOVE); + Close(); + if (!fMockDb) { + fs::remove_all(fs::path(strPath) / "database"); + } + } + } + } +} + +bool BerkeleyDatabase::PeriodicFlush() +{ + bool ret = false; + TRY_LOCK(cs_db, lockDb); + if (lockDb) + { + // Don't do this if any databases are in use + int nRefCount = 0; + std::map::iterator mit = env->mapFileUseCount.begin(); + while (mit != env->mapFileUseCount.end()) + { + nRefCount += (*mit).second; + mit++; + } + + if (nRefCount == 0) + { + std::map::iterator mi = env->mapFileUseCount.find(strFile); + if (mi != env->mapFileUseCount.end()) + { + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); + int64_t nStart = GetTimeMillis(); + + // Flush wallet file so it's self contained + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + + env->mapFileUseCount.erase(mi++); + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + ret = true; + } + } + } + + return ret; +} + +bool BerkeleyDatabase::Backup(const std::string& strDest) const +{ + while (true) + { + { + LOCK(cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) + { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); + + // Copy wallet file + fs::path pathSrc = env->Directory() / strFile; + fs::path pathDest(strDest); + if (fs::is_directory(pathDest)) + pathDest /= strFile; + + try { + if (fs::equivalent(pathSrc, pathDest)) { + LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + return false; + } + + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); + LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); + return false; + } + } + } + UninterruptibleSleep(std::chrono::milliseconds{100}); + } +} + +void BerkeleyDatabase::Flush() +{ + env->Flush(false); +} + +void BerkeleyDatabase::Close() +{ + env->Flush(true); +} + +void BerkeleyDatabase::ReloadDbEnv() +{ + env->ReloadDbEnv(); +} + +bool BerkeleyBatch::StartCursor() +{ + assert(!m_cursor); + if (!pdb) + return false; + int ret = pdb->cursor(nullptr, &m_cursor, 0); + return ret == 0; +} + +bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) +{ + complete = false; + if (m_cursor == nullptr) return false; + // Read at cursor + SafeDbt datKey; + SafeDbt datValue; + int ret = m_cursor->get(datKey, datValue, DB_NEXT); + if (ret == DB_NOTFOUND) { + complete = true; + } + if (ret != 0) + return false; + else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) + return false; + + // Convert to streams + ssKey.SetType(SER_DISK); + ssKey.clear(); + ssKey.write((char*)datKey.get_data(), datKey.get_size()); + ssValue.SetType(SER_DISK); + ssValue.clear(); + ssValue.write((char*)datValue.get_data(), datValue.get_size()); + return true; +} + +void BerkeleyBatch::CloseCursor() +{ + if (!m_cursor) return; + m_cursor->close(); + m_cursor = nullptr; +} + +bool BerkeleyBatch::TxnBegin() +{ + if (!pdb || activeTxn) + return false; + DbTxn* ptxn = env->TxnBegin(); + if (!ptxn) + return false; + activeTxn = ptxn; + return true; +} + +bool BerkeleyBatch::TxnCommit() +{ + if (!pdb || !activeTxn) + return false; + int ret = activeTxn->commit(0); + activeTxn = nullptr; + return (ret == 0); +} + +bool BerkeleyBatch::TxnAbort() +{ + if (!pdb || !activeTxn) + return false; + int ret = activeTxn->abort(); + activeTxn = nullptr; + return (ret == 0); +} + +std::string BerkeleyDatabaseVersion() +{ + return DbEnv::version(nullptr, nullptr, nullptr); +} + +bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) +{ + if (!pdb) + return false; + + SafeDbt datKey(key.data(), key.size()); + + SafeDbt datValue; + int ret = pdb->get(activeTxn, datKey, datValue, 0); + if (ret == 0 && datValue.get_data() != nullptr) { + value.write((char*)datValue.get_data(), datValue.get_size()); + return true; + } + return false; +} + +bool BerkeleyBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) +{ + if (!pdb) + return false; + if (fReadOnly) + assert(!"Write called on database in read-only mode"); + + SafeDbt datKey(key.data(), key.size()); + + SafeDbt datValue(value.data(), value.size()); + + int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE)); + return (ret == 0); +} + +bool BerkeleyBatch::EraseKey(CDataStream&& key) +{ + if (!pdb) + return false; + if (fReadOnly) + assert(!"Erase called on database in read-only mode"); + + SafeDbt datKey(key.data(), key.size()); + + int ret = pdb->del(activeTxn, datKey, 0); + return (ret == 0 || ret == DB_NOTFOUND); +} + +bool BerkeleyBatch::HasKey(CDataStream&& key) +{ + if (!pdb) + return false; + + SafeDbt datKey(key.data(), key.size()); + + int ret = pdb->exists(activeTxn, datKey, 0); + return ret == 0; +} + +void BerkeleyDatabase::AddRef() +{ + LOCK(cs_db); + ++env->mapFileUseCount[strFile]; +} + +void BerkeleyDatabase::RemoveRef() +{ + { + LOCK(cs_db); + --env->mapFileUseCount[strFile]; + } + env->m_db_in_use.notify_all(); +} + +std::unique_ptr BerkeleyDatabase::MakeBatch(const char* mode, bool flush_on_close) +{ + return MakeUnique(*this, mode, flush_on_close); +} diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h new file mode 100644 index 000000000000..0b857190c8dc --- /dev/null +++ b/src/wallet/bdb.h @@ -0,0 +1,230 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_BDB_H +#define BITCOIN_WALLET_BDB_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif +#include +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + +static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; +static const bool DEFAULT_WALLET_PRIVDB = true; + +struct WalletDatabaseFileId { + u_int8_t value[DB_FILE_ID_LEN]; + bool operator==(const WalletDatabaseFileId& rhs) const; +}; + +class BerkeleyDatabase; + +class BerkeleyEnvironment +{ +private: + bool fDbEnvInit; + bool fMockDb; + // Don't change into fs::path, as that can result in + // shutdown problems/crashes caused by a static initialized internal pointer. + std::string strPath; + +public: + std::unique_ptr dbenv; + std::map mapFileUseCount; + std::map> m_databases; + std::unordered_map m_fileids; + std::condition_variable_any m_db_in_use; + + BerkeleyEnvironment(const fs::path& env_directory); + BerkeleyEnvironment(); + ~BerkeleyEnvironment(); + void Reset(); + + bool IsMock() const { return fMockDb; } + bool IsInitialized() const { return fDbEnvInit; } + bool IsDatabaseLoaded(const std::string& db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } + fs::path Directory() const { return strPath; } + + bool Verify(const std::string& strFile); + + bool Open(bool retry); + void Close(); + void Flush(bool fShutdown); + void CheckpointLSN(const std::string& strFile); + + void CloseDb(const std::string& strFile); + void ReloadDbEnv(); + + DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) + { + DbTxn* ptxn = nullptr; + int ret = dbenv->txn_begin(nullptr, &ptxn, flags); + if (!ptxn || ret != 0) + return nullptr; + return ptxn; + } +}; + +/** Get BerkeleyEnvironment and database filename given a wallet path. */ +std::shared_ptr GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); + +/** Return whether a wallet database is currently loaded. */ +bool IsBDBWalletLoaded(const fs::path& wallet_path); + +class BerkeleyBatch; + +/** An instance of this class represents one database. + * For BerkeleyDB this is just a (env, strFile) tuple. + **/ +class BerkeleyDatabase : public WalletDatabase +{ + friend class BerkeleyBatch; +public: + BerkeleyDatabase() = delete; + + /** Create DB handle to real database */ + BerkeleyDatabase(std::shared_ptr env, std::string filename) : + WalletDatabase(), env(std::move(env)), strFile(std::move(filename)) + { + auto inserted = this->env->m_databases.emplace(strFile, std::ref(*this)); + assert(inserted.second); + } + + ~BerkeleyDatabase() override; + + /** Open the database if it is not already opened. + * Dummy function, doesn't do anything right now, but is needed for class abstraction */ + void Open(const char* mode) override; + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip=nullptr) override; + + /** Indicate the a new database user has began using the database. */ + void AddRef() override; + /** Indicate that database user has stopped using the database and that it could be flushed or closed. */ + void RemoveRef() override; + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest) const override; + + /** Make sure all changes are flushed to database file. + */ + void Flush() override; + /** Flush to the database file and close the database. + * Also close the environment if no other databases are open in it. + */ + void Close() override; + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + bool PeriodicFlush() override; + + void IncrementUpdateCounter() override; + + void ReloadDbEnv() override; + + /** Verifies the environment and database file */ + bool Verify(std::string& error) override; + + /** + * Pointer to shared database environment. + * + * Normally there is only one BerkeleyDatabase object per + * BerkeleyEnvivonment, but in the special, backwards compatible case where + * multiple wallet BDB data files are loaded from the same directory, this + * will point to a shared instance that gets freed when the last data file + * is closed. + */ + std::shared_ptr env; + + /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ + std::unique_ptr m_db; + + /** Make a BerkeleyBatch connected to this database */ + std::unique_ptr MakeBatch(const char* mode = "r+", bool flush_on_close = true) override; + +private: + std::string strFile; +}; + +/** RAII class that provides access to a Berkeley database */ +class BerkeleyBatch : public DatabaseBatch +{ + /** RAII class that automatically cleanses its data on destruction */ + class SafeDbt final + { + Dbt m_dbt; + + public: + // construct Dbt with internally-managed data + SafeDbt(); + // construct Dbt with provided data + SafeDbt(void* data, size_t size); + ~SafeDbt(); + + // delegate to Dbt + const void* get_data() const; + u_int32_t get_size() const; + + // conversion operator to access the underlying Dbt + operator Dbt*(); + }; + +private: + bool ReadKey(CDataStream&& key, CDataStream& value) override; + bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true) override; + bool EraseKey(CDataStream&& key) override; + bool HasKey(CDataStream&& key) override; + +protected: + Db* pdb; + std::string strFile; + DbTxn* activeTxn; + Dbc* m_cursor; + bool fReadOnly; + bool fFlushOnClose; + BerkeleyEnvironment *env; + BerkeleyDatabase& m_database; + +public: + explicit BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode = "r+", bool fFlushOnCloseIn=true); + ~BerkeleyBatch() override { Close(); } + + BerkeleyBatch(const BerkeleyBatch&) = delete; + BerkeleyBatch& operator=(const BerkeleyBatch&) = delete; + + void Flush() override; + void Close() override; + + bool StartCursor() override; + bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) override; + void CloseCursor() override; + bool TxnBegin() override; + bool TxnCommit() override; + bool TxnAbort() override; +}; + +std::string BerkeleyDatabaseVersion(); + +#endif // BITCOIN_WALLET_BDB_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 4ac6b0778826..fb1a9fc418f2 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -3,54 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include -#include - -#include - -#ifndef WIN32 -#include -#endif - -namespace { - -//! Make sure database has a unique fileid within the environment. If it -//! doesn't, throw an error. BDB caches do not work properly when more than one -//! open database has the same fileid (values written to one database may show -//! up in reads to other databases). -//! -//! BerkeleyDB generates unique fileids by default -//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), -//! so bitcoin should never create different databases with the same fileid, but -//! this error can be triggered if users manually copy database files. -void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid) -{ - if (env.IsMock()) return; - - int ret = db.get_mpf()->get_fileid(fileid.value); - if (ret != 0) { - throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret)); - } - - for (const auto& item : env.m_fileids) { - if (fileid == item.second && &fileid != &item.second) { - throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (duplicates fileid %s from %s)", filename, - HexStr(item.second.value), item.first)); - } - } -} - -CCriticalSection cs_db; -std::map> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. -} // namespace - -bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const -{ - return memcmp(value, &rhs.value, sizeof(value)) == 0; -} - -static void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename) +void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename) { if (fs::is_regular_file(wallet_path)) { // Special case for backwards compatibility: if wallet path points to an @@ -66,18 +22,6 @@ static void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory } } -bool IsWalletLoaded(const fs::path& wallet_path) -{ - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - LOCK(cs_db); - auto env = g_dbenvs.find(env_directory.string()); - if (env == g_dbenvs.end()) return false; - auto database = env->second.lock(); - return database && database->IsDatabaseLoaded(database_filename); -} - fs::path WalletDataFilePath(const fs::path& wallet_path) { fs::path env_directory; @@ -85,835 +29,3 @@ fs::path WalletDataFilePath(const fs::path& wallet_path) SplitWalletPath(wallet_path, env_directory, database_filename); return env_directory / database_filename; } - -/** - * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. - * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. - * @return A shared pointer to the BerkeleyEnvironment object for the wallet directory, never empty because ~BerkeleyEnvironment - * erases the weak pointer from the g_dbenvs map. - * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map. - */ -std::shared_ptr GetWalletEnv(const fs::path& wallet_path, std::string& database_filename) -{ - fs::path env_directory; - SplitWalletPath(wallet_path, env_directory, database_filename); - LOCK(cs_db); - auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr()); - if (inserted.second) { - auto env = std::make_shared(env_directory.string()); - inserted.first->second = env; - return env; - } - return inserted.first->second.lock(); -} - -// -// BerkeleyBatch -// - -void BerkeleyEnvironment::Close() -{ - if (!fDbEnvInit) - return; - - fDbEnvInit = false; - - for (auto& db : m_databases) { - auto count = mapFileUseCount.find(db.first); - assert(count == mapFileUseCount.end() || count->second == 0); - BerkeleyDatabase& database = db.second.get(); - if (database.m_db) { - database.m_db->close(0); - database.m_db.reset(); - } - } - - FILE* error_file = nullptr; - dbenv->get_errfile(&error_file); - - int ret = dbenv->close(0); - if (ret != 0) - LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret)); - if (!fMockDb) - DbEnv((u_int32_t)0).remove(strPath.c_str(), 0); - - if (error_file) fclose(error_file); - - UnlockDirectory(strPath, ".walletlock"); -} - -void BerkeleyEnvironment::Reset() -{ - dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); - fDbEnvInit = false; - fMockDb = false; -} - -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string()) -{ - Reset(); -} - -BerkeleyEnvironment::~BerkeleyEnvironment() -{ - LOCK(cs_db); - g_dbenvs.erase(strPath); - Close(); -} - -bool BerkeleyEnvironment::Open(bool retry) -{ - if (fDbEnvInit) { - return true; - } - - fs::path pathIn = strPath; - TryCreateDirectories(pathIn); - if (!LockDirectory(pathIn, ".walletlock")) { - LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); - return false; - } - - fs::path pathLogDir = pathIn / "database"; - TryCreateDirectories(pathLogDir); - fs::path pathErrorFile = pathIn / "db.log"; - LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); - - unsigned int nEnvFlags = 0; - if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) - nEnvFlags |= DB_PRIVATE; - - dbenv->set_lg_dir(pathLogDir.string().c_str()); - dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet - dbenv->set_lg_bsize(0x10000); - dbenv->set_lg_max(1048576); - dbenv->set_lk_max_locks(40000); - dbenv->set_lk_max_objects(40000); - dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); - dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); - int ret = dbenv->open(strPath.c_str(), - DB_CREATE | - DB_INIT_LOCK | - DB_INIT_LOG | - DB_INIT_MPOOL | - DB_INIT_TXN | - DB_THREAD | - DB_RECOVER | - nEnvFlags, - S_IRUSR | S_IWUSR); - if (ret != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret)); - int ret2 = dbenv->close(0); - if (ret2 != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2)); - } - Reset(); - if (retry) { - // try moving the database env out of the way - fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime()); - try { - fs::rename(pathLogDir, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string()); - } catch (const fs::filesystem_error&) { - // failure is ok (well, not really, but it's not worse than what we started with) - } - // try opening it again one more time - if (!Open(false /* retry */)) { - // if it still fails, it probably means we can't even create the database env - return false; - } - } else { - return false; - } - } - - fDbEnvInit = true; - fMockDb = false; - return true; -} - -//! Construct an in-memory mock Berkeley environment for testing -BerkeleyEnvironment::BerkeleyEnvironment() -{ - Reset(); - - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); - - dbenv->set_cachesize(1, 0, 1); - dbenv->set_lg_bsize(10485760 * 4); - dbenv->set_lg_max(10485760); - dbenv->set_lk_max_locks(10000); - dbenv->set_lk_max_objects(10000); - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); - int ret = dbenv->open(nullptr, - DB_CREATE | - DB_INIT_LOCK | - DB_INIT_LOG | - DB_INIT_MPOOL | - DB_INIT_TXN | - DB_THREAD | - DB_PRIVATE, - S_IRUSR | S_IWUSR); - if (ret > 0) { - throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret)); - } - - fDbEnvInit = true; - fMockDb = true; -} - -BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename) -{ - LOCK(cs_db); - assert(mapFileUseCount.count(strFile) == 0); - - Db db(dbenv.get(), 0); - int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); - if (result == 0) - return VerifyResult::VERIFY_OK; - else if (recoverFunc == nullptr) - return VerifyResult::RECOVER_FAIL; - - // Try to recover: - bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename); - return (fRecovered ? VerifyResult::RECOVER_OK : VerifyResult::RECOVER_FAIL); -} - -BerkeleyBatch::SafeDbt::SafeDbt() -{ - m_dbt.set_flags(DB_DBT_MALLOC); -} - -BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size) - : m_dbt(data, size) -{ -} - -BerkeleyBatch::SafeDbt::~SafeDbt() -{ - if (m_dbt.get_data() != nullptr) { - // Clear memory, e.g. in case it was a private key - memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); - // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be - // freed by the caller. - // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html - if (m_dbt.get_flags() & DB_DBT_MALLOC) { - free(m_dbt.get_data()); - } - } -} - -const void* BerkeleyBatch::SafeDbt::get_data() const -{ - return m_dbt.get_data(); -} - -u_int32_t BerkeleyBatch::SafeDbt::get_size() const -{ - return m_dbt.get_size(); -} - -BerkeleyBatch::SafeDbt::operator Dbt*() -{ - return &m_dbt; -} - -bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename) -{ - std::string filename; - std::shared_ptr env = GetWalletEnv(file_path, filename); - - // Recovery procedure: - // move wallet file to walletfilename.timestamp.bak - // Call Salvage with fAggressive=true to - // get as much data as possible. - // Rewrite salvaged data to fresh wallet file - // Set -rescan so any missing transactions will be - // found. - int64_t now = GetTime(); - newFilename = strprintf("%s.%d.bak", filename, now); - - int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, - newFilename.c_str(), DB_AUTO_COMMIT); - if (result == 0) - LogPrintf("Renamed %s to %s\n", filename, newFilename); - else - { - LogPrintf("Failed to rename %s to %s\n", filename, newFilename); - return false; - } - - std::vector salvagedData; - bool fSuccess = env->Salvage(newFilename, true, salvagedData); - if (salvagedData.empty()) - { - LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); - return false; - } - LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - - std::unique_ptr pdbCopy = MakeUnique(env->dbenv.get(), 0); - int ret = pdbCopy->open(nullptr, // Txn pointer - filename.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("Cannot create database file %s\n", filename); - pdbCopy->close(0); - return false; - } - - DbTxn* ptxn = env->TxnBegin(); - for (BerkeleyEnvironment::KeyValPair& row : salvagedData) - { - if (recoverKVcallback) - { - CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); - CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); - if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) - continue; - } - Dbt datKey(&row.first[0], row.first.size()); - Dbt datValue(&row.second[0], row.second.size()); - int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) - fSuccess = false; - } - ptxn->commit(0); - pdbCopy->close(0); - - return fSuccess; -} - -bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& errorStr) -{ - std::string walletFile; - std::shared_ptr env = GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); - LogPrintf("Using wallet %s\n", file_path.string()); - - if (!env->Open(true /* retry */)) { - errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); - return false; - } - - return true; -} - -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::string& warningStr, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc) -{ - std::string walletFile; - std::shared_ptr env = GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - if (fs::exists(walletDir / walletFile)) - { - std::string backup_filename; - BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename); - if (r == BerkeleyEnvironment::VerifyResult::RECOVER_OK) - { - warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!" - " Original %s saved as %s in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup."), - walletFile, backup_filename, walletDir); - } - if (r == BerkeleyEnvironment::VerifyResult::RECOVER_FAIL) - { - errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); - return false; - } - } - // also return true if files does not exists - return true; -} - -/* End of headers, beginning of key/value data */ -static const char *HEADER_END = "HEADER=END"; -/* End of key/value data */ -static const char *DATA_END = "DATA=END"; - -bool BerkeleyEnvironment::Salvage(const std::string& strFile, bool fAggressive, std::vector& vResult) -{ - LOCK(cs_db); - assert(mapFileUseCount.count(strFile) == 0); - - u_int32_t flags = DB_SALVAGE; - if (fAggressive) - flags |= DB_AGGRESSIVE; - - std::stringstream strDump; - - Db db(dbenv.get(), 0); - int result = db.verify(strFile.c_str(), nullptr, &strDump, flags); - if (result == DB_VERIFY_BAD) { - LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found errors, all data may not be recoverable.\n"); - if (!fAggressive) { - LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive mode to ignore errors and continue.\n"); - return false; - } - } - if (result != 0 && result != DB_VERIFY_BAD) { - LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with result %d.\n", result); - return false; - } - - // Format of bdb dump is ascii lines: - // header lines... - // HEADER=END - // hexadecimal key - // hexadecimal value - // ... repeated - // DATA=END - - std::string strLine; - while (!strDump.eof() && strLine != HEADER_END) - getline(strDump, strLine); // Skip past header - - std::string keyHex, valueHex; - while (!strDump.eof() && keyHex != DATA_END) { - getline(strDump, keyHex); - if (keyHex != DATA_END) { - if (strDump.eof()) - break; - getline(strDump, valueHex); - if (valueHex == DATA_END) { - LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of keys in data does not match number of values.\n"); - break; - } - vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); - } - } - - if (keyHex != DATA_END) { - LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of file while reading salvage output.\n"); - return false; - } - - return (result == 0); -} - - -void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) -{ - dbenv->txn_checkpoint(0, 0, 0); - if (fMockDb) - return; - dbenv->lsn_reset(strFile.c_str(), 0); -} - - -BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr) -{ - fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); - fFlushOnClose = fFlushOnCloseIn; - env = database.env.get(); - if (database.IsDummy()) { - return; - } - const std::string &strFilename = database.strFile; - - bool fCreate = strchr(pszMode, 'c') != nullptr; - unsigned int nFlags = DB_THREAD; - if (fCreate) - nFlags |= DB_CREATE; - - { - LOCK(cs_db); - if (!env->Open(false /* retry */)) - throw std::runtime_error("BerkeleyBatch: Failed to open database environment."); - - pdb = database.m_db.get(); - if (pdb == nullptr) { - int ret; - std::unique_ptr pdb_temp = MakeUnique(env->dbenv.get(), 0); - - bool fMockDb = env->IsMock(); - if (fMockDb) { - DbMpoolFile* mpf = pdb_temp->get_mpf(); - ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) { - throw std::runtime_error(strprintf("BerkeleyBatch: Failed to configure for no temp file backing for database %s", strFilename)); - } - } - - ret = pdb_temp->open(nullptr, // Txn pointer - fMockDb ? nullptr : strFilename.c_str(), // Filename - fMockDb ? strFilename.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags - 0); - - if (ret != 0) { - throw std::runtime_error(strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename)); - } - - // Call CheckUniqueFileid on the containing BDB environment to - // avoid BDB data consistency bugs that happen when different data - // files in the same environment have the same fileid. - // - // Also call CheckUniqueFileid on all the other g_dbenvs to prevent - // bitcoin from opening the same data file through another - // environment when the file is referenced through equivalent but - // not obviously identical symlinked or hard linked or bind mounted - // paths. In the future a more relaxed check for equal inode and - // device ids could be done instead, which would allow opening - // different backup copies of a wallet at the same time. Maybe even - // more ideally, an exclusive lock for accessing the database could - // be implemented, so no equality checks are needed at all. (Newer - // versions of BDB have an set_lk_exclusive method for this - // purpose, but the older version we use does not.) - for (const auto& env : g_dbenvs) { - CheckUniqueFileid(*env.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]); - } - - pdb = pdb_temp.release(); - database.m_db.reset(pdb); - - if (fCreate && !Exists(std::string("version"))) { - bool fTmp = fReadOnly; - fReadOnly = false; - Write(std::string("version"), CLIENT_VERSION); - fReadOnly = fTmp; - } - } - ++env->mapFileUseCount[strFilename]; - strFile = strFilename; - } -} - -void BerkeleyBatch::Flush() -{ - if (activeTxn) - return; - - // Flush database activity from memory pool to disk log - unsigned int nMinutes = 0; - if (fReadOnly) - nMinutes = 1; - - if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault - env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); - } -} - -void BerkeleyDatabase::IncrementUpdateCounter() -{ - ++nUpdateCounter; -} - -void BerkeleyBatch::Close() -{ - if (!pdb) - return; - if (activeTxn) - activeTxn->abort(); - activeTxn = nullptr; - pdb = nullptr; - - if (fFlushOnClose) - Flush(); - - { - LOCK(cs_db); - --env->mapFileUseCount[strFile]; - } - env->m_db_in_use.notify_all(); -} - -void BerkeleyEnvironment::CloseDb(const std::string& strFile) -{ - { - LOCK(cs_db); - auto it = m_databases.find(strFile); - assert(it != m_databases.end()); - BerkeleyDatabase& database = it->second.get(); - if (database.m_db) { - // Close the database handle - database.m_db->close(0); - database.m_db.reset(); - } - } -} - -void BerkeleyEnvironment::ReloadDbEnv() -{ - // Make sure that no Db's are in use - AssertLockNotHeld(cs_db); - std::unique_lock lock(cs_db); - m_db_in_use.wait(lock, [this](){ - for (auto& count : mapFileUseCount) { - if (count.second > 0) return false; - } - return true; - }); - - std::vector filenames; - for (auto it : m_databases) { - filenames.push_back(it.first); - } - // Close the individual Db's - for (const std::string& filename : filenames) { - CloseDb(filename); - } - // Reset the environment - Flush(true); // This will flush and close the environment - Reset(); - Open(true); -} - -bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) -{ - if (database.IsDummy()) { - return true; - } - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; - while (true) { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - bool fSuccess = true; - LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); - std::string strFileRes = strFile + ".rewrite"; - { // surround usage of db with extra {} - BerkeleyBatch db(database, "r"); - std::unique_ptr pdbCopy = MakeUnique(env->dbenv.get(), 0); - - int ret = pdbCopy->open(nullptr, // Txn pointer - strFileRes.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes); - fSuccess = false; - } - - Dbc* pcursor = db.GetCursor(); - if (pcursor) - while (fSuccess) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret1 == DB_NOTFOUND) { - pcursor->close(); - break; - } else if (ret1 != 0) { - pcursor->close(); - fSuccess = false; - break; - } - if (pszSkip && - strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) - continue; - if (strncmp(ssKey.data(), "\x07version", 8) == 0) { - // Update version: - ssValue.clear(); - ssValue << CLIENT_VERSION; - } - Dbt datKey(ssKey.data(), ssKey.size()); - Dbt datValue(ssValue.data(), ssValue.size()); - int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) - fSuccess = false; - } - if (fSuccess) { - db.Close(); - env->CloseDb(strFile); - if (pdbCopy->close(0)) - fSuccess = false; - } else { - pdbCopy->close(0); - } - } - if (fSuccess) { - Db dbA(env->dbenv.get(), 0); - if (dbA.remove(strFile.c_str(), nullptr, 0)) - fSuccess = false; - Db dbB(env->dbenv.get(), 0); - if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) - fSuccess = false; - } - if (!fSuccess) - LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes); - return fSuccess; - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - - -void BerkeleyEnvironment::Flush(bool fShutdown) -{ - int64_t nStart = GetTimeMillis(); - // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); - if (!fDbEnvInit) - return; - { - LOCK(cs_db); - std::map::iterator mi = mapFileUseCount.begin(); - while (mi != mapFileUseCount.end()) { - std::string strFile = (*mi).first; - int nRefCount = (*mi).second; - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); - if (nRefCount == 0) { - // Move log data to the dat file - CloseDb(strFile); - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); - dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); - if (!fMockDb) - dbenv->lsn_reset(strFile.c_str(), 0); - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); - mapFileUseCount.erase(mi++); - } else - mi++; - } - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); - if (fShutdown) { - char** listp; - if (mapFileUseCount.empty()) { - dbenv->log_archive(&listp, DB_ARCH_REMOVE); - Close(); - if (!fMockDb) { - fs::remove_all(fs::path(strPath) / "database"); - } - } - } - } -} - -bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) -{ - if (database.IsDummy()) { - return true; - } - bool ret = false; - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; - TRY_LOCK(cs_db, lockDb); - if (lockDb) - { - // Don't do this if any databases are in use - int nRefCount = 0; - std::map::iterator mit = env->mapFileUseCount.begin(); - while (mit != env->mapFileUseCount.end()) - { - nRefCount += (*mit).second; - mit++; - } - - if (nRefCount == 0) - { - std::map::iterator mi = env->mapFileUseCount.find(strFile); - if (mi != env->mapFileUseCount.end()) - { - LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - - env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); - ret = true; - } - } - } - - return ret; -} - -bool BerkeleyDatabase::Rewrite(const char* pszSkip) -{ - return BerkeleyBatch::Rewrite(*this, pszSkip); -} - -bool BerkeleyDatabase::Backup(const std::string& strDest) -{ - if (IsDummy()) { - return false; - } - while (true) - { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) - { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - // Copy wallet file - fs::path pathSrc = env->Directory() / strFile; - fs::path pathDest(strDest); - if (fs::is_directory(pathDest)) - pathDest /= strFile; - - try { - if (fs::equivalent(pathSrc, pathDest)) { - LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); - return false; - } - - fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strFile, pathDest.string()); - return true; - } catch (const fs::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); - return false; - } - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - -void BerkeleyDatabase::Flush(bool shutdown) -{ - if (!IsDummy()) { - env->Flush(shutdown); - if (shutdown) { - LOCK(cs_db); - g_dbenvs.erase(env->Directory().string()); - env = nullptr; - } else { - // TODO: To avoid g_dbenvs.erase erasing the environment prematurely after the - // first database shutdown when multiple databases are open in the same - // environment, should replace raw database `env` pointers with shared or weak - // pointers, or else separate the database and environment shutdowns so - // environments can be shut down after databases. - env->m_fileids.erase(strFile); - } - } -} - -void BerkeleyDatabase::ReloadDbEnv() -{ - if (!IsDummy()) { - env->ReloadDbEnv(); - } -} - -std::string BerkeleyDatabaseVersion() -{ - return DbEnv::version(nullptr, nullptr, nullptr); -} diff --git a/src/wallet/db.h b/src/wallet/db.h index bd7a2424f59f..adec3b3464b3 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -8,405 +8,189 @@ #include #include -#include #include -#include +#include #include -#include #include #include -#include -#include - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsuggest-override" -#endif -#include -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - -static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; -static const bool DEFAULT_WALLET_PRIVDB = true; - -struct WalletDatabaseFileId { - u_int8_t value[DB_FILE_ID_LEN]; - bool operator==(const WalletDatabaseFileId& rhs) const; -}; - -class BerkeleyDatabase; - -class BerkeleyEnvironment -{ -private: - bool fDbEnvInit; - bool fMockDb; - // Don't change into fs::path, as that can result in - // shutdown problems/crashes caused by a static initialized internal pointer. - std::string strPath; - -public: - std::unique_ptr dbenv; - std::map mapFileUseCount; - std::map> m_databases; - std::unordered_map m_fileids; - std::condition_variable_any m_db_in_use; - - BerkeleyEnvironment(const fs::path& env_directory); - BerkeleyEnvironment(); - ~BerkeleyEnvironment(); - void Reset(); - - bool IsMock() const { return fMockDb; } - bool IsInitialized() const { return fDbEnvInit; } - bool IsDatabaseLoaded(const std::string& db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } - fs::path Directory() const { return strPath; } - - /** - * Verify that database file strFile is OK. If it is not, - * call the callback to try to recover. - * This must be called BEFORE strFile is opened. - * Returns true if strFile is OK. - */ - enum class VerifyResult { VERIFY_OK, - RECOVER_OK, - RECOVER_FAIL }; - typedef bool (*recoverFunc_type)(const fs::path& file_path, std::string& out_backup_filename); - VerifyResult Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename); - /** - * Salvage data from a file that Verify says is bad. - * fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method documentation). - * Appends binary key/value pairs to vResult, returns true if successful. - * NOTE: reads the entire database into memory, so cannot be used - * for huge databases. - */ - typedef std::pair, std::vector > KeyValPair; - bool Salvage(const std::string& strFile, bool fAggressive, std::vector& vResult); - - bool Open(bool retry); - void Close(); - void Flush(bool fShutdown); - void CheckpointLSN(const std::string& strFile); - - void CloseDb(const std::string& strFile); - void ReloadDbEnv(); - - DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) - { - DbTxn* ptxn = nullptr; - int ret = dbenv->txn_begin(nullptr, &ptxn, flags); - if (!ptxn || ret != 0) - return nullptr; - return ptxn; - } -}; - -/** Return whether a wallet database is currently loaded. */ -bool IsWalletLoaded(const fs::path& wallet_path); /** Given a wallet directory path or legacy file path, return path to main data file in the wallet database. */ fs::path WalletDataFilePath(const fs::path& wallet_path); +void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename); -/** Get BerkeleyEnvironment and database filename given a wallet path. */ -std::shared_ptr GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); - -/** An instance of this class represents one database. - * For BerkeleyDB this is just a (env, strFile) tuple. - **/ -class BerkeleyDatabase +/** RAII class that provides access to a WalletDatabase */ +class DatabaseBatch { - friend class BerkeleyBatch; -public: - /** Create dummy DB handle */ - BerkeleyDatabase() : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(nullptr) - { - } - - /** Create DB handle to real database */ - BerkeleyDatabase(std::shared_ptr env, std::string filename) : - nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(std::move(env)), strFile(std::move(filename)) - { - auto inserted = this->env->m_databases.emplace(strFile, std::ref(*this)); - assert(inserted.second); - } - - ~BerkeleyDatabase() { - if (env) { - size_t erased = env->m_databases.erase(strFile); - assert(erased == 1); - } - } - - /** Return object for accessing database at specified path. */ - static std::unique_ptr Create(const fs::path& path) - { - std::string filename; - return MakeUnique(GetWalletEnv(path, filename), std::move(filename)); - } - - /** Return object for accessing dummy database with no read/write capabilities. */ - static std::unique_ptr CreateDummy() - { - return MakeUnique(); - } - - /** Return object for accessing temporary in-memory database. */ - static std::unique_ptr CreateMock() - { - return MakeUnique(std::make_shared(), ""); - } - - /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero - */ - bool Rewrite(const char* pszSkip=nullptr); - - /** Back up the entire database to a file. - */ - bool Backup(const std::string& strDest); - - /** Make sure all changes are flushed to disk. - */ - void Flush(bool shutdown); - - void IncrementUpdateCounter(); - - void ReloadDbEnv(); - - std::atomic nUpdateCounter; - unsigned int nLastSeen; - unsigned int nLastFlushed; - int64_t nLastWalletUpdate; - - /** - * Pointer to shared database environment. - * - * Normally there is only one BerkeleyDatabase object per - * BerkeleyEnvivonment, but in the special, backwards compatible case where - * multiple wallet BDB data files are loaded from the same directory, this - * will point to a shared instance that gets freed when the last data file - * is closed. - */ - std::shared_ptr env; - - /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ - std::unique_ptr m_db; - private: - std::string strFile; - - /** Return whether this database handle is a dummy for testing. - * Only to be used at a low level, application should ideally not care - * about this. - */ - bool IsDummy() { return env == nullptr; } -}; - -/** RAII class that provides access to a Berkeley database */ -class BerkeleyBatch -{ - /** RAII class that automatically cleanses its data on destruction */ - class SafeDbt final - { - Dbt m_dbt; - - public: - // construct Dbt with internally-managed data - SafeDbt(); - // construct Dbt with provided data - SafeDbt(void* data, size_t size); - ~SafeDbt(); - - // delegate to Dbt - const void* get_data() const; - u_int32_t get_size() const; - - // conversion operator to access the underlying Dbt - operator Dbt*(); - }; - -protected: - Db* pdb; - std::string strFile; - DbTxn* activeTxn; - bool fReadOnly; - bool fFlushOnClose; - BerkeleyEnvironment *env; + virtual bool ReadKey(CDataStream&& key, CDataStream& value) = 0; + virtual bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite=true) = 0; + virtual bool EraseKey(CDataStream&& key) = 0; + virtual bool HasKey(CDataStream&& key) = 0; public: - explicit BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode = "r+", bool fFlushOnCloseIn=true); - ~BerkeleyBatch() { Close(); } + explicit DatabaseBatch() {} + virtual ~DatabaseBatch() {} - BerkeleyBatch(const BerkeleyBatch&) = delete; - BerkeleyBatch& operator=(const BerkeleyBatch&) = delete; + DatabaseBatch(const DatabaseBatch&) = delete; + DatabaseBatch& operator=(const DatabaseBatch&) = delete; - void Flush(); - void Close(); - static bool Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename); - - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ - static bool PeriodicFlush(BerkeleyDatabase& database); - /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& file_path, std::string& errorStr); - /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& file_path, std::string& warningStr, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc); + virtual void Flush() = 0; + virtual void Close() = 0; template bool Read(const K& key, T& value) { - if (!pdb) - return false; - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Read - SafeDbt datValue; - int ret = pdb->get(activeTxn, datKey, datValue, 0); - bool success = false; - if (datValue.get_data() != nullptr) { - // Unserialize value - try { - CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); - ssValue >> value; - success = true; - } catch (const std::exception&) { - // In this case success remains 'false' - } + + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + if (!ReadKey(std::move(ssKey), ssValue)) return false; + try { + ssValue >> value; + return true; + } catch (const std::exception&) { + return false; } - return ret == 0 && success; } template bool Write(const K& key, const T& value, bool fOverwrite = true) { - if (!pdb) - return true; - if (fReadOnly) - assert(!"Write called on database in read-only mode"); - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - // Value CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(10000); ssValue << value; - SafeDbt datValue(ssValue.data(), ssValue.size()); - // Write - int ret = pdb->put(activeTxn, datKey, datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); - return (ret == 0); + return WriteKey(std::move(ssKey), std::move(ssValue), fOverwrite); } template bool Erase(const K& key) { - if (!pdb) - return false; - if (fReadOnly) - assert(!"Erase called on database in read-only mode"); - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - // Erase - int ret = pdb->del(activeTxn, datKey, 0); - return (ret == 0 || ret == DB_NOTFOUND); + return EraseKey(std::move(ssKey)); } template bool Exists(const K& key) { - if (!pdb) - return false; - - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - // Exists - int ret = pdb->exists(activeTxn, datKey, 0); - return (ret == 0); + return HasKey(std::move(ssKey)); } - Dbc* GetCursor() - { - if (!pdb) - return nullptr; - Dbc* pcursor = nullptr; - int ret = pdb->cursor(nullptr, &pcursor, 0); - if (ret != 0) - return nullptr; - return pcursor; - } + virtual bool StartCursor() = 0; + virtual bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) = 0; + virtual void CloseCursor() = 0; + virtual bool TxnBegin() = 0; + virtual bool TxnCommit() = 0; + virtual bool TxnAbort() = 0; +}; - int ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& ssValue) - { - // Read at cursor - SafeDbt datKey; - SafeDbt datValue; - int ret = pcursor->get(datKey, datValue, DB_NEXT); - if (ret != 0) - return ret; - else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) - return 99999; - - // Convert to streams - ssKey.SetType(SER_DISK); - ssKey.clear(); - ssKey.write((char*)datKey.get_data(), datKey.get_size()); - ssValue.SetType(SER_DISK); - ssValue.clear(); - ssValue.write((char*)datValue.get_data(), datValue.get_size()); - return 0; - } +/** An instance of this class represents one database. + **/ +class WalletDatabase +{ +public: + /** Create dummy DB handle */ + WalletDatabase() : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0) {} + virtual ~WalletDatabase() {}; - bool TxnBegin() - { - if (!pdb || activeTxn) - return false; - DbTxn* ptxn = env->TxnBegin(); - if (!ptxn) - return false; - activeTxn = ptxn; - return true; - } + /** Open the database if it is not already opened. */ + virtual void Open(const char* mode) = 0; - bool TxnCommit() - { - if (!pdb || !activeTxn) - return false; - int ret = activeTxn->commit(0); - activeTxn = nullptr; - return (ret == 0); - } + //! Counts the number of active database users to be sure that the database is not closed while someone is using it + std::atomic m_refcount{0}; + /** Indicate the a new database user has began using the database. Increments m_refcount */ + virtual void AddRef() = 0; + /** Indicate that database user has stopped using the database and that it could be flushed or closed. Decrement m_refcount */ + virtual void RemoveRef() = 0; - bool TxnAbort() - { - if (!pdb || !activeTxn) - return false; - int ret = activeTxn->abort(); - activeTxn = nullptr; - return (ret == 0); - } + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + virtual bool Rewrite(const char* pszSkip=nullptr) = 0; + + /** Back up the entire database to a file. + */ + virtual bool Backup(const std::string& strDest) const = 0; + + /** Make sure all changes are flushed to database file. + */ + virtual void Flush() = 0; + /** Flush to the database file and close the database. + * Also close the environment if no other databases are open in it. + */ + virtual void Close() = 0; + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + virtual bool PeriodicFlush() = 0; - bool static Rewrite(BerkeleyDatabase& database, const char* pszSkip = nullptr); + virtual void IncrementUpdateCounter() = 0; + + virtual void ReloadDbEnv() = 0; + + std::atomic nUpdateCounter; + unsigned int nLastSeen; + unsigned int nLastFlushed; + int64_t nLastWalletUpdate; + + /** Verifies the environment and database file */ + virtual bool Verify(std::string& error) = 0; + + std::string m_file_path; + + /** Make a DatabaseBatch connected to this database */ + virtual std::unique_ptr MakeBatch(const char* mode = "r+", bool flush_on_close = true) = 0; }; -std::string BerkeleyDatabaseVersion(); +/** RAII class that provides access to a DummyDatabase. Never fails. */ +class DummyBatch : public DatabaseBatch +{ +private: + bool ReadKey(CDataStream&& key, CDataStream& value) override { return true; } + bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite=true) override { return true; } + bool EraseKey(CDataStream&& key) override { return true; } + bool HasKey(CDataStream&& key) override { return true; } + +public: + void Flush() override {} + void Close() override {} + + bool StartCursor() override { return true; } + bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) override { return true; } + void CloseCursor() override {} + bool TxnBegin() override { return true; } + bool TxnCommit() override { return true; } + bool TxnAbort() override { return true; } +}; + +/** A dummy WalletDatabase that does nothing and never fails. Only used by unit tests. + **/ +class DummyDatabase : public WalletDatabase +{ +public: + void Open(const char* mode) override {}; + void AddRef() override {} + void RemoveRef() override {} + bool Rewrite(const char* pszSkip=nullptr) override { return true; } + bool Backup(const std::string& strDest) const override { return true; } + void Close() override {} + void Flush() override {} + bool PeriodicFlush() override { return true; } + void IncrementUpdateCounter() override { ++nUpdateCounter; } + void ReloadDbEnv() override {} + bool Verify(std::string& errorStr) override { return true; } + std::unique_ptr MakeBatch(const char* mode = "r+", bool flush_on_close = true) override { return MakeUnique(); } +}; #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 9cb4b81272ae..ad5a967cf636 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -49,7 +49,6 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-keypool=", strprintf("Set key pool size to (default: %u)", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-rescan=", "Rescan the block chain for missing wallet transactions on startup" " (1 = start from wallet creation time, 2 = start from genesis block)", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in .)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -111,16 +110,6 @@ bool WalletInit::ParameterInteraction() const LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - if (gArgs.GetBoolArg("-salvagewallet", false)) { - if (is_multiwallet) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-salvagewallet")); - } - // Rewrite just private keys: rescan to find transactions - if (gArgs.SoftSetBoolArg("-rescan", true)) { - LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__); - } - } - bool zapwallettxes = gArgs.GetBoolArg("-zapwallettxes", false); // -zapwallettxes implies dropping the mempool on startup if (zapwallettxes && gArgs.SoftSetBoolArg("-persistmempool", false)) { diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 4b3f465f523a..b88b5cfac0e0 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -38,11 +38,6 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector& wal chain.initMessage(_("Verifying wallet(s)...")); - // Parameter interaction code should have thrown an error if -salvagewallet - // was enabled with more than wallet file, so the wallet_files size check - // here should have no effect. - bool salvage_wallet = gArgs.GetBoolArg("-salvagewallet", false) && wallet_files.size() <= 1; - // Keep track of each wallet absolute path to detect duplicates. std::set wallet_paths; @@ -56,7 +51,7 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector& wal std::string error_string; std::string warning_string; - bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warning_string); + bool verify_success = CWallet::Verify(chain, location, error_string, warning_string); if (!error_string.empty()) chain.initError(error_string); if (!warning_string.empty()) chain.initWarning(warning_string); if (!verify_success) return false; @@ -101,14 +96,14 @@ void FlushWallets() it->second->ResetPool(); it->second->StopMixing(); } - pwallet->Flush(false); + pwallet->Flush(); } } void StopWallets() { for (const std::shared_ptr& pwallet : GetWallets()) { - pwallet->Flush(true); + pwallet->Close(); } } diff --git a/src/wallet/load.h b/src/wallet/load.h index 7420d81b1671..29b81844e52b 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -16,8 +16,6 @@ class Chain; } // namespace interfaces //! Responsible for reading and validating the -wallet arguments and verifying the wallet database. -// This function will perform salvage on the wallet if requested, as long as only one wallet is -// being loaded (WalletInit::ParameterInteraction() forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet). bool VerifyWallets(interfaces::Chain& chain, const std::vector& wallet_files); //! Load wallet databases. diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index b7cf913f742b..730afa6163c3 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2909,7 +2909,7 @@ static UniValue createwallet(const JSONRPCRequest& request) } // Wallet::Verify will check if we're trying to create a wallet with a duplication name. - if (!CWallet::Verify(*g_rpc_interfaces->chain, location, false, error, warning)) { + if (!CWallet::Verify(*g_rpc_interfaces->chain, location, error, warning)) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet file verification failed: " + error); } diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp new file mode 100644 index 000000000000..a786bdcfc4af --- /dev/null +++ b/src/wallet/salvage.cpp @@ -0,0 +1,156 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +/* End of headers, beginning of key/value data */ +static const char *HEADER_END = "HEADER=END"; +/* End of key/value data */ +static const char *DATA_END = "DATA=END"; +typedef std::pair, std::vector > KeyValPair; + +bool RecoverDatabaseFile(const fs::path& file_path) +{ + std::string filename; + std::shared_ptr env = GetWalletEnv(file_path, filename); + + if (!env->Open(true /* retry */)) { + tfm::format(std::cerr, "Error initializing wallet database environment %s!", env->Directory()); + return false; + } + + // Recovery procedure: + // move wallet file to walletfilename.timestamp.bak + // Call Salvage with fAggressive=true to + // get as much data as possible. + // Rewrite salvaged data to fresh wallet file + // Set -rescan so any missing transactions will be + // found. + int64_t now = GetTime(); + std::string newFilename = strprintf("%s.%d.bak", filename, now); + + int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, + newFilename.c_str(), DB_AUTO_COMMIT); + if (result == 0) + LogPrintf("Renamed %s to %s\n", filename, newFilename); + else + { + LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + return false; + } + + /** + * Salvage data from a file. The DB_AGGRESSIVE flag is being used (see berkeley DB->verify() method documentation). + * key/value pairs are appended to salvagedData which are then written out to a new wallet file. + * NOTE: reads the entire database into memory, so cannot be used + * for huge databases. + */ + std::vector salvagedData; + + std::stringstream strDump; + + Db db(env->dbenv.get(), 0); + result = db.verify(newFilename.c_str(), nullptr, &strDump, DB_SALVAGE | DB_AGGRESSIVE); + if (result == DB_VERIFY_BAD) { + LogPrintf("Salvage: Database salvage found errors, all data may not be recoverable.\n"); + } + if (result != 0 && result != DB_VERIFY_BAD) { + LogPrintf("Salvage: Database salvage failed with result %d.\n", result); + return false; + } + + // Format of bdb dump is ascii lines: + // header lines... + // HEADER=END + // hexadecimal key + // hexadecimal value + // ... repeated + // DATA=END + + std::string strLine; + while (!strDump.eof() && strLine != HEADER_END) + getline(strDump, strLine); // Skip past header + + std::string keyHex, valueHex; + while (!strDump.eof() && keyHex != DATA_END) { + getline(strDump, keyHex); + if (keyHex != DATA_END) { + if (strDump.eof()) + break; + getline(strDump, valueHex); + if (valueHex == DATA_END) { + LogPrintf("Salvage: WARNING: Number of keys in data does not match number of values.\n"); + break; + } + salvagedData.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); + } + } + + bool fSuccess; + if (keyHex != DATA_END) { + LogPrintf("Salvage: WARNING: Unexpected end of file while reading salvage output.\n"); + fSuccess = false; + } else { + fSuccess = (result == 0); + } + + if (salvagedData.empty()) + { + LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + return false; + } + LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); + + std::unique_ptr pdbCopy = MakeUnique(env->dbenv.get(), 0); + int ret = pdbCopy->open(nullptr, // Txn pointer + filename.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) { + LogPrintf("Cannot create database file %s\n", filename); + pdbCopy->close(0); + return false; + } + + auto chain = interfaces::MakeChain(); + DbTxn* ptxn = env->TxnBegin(); + CWallet dummyWallet(*chain, WalletLocation(), CreateDummyWalletDatabase()); + for (KeyValPair& row : salvagedData) + { + /* Filter for only private key type KV pairs to be added to the salvaged wallet */ + CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + std::string strType, strErr; + bool fReadOK; + { + // Required in LoadKeyMetadata(): + LOCK(dummyWallet.cs_wallet); + fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, strType, strErr); + } + if (!WalletBatch::IsKeyType(strType) && strType != DBKeys::HDCHAIN) { + continue; + } + if (!fReadOK) + { + LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); + continue; + } + Dbt datKey(&row.first[0], row.first.size()); + Dbt datValue(&row.second[0], row.second.size()); + int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + ptxn->commit(0); + pdbCopy->close(0); + + return fSuccess; +} diff --git a/src/wallet/salvage.h b/src/wallet/salvage.h new file mode 100644 index 000000000000..e361930f5ed9 --- /dev/null +++ b/src/wallet/salvage.h @@ -0,0 +1,14 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_SALVAGE_H +#define BITCOIN_WALLET_SALVAGE_H + +#include +#include + +bool RecoverDatabaseFile(const fs::path& file_path); + +#endif // BITCOIN_WALLET_SALVAGE_H diff --git a/src/wallet/test/coinjoin_tests.cpp b/src/wallet/test/coinjoin_tests.cpp index d9058c5c1d6c..8008092a6e90 100644 --- a/src/wallet/test/coinjoin_tests.cpp +++ b/src/wallet/test/coinjoin_tests.cpp @@ -37,7 +37,7 @@ class CTransactionBuilderTestSetup : public TestChain100Setup { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); chain = interfaces::MakeChain(); - wallet = MakeUnique(*chain, WalletLocation(), WalletDatabase::CreateMock()); + wallet = MakeUnique(*chain, WalletLocation(), CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); AddWallet(wallet); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 1825a4231118..9fde830e684c 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -30,7 +30,7 @@ typedef std::set CoinSet; static std::vector vCoins; static auto testChain = interfaces::MakeChain(); -static CWallet testWallet(*testChain, WalletLocation(), WalletDatabase::CreateDummy()); +static CWallet testWallet(*testChain, WalletLocation(), CreateDummyWalletDatabase()); static CAmount balance = 0; CoinEligibilityFilter filter_standard(1, 6, 0); diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index babb3bd9c100..1a020fc35c31 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index f6db5196210f..b818b0d18d63 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -7,7 +7,7 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), - m_wallet(*m_chain, WalletLocation(), WalletDatabase::CreateMock()) + m_wallet(*m_chain, WalletLocation(), CreateMockWalletDatabase()) { bool fFirstRun; m_wallet.LoadWallet(fFirstRun); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 166e2ba774d7..f25d388b1eef 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -49,7 +49,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions accommodates a null start block. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(*chain, WalletLocation(), CreateDummyWalletDatabase()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -64,7 +64,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(*chain, WalletLocation(), CreateDummyWalletDatabase()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -83,7 +83,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(*chain, WalletLocation(), CreateDummyWalletDatabase()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -101,7 +101,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(*chain, WalletLocation(), CreateDummyWalletDatabase()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -134,7 +134,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // before the missing block, and success for a key whose creation time is // after. { - std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), CreateDummyWalletDatabase()); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -197,7 +197,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { - std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), CreateDummyWalletDatabase()); LOCK(wallet->cs_wallet); wallet->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); @@ -213,7 +213,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), CreateDummyWalletDatabase()); JSONRPCRequest request; request.params.setArray(); @@ -245,7 +245,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { auto chain = interfaces::MakeChain(); - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(*chain, WalletLocation(), CreateDummyWalletDatabase()); CWalletTx wtx(&wallet, m_coinbase_txns.back()); auto locked_chain = chain->lock(); @@ -343,7 +343,7 @@ class ListCoinsTestingSetup : public TestChain100Setup ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = MakeUnique(*m_chain, WalletLocation(), WalletDatabase::CreateMock()); + wallet = MakeUnique(*m_chain, WalletLocation(), CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); @@ -485,7 +485,7 @@ class CreateTransactionTestSetup : public TestChain100Setup CreateTransactionTestSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = MakeUnique(*m_chain, WalletLocation(), WalletDatabase::CreateMock()); + wallet = MakeUnique(*m_chain, WalletLocation(), CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); AddWallet(wallet); @@ -958,7 +958,7 @@ BOOST_FIXTURE_TEST_CASE(select_coins_grouped_by_addresses, ListCoinsTestingSetup BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { auto chain = interfaces::MakeChain(); - std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr wallet = std::make_shared(*chain, WalletLocation(), CreateDummyWalletDatabase()); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 2cc3dc4742f9..7bc3a386d976 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -150,7 +150,7 @@ void UnloadWallet(std::shared_ptr&& wallet) std::shared_ptr LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::string& warning) { - if (!CWallet::Verify(chain, location, false, error, warning)) { + if (!CWallet::Verify(chain, location, error, warning)) { error = "Wallet file verification failed: " + error; return nullptr; } @@ -811,9 +811,14 @@ std::set CWallet::GetConflicts(const uint256& txid) const return result; } -void CWallet::Flush(bool shutdown) +void CWallet::Flush() { - database->Flush(shutdown); + database->Flush(); +} + +void CWallet::Close() +{ + database->Close(); } void CWallet::SyncMetaData(std::pair range) @@ -4916,7 +4921,7 @@ std::vector CWallet::GetDestValues(const std::string& prefix) const return values; } -bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::string& warning_string) +bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, std::string& error_string, std::string& warning_string) { // Do some checking on wallet path. It should be either a: // @@ -4945,32 +4950,20 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b } // Keep same database environment instance across Verify/Recover calls below. - // Let tempWallet hold the pointer to the corresponding wallet database. - std::unique_ptr tempWallet = MakeUnique(chain, location, WalletDatabase::Create(wallet_path)); + std::unique_ptr database = CreateWalletDatabase(wallet_path); try { - if (!WalletBatch::VerifyEnvironment(wallet_path, error_string)) { - return false; - } + return database->Verify(error_string); } catch (const fs::filesystem_error& e) { error_string = strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e)); return false; } + // Let tempWallet hold the pointer to the corresponding wallet database. + std::unique_ptr tempWallet = MakeUnique(chain, location, std::move(database)); if (!tempWallet->AutoBackupWallet(wallet_path, warning_string, error_string) && !error_string.empty()) { return false; } - - if (salvage_wallet) { - // Recover readable keypairs: - CWallet dummyWallet(chain, WalletLocation(), WalletDatabase::CreateDummy()); - std::string backup_filename; - if (!WalletBatch::Recover(wallet_path, (void *)&dummyWallet, WalletBatch::RecoverKeysOnlyFilter, backup_filename)) { - return false; - } - } - - return WalletBatch::VerifyDatabaseFile(wallet_path, warning_string, error_string); } std::shared_ptr CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, uint64_t wallet_creation_flags) @@ -4983,7 +4976,7 @@ std::shared_ptr CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.GetBoolArg("-zapwallettxes", false)) { chain.initMessage(_("Zapping all transactions from wallet...")); - std::unique_ptr tempWallet = MakeUnique(chain, location, WalletDatabase::Create(location.GetPath())); + std::unique_ptr tempWallet = MakeUnique(chain, location, CreateWalletDatabase(location.GetPath())); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DBErrors::LOAD_OK) { chain.initError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); @@ -4997,7 +4990,7 @@ std::shared_ptr CWallet::CreateWalletFromFile(interfaces::Chain& chain, bool fFirstRun = true; // TODO: Can't use std::make_shared because we need a custom deleter but // should be possible to use std::allocate_shared. - std::shared_ptr walletInstance(new CWallet(chain, location, WalletDatabase::Create(location.GetPath())), ReleaseWallet); + std::shared_ptr walletInstance(new CWallet(chain, location, CreateWalletDatabase(location.GetPath())), ReleaseWallet); AddWallet(walletInstance); auto error = [&](const std::string& strError) { RemoveWallet(walletInstance); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 3c6b4fad33b0..86ef1376a0ca 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1113,7 +1113,10 @@ class CWallet final : public CCryptoKeyStore, private interfaces::Chain::Notific std::set GetConflicts(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Flush wallet (bitdb flush) - void Flush(bool shutdown=false); + void Flush(); + + //! Close wallet database + void Close(); /** Wallet is about to be unloaded */ boost::signals2::signal NotifyUnload; @@ -1161,7 +1164,7 @@ class CWallet final : public CCryptoKeyStore, private interfaces::Chain::Notific bool AbandonTransaction(interfaces::Chain::Lock& locked_chain, const uint256& hashTx); //! Verify wallet naming and perform salvage on the wallet if required - static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::string& warning_string); + static bool Verify(interfaces::Chain& chain, const WalletLocation& location, std::string& error_string, std::string& warning_string); /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ static std::shared_ptr CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, uint64_t wallet_creation_flags = 0); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 622037900a34..307844409909 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -158,8 +158,8 @@ bool WalletBatch::WriteBestBlock(const CBlockLocator& locator) bool WalletBatch::ReadBestBlock(CBlockLocator& locator) { - if (m_batch.Read(DBKeys::BESTBLOCK, locator) && !locator.vHave.empty()) return true; - return m_batch.Read(DBKeys::BESTBLOCK_NOMERKLE, locator); + if (m_batch->Read(DBKeys::BESTBLOCK, locator) && !locator.vHave.empty()) return true; + return m_batch->Read(DBKeys::BESTBLOCK_NOMERKLE, locator); } bool WalletBatch::WriteOrderPosNext(int64_t nOrderPosNext) @@ -169,7 +169,7 @@ bool WalletBatch::WriteOrderPosNext(int64_t nOrderPosNext) bool WalletBatch::ReadPool(int64_t nPool, CKeyPool& keypool) { - return m_batch.Read(std::make_pair(DBKeys::POOL, nPool), keypool); + return m_batch->Read(std::make_pair(DBKeys::POOL, nPool), keypool); } bool WalletBatch::WritePool(int64_t nPool, const CKeyPool& keypool) @@ -190,7 +190,7 @@ bool WalletBatch::WriteMinVersion(int nVersion) bool WalletBatch::ReadCoinJoinSalt(uint256& salt, bool fLegacy) { // TODO: Remove legacy checks after few major releases - return m_batch.Read(std::string(fLegacy ? DBKeys::PRIVATESEND_SALT : DBKeys::COINJOIN_SALT), salt); + return m_batch->Read(std::string(fLegacy ? DBKeys::PRIVATESEND_SALT : DBKeys::COINJOIN_SALT), salt); } bool WalletBatch::WriteCoinJoinSalt(const uint256& salt) @@ -489,6 +489,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return true; } +bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr) +{ + CWalletScanState dummy_wss; + LOCK(pwallet->cs_wallet); + return ReadKeyValue(pwallet, ssKey, ssValue, dummy_wss, strType, strErr); +} + bool WalletBatch::IsKeyType(const std::string& strType) { return (strType == DBKeys::KEY || @@ -507,15 +514,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) LOCK(pwallet->cs_wallet); try { int nMinVersion = 0; - if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { + if (m_batch->Read(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) return DBErrors::TOO_NEW; pwallet->LoadMinVersion(nMinVersion); } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch->StartCursor()) { pwallet->WalletLogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -526,11 +532,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) + bool complete; + bool ret = m_batch->ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) + } + else if (!ret) { + m_batch->CloseCursor(); pwallet->WalletLogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -557,7 +566,6 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (!strErr.empty()) pwallet->WalletLogPrintf("%s\n", strErr); } - pcursor->close(); // Store initial external keypool size since we mostly use external keys in mixing pwallet->nKeysLeftSinceAutoBackup = pwallet->KeypoolCountExternalKeys(); @@ -565,6 +573,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) } catch (...) { result = DBErrors::CORRUPT; } + m_batch->CloseCursor(); if (fNoncriticalErrors && result == DBErrors::LOAD_OK) result = DBErrors::NONCRITICAL_ERROR; @@ -576,7 +585,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Last client version to open this wallet, was previously the file version number int last_client = CLIENT_VERSION; - m_batch.Read(DBKeys::VERSION, last_client); + m_batch->Read(DBKeys::VERSION, last_client); int wallet_version = pwallet->GetVersion(); pwallet->WalletLogPrintf("Wallet File Version = %d\n", wallet_version > 0 ? wallet_version : last_client); @@ -597,7 +606,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) return DBErrors::NEED_REWRITE; if (last_client < CLIENT_VERSION) // Update - m_batch.Write(DBKeys::VERSION, CLIENT_VERSION); + m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); if (wss.fAnyUnordered) result = pwallet->ReorderTransactions(); @@ -619,14 +628,13 @@ DBErrors WalletBatch::FindWalletTx(std::vector& vTxHash, std::vectorRead(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) return DBErrors::TOO_NEW; } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch->StartCursor()) { LogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -637,11 +645,12 @@ DBErrors WalletBatch::FindWalletTx(std::vector& vTxHash, std::vectorReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) - { + } else if (!ret) { + m_batch->CloseCursor(); LogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -659,10 +668,10 @@ DBErrors WalletBatch::FindWalletTx(std::vector& vTxHash, std::vectorclose(); } catch (...) { result = DBErrors::CORRUPT; } + m_batch->CloseCursor(); return result; } @@ -743,7 +752,7 @@ void MaybeCompactWalletDB() } if (dbh.nLastFlushed != nUpdateCounter && GetTime() - dbh.nLastWalletUpdate >= 2) { - if (BerkeleyBatch::PeriodicFlush(dbh)) { + if (dbh.PeriodicFlush()) { dbh.nLastFlushed = nUpdateCounter; } } @@ -752,56 +761,6 @@ void MaybeCompactWalletDB() fOneThread = false; } -// -// Try to (very carefully!) recover wallet file if there is a problem. -// -bool WalletBatch::Recover(const fs::path& wallet_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename) -{ - return BerkeleyBatch::Recover(wallet_path, callbackDataIn, recoverKVcallback, out_backup_filename); -} - -bool WalletBatch::Recover(const fs::path& wallet_path, std::string& out_backup_filename) -{ - // recover without a key filter callback - // results in recovering all record types - return WalletBatch::Recover(wallet_path, nullptr, nullptr, out_backup_filename); -} - -bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue) -{ - CWallet *dummyWallet = reinterpret_cast(callbackData); - CWalletScanState dummyWss; - std::string strType, strErr; - bool fReadOK; - { - // Required in LoadKeyMetadata(): - LOCK(dummyWallet->cs_wallet); - fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, - dummyWss, strType, strErr); - } - if (!IsKeyType(strType) && strType != DBKeys::HDCHAIN) { - return false; - } - - if (!fReadOK) - { - LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); - return false; - } - - return true; -} - -bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, std::string& errorStr) -{ - return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); -} - -bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, std::string& warningStr, std::string& errorStr) -{ - return BerkeleyBatch::VerifyDatabaseFile(wallet_path, warningStr, errorStr, WalletBatch::Recover); -} - bool WalletBatch::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { return WriteIC(std::make_pair(DBKeys::DESTDATA, std::make_pair(address, key)), value); @@ -842,15 +801,39 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) bool WalletBatch::TxnBegin() { - return m_batch.TxnBegin(); + return m_batch->TxnBegin(); } bool WalletBatch::TxnCommit() { - return m_batch.TxnCommit(); + return m_batch->TxnCommit(); } bool WalletBatch::TxnAbort() { - return m_batch.TxnAbort(); + return m_batch->TxnAbort(); +} + +bool IsWalletLoaded(const fs::path& wallet_path) +{ + return IsBDBWalletLoaded(wallet_path); +} + +/** Return object for accessing database at specified path. */ +std::unique_ptr CreateWalletDatabase(const fs::path& path) +{ + std::string filename; + return MakeUnique(GetWalletEnv(path, filename), std::move(filename)); +} + +/** Return object for accessing dummy database with no read/write capabilities. */ +std::unique_ptr CreateDummyWalletDatabase() +{ + return MakeUnique(); +} + +/** Return object for accessing temporary in-memory database. */ +std::unique_ptr CreateMockWalletDatabase() +{ + return MakeUnique(std::make_shared(), ""); } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index a0e5f7a60719..5b8cf3eb50df 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -8,6 +8,7 @@ #include #include