From 7ff6515c8814210010bc4f4f02f32cf22d20e9d6 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 May 2021 13:48:04 +0300 Subject: [PATCH] Merge #13033: Build txindex in parallel with validation 9b2704777c [doc] Include txindex changes in the release notes. (Jim Posen) ed77dd6b30 [test] Simple unit test for TxIndex. (Jim Posen) 6d772a3d44 [rpc] Public interfaces to GetTransaction block until synced. (Jim Posen) a03f804f2a [index] Move disk IO logic from GetTransaction to TxIndex::FindTx. (Jim Posen) e0a3b80033 [validation] Replace tx index code in validation code with TxIndex. (Jim Posen) 8181db88f6 [init] Initialize and start TxIndex in init code. (Jim Posen) f90c3a62f5 [index] TxIndex method to wait until caught up. (Jim Posen) 70d510d93c [index] Allow TxIndex sync thread to be interrupted. (Jim Posen) 94b4f8bbb9 [index] TxIndex initial sync thread. (Jim Posen) 34d68bf3a3 [index] Create new TxIndex class. (Jim Posen) c88bcec93f [db] Migration for txindex data to new, separate database. (Jim Posen) 0cb8303241 [db] Create separate database for txindex. (Jim Posen) Pull request description: I'm re-opening #11857 as a new pull request because the last one stopped loading for people ------------------------------- This refactors the tx index code to be in it's own class and get built concurrently with validation code. The main benefit is decoupling and moving the txindex into a separate DB. The primary motivation is to lay the groundwork for other indexers that might be desired (such as the [compact filters](https://github.com/bitcoin/bips/pull/636)). The basic idea is that the TxIndex spins up its own thread, which first syncs the txindex to the current block index, then once in sync the BlockConnected ValidationInterface hook writes new blocks. ### DB changes At the suggestion of some other developers, the txindex has been split out into a separate database. A data migration runs at startup on any nodes with a legacy txindex. Currently the migration blocks node initialization until complete. ### Open questions - Should the migration of txindex data from the old DB to the new DB block in init or should it happen in a background thread? The downside to backgrounding it is that `getrawtransaction` would return an error message saying the txindex is syncing while the migration is running. ### Impact In a sample size n=1 test where I synced nodes from scratch, the average time [Index writing](https://github.com/bitcoin/bitcoin/blob/master/src/validation.cpp#L1903) was 3.36ms in master and 1.72ms in this branch. The average time between `UpdateTip` log lines for sequential blocks between 400,000 and IBD end on mainnet was 0.297204s in master and 0.286134s in this branch. Most likely this is just variance in IBD times, but I can try with some more trials if people want. Tree-SHA512: 451fd7d95df89dfafceaa723cdf0f7b137615b531cf5c5035cfb54e9ccc2026cec5ac85edbcf71b7f4e2f102e36e9202b8b3a667e1504a9e1a9976ab1f0079c4 --- doc/files.md | 1 + src/Makefile.am | 2 + src/Makefile.test.include | 1 + src/dbwrapper.h | 3 + src/index/txindex.cpp | 315 ++++++++++++++++++ src/index/txindex.h | 95 ++++++ src/init.cpp | 40 ++- src/llmq/quorums_instantsend.cpp | 5 + src/net_processing.cpp | 3 +- src/rest.cpp | 5 + src/rpc/blockchain.cpp | 8 +- src/rpc/governance.cpp | 17 + src/rpc/masternode.cpp | 5 + src/rpc/rawtransaction.cpp | 37 +- src/rpc/rpcevo.cpp | 9 + src/rpc/rpcquorums.cpp | 5 + src/test/test_dash.cpp | 11 + src/test/txindex_tests.cpp | 66 ++++ src/threadinterrupt.cpp | 2 + src/threadinterrupt.h | 1 + src/txdb.cpp | 171 ++++++++++ src/txdb.h | 38 ++- src/validation.cpp | 59 +--- src/validation.h | 1 - test/functional/feature_txindex.py | 13 - .../test_framework/test_framework.py | 2 +- test/lint/lint-circular-dependencies.sh | 6 +- test/lint/lint-format-strings.py | 1 + 28 files changed, 817 insertions(+), 105 deletions(-) create mode 100644 src/index/txindex.cpp create mode 100644 src/index/txindex.h create mode 100644 src/test/txindex_tests.cpp diff --git a/doc/files.md b/doc/files.md index ae5067b80fee7..164ca3ef9d25a 100644 --- a/doc/files.md +++ b/doc/files.md @@ -12,6 +12,7 @@ * evodb/*: special txes and quorums database * fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation * governance.dat: stores data for governance obgects +* indexes/txindex/*: optional transaction index database (LevelDB); since 0.17.0 * llmq/*: quorum signatures database * mempool.dat: dump of the mempool's transactions * mncache.dat: stores data for masternode list diff --git a/src/Makefile.am b/src/Makefile.am index c85d1b5c2b907..a1757c6a7229c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -173,6 +173,7 @@ BITCOIN_CORE_H = \ fs.h \ httprpc.h \ httpserver.h \ + index/txindex.h \ indirectmap.h \ init.h \ interfaces/handler.h \ @@ -315,6 +316,7 @@ libdash_server_a_SOURCES = \ evo/specialtx.cpp \ httprpc.cpp \ httpserver.cpp \ + index/txindex.cpp \ init.cpp \ dbwrapper.cpp \ governance/governance.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0ad3d24da0187..6fc8a1e4ce642 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -104,6 +104,7 @@ BITCOIN_TESTS =\ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ + test/txindex_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b058807a11b03..c795054686e86 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -245,6 +245,9 @@ class CDBWrapper CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); ~CDBWrapper(); + CDBWrapper(const CDBWrapper&) = delete; + CDBWrapper& operator=(const CDBWrapper&) = delete; + template bool ReadDataStream(const K& key, CDataStream& ssValue) const { diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp new file mode 100644 index 0000000000000..9ccbc95401840 --- /dev/null +++ b/src/index/txindex.cpp @@ -0,0 +1,315 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds +constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds + +std::unique_ptr g_txindex; + +template +static void FatalError(const char* fmt, const Args&... args) +{ + std::string strMessage = tfm::format(fmt, args...); + SetMiscWarning(strMessage); + LogPrintf("*** %s\n", strMessage); + uiInterface.ThreadSafeMessageBox( + "Error: A fatal internal error occurred, see debug.log for details", + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); +} + +TxIndex::TxIndex(std::unique_ptr db) : + m_db(std::move(db)), m_synced(false), m_best_block_index(nullptr) +{} + +TxIndex::~TxIndex() +{ + Interrupt(); + Stop(); +} + +bool TxIndex::Init() +{ + LOCK(cs_main); + + // Attempt to migrate txindex from the old database to the new one. Even if + // chain_tip is null, the node could be reindexing and we still want to + // delete txindex records in the old database. + if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) { + return false; + } + + CBlockLocator locator; + if (!m_db->ReadBestBlock(locator)) { + locator.SetNull(); + } + + m_best_block_index = FindForkInGlobalIndex(chainActive, locator); + m_synced = m_best_block_index.load() == chainActive.Tip(); + return true; +} + +static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev) +{ + AssertLockHeld(cs_main); + + if (!pindex_prev) { + return chainActive.Genesis(); + } + + const CBlockIndex* pindex = chainActive.Next(pindex_prev); + if (pindex) { + return pindex; + } + + return chainActive.Next(chainActive.FindFork(pindex_prev)); +} + +void TxIndex::ThreadSync() +{ + const CBlockIndex* pindex = m_best_block_index.load(); + if (!m_synced) { + auto& consensus_params = Params().GetConsensus(); + + int64_t last_log_time = 0; + int64_t last_locator_write_time = 0; + while (true) { + if (m_interrupt) { + WriteBestBlock(pindex); + return; + } + + { + LOCK(cs_main); + const CBlockIndex* pindex_next = NextSyncBlock(pindex); + if (!pindex_next) { + WriteBestBlock(pindex); + m_best_block_index = pindex; + m_synced = true; + break; + } + pindex = pindex_next; + } + + int64_t current_time = GetTime(); + if (last_log_time + SYNC_LOG_INTERVAL < current_time) { + LogPrintf("Syncing txindex with block chain from height %d\n", pindex->nHeight); + last_log_time = current_time; + } + + if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { + WriteBestBlock(pindex); + last_locator_write_time = current_time; + } + + CBlock block; + if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + FatalError("%s: Failed to read block %s from disk", + __func__, pindex->GetBlockHash().ToString()); + return; + } + if (!WriteBlock(block, pindex)) { + FatalError("%s: Failed to write block %s to tx index database", + __func__, pindex->GetBlockHash().ToString()); + return; + } + } + } + + if (pindex) { + LogPrintf("txindex is enabled at height %d\n", pindex->nHeight); + } else { + LogPrintf("txindex is enabled\n"); + } +} + +bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); + std::vector> vPos; + vPos.reserve(block.vtx.size()); + for (const auto& tx : block.vtx) { + vPos.emplace_back(tx->GetHash(), pos); + pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION); + } + return m_db->WriteTxs(vPos); +} + +bool TxIndex::WriteBestBlock(const CBlockIndex* block_index) +{ + LOCK(cs_main); + if (!m_db->WriteBestBlock(chainActive.GetLocator(block_index))) { + return error("%s: Failed to write locator to disk", __func__); + } + return true; +} + +void TxIndex::BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, + const std::vector& txn_conflicted) +{ + if (!m_synced) { + return; + } + + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (!best_block_index) { + if (pindex->nHeight != 0) { + FatalError("%s: First block connected is not the genesis block (height=%d)", + __func__, pindex->nHeight); + return; + } + } else { + // Ensure block connects to an ancestor of the current best block. This should be the case + // most of the time, but may not be immediately after the the sync thread catches up and sets + // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are + // in the ValidationInterface queue backlog even after the sync thread has caught up to the + // new chain tip. In this unlikely event, log a warning and let the queue clear. + if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { + LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ + "known best chain (tip=%s); not updating txindex\n", + __func__, pindex->GetBlockHash().ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + } + + if (WriteBlock(*block, pindex)) { + m_best_block_index = pindex; + } else { + FatalError("%s: Failed to write block %s to txindex", + __func__, pindex->GetBlockHash().ToString()); + return; + } +} + +void TxIndex::SetBestChain(const CBlockLocator& locator) +{ + if (!m_synced) { + return; + } + + const uint256& locator_tip_hash = locator.vHave.front(); + const CBlockIndex* locator_tip_index; + { + LOCK(cs_main); + locator_tip_index = LookupBlockIndex(locator_tip_hash); + } + + if (!locator_tip_index) { + FatalError("%s: First block (hash=%s) in locator was not found", + __func__, locator_tip_hash.ToString()); + return; + } + + // This checks that SetBestChain callbacks are received after BlockConnected. The check may fail + // immediately after the the sync thread catches up and sets m_synced. Consider the case where + // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue + // backlog even after the sync thread has caught up to the new chain tip. In this unlikely + // event, log a warning and let the queue clear. + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { + LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ + "chain (tip=%s); not writing txindex locator\n", + __func__, locator_tip_hash.ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + + if (!m_db->WriteBestBlock(locator)) { + error("%s: Failed to write locator to disk", __func__); + } +} + +bool TxIndex::BlockUntilSyncedToCurrentChain() +{ + AssertLockNotHeld(cs_main); + + if (!m_synced) { + return false; + } + + { + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip(). + LOCK(cs_main); + const CBlockIndex* chain_tip = chainActive.Tip(); + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) { + return true; + } + } + + LogPrintf("%s: txindex is catching up on block notifications\n", __func__); + SyncWithValidationInterfaceQueue(); + return true; +} + +bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const +{ + CDiskTxPos postx; + if (!m_db->ReadTxPos(tx_hash, postx)) { + return false; + } + + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + return error("%s: OpenBlockFile failed", __func__); + } + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> tx; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + if (tx->GetHash() != tx_hash) { + return error("%s: txid mismatch", __func__); + } + block_hash = header.GetHash(); + return true; +} + +bool TxIndex::HasTx(const uint256& tx_hash) const +{ + CDiskTxPos postx; + return m_db->ReadTxPos(tx_hash, postx); +} + +void TxIndex::Interrupt() +{ + m_interrupt(); +} + +void TxIndex::Start() +{ + // Need to register this ValidationInterface before running Init(), so that + // callbacks are not missed if Init sets m_synced to true. + RegisterValidationInterface(this); + if (!Init()) { + FatalError("%s: txindex failed to initialize", __func__); + return; + } + + m_thread_sync = std::thread(&TraceThread>, "txindex", + std::bind(&TxIndex::ThreadSync, this)); +} + +void TxIndex::Stop() +{ + UnregisterValidationInterface(this); + + if (m_thread_sync.joinable()) { + m_thread_sync.join(); + } +} diff --git a/src/index/txindex.h b/src/index/txindex.h new file mode 100644 index 0000000000000..eb854763ad34d --- /dev/null +++ b/src/index/txindex.h @@ -0,0 +1,95 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_TXINDEX_H +#define BITCOIN_INDEX_TXINDEX_H + +#include +#include +#include +#include +#include +#include + +class CBlockIndex; + +/** + * TxIndex is used to look up transactions included in the blockchain by hash. + * The index is written to a LevelDB database and records the filesystem + * location of each transaction by transaction hash. + */ +class TxIndex final : public CValidationInterface +{ +private: + const std::unique_ptr m_db; + + /// Whether the index is in sync with the main chain. The flag is flipped + /// from false to true once, after which point this starts processing + /// ValidationInterface notifications to stay in sync. + std::atomic m_synced; + + /// The last block in the chain that the TxIndex is in sync with. + std::atomic m_best_block_index; + + std::thread m_thread_sync; + CThreadInterrupt m_interrupt; + + /// Initialize internal state from the database and block index. + bool Init(); + + /// Sync the tx index with the block index starting from the current best + /// block. Intended to be run in its own thread, m_thread_sync, and can be + /// interrupted with m_interrupt. Once the txindex gets in sync, the + /// m_synced flag is set and the BlockConnected ValidationInterface callback + /// takes over and the sync thread exits. + void ThreadSync(); + + /// Write update index entries for a newly connected block. + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex); + + /// Write the current chain block locator to the DB. + bool WriteBestBlock(const CBlockIndex* block_index); + +protected: + void BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, + const std::vector& txn_conflicted) override; + + void SetBestChain(const CBlockLocator& locator) override; + +public: + /// Constructs the TxIndex, which becomes available to be queried. + explicit TxIndex(std::unique_ptr db); + + /// Destructor interrupts sync thread if running and blocks until it exits. + ~TxIndex(); + + /// Blocks the current thread until the transaction index is caught up to + /// the current state of the block chain. This only blocks if the index has gotten in sync once + /// and only needs to process blocks in the ValidationInterface queue. If the index is catching + /// up from far behind, this method does not block and immediately returns false. + bool BlockUntilSyncedToCurrentChain(); + + /// Look up a transaction by hash. + /// + /// @param[in] tx_hash The hash of the transaction to be returned. + /// @param[out] block_hash The hash of the block the transaction is found in. + /// @param[out] tx The transaction itself. + /// @return true if transaction is found, false otherwise + bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const; + bool HasTx(const uint256& tx_hash) const; + + void Interrupt(); + + /// Start initializes the sync state and registers the instance as a + /// ValidationInterface so that it stays in sync with blockchain updates. + void Start(); + + /// Stops the instance from staying in sync with blockchain updates. + void Stop(); +}; + +/// The global transaction index, used in GetTransaction. May be null. +extern std::unique_ptr g_txindex; + +#endif // BITCOIN_INDEX_TXINDEX_H diff --git a/src/init.cpp b/src/init.cpp index a7f544d4f1fd7..3a6cc0ba8c93f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -224,6 +225,9 @@ void Interrupt() InterruptMapPort(); if (g_connman) g_connman->Interrupt(); + if (g_txindex) { + g_txindex->Interrupt(); + } } /** Preparing steps before shutting down or restarting the wallet */ @@ -259,7 +263,7 @@ void PrepareShutdown() // using the other before destroying them. if (peerLogic) UnregisterValidationInterface(peerLogic.get()); if (g_connman) g_connman->Stop(); - // if (g_txindex) g_txindex->Stop(); //TODO watch out when backporting bitcoin#13033 (don't accidently put the reset here, as we've already backported bitcoin#13894) + if (g_txindex) g_txindex->Stop(); StopTorControl(); @@ -290,7 +294,7 @@ void PrepareShutdown() // destruct and reset all to nullptr. peerLogic.reset(); g_connman.reset(); - //g_txindex.reset(); //TODO watch out when backporting bitcoin#13033 (re-enable this, was backported via bitcoin#13894) + g_txindex.reset(); if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); @@ -1961,9 +1965,10 @@ bool AppInitMain() int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache - int64_t nBlockTreeDBCache = nTotalCache / 8; - nBlockTreeDBCache = std::min(nBlockTreeDBCache, (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxBlockDBAndTxIndexCache : nMaxBlockDBCache) << 20); + int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20); nTotalCache -= nBlockTreeDBCache; + int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); + nTotalCache -= nTxIndexCache; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; @@ -1972,6 +1977,9 @@ bool AppInitMain() int64_t nEvoDbCache = 1024 * 1024 * 16; // TODO LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); + if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + LogPrintf("* Using %.1fMiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024)); + } LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); @@ -2015,9 +2023,8 @@ bool AppInitMain() if (fRequestShutdown) break; - // LoadBlockIndex will load fTxIndex from the db, or set it if - // we're reindexing. It will also load fHavePruned if we've - // ever removed a block file from disk. + // LoadBlockIndex will load fHavePruned if we've ever removed a + // block file from disk. // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something different! if (!LoadBlockIndex(chainparams)) { @@ -2025,7 +2032,7 @@ bool AppInitMain() break; } - if (!fDisableGovernance && !fTxIndex + if (!fDisableGovernance && !gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) && chainparams.NetworkIDString() != CBaseChainParams::REGTEST) { // TODO remove this when pruning is fixed. See https://github.com/dashpay/dash/pull/1817 and https://github.com/dashpay/dash/pull/1743 return InitError(_("Transaction index can't be disabled with governance validation enabled. Either start with -disablegovernance command line switch or enable transaction index.")); } @@ -2039,12 +2046,6 @@ bool AppInitMain() if (!chainparams.GetConsensus().hashDevnetGenesisBlock.IsNull() && !mapBlockIndex.empty() && mapBlockIndex.count(chainparams.GetConsensus().hashDevnetGenesisBlock) == 0) return InitError(_("Incorrect or no devnet genesis block found. Wrong datadir for devnet specified?")); - // Check for changed -txindex state - if (fTxIndex != gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - strLoadError = _("You need to rebuild the database using -reindex to change -txindex"); - break; - } - // Check for changed -addressindex state if (fAddressIndex != gArgs.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX)) { strLoadError = _("You need to rebuild the database using -reindex to change -addressindex"); @@ -2198,7 +2199,14 @@ bool AppInitMain() ::feeEstimator.Read(est_filein); fFeeEstimatesInitialized = true; - // ********************************************************* Step 8: load wallet + // ********************************************************* Step 8: start indexers + if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + auto txindex_db = MakeUnique(nTxIndexCache, false, fReindex); + g_txindex = MakeUnique(std::move(txindex_db)); + g_txindex->Start(); + } + + // ********************************************************* Step 9: load wallet if (!g_wallet_init_interface.Open()) return false; // As InitLoadWallet can take several minutes, it's possible the user @@ -2208,8 +2216,8 @@ bool AppInitMain() LogPrintf("Shutdown requested. Exiting.\n"); return false; } + // ********************************************************* Step 10: data directory maintenance - // ********************************************************* Step 9: data directory maintenance // if pruning, unset the service bit and perform the initial blockstore prune // after any wallet rescanning has taken place. diff --git a/src/llmq/quorums_instantsend.cpp b/src/llmq/quorums_instantsend.cpp index d439aa11abe01..f57aba35a8274 100644 --- a/src/llmq/quorums_instantsend.cpp +++ b/src/llmq/quorums_instantsend.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -644,6 +645,10 @@ void CInstantSendManager::HandleNewRecoveredSig(const CRecoveredSig& recoveredSi void CInstantSendManager::HandleNewInputLockRecoveredSig(const CRecoveredSig& recoveredSig, const uint256& txid) { + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hashBlock; if (!GetTransaction(txid, tx, Params().GetConsensus(), hashBlock, true)) { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 82904d6d55344..2024585012727 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1351,7 +1352,7 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) mempool.exists(inv.hash) || pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)) || - (fTxIndex && pblocktree->HasTxIndex(inv.hash)); + (g_txindex != nullptr && g_txindex->HasTx(inv.hash)); } case MSG_BLOCK: diff --git a/src/rest.cpp b/src/rest.cpp index 008dbf9a7bb41..27378f34c85a0 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -351,6 +352,10 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hashBlock = uint256(); if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 34f5791fcc44a..714e2645a7687 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,7 +14,7 @@ #include #include #include -// #include +#include #include #include #include @@ -1922,6 +1922,10 @@ static UniValue getblockstats(const JSONRPCRequest& request) ); } + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK(cs_main); CBlockIndex* pindex; @@ -2021,7 +2025,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) if (loop_inputs) { - if (!fTxIndex) { + if (g_txindex == nullptr) { throw JSONRPCError(RPC_INVALID_PARAMETER, "One or more of the selected stats requires -txindex enabled"); } CAmount tx_total_in = 0; diff --git a/src/rpc/governance.cpp b/src/rpc/governance.cpp index aeaa3d7baa44a..8adde3ee97284 100644 --- a/src/rpc/governance.cpp +++ b/src/rpc/governance.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -186,6 +187,10 @@ UniValue gobject_prepare(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Trigger objects need not be prepared (however only masternodes can create them)"); } + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK2(cs_main, mempool.cs); LOCK(pwallet->cs_wallet); @@ -362,6 +367,10 @@ UniValue gobject_submit(const JSONRPCRequest& request) std::string strError = ""; bool fMissingConfirmations; { + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK(cs_main); if (!govobj.IsValidLocally(strError, fMissingConfirmations, true) && !fMissingConfirmations) { LogPrintf("gobject(submit) -- Object submission rejected because object is not valid - hash = %s, strError = %s\n", strHash, strError); @@ -677,6 +686,10 @@ UniValue ListObjects(const std::string& strCachedSignal, const std::string& strT // GET MATCHING GOVERNANCE OBJECTS + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK2(cs_main, governance.cs); std::vector objs = governance.GetAllNewerThan(nStartTime); @@ -810,6 +823,10 @@ UniValue gobject_get(const JSONRPCRequest& request) // COLLECT VARIABLES FROM OUR USER uint256 hash = ParseHashV(request.params[1], "GovObj hash"); + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK2(cs_main, governance.cs); // FIND THE GOVERNANCE OBJECT THE USER IS LOOKING FOR diff --git a/src/rpc/masternode.cpp b/src/rpc/masternode.cpp index fbf4f95249c14..0ba6c768898e5 100644 --- a/src/rpc/masternode.cpp +++ b/src/rpc/masternode.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -385,6 +386,10 @@ UniValue masternode_payments(const JSONRPCRequest& request) CBlockIndex* pindex{nullptr}; + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + if (request.params[1].isNull()) { LOCK(cs_main); pindex = chainActive.Tip(); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 28f941e6f7872..024a0f3aeceda 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -77,6 +78,8 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) bool chainLock = false; if (!hashBlock.IsNull()) { + LOCK(cs_main); + entry.pushKV("blockhash", hashBlock.GetHex()); CBlockIndex* pindex = LookupBlockIndex(hashBlock); if (pindex) { @@ -182,8 +185,6 @@ UniValue getrawtransaction(const JSONRPCRequest& request) + HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"") ); - LOCK(cs_main); - bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); CBlockIndex* blockindex = nullptr; @@ -200,6 +201,8 @@ UniValue getrawtransaction(const JSONRPCRequest& request) } if (!request.params[2].isNull()) { + LOCK(cs_main); + uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); blockindex = LookupBlockIndex(blockhash); if (!blockindex) { @@ -208,6 +211,11 @@ UniValue getrawtransaction(const JSONRPCRequest& request) in_active_chain = chainActive.Contains(blockindex); } + bool f_txindex_ready = false; + if (g_txindex && !blockindex) { + f_txindex_ready = g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hash_block; if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, true, blockindex)) { @@ -217,10 +225,12 @@ UniValue getrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_MISC_ERROR, "Block not available"); } errmsg = "No such transaction found in the provided block"; + } else if (!g_txindex) { + errmsg = "No such mempool transaction. Use -txindex to enable blockchain transaction queries"; + } else if (!f_txindex_ready) { + errmsg = "No such mempool transaction. Blockchain transactions are still in the process of being indexed"; } else { - errmsg = fTxIndex - ? "No such mempool or blockchain transaction" - : "No such mempool transaction. Use -txindex to enable blockchain transaction queries"; + errmsg = "No such mempool or blockchain transaction"; } throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions."); } @@ -275,19 +285,18 @@ UniValue gettxoutproof(const JSONRPCRequest& request) oneTxid = hash; } - LOCK(cs_main); - CBlockIndex* pblockindex = nullptr; - uint256 hashBlock; - if (!request.params[1].isNull()) - { + if (!request.params[1].isNull()) { + LOCK(cs_main); hashBlock = uint256S(request.params[1].get_str()); pblockindex = LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } else { + LOCK(cs_main); + // Loop through txids and try to find which block they're in. Exit loop once a block is found. for (const auto& tx : setTxids) { const Coin& coin = AccessByTxid(*pcoinsTip, tx); @@ -298,6 +307,14 @@ UniValue gettxoutproof(const JSONRPCRequest& request) } } + + // Allow txindex to catch up if we need to query it and before we acquire cs_main. + if (g_txindex && !pblockindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + + LOCK(cs_main); + if (pblockindex == nullptr) { CTransactionRef tx; diff --git a/src/rpc/rpcevo.cpp b/src/rpc/rpcevo.cpp index 909d4e04e0e45..eb25b9c04cd40 100644 --- a/src/rpc/rpcevo.cpp +++ b/src/rpc/rpcevo.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -980,6 +981,10 @@ UniValue protx_list(const JSONRPCRequest& request) UniValue ret(UniValue::VARR); + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK(cs_main); if (type == "wallet") { @@ -1072,6 +1077,10 @@ UniValue protx_info(const JSONRPCRequest& request) CWallet* const pwallet = nullptr; #endif + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + uint256 proTxHash = ParseHashV(request.params[1], "proTxHash"); auto mnList = deterministicMNManager->GetListAtChainTip(); auto dmn = mnList.GetMN(proTxHash); diff --git a/src/rpc/rpcquorums.cpp b/src/rpc/rpcquorums.cpp index cad19c834c1dc..c873027f25336 100644 --- a/src/rpc/rpcquorums.cpp +++ b/src/rpc/rpcquorums.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include @@ -739,6 +740,10 @@ UniValue verifyislock(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "invalid signature format"); } + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + CBlockIndex* pindexMined{nullptr}; { LOCK(cs_main); diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index 2f2f6ab640460..3ac6ca4eece7c 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,8 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha connman = g_connman.get(); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); + g_txindex = MakeUnique(MakeUnique(1 << 20, true)); + g_txindex->Start(); llmq::InitLLMQSystem(*evoDb, true); pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); if (!LoadGenesisBlock(chainparams)) { @@ -128,6 +131,7 @@ TestingSetup::~TestingSetup() { llmq::InterruptLLMQSystem(); llmq::StopLLMQSystem(); + g_txindex.reset(); threadGroup.interrupt_all(); threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); @@ -152,6 +156,13 @@ TestChainSetup::TestChainSetup(int blockCount) : TestingSetup(CBaseChainParams:: CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey); m_coinbase_txns.push_back(b.vtx[0]); } + // Allow tx index to catch up with the block index. + constexpr int64_t timeout_ms = 10 * 1000; + int64_t time_start = GetTimeMillis(); + while (!g_txindex->BlockUntilSyncedToCurrentChain()) { + assert(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } } // diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp new file mode 100644 index 0000000000000..a46e59d447cb1 --- /dev/null +++ b/src/test/txindex_tests.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include