diff --git a/doc/files.md b/doc/files.md index ae5067b80fee7..164ca3ef9d25a 100644 --- a/doc/files.md +++ b/doc/files.md @@ -12,6 +12,7 @@ * evodb/*: special txes and quorums database * fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation * governance.dat: stores data for governance obgects +* indexes/txindex/*: optional transaction index database (LevelDB); since 0.17.0 * llmq/*: quorum signatures database * mempool.dat: dump of the mempool's transactions * mncache.dat: stores data for masternode list diff --git a/src/Makefile.am b/src/Makefile.am index c85d1b5c2b907..a1757c6a7229c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -173,6 +173,7 @@ BITCOIN_CORE_H = \ fs.h \ httprpc.h \ httpserver.h \ + index/txindex.h \ indirectmap.h \ init.h \ interfaces/handler.h \ @@ -315,6 +316,7 @@ libdash_server_a_SOURCES = \ evo/specialtx.cpp \ httprpc.cpp \ httpserver.cpp \ + index/txindex.cpp \ init.cpp \ dbwrapper.cpp \ governance/governance.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0ad3d24da0187..6fc8a1e4ce642 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -104,6 +104,7 @@ BITCOIN_TESTS =\ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ + test/txindex_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b058807a11b03..c795054686e86 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -245,6 +245,9 @@ class CDBWrapper CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); ~CDBWrapper(); + CDBWrapper(const CDBWrapper&) = delete; + CDBWrapper& operator=(const CDBWrapper&) = delete; + template bool ReadDataStream(const K& key, CDataStream& ssValue) const { diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp new file mode 100644 index 0000000000000..9ccbc95401840 --- /dev/null +++ b/src/index/txindex.cpp @@ -0,0 +1,315 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds +constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds + +std::unique_ptr g_txindex; + +template +static void FatalError(const char* fmt, const Args&... args) +{ + std::string strMessage = tfm::format(fmt, args...); + SetMiscWarning(strMessage); + LogPrintf("*** %s\n", strMessage); + uiInterface.ThreadSafeMessageBox( + "Error: A fatal internal error occurred, see debug.log for details", + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); +} + +TxIndex::TxIndex(std::unique_ptr db) : + m_db(std::move(db)), m_synced(false), m_best_block_index(nullptr) +{} + +TxIndex::~TxIndex() +{ + Interrupt(); + Stop(); +} + +bool TxIndex::Init() +{ + LOCK(cs_main); + + // Attempt to migrate txindex from the old database to the new one. Even if + // chain_tip is null, the node could be reindexing and we still want to + // delete txindex records in the old database. + if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) { + return false; + } + + CBlockLocator locator; + if (!m_db->ReadBestBlock(locator)) { + locator.SetNull(); + } + + m_best_block_index = FindForkInGlobalIndex(chainActive, locator); + m_synced = m_best_block_index.load() == chainActive.Tip(); + return true; +} + +static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev) +{ + AssertLockHeld(cs_main); + + if (!pindex_prev) { + return chainActive.Genesis(); + } + + const CBlockIndex* pindex = chainActive.Next(pindex_prev); + if (pindex) { + return pindex; + } + + return chainActive.Next(chainActive.FindFork(pindex_prev)); +} + +void TxIndex::ThreadSync() +{ + const CBlockIndex* pindex = m_best_block_index.load(); + if (!m_synced) { + auto& consensus_params = Params().GetConsensus(); + + int64_t last_log_time = 0; + int64_t last_locator_write_time = 0; + while (true) { + if (m_interrupt) { + WriteBestBlock(pindex); + return; + } + + { + LOCK(cs_main); + const CBlockIndex* pindex_next = NextSyncBlock(pindex); + if (!pindex_next) { + WriteBestBlock(pindex); + m_best_block_index = pindex; + m_synced = true; + break; + } + pindex = pindex_next; + } + + int64_t current_time = GetTime(); + if (last_log_time + SYNC_LOG_INTERVAL < current_time) { + LogPrintf("Syncing txindex with block chain from height %d\n", pindex->nHeight); + last_log_time = current_time; + } + + if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { + WriteBestBlock(pindex); + last_locator_write_time = current_time; + } + + CBlock block; + if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + FatalError("%s: Failed to read block %s from disk", + __func__, pindex->GetBlockHash().ToString()); + return; + } + if (!WriteBlock(block, pindex)) { + FatalError("%s: Failed to write block %s to tx index database", + __func__, pindex->GetBlockHash().ToString()); + return; + } + } + } + + if (pindex) { + LogPrintf("txindex is enabled at height %d\n", pindex->nHeight); + } else { + LogPrintf("txindex is enabled\n"); + } +} + +bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); + std::vector> vPos; + vPos.reserve(block.vtx.size()); + for (const auto& tx : block.vtx) { + vPos.emplace_back(tx->GetHash(), pos); + pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION); + } + return m_db->WriteTxs(vPos); +} + +bool TxIndex::WriteBestBlock(const CBlockIndex* block_index) +{ + LOCK(cs_main); + if (!m_db->WriteBestBlock(chainActive.GetLocator(block_index))) { + return error("%s: Failed to write locator to disk", __func__); + } + return true; +} + +void TxIndex::BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, + const std::vector& txn_conflicted) +{ + if (!m_synced) { + return; + } + + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (!best_block_index) { + if (pindex->nHeight != 0) { + FatalError("%s: First block connected is not the genesis block (height=%d)", + __func__, pindex->nHeight); + return; + } + } else { + // Ensure block connects to an ancestor of the current best block. This should be the case + // most of the time, but may not be immediately after the the sync thread catches up and sets + // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are + // in the ValidationInterface queue backlog even after the sync thread has caught up to the + // new chain tip. In this unlikely event, log a warning and let the queue clear. + if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { + LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ + "known best chain (tip=%s); not updating txindex\n", + __func__, pindex->GetBlockHash().ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + } + + if (WriteBlock(*block, pindex)) { + m_best_block_index = pindex; + } else { + FatalError("%s: Failed to write block %s to txindex", + __func__, pindex->GetBlockHash().ToString()); + return; + } +} + +void TxIndex::SetBestChain(const CBlockLocator& locator) +{ + if (!m_synced) { + return; + } + + const uint256& locator_tip_hash = locator.vHave.front(); + const CBlockIndex* locator_tip_index; + { + LOCK(cs_main); + locator_tip_index = LookupBlockIndex(locator_tip_hash); + } + + if (!locator_tip_index) { + FatalError("%s: First block (hash=%s) in locator was not found", + __func__, locator_tip_hash.ToString()); + return; + } + + // This checks that SetBestChain callbacks are received after BlockConnected. The check may fail + // immediately after the the sync thread catches up and sets m_synced. Consider the case where + // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue + // backlog even after the sync thread has caught up to the new chain tip. In this unlikely + // event, log a warning and let the queue clear. + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { + LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ + "chain (tip=%s); not writing txindex locator\n", + __func__, locator_tip_hash.ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + + if (!m_db->WriteBestBlock(locator)) { + error("%s: Failed to write locator to disk", __func__); + } +} + +bool TxIndex::BlockUntilSyncedToCurrentChain() +{ + AssertLockNotHeld(cs_main); + + if (!m_synced) { + return false; + } + + { + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip(). + LOCK(cs_main); + const CBlockIndex* chain_tip = chainActive.Tip(); + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) { + return true; + } + } + + LogPrintf("%s: txindex is catching up on block notifications\n", __func__); + SyncWithValidationInterfaceQueue(); + return true; +} + +bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const +{ + CDiskTxPos postx; + if (!m_db->ReadTxPos(tx_hash, postx)) { + return false; + } + + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + return error("%s: OpenBlockFile failed", __func__); + } + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> tx; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + if (tx->GetHash() != tx_hash) { + return error("%s: txid mismatch", __func__); + } + block_hash = header.GetHash(); + return true; +} + +bool TxIndex::HasTx(const uint256& tx_hash) const +{ + CDiskTxPos postx; + return m_db->ReadTxPos(tx_hash, postx); +} + +void TxIndex::Interrupt() +{ + m_interrupt(); +} + +void TxIndex::Start() +{ + // Need to register this ValidationInterface before running Init(), so that + // callbacks are not missed if Init sets m_synced to true. + RegisterValidationInterface(this); + if (!Init()) { + FatalError("%s: txindex failed to initialize", __func__); + return; + } + + m_thread_sync = std::thread(&TraceThread>, "txindex", + std::bind(&TxIndex::ThreadSync, this)); +} + +void TxIndex::Stop() +{ + UnregisterValidationInterface(this); + + if (m_thread_sync.joinable()) { + m_thread_sync.join(); + } +} diff --git a/src/index/txindex.h b/src/index/txindex.h new file mode 100644 index 0000000000000..eb854763ad34d --- /dev/null +++ b/src/index/txindex.h @@ -0,0 +1,95 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_TXINDEX_H +#define BITCOIN_INDEX_TXINDEX_H + +#include +#include +#include +#include +#include +#include + +class CBlockIndex; + +/** + * TxIndex is used to look up transactions included in the blockchain by hash. + * The index is written to a LevelDB database and records the filesystem + * location of each transaction by transaction hash. + */ +class TxIndex final : public CValidationInterface +{ +private: + const std::unique_ptr m_db; + + /// Whether the index is in sync with the main chain. The flag is flipped + /// from false to true once, after which point this starts processing + /// ValidationInterface notifications to stay in sync. + std::atomic m_synced; + + /// The last block in the chain that the TxIndex is in sync with. + std::atomic m_best_block_index; + + std::thread m_thread_sync; + CThreadInterrupt m_interrupt; + + /// Initialize internal state from the database and block index. + bool Init(); + + /// Sync the tx index with the block index starting from the current best + /// block. Intended to be run in its own thread, m_thread_sync, and can be + /// interrupted with m_interrupt. Once the txindex gets in sync, the + /// m_synced flag is set and the BlockConnected ValidationInterface callback + /// takes over and the sync thread exits. + void ThreadSync(); + + /// Write update index entries for a newly connected block. + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex); + + /// Write the current chain block locator to the DB. + bool WriteBestBlock(const CBlockIndex* block_index); + +protected: + void BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, + const std::vector& txn_conflicted) override; + + void SetBestChain(const CBlockLocator& locator) override; + +public: + /// Constructs the TxIndex, which becomes available to be queried. + explicit TxIndex(std::unique_ptr db); + + /// Destructor interrupts sync thread if running and blocks until it exits. + ~TxIndex(); + + /// Blocks the current thread until the transaction index is caught up to + /// the current state of the block chain. This only blocks if the index has gotten in sync once + /// and only needs to process blocks in the ValidationInterface queue. If the index is catching + /// up from far behind, this method does not block and immediately returns false. + bool BlockUntilSyncedToCurrentChain(); + + /// Look up a transaction by hash. + /// + /// @param[in] tx_hash The hash of the transaction to be returned. + /// @param[out] block_hash The hash of the block the transaction is found in. + /// @param[out] tx The transaction itself. + /// @return true if transaction is found, false otherwise + bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const; + bool HasTx(const uint256& tx_hash) const; + + void Interrupt(); + + /// Start initializes the sync state and registers the instance as a + /// ValidationInterface so that it stays in sync with blockchain updates. + void Start(); + + /// Stops the instance from staying in sync with blockchain updates. + void Stop(); +}; + +/// The global transaction index, used in GetTransaction. May be null. +extern std::unique_ptr g_txindex; + +#endif // BITCOIN_INDEX_TXINDEX_H diff --git a/src/init.cpp b/src/init.cpp index a7f544d4f1fd7..3a6cc0ba8c93f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -224,6 +225,9 @@ void Interrupt() InterruptMapPort(); if (g_connman) g_connman->Interrupt(); + if (g_txindex) { + g_txindex->Interrupt(); + } } /** Preparing steps before shutting down or restarting the wallet */ @@ -259,7 +263,7 @@ void PrepareShutdown() // using the other before destroying them. if (peerLogic) UnregisterValidationInterface(peerLogic.get()); if (g_connman) g_connman->Stop(); - // if (g_txindex) g_txindex->Stop(); //TODO watch out when backporting bitcoin#13033 (don't accidently put the reset here, as we've already backported bitcoin#13894) + if (g_txindex) g_txindex->Stop(); StopTorControl(); @@ -290,7 +294,7 @@ void PrepareShutdown() // destruct and reset all to nullptr. peerLogic.reset(); g_connman.reset(); - //g_txindex.reset(); //TODO watch out when backporting bitcoin#13033 (re-enable this, was backported via bitcoin#13894) + g_txindex.reset(); if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); @@ -1961,9 +1965,10 @@ bool AppInitMain() int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache - int64_t nBlockTreeDBCache = nTotalCache / 8; - nBlockTreeDBCache = std::min(nBlockTreeDBCache, (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxBlockDBAndTxIndexCache : nMaxBlockDBCache) << 20); + int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20); nTotalCache -= nBlockTreeDBCache; + int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); + nTotalCache -= nTxIndexCache; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; @@ -1972,6 +1977,9 @@ bool AppInitMain() int64_t nEvoDbCache = 1024 * 1024 * 16; // TODO LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); + if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + LogPrintf("* Using %.1fMiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024)); + } LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); @@ -2015,9 +2023,8 @@ bool AppInitMain() if (fRequestShutdown) break; - // LoadBlockIndex will load fTxIndex from the db, or set it if - // we're reindexing. It will also load fHavePruned if we've - // ever removed a block file from disk. + // LoadBlockIndex will load fHavePruned if we've ever removed a + // block file from disk. // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something different! if (!LoadBlockIndex(chainparams)) { @@ -2025,7 +2032,7 @@ bool AppInitMain() break; } - if (!fDisableGovernance && !fTxIndex + if (!fDisableGovernance && !gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) && chainparams.NetworkIDString() != CBaseChainParams::REGTEST) { // TODO remove this when pruning is fixed. See https://github.com/dashpay/dash/pull/1817 and https://github.com/dashpay/dash/pull/1743 return InitError(_("Transaction index can't be disabled with governance validation enabled. Either start with -disablegovernance command line switch or enable transaction index.")); } @@ -2039,12 +2046,6 @@ bool AppInitMain() if (!chainparams.GetConsensus().hashDevnetGenesisBlock.IsNull() && !mapBlockIndex.empty() && mapBlockIndex.count(chainparams.GetConsensus().hashDevnetGenesisBlock) == 0) return InitError(_("Incorrect or no devnet genesis block found. Wrong datadir for devnet specified?")); - // Check for changed -txindex state - if (fTxIndex != gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - strLoadError = _("You need to rebuild the database using -reindex to change -txindex"); - break; - } - // Check for changed -addressindex state if (fAddressIndex != gArgs.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX)) { strLoadError = _("You need to rebuild the database using -reindex to change -addressindex"); @@ -2198,7 +2199,14 @@ bool AppInitMain() ::feeEstimator.Read(est_filein); fFeeEstimatesInitialized = true; - // ********************************************************* Step 8: load wallet + // ********************************************************* Step 8: start indexers + if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + auto txindex_db = MakeUnique(nTxIndexCache, false, fReindex); + g_txindex = MakeUnique(std::move(txindex_db)); + g_txindex->Start(); + } + + // ********************************************************* Step 9: load wallet if (!g_wallet_init_interface.Open()) return false; // As InitLoadWallet can take several minutes, it's possible the user @@ -2208,8 +2216,8 @@ bool AppInitMain() LogPrintf("Shutdown requested. Exiting.\n"); return false; } + // ********************************************************* Step 10: data directory maintenance - // ********************************************************* Step 9: data directory maintenance // if pruning, unset the service bit and perform the initial blockstore prune // after any wallet rescanning has taken place. diff --git a/src/llmq/quorums_instantsend.cpp b/src/llmq/quorums_instantsend.cpp index d439aa11abe01..f57aba35a8274 100644 --- a/src/llmq/quorums_instantsend.cpp +++ b/src/llmq/quorums_instantsend.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -644,6 +645,10 @@ void CInstantSendManager::HandleNewRecoveredSig(const CRecoveredSig& recoveredSi void CInstantSendManager::HandleNewInputLockRecoveredSig(const CRecoveredSig& recoveredSig, const uint256& txid) { + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hashBlock; if (!GetTransaction(txid, tx, Params().GetConsensus(), hashBlock, true)) { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 82904d6d55344..2024585012727 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1351,7 +1352,7 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) mempool.exists(inv.hash) || pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)) || - (fTxIndex && pblocktree->HasTxIndex(inv.hash)); + (g_txindex != nullptr && g_txindex->HasTx(inv.hash)); } case MSG_BLOCK: diff --git a/src/rest.cpp b/src/rest.cpp index 008dbf9a7bb41..27378f34c85a0 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -351,6 +352,10 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) if (!ParseHashStr(hashStr, hash)) return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hashBlock = uint256(); if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 34f5791fcc44a..714e2645a7687 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,7 +14,7 @@ #include #include #include -// #include +#include #include #include #include @@ -1922,6 +1922,10 @@ static UniValue getblockstats(const JSONRPCRequest& request) ); } + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK(cs_main); CBlockIndex* pindex; @@ -2021,7 +2025,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) if (loop_inputs) { - if (!fTxIndex) { + if (g_txindex == nullptr) { throw JSONRPCError(RPC_INVALID_PARAMETER, "One or more of the selected stats requires -txindex enabled"); } CAmount tx_total_in = 0; diff --git a/src/rpc/governance.cpp b/src/rpc/governance.cpp index aeaa3d7baa44a..8adde3ee97284 100644 --- a/src/rpc/governance.cpp +++ b/src/rpc/governance.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -186,6 +187,10 @@ UniValue gobject_prepare(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Trigger objects need not be prepared (however only masternodes can create them)"); } + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK2(cs_main, mempool.cs); LOCK(pwallet->cs_wallet); @@ -362,6 +367,10 @@ UniValue gobject_submit(const JSONRPCRequest& request) std::string strError = ""; bool fMissingConfirmations; { + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK(cs_main); if (!govobj.IsValidLocally(strError, fMissingConfirmations, true) && !fMissingConfirmations) { LogPrintf("gobject(submit) -- Object submission rejected because object is not valid - hash = %s, strError = %s\n", strHash, strError); @@ -677,6 +686,10 @@ UniValue ListObjects(const std::string& strCachedSignal, const std::string& strT // GET MATCHING GOVERNANCE OBJECTS + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK2(cs_main, governance.cs); std::vector objs = governance.GetAllNewerThan(nStartTime); @@ -810,6 +823,10 @@ UniValue gobject_get(const JSONRPCRequest& request) // COLLECT VARIABLES FROM OUR USER uint256 hash = ParseHashV(request.params[1], "GovObj hash"); + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK2(cs_main, governance.cs); // FIND THE GOVERNANCE OBJECT THE USER IS LOOKING FOR diff --git a/src/rpc/masternode.cpp b/src/rpc/masternode.cpp index fbf4f95249c14..0ba6c768898e5 100644 --- a/src/rpc/masternode.cpp +++ b/src/rpc/masternode.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -385,6 +386,10 @@ UniValue masternode_payments(const JSONRPCRequest& request) CBlockIndex* pindex{nullptr}; + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + if (request.params[1].isNull()) { LOCK(cs_main); pindex = chainActive.Tip(); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 28f941e6f7872..024a0f3aeceda 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -77,6 +78,8 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) bool chainLock = false; if (!hashBlock.IsNull()) { + LOCK(cs_main); + entry.pushKV("blockhash", hashBlock.GetHex()); CBlockIndex* pindex = LookupBlockIndex(hashBlock); if (pindex) { @@ -182,8 +185,6 @@ UniValue getrawtransaction(const JSONRPCRequest& request) + HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"") ); - LOCK(cs_main); - bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); CBlockIndex* blockindex = nullptr; @@ -200,6 +201,8 @@ UniValue getrawtransaction(const JSONRPCRequest& request) } if (!request.params[2].isNull()) { + LOCK(cs_main); + uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); blockindex = LookupBlockIndex(blockhash); if (!blockindex) { @@ -208,6 +211,11 @@ UniValue getrawtransaction(const JSONRPCRequest& request) in_active_chain = chainActive.Contains(blockindex); } + bool f_txindex_ready = false; + if (g_txindex && !blockindex) { + f_txindex_ready = g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hash_block; if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, true, blockindex)) { @@ -217,10 +225,12 @@ UniValue getrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_MISC_ERROR, "Block not available"); } errmsg = "No such transaction found in the provided block"; + } else if (!g_txindex) { + errmsg = "No such mempool transaction. Use -txindex to enable blockchain transaction queries"; + } else if (!f_txindex_ready) { + errmsg = "No such mempool transaction. Blockchain transactions are still in the process of being indexed"; } else { - errmsg = fTxIndex - ? "No such mempool or blockchain transaction" - : "No such mempool transaction. Use -txindex to enable blockchain transaction queries"; + errmsg = "No such mempool or blockchain transaction"; } throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions."); } @@ -275,19 +285,18 @@ UniValue gettxoutproof(const JSONRPCRequest& request) oneTxid = hash; } - LOCK(cs_main); - CBlockIndex* pblockindex = nullptr; - uint256 hashBlock; - if (!request.params[1].isNull()) - { + if (!request.params[1].isNull()) { + LOCK(cs_main); hashBlock = uint256S(request.params[1].get_str()); pblockindex = LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } else { + LOCK(cs_main); + // Loop through txids and try to find which block they're in. Exit loop once a block is found. for (const auto& tx : setTxids) { const Coin& coin = AccessByTxid(*pcoinsTip, tx); @@ -298,6 +307,14 @@ UniValue gettxoutproof(const JSONRPCRequest& request) } } + + // Allow txindex to catch up if we need to query it and before we acquire cs_main. + if (g_txindex && !pblockindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + + LOCK(cs_main); + if (pblockindex == nullptr) { CTransactionRef tx; diff --git a/src/rpc/rpcevo.cpp b/src/rpc/rpcevo.cpp index 909d4e04e0e45..eb25b9c04cd40 100644 --- a/src/rpc/rpcevo.cpp +++ b/src/rpc/rpcevo.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -980,6 +981,10 @@ UniValue protx_list(const JSONRPCRequest& request) UniValue ret(UniValue::VARR); + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + LOCK(cs_main); if (type == "wallet") { @@ -1072,6 +1077,10 @@ UniValue protx_info(const JSONRPCRequest& request) CWallet* const pwallet = nullptr; #endif + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + uint256 proTxHash = ParseHashV(request.params[1], "proTxHash"); auto mnList = deterministicMNManager->GetListAtChainTip(); auto dmn = mnList.GetMN(proTxHash); diff --git a/src/rpc/rpcquorums.cpp b/src/rpc/rpcquorums.cpp index cad19c834c1dc..c873027f25336 100644 --- a/src/rpc/rpcquorums.cpp +++ b/src/rpc/rpcquorums.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include @@ -739,6 +740,10 @@ UniValue verifyislock(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "invalid signature format"); } + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + CBlockIndex* pindexMined{nullptr}; { LOCK(cs_main); diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index 2f2f6ab640460..3ac6ca4eece7c 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,8 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha connman = g_connman.get(); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); + g_txindex = MakeUnique(MakeUnique(1 << 20, true)); + g_txindex->Start(); llmq::InitLLMQSystem(*evoDb, true); pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); if (!LoadGenesisBlock(chainparams)) { @@ -128,6 +131,7 @@ TestingSetup::~TestingSetup() { llmq::InterruptLLMQSystem(); llmq::StopLLMQSystem(); + g_txindex.reset(); threadGroup.interrupt_all(); threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); @@ -152,6 +156,13 @@ TestChainSetup::TestChainSetup(int blockCount) : TestingSetup(CBaseChainParams:: CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey); m_coinbase_txns.push_back(b.vtx[0]); } + // Allow tx index to catch up with the block index. + constexpr int64_t timeout_ms = 10 * 1000; + int64_t time_start = GetTimeMillis(); + while (!g_txindex->BlockUntilSyncedToCurrentChain()) { + assert(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } } // diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp new file mode 100644 index 0000000000000..a46e59d447cb1 --- /dev/null +++ b/src/test/txindex_tests.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include