backport: merge bitcoin#23280 (coalesce chainstate loading sequence between {,non-}unittest codepaths)

src/Makefile.am

@@ -269,6 +269,8 @@ BITCOIN_CORE_H = \
   netgroup.h \
   netmessagemaker.h \
   node/blockstorage.h \
+  node/caches.h \
+  node/chainstate.h \
   node/coin.h \
   node/coinstats.h \
   node/connection_types.h \
@@ -501,6 +503,8 @@ libbitcoin_server_a_SOURCES = \
   netgroup.cpp \
   net_processing.cpp \
   node/blockstorage.cpp \
+  node/caches.cpp \
+  node/chainstate.cpp \
   node/coin.cpp \
   node/coinstats.cpp \
   node/connection_types.cpp \

src/node/caches.cpp

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <node/caches.h>
+
+#include <txdb.h>
+#include <util/system.h>
+#include <validation.h>
+
+CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
+{
+    int64_t nTotalCache = (args.GetArg("-dbcache", nDefaultDbCache) << 20);
+    nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
+    nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache
+    CacheSizes sizes;
+    sizes.block_tree_db = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
+    nTotalCache -= sizes.block_tree_db;
+    sizes.tx_index = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
+    nTotalCache -= sizes.tx_index;
+    sizes.filter_index = 0;
+    if (n_indexes > 0) {
+        int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
+        sizes.filter_index = max_cache / n_indexes;
+        nTotalCache -= sizes.filter_index * n_indexes;
+    }
+    sizes.coins_db = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
+    sizes.coins_db = std::min(sizes.coins_db, nMaxCoinsDBCache << 20); // cap total coins db cache
+    nTotalCache -= sizes.coins_db;
+    sizes.coins = nTotalCache; // the rest goes to in-memory cache
+    return sizes;
+}

src/node/caches.h

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_NODE_CACHES_H
+#define BITCOIN_NODE_CACHES_H
+
+#include <cstddef>
+#include <cstdint>
+
+class ArgsManager;
+
+struct CacheSizes {
+    int64_t block_tree_db;
+    int64_t coins_db;
+    int64_t coins;
+    int64_t tx_index;
+    int64_t filter_index;
+};
+CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0);
+
+#endif // BITCOIN_NODE_CACHES_H

src/node/chainstate.h

@@ -0,0 +1,162 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_NODE_CHAINSTATE_H
+#define BITCOIN_NODE_CHAINSTATE_H
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+
+class CActiveMasternodeManager;
+class CChainstateHelper;
+class CCreditPoolManager;
+class CDeterministicMNManager;
+class CEvoDB;
+class CGovernanceManager;
+class ChainstateManager;
+class CMasternodeMetaMan;
+class CMasternodeSync;
+class CMNHFManager;
+class CSporkManager;
+class CTxMemPool;
+struct LLMQContext;
+
+namespace llmq {
+class CChainLocksHandler;
+class CInstantSendManager;
+class CQuorumSnapshotManager;
+}
+
+namespace Consensus {
+struct Params;
+}
+
+enum class ChainstateLoadingError {
+    ERROR_LOADING_BLOCK_DB,
+    ERROR_BAD_GENESIS_BLOCK,
+    ERROR_BAD_DEVNET_GENESIS_BLOCK,
+    ERROR_TXINDEX_DISABLED_WHEN_GOV_ENABLED,
+    ERROR_ADDRIDX_NEEDS_REINDEX,
+    ERROR_SPENTIDX_NEEDS_REINDEX,
+    ERROR_TIMEIDX_NEEDS_REINDEX,
+    ERROR_PRUNED_NEEDS_REINDEX,
+    ERROR_LOAD_GENESIS_BLOCK_FAILED,
+    ERROR_CHAINSTATE_UPGRADE_FAILED,
+    ERROR_REPLAYBLOCKS_FAILED,
+    ERROR_LOADCHAINTIP_FAILED,
+    ERROR_GENERIC_BLOCKDB_OPEN_FAILED,
+    ERROR_COMMITING_EVO_DB,
+    ERROR_UPGRADING_EVO_DB,
+    ERROR_UPGRADING_SIGNALS_DB,
+    SHUTDOWN_PROBED,
+};
+
+/** This sequence can have 4 types of outcomes:
+ *
+ *  1. Success
+ *  2. Shutdown requested
+ *    - nothing failed but a shutdown was triggered in the middle of the
+ *      sequence
+ *  3. Soft failure
+ *    - a failure that might be recovered from with a reindex
+ *  4. Hard failure
+ *    - a failure that definitively cannot be recovered from with a reindex
+ *
+ *  Currently, LoadChainstate returns a std::optional<ChainstateLoadingError>
+ *  which:
+ *
+ *  - if has_value()
+ *      - Either "Soft failure", "Hard failure", or "Shutdown requested",
+ *        differentiable by the specific enumerator.
+ *
+ *        Note that a return value of SHUTDOWN_PROBED means ONLY that "during
+ *        this sequence, when we explicitly checked shutdown_requested() at
+ *        arbitrary points, one of those calls returned true". Therefore, a
+ *        return value other than SHUTDOWN_PROBED does not guarantee that
+ *        shutdown_requested() hasn't been called indirectly.
+ *  - else
+ *      - Success!
+ */
+std::optional<ChainstateLoadingError> LoadChainstate(bool fReset,
+                                                     ChainstateManager& chainman,
+                                                     CGovernanceManager& govman,
+                                                     CMasternodeMetaMan& mn_metaman,
+                                                     CMasternodeSync& mn_sync,
+                                                     CSporkManager& sporkman,
+                                                     std::unique_ptr<CActiveMasternodeManager>& mn_activeman,
+                                                     std::unique_ptr<CChainstateHelper>& chain_helper,
+                                                     std::unique_ptr<CCreditPoolManager>& cpoolman,
+                                                     std::unique_ptr<CDeterministicMNManager>& dmnman,
+                                                     std::unique_ptr<CEvoDB>& evodb,
+                                                     std::unique_ptr<CMNHFManager>& mnhf_manager,
+                                                     std::unique_ptr<llmq::CChainLocksHandler>& clhandler,
+                                                     std::unique_ptr<llmq::CInstantSendManager>& isman,
+                                                     std::unique_ptr<llmq::CQuorumSnapshotManager>& qsnapman,
+                                                     std::unique_ptr<LLMQContext>& llmq_ctx,
+                                                     CTxMemPool* mempool,
+                                                     bool fPruneMode,
+                                                     bool is_addrindex_enabled,
+                                                     bool is_governance_enabled,
+                                                     bool is_spentindex_enabled,
+                                                     bool is_timeindex_enabled,
+                                                     bool is_txindex_enabled,
+                                                     const Consensus::Params& consensus_params,
+                                                     const std::string& network_id,
+                                                     bool fReindexChainState,
+                                                     int64_t nBlockTreeDBCache,
+                                                     int64_t nCoinDBCache,
+                                                     int64_t nCoinCacheUsage,
+                                                     bool block_tree_db_in_memory,
+                                                     bool coins_db_in_memory,
+                                                     std::function<bool()> shutdown_requested = nullptr,
+                                                     std::function<void()> coins_error_cb = nullptr);
+
+/** Initialize Dash-specific components during chainstate initialization */
+void DashChainstateSetup(ChainstateManager& chainman,
+                         CGovernanceManager& govman,
+                         CMasternodeMetaMan& mn_metaman,
+                         CMasternodeSync& mn_sync,
+                         CSporkManager& sporkman,
+                         std::unique_ptr<CActiveMasternodeManager>& mn_activeman,
+                         std::unique_ptr<CChainstateHelper>& chain_helper,
+                         std::unique_ptr<CCreditPoolManager>& cpoolman,
+                         std::unique_ptr<CDeterministicMNManager>& dmnman,
+                         std::unique_ptr<CEvoDB>& evodb,
+                         std::unique_ptr<CMNHFManager>& mnhf_manager,
+                         std::unique_ptr<llmq::CQuorumSnapshotManager>& qsnapman,
+                         std::unique_ptr<LLMQContext>& llmq_ctx,
+                         CTxMemPool* mempool,
+                         bool fReset,
+                         bool fReindexChainState,
+                         const Consensus::Params& consensus_params);
+
+void DashChainstateSetupClose(std::unique_ptr<CChainstateHelper>& chain_helper,
+                              std::unique_ptr<CCreditPoolManager>& cpoolman,
+                              std::unique_ptr<CDeterministicMNManager>& dmnman,
+                              std::unique_ptr<CMNHFManager>& mnhf_manager,
+                              std::unique_ptr<llmq::CQuorumSnapshotManager>& qsnapman,
+                              std::unique_ptr<LLMQContext>& llmq_ctx,
+                              CTxMemPool* mempool);
+
+enum class ChainstateLoadVerifyError {
+    ERROR_BLOCK_FROM_FUTURE,
+    ERROR_CORRUPTED_BLOCK_DB,
+    ERROR_EVO_DB_SANITY_FAILED,
+    ERROR_GENERIC_FAILURE,
+};
+
+std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManager& chainman,
+                                                                CEvoDB& evodb,
+                                                                bool fReset,
+                                                                bool fReindexChainState,
+                                                                const Consensus::Params& consensus_params,
+                                                                unsigned int check_blocks,
+                                                                unsigned int check_level,
+                                                                std::function<int64_t()> get_unix_time_seconds,
+                                                                std::function<void(bool)> notify_bls_state = nullptr);
+
+#endif // BITCOIN_NODE_CHAINSTATE_H

src/rpc/blockchain.cpp

@@ -1642,7 +1642,7 @@ static RPCHelpMan verifychain()
 
     CChainState& active_chainstate = chainman.ActiveChainstate();
     return CVerifyDB().VerifyDB(
-        active_chainstate, Params(), active_chainstate.CoinsTip(), *CHECK_NONFATAL(node.evodb), check_level, check_depth);
+        active_chainstate, Params().GetConsensus(), active_chainstate.CoinsTip(), *CHECK_NONFATAL(node.evodb), check_level, check_depth);
 },
     };
 }

src/test/evo_deterministicmns_tests.cpp

@@ -816,8 +816,8 @@ void FuncVerifyDB(TestChainSetup& setup)
 
     // Verify db consistency
     LOCK(cs_main);
-    BOOST_REQUIRE(CVerifyDB().VerifyDB(chainman.ActiveChainstate(), Params(), chainman.ActiveChainstate().CoinsTip(),
-                                       *(setup.m_node.evodb), 4, 2));
+    BOOST_REQUIRE(CVerifyDB().VerifyDB(chainman.ActiveChainstate(), Params().GetConsensus(),
+                                       chainman.ActiveChainstate().CoinsTip(), *(setup.m_node.evodb), 4, 2));
 }
 
 BOOST_AUTO_TEST_SUITE(evo_dip3_activation_tests)