Skip to content

perf: use cache for SML #6738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jul 22, 2025
Merged

perf: use cache for SML #6738

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
e8949c8
refactor: move CSimplifiedMNListDiffs::ToJson to evo/core_write
knst Jun 27, 2025
c2e053f
refactor: move diff related code from evo/simplifiedmns to new module…
knst Jun 27, 2025
bbd8731
fmt: apply clang-format suggestions
knst Jul 1, 2025
3b4a1a0
feat: new GetSML method for CDeterministicMNList
knst Jun 23, 2025
7bed080
perf: cache shared_ptr to SML instead of SML list
knst Jun 22, 2025
63031da
refactor: drop dependency SML on DMN
knst Jun 30, 2025
83c667e
refactor: update comment for SML cache population
PastaPastaPasta Jul 1, 2025
53190a5
refactor: enhance thread safety for SML caching with mutex protection
PastaPastaPasta Jul 1, 2025
d61f44e
refactor: centralize SML cache invalidation logic
PastaPastaPasta Jul 6, 2025
d0a2791
test: add comprehensive unit tests for SML caching mechanism
PastaPastaPasta Jul 6, 2025
b75e640
refactor: add gsl::not_null annotation for cached_sml and CalcCbTxMer…
knst Jul 16, 2025
8a68ec6
fix: remove duplicated check for hash existance in map, avoid hash re…
knst Jul 19, 2025
bc01e96
fmt: re-order include accorindgly to clang formatter
knst Jul 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ BITCOIN_CORE_H = \
evo/netinfo.h \
evo/providertx.h \
evo/simplifiedmns.h \
evo/smldiff.h \
evo/specialtx.h \
evo/specialtxman.h \
dsnotificationinterface.h \
Expand Down Expand Up @@ -472,6 +473,7 @@ libbitcoin_node_a_SOURCES = \
evo/mnhftx.cpp \
evo/providertx.cpp \
evo/simplifiedmns.cpp \
evo/smldiff.cpp \
evo/specialtx.cpp \
evo/specialtxman.cpp \
flatfile.cpp \
Expand Down
2 changes: 2 additions & 0 deletions src/evo/cbtx.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
class BlockValidationState;
class CBlock;
class CBlockIndex;
class CDeterministicMNList;
class TxValidationState;

namespace llmq {
class CQuorumBlockProcessor;
}// namespace llmq
Expand Down
96 changes: 96 additions & 0 deletions src/evo/core_write.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
#include <evo/mnhftx.h>
#include <evo/netinfo.h>
#include <evo/providertx.h>
#include <evo/simplifiedmns.h>
#include <evo/smldiff.h>
#include <llmq/commitment.h>

#include <univalue.h>
Expand Down Expand Up @@ -149,3 +151,97 @@
ret.pushKV("commitment", commitment.ToJson());
return ret;
}

[[nodiscard]] UniValue CSimplifiedMNListEntry::ToJson(bool extended) const
{
UniValue obj(UniValue::VOBJ);
obj.pushKV("nVersion", nVersion);
obj.pushKV("nType", ToUnderlying(nType));
obj.pushKV("proRegTxHash", proRegTxHash.ToString());
obj.pushKV("confirmedHash", confirmedHash.ToString());
if (IsServiceDeprecatedRPCEnabled()) {
obj.pushKV("service", netInfo->GetPrimary().ToStringAddrPort());
}
obj.pushKV("addresses", netInfo->ToJson());
obj.pushKV("pubKeyOperator", pubKeyOperator.ToString());
obj.pushKV("votingAddress", EncodeDestination(PKHash(keyIDVoting)));
obj.pushKV("isValid", isValid);
if (nType == MnType::Evo) {
obj.pushKV("platformHTTPPort", platformHTTPPort);
obj.pushKV("platformNodeID", platformNodeID.ToString());
}

if (extended) {
CTxDestination dest;
if (ExtractDestination(scriptPayout, dest)) {
obj.pushKV("payoutAddress", EncodeDestination(dest));
}
if (ExtractDestination(scriptOperatorPayout, dest)) {
obj.pushKV("operatorPayoutAddress", EncodeDestination(dest));
}
}
return obj;
}

[[nodiscard]] UniValue CSimplifiedMNListDiff::ToJson(bool extended) const
{
UniValue obj(UniValue::VOBJ);

obj.pushKV("nVersion", nVersion);
obj.pushKV("baseBlockHash", baseBlockHash.ToString());
obj.pushKV("blockHash", blockHash.ToString());

CDataStream ssCbTxMerkleTree(SER_NETWORK, PROTOCOL_VERSION);
ssCbTxMerkleTree << cbTxMerkleTree;
obj.pushKV("cbTxMerkleTree", HexStr(ssCbTxMerkleTree));

obj.pushKV("cbTx", EncodeHexTx(*cbTx));

UniValue deletedMNsArr(UniValue::VARR);
for (const auto& h : deletedMNs) {
deletedMNsArr.push_back(h.ToString());
}
obj.pushKV("deletedMNs", deletedMNsArr);

UniValue mnListArr(UniValue::VARR);
for (const auto& e : mnList) {
mnListArr.push_back(e.ToJson(extended));
}
obj.pushKV("mnList", mnListArr);

UniValue deletedQuorumsArr(UniValue::VARR);
for (const auto& e : deletedQuorums) {
UniValue eObj(UniValue::VOBJ);
eObj.pushKV("llmqType", e.first);
eObj.pushKV("quorumHash", e.second.ToString());
deletedQuorumsArr.push_back(eObj);
}
obj.pushKV("deletedQuorums", deletedQuorumsArr);

UniValue newQuorumsArr(UniValue::VARR);
for (const auto& e : newQuorums) {
newQuorumsArr.push_back(e.ToJson());
}
obj.pushKV("newQuorums", newQuorumsArr);

// Do not assert special tx type here since this can be called prior to DIP0003 activation
if (const auto opt_cbTxPayload = GetTxPayload<CCbTx>(*cbTx, /*assert_type=*/false)) {
obj.pushKV("merkleRootMNList", opt_cbTxPayload->merkleRootMNList.ToString());
if (opt_cbTxPayload->nVersion >= CCbTx::Version::MERKLE_ROOT_QUORUMS) {
obj.pushKV("merkleRootQuorums", opt_cbTxPayload->merkleRootQuorums.ToString());
}
}

UniValue quorumsCLSigsArr(UniValue::VARR);
for (const auto& [signature, quorumsIndexes] : quorumsCLSigs) {
UniValue j(UniValue::VOBJ);
UniValue idxArr(UniValue::VARR);
for (const auto& idx : quorumsIndexes) {
idxArr.push_back(idx);
}
j.pushKV(signature.ToString(), idxArr);
quorumsCLSigsArr.push_back(j);
}
obj.pushKV("quorumsCLSigs", quorumsCLSigsArr);
return obj;
}
39 changes: 38 additions & 1 deletion src/evo/deterministicmns.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <evo/dmnstate.h>
#include <evo/evodb.h>
#include <evo/providertx.h>
#include <evo/simplifiedmns.h>
#include <evo/specialtx.h>
#include <index/txindex.h>

Expand Down Expand Up @@ -36,6 +37,14 @@ uint64_t CDeterministicMN::GetInternalId() const
return internalId;
}

CSimplifiedMNListEntry CDeterministicMN::to_sml_entry() const
{
const CDeterministicMNState& state{*pdmnState};
return CSimplifiedMNListEntry(proTxHash, state.confirmedHash, state.netInfo, state.pubKeyOperator,
state.keyIDVoting, !state.IsBanned(), state.platformHTTPPort, state.platformNodeID,
state.scriptPayout, state.scriptOperatorPayout, state.nVersion, nType);
}

std::string CDeterministicMN::ToString() const
{
return strprintf("CDeterministicMN(proTxHash=%s, collateralOutpoint=%s, nOperatorReward=%f, state=%s", proTxHash.ToString(), collateralOutpoint.ToStringShort(), (double)nOperatorReward / 100, pdmnState->ToString());
Expand Down Expand Up @@ -258,6 +267,22 @@ std::vector<CDeterministicMNCPtr> CDeterministicMNList::GetProjectedMNPayees(gsl
return result;
}

gsl::not_null<std::shared_ptr<const CSimplifiedMNList>> CDeterministicMNList::to_sml() const
{
LOCK(m_cached_sml_mutex);
if (!m_cached_sml) {
std::vector<std::unique_ptr<CSimplifiedMNListEntry>> sml_entries;
sml_entries.reserve(mnMap.size());

ForEachMN(false, [&sml_entries](auto& dmn) {
sml_entries.emplace_back(std::make_unique<CSimplifiedMNListEntry>(dmn.to_sml_entry()));
});
m_cached_sml = std::make_shared<CSimplifiedMNList>(std::move(sml_entries));
}

return m_cached_sml;
}
Comment on lines +270 to +284
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Consider using immutable data structures for thread safety.

According to the coding guidelines, masternode lists must use immutable data structures (Immer library) for thread safety. The current implementation uses std::mutex for synchronization, which may not align with the required approach.

Consider refactoring to use Immer's immutable data structures instead of mutex-based synchronization to comply with the coding guidelines for the src/{masternode,evo}/** modules.

🤖 Prompt for AI Agents 
In src/evo/deterministicmns.cpp around lines 270 to 284, the current to_sml()
method uses a mutex to protect the cached masternode list, but the coding
guidelines require using Immer immutable data structures for thread safety
instead of mutexes. Refactor the caching mechanism to store and update the
masternode list using Immer's immutable containers, eliminating the need for
explicit locking and ensuring thread-safe access according to the project's
standards.


int CDeterministicMNList::CalcMaxPoSePenalty() const
{
// Maximum PoSe penalty is dynamic and equals the number of registered MNs
Expand Down Expand Up @@ -443,6 +468,7 @@ void CDeterministicMNList::AddMN(const CDeterministicMNCPtr& dmn, bool fBumpTota

mnMap = mnMap.set(dmn->proTxHash, dmn);
mnInternalIdMap = mnInternalIdMap.set(dmn->GetInternalId(), dmn->proTxHash);
InvalidateSMLCache();
if (fBumpTotalCount) {
// nTotalRegisteredCount acts more like a checkpoint, not as a limit,
nTotalRegisteredCount = std::max(dmn->GetInternalId() + 1, (uint64_t)nTotalRegisteredCount);
Expand Down Expand Up @@ -514,6 +540,10 @@ void CDeterministicMNList::UpdateMN(const CDeterministicMN& oldDmn, const std::s

dmn->pdmnState = pdmnState;
mnMap = mnMap.set(oldDmn.proTxHash, dmn);
LOCK(m_cached_sml_mutex);
if (m_cached_sml && oldDmn.to_sml_entry() != dmn->to_sml_entry()) {
m_cached_sml = nullptr;
}
}

void CDeterministicMNList::UpdateMN(const uint256& proTxHash, const std::shared_ptr<const CDeterministicMNState>& pdmnState)
Expand Down Expand Up @@ -585,6 +615,7 @@ void CDeterministicMNList::RemoveMN(const uint256& proTxHash)

mnMap = mnMap.erase(proTxHash);
mnInternalIdMap = mnInternalIdMap.erase(dmn->GetInternalId());
InvalidateSMLCache();
}

bool CDeterministicMNManager::ProcessBlock(const CBlock& block, gsl::not_null<const CBlockIndex*> pindex,
Expand All @@ -604,6 +635,8 @@ bool CDeterministicMNManager::ProcessBlock(const CBlock& block, gsl::not_null<co
int nHeight = pindex->nHeight;

try {
newList.to_sml(); // to populate the SML cache

LOCK(cs);

oldList = GetListForBlockInternal(pindex->pprev);
Expand All @@ -619,6 +652,7 @@ bool CDeterministicMNManager::ProcessBlock(const CBlock& block, gsl::not_null<co

diff.nHeight = pindex->nHeight;
mnListDiffsCache.emplace(pindex->GetBlockHash(), diff);
mnListsCache.emplace(newList.GetBlockHash(), newList);
} catch (const std::exception& e) {
LogPrintf("CDeterministicMNManager::%s -- internal error: %s\n", __func__, e.what());
return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "failed-dmn-block");
Expand Down Expand Up @@ -752,7 +786,10 @@ CDeterministicMNList CDeterministicMNManager::GetListForBlockInternal(gsl::not_n
if (tipIndex) {
// always keep a snapshot for the tip
if (snapshot.GetBlockHash() == tipIndex->GetBlockHash()) {
mnListsCache.emplace(snapshot.GetBlockHash(), snapshot);
auto hash = snapshot.GetBlockHash();
if (mnListsCache.find(hash) == mnListsCache.end()) {
mnListsCache.emplace(snapshot.GetBlockHash(), snapshot);
}
} else {
// keep snapshots for yet alive quorums
if (ranges::any_of(Params().GetConsensus().llmqs,
Expand Down
56 changes: 56 additions & 0 deletions src/evo/deterministicmns.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ class CBlock;
class CBlockIndex;
class CCoinsViewCache;
class CEvoDB;
class CSimplifiedMNList;
class CSimplifiedMNListEntry;
class TxValidationState;

extern RecursiveMutex cs_main;
Expand Down Expand Up @@ -79,6 +81,7 @@ class CDeterministicMN

[[nodiscard]] uint64_t GetInternalId() const;

[[nodiscard]] CSimplifiedMNListEntry to_sml_entry() const;
[[nodiscard]] std::string ToString() const;
[[nodiscard]] UniValue ToJson() const;
};
Expand Down Expand Up @@ -145,6 +148,22 @@ class CDeterministicMNList
// we keep track of this as checking for duplicates would otherwise be painfully slow
MnUniquePropertyMap mnUniquePropertyMap;

// This SML could be null
// This cache is used to improve performance and meant to be reused
// for multiple CDeterministicMNList until mnMap is actually changed.
// Calls of AddMN, RemoveMN and (in some cases) UpdateMN reset this cache;
// it happens also for indirect calls such as ApplyDiff
// Thread safety: Protected by its own mutex for thread-safe access
mutable Mutex m_cached_sml_mutex;
mutable std::shared_ptr<const CSimplifiedMNList> m_cached_sml GUARDED_BY(m_cached_sml_mutex);

// Private helper method to invalidate SML cache
void InvalidateSMLCache()
{
LOCK(m_cached_sml_mutex);
m_cached_sml = nullptr;
}

public:
CDeterministicMNList() = default;
explicit CDeterministicMNList(const uint256& _blockHash, int _height, uint32_t _totalRegisteredCount) :
Expand All @@ -155,6 +174,36 @@ class CDeterministicMNList
assert(nHeight >= 0);
}

// Copy constructor
CDeterministicMNList(const CDeterministicMNList& other) :
blockHash(other.blockHash),
nHeight(other.nHeight),
nTotalRegisteredCount(other.nTotalRegisteredCount),
mnMap(other.mnMap),
mnInternalIdMap(other.mnInternalIdMap),
mnUniquePropertyMap(other.mnUniquePropertyMap)
{
LOCK(other.m_cached_sml_mutex);
m_cached_sml = other.m_cached_sml;
}

// Assignment operator
CDeterministicMNList& operator=(const CDeterministicMNList& other)
{
if (this != &other) {
blockHash = other.blockHash;
nHeight = other.nHeight;
nTotalRegisteredCount = other.nTotalRegisteredCount;
mnMap = other.mnMap;
mnInternalIdMap = other.mnInternalIdMap;
mnUniquePropertyMap = other.mnUniquePropertyMap;

LOCK2(m_cached_sml_mutex, other.m_cached_sml_mutex);
m_cached_sml = other.m_cached_sml;
}
return *this;
}

template <typename Stream, typename Operation>
inline void SerializationOpBase(Stream& s, Operation ser_action)
{
Expand Down Expand Up @@ -195,6 +244,7 @@ class CDeterministicMNList
mnMap = MnMap();
mnUniquePropertyMap = MnUniquePropertyMap();
mnInternalIdMap = MnInternalIdMap();
InvalidateSMLCache();
}

[[nodiscard]] size_t GetAllMNsCount() const
Expand Down Expand Up @@ -314,6 +364,12 @@ class CDeterministicMNList
*/
[[nodiscard]] std::vector<CDeterministicMNCPtr> GetProjectedMNPayees(gsl::not_null<const CBlockIndex* const> pindexPrev, int nCount = std::numeric_limits<int>::max()) const;

/**
* Calculates CSimplifiedMNList for current list and cache it
* Thread safety: Uses internal mutex for thread-safe cache access
*/
gsl::not_null<std::shared_ptr<const CSimplifiedMNList>> to_sml() const;

/**
* Calculates the maximum penalty which is allowed at the height of this MN list. It is dynamic and might change
* for every block.
Expand Down
Loading
Loading