merge #17812, #16702, #16730, #18023: supplying and using asmap to improve IP bucketing in addrman #4028
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,16 +6,21 @@ | |
| #ifndef BITCOIN_ADDRMAN_H | ||
| #define BITCOIN_ADDRMAN_H | ||
|
|
||
| #include <clientversion.h> | ||
| #include <netaddress.h> | ||
| #include <protocol.h> | ||
| #include <random.h> | ||
| #include <sync.h> | ||
| #include <timedata.h> | ||
| #include <util.h> | ||
|
|
||
| #include <fs.h> | ||
| #include <hash.h> | ||
| #include <iostream> | ||
| #include <map> | ||
| #include <set> | ||
| #include <stdint.h> | ||
| #include <streams.h> | ||
| #include <vector> | ||
|
|
||
| /** | ||
|
|
@@ -83,15 +88,15 @@ class CAddrInfo : public CAddress | |
| } | ||
|
|
||
| //! Calculate in which "tried" bucket this entry belongs | ||
| int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; | ||
|
|
||
| //! Calculate in which "new" bucket this entry belongs, given a certain source | ||
| int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; | ||
|
|
||
| //! Calculate in which "new" bucket this entry belongs, using its default source | ||
| int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const | ||
| { | ||
| return GetNewBucket(nKey, source, asmap); | ||
| } | ||
|
|
||
| //! Calculate in which position of a bucket to store this entry. | ||
|
|
@@ -184,6 +189,9 @@ class CAddrInfo : public CAddress | |
| class CAddrMan | ||
| { | ||
| private: | ||
| friend class CAddrManTest; | ||
|
|
||
| protected: | ||
|
||
| //! critical section to protect the inner data structures | ||
| mutable CCriticalSection cs; | ||
|
|
||
|
|
@@ -285,9 +293,29 @@ class CAddrMan | |
| CAddrInfo GetAddressInfo_(const CService& addr); | ||
|
|
||
| public: | ||
| // Compressed IP->ASN mapping, loaded from a file when a node starts. | ||
| // Should be always empty if no file was provided. | ||
| // This mapping is then used for bucketing nodes in Addrman. | ||
| // | ||
| // If asmap is provided, nodes will be bucketed by | ||
| // AS they belong to, in order to make impossible for a node | ||
| // to connect to several nodes hosted in a single AS. | ||
| // This is done in response to Erebus attack, but also to generally | ||
| // diversify the connections every node creates, | ||
| // especially useful when a large fraction of nodes | ||
| // operate under a couple of cloud providers. | ||
| // | ||
| // If a new asmap was provided, the existing records | ||
| // would be re-bucketed accordingly. | ||
| std::vector<bool> m_asmap; | ||
|
|
||
| // Read asmap from provided binary file | ||
| static std::vector<bool> DecodeAsmap(fs::path path); | ||
|
|
||
|
|
||
| /** | ||
| * serialized format: | ||
| * * version byte (1 for pre-asmap files, 2 for files including asmap version) | ||
| * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) | ||
| * * nNew | ||
| * * nTried | ||
|
|
@@ -319,7 +347,7 @@ class CAddrMan | |
| { | ||
| LOCK(cs); | ||
|
|
||
| unsigned char nVersion = 2; | ||
| s << nVersion; | ||
| s << ((unsigned char)32); | ||
| s << nKey; | ||
|
|
@@ -362,6 +390,13 @@ class CAddrMan | |
| } | ||
| } | ||
| } | ||
| // Store asmap version after bucket entries so that it | ||
| // can be ignored by older clients for backward compatibility. | ||
| uint256 asmap_version; | ||
| if (m_asmap.size() != 0) { | ||
| asmap_version = SerializeHash(m_asmap); | ||
| } | ||
| s << asmap_version; | ||
| } | ||
|
|
||
| template<typename Stream> | ||
|
|
@@ -370,7 +405,6 @@ class CAddrMan | |
| LOCK(cs); | ||
|
|
||
| Clear(); | ||
| unsigned char nVersion; | ||
| s >> nVersion; | ||
| unsigned char nKeySize; | ||
|
|
@@ -400,16 +434,6 @@ class CAddrMan | |
| mapAddr[info] = n; | ||
| info.nRandomPos = vRandom.size(); | ||
| vRandom.push_back(n); | ||
| } | ||
| nIdCount = nNew; | ||
|
|
||
|
|
@@ -418,7 +442,7 @@ class CAddrMan | |
| for (int n = 0; n < nTried; n++) { | ||
| CAddrInfo info; | ||
| s >> info; | ||
| int nKBucket = info.GetTriedBucket(nKey, m_asmap); | ||
| int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); | ||
| if (vvTried[nKBucket][nKBucketPos] == -1) { | ||
| info.nRandomPos = vRandom.size(); | ||
|
|
@@ -434,20 +458,48 @@ class CAddrMan | |
| } | ||
| nTried -= nLost; | ||
|
|
||
| // Store positions in the new table buckets to apply later (if possible). | ||
| std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing | ||
|
|
||
| for (int bucket = 0; bucket < nUBuckets; bucket++) { | ||
| int nSize = 0; | ||
| s >> nSize; | ||
| for (int n = 0; n < nSize; n++) { | ||
| int nIndex = 0; | ||
| s >> nIndex; | ||
| if (nIndex >= 0 && nIndex < nNew) { | ||
| entryToBucket[nIndex] = bucket; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| uint256 supplied_asmap_version; | ||
| if (m_asmap.size() != 0) { | ||
| supplied_asmap_version = SerializeHash(m_asmap); | ||
| } | ||
| uint256 serialized_asmap_version; | ||
| if (nVersion > 1) { | ||
| s >> serialized_asmap_version; | ||
| } | ||
|
|
||
| for (int n = 0; n < nNew; n++) { | ||
| CAddrInfo &info = mapInfo[n]; | ||
| int bucket = entryToBucket[n]; | ||
| int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); | ||
| if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && | ||
| info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { | ||
| // Bucketing has not changed, using existing bucket positions for the new table | ||
| vvNew[bucket][nUBucketPos] = n; | ||
| info.nRefCount++; | ||
| } else { | ||
| // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), | ||
| // try to give them a reference based on their primary source address. | ||
| LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); | ||
| bucket = info.GetNewBucket(nKey, m_asmap); | ||
| nUBucketPos = info.GetBucketPosition(nKey, true, bucket); | ||
| if (vvNew[bucket][nUBucketPos] == -1) { | ||
| vvNew[bucket][nUBucketPos] = n; | ||
| info.nRefCount++; | ||
| } | ||
| } | ||
| } | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.