Skip to content

merge #12752, #10785, #17896, #17957, #18021, #18112: serialization improvements #4037

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
UdjinM6 merged 6 commits into from
May 14, 2021
Merged

merge #12752, #10785, #17896, #17957, #18021, #18112: serialization improvements #4037

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e483a68
Merge #17850: Convert addrdb/addrman to new serialization
kwvg Jul 7, 2017
46cb6e2
Merge #17896: Convert chain to new serialization
kwvg Jan 8, 2020
0802bdf
Merge #18021: Convert undo.h to new serialization framework
kwvg Sep 6, 2017
2f542a9
Merge #12752: Move compressor utility functions out of class
kwvg Jul 8, 2017
bfe5971
Merge #17957: Convert compression.h to new serialization framework
kwvg Jan 18, 2020
97caca3
Merge #18112: Convert blockencodings.h to new serialization framework
kwvg Feb 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 1 addition & 9 deletions src/addrdb.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,7 @@ class CBanEntry
nCreateTime = nCreateTimeIn;
}

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(this->nVersion);
READWRITE(nCreateTime);
READWRITE(nBanUntil);
READWRITE(banReason);
}
SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); }

void SetNull()
{
Expand Down
14 changes: 5 additions & 9 deletions src/addrman.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,10 @@ class CAddrInfo : public CAddress

public:

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITEAS(CAddress, *this);
READWRITE(source);
READWRITE(nLastSuccess);
READWRITE(nAttempts);
SERIALIZE_METHODS(CAddrInfo, obj)
{
READWRITEAS(CAddress, obj);
READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts);
}

void Init()
Expand Down Expand Up @@ -315,7 +311,7 @@ class CAddrMan
* This format is more complex, but significantly smaller (at most 1.5 MiB), and supports
* changes to the ADDRMAN_ parameters without breaking the on-disk structure.
*
* We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has
* We don't use SERIALIZE_METHODS since the serialization and deserialization code has
* very little in common.
*/
template<typename Stream>
Expand Down
140 changes: 34 additions & 106 deletions src/blockencodings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,29 @@

class CTxMemPool;

// Dumb helper to handle CTransaction compression at serialize-time
struct TransactionCompressor {
private:
CTransactionRef& tx;
public:
explicit TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {}
// Transaction compression schemes for compact block relay can be introduced by writing
// an actual formatter here.
using TransactionCompression = DefaultFormatter;

ADD_SERIALIZE_METHODS;
class DifferenceFormatter
{
uint64_t m_shift = 0;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(tx); //TODO: Compress tx encoding
public:
template<typename Stream, typename I>
void Ser(Stream& s, I v)
{
if (v < m_shift || v >= std::numeric_limits<uint64_t>::max()) throw std::ios_base::failure("differential value overflow");
WriteCompactSize(s, v - m_shift);
m_shift = uint64_t(v) + 1;
}
template<typename Stream, typename I>
void Unser(Stream& s, I& v)
{
uint64_t n = ReadCompactSize(s);
m_shift += n;
if (m_shift < n || m_shift >= std::numeric_limits<uint64_t>::max() || m_shift < std::numeric_limits<I>::min() || m_shift > std::numeric_limits<I>::max()) throw std::ios_base::failure("differential value overflow");
v = I(m_shift++);
}
};

Expand All @@ -32,39 +43,9 @@ class BlockTransactionsRequest {
uint256 blockhash;
std::vector<uint16_t> indexes;

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(blockhash);
uint64_t indexes_size = (uint64_t)indexes.size();
READWRITE(COMPACTSIZE(indexes_size));
if (ser_action.ForRead()) {
size_t i = 0;
while (indexes.size() < indexes_size) {
indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size));
for (; i < indexes.size(); i++) {
uint64_t index = 0;
READWRITE(COMPACTSIZE(index));
if (index > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("index overflowed 16 bits");
indexes[i] = index;
}
}

int32_t offset = 0;
for (size_t j = 0; j < indexes.size(); j++) {
if (int32_t(indexes[j]) + offset > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("indexes overflowed 16 bits");
indexes[j] = indexes[j] + offset;
offset = int32_t(indexes[j]) + 1;
}
} else {
for (size_t i = 0; i < indexes.size(); i++) {
uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1));
READWRITE(COMPACTSIZE(index));
}
}
SERIALIZE_METHODS(BlockTransactionsRequest, obj)
{
READWRITE(obj.blockhash, Using<VectorFormatter<DifferenceFormatter>>(obj.indexes));
}
};

Expand All @@ -78,24 +59,9 @@ class BlockTransactions {
explicit BlockTransactions(const BlockTransactionsRequest& req) :
blockhash(req.blockhash), txn(req.indexes.size()) {}

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(blockhash);
uint64_t txn_size = (uint64_t)txn.size();
READWRITE(COMPACTSIZE(txn_size));
if (ser_action.ForRead()) {
size_t i = 0;
while (txn.size() < txn_size) {
txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size));
for (; i < txn.size(); i++)
READWRITE(TransactionCompressor(txn[i]));
}
} else {
for (size_t i = 0; i < txn.size(); i++)
READWRITE(TransactionCompressor(txn[i]));
}
SERIALIZE_METHODS(BlockTransactions, obj)
{
READWRITE(obj.blockhash, Using<VectorFormatter<TransactionCompression>>(obj.txn));
}
};

Expand All @@ -106,17 +72,7 @@ struct PrefilledTransaction {
uint16_t index;
CTransactionRef tx;

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
uint64_t idx = index;
READWRITE(COMPACTSIZE(idx));
if (idx > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("index overflowed 16-bits");
index = idx;
READWRITE(TransactionCompressor(tx));
}
SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using<TransactionCompression>(obj.tx)); }
};

typedef enum ReadStatus_t
Expand Down Expand Up @@ -154,43 +110,15 @@ class CBlockHeaderAndShortTxIDs {

size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); }

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(header);
READWRITE(nonce);

uint64_t shorttxids_size = (uint64_t)shorttxids.size();
READWRITE(COMPACTSIZE(shorttxids_size));
SERIALIZE_METHODS(CBlockHeaderAndShortTxIDs, obj)
{
READWRITE(obj.header, obj.nonce, Using<VectorFormatter<CustomUintFormatter<SHORTTXIDS_LENGTH>>>(obj.shorttxids), obj.prefilledtxn);
if (ser_action.ForRead()) {
size_t i = 0;
while (shorttxids.size() < shorttxids_size) {
shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size));
for (; i < shorttxids.size(); i++) {
uint32_t lsb = 0; uint16_t msb = 0;
READWRITE(lsb);
READWRITE(msb);
shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb);
static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids");
}
}
} else {
for (size_t i = 0; i < shorttxids.size(); i++) {
uint32_t lsb = shorttxids[i] & 0xffffffff;
uint16_t msb = (shorttxids[i] >> 32) & 0xffff;
READWRITE(lsb);
READWRITE(msb);
if (obj.BlockTxCount() > std::numeric_limits<uint16_t>::max()) {
throw std::ios_base::failure("indexes overflowed 16 bits");
}
obj.FillShortTxIDSelector();
}

READWRITE(prefilledtxn);

if (BlockTxCount() > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("indexes overflowed 16 bits");

if (ser_action.ForRead())
FillShortTxIDSelector();
}
};

Expand Down
60 changes: 26 additions & 34 deletions src/chain.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,15 @@ class CBlockFileInfo
uint64_t nTimeFirst; //!< earliest time of block in file
uint64_t nTimeLast; //!< latest time of block in file

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(VARINT(nBlocks));
READWRITE(VARINT(nSize));
READWRITE(VARINT(nUndoSize));
READWRITE(VARINT(nHeightFirst));
READWRITE(VARINT(nHeightLast));
READWRITE(VARINT(nTimeFirst));
READWRITE(VARINT(nTimeLast));
SERIALIZE_METHODS(CBlockFileInfo, obj)
{
READWRITE(VARINT(obj.nBlocks));
READWRITE(VARINT(obj.nSize));
READWRITE(VARINT(obj.nUndoSize));
READWRITE(VARINT(obj.nHeightFirst));
READWRITE(VARINT(obj.nHeightLast));
READWRITE(VARINT(obj.nTimeFirst));
READWRITE(VARINT(obj.nTimeLast));
}

void SetNull() {
Expand Down Expand Up @@ -383,33 +381,27 @@ class CDiskBlockIndex : public CBlockIndex
hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
}

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
SERIALIZE_METHODS(CDiskBlockIndex, obj)
{
int _nVersion = s.GetVersion();
if (!(s.GetType() & SER_GETHASH))
READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));

READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
READWRITE(VARINT(nStatus));
READWRITE(VARINT(nTx));
if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO))
READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
if (nStatus & BLOCK_HAVE_DATA)
READWRITE(VARINT(nDataPos));
if (nStatus & BLOCK_HAVE_UNDO)
READWRITE(VARINT(nUndoPos));
if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));

READWRITE(VARINT(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED));
READWRITE(VARINT(obj.nStatus));
READWRITE(VARINT(obj.nTx));
if (obj.nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED));
if (obj.nStatus & BLOCK_HAVE_DATA) READWRITE(VARINT(obj.nDataPos));
if (obj.nStatus & BLOCK_HAVE_UNDO) READWRITE(VARINT(obj.nUndoPos));

// block hash
READWRITE(hash);
READWRITE(obj.hash);
// block header
READWRITE(this->nVersion);
READWRITE(hashPrev);
READWRITE(hashMerkleRoot);
READWRITE(nTime);
READWRITE(nBits);
READWRITE(nNonce);
READWRITE(obj.nVersion);
READWRITE(obj.hashPrev);
READWRITE(obj.hashMerkleRoot);
READWRITE(obj.nTime);
READWRITE(obj.nBits);
READWRITE(obj.nNonce);
}

uint256 GetBlockHash() const
Expand Down
4 changes: 2 additions & 2 deletions src/coins.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ class Coin
assert(!IsSpent());
uint32_t code = nHeight * 2 + fCoinBase;
::Serialize(s, VARINT(code));
::Serialize(s, CTxOutCompressor(REF(out)));
::Serialize(s, Using<TxOutCompression>(out));
}

template<typename Stream>
Expand All @@ -69,7 +69,7 @@ class Coin
::Unserialize(s, VARINT(code));
nHeight = code >> 1;
fCoinBase = code & 1;
::Unserialize(s, CTxOutCompressor(out));
::Unserialize(s, Using<TxOutCompression>(out));
}

bool IsSpent() const {
Expand Down
Loading