backport: merge bitcoin#21817, #21966, #21824, #21969, #23653, #23438, #24190, #24253, #24231, #26258, #28012, partial bitcoin#25001, #25296, #23595, #27927 (serialization updates)

src/Makefile.test.include

@@ -151,6 +151,7 @@ BITCOIN_TESTS =\
   test/script_tests.cpp \
   test/script_standard_tests.cpp \
   test/scriptnum_tests.cpp \
+  test/serfloat_tests.cpp \
   test/serialize_tests.cpp \
   test/settings_tests.cpp \
   test/sighash_tests.cpp \

src/addrman.cpp

@@ -39,7 +39,7 @@ int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std:
 
 int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const
 {
-    uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? 'N' : 'K') << nBucket << GetKey()).GetCheapHash();
+    uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? uint8_t{'N'} : uint8_t{'K'}) << nBucket << GetKey()).GetCheapHash();
     return hash1 % ADDRMAN_BUCKET_SIZE;
 }
 
@@ -762,7 +762,7 @@ std::vector<bool> CAddrMan::DecodeAsmap(fs::path path)
     int length = ftell(filestr);
     LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length);
     fseek(filestr, 0, SEEK_SET);
-    char cur_byte;
+    uint8_t cur_byte;
     for (int i = 0; i < length; ++i) {
         file >> cur_byte;
         for (int bit = 0; bit < 8; ++bit) {

src/bench/checkblock.cpp

@@ -17,8 +17,8 @@
 static void DeserializeBlockTest(benchmark::Bench& bench)
 {
     CDataStream stream(benchmark::data::block813851, SER_NETWORK, PROTOCOL_VERSION);
-    char a = '\0';
-    stream.write(&a, 1); // Prevent compaction
+    std::byte a{0};
+    stream.write({&a, 1}); // Prevent compaction
 
     bench.unit("block").run([&] {
         CBlock block;
@@ -31,8 +31,8 @@ static void DeserializeBlockTest(benchmark::Bench& bench)
 static void DeserializeAndCheckBlockTest(benchmark::Bench& bench)
 {
     CDataStream stream(benchmark::data::block813851, SER_NETWORK, PROTOCOL_VERSION);
-    char a = '\0';
-    stream.write(&a, 1); // Prevent compaction
+    std::byte a{0};
+    stream.write({&a, 1}); // Prevent compaction
 
     ArgsManager bench_args;
     const auto chainParams = CreateChainParams(bench_args, CBaseChainParams::MAIN);

src/bench/rpc_blockchain.cpp

@@ -27,8 +27,8 @@ struct TestBlockAndIndex {
     TestBlockAndIndex()
     {
         CDataStream stream(benchmark::data::block813851, SER_NETWORK, PROTOCOL_VERSION);
-        char a = '\0';
-        stream.write(&a, 1); // Prevent compaction
+        std::byte a{0};
+        stream.write({&a, 1}); // Prevent compaction
 
         stream >> block;
 

src/blockfilter.cpp

@@ -50,7 +50,7 @@ GCSFilter::GCSFilter(const Params& params)
 GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter)
     : m_params(params), m_encoded(std::move(encoded_filter))
 {
-    VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+    SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0};
 
     uint64_t N = ReadCompactSize(stream);
     m_N = static_cast<uint32_t>(N);
@@ -61,7 +61,7 @@ GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_fi
 
     // Verify that the encoded filter contains exactly N elements. If it has too much or too little
     // data, a std::ios_base::failure exception will be raised.
-    BitStreamReader<VectorReader> bitreader(stream);
+    BitStreamReader<SpanReader> bitreader{stream};
     for (uint64_t i = 0; i < m_N; ++i) {
         GolombRiceDecode(bitreader, m_params.m_P);
     }
@@ -102,13 +102,13 @@ GCSFilter::GCSFilter(const Params& params, const ElementSet& elements)
 
 bool GCSFilter::MatchInternal(const uint64_t* element_hashes, size_t size) const
 {
-    VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0);
+    SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0};
 
     // Seek forward by size of N
     uint64_t N = ReadCompactSize(stream);
     assert(N == m_N);
 
-    BitStreamReader<VectorReader> bitreader(stream);
+    BitStreamReader<SpanReader> bitreader{stream};
 
     uint64_t value = 0;
     size_t hashes_index = 0;

src/bloom.cpp

@@ -66,7 +66,7 @@ void CBloomFilter::insert(const COutPoint& outpoint)
 {
     CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
     stream << outpoint;
-    insert(stream);
+    insert(MakeUCharSpan(stream));
 }
 
 bool CBloomFilter::contains(Span<const unsigned char> vKey) const

src/bls/bls.h

@@ -167,7 +167,7 @@ class CBLSWrapper
     template <typename Stream>
     inline void Serialize(Stream& s, const bool specificLegacyScheme) const
     {
-        s.write(reinterpret_cast<const char*>(ToByteVector(specificLegacyScheme).data()), SerSize);
+        s.write(AsBytes(Span{ToByteVector(specificLegacyScheme).data(), SerSize}));
     }
 
     template <typename Stream>
@@ -180,7 +180,7 @@ class CBLSWrapper
     inline void Unserialize(Stream& s, const bool specificLegacyScheme)
     {
         std::array<uint8_t, SerSize> vecBytes{};
-        s.read(reinterpret_cast<char*>(vecBytes.data()), SerSize);
+        s.read(AsWritableBytes(Span{vecBytes.data(), SerSize}));
         SetByteVector(vecBytes, specificLegacyScheme);
 
         if (!CheckMalleable(vecBytes, specificLegacyScheme)) {
@@ -456,7 +456,7 @@ class CBLSLazyWrapper
             bufLegacyScheme = specificLegacyScheme;
             hash.SetNull();
         }
-        s.write(reinterpret_cast<const char*>(vecBytes.data()), vecBytes.size());
+        s.write(MakeByteSpan(vecBytes));
     }
 
     template<typename Stream>
@@ -469,7 +469,7 @@ class CBLSLazyWrapper
     inline void Unserialize(Stream& s, const bool specificLegacyScheme) const
     {
         std::unique_lock<std::mutex> l(mutex);
-        s.read(reinterpret_cast<char*>(vecBytes.data()), BLSObject::SerSize);
+        s.read(AsWritableBytes(Span{vecBytes.data(), BLSObject::SerSize}));
         bufValid = true;
         bufLegacyScheme = specificLegacyScheme;
         objInitialized = false;
@@ -543,7 +543,7 @@ class CBLSLazyWrapper
         }
         if (hash.IsNull()) {
             CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION);
-            ss.write(reinterpret_cast<const char*>(vecBytes.data()), vecBytes.size());
+            ss.write(MakeByteSpan(vecBytes));
             hash = ss.GetHash();
         }
         return hash;

src/bls/bls_ies.h

@@ -109,7 +109,7 @@ class CBLSIESMultiRecipientObjects : public CBLSIESMultiRecipientBlobs
                 ds.clear();
 
                 ds << _objects[i];
-                blobs[i].assign(ds.begin(), ds.end());
+                blobs[i].assign(UCharCast(ds.data()), UCharCast(ds.data() + ds.size()));
             }
         } catch (const std::exception&) {
             return false;
@@ -122,7 +122,7 @@ class CBLSIESMultiRecipientObjects : public CBLSIESMultiRecipientBlobs
     {
         CDataStream ds(SER_NETWORK, nVersion);
         ds << obj;
-        Blob blob(ds.begin(), ds.end());
+        Blob blob(UCharCast(ds.data()), UCharCast(ds.data() + ds.size()));
         return CBLSIESMultiRecipientBlobs::Encrypt(idx, recipient, blob);
     }
 

src/compat/assumptions.h

@@ -36,11 +36,6 @@ static_assert(std::numeric_limits<double>::is_iec559, "IEEE 754 double assumed")
 // Example(s): Everywhere :-)
 static_assert(std::numeric_limits<unsigned char>::digits == 8, "8-bit byte assumed");
 
-// Assumption: We assume floating-point widths.
-// Example(s): Type punning in serialization code (ser_{float,double}_to_uint{32,64}).
-static_assert(sizeof(float) == 4, "32-bit float assumed");
-static_assert(sizeof(double) == 8, "64-bit double assumed");
-
 // Assumption: We assume integer widths.
 // Example(s): GetSizeOfCompactSize and WriteCompactSize in the serialization
 //             code.

src/compressor.cpp

@@ -124,7 +124,7 @@ bool DecompressScript(CScript& script, unsigned int nSize, const CompressedScrip
         unsigned char vch[33] = {};
         vch[0] = nSize - 2;
         memcpy(&vch[1], in.data(), 32);
-        CPubKey pubkey(&vch[0], &vch[33]);
+        CPubKey pubkey{vch};
         if (!pubkey.Decompress())
             return false;
         assert(pubkey.size() == 65);

src/dbwrapper.cpp

@@ -223,10 +223,9 @@ const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
  */
 std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
 {
-    unsigned char buff[OBFUSCATE_KEY_NUM_BYTES];
-    GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES);
-    return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);
-
+    std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
+    GetRandBytes(ret.data(), OBFUSCATE_KEY_NUM_BYTES);
+    return ret;
 }
 
 bool CDBWrapper::IsEmpty()

src/dbwrapper.h

@@ -169,7 +169,7 @@ class CDBIterator
 
     CDataStream GetKey() {
         leveldb::Slice slKey = piter->key();
-        return CDataStream(MakeUCharSpan(slKey), SER_DISK, CLIENT_VERSION);
+        return CDataStream{MakeByteSpan(slKey), SER_DISK, CLIENT_VERSION};
     }
 
     unsigned int GetKeySize() {
@@ -179,7 +179,7 @@ class CDBIterator
     template<typename V> bool GetValue(V& value) {
         leveldb::Slice slValue = piter->value();
         try {
-            CDataStream ssValue(MakeUCharSpan(slValue), SER_DISK, CLIENT_VERSION);
+            CDataStream ssValue{MakeByteSpan(slValue), SER_DISK, CLIENT_VERSION};
             ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
             ssValue >> value;
         } catch (const std::exception&) {
@@ -269,7 +269,7 @@ class CDBWrapper
             LogPrintf("LevelDB read failure: %s\n", status.ToString());
             dbwrapper_private::HandleError(status);
         }
-        CDataStream ssValueTmp(MakeUCharSpan(strValue), SER_DISK, CLIENT_VERSION);
+        CDataStream ssValueTmp{MakeByteSpan(strValue), SER_DISK, CLIENT_VERSION};
         ssValueTmp.Xor(obfuscate_key);
         ssValue = std::move(ssValueTmp);
         return true;

src/evo/specialtx.h

@@ -40,7 +40,7 @@ void SetTxPayload(CMutableTransaction& tx, const T& payload)
 {
     CDataStream ds(SER_NETWORK, PROTOCOL_VERSION);
     ds << payload;
-    tx.vExtraPayload.assign(ds.begin(), ds.end());
+    tx.vExtraPayload.assign(UCharCast(ds.data()), UCharCast(ds.data() + ds.size()));
 }
 
 uint256 CalcTxInputsHash(const CTransaction& tx);

src/flat-database.h

@@ -97,7 +97,7 @@ class CFlatDB
 
         // read data and checksum from file
         try {
-            filein.read((char *)vchData.data(), dataSize);
+            filein.read(MakeWritableByteSpan(vchData));
             filein >> hashIn;
         }
         catch (std::exception &e) {

src/hash.h

@@ -124,8 +124,9 @@ class CHashWriter
     int GetType() const { return nType; }
     int GetVersion() const { return nVersion; }
 
-    void write(const char *pch, size_t size) {
-        ctx.Write((const unsigned char*)pch, size);
+    void write(Span<const std::byte> src)
+    {
+        ctx.Write(UCharCast(src.data()), src.size());
     }
 
     /** Compute the double-SHA256 hash of all data written to this object.
@@ -175,18 +176,18 @@ class CHashVerifier : public CHashWriter
 public:
     explicit CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {}
 
-    void read(char* pch, size_t nSize)
+    void read(Span<std::byte> dst)
     {
-        source->read(pch, nSize);
-        this->write(pch, nSize);
+        source->read(dst);
+        this->write(dst);
     }
 
     void ignore(size_t nSize)
     {
-        char data[1024];
+        std::byte data[1024];
         while (nSize > 0) {
             size_t now = std::min<size_t>(nSize, 1024);
-            read(data, now);
+            read({data, now});
             nSize -= now;
         }
     }

src/httprpc.cpp

@@ -132,7 +132,9 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna
     if (strAuth.substr(0, 6) != "Basic ")
         return false;
     std::string strUserPass64 = TrimString(strAuth.substr(6));
-    std::string strUserPass = DecodeBase64(strUserPass64);
+    bool invalid;
+    std::string strUserPass = DecodeBase64(strUserPass64, &invalid);
+    if (invalid) return false;
 
     if (strUserPass.find(':') != std::string::npos)
         strAuthUsernameOut = strUserPass.substr(0, strUserPass.find(':'));

src/index/base.cpp

@@ -13,7 +13,7 @@
 #include <validation.h> // For g_chainman
 #include <warnings.h>
 
-constexpr char DB_BEST_BLOCK = 'B';
+constexpr uint8_t DB_BEST_BLOCK{'B'};
 
 constexpr auto SYNC_LOG_INTERVAL{30s};
 constexpr auto SYNC_LOCATOR_WRITE_INTERVAL{30s};

src/index/blockfilterindex.cpp

@@ -24,9 +24,9 @@
  * as big-endian so that sequential reads of filters by height are fast.
  * Keys for the hash index have the type [DB_BLOCK_HASH, uint256].
  */
-constexpr char DB_BLOCK_HASH = 's';
-constexpr char DB_BLOCK_HEIGHT = 't';
-constexpr char DB_FILTER_POS = 'P';
+constexpr uint8_t DB_BLOCK_HASH{'s'};
+constexpr uint8_t DB_BLOCK_HEIGHT{'t'};
+constexpr uint8_t DB_FILTER_POS{'P'};
 
 constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
 /** The pre-allocation chunk size for fltr?????.dat files */
@@ -68,7 +68,7 @@ struct DBHeightKey {
     template<typename Stream>
     void Unserialize(Stream& s)
     {
-        char prefix = ser_readdata8(s);
+        const uint8_t prefix{ser_readdata8(s)};
         if (prefix != DB_BLOCK_HEIGHT) {
             throw std::ios_base::failure("Invalid format for block filter index DB height key");
         }
@@ -81,9 +81,8 @@ struct DBHashKey {
 
     explicit DBHashKey(const uint256& hash_in) : hash(hash_in) {}
 
-    SERIALIZE_METHODS(DBHashKey, obj)
-    {
-        char prefix = DB_BLOCK_HASH;
+    SERIALIZE_METHODS(DBHashKey, obj) {
+        uint8_t prefix{DB_BLOCK_HASH};
         READWRITE(prefix);
         if (prefix != DB_BLOCK_HASH) {
             throw std::ios_base::failure("Invalid format for block filter index DB hash key");
@@ -155,7 +154,7 @@ bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& f
     }
 
     uint256 block_hash;
-    std::vector<unsigned char> encoded_filter;
+    std::vector<uint8_t> encoded_filter;
     try {
         filein >> block_hash >> encoded_filter;
         filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));

src/index/coinstatsindex.cpp

@@ -13,9 +13,9 @@
 #include <validation.h>
 #include <util/check.h>
 
-static constexpr char DB_BLOCK_HASH = 's';
-static constexpr char DB_BLOCK_HEIGHT = 't';
-static constexpr char DB_MUHASH = 'M';
+static constexpr uint8_t DB_BLOCK_HASH{'s'};
+static constexpr uint8_t DB_BLOCK_HEIGHT{'t'};
+static constexpr uint8_t DB_MUHASH{'M'};
 
 namespace {
 
@@ -67,7 +67,7 @@ struct DBHeightKey {
     template <typename Stream>
     void Unserialize(Stream& s)
     {
-        char prefix{static_cast<char>(ser_readdata8(s))};
+        const uint8_t prefix{ser_readdata8(s)};
         if (prefix != DB_BLOCK_HEIGHT) {
             throw std::ios_base::failure("Invalid format for coinstatsindex DB height key");
         }
@@ -82,7 +82,7 @@ struct DBHashKey {
 
     SERIALIZE_METHODS(DBHashKey, obj)
     {
-        char prefix{DB_BLOCK_HASH};
+        uint8_t prefix{DB_BLOCK_HASH};
         READWRITE(prefix);
         if (prefix != DB_BLOCK_HASH) {
             throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key");