merge bitcoin#17071...#18206: fuzzing harness backports (part 1)

.gitlab-ci.yml

@@ -132,6 +132,7 @@ builder-image:
       - build-ci/dashcore-$BUILD_TARGET/src/qt/test/test_dash-qt.exe
       - build-ci/dashcore-$BUILD_TARGET/src/test/test_dash
       - build-ci/dashcore-$BUILD_TARGET/src/test/test_dash.exe
+      - build-ci/dashcore-$BUILD_TARGET/src/test/fuzz/*
     expire_in: 3 days
 
 .test-template:

doc/fuzzing.md

@@ -102,7 +102,7 @@ libFuzzer is needed (all found in the `compiler-rt` runtime libraries package).
 To build all fuzz targets with libFuzzer, run
 
 ```
-./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
+./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++
 make
 ```
 

src/addrman.cpp

@@ -680,5 +680,9 @@ std::vector<bool> CAddrMan::DecodeAsmap(fs::path path)
             bits.push_back((cur_byte >> bit) & 1);
         }
     }
+    if (!SanityCheckASMap(bits)) {
+        LogPrintf("Sanity check of asmap file %s failed\n", path);
+        return {};
+    }
     return bits;
 }

src/indirectmap.h

@@ -5,6 +5,8 @@
 #ifndef BITCOIN_INDIRECTMAP_H
 #define BITCOIN_INDIRECTMAP_H
 
+#include <map>
+
 template <class T>
 struct DereferencingComparator { bool operator()(const T a, const T b) const { return *a < *b; } };
 

src/memusage.h

@@ -6,9 +6,11 @@
 #define BITCOIN_MEMUSAGE_H
 
 #include <indirectmap.h>
+#include <prevector.h>
 
 #include <stdlib.h>
 
+#include <cassert>
 #include <map>
 #include <memory>
 #include <set>

src/netaddress.cpp

@@ -1095,3 +1095,8 @@ bool operator<(const CSubNet& a, const CSubNet& b)
 {
     return (a.network < b.network || (a.network == b.network && memcmp(a.netmask, b.netmask, 16) < 0));
 }
+
+bool SanityCheckASMap(const std::vector<bool>& asmap)
+{
+    return SanityCheckASMap(asmap, 128); // For IP address lookups, the input is 128 bits
+}

src/netaddress.h

@@ -146,7 +146,6 @@ class CNetAddr
          */
         void SetLegacyIPv6(Span<const uint8_t> ipv6);
 
-    private:
         /**
          * Set raw IPv4 or IPv6 address (in network byte order)
          * @note Only NET_IPV4 and NET_IPV6 are allowed for network.
@@ -531,4 +530,6 @@ class CService : public CNetAddr
         }
 };
 
+bool SanityCheckASMap(const std::vector<bool>& asmap);
+
 #endif // BITCOIN_NETADDRESS_H

src/script/interpreter.cpp

@@ -1437,16 +1437,30 @@ uint256 GetOutputsHash(const T& txTo)
 } // namespace
 
 template <class T>
-PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo)
+void PrecomputedTransactionData::Init(const T& txTo, std::vector<CTxOut>&& spent_outputs)
 {
+    assert(!m_ready);
+
+    m_spent_outputs = std::move(spent_outputs);
+
     hashPrevouts = GetPrevoutHash(txTo);
     hashSequence = GetSequenceHash(txTo);
     hashOutputs = GetOutputsHash(txTo);
+
+    m_ready = true;
+}
+
+template <class T>
+PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo)
+{
+    Init(txTo, {});
 }
 
 // explicit instantiation
 template PrecomputedTransactionData::PrecomputedTransactionData(const CTransaction& txTo);
 template PrecomputedTransactionData::PrecomputedTransactionData(const CMutableTransaction& txTo);
+template void PrecomputedTransactionData::Init(const CTransaction& txTo, std::vector<CTxOut>&& spent_outputs);
+template void PrecomputedTransactionData::Init(const CMutableTransaction& txTo, std::vector<CTxOut>&& spent_outputs);
 
 template <class T>
 uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn, int nHashType, const CAmount& amount, SigVersion sigversion, const PrecomputedTransactionData* cache)

src/script/interpreter.h

@@ -15,6 +15,7 @@
 class CPubKey;
 class CScript;
 class CTransaction;
+class CTxOut;
 class uint256;
 
 /** Signature hash types/flags */
@@ -108,6 +109,13 @@ bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned i
 struct PrecomputedTransactionData
 {
     uint256 hashPrevouts, hashSequence, hashOutputs;
+    bool m_ready = false;
+    std::vector<CTxOut> m_spent_outputs;
+
+    PrecomputedTransactionData() = default;
+
+    template <class T>
+    void Init(const T& tx, std::vector<CTxOut>&& spent_outputs);
 
     template <class T>
     explicit PrecomputedTransactionData(const T& tx);

src/test/fuzz/FuzzedDataProvider.h

@@ -13,11 +13,10 @@
 #ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
 #define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
 
-#include <limits.h>
-#include <stddef.h>
-#include <stdint.h>
-
 #include <algorithm>
+#include <climits>
+#include <cstddef>
+#include <cstdint>
 #include <cstring>
 #include <initializer_list>
 #include <limits>
@@ -26,8 +25,10 @@
 #include <utility>
 #include <vector>
 
+// In addition to the comments below, the API is also briefly documented at
+// https://github.com/google/fuzzing/blob/master/docs/split-inputs.md#fuzzed-data-provider
 class FuzzedDataProvider {
-public:
+ public:
   // |data| is an array of length |size| that the FuzzedDataProvider wraps to
   // provide more granular access. |data| must outlive the FuzzedDataProvider.
   FuzzedDataProvider(const uint8_t *data, size_t size)
@@ -144,9 +145,9 @@ class FuzzedDataProvider {
     return ConsumeBytes<T>(remaining_bytes_);
   }
 
+  // Returns a std::string containing all remaining bytes of the input data.
   // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string
   // object.
-  // Returns a std::vector containing all remaining bytes of the input data.
   std::string ConsumeRemainingBytesAsString() {
     return ConsumeBytesAsString(remaining_bytes_);
   }
@@ -162,7 +163,7 @@ class FuzzedDataProvider {
   // Reads one byte and returns a bool, or false when no data remains.
   bool ConsumeBool() { return 1 & ConsumeIntegral<uint8_t>(); }
 
-  // Returns a copy of a value selected from a fixed-size |array|.
+  // Returns a copy of the value selected from the given fixed-size |array|.
   template <typename T, size_t size>
   T PickValueInArray(const T (&array)[size]) {
     static_assert(size > 0, "The array must be non empty.");
@@ -171,11 +172,14 @@ class FuzzedDataProvider {
 
   template <typename T>
   T PickValueInArray(std::initializer_list<const T> list) {
-    // static_assert(list.size() > 0, "The array must be non empty.");
+    // TODO(Dor1s): switch to static_assert once C++14 is allowed.
+    if (!list.size())
+      abort();
+
     return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1));
   }
 
-  // Return an enum value. The enum must start at 0 and be contiguous. It must
+  // Returns an enum value. The enum must start at 0 and be contiguous. It must
   // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as:
   // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue };
   template <typename T> T ConsumeEnum() {
@@ -184,10 +188,60 @@ class FuzzedDataProvider {
         0, static_cast<uint32_t>(T::kMaxValue)));
   }
 
+  // Returns a floating point number in the range [0.0, 1.0]. If there's no
+  // input data left, always returns 0.
+  template <typename T> T ConsumeProbability() {
+    static_assert(std::is_floating_point<T>::value,
+                  "A floating point type is required.");
+
+    // Use different integral types for different floating point types in order
+    // to provide better density of the resulting values.
+    using IntegralType =
+        typename std::conditional<(sizeof(T) <= sizeof(uint32_t)), uint32_t,
+                                  uint64_t>::type;
+
+    T result = static_cast<T>(ConsumeIntegral<IntegralType>());
+    result /= static_cast<T>(std::numeric_limits<IntegralType>::max());
+    return result;
+  }
+
+  // Returns a floating point value in the range [Type's lowest, Type's max] by
+  // consuming bytes from the input data. If there's no input data left, always
+  // returns approximately 0.
+  template <typename T> T ConsumeFloatingPoint() {
+    return ConsumeFloatingPointInRange<T>(std::numeric_limits<T>::lowest(),
+                                          std::numeric_limits<T>::max());
+  }
+
+  // Returns a floating point value in the given range by consuming bytes from
+  // the input data. If there's no input data left, returns |min|. Note that
+  // |min| must be less than or equal to |max|.
+  template <typename T> T ConsumeFloatingPointInRange(T min, T max) {
+    if (min > max)
+      abort();
+
+    T range = .0;
+    T result = min;
+    constexpr T zero(.0);
+    if (max > zero && min < zero && max > min + std::numeric_limits<T>::max()) {
+      // The diff |max - min| would overflow the given floating point type. Use
+      // the half of the diff as the range and consume a bool to decide whether
+      // the result is in the first of the second part of the diff.
+      range = (max / 2.0) - (min / 2.0);
+      if (ConsumeBool()) {
+        result += range;
+      }
+    } else {
+      range = max - min;
+    }
+
+    return result + range * ConsumeProbability<T>();
+  }
+
   // Reports the remaining bytes available for fuzzed input.
   size_t remaining_bytes() { return remaining_bytes_; }
 
-private:
+ private:
   FuzzedDataProvider(const FuzzedDataProvider &) = delete;
   FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete;
 
@@ -210,6 +264,12 @@ class FuzzedDataProvider {
     // which seems to be a natural choice for other implementations as well.
     // To increase the odds even more, we also call |shrink_to_fit| below.
     std::vector<T> result(size);
+    if (size == 0) {
+      if (num_bytes_to_consume != 0)
+        abort();
+      return result;
+    }
+
     std::memcpy(result.data(), data_ptr_, num_bytes_to_consume);
     Advance(num_bytes_to_consume);
 
@@ -231,9 +291,9 @@ class FuzzedDataProvider {
 
     // Avoid using implementation-defined unsigned to signer conversions.
     // To learn more, see https://stackoverflow.com/questions/13150449.
-    if (value <= std::numeric_limits<TS>::max())
+    if (value <= std::numeric_limits<TS>::max()) {
       return static_cast<TS>(value);
-    else {
+    } else {
       constexpr auto TS_min = std::numeric_limits<TS>::min();
       return TS_min + static_cast<char>(value - TS_min);
     }

src/test/fuzz/asmap.cpp

@@ -0,0 +1,60 @@
+// Copyright (c) 2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <netaddress.h>
+#include <test/fuzz/fuzz.h>
+#include <util/asmap.h>
+
+#include <cstdint>
+#include <vector>
+
+//! asmap code that consumes nothing
+static const std::vector<bool> IPV6_PREFIX_ASMAP = {};
+
+//! asmap code that consumes the 96 prefix bits of ::ffff:0/96 (IPv4-in-IPv6 map)
+static const std::vector<bool> IPV4_PREFIX_ASMAP = {
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
+    true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,         // Match 0xFF
+    true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true          // Match 0xFF
+};
+
+void test_one_input(const std::vector<uint8_t>& buffer)
+{
+    // Encoding: [7 bits: asmap size] [1 bit: ipv6?] [3-130 bytes: asmap] [4 or 16 bytes: addr]
+    if (buffer.size() < 1 + 3 + 4) return;
+    int asmap_size = 3 + (buffer[0] & 127);
+    bool ipv6 = buffer[0] & 128;
+    const size_t addr_size = ipv6 ? ADDR_IPV6_SIZE : ADDR_IPV4_SIZE;
+    if (buffer.size() < size_t(1 + asmap_size + addr_size)) return;
+    std::vector<bool> asmap = ipv6 ? IPV6_PREFIX_ASMAP : IPV4_PREFIX_ASMAP;
+    asmap.reserve(asmap.size() + 8 * asmap_size);
+    for (int i = 0; i < asmap_size; ++i) {
+        for (int j = 0; j < 8; ++j) {
+            asmap.push_back((buffer[1 + i] >> j) & 1);
+        }
+    }
+    if (!SanityCheckASMap(asmap, 128)) return;
+
+    const uint8_t* addr_data = buffer.data() + 1 + asmap_size;
+    CNetAddr net_addr;
+    if (ipv6) {
+        assert(addr_size == ADDR_IPV6_SIZE);
+        net_addr.SetLegacyIPv6({addr_data, addr_size});
+    } else {
+        assert(addr_size == ADDR_IPV4_SIZE);
+        in_addr ipv4;
+        memcpy(&ipv4, addr_data, addr_size);
+        net_addr.SetIP(CNetAddr{ipv4});
+    }
+    (void)net_addr.GetMappedAS(asmap);
+}

src/test/fuzz/asmap_direct.cpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <util/asmap.h>
+#include <test/fuzz/fuzz.h>
+
+#include <cstdint>
+#include <vector>
+
+#include <assert.h>
+
+void test_one_input(const std::vector<uint8_t>& buffer)
+{
+    // Encoding: [asmap using 1 bit / byte] 0xFF [addr using 1 bit / byte]
+    bool have_sep = false;
+    size_t sep_pos;
+    for (size_t pos = 0; pos < buffer.size(); ++pos) {
+        uint8_t x = buffer[pos];
+        if ((x & 0xFE) == 0) continue;
+        if (x == 0xFF) {
+            if (have_sep) return;
+            have_sep = true;
+            sep_pos = pos;
+        } else {
+            return;
+        }
+    }
+    if (!have_sep) return; // Needs exactly 1 separator
+    if (buffer.size() - sep_pos - 1 > 128) return; // At most 128 bits in IP address
+
+    // Checks on asmap
+    std::vector<bool> asmap(buffer.begin(), buffer.begin() + sep_pos);
+    if (SanityCheckASMap(asmap, buffer.size() - 1 - sep_pos)) {
+        // Verify that for valid asmaps, no prefix (except up to 7 zero padding bits) is valid.
+        std::vector<bool> asmap_prefix = asmap;
+        while (!asmap_prefix.empty() && asmap_prefix.size() + 7 > asmap.size() && asmap_prefix.back() == false) asmap_prefix.pop_back();
+        while (!asmap_prefix.empty()) {
+            asmap_prefix.pop_back();
+            assert(!SanityCheckASMap(asmap_prefix, buffer.size() - 1 - sep_pos));
+        }
+        // No address input should trigger assertions in interpreter
+        std::vector<bool> addr(buffer.begin() + sep_pos + 1, buffer.end());
+        (void)Interpret(asmap, addr);
+    }
+}