backport: merge bitcoin#24469, #24470, #29040, #29181, #29085, #29484, #29577, #29081, #29815, #28687, #31502, partial bitcoin#26707, #28894 (c++20 backports: part 2)

configure.ac

@@ -851,7 +851,10 @@ case $host in
      fi
 
      CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0"
-     OBJCXXFLAGS="$CXXFLAGS"
+
+     dnl ignore deprecated-declarations warnings coming from objcxx code
+     dnl "'NSUserNotificationCenter' is deprecated: first deprecated in macOS 11.0".
+     OBJCXXFLAGS="$CXXFLAGS -Wno-deprecated-declarations"
      ;;
    *android*)
      dnl make sure android stays above linux for hosts like *linux-android*
@@ -1056,8 +1059,6 @@ AC_CHECK_DECLS([setsid])
 
 AC_CHECK_DECLS([pipe2])
 
-AC_CHECK_FUNCS([timingsafe_bcmp])
-
 dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas)
 AC_MSG_CHECKING([for mallopt M_ARENA_MAX])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]],
@@ -1158,22 +1159,6 @@ if test "$use_thread_local" = "yes" || test "$use_thread_local" = "auto"; then
   LDFLAGS="$TEMP_LDFLAGS"
 fi
 
-dnl check for gmtime_r(), fallback to gmtime_s() if that is unavailable
-dnl fail if neither are available.
-AC_MSG_CHECKING([for gmtime_r])
-AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]],
-  [[ gmtime_r((const time_t *) nullptr, (struct tm *) nullptr); ]])],
-  [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GMTIME_R], [1], [Define this symbol if gmtime_r is available]) ],
-  [ AC_MSG_RESULT([no]);
-    AC_MSG_CHECKING([for gmtime_s]);
-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]],
-       [[ gmtime_s((struct tm *) nullptr, (const time_t *) nullptr); ]])],
-       [ AC_MSG_RESULT([yes])],
-       [ AC_MSG_RESULT([no]); AC_MSG_ERROR([Both gmtime_r and gmtime_s are unavailable]) ]
-    )
-  ]
-)
-
 dnl Check for different ways of gathering OS randomness
 AC_MSG_CHECKING([for Linux getrandom syscall])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h>
@@ -1215,14 +1200,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
  [ AC_MSG_RESULT([no])]
 )
 
-AC_MSG_CHECKING([for if type char equals int8_t])
-AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <stdint.h>
-  #include <type_traits>]],
- [[ static_assert(std::is_same<int8_t, char>::value, ""); ]])],
- [ AC_MSG_RESULT([yes]); AC_DEFINE([CHAR_EQUALS_INT8], [1], [Define this symbol if type char equals int8_t]) ],
- [ AC_MSG_RESULT([no])]
-)
-
 dnl ensure backtrace() is found, check -lexecinfo if necessary
 if test "$TARGET_OS" != "windows"; then
   if test "$enable_stacktraces" != "no"; then
@@ -1910,7 +1887,6 @@ AC_SUBST(HAVE_O_CLOEXEC)
 AC_SUBST(HAVE_BUILTIN_PREFETCH)
 AC_SUBST(HAVE_MM_PREFETCH)
 AC_SUBST(HAVE_STRONG_GETAUXVAL)
-AC_SUBST(HAVE_GMTIME_R)
 AC_SUBST(ANDROID_ARCH)
 AC_SUBST(HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR)
 AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini])

contrib/devtools/copyright_header.py

@@ -23,7 +23,6 @@
     'src/bench/nanobench.h',
     'src/crypto/*',
     'src/ctpl_stl.h',
-    'src/reverse_iterator.h',
     'src/test/fuzz/FuzzedDataProvider.h',
     'src/tinyformat.h',
     'src/util/expected.h',

depends/hosts/netbsd.mk

@@ -7,8 +7,6 @@ netbsd_NM = $(host_toolchain)gcc-nm
 netbsd_RANLIB = $(host_toolchain)gcc-ranlib
 endif
 
-netbsd_CXXFLAGS=$(netbsd_CFLAGS)
-
 netbsd_release_CFLAGS=-O2
 netbsd_release_CXXFLAGS=$(netbsd_release_CFLAGS)
 

depends/packages/systemtap.mk

@@ -3,11 +3,10 @@ $(package)_version=4.8
 $(package)_download_path=https://sourceware.org/ftp/systemtap/releases/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=cbd50a4eba5b261394dc454c12448ddec73e55e6742fda7f508f9fbc1331c223
-$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch fix_variadic_warning.patch
+$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch
 
 define $(package)_preprocess_cmds
   patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \
-  patch -p1 < $($(package)_patch_dir)/fix_variadic_warning.patch && \
   mkdir -p $($(package)_staging_prefix_dir)/include/sys && \
   cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h
 endef

doc/developer-notes.md

@@ -1408,7 +1408,7 @@ A few guidelines for introducing and reviewing new RPC interfaces:
 
   - *Rationale*: User-facing consistency.
 
-- Use `fs::path::u8string()` and `fs::u8path()` functions when converting path
+- Use `fs::path::u8string()`/`fs::path::utf8string()` and `fs::u8path()` functions when converting path
   to JSON strings, not `fs::PathToString` and `fs::PathFromString`
 
   - *Rationale*: JSON strings are Unicode strings, not byte strings, and

src/Makefile.am

@@ -12,6 +12,7 @@ DIST_SUBDIRS = secp256k1
 AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS)
 AM_CFLAGS = $(DEBUG_CFLAGS)
 AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(CORE_CXXFLAGS)
+AM_OBJCXXFLAGS = $(AM_CXXFLAGS)
 AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS)
 AM_LIBTOOLFLAGS = --preserve-dup-deps
 PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS)
@@ -296,7 +297,6 @@ BITCOIN_CORE_H = \
   psbt.h \
   random.h \
   randomenv.h \
-  reverse_iterator.h \
   rpc/blockchain.h \
   rpc/client.h \
   rpc/index_util.h \

src/Makefile.bench.include

@@ -83,6 +83,7 @@ endif
 if ENABLE_WALLET
 bench_bench_dash_SOURCES += bench/coin_selection.cpp
 bench_bench_dash_SOURCES += bench/wallet_balance.cpp
+bench_bench_dash_SOURCES += bench/wallet_create.cpp
 endif
 
 bench_bench_dash_LDADD += $(BACKTRACE_LIB) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(GMP_LIBS)

src/addrdb.cpp

@@ -54,7 +54,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data
     std::string tmpfn = strprintf("%s.%04x", prefix, randv);
 
     // open temp output file, and associate with CAutoFile
-    fs::path pathTmp = gArgs.GetDataDirNet() / tmpfn;
+    fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn);
     FILE *file = fsbridge::fopen(pathTmp, "wb");
     CAutoFile fileout(file, SER_DISK, version);
     if (fileout.IsNull()) {

src/bench/wallet_create.cpp

@@ -0,0 +1,56 @@
+// Copyright (c) 2023-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <node/context.h>
+#include <random.h>
+#include <test/util/setup_common.h>
+#include <util/translation.h>
+#include <wallet/context.h>
+#include <wallet/wallet.h>
+
+namespace wallet {
+static void WalletCreate(benchmark::Bench& bench, bool encrypted)
+{
+    auto test_setup = MakeNoLogFileContext<TestingSetup>();
+    FastRandomContext random;
+
+    WalletContext context;
+    context.args = &test_setup->m_args;
+    context.chain = test_setup->m_node.chain.get();
+
+    DatabaseOptions options;
+    options.require_format = DatabaseFormat::SQLITE;
+    options.require_create = true;
+    options.create_flags = WALLET_FLAG_DESCRIPTORS;
+
+    if (encrypted) {
+        options.create_passphrase = random.rand256().ToString();
+    }
+
+    DatabaseStatus status;
+    bilingual_str error_string;
+    std::vector<bilingual_str> warnings;
+
+    fs::path wallet_path = test_setup->m_path_root / strprintf("test_wallet_%d", random.rand32()).c_str();
+    bench.run([&] {
+        auto wallet = CreateWallet(context, wallet_path.utf8string(), /*load_on_start=*/std::nullopt, options, status, error_string, warnings);
+        assert(status == DatabaseStatus::SUCCESS);
+        assert(wallet != nullptr);
+
+        // Cleanup
+        wallet.reset();
+        fs::remove_all(wallet_path);
+    });
+}
+
+static void WalletCreatePlain(benchmark::Bench& bench) { WalletCreate(bench, /*encrypted=*/false); }
+static void WalletCreateEncrypted(benchmark::Bench& bench) { WalletCreate(bench, /*encrypted=*/true); }
+
+#ifdef USE_SQLITE
+BENCHMARK(WalletCreatePlain);
+BENCHMARK(WalletCreateEncrypted);
+#endif
+
+} // namespace wallet

src/crypto/chacha20.cpp

@@ -11,15 +11,14 @@
 #include <span.h>
 
 #include <algorithm>
+#include <bit>
 #include <string.h>
 
-constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); }
-
 #define QUARTERROUND(a,b,c,d) \
-  a += b; d = rotl32(d ^ a, 16); \
-  c += d; b = rotl32(b ^ c, 12); \
-  a += b; d = rotl32(d ^ a, 8); \
-  c += d; b = rotl32(b ^ c, 7);
+  a += b; d = std::rotl(d ^ a, 16); \
+  c += d; b = std::rotl(b ^ c, 12); \
+  a += b; d = std::rotl(d ^ a, 8); \
+  c += d; b = std::rotl(b ^ c, 7);
 
 #define REPEAT10(a) do { {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; } while(0)
 

src/crypto/chacha20poly1305.cpp

@@ -26,10 +26,7 @@ void AEADChaCha20Poly1305::SetKey(Span<const std::byte> key) noexcept
 
 namespace {
 
-#ifndef HAVE_TIMINGSAFE_BCMP
-#define HAVE_TIMINGSAFE_BCMP
-
-int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept
+int timingsafe_bcmp_internal(const unsigned char* b1, const unsigned char* b2, size_t n) noexcept
 {
     const unsigned char *p1 = b1, *p2 = b2;
     int ret = 0;
@@ -38,8 +35,6 @@ int timingsafe_bcmp(const unsigned char* b1, const unsigned char* b2, size_t n)
     return (ret != 0);
 }
 
-#endif
-
 /** Compute poly1305 tag. chacha20 must be set to the right nonce, block 0. Will be at block 1 after. */
 void ComputeTag(ChaCha20& chacha20, Span<const std::byte> aad, Span<const std::byte> cipher, Span<std::byte> tag) noexcept
 {
@@ -93,7 +88,7 @@ bool AEADChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std:
     m_chacha20.Seek(nonce, 0);
     std::byte expected_tag[EXPANSION];
     ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), expected_tag);
-    if (timingsafe_bcmp(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false;
+    if (timingsafe_bcmp_internal(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false;
 
     // Decrypt (starting at block 1).
     m_chacha20.Crypt(cipher.first(plain1.size()), plain1);

src/crypto/sha3.cpp

@@ -10,15 +10,10 @@
 #include <span.h>
 
 #include <array> // For std::begin and std::end.
+#include <bit>
 
 #include <stdint.h>
 
-// Internal implementation code.
-namespace
-{
-uint64_t Rotl(uint64_t x, int n) { return (x << n) | (x >> (64 - n)); }
-} // namespace
-
 void KeccakF(uint64_t (&st)[25])
 {
     static constexpr uint64_t RNDC[24] = {
@@ -40,38 +35,38 @@ void KeccakF(uint64_t (&st)[25])
         bc2 = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22];
         bc3 = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23];
         bc4 = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24];
-        t = bc4 ^ Rotl(bc1, 1); st[0] ^= t; st[5] ^= t; st[10] ^= t; st[15] ^= t; st[20] ^= t;
-        t = bc0 ^ Rotl(bc2, 1); st[1] ^= t; st[6] ^= t; st[11] ^= t; st[16] ^= t; st[21] ^= t;
-        t = bc1 ^ Rotl(bc3, 1); st[2] ^= t; st[7] ^= t; st[12] ^= t; st[17] ^= t; st[22] ^= t;
-        t = bc2 ^ Rotl(bc4, 1); st[3] ^= t; st[8] ^= t; st[13] ^= t; st[18] ^= t; st[23] ^= t;
-        t = bc3 ^ Rotl(bc0, 1); st[4] ^= t; st[9] ^= t; st[14] ^= t; st[19] ^= t; st[24] ^= t;
+        t = bc4 ^ std::rotl(bc1, 1); st[0] ^= t; st[5] ^= t; st[10] ^= t; st[15] ^= t; st[20] ^= t;
+        t = bc0 ^ std::rotl(bc2, 1); st[1] ^= t; st[6] ^= t; st[11] ^= t; st[16] ^= t; st[21] ^= t;
+        t = bc1 ^ std::rotl(bc3, 1); st[2] ^= t; st[7] ^= t; st[12] ^= t; st[17] ^= t; st[22] ^= t;
+        t = bc2 ^ std::rotl(bc4, 1); st[3] ^= t; st[8] ^= t; st[13] ^= t; st[18] ^= t; st[23] ^= t;
+        t = bc3 ^ std::rotl(bc0, 1); st[4] ^= t; st[9] ^= t; st[14] ^= t; st[19] ^= t; st[24] ^= t;
 
         // Rho Pi
         t = st[1];
-        bc0 = st[10]; st[10] = Rotl(t, 1); t = bc0;
-        bc0 = st[7]; st[7] = Rotl(t, 3); t = bc0;
-        bc0 = st[11]; st[11] = Rotl(t, 6); t = bc0;
-        bc0 = st[17]; st[17] = Rotl(t, 10); t = bc0;
-        bc0 = st[18]; st[18] = Rotl(t, 15); t = bc0;
-        bc0 = st[3]; st[3] = Rotl(t, 21); t = bc0;
-        bc0 = st[5]; st[5] = Rotl(t, 28); t = bc0;
-        bc0 = st[16]; st[16] = Rotl(t, 36); t = bc0;
-        bc0 = st[8]; st[8] = Rotl(t, 45); t = bc0;
-        bc0 = st[21]; st[21] = Rotl(t, 55); t = bc0;
-        bc0 = st[24]; st[24] = Rotl(t, 2); t = bc0;
-        bc0 = st[4]; st[4] = Rotl(t, 14); t = bc0;
-        bc0 = st[15]; st[15] = Rotl(t, 27); t = bc0;
-        bc0 = st[23]; st[23] = Rotl(t, 41); t = bc0;
-        bc0 = st[19]; st[19] = Rotl(t, 56); t = bc0;
-        bc0 = st[13]; st[13] = Rotl(t, 8); t = bc0;
-        bc0 = st[12]; st[12] = Rotl(t, 25); t = bc0;
-        bc0 = st[2]; st[2] = Rotl(t, 43); t = bc0;
-        bc0 = st[20]; st[20] = Rotl(t, 62); t = bc0;
-        bc0 = st[14]; st[14] = Rotl(t, 18); t = bc0;
-        bc0 = st[22]; st[22] = Rotl(t, 39); t = bc0;
-        bc0 = st[9]; st[9] = Rotl(t, 61); t = bc0;
-        bc0 = st[6]; st[6] = Rotl(t, 20); t = bc0;
-        st[1] = Rotl(t, 44);
+        bc0 = st[10]; st[10] = std::rotl(t, 1); t = bc0;
+        bc0 = st[7]; st[7] = std::rotl(t, 3); t = bc0;
+        bc0 = st[11]; st[11] = std::rotl(t, 6); t = bc0;
+        bc0 = st[17]; st[17] = std::rotl(t, 10); t = bc0;
+        bc0 = st[18]; st[18] = std::rotl(t, 15); t = bc0;
+        bc0 = st[3]; st[3] = std::rotl(t, 21); t = bc0;
+        bc0 = st[5]; st[5] = std::rotl(t, 28); t = bc0;
+        bc0 = st[16]; st[16] = std::rotl(t, 36); t = bc0;
+        bc0 = st[8]; st[8] = std::rotl(t, 45); t = bc0;
+        bc0 = st[21]; st[21] = std::rotl(t, 55); t = bc0;
+        bc0 = st[24]; st[24] = std::rotl(t, 2); t = bc0;
+        bc0 = st[4]; st[4] = std::rotl(t, 14); t = bc0;
+        bc0 = st[15]; st[15] = std::rotl(t, 27); t = bc0;
+        bc0 = st[23]; st[23] = std::rotl(t, 41); t = bc0;
+        bc0 = st[19]; st[19] = std::rotl(t, 56); t = bc0;
+        bc0 = st[13]; st[13] = std::rotl(t, 8); t = bc0;
+        bc0 = st[12]; st[12] = std::rotl(t, 25); t = bc0;
+        bc0 = st[2]; st[2] = std::rotl(t, 43); t = bc0;
+        bc0 = st[20]; st[20] = std::rotl(t, 62); t = bc0;
+        bc0 = st[14]; st[14] = std::rotl(t, 18); t = bc0;
+        bc0 = st[22]; st[22] = std::rotl(t, 39); t = bc0;
+        bc0 = st[9]; st[9] = std::rotl(t, 61); t = bc0;
+        bc0 = st[6]; st[6] = std::rotl(t, 20); t = bc0;
+        st[1] = std::rotl(t, 44);
 
         // Chi Iota
         bc0 = st[0]; bc1 = st[1]; bc2 = st[2]; bc3 = st[3]; bc4 = st[4];

src/crypto/siphash.cpp

@@ -4,15 +4,15 @@
 
 #include <crypto/siphash.h>
 
-#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
+#include <bit>
 
 #define SIPROUND do { \
-    v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \
-    v0 = ROTL(v0, 32); \
-    v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \
-    v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \
-    v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \
-    v2 = ROTL(v2, 32); \
+    v0 += v1; v1 = std::rotl(v1, 13); v1 ^= v0; \
+    v0 = std::rotl(v0, 32); \
+    v2 += v3; v3 = std::rotl(v3, 16); v3 ^= v2; \
+    v0 += v3; v3 = std::rotl(v3, 21); v3 ^= v0; \
+    v2 += v1; v1 = std::rotl(v1, 17); v1 ^= v2; \
+    v2 = std::rotl(v2, 32); \
 } while (0)
 
 CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)

src/flat-database.h

@@ -175,7 +175,7 @@ class CFlatDB
 
 public:
     CFlatDB(std::string&& strFilenameIn, std::string&& strMagicMessageIn) :
-        pathDB{gArgs.GetDataDirNet() / strFilenameIn},
+        pathDB{gArgs.GetDataDirNet() / fs::u8path(strFilenameIn)},
         strFilename{strFilenameIn},
         strMagicMessage{strMagicMessageIn}
     {

src/flatfile.cpp

@@ -27,7 +27,7 @@ std::string FlatFilePos::ToString() const
 
 fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const
 {
-    return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile);
+    return m_dir / fs::u8path(strprintf("%s%05u.dat", m_prefix, pos.nFile));
 }
 
 FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only)