backport: bitcoin#24374, #24583, #24594, #24653, #24656, #24667, #24670, #24691, #24710

ci/test/00_setup_env_native_qt5.sh

@@ -15,4 +15,4 @@ export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"
 export GOAL="install"
 export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.12.1.5 v0.15.0.0 v0.16.1.1 v0.17.0.3 v18.2.2 v19.3.0 v20.1.1 v21.1.1"
-export BITCOIN_CONFIG="--enable-zmq --with-libs=no --enable-reduce-exports --disable-fuzz-binary LDFLAGS=-static-libstdc++ --with-boost-process"
+export BITCOIN_CONFIG="--enable-zmq --with-libs=no --enable-reduce-exports LDFLAGS=-static-libstdc++ --with-boost-process"

contrib/linearize/linearize-data.py

@@ -20,41 +20,9 @@
 
 settings = {}
 
-def hex_switchEndian(s):
-    """ Switches the endianness of a hex string (in pairs of hex chars) """
-    pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
-    return b''.join(pairList[::-1]).decode()
-
-def uint32(x):
-    return x & 0xffffffff
-
-def bytereverse(x):
-    return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
-               (((x) >> 8) & 0x0000ff00) | ((x) >> 24) ))
-
-def bufreverse(in_buf):
-    out_words = []
-    for i in range(0, len(in_buf), 4):
-        word = struct.unpack('@I', in_buf[i:i+4])[0]
-        out_words.append(struct.pack('@I', bytereverse(word)))
-    return b''.join(out_words)
-
-def wordreverse(in_buf):
-    out_words = []
-    for i in range(0, len(in_buf), 4):
-        out_words.append(in_buf[i:i+4])
-    out_words.reverse()
-    return b''.join(out_words)
-
-def calc_hdr_hash(blk_hdr):
-    return dash_hash.getPoWHash(blk_hdr)
-
 def calc_hash_str(blk_hdr):
-    hash = calc_hdr_hash(blk_hdr)
-    hash = bufreverse(hash)
-    hash = wordreverse(hash)
-    hash_str = hash.hex()
-    return hash_str
+    blk_hdr_hash = dash_hash.getPoWHash(blk_hdr)
+    return blk_hdr_hash[::-1].hex()
 
 def get_blk_dt(blk_hdr):
     members = struct.unpack("<I", blk_hdr[68:68+4])
@@ -70,7 +38,7 @@ def get_block_hashes(settings):
     for line in f:
         line = line.rstrip()
         if settings['rev_hash_bytes'] == 'true':
-            line = hex_switchEndian(line)
+            line = bytes.fromhex(line)[::-1].hex()
         blkindex.append(line)
 
     print("Read " + str(len(blkindex)) + " hashes")

contrib/linearize/linearize-hashes.py

@@ -17,11 +17,6 @@
 
 settings = {}
 
-def hex_switchEndian(s):
-    """ Switches the endianness of a hex string (in pairs of hex chars) """
-    pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
-    return b''.join(pairList[::-1]).decode()
-
 class BitcoinRPC:
     def __init__(self, host, port, username, password):
         authpair = "%s:%s" % (username, password)
@@ -85,7 +80,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
                 sys.exit(1)
             assert(resp_obj['id'] == x) # assume replies are in-sequence
             if settings['rev_hash_bytes'] == 'true':
-                resp_obj['result'] = hex_switchEndian(resp_obj['result'])
+                resp_obj['result'] = bytes.fromhex(resp_obj['result'])[::-1].hex()
             print(resp_obj['result'])
 
         height += num_blocks

doc/cjdns.md

@@ -10,7 +10,8 @@ CJDNS is like a distributed, shared VPN with multiple entry points where every
 participant can reach any other participant. All participants use addresses from
 the `fc00::/8` network (reserved IPv6 range). Installation and configuration is
 done outside of Dash Core, similarly to a VPN (either in the host/OS or on
-the network router).
+the network router). See https://github.com/cjdelisle/cjdns#readme and
+https://github.com/hyperboria/docs#hyperboriadocs for more information.
 
 Compared to IPv4/IPv6, CJDNS provides end-to-end encryption and protects nodes
 from traffic analysis and filtering.
@@ -23,17 +24,37 @@ somewhat centralized. I2P connections have a source address and I2P is slow.
 CJDNS is fast but does not hide the sender and the recipient from intermediate
 routers.
 
-## Installing CJDNS and connecting to the network
+## Installing CJDNS and finding a peer to connect to the network
 
 To install and set up CJDNS, follow the instructions at
-https://github.com/cjdelisle/cjdns#cjdns.
+https://github.com/cjdelisle/cjdns#how-to-install-cjdns.
 
-Don't skip steps
+You need to initiate an outbound connection to a peer on the CJDNS network
+before it will work with your Dash Core node. This is described in steps
 ["2. Find a friend"](https://github.com/cjdelisle/cjdns#2-find-a-friend) and
 ["3. Connect your node to your friend's
-node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node).
-You need to be connected to the CJDNS network before it will work with your
-Dash Core node.
+node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node)
+in the CJDNS documentation.
+
+One quick way to accomplish these two steps is to query for available public
+peers on [Hyperboria](https://github.com/hyperboria) by running the following:
+
+```
+git clone https://github.com/hyperboria/peers hyperboria-peers
+cd hyperboria-peers
+./testAvailable.py
+```
+
+For each peer, the `./testAvailable.py` script prints the filename of the peer's
+credentials followed by the ping result.
+
+Choose one or several peers, copy their credentials from their respective files,
+paste them into the relevant IPv4 or IPv6 "connectTo" JSON object in the
+`cjdroute.conf` file you created in step ["1. Generate a new configuration
+file"](https://github.com/cjdelisle/cjdns#1-generate-a-new-configuration-file),
+and save the file.
+
+## Launching CJDNS
 
 Typically, CJDNS might be launched from its directory with
 `sudo ./cjdroute < cjdroute.conf` and it sheds permissions after setting up the

doc/release-notes-empty-template.md

@@ -0,0 +1,100 @@
+*The release notes draft is a temporary file that can be added to by anyone. See
+[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes)
+for the process.*
+
+# Dash Core version *version*
+===============================
+
+This is a new minor version release, bringing various bugfixes and performance improvements.
+This release is **optional** for all nodes, although recommended.
+
+Please report bugs using the issue tracker at GitHub:
+
+  <https://github.com/dashpay/dash/issues>
+
+
+# Upgrading and downgrading
+
+## How to Upgrade
+
+If you are running an older version, shut it down. Wait until it has completely
+shut down (which might take a few minutes in some cases), then run the
+installer (on Windows) or just copy over `/Applications/Dash-Qt` (on Mac) or
+`dashd`/`dash-qt` (on Linux).
+
+## Downgrade warning
+
+### Downgrade to a version < *version*
+
+Downgrading to a version older than *version* may not be supported, and will
+likely require a reindex.
+
+# Release Notes
+
+Notable changes
+===============
+
+P2P and network changes
+-----------------------
+
+Updated RPCs
+------------
+
+
+Changes to wallet related RPCs can be found in the Wallet section below.
+
+New RPCs
+--------
+
+Build System
+------------
+
+Updated settings
+----------------
+
+
+Changes to GUI or wallet related settings can be found in the GUI or Wallet section below.
+
+New settings
+------------
+
+Tools and Utilities
+-------------------
+
+Wallet
+------
+
+GUI changes
+-----------
+
+Low-level changes
+=================
+
+RPC
+---
+
+Tests
+-----
+
+See detailed [set of changes][set-of-changes].
+
+# Credits
+
+Thanks to everyone who directly contributed to this release:
+
+-
+-
+-
+
+As well as everyone that submitted issues, reviewed pull requests and helped
+debug the release candidates.
+
+# Older releases
+
+These releases are considered obsolete. Old release notes can be found here:
+
+-
+-
+-
+
+[set-of-changes]: https://github.com/dashpay/dash/compare/*version*...dashpay:*version*

doc/release-process.md

@@ -9,7 +9,7 @@ Before every minor and major release:
 * [ ] Review ["Needs backport" labels](https://github.com/dashpay/dash/labels?q=backport).
 * [ ] Update DIPs with any changes introduced by this release (see [this pull request](https://github.com/dashpay/dips/pull/142) for an example)
 * [ ] Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_IS_RELEASE` to `true`)
-* [ ] Write release notes (see below)
+* [ ] Write release notes (see below). To clear the release notes: `cp doc/release-notes-empty-template.md doc/release-notes.md`
 * [ ] Update `src/chainparams.cpp` `nMinimumChainWork` with information from the `getblockchaininfo` rpc.
 * [ ] Update `src/chainparams.cpp` `defaultAssumeValid` with information from the `getblockhash` rpc.
   - The selected value must not be orphaned so it may be useful to set the value two blocks back from the tip.
@@ -83,32 +83,28 @@ against other `guix-attest` signatures.
 git -C ./guix.sigs pull
 ```
 
-### Create the macOS SDK tarball: (first time, or when SDK version changes)
+### Create the macOS SDK tarball (first time, or when SDK version changes)
 
 _Note: this step can be skipped if [our CI](https://github.com/dashpay/dash/blob/master/ci/test/00_setup_env.sh#L64) still uses bitcoin's SDK package (see SDK_URL)_
 
 Create the macOS SDK tarball, see the [macOS build
 instructions](build-osx.md#deterministic-macos-app-notes) for
 details.
 
-### Build and attest to build outputs:
+### Build and attest to build outputs
 
 Follow the relevant Guix README.md sections:
 - [Building](/contrib/guix/README.md#building)
 - [Attesting to build outputs](/contrib/guix/README.md#attesting-to-build-outputs)
 
 _Note: we ship releases for only some supported HOSTs so consider providing limited `HOSTS` variable or run `./contrib/containers/guix/scripts/guix-start` instead of `./contrib/guix/guix-build` when building binaries for quicker builds that exclude the supported but not shipped HOSTs_
 
-### Verify other builders' signatures to your own. (Optional)
+### Verify other builders' signatures to your own (optional)
 
-Add other builders keys to your gpg keyring, and/or refresh keys: See `../dash/contrib/builder-keys/README.md`.
-
-Follow the relevant Guix README.md sections:
+- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md)
 - [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations)
 
-### Next steps:
-
-Commit your signature to `guix.sigs`:
+### Commit your non codesigned signature to guix.sigs
 
 ```sh
 pushd guix.sigs
@@ -118,24 +114,22 @@ git push  # Assuming you can push to the guix.sigs tree
 popd
 ```
 
-Codesigner only: Create Windows/macOS detached signatures:
-- Only one person handles codesigning. Everyone else should skip to the next step.
-- Only once the Windows/macOS builds each have 3 matching signatures may they be signed with their respective release keys.
+## Codesigning
 
-Codesigner only: Sign the macOS binary:
+### macOS codesigner only: Create detached macOS signatures (assuming [signapple](https://github.com/achow101/signapple/) is installed and up to date with master branch)
 
 * Transfer `dashcore-osx-unsigned.tar.gz` to macOS for signing
 * Extract and sign:
 
     ```sh
     tar xf dashcore-osx-unsigned.tar.gz
-    ./detached-sig-create.sh -s "Key ID" -o runtime
+    ./detached-sig-create.sh /path/to/codesign.p12 -o runtime
     ```
 
 * Enter the keychain password and authorize the signature
-* Move `signature-osx.tar.gz` back to the guix-build host
+* `signature-osx.tar.gz` will be created
 
-Codesigner only: Sign the windows binaries:
+### Windows codesigner only: Create detached Windows signatures
 
 * Extract and sign:
 
@@ -147,10 +141,11 @@ Codesigner only: Sign the windows binaries:
 * Enter the passphrase for the key when prompted
 * `signature-win.tar.gz` will be created
 
-Code-signer only: It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step.
-However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds.
+### Windows and macOS codesigners only: test code signatures
+It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step.
+However if this is done, once the release has been tagged in the dash-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds.
 
-Codesigner only: Commit the detached codesign payloads:
+### Windows and macOS codesigners only: Commit the detached codesign payloads
 
 ```sh
 pushd ~/dashcore-detached-sigs
@@ -165,15 +160,20 @@ git push
 popd
 ```
 
-Non-codesigners: wait for Windows/macOS detached signatures:
+### Non-codesigners: wait for Windows and macOS detached signatures
 
-- Once the Windows/macOS builds each have 3 matching signatures, they will be signed with their respective release keys.
+- Once the Windows and macOS builds each have 3 matching signatures, they will be signed with their respective release keys.
 - Detached signatures will then be committed to the [dash-detached-sigs](https://github.com/dashpay/dash-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries.
 
-Create (and optionally verify) the codesigned outputs:
-- [Codesigning](/contrib/guix/README.md#codesigning)
+### Create the codesigned build outputs
+- [Codesigning build outputs](/contrib/guix/README.md#codesigning-build-outputs)
+
+### Verify other builders' signatures to your own (optional)
+
+- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md)
+- [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations)
 
-Commit your signature for the signed macOS/Windows binaries:
+### Commit your codesigned signature to guix.sigs (for the signed macOS/Windows binaries)
 
 ```sh
 pushd ./guix.sigs
@@ -183,7 +183,7 @@ git push  # Assuming you can push to the guix.sigs tree
 popd
 ```
 
-### After 3 or more people have guix-built and their results match:
+## After 3 or more people have guix-built and their results match
 
 * [ ] Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`:
     ```sh