Merge #17119,1683,17102,17134,17691,17992,18170,18382,16975,18472, 18486,17633,7833

contrib/devtools/circular-dependencies.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import sys
 import re

contrib/devtools/copyright_header.py

@@ -20,6 +20,8 @@
     'src/qt/bitcoinstrings.cpp',
     'src/chainparamsseeds.h',
     # other external copyrights:
+    'src/reverse_iterator.h',
+    'src/test/fuzz/FuzzedDataProvider.h',
     'src/tinyformat.h',
     'src/bench/nanobench.h',
     'test/functional/test_framework/bignum.py',
@@ -459,14 +461,14 @@ def get_header_lines(header, start_year, end_year):
 def get_cpp_header_lines_to_insert(start_year, end_year):
     return reversed(get_header_lines(CPP_HEADER, start_year, end_year))
 
-PYTHON_HEADER = '''
+SCRIPT_HEADER = '''
 # Copyright (c) %s The Dash Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
 
-def get_python_header_lines_to_insert(start_year, end_year):
-    return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year))
+def get_script_header_lines_to_insert(start_year, end_year):
+    return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year))
 
 ################################################################################
 # query git for year of last change
@@ -495,17 +497,18 @@ def file_has_hashbang(file_lines):
         return False
     return file_lines[0][:2] == '#!'
 
-def insert_python_header(filename, file_lines, start_year, end_year):
+def insert_script_header(filename, file_lines, start_year, end_year):
     if file_has_hashbang(file_lines):
         insert_idx = 1
     else:
         insert_idx = 0
-    header_lines = get_python_header_lines_to_insert(start_year, end_year)
+    header_lines = get_script_header_lines_to_insert(start_year, end_year)
     for line in header_lines:
         file_lines.insert(insert_idx, line)
     write_file_lines(filename, file_lines)
 
 def insert_cpp_header(filename, file_lines, start_year, end_year):
+    file_lines.insert(0, '\n')
     header_lines = get_cpp_header_lines_to_insert(start_year, end_year)
     for line in header_lines:
         file_lines.insert(0, line)
@@ -517,8 +520,8 @@ def exec_insert_header(filename, style):
         sys.exit('*** %s already has a copyright by The Dash Core developers'
                  % (filename))
     start_year, end_year = get_git_change_year_range(filename)
-    if style == 'python':
-        insert_python_header(filename, file_lines, start_year, end_year)
+    if style in ['python', 'shell']:
+        insert_script_header(filename, file_lines, start_year, end_year)
     else:
         insert_cpp_header(filename, file_lines, start_year, end_year)
 
@@ -559,11 +562,13 @@ def insert_cmd(argv):
     if not os.path.isfile(filename):
         sys.exit("*** bad filename: %s" % filename)
     _, extension = os.path.splitext(filename)
-    if extension not in ['.h', '.cpp', '.cc', '.c', '.py']:
+    if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']:
         sys.exit("*** cannot insert for file extension %s" % extension)
 
     if extension == '.py':
         style = 'python'
+    elif extension == '.sh':
+        style = 'shell'
     else:
         style = 'cpp'
     exec_insert_header(filename, style)

contrib/devtools/gen-manpages.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+# Copyright (c) 2016-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C
 TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)}

contrib/filter-lcov.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# Copyright (c) 2017-2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import argparse
 

contrib/gitian-build.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# Copyright (c) 2018-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import argparse
 import os

doc/developer-notes.md

@@ -95,7 +95,6 @@ code.
   - `nullptr` is preferred over `NULL` or `(void*)0`.
   - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking.
   - Align pointers and references to the left i.e. use `type& var` and not `type &var`.
-  - `enum class` is preferred over `enum` where possible. Scoped enumerations avoid two potential pitfalls/problems with traditional C++ enumerations: implicit conversions to int, and name clashes due to enumerators being exported to the surrounding scope.
 
 Block style example:
 ```c++
@@ -257,6 +256,7 @@ $ valgrind --suppressions=contrib/valgrind.supp src/test/test_dash
 $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \
       --show-leak-kinds=all src/test/test_dash --log_level=test_suite
 $ valgrind -v --leak-check=full src/dashd -printtoconsole
+$ ./test/functional/test_runner.py --valgrind
 ```
 
 ### Compiling for test coverage
@@ -582,6 +582,34 @@ class A
    int. If the signed int is some negative `N`, it'll become `INT_MAX - N`  which might cause unexpected consequences.
 
 
+- Prefer `enum class` (scoped enumerations) over `enum` (traditional enumerations) where possible.
+
+  - *Rationale*: Scoped enumerations avoid two potential pitfalls/problems with traditional C++ enumerations: implicit conversions to `int`, and name clashes due to enumerators being exported to the surrounding scope.
+
+- `switch` statement on an enumeration example:
+
+```cpp
+enum class Tabs {
+    INFO,
+    CONSOLE,
+    GRAPH,
+    PEERS
+};
+
+int GetInt(Tabs tab)
+{
+    switch (tab) {
+    case Tabs::INFO: return 0;
+    case Tabs::CONSOLE: return 1;
+    case Tabs::GRAPH: return 2;
+    case Tabs::PEERS: return 3;
+    } // no default case, so the compiler can warn about missing cases
+    assert(false);
+}
+```
+
+*Rationale*: The comment documents skipping `default:` label, and it complies with `clang-format` rules. The assertion prevents firing of `-Wreturn-type` warning on some compilers.
+
 Strings and formatting
 ------------------------
 

doc/files.md

@@ -1,27 +1,107 @@
+# Dash Core file system
+
+**Contents**
+
+- [Data directory location](#data-directory-location)
+
+- [Data directory layout](#data-directory-layout)
+
+- [Multi-wallet environment](#multi-wallet-environment)
+
+- [GUI settings](#gui-settings)
+
+- [Legacy subdirectories and files](#legacy-subdirectories-and-files)
+
+- [Notes](#notes)
+
+## Data directory location
+
+The data directory is the default location where the Dash Core files are stored.
+
+1. The default data directory paths for supported platforms are:
+
+Platform | Data directory path
+---------|--------------------
+Linux    | `$HOME/.dashcore/`
+macOS    | `$HOME/Library/Application Support/Dashcore/`
+Windows  | `%APPDATA%\Dashcore\` <sup>[\[1\]](#note1)</sup>
+
+2. The non-default data directory path can be specified by `-datadir` option.
+
+3. All content of the data directory, except for `dash.conf` file, is chain-specific. This means the actual data directory paths for non-mainnet cases differ:
+
+Chain option        | Data directory path
+--------------------|--------------------
+no option (mainnet) | *path_to_datadir*`/`
+`-testnet`          | *path_to_datadir*`/testnet3/`
+`-regtest`          | *path_to_datadir*`/regtest/`
+
+## Data directory layout
+
+Subdirectory       | File(s)               | Description
+-------------------|-----------------------|------------
+`blocks/`          |                       | Blocks directory; can be specified by `-blocksdir` option (except for `blocks/index/`)
+`blocks/index/`    | LevelDB database      | Block index; `-blocksdir` option does not affect this path
+`blocks/`          | `blkNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Actual Dash blocks (in network format, dumped in raw on disk, 128 MiB per file)
+`blocks/`          | `revNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Block undo data (custom format)
+`chainstate/`      | LevelDB database      | Blockchain state (a compact representation of all currently unspent transaction outputs and some metadata about the transactions they are from)
+`indexes/txindex/` | LevelDB database      | Transaction index; *optional*, used if `-txindex=1`
+`indexes/blockfilter/basic/db/` | LevelDB database      | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
+`indexes/blockfilter/basic/`    | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
+`wallets/`         |                       | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, a wallet resides in the data directory
+`evodb/`         |                       |special txes and quorums database
+`llmq/`          |                       |quorum signatures database
+`./`               | `banlist.dat`         | Stores the IPs/subnets of banned nodes
+`./`               | `dash.conf`        | Contains [configuration settings](dash-conf.md) for `dashd` or `dash-qt`; can be specified by `-conf` option
+`./`               | `dashd.pid`        | Stores the process ID (PID) of `dashd` or `dash-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option
+`./`               | `debug.log`           | Contains debug information and general logging generated by `dashd` or `dash-qt`; can be specified by `-debuglogfile` option
+`./`               | `governance.dat`      | stores data for governance objects
+`./`               | `mncache.dat`         | stores data for masternode list
+`./`               | `netfulfilled.dat`    | stores data about recently made network requests
+`./`               | `fee_estimates.dat`   | Stores statistics used to estimate minimum transaction fees and priorities required for confirmation
+`./`               | `guisettings.ini.bak` | Backup of former [GUI settings](#gui-settings) after `-resetguisettings` option is used
+`./`               | `mempool.dat`         | Dump of the mempool's transactions
+`./`               | `onion_v3_private_key` | Cached Tor hidden service private key for `-listenonion` option
+`./`               | `peers.dat`           | Peer IP address database (custom format)
+`./`               | `.cookie`             | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option
+`./`               | `.lock`               | Data directory lock file
+
+## Multi-wallet environment
+
+Wallets are Berkeley DB (BDB) databases:
+
+Subdirectory | File(s)           | Description
+-------------|-------------------|------------
+`database/`  | BDB logging files | Part of BDB environment; created at start and deleted on shutdown; a user *must keep it as safe* as personal wallet `wallet.dat`
+`./`         | `db.log`          | BDB error file
+`./`         | `wallet.dat`      | Personal wallet (BDB) with keys and transactions
+`./`         | `.walletlock`     | Wallet lock file
+
+1. Each user-defined wallet named "wallet_name" resides in `wallets/wallet_name/` subdirectory.
+
+2. The default (unnamed) wallet resides in `wallets/` subdirectory; if the latter does not exist, the wallet resides in the data directory.
+
+3. A wallet database path can be specified by `-wallet` option.
+
+## GUI settings
+
+`dash-qt` uses [`QSettings`](https://doc.qt.io/qt-5/qsettings.html) class; this implies platform-specific [locations where application settings are stored](https://doc.qt.io/qt-5/qsettings.html#locations-where-application-settings-are-stored).
+
+## Legacy subdirectories and files
+
+These subdirectories and files are no longer used by the Dash Core:
+
+Path           | Description | Repository notes
+---------------|-------------|-----------------
+`blktree/`     | Blockchain index; replaced by `blocks/index/` in [0.8.0](https://github.com/dash/dash/blob/master/doc/release-notes/release-notes-0.8.0.md#improvements) | [PR #2231](https://github.com/dash/dash/pull/2231), [`8fdc94cc`](https://github.com/dash/dash/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15)
+`coins/`       | Unspent transaction output database; replaced by `chainstate/` in 0.8.0 | [PR #2231](https://github.com/dash/dash/pull/2231), [`8fdc94cc`](https://github.com/dash/dash/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15)
+`blkindex.dat` | Blockchain index BDB database; replaced by {`chainstate/`, `blocks/index/`, `blocks/revNNNNN.dat`<sup>[\[2\]](#note2)</sup>} in 0.8.0 | [PR #1677](https://github.com/dash/dash/pull/1677)
+`blk000?.dat`  | Block data (custom format, 2 GiB per file); replaced by `blocks/blkNNNNN.dat`<sup>[\[2\]](#note2)</sup> in 0.8.0 | [PR #1677](https://github.com/dash/dash/pull/1677)
+`addr.dat`     | Peer IP address BDB database; replaced by `peers.dat` in [0.7.0](https://github.com/dash/dash/blob/master/doc/release-notes/release-notes-0.7.0.md) | [PR #1198](https://github.com/dash/dash/pull/1198), [`928d3a01`](https://github.com/dash/dash/commit/928d3a011cc66c7f907c4d053f674ea77dc611cc)
+
+## Notes
+
+<a name="note1">1</a>. The `/` (slash, U+002F) is used as the platform-independent path component separator in this paper.
+
+<a name="note2">2</a>. `NNNNN` matches `[0-9]{5}` regex.
 
-* banlist.dat: stores the IPs/Subnets of banned nodes
-* blocks/blk000??.dat: block data (custom, 128 MiB per file)
-* blocks/rev000??.dat; block undo data (custom)
-* blocks/index/*; block index (LevelDB)
-* chainstate/*; block chain state database (LevelDB)
-* dash.conf: contains configuration settings for dashd or dash-qt
-* dashd.pid: stores the process id of dashd while running
-* database/*: BDB database environment; only used for wallet; moved to wallets/ directory on new installs since 0.16.0
-* db.log: wallet database log file; moved to wallets/ directory on new installs since 0.16.0
-* debug.log: contains debug information and general logging generated by dashd or dash-qt
-* evodb/*: special txes and quorums database
-* fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation
-* governance.dat: stores data for governance objects
-* indexes/txindex/*: optional transaction index database (LevelDB); since 0.17.0
-* llmq/*: quorum signatures database
-* mempool.dat: dump of the mempool's transactions
-* mncache.dat: stores data for masternode list
-* netfulfilled.dat: stores data about recently made network requests
-* peers.dat: peer IP address database (custom format)
-* wallet.dat: personal wallet (BDB) with keys and transactions; moved to wallets/ directory on new installs since 0.16.0
-* wallets/database/*: BDB database environment; used for wallets since 0.16.0
-* wallets/db.log: wallet database log file; since 0.16.0
-* wallets/wallet.dat: personal wallet (BDB) with keys and transactions; since 0.16.0
-* .cookie: session RPC authentication cookie (written at start when cookie authentication is used, deleted on shutdown)
-* onion_v3_private_key: cached Tor hidden service private key for `-listenonion`
-* guisettings.ini.bak: backup of former GUI settings after `-resetguisettings` is used

src/.clang-format

@@ -5,6 +5,7 @@ AlignEscapedNewlinesLeft: true
 AlignTrailingComments: true
 AllowAllParametersOfDeclarationOnNextLine: true
 AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: true
 AllowShortFunctionsOnASingleLine: All
 AllowShortIfStatementsOnASingleLine: true
 AllowShortLoopsOnASingleLine: false

src/Makefile.test.include

@@ -685,7 +685,7 @@ endif
 
 %.cpp.test: %.cpp
 	@echo Running tests: `cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1` from $<
-	$(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" > $<.log 2>&1 || (cat $<.log && false)
+	$(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" -- DEBUG_LOG_OUT > $<.log 2>&1 || (cat $<.log && false)
 
 test/data/%.json.h: test/data/%.json
 	@$(MKDIR_P) $(@D)

src/bench/bench.cpp

@@ -13,6 +13,7 @@
 #include <iostream>
 #include <regex>
 
+const std::function<void(const std::string&)> G_TEST_LOG_FUN{};
 namespace {
 
 void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& benchmarkResults, const std::string& filename, const char* tpl)

src/crypto/sha256_avx2.cpp

@@ -1,3 +1,7 @@
+// Copyright (c) 2017-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #ifdef ENABLE_AVX2
 
 #include <stdint.h>

src/crypto/sha256_sse41.cpp

@@ -1,3 +1,7 @@
+// Copyright (c) 2018-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #ifdef ENABLE_SSE41
 
 #include <stdint.h>

src/fs.cpp

@@ -1,3 +1,7 @@
+// Copyright (c) 2017-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #include <fs.h>
 
 #ifndef WIN32

src/qt/test/addressbooktests.cpp

@@ -1,3 +1,7 @@
+// Copyright (c) 2017-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #include <qt/test/addressbooktests.h>
 #include <qt/test/util.h>
 #include <test/util/setup_common.h>

src/qt/test/addressbooktests.h

@@ -1,3 +1,7 @@
+// Copyright (c) 2018-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #ifndef BITCOIN_QT_TEST_ADDRESSBOOKTESTS_H
 #define BITCOIN_QT_TEST_ADDRESSBOOKTESTS_H
 

src/qt/test/test_main.cpp

@@ -46,6 +46,8 @@ Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin);
 #endif
 #endif
 
+const std::function<void(const std::string&)> G_TEST_LOG_FUN{};
+
 // This is all you need to run all the tests
 int main(int argc, char *argv[])
 {

src/qt/test/util.cpp

@@ -1,3 +1,7 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #include <QApplication>
 #include <QMessageBox>
 #include <QPushButton>

src/qt/test/util.h

@@ -1,3 +1,7 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #ifndef BITCOIN_QT_TEST_UTIL_H
 #define BITCOIN_QT_TEST_UTIL_H
 

src/qt/test/wallettests.cpp

@@ -1,3 +1,7 @@
+// Copyright (c) 2015-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #include <qt/test/wallettests.h>
 #include <qt/test/util.h>
 

src/qt/test/wallettests.h

@@ -1,3 +1,7 @@
+// Copyright (c) 2017-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #ifndef BITCOIN_QT_TEST_WALLETTESTS_H
 #define BITCOIN_QT_TEST_WALLETTESTS_H