bench: introduce -min_time argument

When it is not easily possible to stabilize benchmark machine and code
the argument -min_time can be used to specify a minimum duration
that a benchmark should take. E.g. choose -min_time=1000 if you
are willing to wait about 1 second for each benchmark result.

The default is now set to 10ms instead of 0, which should make runs on
fast machines more stable with negligible slowdown.

Author: martinus

src/bench/bench.cpp

@@ -61,6 +61,12 @@ void benchmark::BenchRunner::RunAll(const Args& args)
 
         Bench bench;
         bench.name(p.first);
+        if (args.min_time > 0ms) {
+            // convert to nanos before dividing to reduce rounding errors
+            std::chrono::nanoseconds min_time_ns = args.min_time;
+            bench.minEpochTime(min_time_ns / bench.epochs());
+        }
+
         if (args.asymptote.empty()) {
             p.second(bench);
         } else {

src/bench/bench.h

@@ -43,6 +43,7 @@ typedef std::function<void(Bench&)> BenchFunction;
 struct Args {
     std::string regex_filter;
     bool is_list_only;
+    std::chrono::milliseconds min_time;
     std::vector<double> asymptote;
     std::string output_csv;
     std::string output_json;

src/bench/bench_bitcoin.cpp

@@ -11,6 +11,7 @@
 #include <memory>
 
 static const char* DEFAULT_BENCH_FILTER = ".*";
+static constexpr int64_t DEFAULT_MIN_TIME_MS{10};
 
 static void SetupBenchArgs(ArgsManager& argsman)
 {
@@ -19,6 +20,7 @@ static void SetupBenchArgs(ArgsManager& argsman)
     argsman.AddArg("-asymptote=n1,n2,n3,...", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-min_time=<milliseconds>", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_INT, OptionsCategory::OPTIONS);
     argsman.AddArg("-output_csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-output_json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 }
@@ -57,6 +59,7 @@ int main(int argc, char** argv)
     args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
     args.is_list_only = argsman.GetBoolArg("-list", false);
     args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", ""));
+    args.min_time = std::chrono::milliseconds(argsman.GetArg("-min_time", DEFAULT_MIN_TIME_MS));
     args.output_csv = argsman.GetArg("-output_csv", "");
     args.output_json = argsman.GetArg("-output_json", "");