From 04f53c432be44a70e8281e81d7e1706f5e4a180d Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 11:16:09 -0500 Subject: [PATCH 01/34] Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + x: number of iterations + s: minimum number of seconds. This matches the internal benchmark API. --- include/benchmark/benchmark.h | 3 +- src/benchmark.cc | 22 ++++++-- src/benchmark_runner.cc | 65 +++++++++++++++++++++- test/BUILD | 23 ++++---- test/benchmark_min_time_flag_iters_test.cc | 65 ++++++++++++++++++++++ 5 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 test/benchmark_min_time_flag_iters_test.cc diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index e8b1a7d6d4..1222aedcee 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1081,11 +1081,12 @@ class BENCHMARK_EXPORT Benchmark { Benchmark* MinWarmUpTime(double t); // Specify the amount of iterations that should be run by this benchmark. + // This option overrides the `benchmark_min_time` flag. // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. // // NOTE: This function should only be used when *exact* iteration control is // needed and never to control or limit how long a benchmark runs, where - // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. + // `--benchmark_min_time=s` or `MinTime(...)` should be used instead. Benchmark* Iterations(IterationCount n); // Specify the amount of times to repeat this benchmark. This option overrides diff --git a/src/benchmark.cc b/src/benchmark.cc index 539f0de429..7607bebfd7 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -65,12 +65,24 @@ BM_DEFINE_bool(benchmark_list_tests, false); // linked into the binary are run. BM_DEFINE_string(benchmark_filter, ""); -// Minimum number of seconds we should run benchmark before results are -// considered significant. For cpu-time based tests, this is the lower bound +// Specification of how long to run the benchmark. +// +// It can be either an exact number of iterations (specified as `x`), +// or a minimum number of seconds (specified as `s`). If the latter +// format (ie., min seconds) is used, the system may run the benchmark longer +// until the results are considered significatn. +// +// For backward compatibility, the `s` suffix may be omitted, in which case, +// the specified number is interpreted as the number of seconds. +// +// For cpu-time based tests, this is the lower bound // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -BM_DEFINE_double(benchmark_min_time, 0.5); +// +// FIXME: Renmae this flag to --benchmark_time for clarity (and for consistency +// with other Go's) +BM_DEFINE_double(benchmark_min_time, "0.5s"); // Minimum number of seconds a benchmark should be run before results should be // taken into account. This e.g can be necessary for benchmarks of code which @@ -605,7 +617,7 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || - ParseDoubleFlag(argv[i], "benchmark_min_time", + ParseStringFlag(argv[i], "benchmark_min_time", &FLAGS_benchmark_min_time) || ParseDoubleFlag(argv[i], "benchmark_min_warmup_time", &FLAGS_benchmark_min_warmup_time) || @@ -666,7 +678,7 @@ void PrintDefaultHelp() { "benchmark" " [--benchmark_list_tests={true|false}]\n" " [--benchmark_filter=]\n" - " [--benchmark_min_time=]\n" + " [--benchmark_min_time=`x` OR `s` ]\n" " [--benchmark_min_warmup_time=]\n" " [--benchmark_repetitions=]\n" " [--benchmark_enable_random_interleaving={true|false}]\n" diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 04e5c2a758..83aeee53d2 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -37,6 +37,7 @@ #include #include #include +#include #include "check.h" #include "colorprint.h" @@ -140,6 +141,63 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } +std::variant ParseBenchMinTime(const std::string& value) { + if (value.empty()) return 0; + + if (value.back() == 'x') { + std::string num_iters_str = value.substr(0, value.length() - 1); + int num_iters = std::atoi(num_iters_str); + + // std::atoi doesn't provide useful error messages, so we do some + // sanity checks. + // Also check that the iters is not negative. + BM_CHECK(num_iters > 0 && !(num_iters == 0 && num_iters_str != "0")) + << "Malformed iters value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + return num_iters; + } + + std::string min_time_str; + if (value.back() != 's') { + BM_VLOG(2) << "Value passed to --benchmark_min_time should have a suffix. " + "Eg., `30s` for 30-seconds."; + min_time_str = value; + } else { + min_time_str = value.substr(0, value.length() - 1); + } + double min_time = std::atof(min_time_str); + + BM_CHECK(min_time > 0 && + !(min_time == 0 && (min_time_str != "0" || min_time_str != "0.0"))) + << "Malformed seconds value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + return min_time; +} + +double GetMinTime(const benchmark::internal::BenchmarkInstance& b) { + if (!IsZero(b.min_time())) return b.min_time(); + + std::variant iters_or_time = + ParseBenchMinTime(FLAGS_benchmark_min_time); + + // If the flag was used to specify number of iters, then return 0 for time. + if (iters_or_time.index() == 0) return 0; + return std::get(iters_or_time); +} + +bool BenchMinTimeHasIters() { + return !FLAGS_benchmark_min_time.empty() && + FLAGS_benchmark_min_time.empty.back() == 'x'; +} + +int GetIters(const benchmark::internal::BenchmarkInstance& b) { + if (b.iterations() != 0) return b.iterations(); + + // We've already checked that this flag is currently used to pass + // iters. + return std::get(ParseBenchMinTime(FLAGS_benchmark_min_time)); +} + } // end namespace BenchmarkRunner::BenchmarkRunner( @@ -147,16 +205,17 @@ BenchmarkRunner::BenchmarkRunner( BenchmarkReporter::PerFamilyRunReports* reports_for_family_) : b(b_), reports_for_family(reports_for_family_), - min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), + min_time(GetMinTime(b_)), min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) ? b.min_warmup_time() : FLAGS_benchmark_min_warmup_time), warmup_done(!(min_warmup_time > 0.0)), repeats(b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions), - has_explicit_iteration_count(b.iterations() != 0), + has_explicit_iteration_count(b.iterations() != 0 || + BenchMinTimeHasIters()), pool(b.threads() - 1), - iters(has_explicit_iteration_count ? b.iterations() : 1), + iters(has_explicit_iteration_count ? GetIters(b_) : 1), perf_counters_measurement(StrSplit(FLAGS_benchmark_perf_counters, ',')), perf_counters_measurement_ptr(perf_counters_measurement.IsValid() ? &perf_counters_measurement diff --git a/test/BUILD b/test/BUILD index 0a66bf3d53..1909b0a133 100644 --- a/test/BUILD +++ b/test/BUILD @@ -25,14 +25,15 @@ PER_SRC_COPTS = { "donotoptimize_test.cc": ["-O3"], } -TEST_ARGS = ["--benchmark_min_time=0.01"] +TEST_ARGS = ["--benchmark_min_time=0.01s"] -PER_SRC_TEST_ARGS = ({ +PER_SRC_TEST_ARGS = { "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], "repetitions_test.cc": [" --benchmark_repetitions=3"], - "spec_arg_test.cc" : ["--benchmark_filter=BM_NotChosen"], - "spec_arg_verbosity_test.cc" : ["--v=42"], -}) + "spec_arg_test.cc": ["--benchmark_filter=BM_NotChosen"], + "spec_arg_verbosity_test.cc": ["--v=42"], + "benchmark_min_time_flag_iters_test.cc": ["--benchmark_min_time=4x"], +} cc_library( name = "output_test_helper", @@ -58,14 +59,14 @@ cc_library( copts = select({ "//:windows": [], "//conditions:default": TEST_COPTS, - }) + PER_SRC_COPTS.get(test_src, []) , + }) + PER_SRC_COPTS.get(test_src, []), deps = [ ":output_test_helper", "//:benchmark", "//:benchmark_internal_headers", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", - ] + ], # FIXME: Add support for assembly tests to bazel. # See Issue #556 # https://github.com/google/benchmark/issues/556 @@ -85,6 +86,10 @@ cc_test( size = "small", srcs = ["cxx03_test.cc"], copts = TEST_COPTS + ["-std=c++03"], + target_compatible_with = select({ + "//:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), deps = [ ":output_test_helper", "//:benchmark", @@ -92,10 +97,6 @@ cc_test( "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], - target_compatible_with = select({ - "//:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }) ) cc_test( diff --git a/test/benchmark_min_time_flag_iters_test.cc b/test/benchmark_min_time_flag_iters_test.cc new file mode 100644 index 0000000000..40498b86d5 --- /dev/null +++ b/test/benchmark_min_time_flag_iters_test.cc @@ -0,0 +1,65 @@ +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +// Tests that we can specify the number of iterations with +// --benchmark_min_time=x. +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + return ConsoleReporter::ReportContext(context); + }; + + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + assert(report.size() == 1); + iter_nums.push_back(report[0].iterations); + ConsoleReporter::ReportRuns(report); + }; + + TestReporter() {} + + virtual ~TestReporter() {} + + const std::vector& GetIters() const { return iter_nums_; } + + private: + std::vector iter_nums_; +}; + +} // end namespace + +static void BM_MyBench(benchmark::State& state) { + for (auto s : state) { + } +} +BENCHMARK(BM_MyBench); + +int main(int argc, char** argv) { + // Verify that argv specify --benchmark_min_time=4x + bool found = false; + for (int i = 0; i < argc; ++i) { + if (strcmp("--benchmark_min_time=4x", argv[i]) == 0) { + found = true; + break; + } + } + assert(found); + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench"); + assert(returned_count == 1); + + // Check the executed iters. + const std::vector iters = test_reporter.GetIters(); + assert(!iters.empty() && iters[0] == 4); + return 0; +} From 7e2aeaa61baa0710b8a8f9b356f3cdc4dc3e9e0f Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 11:22:55 -0500 Subject: [PATCH 02/34] forgot to change flag type to string --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 7607bebfd7..7483b80774 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -82,7 +82,7 @@ BM_DEFINE_string(benchmark_filter, ""); // // FIXME: Renmae this flag to --benchmark_time for clarity (and for consistency // with other Go's) -BM_DEFINE_double(benchmark_min_time, "0.5s"); +BM_DEFINE_string(benchmark_min_time, "0.5s"); // Minimum number of seconds a benchmark should be run before results should be // taken into account. This e.g can be necessary for benchmarks of code which From 0ba28c5ee174dbe895ca2b9185227c1e4cd12e50 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 13:07:59 -0500 Subject: [PATCH 03/34] used tagged union instead of std::variant, which is not available pre C++14 --- src/benchmark_runner.cc | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 83aeee53d2..d0a2935a3f 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -37,7 +37,6 @@ #include #include #include -#include #include "check.h" #include "colorprint.h" @@ -141,7 +140,15 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } -std::variant ParseBenchMinTime(const std::string& value) { +struct BenchTimeType { + enum { ITERS, TIME } tag; + union { + int iter; + double time; + }; +}; + +BenchTimeType ParseBenchMinTime(const std::string& value) { if (value.empty()) return 0; if (value.back() == 'x') { @@ -154,7 +161,9 @@ std::variant ParseBenchMinTime(const std::string& value) { BM_CHECK(num_iters > 0 && !(num_iters == 0 && num_iters_str != "0")) << "Malformed iters value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; - return num_iters; + + BenchTimeType ret = {BenchTimeType::ITERS, num_iters}; + return ret; } std::string min_time_str; @@ -171,18 +180,19 @@ std::variant ParseBenchMinTime(const std::string& value) { !(min_time == 0 && (min_time_str != "0" || min_time_str != "0.0"))) << "Malformed seconds value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; - return min_time; + + BenchTimeType ret = {BenchTimeType::TIME, min_time}; + return ret; } double GetMinTime(const benchmark::internal::BenchmarkInstance& b) { if (!IsZero(b.min_time())) return b.min_time(); - std::variant iters_or_time = - ParseBenchMinTime(FLAGS_benchmark_min_time); + BenchTimeType iters_or_time = ParseBenchMinTime(FLAGS_benchmark_min_time); // If the flag was used to specify number of iters, then return 0 for time. - if (iters_or_time.index() == 0) return 0; - return std::get(iters_or_time); + if (iters_or_time.tag == BenchTimeType::ITERS) return 0; + return return iters_or_time.t; } bool BenchMinTimeHasIters() { @@ -194,8 +204,10 @@ int GetIters(const benchmark::internal::BenchmarkInstance& b) { if (b.iterations() != 0) return b.iterations(); // We've already checked that this flag is currently used to pass - // iters. - return std::get(ParseBenchMinTime(FLAGS_benchmark_min_time)); + // iters but do a sanity check anyway. + BenchTimeType parsed = ParseBenchMinTime(FLAGS_benchmark_min_time); + assert(parsed.tag == BenchTimeType::ITERS); + return parsed.i; } } // end namespace From de4c454ea5893401102151e47e986dba88332924 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 13:15:54 -0500 Subject: [PATCH 04/34] update decl in benchmark_runner.h too --- src/benchmark_runner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 0174bd3401..4d05717add 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -25,7 +25,7 @@ namespace benchmark { -BM_DECLARE_double(benchmark_min_time); +BM_DECLARE_string(benchmark_min_time); BM_DECLARE_double(benchmark_min_warmup_time); BM_DECLARE_int32(benchmark_repetitions); BM_DECLARE_bool(benchmark_report_aggregates_only); From fab90540311fc2c04de63d14357f845c03cec1a9 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 13:23:04 -0500 Subject: [PATCH 05/34] fixed errors --- src/benchmark_runner.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index d0a2935a3f..4dae8f1571 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -149,11 +149,11 @@ struct BenchTimeType { }; BenchTimeType ParseBenchMinTime(const std::string& value) { - if (value.empty()) return 0; + if (value.empty()) return {BenchTimeType::TIME, 0.0}; if (value.back() == 'x') { std::string num_iters_str = value.substr(0, value.length() - 1); - int num_iters = std::atoi(num_iters_str); + int num_iters = std::atoi(num_iters_str.c_str()); // std::atoi doesn't provide useful error messages, so we do some // sanity checks. @@ -174,7 +174,7 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { } else { min_time_str = value.substr(0, value.length() - 1); } - double min_time = std::atof(min_time_str); + double min_time = std::atof(min_time_str.c_str()); BM_CHECK(min_time > 0 && !(min_time == 0 && (min_time_str != "0" || min_time_str != "0.0"))) @@ -187,17 +187,16 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { double GetMinTime(const benchmark::internal::BenchmarkInstance& b) { if (!IsZero(b.min_time())) return b.min_time(); - BenchTimeType iters_or_time = ParseBenchMinTime(FLAGS_benchmark_min_time); - // If the flag was used to specify number of iters, then return 0 for time. if (iters_or_time.tag == BenchTimeType::ITERS) return 0; - return return iters_or_time.t; + + return iters_or_time.t; } bool BenchMinTimeHasIters() { return !FLAGS_benchmark_min_time.empty() && - FLAGS_benchmark_min_time.empty.back() == 'x'; + FLAGS_benchmark_min_time.back() == 'x'; } int GetIters(const benchmark::internal::BenchmarkInstance& b) { From 6d98c387e55a95ce962b81a500bd988539f7e5c2 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 13:33:42 -0500 Subject: [PATCH 06/34] refactor --- src/benchmark_runner.cc | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 4dae8f1571..8cc0c79b77 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -149,7 +149,12 @@ struct BenchTimeType { }; BenchTimeType ParseBenchMinTime(const std::string& value) { - if (value.empty()) return {BenchTimeType::TIME, 0.0}; + BenchTimeType ret = {BenchTimeType::TIME, 0}; + + if (value.empty()) { + ret.time = 0.0; + return ret; + } if (value.back() == 'x') { std::string num_iters_str = value.substr(0, value.length() - 1); @@ -162,7 +167,8 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { << "Malformed iters value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; - BenchTimeType ret = {BenchTimeType::ITERS, num_iters}; + ret.tag = BenchTimeType::ITERS; + ret.iters = num_iters; return ret; } @@ -181,7 +187,9 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { << "Malformed seconds value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; - BenchTimeType ret = {BenchTimeType::TIME, min_time}; + ret.tag = BenchTimeType::TIME; + ret.time = min_time; + return ret; } @@ -191,7 +199,7 @@ double GetMinTime(const benchmark::internal::BenchmarkInstance& b) { // If the flag was used to specify number of iters, then return 0 for time. if (iters_or_time.tag == BenchTimeType::ITERS) return 0; - return iters_or_time.t; + return iters_or_time.time; } bool BenchMinTimeHasIters() { @@ -206,7 +214,7 @@ int GetIters(const benchmark::internal::BenchmarkInstance& b) { // iters but do a sanity check anyway. BenchTimeType parsed = ParseBenchMinTime(FLAGS_benchmark_min_time); assert(parsed.tag == BenchTimeType::ITERS); - return parsed.i; + return parsed.iters; } } // end namespace From 2d7206da0aeee7665ba4ac5957c725ef9949de9e Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 14:30:03 -0500 Subject: [PATCH 07/34] backward compat --- src/benchmark.cc | 5 +- src/benchmark_runner.cc | 2 +- test/BUILD | 2 + test/benchmark_min_time_flag_time_test.cc | 76 +++++++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 test/benchmark_min_time_flag_time_test.cc diff --git a/src/benchmark.cc b/src/benchmark.cc index 7483b80774..aa997bbd01 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -70,7 +70,7 @@ BM_DEFINE_string(benchmark_filter, ""); // It can be either an exact number of iterations (specified as `x`), // or a minimum number of seconds (specified as `s`). If the latter // format (ie., min seconds) is used, the system may run the benchmark longer -// until the results are considered significatn. +// until the results are considered significant. // // For backward compatibility, the `s` suffix may be omitted, in which case, // the specified number is interpreted as the number of seconds. @@ -79,9 +79,6 @@ BM_DEFINE_string(benchmark_filter, ""); // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -// -// FIXME: Renmae this flag to --benchmark_time for clarity (and for consistency -// with other Go's) BM_DEFINE_string(benchmark_min_time, "0.5s"); // Minimum number of seconds a benchmark should be run before results should be diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 8cc0c79b77..82f544c385 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -143,7 +143,7 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, struct BenchTimeType { enum { ITERS, TIME } tag; union { - int iter; + int iters; double time; }; }; diff --git a/test/BUILD b/test/BUILD index 1909b0a133..c05e47a6e3 100644 --- a/test/BUILD +++ b/test/BUILD @@ -33,6 +33,8 @@ PER_SRC_TEST_ARGS = { "spec_arg_test.cc": ["--benchmark_filter=BM_NotChosen"], "spec_arg_verbosity_test.cc": ["--v=42"], "benchmark_min_time_flag_iters_test.cc": ["--benchmark_min_time=4x"], + # Backward compat test. + "benchmark_min_time_flag_time_test.cc": ["--benchmark_min_time=2"], } cc_library( diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc new file mode 100644 index 0000000000..7539f3bb93 --- /dev/null +++ b/test/benchmark_min_time_flag_time_test.cc @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +// Tests that we can specify the min time with +// --benchmark_min_time= (no suffix needed) OR +// --benchmark_min_time=s +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + return ConsoleReporter::ReportContext(context); + }; + + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + assert(report.size() == 1); + min_times_.push_back(report[0].run_name.min_time); + ConsoleReporter::ReportRuns(report); + }; + + TestReporter() {} + + virtual ~TestReporter() {} + + const std::vector& GetMinTimes() const { return min_times__; } + + private: + std::vector min_times_; +}; + +void DoTestHelper(int* argc, const char** argv, const std::string& expected) { + benchmark::Initialize(argc, argv); + + TestReporter test_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench"); + assert(returned_count == 1); + + // Check the min_time + const std::vector& min_times = test_reporter.GetMinTimes(); + assert(!iters.empty() && min_times[0] == expected); +} + +} // end namespace + +static void BM_MyBench(benchmark::State& state) { + for (auto s : state) { + } +} +BENCHMARK(BM_MyBench); + +int main(int argc, char** argv) { + // Make a fake argv and append the new --benchmark_min_time= to it. + const int fake_argc = argc + 1; + const char** fake_argv = new const char*[fake_argc]; + + for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i]; + + const char[] no_suffix = "--benchmark_min_time=4.0"; + const char[] with_suffix = "--benchmark_min_time=4.0s"; + std::string expected = "min_time:4.0s"; + + fake_argv[argc] = no_suffix; + DoTestHelper(&fake_argc, fake_argv, expected); + + fake_argv[argc] = with_suffix; + DoTestHelper(&fake_argc, fake_argv, expected); + + return 0; +} From 807322b61ee1ee5a1216d848d7f61008bb75c0b8 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 14:36:25 -0500 Subject: [PATCH 08/34] typo --- test/benchmark_min_time_flag_time_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index 7539f3bb93..2bb1f1dfef 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -44,7 +44,7 @@ void DoTestHelper(int* argc, const char** argv, const std::string& expected) { // Check the min_time const std::vector& min_times = test_reporter.GetMinTimes(); - assert(!iters.empty() && min_times[0] == expected); + assert(!min_times.empty() && min_times[0] == expected); } } // end namespace @@ -57,13 +57,13 @@ BENCHMARK(BM_MyBench); int main(int argc, char** argv) { // Make a fake argv and append the new --benchmark_min_time= to it. - const int fake_argc = argc + 1; + int fake_argc = argc + 1; const char** fake_argv = new const char*[fake_argc]; for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i]; - const char[] no_suffix = "--benchmark_min_time=4.0"; - const char[] with_suffix = "--benchmark_min_time=4.0s"; + const char* no_suffix = "--benchmark_min_time=4.0"; + const char* with_suffix = "--benchmark_min_time=4.0s"; std::string expected = "min_time:4.0s"; fake_argv[argc] = no_suffix; From 80c349f4d81ed24b05c7d9d0fff9d76c76f69fac Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 14:40:44 -0500 Subject: [PATCH 09/34] use IterationCount type --- src/benchmark_runner.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 82f544c385..2b30eac439 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -149,9 +149,10 @@ struct BenchTimeType { }; BenchTimeType ParseBenchMinTime(const std::string& value) { - BenchTimeType ret = {BenchTimeType::TIME, 0}; + BenchTimeType ret; if (value.empty()) { + ret.tag = BenchTimeType::TIME; ret.time = 0.0; return ret; } @@ -207,7 +208,7 @@ bool BenchMinTimeHasIters() { FLAGS_benchmark_min_time.back() == 'x'; } -int GetIters(const benchmark::internal::BenchmarkInstance& b) { +IterationCount GetIters(const benchmark::internal::BenchmarkInstance& b) { if (b.iterations() != 0) return b.iterations(); // We've already checked that this flag is currently used to pass From 2432374aacec166f8422dd165dafddf76fd94bb3 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 14:45:23 -0500 Subject: [PATCH 10/34] fixed test --- test/benchmark_min_time_flag_iters_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark_min_time_flag_iters_test.cc b/test/benchmark_min_time_flag_iters_test.cc index 40498b86d5..9b67717998 100644 --- a/test/benchmark_min_time_flag_iters_test.cc +++ b/test/benchmark_min_time_flag_iters_test.cc @@ -19,7 +19,7 @@ class TestReporter : public benchmark::ConsoleReporter { virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { assert(report.size() == 1); - iter_nums.push_back(report[0].iterations); + iter_nums_.push_back(report[0].iterations); ConsoleReporter::ReportRuns(report); }; From cacbc83303cf436e5c27a423dc7f859ecdb70811 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 15:26:13 -0500 Subject: [PATCH 11/34] const_cast --- test/benchmark_min_time_flag_time_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index 2bb1f1dfef..e74339ec7d 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -31,11 +31,11 @@ class TestReporter : public benchmark::ConsoleReporter { const std::vector& GetMinTimes() const { return min_times__; } private: - std::vector min_times_; + std::vector min_times_; }; void DoTestHelper(int* argc, const char** argv, const std::string& expected) { - benchmark::Initialize(argc, argv); + benchmark::Initialize(argc, const_cast(argv)); TestReporter test_reporter; const size_t returned_count = From 774dbaef9468ca99527f03af86b7b122cdd076c2 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 15:32:59 -0500 Subject: [PATCH 12/34] ret type --- test/benchmark_min_time_flag_time_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index e74339ec7d..3e2a4a166c 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -28,7 +28,7 @@ class TestReporter : public benchmark::ConsoleReporter { virtual ~TestReporter() {} - const std::vector& GetMinTimes() const { return min_times__; } + const std::vector& GetMinTimes() const { return min_times__; } private: std::vector min_times_; From 26455dc393639ad016a9767d5a0e1dc77eac929b Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 15:47:08 -0500 Subject: [PATCH 13/34] remove extra _ --- test/benchmark_min_time_flag_time_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index 3e2a4a166c..5bbe86c7a6 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -28,7 +28,7 @@ class TestReporter : public benchmark::ConsoleReporter { virtual ~TestReporter() {} - const std::vector& GetMinTimes() const { return min_times__; } + const std::vector& GetMinTimes() const { return min_times_; } private: std::vector min_times_; From f64dc35146f8c79f8bba306eb17bc728fa4626e2 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Wed, 11 Jan 2023 15:53:42 -0500 Subject: [PATCH 14/34] debug --- test/benchmark_min_time_flag_time_test.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index 5bbe86c7a6..6d509f783f 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -44,6 +44,12 @@ void DoTestHelper(int* argc, const char** argv, const std::string& expected) { // Check the min_time const std::vector& min_times = test_reporter.GetMinTimes(); + if (min_times.empty()) std::cout << "**** min_times empty\n"; + else { + std::cout << " *** EXPECTED = " << expected << "\n"; + for (const std::string& v : min_times) + std::cout << " ** v = " << v << "\n"; + } assert(!min_times.empty() && min_times[0] == expected); } From d145ff9e93d9ba6a3b7b76167350f3f1ca0e6df8 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 13:21:53 -0500 Subject: [PATCH 15/34] fixed bug from reporting that caused the new configs not to be included in the final report --- src/benchmark.cc | 4 ++++ src/benchmark_runner.cc | 18 ++++++++++++++++++ src/benchmark_runner.h | 2 ++ test/CMakeLists.txt | 6 ++++++ test/benchmark_min_time_flag_iters_test.cc | 17 +++++++---------- test/benchmark_min_time_flag_time_test.cc | 10 ++-------- 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index aa997bbd01..df0e68f9e3 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -401,6 +401,10 @@ void RunBenchmarks(const std::vector& benchmarks, } } + // Final step: fixup the configs that were overrriden with + // non-default values. + runner.UpdateReport(run_results); + Report(display_reporter, file_reporter, run_results); } } diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 2b30eac439..5e2dcf50b6 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -257,6 +257,24 @@ BenchmarkRunner::BenchmarkRunner( } } +void BenchmarkRunner::UpdateReport(RunResults& run_results) { + bool update_time = min_time != b.min_time(); + bool update_iters = has_explicit_iteration_count && iters != b.iterations(); + + if (!update_time && !update_iters) return; + + auto UpdateRun = [](bool update_time, double min_time, bool update_iters, IterationCount iters, + BenchmarkReporter::Run& run) { + if (update_time) + run.run_name.min_time = StrFormat("min_time:%0.3fs", min_time); + if (update_iters) + run.iterations = iters; + }; + + for (auto& run : run_results.non_aggregates) UpdateRun(update_time, min_time, update_iters, iters, run); + for (auto& run: run_results.aggregates_only) UpdateRun(update_time, min_time, update_iters, iters, run); +} + BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { BM_VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 4d05717add..de9fa9e514 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -63,6 +63,8 @@ class BenchmarkRunner { return reports_for_family; } + void UpdateReport(RunResults& run_results); + private: RunResults run_results; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a49ab195e7..ca21b9b277 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -78,6 +78,12 @@ macro(add_filter_test name filter expect) add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) +compile_benchmark_test(benchmark_min_time_flag_time_test) +add_test(NAME min_time_flag_time COMMAND benchmark_min_time_flag_time_test) + +compile_benchmark_test(benchmark_min_time_flag_iters_test) +add_test(NAME min_time_flag_iters COMMAND benchmark_min_time_flag_iters_test) + add_filter_test(filter_simple "Foo" 3) add_filter_test(filter_simple_negative "-Foo" 2) add_filter_test(filter_suffix "BM_.*" 4) diff --git a/test/benchmark_min_time_flag_iters_test.cc b/test/benchmark_min_time_flag_iters_test.cc index 9b67717998..3e2fadd288 100644 --- a/test/benchmark_min_time_flag_iters_test.cc +++ b/test/benchmark_min_time_flag_iters_test.cc @@ -42,16 +42,13 @@ static void BM_MyBench(benchmark::State& state) { BENCHMARK(BM_MyBench); int main(int argc, char** argv) { - // Verify that argv specify --benchmark_min_time=4x - bool found = false; - for (int i = 0; i < argc; ++i) { - if (strcmp("--benchmark_min_time=4x", argv[i]) == 0) { - found = true; - break; - } - } - assert(found); - benchmark::Initialize(&argc, argv); + // Make a fake argv and append the new --benchmark_min_time= to it. + int fake_argc = argc + 1; + const char** fake_argv = new const char*[fake_argc]; + for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i]; + fake_argv[argc] = "--benchmark_min_time=4x"; + + benchmark::Initialize(&fake_argc, const_cast(fake_argv)); TestReporter test_reporter; const size_t returned_count = diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index 6d509f783f..dccc193790 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -44,12 +44,6 @@ void DoTestHelper(int* argc, const char** argv, const std::string& expected) { // Check the min_time const std::vector& min_times = test_reporter.GetMinTimes(); - if (min_times.empty()) std::cout << "**** min_times empty\n"; - else { - std::cout << " *** EXPECTED = " << expected << "\n"; - for (const std::string& v : min_times) - std::cout << " ** v = " << v << "\n"; - } assert(!min_times.empty() && min_times[0] == expected); } @@ -68,9 +62,9 @@ int main(int argc, char** argv) { for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i]; - const char* no_suffix = "--benchmark_min_time=4.0"; + const char* no_suffix = "--benchmark_min_time=4"; const char* with_suffix = "--benchmark_min_time=4.0s"; - std::string expected = "min_time:4.0s"; + std::string expected = "min_time:4.000s"; fake_argv[argc] = no_suffix; DoTestHelper(&fake_argc, fake_argv, expected); From d7f8701bd1e3724add749a3166df8628f2e8a575 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 14:17:48 -0500 Subject: [PATCH 16/34] addressed review comments --- src/benchmark_runner.cc | 95 ++++++++++++---------- src/benchmark_runner.h | 9 ++ test/benchmark_min_time_flag_iters_test.cc | 2 +- 3 files changed, 60 insertions(+), 46 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 5e2dcf50b6..944088ff29 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -140,14 +141,6 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } -struct BenchTimeType { - enum { ITERS, TIME } tag; - union { - int iters; - double time; - }; -}; - BenchTimeType ParseBenchMinTime(const std::string& value) { BenchTimeType ret; @@ -158,13 +151,18 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { } if (value.back() == 'x') { - std::string num_iters_str = value.substr(0, value.length() - 1); - int num_iters = std::atoi(num_iters_str.c_str()); - - // std::atoi doesn't provide useful error messages, so we do some - // sanity checks. - // Also check that the iters is not negative. - BM_CHECK(num_iters > 0 && !(num_iters == 0 && num_iters_str != "0")) + const char* iters_str = value.c_str(); + char* p_end; + // Reset error since atol will change it. + const int old_errno = errno; + errno = 0; + IterationCount num_iters = std::strtol(iters_str, &p_end, 10); + const int new_errno = errno; + errno = old_errno; + + // After a valid parse, p_end should have been set to + // point to the 'x' suffix. + BM_CHECK(new_errno == 0 && p_end != nullptr && *p_end == 'x') << "Malformed iters value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; @@ -173,18 +171,25 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { return ret; } - std::string min_time_str; - if (value.back() != 's') { + const char* time_str = value.c_str(); + bool has_suffix = value.back() == 's'; + if (!has_suffix) { BM_VLOG(2) << "Value passed to --benchmark_min_time should have a suffix. " "Eg., `30s` for 30-seconds."; - min_time_str = value; - } else { - min_time_str = value.substr(0, value.length() - 1); } - double min_time = std::atof(min_time_str.c_str()); - BM_CHECK(min_time > 0 && - !(min_time == 0 && (min_time_str != "0" || min_time_str != "0.0"))) + char* p_end; + // Reset error before it's changed by strtod. + const int old_errno = errno; + errno = 0; + double min_time = std::strtod(time_str, &p_end); + const int new_errno = errno; + errno = old_errno; + + // After a successfull parse, p_end should point to the suffix 's' + // or the end of the string, if the suffix was omitted. + BM_CHECK(new_errno == 0 && p_end != nullptr && + (has_suffix && *p_end == 's' || *p_end == '\0')) << "Malformed seconds value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; @@ -194,27 +199,23 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { return ret; } -double GetMinTime(const benchmark::internal::BenchmarkInstance& b) { +double GetMinTime(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { if (!IsZero(b.min_time())) return b.min_time(); - BenchTimeType iters_or_time = ParseBenchMinTime(FLAGS_benchmark_min_time); // If the flag was used to specify number of iters, then return 0 for time. if (iters_or_time.tag == BenchTimeType::ITERS) return 0; return iters_or_time.time; } -bool BenchMinTimeHasIters() { - return !FLAGS_benchmark_min_time.empty() && - FLAGS_benchmark_min_time.back() == 'x'; -} - -IterationCount GetIters(const benchmark::internal::BenchmarkInstance& b) { +IterationCount GetIters(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { if (b.iterations() != 0) return b.iterations(); // We've already checked that this flag is currently used to pass - // iters but do a sanity check anyway. + // iters but do a check anyway. BenchTimeType parsed = ParseBenchMinTime(FLAGS_benchmark_min_time); - assert(parsed.tag == BenchTimeType::ITERS); + BM_CHECK(parsed.tag == BenchTimeType::ITERS); return parsed.iters; } @@ -225,7 +226,8 @@ BenchmarkRunner::BenchmarkRunner( BenchmarkReporter::PerFamilyRunReports* reports_for_family_) : b(b_), reports_for_family(reports_for_family_), - min_time(GetMinTime(b_)), + parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), + min_time(GetMinTime(b_, parsed_benchtime_flag)), min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) ? b.min_warmup_time() : FLAGS_benchmark_min_warmup_time), @@ -233,9 +235,11 @@ BenchmarkRunner::BenchmarkRunner( repeats(b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions), has_explicit_iteration_count(b.iterations() != 0 || - BenchMinTimeHasIters()), + parsed_benchtime_flag.tag == + BenchTimeType::ITERS), pool(b.threads() - 1), - iters(has_explicit_iteration_count ? GetIters(b_) : 1), + iters(has_explicit_iteration_count ? GetIters(b_, parsed_benchtime_flag) + : 1), perf_counters_measurement(StrSplit(FLAGS_benchmark_perf_counters, ',')), perf_counters_measurement_ptr(perf_counters_measurement.IsValid() ? &perf_counters_measurement @@ -263,16 +267,17 @@ void BenchmarkRunner::UpdateReport(RunResults& run_results) { if (!update_time && !update_iters) return; - auto UpdateRun = [](bool update_time, double min_time, bool update_iters, IterationCount iters, - BenchmarkReporter::Run& run) { - if (update_time) - run.run_name.min_time = StrFormat("min_time:%0.3fs", min_time); - if (update_iters) - run.iterations = iters; - }; + auto UpdateRun = [](bool update_time, double min_time, bool update_iters, + IterationCount iters, BenchmarkReporter::Run& run) { + if (update_time) + run.run_name.min_time = StrFormat("min_time:%0.3fs", min_time); + if (update_iters) run.iterations = iters; + }; - for (auto& run : run_results.non_aggregates) UpdateRun(update_time, min_time, update_iters, iters, run); - for (auto& run: run_results.aggregates_only) UpdateRun(update_time, min_time, update_iters, iters, run); + for (auto& run : run_results.non_aggregates) + UpdateRun(update_time, min_time, update_iters, iters, run); + for (auto& run : run_results.aggregates_only) + UpdateRun(update_time, min_time, update_iters, iters, run); } BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index de9fa9e514..1039afccdb 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -44,6 +44,14 @@ struct RunResults { bool file_report_aggregates_only = false; }; +struct BenchTimeType { + enum { ITERS, TIME } tag; + union { + int iters; + double time; + }; +}; + class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, @@ -71,6 +79,7 @@ class BenchmarkRunner { const benchmark::internal::BenchmarkInstance& b; BenchmarkReporter::PerFamilyRunReports* reports_for_family; + BenchTimeType parsed_benchtime_flag; const double min_time; const double min_warmup_time; bool warmup_done; diff --git a/test/benchmark_min_time_flag_iters_test.cc b/test/benchmark_min_time_flag_iters_test.cc index 3e2fadd288..9a36ece517 100644 --- a/test/benchmark_min_time_flag_iters_test.cc +++ b/test/benchmark_min_time_flag_iters_test.cc @@ -46,7 +46,7 @@ int main(int argc, char** argv) { int fake_argc = argc + 1; const char** fake_argv = new const char*[fake_argc]; for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i]; - fake_argv[argc] = "--benchmark_min_time=4x"; + fake_argv[argc] = "--benchmark_min_time=4x"; benchmark::Initialize(&fake_argc, const_cast(fake_argv)); From c37b2a17afcf9f2d0b43f19db6fe151bfc859a07 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 14:55:17 -0500 Subject: [PATCH 17/34] restore unnecessary changes in test/BUILD --- test/BUILD | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/BUILD b/test/BUILD index c05e47a6e3..a29abe5ae1 100644 --- a/test/BUILD +++ b/test/BUILD @@ -25,16 +25,13 @@ PER_SRC_COPTS = { "donotoptimize_test.cc": ["-O3"], } -TEST_ARGS = ["--benchmark_min_time=0.01s"] +TEST_ARGS = ["--benchmark_min_time=0.01"] PER_SRC_TEST_ARGS = { "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], "repetitions_test.cc": [" --benchmark_repetitions=3"], "spec_arg_test.cc": ["--benchmark_filter=BM_NotChosen"], "spec_arg_verbosity_test.cc": ["--v=42"], - "benchmark_min_time_flag_iters_test.cc": ["--benchmark_min_time=4x"], - # Backward compat test. - "benchmark_min_time_flag_time_test.cc": ["--benchmark_min_time=2"], } cc_library( From 48c7d820e5c1c64fb5d5ac7b6899557ec767dfb8 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 15:24:47 -0500 Subject: [PATCH 18/34] fix float comparisons warnings from Release builds --- src/benchmark_api_internal.cc | 4 ++-- src/benchmark_runner.cc | 36 +++++++++++++++++++++++------------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 286f986530..01b3ebd60d 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -48,12 +48,12 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, } if (!IsZero(benchmark->min_time_)) { - name_.min_time = StrFormat("min_time:%0.3f", benchmark_.min_time_); + name_.min_time = StrFormat("min_time:%0.3fs", benchmark_.min_time_); } if (!IsZero(benchmark->min_warmup_time_)) { name_.min_warmup_time = - StrFormat("min_warmup_time:%0.3f", benchmark_.min_warmup_time_); + StrFormat("min_warmup_time:%0.3fs", benchmark_.min_warmup_time_); } if (benchmark_.iterations_ != 0) { diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 944088ff29..03da9aa72d 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -29,16 +29,19 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include #include + #include "check.h" #include "colorprint.h" #include "commandlineflags.h" @@ -160,6 +163,8 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { const int new_errno = errno; errno = old_errno; + (void)new_errno; + // After a valid parse, p_end should have been set to // point to the 'x' suffix. BM_CHECK(new_errno == 0 && p_end != nullptr && *p_end == 'x') @@ -186,6 +191,8 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { const int new_errno = errno; errno = old_errno; + (void)new_errno; + // After a successfull parse, p_end should point to the suffix 's' // or the end of the string, if the suffix was omitted. BM_CHECK(new_errno == 0 && p_end != nullptr && @@ -214,9 +221,8 @@ IterationCount GetIters(const benchmark::internal::BenchmarkInstance& b, // We've already checked that this flag is currently used to pass // iters but do a check anyway. - BenchTimeType parsed = ParseBenchMinTime(FLAGS_benchmark_min_time); - BM_CHECK(parsed.tag == BenchTimeType::ITERS); - return parsed.iters; + BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); + return iters_or_time.iters; } } // end namespace @@ -261,22 +267,28 @@ BenchmarkRunner::BenchmarkRunner( } } -void BenchmarkRunner::UpdateReport(RunResults& run_results) { - bool update_time = min_time != b.min_time(); +static bool AlmostEqual(double a, double b) { + return std::fabs(a - b) < std::numeric_limits::epsilon(); +} + +void BenchmarkRunner::UpdateReport(RunResults& current_run_results) { + // If we the default min_time is different from the requested min_time + // then update it. + bool update_time = !AlmostEqual(min_time, b.min_time()) || !AlmostEqual(min_time, 0.0); bool update_iters = has_explicit_iteration_count && iters != b.iterations(); if (!update_time && !update_iters) return; - auto UpdateRun = [](bool update_time, double min_time, bool update_iters, - IterationCount iters, BenchmarkReporter::Run& run) { - if (update_time) - run.run_name.min_time = StrFormat("min_time:%0.3fs", min_time); - if (update_iters) run.iterations = iters; + auto UpdateRun = [](bool update_t, double new_min_time, bool update_i, + IterationCount new_iters, BenchmarkReporter::Run& run) { + if (update_t) + run.run_name.min_time = StrFormat("min_time:%0.3fs", new_min_time); + if (update_i) run.iterations = new_iters; }; - for (auto& run : run_results.non_aggregates) + for (auto& run : current_run_results.non_aggregates) UpdateRun(update_time, min_time, update_iters, iters, run); - for (auto& run : run_results.aggregates_only) + for (auto& run : current_run_results.aggregates_only) UpdateRun(update_time, min_time, update_iters, iters, run); } From 0d69ed02a7730b7401ddb20ca02447a871b2a2e8 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 16:03:56 -0500 Subject: [PATCH 19/34] clang format --- include/benchmark/benchmark.h | 2 ++ src/benchmark_api_internal.cc | 4 ++-- src/benchmark_runner.cc | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 1222aedcee..6f2ff32b16 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -280,6 +280,8 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); namespace benchmark { class BenchmarkReporter; +BENCHMARK_EXPORT const double kDefaultMinTime = 0.5; + BENCHMARK_EXPORT void PrintDefaultHelp(); BENCHMARK_EXPORT void Initialize(int* argc, char** argv, diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 01b3ebd60d..286f986530 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -48,12 +48,12 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, } if (!IsZero(benchmark->min_time_)) { - name_.min_time = StrFormat("min_time:%0.3fs", benchmark_.min_time_); + name_.min_time = StrFormat("min_time:%0.3f", benchmark_.min_time_); } if (!IsZero(benchmark->min_warmup_time_)) { name_.min_warmup_time = - StrFormat("min_warmup_time:%0.3fs", benchmark_.min_warmup_time_); + StrFormat("min_warmup_time:%0.3f", benchmark_.min_warmup_time_); } if (benchmark_.iterations_ != 0) { diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 03da9aa72d..8eb72e7fed 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -272,9 +272,10 @@ static bool AlmostEqual(double a, double b) { } void BenchmarkRunner::UpdateReport(RunResults& current_run_results) { - // If we the default min_time is different from the requested min_time - // then update it. - bool update_time = !AlmostEqual(min_time, b.min_time()) || !AlmostEqual(min_time, 0.0); + // If we the actual min_time is different from the value in the benchmark instance + // AND if it's not the default value then update it. + bool update_time = !AlmostEqual(min_time, b.min_time()) + && !AlmostEqual(min_time, kDefaultMinTime); bool update_iters = has_explicit_iteration_count && iters != b.iterations(); if (!update_time && !update_iters) return; From bce641344d35f2e9be8431549a36eb2ada176176 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 16:08:02 -0500 Subject: [PATCH 20/34] fix visibility warning --- include/benchmark/#benchmark.h# | 1873 +++++++++++++++++++++++++++++++ include/benchmark/.#benchmark.h | 1 + include/benchmark/benchmark.h | 2 +- src/benchmark_runner.cc | 33 +- 4 files changed, 1892 insertions(+), 17 deletions(-) create mode 100644 include/benchmark/#benchmark.h# create mode 120000 include/benchmark/.#benchmark.h diff --git a/include/benchmark/#benchmark.h# b/include/benchmark/#benchmark.h# new file mode 100644 index 0000000000..1222aedcee --- /dev/null +++ b/include/benchmark/#benchmark.h# @@ -0,0 +1,1873 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Support for registering benchmarks for functions. + +/* Example usage: +// Define a function that executes the code to be measured a +// specified number of times: +static void BM_StringCreation(benchmark::State& state) { + for (auto _ : state) + std::string empty_string; +} + +// Register the function as a benchmark +BENCHMARK(BM_StringCreation); + +// Define another benchmark +static void BM_StringCopy(benchmark::State& state) { + std::string x = "hello"; + for (auto _ : state) + std::string copy(x); +} +BENCHMARK(BM_StringCopy); + +// Augment the main() program to invoke benchmarks if specified +// via the --benchmark_filter command line flag. E.g., +// my_unittest --benchmark_filter=all +// my_unittest --benchmark_filter=BM_StringCreation +// my_unittest --benchmark_filter=String +// my_unittest --benchmark_filter='Copy|Creation' +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} + +// Sometimes a family of microbenchmarks can be implemented with +// just one routine that takes an extra argument to specify which +// one of the family of benchmarks to run. For example, the following +// code defines a family of microbenchmarks for measuring the speed +// of memcpy() calls of different lengths: + +static void BM_memcpy(benchmark::State& state) { + char* src = new char[state.range(0)]; char* dst = new char[state.range(0)]; + memset(src, 'x', state.range(0)); + for (auto _ : state) + memcpy(dst, src, state.range(0)); + state.SetBytesProcessed(state.iterations() * state.range(0)); + delete[] src; delete[] dst; +} +BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); + +// The preceding code is quite repetitive, and can be replaced with the +// following short-hand. The following invocation will pick a few +// appropriate arguments in the specified range and will generate a +// microbenchmark for each such argument. +BENCHMARK(BM_memcpy)->Range(8, 8<<10); + +// You might have a microbenchmark that depends on two inputs. For +// example, the following code defines a family of microbenchmarks for +// measuring the speed of set insertion. +static void BM_SetInsert(benchmark::State& state) { + set data; + for (auto _ : state) { + state.PauseTiming(); + data = ConstructRandomSet(state.range(0)); + state.ResumeTiming(); + for (int j = 0; j < state.range(1); ++j) + data.insert(RandomNumber()); + } +} +BENCHMARK(BM_SetInsert) + ->Args({1<<10, 128}) + ->Args({2<<10, 128}) + ->Args({4<<10, 128}) + ->Args({8<<10, 128}) + ->Args({1<<10, 512}) + ->Args({2<<10, 512}) + ->Args({4<<10, 512}) + ->Args({8<<10, 512}); + +// The preceding code is quite repetitive, and can be replaced with +// the following short-hand. The following macro will pick a few +// appropriate arguments in the product of the two specified ranges +// and will generate a microbenchmark for each such pair. +BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); + +// For more complex patterns of inputs, passing a custom function +// to Apply allows programmatic specification of an +// arbitrary set of arguments to run the microbenchmark on. +// The following example enumerates a dense range on +// one parameter, and a sparse range on the second. +static void CustomArguments(benchmark::internal::Benchmark* b) { + for (int i = 0; i <= 10; ++i) + for (int j = 32; j <= 1024*1024; j *= 8) + b->Args({i, j}); +} +BENCHMARK(BM_SetInsert)->Apply(CustomArguments); + +// Templated microbenchmarks work the same way: +// Produce then consume 'size' messages 'iters' times +// Measures throughput in the absence of multiprogramming. +template int BM_Sequential(benchmark::State& state) { + Q q; + typename Q::value_type v; + for (auto _ : state) { + for (int i = state.range(0); i--; ) + q.push(v); + for (int e = state.range(0); e--; ) + q.Wait(&v); + } + // actually messages, not bytes: + state.SetBytesProcessed(state.iterations() * state.range(0)); +} +BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); + +Use `Benchmark::MinTime(double t)` to set the minimum time used to run the +benchmark. This option overrides the `benchmark_min_time` flag. + +void BM_test(benchmark::State& state) { + ... body ... +} +BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds. + +In a multithreaded test, it is guaranteed that none of the threads will start +until all have reached the loop start, and all will have finished before any +thread exits the loop body. As such, any global setup or teardown you want to +do can be wrapped in a check against the thread index: + +static void BM_MultiThreaded(benchmark::State& state) { + if (state.thread_index() == 0) { + // Setup code here. + } + for (auto _ : state) { + // Run the test as normal. + } + if (state.thread_index() == 0) { + // Teardown code here. + } +} +BENCHMARK(BM_MultiThreaded)->Threads(4); + + +If a benchmark runs a few milliseconds it may be hard to visually compare the +measured times, since the output data is given in nanoseconds per default. In +order to manually set the time unit, you can specify it manually: + +BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); +*/ + +#ifndef BENCHMARK_BENCHMARK_H_ +#define BENCHMARK_BENCHMARK_H_ + +// The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. +#if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) +#define BENCHMARK_HAS_CXX11 +#endif + +// This _MSC_VER check should detect VS 2017 v15.3 and newer. +#if __cplusplus >= 201703L || \ + (defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L) +#define BENCHMARK_HAS_CXX17 +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "benchmark/export.h" + +#if defined(BENCHMARK_HAS_CXX11) +#include +#include +#include +#include +#endif + +#if defined(_MSC_VER) +#include // for _ReadWriteBarrier +#endif + +#ifndef BENCHMARK_HAS_CXX11 +#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + TypeName& operator=(const TypeName&) +#else +#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&) = delete; \ + TypeName& operator=(const TypeName&) = delete +#endif + +#ifdef BENCHMARK_HAS_CXX17 +#define BENCHMARK_UNUSED [[maybe_unused]] +#elif defined(__GNUC__) || defined(__clang__) +#define BENCHMARK_UNUSED __attribute__((unused)) +#else +#define BENCHMARK_UNUSED +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) && !defined(__clang__) +#define BENCHMARK_ALWAYS_INLINE __forceinline +#define __func__ __FUNCTION__ +#else +#define BENCHMARK_ALWAYS_INLINE +#endif + +#define BENCHMARK_INTERNAL_TOSTRING2(x) #x +#define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) + +// clang-format off +#if defined(__GNUC__) && !defined(__NVCC__) || defined(__clang__) +#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) +#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop") +#else +#define BENCHMARK_BUILTIN_EXPECT(x, y) x +#define BENCHMARK_DEPRECATED_MSG(msg) +#define BENCHMARK_WARNING_MSG(msg) \ + __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \ + __LINE__) ") : warning note: " msg)) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING +#define BENCHMARK_RESTORE_DEPRECATED_WARNING +#endif +// clang-format on + +#if defined(__GNUC__) && !defined(__clang__) +#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if defined(__GNUC__) || __has_builtin(__builtin_unreachable) +#define BENCHMARK_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +#define BENCHMARK_UNREACHABLE() __assume(false) +#else +#define BENCHMARK_UNREACHABLE() ((void)0) +#endif + +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK_OVERRIDE override +#else +#define BENCHMARK_OVERRIDE +#endif + +#if defined(_MSC_VER) +#pragma warning(push) +// C4251: needs to have dll-interface to be used by clients of class +#pragma warning(disable : 4251) +#endif + +namespace benchmark { +class BenchmarkReporter; + +BENCHMARK_EXPORT void PrintDefaultHelp(); + +BENCHMARK_EXPORT void Initialize(int* argc, char** argv, + void (*HelperPrinterf)() = PrintDefaultHelp); +BENCHMARK_EXPORT void Shutdown(); + +// Report to stdout all arguments in 'argv' as unrecognized except the first. +// Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). +BENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv); + +// Returns the current value of --benchmark_filter. +BENCHMARK_EXPORT std::string GetBenchmarkFilter(); + +// Sets a new value to --benchmark_filter. (This will override this flag's +// current value). +// Should be called after `benchmark::Initialize()`, as +// `benchmark::Initialize()` will override the flag's value. +BENCHMARK_EXPORT void SetBenchmarkFilter(std::string value); + +// Returns the current value of --v (command line value for verbosity). +BENCHMARK_EXPORT int32_t GetBenchmarkVerbosity(); + +// Creates a default display reporter. Used by the library when no display +// reporter is provided, but also made available for external use in case a +// custom reporter should respect the `--benchmark_format` flag as a fallback +BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); + +// Generate a list of benchmarks matching the specified --benchmark_filter flag +// and if --benchmark_list_tests is specified return after printing the name +// of each matching benchmark. Otherwise run each matching benchmark and +// report the results. +// +// spec : Specify the benchmarks to run. If users do not specify this arg, +// then the value of FLAGS_benchmark_filter +// will be used. +// +// The second and third overload use the specified 'display_reporter' and +// 'file_reporter' respectively. 'file_reporter' will write to the file +// specified +// by '--benchmark_output'. If '--benchmark_output' is not given the +// 'file_reporter' is ignored. +// +// RETURNS: The number of matching benchmarks. +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec); + +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec); + +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks( + BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter, std::string spec); + +// TimeUnit is passed to a benchmark in order to specify the order of magnitude +// for the measured time. +enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; + +BENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit(); + +// Sets the default time unit the benchmarks use +// Has to be called before the benchmark loop to take effect +BENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit); + +// If a MemoryManager is registered (via RegisterMemoryManager()), +// it can be used to collect and report allocation metrics for a run of the +// benchmark. +class MemoryManager { + public: + static const int64_t TombstoneValue; + + struct Result { + Result() + : num_allocs(0), + max_bytes_used(0), + total_allocated_bytes(TombstoneValue), + net_heap_growth(TombstoneValue) {} + + // The number of allocations made in total between Start and Stop. + int64_t num_allocs; + + // The peak memory use between Start and Stop. + int64_t max_bytes_used; + + // The total memory allocated, in bytes, between Start and Stop. + // Init'ed to TombstoneValue if metric not available. + int64_t total_allocated_bytes; + + // The net changes in memory, in bytes, between Start and Stop. + // ie., total_allocated_bytes - total_deallocated_bytes. + // Init'ed to TombstoneValue if metric not available. + int64_t net_heap_growth; + }; + + virtual ~MemoryManager() {} + + // Implement this to start recording allocation information. + virtual void Start() = 0; + + // Implement this to stop recording and fill out the given Result structure. + virtual void Stop(Result& result) = 0; +}; + +// Register a MemoryManager instance that will be used to collect and report +// allocation measurements for benchmark runs. +BENCHMARK_EXPORT +void RegisterMemoryManager(MemoryManager* memory_manager); + +// Add a key-value pair to output as part of the context stanza in the report. +BENCHMARK_EXPORT +void AddCustomContext(const std::string& key, const std::string& value); + +namespace internal { +class Benchmark; +class BenchmarkImp; +class BenchmarkFamilies; + +BENCHMARK_EXPORT std::map*& GetGlobalContext(); + +BENCHMARK_EXPORT +void UseCharPointer(char const volatile*); + +// Take ownership of the pointer and register the benchmark. Return the +// registered benchmark. +BENCHMARK_EXPORT Benchmark* RegisterBenchmarkInternal(Benchmark*); + +// Ensure that the standard streams are properly initialized in every TU. +BENCHMARK_EXPORT int InitializeStreams(); +BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); + +} // namespace internal + +#if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ + defined(__EMSCRIPTEN__) +#define BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#endif + +// Force the compiler to flush pending writes to global memory. Acts as an +// effective read/write barrier +#ifdef BENCHMARK_HAS_CXX11 +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { + std::atomic_signal_fence(std::memory_order_acq_rel); +} +#endif + +// The DoNotOptimize(...) function can be used to prevent a value or +// expression from being optimized away by the compiler. This function is +// intended to add little to no overhead. +// See: https://youtu.be/nXaxk27zwlk?t=2441 +#ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER) +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { +#if defined(__clang__) + asm volatile("" : "+r,m"(value) : : "memory"); +#else + asm volatile("" : "+m,r"(value) : : "memory"); +#endif +} +#elif defined(BENCHMARK_HAS_CXX11) && (__GNUC__ >= 5) +// Workaround for a bug with full argument copy overhead with GCC. +// See: #1340 and https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105519 +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m,r"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +#else +// Fallback for GCC < 5. Can add some overhead because the compiler is forced +// to use memory operations instead of operations with registers. +// TODO: Remove if GCC < 5 will be unsupported. +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + asm volatile("" : : "m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { + asm volatile("" : "+m"(value) : : "memory"); +} +#endif + +#ifndef BENCHMARK_HAS_CXX11 +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { + asm volatile("" : : : "memory"); +} +#endif +#elif defined(_MSC_VER) +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + internal::UseCharPointer(&reinterpret_cast(value)); + _ReadWriteBarrier(); +} + +#ifndef BENCHMARK_HAS_CXX11 +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } +#endif +#else +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + internal::UseCharPointer(&reinterpret_cast(value)); +} +// FIXME Add ClobberMemory() for non-gnu and non-msvc compilers, before C++11. +#endif + +// This class is used for user-defined counters. +class Counter { + public: + enum Flags { + kDefaults = 0, + // Mark the counter as a rate. It will be presented divided + // by the duration of the benchmark. + kIsRate = 1 << 0, + // Mark the counter as a thread-average quantity. It will be + // presented divided by the number of threads. + kAvgThreads = 1 << 1, + // Mark the counter as a thread-average rate. See above. + kAvgThreadsRate = kIsRate | kAvgThreads, + // Mark the counter as a constant value, valid/same for *every* iteration. + // When reporting, it will be *multiplied* by the iteration count. + kIsIterationInvariant = 1 << 2, + // Mark the counter as a constant rate. + // When reporting, it will be *multiplied* by the iteration count + // and then divided by the duration of the benchmark. + kIsIterationInvariantRate = kIsRate | kIsIterationInvariant, + // Mark the counter as a iteration-average quantity. + // It will be presented divided by the number of iterations. + kAvgIterations = 1 << 3, + // Mark the counter as a iteration-average rate. See above. + kAvgIterationsRate = kIsRate | kAvgIterations, + + // In the end, invert the result. This is always done last! + kInvert = 1 << 31 + }; + + enum OneK { + // 1'000 items per 1k + kIs1000 = 1000, + // 1'024 items per 1k + kIs1024 = 1024 + }; + + double value; + Flags flags; + OneK oneK; + + BENCHMARK_ALWAYS_INLINE + Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000) + : value(v), flags(f), oneK(k) {} + + BENCHMARK_ALWAYS_INLINE operator double const &() const { return value; } + BENCHMARK_ALWAYS_INLINE operator double&() { return value; } +}; + +// A helper for user code to create unforeseen combinations of Flags, without +// having to do this cast manually each time, or providing this operator. +Counter::Flags inline operator|(const Counter::Flags& LHS, + const Counter::Flags& RHS) { + return static_cast(static_cast(LHS) | + static_cast(RHS)); +} + +// This is the container for the user-defined counters. +typedef std::map UserCounters; + +// BigO is passed to a benchmark in order to specify the asymptotic +// computational +// complexity for the benchmark. In case oAuto is selected, complexity will be +// calculated automatically to the best fit. +enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; + +typedef int64_t IterationCount; + +enum StatisticUnit { kTime, kPercentage }; + +// BigOFunc is passed to a benchmark in order to specify the asymptotic +// computational complexity for the benchmark. +typedef double(BigOFunc)(IterationCount); + +// StatisticsFunc is passed to a benchmark in order to compute some descriptive +// statistics over all the measurements of some type +typedef double(StatisticsFunc)(const std::vector&); + +namespace internal { +struct Statistics { + std::string name_; + StatisticsFunc* compute_; + StatisticUnit unit_; + + Statistics(const std::string& name, StatisticsFunc* compute, + StatisticUnit unit = kTime) + : name_(name), compute_(compute), unit_(unit) {} +}; + +class BenchmarkInstance; +class ThreadTimer; +class ThreadManager; +class PerfCountersMeasurement; + +enum AggregationReportMode +#if defined(BENCHMARK_HAS_CXX11) + : unsigned +#else +#endif +{ + // The mode has not been manually specified + ARM_Unspecified = 0, + // The mode is user-specified. + // This may or may not be set when the following bit-flags are set. + ARM_Default = 1U << 0U, + // File reporter should only output aggregates. + ARM_FileReportAggregatesOnly = 1U << 1U, + // Display reporter should only output aggregates + ARM_DisplayReportAggregatesOnly = 1U << 2U, + // Both reporters should only display aggregates. + ARM_ReportAggregatesOnly = + ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly +}; + +} // namespace internal + +// State is passed to a running Benchmark and contains state for the +// benchmark to use. +class BENCHMARK_EXPORT State { + public: + struct StateIterator; + friend struct StateIterator; + + // Returns iterators used to run each iteration of a benchmark using a + // C++11 ranged-based for loop. These functions should not be called directly. + // + // REQUIRES: The benchmark has not started running yet. Neither begin nor end + // have been called previously. + // + // NOTE: KeepRunning may not be used after calling either of these functions. + BENCHMARK_ALWAYS_INLINE StateIterator begin(); + BENCHMARK_ALWAYS_INLINE StateIterator end(); + + // Returns true if the benchmark should continue through another iteration. + // NOTE: A benchmark may not return from the test until KeepRunning() has + // returned false. + bool KeepRunning(); + + // Returns true iff the benchmark should run n more iterations. + // REQUIRES: 'n' > 0. + // NOTE: A benchmark must not return from the test until KeepRunningBatch() + // has returned false. + // NOTE: KeepRunningBatch() may overshoot by up to 'n' iterations. + // + // Intended usage: + // while (state.KeepRunningBatch(1000)) { + // // process 1000 elements + // } + bool KeepRunningBatch(IterationCount n); + + // REQUIRES: timer is running and 'SkipWithError(...)' has not been called + // by the current thread. + // Stop the benchmark timer. If not called, the timer will be + // automatically stopped after the last iteration of the benchmark loop. + // + // For threaded benchmarks the PauseTiming() function only pauses the timing + // for the current thread. + // + // NOTE: The "real time" measurement is per-thread. If different threads + // report different measurements the largest one is reported. + // + // NOTE: PauseTiming()/ResumeTiming() are relatively + // heavyweight, and so their use should generally be avoided + // within each benchmark iteration, if possible. + void PauseTiming(); + + // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called + // by the current thread. + // Start the benchmark timer. The timer is NOT running on entrance to the + // benchmark function. It begins running after control flow enters the + // benchmark loop. + // + // NOTE: PauseTiming()/ResumeTiming() are relatively + // heavyweight, and so their use should generally be avoided + // within each benchmark iteration, if possible. + void ResumeTiming(); + + // REQUIRES: 'SkipWithError(...)' has not been called previously by the + // current thread. + // Report the benchmark as resulting in an error with the specified 'msg'. + // After this call the user may explicitly 'return' from the benchmark. + // + // If the ranged-for style of benchmark loop is used, the user must explicitly + // break from the loop, otherwise all future iterations will be run. + // If the 'KeepRunning()' loop is used the current thread will automatically + // exit the loop at the end of the current iteration. + // + // For threaded benchmarks only the current thread stops executing and future + // calls to `KeepRunning()` will block until all threads have completed + // the `KeepRunning()` loop. If multiple threads report an error only the + // first error message is used. + // + // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit + // the current scope immediately. If the function is called from within + // the 'KeepRunning()' loop the current iteration will finish. It is the users + // responsibility to exit the scope as needed. + void SkipWithError(const char* msg); + + // Returns true if an error has been reported with 'SkipWithError(...)'. + bool error_occurred() const { return error_occurred_; } + + // REQUIRES: called exactly once per iteration of the benchmarking loop. + // Set the manually measured time for this benchmark iteration, which + // is used instead of automatically measured time if UseManualTime() was + // specified. + // + // For threaded benchmarks the final value will be set to the largest + // reported values. + void SetIterationTime(double seconds); + + // Set the number of bytes processed by the current benchmark + // execution. This routine is typically called once at the end of a + // throughput oriented benchmark. + // + // REQUIRES: a benchmark has exited its benchmarking loop. + BENCHMARK_ALWAYS_INLINE + void SetBytesProcessed(int64_t bytes) { + counters["bytes_per_second"] = + Counter(static_cast(bytes), Counter::kIsRate, Counter::kIs1024); + } + + BENCHMARK_ALWAYS_INLINE + int64_t bytes_processed() const { + if (counters.find("bytes_per_second") != counters.end()) + return static_cast(counters.at("bytes_per_second")); + return 0; + } + + // If this routine is called with complexity_n > 0 and complexity report is + // requested for the + // family benchmark, then current benchmark will be part of the computation + // and complexity_n will + // represent the length of N. + BENCHMARK_ALWAYS_INLINE + void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } + + BENCHMARK_ALWAYS_INLINE + int64_t complexity_length_n() const { return complexity_n_; } + + // If this routine is called with items > 0, then an items/s + // label is printed on the benchmark report line for the currently + // executing benchmark. It is typically called at the end of a processing + // benchmark where a processing items/second output is desired. + // + // REQUIRES: a benchmark has exited its benchmarking loop. + BENCHMARK_ALWAYS_INLINE + void SetItemsProcessed(int64_t items) { + counters["items_per_second"] = + Counter(static_cast(items), benchmark::Counter::kIsRate); + } + + BENCHMARK_ALWAYS_INLINE + int64_t items_processed() const { + if (counters.find("items_per_second") != counters.end()) + return static_cast(counters.at("items_per_second")); + return 0; + } + + // If this routine is called, the specified label is printed at the + // end of the benchmark report line for the currently executing + // benchmark. Example: + // static void BM_Compress(benchmark::State& state) { + // ... + // double compress = input_size / output_size; + // state.SetLabel(StrFormat("compress:%.1f%%", 100.0*compression)); + // } + // Produces output that looks like: + // BM_Compress 50 50 14115038 compress:27.3% + // + // REQUIRES: a benchmark has exited its benchmarking loop. + void SetLabel(const char* label); + + void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { + this->SetLabel(str.c_str()); + } + + // Range arguments for this run. CHECKs if the argument has been set. + BENCHMARK_ALWAYS_INLINE + int64_t range(std::size_t pos = 0) const { + assert(range_.size() > pos); + return range_[pos]; + } + + BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") + int64_t range_x() const { return range(0); } + + BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") + int64_t range_y() const { return range(1); } + + // Number of threads concurrently executing the benchmark. + BENCHMARK_ALWAYS_INLINE + int threads() const { return threads_; } + + // Index of the executing thread. Values from [0, threads). + BENCHMARK_ALWAYS_INLINE + int thread_index() const { return thread_index_; } + + BENCHMARK_ALWAYS_INLINE + IterationCount iterations() const { + if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { + return 0; + } + return max_iterations - total_iterations_ + batch_leftover_; + } + + BENCHMARK_ALWAYS_INLINE + std::string name() const { return name_; } + + private: + // items we expect on the first cache line (ie 64 bytes of the struct) + // When total_iterations_ is 0, KeepRunning() and friends will return false. + // May be larger than max_iterations. + IterationCount total_iterations_; + + // When using KeepRunningBatch(), batch_leftover_ holds the number of + // iterations beyond max_iters that were run. Used to track + // completed_iterations_ accurately. + IterationCount batch_leftover_; + + public: + const IterationCount max_iterations; + + private: + bool started_; + bool finished_; + bool error_occurred_; + + // items we don't need on the first cache line + std::vector range_; + + int64_t complexity_n_; + + public: + // Container for user-defined counters. + UserCounters counters; + + private: + State(std::string name, IterationCount max_iters, + const std::vector& ranges, int thread_i, int n_threads, + internal::ThreadTimer* timer, internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement); + + void StartKeepRunning(); + // Implementation of KeepRunning() and KeepRunningBatch(). + // is_batch must be true unless n is 1. + bool KeepRunningInternal(IterationCount n, bool is_batch); + void FinishKeepRunning(); + + const std::string name_; + const int thread_index_; + const int threads_; + + internal::ThreadTimer* const timer_; + internal::ThreadManager* const manager_; + internal::PerfCountersMeasurement* const perf_counters_measurement_; + + friend class internal::BenchmarkInstance; +}; + +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { + return KeepRunningInternal(1, /*is_batch=*/false); +} + +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) { + return KeepRunningInternal(n, /*is_batch=*/true); +} + +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, + bool is_batch) { + // total_iterations_ is set to 0 by the constructor, and always set to a + // nonzero value by StartKepRunning(). + assert(n > 0); + // n must be 1 unless is_batch is true. + assert(is_batch || n == 1); + if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ >= n, true)) { + total_iterations_ -= n; + return true; + } + if (!started_) { + StartKeepRunning(); + if (!error_occurred_ && total_iterations_ >= n) { + total_iterations_ -= n; + return true; + } + } + // For non-batch runs, total_iterations_ must be 0 by now. + if (is_batch && total_iterations_ != 0) { + batch_leftover_ = n - total_iterations_; + total_iterations_ = 0; + return true; + } + FinishKeepRunning(); + return false; +} + +struct State::StateIterator { + struct BENCHMARK_UNUSED Value {}; + typedef std::forward_iterator_tag iterator_category; + typedef Value value_type; + typedef Value reference; + typedef Value pointer; + typedef std::ptrdiff_t difference_type; + + private: + friend class State; + BENCHMARK_ALWAYS_INLINE + StateIterator() : cached_(0), parent_() {} + + BENCHMARK_ALWAYS_INLINE + explicit StateIterator(State* st) + : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {} + + public: + BENCHMARK_ALWAYS_INLINE + Value operator*() const { return Value(); } + + BENCHMARK_ALWAYS_INLINE + StateIterator& operator++() { + assert(cached_ > 0); + --cached_; + return *this; + } + + BENCHMARK_ALWAYS_INLINE + bool operator!=(StateIterator const&) const { + if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) return true; + parent_->FinishKeepRunning(); + return false; + } + + private: + IterationCount cached_; + State* const parent_; +}; + +inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() { + return StateIterator(this); +} +inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() { + StartKeepRunning(); + return StateIterator(); +} + +namespace internal { + +typedef void(Function)(State&); + +// ------------------------------------------------------ +// Benchmark registration object. The BENCHMARK() macro expands +// into an internal::Benchmark* object. Various methods can +// be called on this object to change the properties of the benchmark. +// Each method returns "this" so that multiple method calls can +// chained into one expression. +class BENCHMARK_EXPORT Benchmark { + public: + virtual ~Benchmark(); + + // Note: the following methods all return "this" so that multiple + // method calls can be chained together in one expression. + + // Specify the name of the benchmark + Benchmark* Name(const std::string& name); + + // Run this benchmark once with "x" as the extra argument passed + // to the function. + // REQUIRES: The function passed to the constructor must accept an arg1. + Benchmark* Arg(int64_t x); + + // Run this benchmark with the given time unit for the generated output report + Benchmark* Unit(TimeUnit unit); + + // Run this benchmark once for a number of values picked from the + // range [start..limit]. (start and limit are always picked.) + // REQUIRES: The function passed to the constructor must accept an arg1. + Benchmark* Range(int64_t start, int64_t limit); + + // Run this benchmark once for all values in the range [start..limit] with + // specific step + // REQUIRES: The function passed to the constructor must accept an arg1. + Benchmark* DenseRange(int64_t start, int64_t limit, int step = 1); + + // Run this benchmark once with "args" as the extra arguments passed + // to the function. + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* Args(const std::vector& args); + + // Equivalent to Args({x, y}) + // NOTE: This is a legacy C++03 interface provided for compatibility only. + // New code should use 'Args'. + Benchmark* ArgPair(int64_t x, int64_t y) { + std::vector args; + args.push_back(x); + args.push_back(y); + return Args(args); + } + + // Run this benchmark once for a number of values picked from the + // ranges [start..limit]. (starts and limits are always picked.) + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* Ranges(const std::vector >& ranges); + + // Run this benchmark once for each combination of values in the (cartesian) + // product of the supplied argument lists. + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* ArgsProduct(const std::vector >& arglists); + + // Equivalent to ArgNames({name}) + Benchmark* ArgName(const std::string& name); + + // Set the argument names to display in the benchmark name. If not called, + // only argument values will be shown. + Benchmark* ArgNames(const std::vector& names); + + // Equivalent to Ranges({{lo1, hi1}, {lo2, hi2}}). + // NOTE: This is a legacy C++03 interface provided for compatibility only. + // New code should use 'Ranges'. + Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) { + std::vector > ranges; + ranges.push_back(std::make_pair(lo1, hi1)); + ranges.push_back(std::make_pair(lo2, hi2)); + return Ranges(ranges); + } + + // Have "setup" and/or "teardown" invoked once for every benchmark run. + // If the benchmark is multi-threaded (will run in k threads concurrently), + // the setup callback will be be invoked exactly once (not k times) before + // each run with k threads. Time allowing (e.g. for a short benchmark), there + // may be multiple such runs per benchmark, each run with its own + // "setup"/"teardown". + // + // If the benchmark uses different size groups of threads (e.g. via + // ThreadRange), the above will be true for each size group. + // + // The callback will be passed a State object, which includes the number + // of threads, thread-index, benchmark arguments, etc. + // + // The callback must not be NULL or self-deleting. + Benchmark* Setup(void (*setup)(const benchmark::State&)); + Benchmark* Teardown(void (*teardown)(const benchmark::State&)); + + // Pass this benchmark object to *func, which can customize + // the benchmark by calling various methods like Arg, Args, + // Threads, etc. + Benchmark* Apply(void (*func)(Benchmark* benchmark)); + + // Set the range multiplier for non-dense range. If not called, the range + // multiplier kRangeMultiplier will be used. + Benchmark* RangeMultiplier(int multiplier); + + // Set the minimum amount of time to use when running this benchmark. This + // option overrides the `benchmark_min_time` flag. + // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. + Benchmark* MinTime(double t); + + // Set the minimum amount of time to run the benchmark before taking runtimes + // of this benchmark into account. This + // option overrides the `benchmark_min_warmup_time` flag. + // REQUIRES: `t >= 0` and `Iterations` has not been called on this benchmark. + Benchmark* MinWarmUpTime(double t); + + // Specify the amount of iterations that should be run by this benchmark. + // This option overrides the `benchmark_min_time` flag. + // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. + // + // NOTE: This function should only be used when *exact* iteration control is + // needed and never to control or limit how long a benchmark runs, where + // `--benchmark_min_time=s` or `MinTime(...)` should be used instead. + Benchmark* Iterations(IterationCount n); + + // Specify the amount of times to repeat this benchmark. This option overrides + // the `benchmark_repetitions` flag. + // REQUIRES: `n > 0` + Benchmark* Repetitions(int n); + + // Specify if each repetition of the benchmark should be reported separately + // or if only the final statistics should be reported. If the benchmark + // is not repeated then the single result is always reported. + // Applies to *ALL* reporters (display and file). + Benchmark* ReportAggregatesOnly(bool value = true); + + // Same as ReportAggregatesOnly(), but applies to display reporter only. + Benchmark* DisplayAggregatesOnly(bool value = true); + + // By default, the CPU time is measured only for the main thread, which may + // be unrepresentative if the benchmark uses threads internally. If called, + // the total CPU time spent by all the threads will be measured instead. + // By default, only the main thread CPU time will be measured. + Benchmark* MeasureProcessCPUTime(); + + // If a particular benchmark should use the Wall clock instead of the CPU time + // (be it either the CPU time of the main thread only (default), or the + // total CPU usage of the benchmark), call this method. If called, the elapsed + // (wall) time will be used to control how many iterations are run, and in the + // printing of items/second or MB/seconds values. + // If not called, the CPU time used by the benchmark will be used. + Benchmark* UseRealTime(); + + // If a benchmark must measure time manually (e.g. if GPU execution time is + // being + // measured), call this method. If called, each benchmark iteration should + // call + // SetIterationTime(seconds) to report the measured time, which will be used + // to control how many iterations are run, and in the printing of items/second + // or MB/second values. + Benchmark* UseManualTime(); + + // Set the asymptotic computational complexity for the benchmark. If called + // the asymptotic computational complexity will be shown on the output. + Benchmark* Complexity(BigO complexity = benchmark::oAuto); + + // Set the asymptotic computational complexity for the benchmark. If called + // the asymptotic computational complexity will be shown on the output. + Benchmark* Complexity(BigOFunc* complexity); + + // Add this statistics to be computed over all the values of benchmark run + Benchmark* ComputeStatistics(const std::string& name, + StatisticsFunc* statistics, + StatisticUnit unit = kTime); + + // Support for running multiple copies of the same benchmark concurrently + // in multiple threads. This may be useful when measuring the scaling + // of some piece of code. + + // Run one instance of this benchmark concurrently in t threads. + Benchmark* Threads(int t); + + // Pick a set of values T from [min_threads,max_threads]. + // min_threads and max_threads are always included in T. Run this + // benchmark once for each value in T. The benchmark run for a + // particular value t consists of t threads running the benchmark + // function concurrently. For example, consider: + // BENCHMARK(Foo)->ThreadRange(1,16); + // This will run the following benchmarks: + // Foo in 1 thread + // Foo in 2 threads + // Foo in 4 threads + // Foo in 8 threads + // Foo in 16 threads + Benchmark* ThreadRange(int min_threads, int max_threads); + + // For each value n in the range, run this benchmark once using n threads. + // min_threads and max_threads are always included in the range. + // stride specifies the increment. E.g. DenseThreadRange(1, 8, 3) starts + // a benchmark with 1, 4, 7 and 8 threads. + Benchmark* DenseThreadRange(int min_threads, int max_threads, int stride = 1); + + // Equivalent to ThreadRange(NumCPUs(), NumCPUs()) + Benchmark* ThreadPerCpu(); + + virtual void Run(State& state) = 0; + + TimeUnit GetTimeUnit() const; + + protected: + explicit Benchmark(const char* name); + void SetName(const char* name); + + int ArgsCnt() const; + + private: + friend class BenchmarkFamilies; + friend class BenchmarkInstance; + + std::string name_; + AggregationReportMode aggregation_report_mode_; + std::vector arg_names_; // Args for all benchmark runs + std::vector > args_; // Args for all benchmark runs + + TimeUnit time_unit_; + bool use_default_time_unit_; + + int range_multiplier_; + double min_time_; + double min_warmup_time_; + IterationCount iterations_; + int repetitions_; + bool measure_process_cpu_time_; + bool use_real_time_; + bool use_manual_time_; + BigO complexity_; + BigOFunc* complexity_lambda_; + std::vector statistics_; + std::vector thread_counts_; + + typedef void (*callback_function)(const benchmark::State&); + callback_function setup_; + callback_function teardown_; + + Benchmark(Benchmark const&) +#if defined(BENCHMARK_HAS_CXX11) + = delete +#endif + ; + + Benchmark& operator=(Benchmark const&) +#if defined(BENCHMARK_HAS_CXX11) + = delete +#endif + ; +}; + +} // namespace internal + +// Create and register a benchmark with the specified 'name' that invokes +// the specified functor 'fn'. +// +// RETURNS: A pointer to the registered benchmark. +internal::Benchmark* RegisterBenchmark(const char* name, + internal::Function* fn); + +#if defined(BENCHMARK_HAS_CXX11) +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); +#endif + +// Remove all registered benchmarks. All pointers to previously registered +// benchmarks are invalidated. +BENCHMARK_EXPORT void ClearRegisteredBenchmarks(); + +namespace internal { +// The class used to hold all Benchmarks created from static function. +// (ie those created using the BENCHMARK(...) macros. +class BENCHMARK_EXPORT FunctionBenchmark : public Benchmark { + public: + FunctionBenchmark(const char* name, Function* func) + : Benchmark(name), func_(func) {} + + void Run(State& st) BENCHMARK_OVERRIDE; + + private: + Function* func_; +}; + +#ifdef BENCHMARK_HAS_CXX11 +template +class LambdaBenchmark : public Benchmark { + public: + void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } + + private: + template + LambdaBenchmark(const char* name, OLambda&& lam) + : Benchmark(name), lambda_(std::forward(lam)) {} + + LambdaBenchmark(LambdaBenchmark const&) = delete; + + template // NOLINTNEXTLINE(readability-redundant-declaration) + friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); + + Lambda lambda_; +}; +#endif +} // namespace internal + +inline internal::Benchmark* RegisterBenchmark(const char* name, + internal::Function* fn) { + return internal::RegisterBenchmarkInternal( + ::new internal::FunctionBenchmark(name, fn)); +} + +#ifdef BENCHMARK_HAS_CXX11 +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { + using BenchType = + internal::LambdaBenchmark::type>; + return internal::RegisterBenchmarkInternal( + ::new BenchType(name, std::forward(fn))); +} +#endif + +#if defined(BENCHMARK_HAS_CXX11) && \ + (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, + Args&&... args) { + return benchmark::RegisterBenchmark( + name, [=](benchmark::State& st) { fn(st, args...); }); +} +#else +#define BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK +#endif + +// The base class for all fixture tests. +class Fixture : public internal::Benchmark { + public: + Fixture() : internal::Benchmark("") {} + + void Run(State& st) BENCHMARK_OVERRIDE { + this->SetUp(st); + this->BenchmarkCase(st); + this->TearDown(st); + } + + // These will be deprecated ... + virtual void SetUp(const State&) {} + virtual void TearDown(const State&) {} + // ... In favor of these. + virtual void SetUp(State& st) { SetUp(const_cast(st)); } + virtual void TearDown(State& st) { TearDown(const_cast(st)); } + + protected: + virtual void BenchmarkCase(State&) = 0; +}; +} // namespace benchmark + +// ------------------------------------------------------ +// Macro to register benchmarks + +// Check that __COUNTER__ is defined and that __COUNTER__ increases by 1 +// every time it is expanded. X + 1 == X + 0 is used in case X is defined to be +// empty. If X is empty the expression becomes (+1 == +0). +#if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0) +#define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__ +#else +#define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__ +#endif + +// Helpers for generating unique variable names +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK_PRIVATE_NAME(...) \ + BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, \ + __VA_ARGS__) +#else +#define BENCHMARK_PRIVATE_NAME(n) \ + BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, n) +#endif // BENCHMARK_HAS_CXX11 + +#define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) +#define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c +// Helper for concatenation with macro name expansion +#define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \ + BaseClass##_##Method##_Benchmark + +#define BENCHMARK_PRIVATE_DECLARE(n) \ + static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \ + BENCHMARK_UNUSED + +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK(...) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark(#__VA_ARGS__, \ + &__VA_ARGS__))) +#else +#define BENCHMARK(n) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark(#n, n))) +#endif // BENCHMARK_HAS_CXX11 + +// Old-style macros +#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) +#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) +#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) +#define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) +#define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ + BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) + +#ifdef BENCHMARK_HAS_CXX11 + +// Register a benchmark which invokes the function specified by `func` +// with the additional arguments specified by `...`. +// +// For example: +// +// template ` +// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { +// [...] +//} +// /* Registers a benchmark named "BM_takes_args/int_string_test` */ +// BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); +#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark( \ + #func "/" #test_case_name, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) + +#endif // BENCHMARK_HAS_CXX11 + +// This will register a benchmark for a templatized function. For example: +// +// template +// void BM_Foo(int iters); +// +// BENCHMARK_TEMPLATE(BM_Foo, 1); +// +// will register BM_Foo<1> as a benchmark. +#define BENCHMARK_TEMPLATE1(n, a) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark(#n "<" #a ">", n))) + +#define BENCHMARK_TEMPLATE2(n, a, b) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark(#n "<" #a "," #b ">", \ + n))) + +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK_TEMPLATE(n, ...) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark( \ + #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) +#else +#define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) +#endif + +#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ + }; + +#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ + }; + +#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ + }; + +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \ + class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ + }; +#else +#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ + BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(n, a) +#endif + +#define BENCHMARK_DEFINE_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE2_DEFINE_F(BaseClass, Method, a, b) \ + BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, ...) \ + BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase +#else +#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) +#endif + +#define BENCHMARK_REGISTER_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_REGISTER_F(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)) + +#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \ + BENCHMARK_PRIVATE_DECLARE(TestName) = \ + (::benchmark::internal::RegisterBenchmarkInternal(new TestName())) + +// This macro will define and register a benchmark within a fixture class. +#define BENCHMARK_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#define BENCHMARK_TEMPLATE2_F(BaseClass, Method, a, b) \ + BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase + +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ + BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase +#else +#define BENCHMARK_TEMPLATE_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) +#endif + +// Helper macro to create a main routine in a test that runs the benchmarks +// Note the workaround for Hexagon simulator passing argc != 0, argv = NULL. +#define BENCHMARK_MAIN() \ + int main(int argc, char** argv) { \ + char arg0_default[] = "benchmark"; \ + char* args_default = arg0_default; \ + if (!argv) { \ + argc = 1; \ + argv = &args_default; \ + } \ + ::benchmark::Initialize(&argc, argv); \ + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ + ::benchmark::RunSpecifiedBenchmarks(); \ + ::benchmark::Shutdown(); \ + return 0; \ + } \ + int main(int, char**) + +// ------------------------------------------------------ +// Benchmark Reporters + +namespace benchmark { + +struct BENCHMARK_EXPORT CPUInfo { + struct CacheInfo { + std::string type; + int level; + int size; + int num_sharing; + }; + + enum Scaling { UNKNOWN, ENABLED, DISABLED }; + + int num_cpus; + Scaling scaling; + double cycles_per_second; + std::vector caches; + std::vector load_avg; + + static const CPUInfo& Get(); + + private: + CPUInfo(); + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); +}; + +// Adding Struct for System Information +struct BENCHMARK_EXPORT SystemInfo { + std::string name; + static const SystemInfo& Get(); + + private: + SystemInfo(); + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); +}; + +// BenchmarkName contains the components of the Benchmark's name +// which allows individual fields to be modified or cleared before +// building the final name using 'str()'. +struct BENCHMARK_EXPORT BenchmarkName { + std::string function_name; + std::string args; + std::string min_time; + std::string min_warmup_time; + std::string iterations; + std::string repetitions; + std::string time_type; + std::string threads; + + // Return the full name of the benchmark with each non-empty + // field separated by a '/' + std::string str() const; +}; + +// Interface for custom benchmark result printers. +// By default, benchmark reports are printed to stdout. However an application +// can control the destination of the reports by calling +// RunSpecifiedBenchmarks and passing it a custom reporter object. +// The reporter object must implement the following interface. +class BENCHMARK_EXPORT BenchmarkReporter { + public: + struct Context { + CPUInfo const& cpu_info; + SystemInfo const& sys_info; + // The number of chars in the longest benchmark name. + size_t name_field_width; + static const char* executable_name; + Context(); + }; + + struct BENCHMARK_EXPORT Run { + static const int64_t no_repetition_index = -1; + enum RunType { RT_Iteration, RT_Aggregate }; + + Run() + : run_type(RT_Iteration), + aggregate_unit(kTime), + error_occurred(false), + iterations(1), + threads(1), + time_unit(GetDefaultTimeUnit()), + real_accumulated_time(0), + cpu_accumulated_time(0), + max_heapbytes_used(0), + complexity(oNone), + complexity_lambda(), + complexity_n(0), + report_big_o(false), + report_rms(false), + memory_result(NULL), + allocs_per_iter(0.0) {} + + std::string benchmark_name() const; + BenchmarkName run_name; + int64_t family_index; + int64_t per_family_instance_index; + RunType run_type; + std::string aggregate_name; + StatisticUnit aggregate_unit; + std::string report_label; // Empty if not set by benchmark. + bool error_occurred; + std::string error_message; + + IterationCount iterations; + int64_t threads; + int64_t repetition_index; + int64_t repetitions; + TimeUnit time_unit; + double real_accumulated_time; + double cpu_accumulated_time; + + // Return a value representing the real time per iteration in the unit + // specified by 'time_unit'. + // NOTE: If 'iterations' is zero the returned value represents the + // accumulated time. + double GetAdjustedRealTime() const; + + // Return a value representing the cpu time per iteration in the unit + // specified by 'time_unit'. + // NOTE: If 'iterations' is zero the returned value represents the + // accumulated time. + double GetAdjustedCPUTime() const; + + // This is set to 0.0 if memory tracing is not enabled. + double max_heapbytes_used; + + // Keep track of arguments to compute asymptotic complexity + BigO complexity; + BigOFunc* complexity_lambda; + int64_t complexity_n; + + // what statistics to compute from the measurements + const std::vector* statistics; + + // Inform print function whether the current run is a complexity report + bool report_big_o; + bool report_rms; + + UserCounters counters; + + // Memory metrics. + const MemoryManager::Result* memory_result; + double allocs_per_iter; + }; + + struct PerFamilyRunReports { + PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {} + + // How many runs will all instances of this benchmark perform? + int num_runs_total; + + // How many runs have happened already? + int num_runs_done; + + // The reports about (non-errneous!) runs of this family. + std::vector Runs; + }; + + // Construct a BenchmarkReporter with the output stream set to 'std::cout' + // and the error stream set to 'std::cerr' + BenchmarkReporter(); + + // Called once for every suite of benchmarks run. + // The parameter "context" contains information that the + // reporter may wish to use when generating its report, for example the + // platform under which the benchmarks are running. The benchmark run is + // never started if this function returns false, allowing the reporter + // to skip runs based on the context information. + virtual bool ReportContext(const Context& context) = 0; + + // Called once for each group of benchmark runs, gives information about + // cpu-time and heap memory usage during the benchmark run. If the group + // of runs contained more than two entries then 'report' contains additional + // elements representing the mean and standard deviation of those runs. + // Additionally if this group of runs was the last in a family of benchmarks + // 'reports' contains additional entries representing the asymptotic + // complexity and RMS of that benchmark family. + virtual void ReportRuns(const std::vector& report) = 0; + + // Called once and only once after ever group of benchmarks is run and + // reported. + virtual void Finalize() {} + + // REQUIRES: The object referenced by 'out' is valid for the lifetime + // of the reporter. + void SetOutputStream(std::ostream* out) { + assert(out); + output_stream_ = out; + } + + // REQUIRES: The object referenced by 'err' is valid for the lifetime + // of the reporter. + void SetErrorStream(std::ostream* err) { + assert(err); + error_stream_ = err; + } + + std::ostream& GetOutputStream() const { return *output_stream_; } + + std::ostream& GetErrorStream() const { return *error_stream_; } + + virtual ~BenchmarkReporter(); + + // Write a human readable string to 'out' representing the specified + // 'context'. + // REQUIRES: 'out' is non-null. + static void PrintBasicContext(std::ostream* out, Context const& context); + + private: + std::ostream* output_stream_; + std::ostream* error_stream_; +}; + +// Simple reporter that outputs benchmark data to the console. This is the +// default reporter used by RunSpecifiedBenchmarks(). +class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { + public: + enum OutputOptions { + OO_None = 0, + OO_Color = 1, + OO_Tabular = 2, + OO_ColorTabular = OO_Color | OO_Tabular, + OO_Defaults = OO_ColorTabular + }; + explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) + : output_options_(opts_), name_field_width_(0), printed_header_(false) {} + + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + + protected: + virtual void PrintRunData(const Run& report); + virtual void PrintHeader(const Run& report); + + OutputOptions output_options_; + size_t name_field_width_; + UserCounters prev_counters_; + bool printed_header_; +}; + +class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { + public: + JSONReporter() : first_report_(true) {} + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + void Finalize() BENCHMARK_OVERRIDE; + + private: + void PrintRunData(const Run& report); + + bool first_report_; +}; + +class BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG( + "The CSV Reporter will be removed in a future release") CSVReporter + : public BenchmarkReporter { + public: + CSVReporter() : printed_header_(false) {} + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + + private: + void PrintRunData(const Run& report); + + bool printed_header_; + std::set user_counter_names_; +}; + +inline const char* GetTimeUnitString(TimeUnit unit) { + switch (unit) { + case kSecond: + return "s"; + case kMillisecond: + return "ms"; + case kMicrosecond: + return "us"; + case kNanosecond: + return "ns"; + } + BENCHMARK_UNREACHABLE(); +} + +inline double GetTimeUnitMultiplier(TimeUnit unit) { + switch (unit) { + case kSecond: + return 1; + case kMillisecond: + return 1e3; + case kMicrosecond: + return 1e6; + case kNanosecond: + return 1e9; + } + BENCHMARK_UNREACHABLE(); +} + +// Creates a list of integer values for the given range and multiplier. +// This can be used together with ArgsProduct() to allow multiple ranges +// with different multipliers. +// Example: +// ArgsProduct({ +// CreateRange(0, 1024, /*multi=*/32), +// CreateRange(0, 100, /*multi=*/4), +// CreateDenseRange(0, 4, /*step=*/1), +// }); +BENCHMARK_EXPORT +std::vector CreateRange(int64_t lo, int64_t hi, int multi); + +// Creates a list of integer values for the given range and step. +BENCHMARK_EXPORT +std::vector CreateDenseRange(int64_t start, int64_t limit, int step); + +} // namespace benchmark + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#endif // BENCHMARK_BENCHMARK_H_ diff --git a/include/benchmark/.#benchmark.h b/include/benchmark/.#benchmark.h new file mode 120000 index 0000000000..73b60e406f --- /dev/null +++ b/include/benchmark/.#benchmark.h @@ -0,0 +1 @@ +vyng@vyng.cam.corp.google.com.2914513:1672780664 \ No newline at end of file diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 6f2ff32b16..359a9c1ba9 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -280,7 +280,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); namespace benchmark { class BenchmarkReporter; -BENCHMARK_EXPORT const double kDefaultMinTime = 0.5; +const double kDefaultMinTime = 0.5; BENCHMARK_EXPORT void PrintDefaultHelp(); diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 8eb72e7fed..7bb51913c9 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -66,6 +66,7 @@ MemoryManager* memory_manager = nullptr; namespace { static constexpr IterationCount kMaxIterations = 1000000000; +static constexpr double kDefaultMinTime = 0.5; BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, @@ -272,22 +273,22 @@ static bool AlmostEqual(double a, double b) { } void BenchmarkRunner::UpdateReport(RunResults& current_run_results) { - // If we the actual min_time is different from the value in the benchmark instance - // AND if it's not the default value then update it. - bool update_time = !AlmostEqual(min_time, b.min_time()) - && !AlmostEqual(min_time, kDefaultMinTime); - bool update_iters = has_explicit_iteration_count && iters != b.iterations(); - - if (!update_time && !update_iters) return; - - auto UpdateRun = [](bool update_t, double new_min_time, bool update_i, - IterationCount new_iters, BenchmarkReporter::Run& run) { - if (update_t) - run.run_name.min_time = StrFormat("min_time:%0.3fs", new_min_time); - if (update_i) run.iterations = new_iters; - }; - - for (auto& run : current_run_results.non_aggregates) + // If we the actual min_time is different from the value in the benchmark + // instance AND if it's not the default value then update it. + bool update_time = !AlmostEqual(min_time, b.min_time()) && + !AlmostEqual(min_time, kDefaultMinTime); + bool update_iters = has_explicit_iteration_count && iters != b.iterations(); + + if (!update_time && !update_iters) return; + + auto UpdateRun = [](bool update_t, double new_min_time, bool update_i, + IterationCount new_iters, BenchmarkReporter::Run& run) { + if (update_t) + run.run_name.min_time = StrFormat("min_time:%0.3fs", new_min_time); + if (update_i) run.iterations = new_iters; + }; + + for (auto& run : current_run_results.non_aggregates) UpdateRun(update_time, min_time, update_iters, iters, run); for (auto& run : current_run_results.aggregates_only) UpdateRun(update_time, min_time, update_iters, iters, run); From 860b2b61e70f62662a754d10878a68c759ef20df Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 16:09:33 -0500 Subject: [PATCH 21/34] remove misc file --- include/benchmark/.#benchmark.h | 1 - 1 file changed, 1 deletion(-) delete mode 120000 include/benchmark/.#benchmark.h diff --git a/include/benchmark/.#benchmark.h b/include/benchmark/.#benchmark.h deleted file mode 120000 index 73b60e406f..0000000000 --- a/include/benchmark/.#benchmark.h +++ /dev/null @@ -1 +0,0 @@ -vyng@vyng.cam.corp.google.com.2914513:1672780664 \ No newline at end of file From 3dc210d5797254c88c4e2023b8972cc1dd07cd91 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 12 Jan 2023 16:10:20 -0500 Subject: [PATCH 22/34] removed backup files --- include/benchmark/#benchmark.h# | 1873 ------------------------------- 1 file changed, 1873 deletions(-) delete mode 100644 include/benchmark/#benchmark.h# diff --git a/include/benchmark/#benchmark.h# b/include/benchmark/#benchmark.h# deleted file mode 100644 index 1222aedcee..0000000000 --- a/include/benchmark/#benchmark.h# +++ /dev/null @@ -1,1873 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Support for registering benchmarks for functions. - -/* Example usage: -// Define a function that executes the code to be measured a -// specified number of times: -static void BM_StringCreation(benchmark::State& state) { - for (auto _ : state) - std::string empty_string; -} - -// Register the function as a benchmark -BENCHMARK(BM_StringCreation); - -// Define another benchmark -static void BM_StringCopy(benchmark::State& state) { - std::string x = "hello"; - for (auto _ : state) - std::string copy(x); -} -BENCHMARK(BM_StringCopy); - -// Augment the main() program to invoke benchmarks if specified -// via the --benchmark_filter command line flag. E.g., -// my_unittest --benchmark_filter=all -// my_unittest --benchmark_filter=BM_StringCreation -// my_unittest --benchmark_filter=String -// my_unittest --benchmark_filter='Copy|Creation' -int main(int argc, char** argv) { - benchmark::Initialize(&argc, argv); - benchmark::RunSpecifiedBenchmarks(); - benchmark::Shutdown(); - return 0; -} - -// Sometimes a family of microbenchmarks can be implemented with -// just one routine that takes an extra argument to specify which -// one of the family of benchmarks to run. For example, the following -// code defines a family of microbenchmarks for measuring the speed -// of memcpy() calls of different lengths: - -static void BM_memcpy(benchmark::State& state) { - char* src = new char[state.range(0)]; char* dst = new char[state.range(0)]; - memset(src, 'x', state.range(0)); - for (auto _ : state) - memcpy(dst, src, state.range(0)); - state.SetBytesProcessed(state.iterations() * state.range(0)); - delete[] src; delete[] dst; -} -BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); - -// The preceding code is quite repetitive, and can be replaced with the -// following short-hand. The following invocation will pick a few -// appropriate arguments in the specified range and will generate a -// microbenchmark for each such argument. -BENCHMARK(BM_memcpy)->Range(8, 8<<10); - -// You might have a microbenchmark that depends on two inputs. For -// example, the following code defines a family of microbenchmarks for -// measuring the speed of set insertion. -static void BM_SetInsert(benchmark::State& state) { - set data; - for (auto _ : state) { - state.PauseTiming(); - data = ConstructRandomSet(state.range(0)); - state.ResumeTiming(); - for (int j = 0; j < state.range(1); ++j) - data.insert(RandomNumber()); - } -} -BENCHMARK(BM_SetInsert) - ->Args({1<<10, 128}) - ->Args({2<<10, 128}) - ->Args({4<<10, 128}) - ->Args({8<<10, 128}) - ->Args({1<<10, 512}) - ->Args({2<<10, 512}) - ->Args({4<<10, 512}) - ->Args({8<<10, 512}); - -// The preceding code is quite repetitive, and can be replaced with -// the following short-hand. The following macro will pick a few -// appropriate arguments in the product of the two specified ranges -// and will generate a microbenchmark for each such pair. -BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); - -// For more complex patterns of inputs, passing a custom function -// to Apply allows programmatic specification of an -// arbitrary set of arguments to run the microbenchmark on. -// The following example enumerates a dense range on -// one parameter, and a sparse range on the second. -static void CustomArguments(benchmark::internal::Benchmark* b) { - for (int i = 0; i <= 10; ++i) - for (int j = 32; j <= 1024*1024; j *= 8) - b->Args({i, j}); -} -BENCHMARK(BM_SetInsert)->Apply(CustomArguments); - -// Templated microbenchmarks work the same way: -// Produce then consume 'size' messages 'iters' times -// Measures throughput in the absence of multiprogramming. -template int BM_Sequential(benchmark::State& state) { - Q q; - typename Q::value_type v; - for (auto _ : state) { - for (int i = state.range(0); i--; ) - q.push(v); - for (int e = state.range(0); e--; ) - q.Wait(&v); - } - // actually messages, not bytes: - state.SetBytesProcessed(state.iterations() * state.range(0)); -} -BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); - -Use `Benchmark::MinTime(double t)` to set the minimum time used to run the -benchmark. This option overrides the `benchmark_min_time` flag. - -void BM_test(benchmark::State& state) { - ... body ... -} -BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds. - -In a multithreaded test, it is guaranteed that none of the threads will start -until all have reached the loop start, and all will have finished before any -thread exits the loop body. As such, any global setup or teardown you want to -do can be wrapped in a check against the thread index: - -static void BM_MultiThreaded(benchmark::State& state) { - if (state.thread_index() == 0) { - // Setup code here. - } - for (auto _ : state) { - // Run the test as normal. - } - if (state.thread_index() == 0) { - // Teardown code here. - } -} -BENCHMARK(BM_MultiThreaded)->Threads(4); - - -If a benchmark runs a few milliseconds it may be hard to visually compare the -measured times, since the output data is given in nanoseconds per default. In -order to manually set the time unit, you can specify it manually: - -BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); -*/ - -#ifndef BENCHMARK_BENCHMARK_H_ -#define BENCHMARK_BENCHMARK_H_ - -// The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. -#if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) -#define BENCHMARK_HAS_CXX11 -#endif - -// This _MSC_VER check should detect VS 2017 v15.3 and newer. -#if __cplusplus >= 201703L || \ - (defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L) -#define BENCHMARK_HAS_CXX17 -#endif - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/export.h" - -#if defined(BENCHMARK_HAS_CXX11) -#include -#include -#include -#include -#endif - -#if defined(_MSC_VER) -#include // for _ReadWriteBarrier -#endif - -#ifndef BENCHMARK_HAS_CXX11 -#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&); \ - TypeName& operator=(const TypeName&) -#else -#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&) = delete; \ - TypeName& operator=(const TypeName&) = delete -#endif - -#ifdef BENCHMARK_HAS_CXX17 -#define BENCHMARK_UNUSED [[maybe_unused]] -#elif defined(__GNUC__) || defined(__clang__) -#define BENCHMARK_UNUSED __attribute__((unused)) -#else -#define BENCHMARK_UNUSED -#endif - -#if defined(__GNUC__) || defined(__clang__) -#define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) -#elif defined(_MSC_VER) && !defined(__clang__) -#define BENCHMARK_ALWAYS_INLINE __forceinline -#define __func__ __FUNCTION__ -#else -#define BENCHMARK_ALWAYS_INLINE -#endif - -#define BENCHMARK_INTERNAL_TOSTRING2(x) #x -#define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) - -// clang-format off -#if defined(__GNUC__) && !defined(__NVCC__) || defined(__clang__) -#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) -#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) -#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") -#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop") -#else -#define BENCHMARK_BUILTIN_EXPECT(x, y) x -#define BENCHMARK_DEPRECATED_MSG(msg) -#define BENCHMARK_WARNING_MSG(msg) \ - __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \ - __LINE__) ") : warning note: " msg)) -#define BENCHMARK_DISABLE_DEPRECATED_WARNING -#define BENCHMARK_RESTORE_DEPRECATED_WARNING -#endif -// clang-format on - -#if defined(__GNUC__) && !defined(__clang__) -#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#if defined(__GNUC__) || __has_builtin(__builtin_unreachable) -#define BENCHMARK_UNREACHABLE() __builtin_unreachable() -#elif defined(_MSC_VER) -#define BENCHMARK_UNREACHABLE() __assume(false) -#else -#define BENCHMARK_UNREACHABLE() ((void)0) -#endif - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_OVERRIDE override -#else -#define BENCHMARK_OVERRIDE -#endif - -#if defined(_MSC_VER) -#pragma warning(push) -// C4251: needs to have dll-interface to be used by clients of class -#pragma warning(disable : 4251) -#endif - -namespace benchmark { -class BenchmarkReporter; - -BENCHMARK_EXPORT void PrintDefaultHelp(); - -BENCHMARK_EXPORT void Initialize(int* argc, char** argv, - void (*HelperPrinterf)() = PrintDefaultHelp); -BENCHMARK_EXPORT void Shutdown(); - -// Report to stdout all arguments in 'argv' as unrecognized except the first. -// Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). -BENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv); - -// Returns the current value of --benchmark_filter. -BENCHMARK_EXPORT std::string GetBenchmarkFilter(); - -// Sets a new value to --benchmark_filter. (This will override this flag's -// current value). -// Should be called after `benchmark::Initialize()`, as -// `benchmark::Initialize()` will override the flag's value. -BENCHMARK_EXPORT void SetBenchmarkFilter(std::string value); - -// Returns the current value of --v (command line value for verbosity). -BENCHMARK_EXPORT int32_t GetBenchmarkVerbosity(); - -// Creates a default display reporter. Used by the library when no display -// reporter is provided, but also made available for external use in case a -// custom reporter should respect the `--benchmark_format` flag as a fallback -BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); - -// Generate a list of benchmarks matching the specified --benchmark_filter flag -// and if --benchmark_list_tests is specified return after printing the name -// of each matching benchmark. Otherwise run each matching benchmark and -// report the results. -// -// spec : Specify the benchmarks to run. If users do not specify this arg, -// then the value of FLAGS_benchmark_filter -// will be used. -// -// The second and third overload use the specified 'display_reporter' and -// 'file_reporter' respectively. 'file_reporter' will write to the file -// specified -// by '--benchmark_output'. If '--benchmark_output' is not given the -// 'file_reporter' is ignored. -// -// RETURNS: The number of matching benchmarks. -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(); -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec); - -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec); - -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks( - BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter, std::string spec); - -// TimeUnit is passed to a benchmark in order to specify the order of magnitude -// for the measured time. -enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; - -BENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit(); - -// Sets the default time unit the benchmarks use -// Has to be called before the benchmark loop to take effect -BENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit); - -// If a MemoryManager is registered (via RegisterMemoryManager()), -// it can be used to collect and report allocation metrics for a run of the -// benchmark. -class MemoryManager { - public: - static const int64_t TombstoneValue; - - struct Result { - Result() - : num_allocs(0), - max_bytes_used(0), - total_allocated_bytes(TombstoneValue), - net_heap_growth(TombstoneValue) {} - - // The number of allocations made in total between Start and Stop. - int64_t num_allocs; - - // The peak memory use between Start and Stop. - int64_t max_bytes_used; - - // The total memory allocated, in bytes, between Start and Stop. - // Init'ed to TombstoneValue if metric not available. - int64_t total_allocated_bytes; - - // The net changes in memory, in bytes, between Start and Stop. - // ie., total_allocated_bytes - total_deallocated_bytes. - // Init'ed to TombstoneValue if metric not available. - int64_t net_heap_growth; - }; - - virtual ~MemoryManager() {} - - // Implement this to start recording allocation information. - virtual void Start() = 0; - - // Implement this to stop recording and fill out the given Result structure. - virtual void Stop(Result& result) = 0; -}; - -// Register a MemoryManager instance that will be used to collect and report -// allocation measurements for benchmark runs. -BENCHMARK_EXPORT -void RegisterMemoryManager(MemoryManager* memory_manager); - -// Add a key-value pair to output as part of the context stanza in the report. -BENCHMARK_EXPORT -void AddCustomContext(const std::string& key, const std::string& value); - -namespace internal { -class Benchmark; -class BenchmarkImp; -class BenchmarkFamilies; - -BENCHMARK_EXPORT std::map*& GetGlobalContext(); - -BENCHMARK_EXPORT -void UseCharPointer(char const volatile*); - -// Take ownership of the pointer and register the benchmark. Return the -// registered benchmark. -BENCHMARK_EXPORT Benchmark* RegisterBenchmarkInternal(Benchmark*); - -// Ensure that the standard streams are properly initialized in every TU. -BENCHMARK_EXPORT int InitializeStreams(); -BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); - -} // namespace internal - -#if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ - defined(__EMSCRIPTEN__) -#define BENCHMARK_HAS_NO_INLINE_ASSEMBLY -#endif - -// Force the compiler to flush pending writes to global memory. Acts as an -// effective read/write barrier -#ifdef BENCHMARK_HAS_CXX11 -inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { - std::atomic_signal_fence(std::memory_order_acq_rel); -} -#endif - -// The DoNotOptimize(...) function can be used to prevent a value or -// expression from being optimized away by the compiler. This function is -// intended to add little to no overhead. -// See: https://youtu.be/nXaxk27zwlk?t=2441 -#ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY -#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER) -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - asm volatile("" : : "r,m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { -#if defined(__clang__) - asm volatile("" : "+r,m"(value) : : "memory"); -#else - asm volatile("" : "+m,r"(value) : : "memory"); -#endif -} -#elif defined(BENCHMARK_HAS_CXX11) && (__GNUC__ >= 5) -// Workaround for a bug with full argument copy overhead with GCC. -// See: #1340 and https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105519 -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value && - (sizeof(Tp) <= sizeof(Tp*))>::type - DoNotOptimize(Tp const& value) { - asm volatile("" : : "r,m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value || - (sizeof(Tp) > sizeof(Tp*))>::type - DoNotOptimize(Tp const& value) { - asm volatile("" : : "m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value && - (sizeof(Tp) <= sizeof(Tp*))>::type - DoNotOptimize(Tp& value) { - asm volatile("" : "+m,r"(value) : : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value || - (sizeof(Tp) > sizeof(Tp*))>::type - DoNotOptimize(Tp& value) { - asm volatile("" : "+m"(value) : : "memory"); -} - -#else -// Fallback for GCC < 5. Can add some overhead because the compiler is forced -// to use memory operations instead of operations with registers. -// TODO: Remove if GCC < 5 will be unsupported. -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - asm volatile("" : : "m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { - asm volatile("" : "+m"(value) : : "memory"); -} -#endif - -#ifndef BENCHMARK_HAS_CXX11 -inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { - asm volatile("" : : : "memory"); -} -#endif -#elif defined(_MSC_VER) -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - internal::UseCharPointer(&reinterpret_cast(value)); - _ReadWriteBarrier(); -} - -#ifndef BENCHMARK_HAS_CXX11 -inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } -#endif -#else -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - internal::UseCharPointer(&reinterpret_cast(value)); -} -// FIXME Add ClobberMemory() for non-gnu and non-msvc compilers, before C++11. -#endif - -// This class is used for user-defined counters. -class Counter { - public: - enum Flags { - kDefaults = 0, - // Mark the counter as a rate. It will be presented divided - // by the duration of the benchmark. - kIsRate = 1 << 0, - // Mark the counter as a thread-average quantity. It will be - // presented divided by the number of threads. - kAvgThreads = 1 << 1, - // Mark the counter as a thread-average rate. See above. - kAvgThreadsRate = kIsRate | kAvgThreads, - // Mark the counter as a constant value, valid/same for *every* iteration. - // When reporting, it will be *multiplied* by the iteration count. - kIsIterationInvariant = 1 << 2, - // Mark the counter as a constant rate. - // When reporting, it will be *multiplied* by the iteration count - // and then divided by the duration of the benchmark. - kIsIterationInvariantRate = kIsRate | kIsIterationInvariant, - // Mark the counter as a iteration-average quantity. - // It will be presented divided by the number of iterations. - kAvgIterations = 1 << 3, - // Mark the counter as a iteration-average rate. See above. - kAvgIterationsRate = kIsRate | kAvgIterations, - - // In the end, invert the result. This is always done last! - kInvert = 1 << 31 - }; - - enum OneK { - // 1'000 items per 1k - kIs1000 = 1000, - // 1'024 items per 1k - kIs1024 = 1024 - }; - - double value; - Flags flags; - OneK oneK; - - BENCHMARK_ALWAYS_INLINE - Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000) - : value(v), flags(f), oneK(k) {} - - BENCHMARK_ALWAYS_INLINE operator double const &() const { return value; } - BENCHMARK_ALWAYS_INLINE operator double&() { return value; } -}; - -// A helper for user code to create unforeseen combinations of Flags, without -// having to do this cast manually each time, or providing this operator. -Counter::Flags inline operator|(const Counter::Flags& LHS, - const Counter::Flags& RHS) { - return static_cast(static_cast(LHS) | - static_cast(RHS)); -} - -// This is the container for the user-defined counters. -typedef std::map UserCounters; - -// BigO is passed to a benchmark in order to specify the asymptotic -// computational -// complexity for the benchmark. In case oAuto is selected, complexity will be -// calculated automatically to the best fit. -enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; - -typedef int64_t IterationCount; - -enum StatisticUnit { kTime, kPercentage }; - -// BigOFunc is passed to a benchmark in order to specify the asymptotic -// computational complexity for the benchmark. -typedef double(BigOFunc)(IterationCount); - -// StatisticsFunc is passed to a benchmark in order to compute some descriptive -// statistics over all the measurements of some type -typedef double(StatisticsFunc)(const std::vector&); - -namespace internal { -struct Statistics { - std::string name_; - StatisticsFunc* compute_; - StatisticUnit unit_; - - Statistics(const std::string& name, StatisticsFunc* compute, - StatisticUnit unit = kTime) - : name_(name), compute_(compute), unit_(unit) {} -}; - -class BenchmarkInstance; -class ThreadTimer; -class ThreadManager; -class PerfCountersMeasurement; - -enum AggregationReportMode -#if defined(BENCHMARK_HAS_CXX11) - : unsigned -#else -#endif -{ - // The mode has not been manually specified - ARM_Unspecified = 0, - // The mode is user-specified. - // This may or may not be set when the following bit-flags are set. - ARM_Default = 1U << 0U, - // File reporter should only output aggregates. - ARM_FileReportAggregatesOnly = 1U << 1U, - // Display reporter should only output aggregates - ARM_DisplayReportAggregatesOnly = 1U << 2U, - // Both reporters should only display aggregates. - ARM_ReportAggregatesOnly = - ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly -}; - -} // namespace internal - -// State is passed to a running Benchmark and contains state for the -// benchmark to use. -class BENCHMARK_EXPORT State { - public: - struct StateIterator; - friend struct StateIterator; - - // Returns iterators used to run each iteration of a benchmark using a - // C++11 ranged-based for loop. These functions should not be called directly. - // - // REQUIRES: The benchmark has not started running yet. Neither begin nor end - // have been called previously. - // - // NOTE: KeepRunning may not be used after calling either of these functions. - BENCHMARK_ALWAYS_INLINE StateIterator begin(); - BENCHMARK_ALWAYS_INLINE StateIterator end(); - - // Returns true if the benchmark should continue through another iteration. - // NOTE: A benchmark may not return from the test until KeepRunning() has - // returned false. - bool KeepRunning(); - - // Returns true iff the benchmark should run n more iterations. - // REQUIRES: 'n' > 0. - // NOTE: A benchmark must not return from the test until KeepRunningBatch() - // has returned false. - // NOTE: KeepRunningBatch() may overshoot by up to 'n' iterations. - // - // Intended usage: - // while (state.KeepRunningBatch(1000)) { - // // process 1000 elements - // } - bool KeepRunningBatch(IterationCount n); - - // REQUIRES: timer is running and 'SkipWithError(...)' has not been called - // by the current thread. - // Stop the benchmark timer. If not called, the timer will be - // automatically stopped after the last iteration of the benchmark loop. - // - // For threaded benchmarks the PauseTiming() function only pauses the timing - // for the current thread. - // - // NOTE: The "real time" measurement is per-thread. If different threads - // report different measurements the largest one is reported. - // - // NOTE: PauseTiming()/ResumeTiming() are relatively - // heavyweight, and so their use should generally be avoided - // within each benchmark iteration, if possible. - void PauseTiming(); - - // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called - // by the current thread. - // Start the benchmark timer. The timer is NOT running on entrance to the - // benchmark function. It begins running after control flow enters the - // benchmark loop. - // - // NOTE: PauseTiming()/ResumeTiming() are relatively - // heavyweight, and so their use should generally be avoided - // within each benchmark iteration, if possible. - void ResumeTiming(); - - // REQUIRES: 'SkipWithError(...)' has not been called previously by the - // current thread. - // Report the benchmark as resulting in an error with the specified 'msg'. - // After this call the user may explicitly 'return' from the benchmark. - // - // If the ranged-for style of benchmark loop is used, the user must explicitly - // break from the loop, otherwise all future iterations will be run. - // If the 'KeepRunning()' loop is used the current thread will automatically - // exit the loop at the end of the current iteration. - // - // For threaded benchmarks only the current thread stops executing and future - // calls to `KeepRunning()` will block until all threads have completed - // the `KeepRunning()` loop. If multiple threads report an error only the - // first error message is used. - // - // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit - // the current scope immediately. If the function is called from within - // the 'KeepRunning()' loop the current iteration will finish. It is the users - // responsibility to exit the scope as needed. - void SkipWithError(const char* msg); - - // Returns true if an error has been reported with 'SkipWithError(...)'. - bool error_occurred() const { return error_occurred_; } - - // REQUIRES: called exactly once per iteration of the benchmarking loop. - // Set the manually measured time for this benchmark iteration, which - // is used instead of automatically measured time if UseManualTime() was - // specified. - // - // For threaded benchmarks the final value will be set to the largest - // reported values. - void SetIterationTime(double seconds); - - // Set the number of bytes processed by the current benchmark - // execution. This routine is typically called once at the end of a - // throughput oriented benchmark. - // - // REQUIRES: a benchmark has exited its benchmarking loop. - BENCHMARK_ALWAYS_INLINE - void SetBytesProcessed(int64_t bytes) { - counters["bytes_per_second"] = - Counter(static_cast(bytes), Counter::kIsRate, Counter::kIs1024); - } - - BENCHMARK_ALWAYS_INLINE - int64_t bytes_processed() const { - if (counters.find("bytes_per_second") != counters.end()) - return static_cast(counters.at("bytes_per_second")); - return 0; - } - - // If this routine is called with complexity_n > 0 and complexity report is - // requested for the - // family benchmark, then current benchmark will be part of the computation - // and complexity_n will - // represent the length of N. - BENCHMARK_ALWAYS_INLINE - void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } - - BENCHMARK_ALWAYS_INLINE - int64_t complexity_length_n() const { return complexity_n_; } - - // If this routine is called with items > 0, then an items/s - // label is printed on the benchmark report line for the currently - // executing benchmark. It is typically called at the end of a processing - // benchmark where a processing items/second output is desired. - // - // REQUIRES: a benchmark has exited its benchmarking loop. - BENCHMARK_ALWAYS_INLINE - void SetItemsProcessed(int64_t items) { - counters["items_per_second"] = - Counter(static_cast(items), benchmark::Counter::kIsRate); - } - - BENCHMARK_ALWAYS_INLINE - int64_t items_processed() const { - if (counters.find("items_per_second") != counters.end()) - return static_cast(counters.at("items_per_second")); - return 0; - } - - // If this routine is called, the specified label is printed at the - // end of the benchmark report line for the currently executing - // benchmark. Example: - // static void BM_Compress(benchmark::State& state) { - // ... - // double compress = input_size / output_size; - // state.SetLabel(StrFormat("compress:%.1f%%", 100.0*compression)); - // } - // Produces output that looks like: - // BM_Compress 50 50 14115038 compress:27.3% - // - // REQUIRES: a benchmark has exited its benchmarking loop. - void SetLabel(const char* label); - - void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { - this->SetLabel(str.c_str()); - } - - // Range arguments for this run. CHECKs if the argument has been set. - BENCHMARK_ALWAYS_INLINE - int64_t range(std::size_t pos = 0) const { - assert(range_.size() > pos); - return range_[pos]; - } - - BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") - int64_t range_x() const { return range(0); } - - BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") - int64_t range_y() const { return range(1); } - - // Number of threads concurrently executing the benchmark. - BENCHMARK_ALWAYS_INLINE - int threads() const { return threads_; } - - // Index of the executing thread. Values from [0, threads). - BENCHMARK_ALWAYS_INLINE - int thread_index() const { return thread_index_; } - - BENCHMARK_ALWAYS_INLINE - IterationCount iterations() const { - if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { - return 0; - } - return max_iterations - total_iterations_ + batch_leftover_; - } - - BENCHMARK_ALWAYS_INLINE - std::string name() const { return name_; } - - private: - // items we expect on the first cache line (ie 64 bytes of the struct) - // When total_iterations_ is 0, KeepRunning() and friends will return false. - // May be larger than max_iterations. - IterationCount total_iterations_; - - // When using KeepRunningBatch(), batch_leftover_ holds the number of - // iterations beyond max_iters that were run. Used to track - // completed_iterations_ accurately. - IterationCount batch_leftover_; - - public: - const IterationCount max_iterations; - - private: - bool started_; - bool finished_; - bool error_occurred_; - - // items we don't need on the first cache line - std::vector range_; - - int64_t complexity_n_; - - public: - // Container for user-defined counters. - UserCounters counters; - - private: - State(std::string name, IterationCount max_iters, - const std::vector& ranges, int thread_i, int n_threads, - internal::ThreadTimer* timer, internal::ThreadManager* manager, - internal::PerfCountersMeasurement* perf_counters_measurement); - - void StartKeepRunning(); - // Implementation of KeepRunning() and KeepRunningBatch(). - // is_batch must be true unless n is 1. - bool KeepRunningInternal(IterationCount n, bool is_batch); - void FinishKeepRunning(); - - const std::string name_; - const int thread_index_; - const int threads_; - - internal::ThreadTimer* const timer_; - internal::ThreadManager* const manager_; - internal::PerfCountersMeasurement* const perf_counters_measurement_; - - friend class internal::BenchmarkInstance; -}; - -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { - return KeepRunningInternal(1, /*is_batch=*/false); -} - -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) { - return KeepRunningInternal(n, /*is_batch=*/true); -} - -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, - bool is_batch) { - // total_iterations_ is set to 0 by the constructor, and always set to a - // nonzero value by StartKepRunning(). - assert(n > 0); - // n must be 1 unless is_batch is true. - assert(is_batch || n == 1); - if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ >= n, true)) { - total_iterations_ -= n; - return true; - } - if (!started_) { - StartKeepRunning(); - if (!error_occurred_ && total_iterations_ >= n) { - total_iterations_ -= n; - return true; - } - } - // For non-batch runs, total_iterations_ must be 0 by now. - if (is_batch && total_iterations_ != 0) { - batch_leftover_ = n - total_iterations_; - total_iterations_ = 0; - return true; - } - FinishKeepRunning(); - return false; -} - -struct State::StateIterator { - struct BENCHMARK_UNUSED Value {}; - typedef std::forward_iterator_tag iterator_category; - typedef Value value_type; - typedef Value reference; - typedef Value pointer; - typedef std::ptrdiff_t difference_type; - - private: - friend class State; - BENCHMARK_ALWAYS_INLINE - StateIterator() : cached_(0), parent_() {} - - BENCHMARK_ALWAYS_INLINE - explicit StateIterator(State* st) - : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {} - - public: - BENCHMARK_ALWAYS_INLINE - Value operator*() const { return Value(); } - - BENCHMARK_ALWAYS_INLINE - StateIterator& operator++() { - assert(cached_ > 0); - --cached_; - return *this; - } - - BENCHMARK_ALWAYS_INLINE - bool operator!=(StateIterator const&) const { - if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) return true; - parent_->FinishKeepRunning(); - return false; - } - - private: - IterationCount cached_; - State* const parent_; -}; - -inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() { - return StateIterator(this); -} -inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() { - StartKeepRunning(); - return StateIterator(); -} - -namespace internal { - -typedef void(Function)(State&); - -// ------------------------------------------------------ -// Benchmark registration object. The BENCHMARK() macro expands -// into an internal::Benchmark* object. Various methods can -// be called on this object to change the properties of the benchmark. -// Each method returns "this" so that multiple method calls can -// chained into one expression. -class BENCHMARK_EXPORT Benchmark { - public: - virtual ~Benchmark(); - - // Note: the following methods all return "this" so that multiple - // method calls can be chained together in one expression. - - // Specify the name of the benchmark - Benchmark* Name(const std::string& name); - - // Run this benchmark once with "x" as the extra argument passed - // to the function. - // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* Arg(int64_t x); - - // Run this benchmark with the given time unit for the generated output report - Benchmark* Unit(TimeUnit unit); - - // Run this benchmark once for a number of values picked from the - // range [start..limit]. (start and limit are always picked.) - // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* Range(int64_t start, int64_t limit); - - // Run this benchmark once for all values in the range [start..limit] with - // specific step - // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* DenseRange(int64_t start, int64_t limit, int step = 1); - - // Run this benchmark once with "args" as the extra arguments passed - // to the function. - // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* Args(const std::vector& args); - - // Equivalent to Args({x, y}) - // NOTE: This is a legacy C++03 interface provided for compatibility only. - // New code should use 'Args'. - Benchmark* ArgPair(int64_t x, int64_t y) { - std::vector args; - args.push_back(x); - args.push_back(y); - return Args(args); - } - - // Run this benchmark once for a number of values picked from the - // ranges [start..limit]. (starts and limits are always picked.) - // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* Ranges(const std::vector >& ranges); - - // Run this benchmark once for each combination of values in the (cartesian) - // product of the supplied argument lists. - // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* ArgsProduct(const std::vector >& arglists); - - // Equivalent to ArgNames({name}) - Benchmark* ArgName(const std::string& name); - - // Set the argument names to display in the benchmark name. If not called, - // only argument values will be shown. - Benchmark* ArgNames(const std::vector& names); - - // Equivalent to Ranges({{lo1, hi1}, {lo2, hi2}}). - // NOTE: This is a legacy C++03 interface provided for compatibility only. - // New code should use 'Ranges'. - Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) { - std::vector > ranges; - ranges.push_back(std::make_pair(lo1, hi1)); - ranges.push_back(std::make_pair(lo2, hi2)); - return Ranges(ranges); - } - - // Have "setup" and/or "teardown" invoked once for every benchmark run. - // If the benchmark is multi-threaded (will run in k threads concurrently), - // the setup callback will be be invoked exactly once (not k times) before - // each run with k threads. Time allowing (e.g. for a short benchmark), there - // may be multiple such runs per benchmark, each run with its own - // "setup"/"teardown". - // - // If the benchmark uses different size groups of threads (e.g. via - // ThreadRange), the above will be true for each size group. - // - // The callback will be passed a State object, which includes the number - // of threads, thread-index, benchmark arguments, etc. - // - // The callback must not be NULL or self-deleting. - Benchmark* Setup(void (*setup)(const benchmark::State&)); - Benchmark* Teardown(void (*teardown)(const benchmark::State&)); - - // Pass this benchmark object to *func, which can customize - // the benchmark by calling various methods like Arg, Args, - // Threads, etc. - Benchmark* Apply(void (*func)(Benchmark* benchmark)); - - // Set the range multiplier for non-dense range. If not called, the range - // multiplier kRangeMultiplier will be used. - Benchmark* RangeMultiplier(int multiplier); - - // Set the minimum amount of time to use when running this benchmark. This - // option overrides the `benchmark_min_time` flag. - // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. - Benchmark* MinTime(double t); - - // Set the minimum amount of time to run the benchmark before taking runtimes - // of this benchmark into account. This - // option overrides the `benchmark_min_warmup_time` flag. - // REQUIRES: `t >= 0` and `Iterations` has not been called on this benchmark. - Benchmark* MinWarmUpTime(double t); - - // Specify the amount of iterations that should be run by this benchmark. - // This option overrides the `benchmark_min_time` flag. - // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. - // - // NOTE: This function should only be used when *exact* iteration control is - // needed and never to control or limit how long a benchmark runs, where - // `--benchmark_min_time=s` or `MinTime(...)` should be used instead. - Benchmark* Iterations(IterationCount n); - - // Specify the amount of times to repeat this benchmark. This option overrides - // the `benchmark_repetitions` flag. - // REQUIRES: `n > 0` - Benchmark* Repetitions(int n); - - // Specify if each repetition of the benchmark should be reported separately - // or if only the final statistics should be reported. If the benchmark - // is not repeated then the single result is always reported. - // Applies to *ALL* reporters (display and file). - Benchmark* ReportAggregatesOnly(bool value = true); - - // Same as ReportAggregatesOnly(), but applies to display reporter only. - Benchmark* DisplayAggregatesOnly(bool value = true); - - // By default, the CPU time is measured only for the main thread, which may - // be unrepresentative if the benchmark uses threads internally. If called, - // the total CPU time spent by all the threads will be measured instead. - // By default, only the main thread CPU time will be measured. - Benchmark* MeasureProcessCPUTime(); - - // If a particular benchmark should use the Wall clock instead of the CPU time - // (be it either the CPU time of the main thread only (default), or the - // total CPU usage of the benchmark), call this method. If called, the elapsed - // (wall) time will be used to control how many iterations are run, and in the - // printing of items/second or MB/seconds values. - // If not called, the CPU time used by the benchmark will be used. - Benchmark* UseRealTime(); - - // If a benchmark must measure time manually (e.g. if GPU execution time is - // being - // measured), call this method. If called, each benchmark iteration should - // call - // SetIterationTime(seconds) to report the measured time, which will be used - // to control how many iterations are run, and in the printing of items/second - // or MB/second values. - Benchmark* UseManualTime(); - - // Set the asymptotic computational complexity for the benchmark. If called - // the asymptotic computational complexity will be shown on the output. - Benchmark* Complexity(BigO complexity = benchmark::oAuto); - - // Set the asymptotic computational complexity for the benchmark. If called - // the asymptotic computational complexity will be shown on the output. - Benchmark* Complexity(BigOFunc* complexity); - - // Add this statistics to be computed over all the values of benchmark run - Benchmark* ComputeStatistics(const std::string& name, - StatisticsFunc* statistics, - StatisticUnit unit = kTime); - - // Support for running multiple copies of the same benchmark concurrently - // in multiple threads. This may be useful when measuring the scaling - // of some piece of code. - - // Run one instance of this benchmark concurrently in t threads. - Benchmark* Threads(int t); - - // Pick a set of values T from [min_threads,max_threads]. - // min_threads and max_threads are always included in T. Run this - // benchmark once for each value in T. The benchmark run for a - // particular value t consists of t threads running the benchmark - // function concurrently. For example, consider: - // BENCHMARK(Foo)->ThreadRange(1,16); - // This will run the following benchmarks: - // Foo in 1 thread - // Foo in 2 threads - // Foo in 4 threads - // Foo in 8 threads - // Foo in 16 threads - Benchmark* ThreadRange(int min_threads, int max_threads); - - // For each value n in the range, run this benchmark once using n threads. - // min_threads and max_threads are always included in the range. - // stride specifies the increment. E.g. DenseThreadRange(1, 8, 3) starts - // a benchmark with 1, 4, 7 and 8 threads. - Benchmark* DenseThreadRange(int min_threads, int max_threads, int stride = 1); - - // Equivalent to ThreadRange(NumCPUs(), NumCPUs()) - Benchmark* ThreadPerCpu(); - - virtual void Run(State& state) = 0; - - TimeUnit GetTimeUnit() const; - - protected: - explicit Benchmark(const char* name); - void SetName(const char* name); - - int ArgsCnt() const; - - private: - friend class BenchmarkFamilies; - friend class BenchmarkInstance; - - std::string name_; - AggregationReportMode aggregation_report_mode_; - std::vector arg_names_; // Args for all benchmark runs - std::vector > args_; // Args for all benchmark runs - - TimeUnit time_unit_; - bool use_default_time_unit_; - - int range_multiplier_; - double min_time_; - double min_warmup_time_; - IterationCount iterations_; - int repetitions_; - bool measure_process_cpu_time_; - bool use_real_time_; - bool use_manual_time_; - BigO complexity_; - BigOFunc* complexity_lambda_; - std::vector statistics_; - std::vector thread_counts_; - - typedef void (*callback_function)(const benchmark::State&); - callback_function setup_; - callback_function teardown_; - - Benchmark(Benchmark const&) -#if defined(BENCHMARK_HAS_CXX11) - = delete -#endif - ; - - Benchmark& operator=(Benchmark const&) -#if defined(BENCHMARK_HAS_CXX11) - = delete -#endif - ; -}; - -} // namespace internal - -// Create and register a benchmark with the specified 'name' that invokes -// the specified functor 'fn'. -// -// RETURNS: A pointer to the registered benchmark. -internal::Benchmark* RegisterBenchmark(const char* name, - internal::Function* fn); - -#if defined(BENCHMARK_HAS_CXX11) -template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); -#endif - -// Remove all registered benchmarks. All pointers to previously registered -// benchmarks are invalidated. -BENCHMARK_EXPORT void ClearRegisteredBenchmarks(); - -namespace internal { -// The class used to hold all Benchmarks created from static function. -// (ie those created using the BENCHMARK(...) macros. -class BENCHMARK_EXPORT FunctionBenchmark : public Benchmark { - public: - FunctionBenchmark(const char* name, Function* func) - : Benchmark(name), func_(func) {} - - void Run(State& st) BENCHMARK_OVERRIDE; - - private: - Function* func_; -}; - -#ifdef BENCHMARK_HAS_CXX11 -template -class LambdaBenchmark : public Benchmark { - public: - void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } - - private: - template - LambdaBenchmark(const char* name, OLambda&& lam) - : Benchmark(name), lambda_(std::forward(lam)) {} - - LambdaBenchmark(LambdaBenchmark const&) = delete; - - template // NOLINTNEXTLINE(readability-redundant-declaration) - friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); - - Lambda lambda_; -}; -#endif -} // namespace internal - -inline internal::Benchmark* RegisterBenchmark(const char* name, - internal::Function* fn) { - return internal::RegisterBenchmarkInternal( - ::new internal::FunctionBenchmark(name, fn)); -} - -#ifdef BENCHMARK_HAS_CXX11 -template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { - using BenchType = - internal::LambdaBenchmark::type>; - return internal::RegisterBenchmarkInternal( - ::new BenchType(name, std::forward(fn))); -} -#endif - -#if defined(BENCHMARK_HAS_CXX11) && \ - (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) -template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, - Args&&... args) { - return benchmark::RegisterBenchmark( - name, [=](benchmark::State& st) { fn(st, args...); }); -} -#else -#define BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK -#endif - -// The base class for all fixture tests. -class Fixture : public internal::Benchmark { - public: - Fixture() : internal::Benchmark("") {} - - void Run(State& st) BENCHMARK_OVERRIDE { - this->SetUp(st); - this->BenchmarkCase(st); - this->TearDown(st); - } - - // These will be deprecated ... - virtual void SetUp(const State&) {} - virtual void TearDown(const State&) {} - // ... In favor of these. - virtual void SetUp(State& st) { SetUp(const_cast(st)); } - virtual void TearDown(State& st) { TearDown(const_cast(st)); } - - protected: - virtual void BenchmarkCase(State&) = 0; -}; -} // namespace benchmark - -// ------------------------------------------------------ -// Macro to register benchmarks - -// Check that __COUNTER__ is defined and that __COUNTER__ increases by 1 -// every time it is expanded. X + 1 == X + 0 is used in case X is defined to be -// empty. If X is empty the expression becomes (+1 == +0). -#if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0) -#define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__ -#else -#define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__ -#endif - -// Helpers for generating unique variable names -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_PRIVATE_NAME(...) \ - BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, \ - __VA_ARGS__) -#else -#define BENCHMARK_PRIVATE_NAME(n) \ - BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, n) -#endif // BENCHMARK_HAS_CXX11 - -#define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) -#define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c -// Helper for concatenation with macro name expansion -#define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \ - BaseClass##_##Method##_Benchmark - -#define BENCHMARK_PRIVATE_DECLARE(n) \ - static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \ - BENCHMARK_UNUSED - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK(...) \ - BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#__VA_ARGS__, \ - &__VA_ARGS__))) -#else -#define BENCHMARK(n) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#n, n))) -#endif // BENCHMARK_HAS_CXX11 - -// Old-style macros -#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) -#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) -#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) -#define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) -#define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ - BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) - -#ifdef BENCHMARK_HAS_CXX11 - -// Register a benchmark which invokes the function specified by `func` -// with the additional arguments specified by `...`. -// -// For example: -// -// template ` -// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { -// [...] -//} -// /* Registers a benchmark named "BM_takes_args/int_string_test` */ -// BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); -#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(func) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark( \ - #func "/" #test_case_name, \ - [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) - -#endif // BENCHMARK_HAS_CXX11 - -// This will register a benchmark for a templatized function. For example: -// -// template -// void BM_Foo(int iters); -// -// BENCHMARK_TEMPLATE(BM_Foo, 1); -// -// will register BM_Foo<1> as a benchmark. -#define BENCHMARK_TEMPLATE1(n, a) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#n "<" #a ">", n))) - -#define BENCHMARK_TEMPLATE2(n, a, b) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark(#n "<" #a "," #b ">", \ - n))) - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE(n, ...) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark( \ - #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) -#else -#define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) -#endif - -#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "/" #Method); \ - } \ - \ - protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; - -#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a ">/" #Method); \ - } \ - \ - protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; - -#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ - } \ - \ - protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \ - class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); \ - } \ - \ - protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ - }; -#else -#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ - BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(n, a) -#endif - -#define BENCHMARK_DEFINE_F(BaseClass, Method) \ - BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE2_DEFINE_F(BaseClass, Method, a, b) \ - BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, ...) \ - BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase -#else -#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) -#endif - -#define BENCHMARK_REGISTER_F(BaseClass, Method) \ - BENCHMARK_PRIVATE_REGISTER_F(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)) - -#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \ - BENCHMARK_PRIVATE_DECLARE(TestName) = \ - (::benchmark::internal::RegisterBenchmarkInternal(new TestName())) - -// This macro will define and register a benchmark within a fixture class. -#define BENCHMARK_F(BaseClass, Method) \ - BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#define BENCHMARK_TEMPLATE2_F(BaseClass, Method, a, b) \ - BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase - -#ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ - BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase -#else -#define BENCHMARK_TEMPLATE_F(BaseClass, Method, a) \ - BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) -#endif - -// Helper macro to create a main routine in a test that runs the benchmarks -// Note the workaround for Hexagon simulator passing argc != 0, argv = NULL. -#define BENCHMARK_MAIN() \ - int main(int argc, char** argv) { \ - char arg0_default[] = "benchmark"; \ - char* args_default = arg0_default; \ - if (!argv) { \ - argc = 1; \ - argv = &args_default; \ - } \ - ::benchmark::Initialize(&argc, argv); \ - if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ - ::benchmark::RunSpecifiedBenchmarks(); \ - ::benchmark::Shutdown(); \ - return 0; \ - } \ - int main(int, char**) - -// ------------------------------------------------------ -// Benchmark Reporters - -namespace benchmark { - -struct BENCHMARK_EXPORT CPUInfo { - struct CacheInfo { - std::string type; - int level; - int size; - int num_sharing; - }; - - enum Scaling { UNKNOWN, ENABLED, DISABLED }; - - int num_cpus; - Scaling scaling; - double cycles_per_second; - std::vector caches; - std::vector load_avg; - - static const CPUInfo& Get(); - - private: - CPUInfo(); - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); -}; - -// Adding Struct for System Information -struct BENCHMARK_EXPORT SystemInfo { - std::string name; - static const SystemInfo& Get(); - - private: - SystemInfo(); - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); -}; - -// BenchmarkName contains the components of the Benchmark's name -// which allows individual fields to be modified or cleared before -// building the final name using 'str()'. -struct BENCHMARK_EXPORT BenchmarkName { - std::string function_name; - std::string args; - std::string min_time; - std::string min_warmup_time; - std::string iterations; - std::string repetitions; - std::string time_type; - std::string threads; - - // Return the full name of the benchmark with each non-empty - // field separated by a '/' - std::string str() const; -}; - -// Interface for custom benchmark result printers. -// By default, benchmark reports are printed to stdout. However an application -// can control the destination of the reports by calling -// RunSpecifiedBenchmarks and passing it a custom reporter object. -// The reporter object must implement the following interface. -class BENCHMARK_EXPORT BenchmarkReporter { - public: - struct Context { - CPUInfo const& cpu_info; - SystemInfo const& sys_info; - // The number of chars in the longest benchmark name. - size_t name_field_width; - static const char* executable_name; - Context(); - }; - - struct BENCHMARK_EXPORT Run { - static const int64_t no_repetition_index = -1; - enum RunType { RT_Iteration, RT_Aggregate }; - - Run() - : run_type(RT_Iteration), - aggregate_unit(kTime), - error_occurred(false), - iterations(1), - threads(1), - time_unit(GetDefaultTimeUnit()), - real_accumulated_time(0), - cpu_accumulated_time(0), - max_heapbytes_used(0), - complexity(oNone), - complexity_lambda(), - complexity_n(0), - report_big_o(false), - report_rms(false), - memory_result(NULL), - allocs_per_iter(0.0) {} - - std::string benchmark_name() const; - BenchmarkName run_name; - int64_t family_index; - int64_t per_family_instance_index; - RunType run_type; - std::string aggregate_name; - StatisticUnit aggregate_unit; - std::string report_label; // Empty if not set by benchmark. - bool error_occurred; - std::string error_message; - - IterationCount iterations; - int64_t threads; - int64_t repetition_index; - int64_t repetitions; - TimeUnit time_unit; - double real_accumulated_time; - double cpu_accumulated_time; - - // Return a value representing the real time per iteration in the unit - // specified by 'time_unit'. - // NOTE: If 'iterations' is zero the returned value represents the - // accumulated time. - double GetAdjustedRealTime() const; - - // Return a value representing the cpu time per iteration in the unit - // specified by 'time_unit'. - // NOTE: If 'iterations' is zero the returned value represents the - // accumulated time. - double GetAdjustedCPUTime() const; - - // This is set to 0.0 if memory tracing is not enabled. - double max_heapbytes_used; - - // Keep track of arguments to compute asymptotic complexity - BigO complexity; - BigOFunc* complexity_lambda; - int64_t complexity_n; - - // what statistics to compute from the measurements - const std::vector* statistics; - - // Inform print function whether the current run is a complexity report - bool report_big_o; - bool report_rms; - - UserCounters counters; - - // Memory metrics. - const MemoryManager::Result* memory_result; - double allocs_per_iter; - }; - - struct PerFamilyRunReports { - PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {} - - // How many runs will all instances of this benchmark perform? - int num_runs_total; - - // How many runs have happened already? - int num_runs_done; - - // The reports about (non-errneous!) runs of this family. - std::vector Runs; - }; - - // Construct a BenchmarkReporter with the output stream set to 'std::cout' - // and the error stream set to 'std::cerr' - BenchmarkReporter(); - - // Called once for every suite of benchmarks run. - // The parameter "context" contains information that the - // reporter may wish to use when generating its report, for example the - // platform under which the benchmarks are running. The benchmark run is - // never started if this function returns false, allowing the reporter - // to skip runs based on the context information. - virtual bool ReportContext(const Context& context) = 0; - - // Called once for each group of benchmark runs, gives information about - // cpu-time and heap memory usage during the benchmark run. If the group - // of runs contained more than two entries then 'report' contains additional - // elements representing the mean and standard deviation of those runs. - // Additionally if this group of runs was the last in a family of benchmarks - // 'reports' contains additional entries representing the asymptotic - // complexity and RMS of that benchmark family. - virtual void ReportRuns(const std::vector& report) = 0; - - // Called once and only once after ever group of benchmarks is run and - // reported. - virtual void Finalize() {} - - // REQUIRES: The object referenced by 'out' is valid for the lifetime - // of the reporter. - void SetOutputStream(std::ostream* out) { - assert(out); - output_stream_ = out; - } - - // REQUIRES: The object referenced by 'err' is valid for the lifetime - // of the reporter. - void SetErrorStream(std::ostream* err) { - assert(err); - error_stream_ = err; - } - - std::ostream& GetOutputStream() const { return *output_stream_; } - - std::ostream& GetErrorStream() const { return *error_stream_; } - - virtual ~BenchmarkReporter(); - - // Write a human readable string to 'out' representing the specified - // 'context'. - // REQUIRES: 'out' is non-null. - static void PrintBasicContext(std::ostream* out, Context const& context); - - private: - std::ostream* output_stream_; - std::ostream* error_stream_; -}; - -// Simple reporter that outputs benchmark data to the console. This is the -// default reporter used by RunSpecifiedBenchmarks(). -class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { - public: - enum OutputOptions { - OO_None = 0, - OO_Color = 1, - OO_Tabular = 2, - OO_ColorTabular = OO_Color | OO_Tabular, - OO_Defaults = OO_ColorTabular - }; - explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) - : output_options_(opts_), name_field_width_(0), printed_header_(false) {} - - bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - - protected: - virtual void PrintRunData(const Run& report); - virtual void PrintHeader(const Run& report); - - OutputOptions output_options_; - size_t name_field_width_; - UserCounters prev_counters_; - bool printed_header_; -}; - -class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { - public: - JSONReporter() : first_report_(true) {} - bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - void Finalize() BENCHMARK_OVERRIDE; - - private: - void PrintRunData(const Run& report); - - bool first_report_; -}; - -class BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG( - "The CSV Reporter will be removed in a future release") CSVReporter - : public BenchmarkReporter { - public: - CSVReporter() : printed_header_(false) {} - bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - - private: - void PrintRunData(const Run& report); - - bool printed_header_; - std::set user_counter_names_; -}; - -inline const char* GetTimeUnitString(TimeUnit unit) { - switch (unit) { - case kSecond: - return "s"; - case kMillisecond: - return "ms"; - case kMicrosecond: - return "us"; - case kNanosecond: - return "ns"; - } - BENCHMARK_UNREACHABLE(); -} - -inline double GetTimeUnitMultiplier(TimeUnit unit) { - switch (unit) { - case kSecond: - return 1; - case kMillisecond: - return 1e3; - case kMicrosecond: - return 1e6; - case kNanosecond: - return 1e9; - } - BENCHMARK_UNREACHABLE(); -} - -// Creates a list of integer values for the given range and multiplier. -// This can be used together with ArgsProduct() to allow multiple ranges -// with different multipliers. -// Example: -// ArgsProduct({ -// CreateRange(0, 1024, /*multi=*/32), -// CreateRange(0, 100, /*multi=*/4), -// CreateDenseRange(0, 4, /*step=*/1), -// }); -BENCHMARK_EXPORT -std::vector CreateRange(int64_t lo, int64_t hi, int multi); - -// Creates a list of integer values for the given range and step. -BENCHMARK_EXPORT -std::vector CreateDenseRange(int64_t start, int64_t limit, int step); - -} // namespace benchmark - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif - -#endif // BENCHMARK_BENCHMARK_H_ From 46b2e7bdaafcba376b4ddc197a9635794ae373ba Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Fri, 13 Jan 2023 10:59:57 -0500 Subject: [PATCH 23/34] addressed review comments --- include/benchmark/benchmark.h | 4 +++ src/benchmark.cc | 11 +++--- src/benchmark_runner.cc | 44 +++++------------------ src/benchmark_runner.h | 5 ++- test/benchmark_min_time_flag_time_test.cc | 26 ++++++++++---- 5 files changed, 43 insertions(+), 47 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 359a9c1ba9..cf8515b88a 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1724,6 +1724,10 @@ class BENCHMARK_EXPORT BenchmarkReporter { // to skip runs based on the context information. virtual bool ReportContext(const Context& context) = 0; + // Called once for each group of benchmark runs, gives information about + // the configurations of the runs. + virtual void ReportRunsConfig(double /*min_time*/, bool /*has_explicit_iters*/, IterationCount /*iters*/) {} + // Called once for each group of benchmark runs, gives information about // cpu-time and heap memory usage during the benchmark run. If the group // of runs contained more than two entries then 'report' contains additional diff --git a/src/benchmark.cc b/src/benchmark.cc index df0e68f9e3..1baaa29592 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -386,6 +386,13 @@ void RunBenchmarks(const std::vector& benchmarks, if (runner.HasRepeatsRemaining()) continue; // FIXME: report each repetition separately, not all of them in bulk. + display_reporter->ReportRunsConfig(runner.GetMinTime(), + runner.HasExplicitIters(), + runner.GetIters()); + if (file_reporter) file_reporter->ReportRunsConfig(runner.GetMinTime(), + runner.HasExplicitIters(), + runner.GetIters()); + RunResults run_results = runner.GetResults(); // Maybe calculate complexity report @@ -401,10 +408,6 @@ void RunBenchmarks(const std::vector& benchmarks, } } - // Final step: fixup the configs that were overrriden with - // non-default values. - runner.UpdateReport(run_results); - Report(display_reporter, file_reporter, run_results); } } diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 7bb51913c9..7e89aa3421 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -180,7 +180,7 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { const char* time_str = value.c_str(); bool has_suffix = value.back() == 's'; if (!has_suffix) { - BM_VLOG(2) << "Value passed to --benchmark_min_time should have a suffix. " + BM_VLOG(0) << "Value passed to --benchmark_min_time should have a suffix. " "Eg., `30s` for 30-seconds."; } @@ -207,21 +207,21 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { return ret; } -double GetMinTime(const benchmark::internal::BenchmarkInstance& b, +double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, const BenchTimeType& iters_or_time) { if (!IsZero(b.min_time())) return b.min_time(); - // If the flag was used to specify number of iters, then return 0 for time. - if (iters_or_time.tag == BenchTimeType::ITERS) return 0; + // If the flag was used to specify number of iters, then return the default min_time. + if (iters_or_time.tag == BenchTimeType::ITERS) return kDefaultMinTime; return iters_or_time.time; } -IterationCount GetIters(const benchmark::internal::BenchmarkInstance& b, +IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, const BenchTimeType& iters_or_time) { if (b.iterations() != 0) return b.iterations(); - // We've already checked that this flag is currently used to pass - // iters but do a check anyway. + // We've already concluded that this flag is currently used to pass + // iters but do a check here again anyway. BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); return iters_or_time.iters; } @@ -234,7 +234,7 @@ BenchmarkRunner::BenchmarkRunner( : b(b_), reports_for_family(reports_for_family_), parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), - min_time(GetMinTime(b_, parsed_benchtime_flag)), + min_time(ComputeMinTime(b_, parsed_benchtime_flag)), min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) ? b.min_warmup_time() : FLAGS_benchmark_min_warmup_time), @@ -245,7 +245,7 @@ BenchmarkRunner::BenchmarkRunner( parsed_benchtime_flag.tag == BenchTimeType::ITERS), pool(b.threads() - 1), - iters(has_explicit_iteration_count ? GetIters(b_, parsed_benchtime_flag) + iters(has_explicit_iteration_count ? ComputeIters(b_, parsed_benchtime_flag) : 1), perf_counters_measurement(StrSplit(FLAGS_benchmark_perf_counters, ',')), perf_counters_measurement_ptr(perf_counters_measurement.IsValid() @@ -268,32 +268,6 @@ BenchmarkRunner::BenchmarkRunner( } } -static bool AlmostEqual(double a, double b) { - return std::fabs(a - b) < std::numeric_limits::epsilon(); -} - -void BenchmarkRunner::UpdateReport(RunResults& current_run_results) { - // If we the actual min_time is different from the value in the benchmark - // instance AND if it's not the default value then update it. - bool update_time = !AlmostEqual(min_time, b.min_time()) && - !AlmostEqual(min_time, kDefaultMinTime); - bool update_iters = has_explicit_iteration_count && iters != b.iterations(); - - if (!update_time && !update_iters) return; - - auto UpdateRun = [](bool update_t, double new_min_time, bool update_i, - IterationCount new_iters, BenchmarkReporter::Run& run) { - if (update_t) - run.run_name.min_time = StrFormat("min_time:%0.3fs", new_min_time); - if (update_i) run.iterations = new_iters; - }; - - for (auto& run : current_run_results.non_aggregates) - UpdateRun(update_time, min_time, update_iters, iters, run); - for (auto& run : current_run_results.aggregates_only) - UpdateRun(update_time, min_time, update_iters, iters, run); -} - BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { BM_VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 1039afccdb..539599e984 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -71,8 +71,11 @@ class BenchmarkRunner { return reports_for_family; } - void UpdateReport(RunResults& run_results); + double GetMinTime() const {return min_time;} + bool HasExplicitIters() const {return has_explicit_iteration_count;} + + IterationCount GetIters() const {return iters;} private: RunResults run_results; diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index dccc193790..ae61bbb91f 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -12,6 +14,9 @@ // --benchmark_min_time=s namespace { +// This is from benchmark.h +typedef int64_t IterationCount; + class TestReporter : public benchmark::ConsoleReporter { public: virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { @@ -20,21 +25,28 @@ class TestReporter : public benchmark::ConsoleReporter { virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { assert(report.size() == 1); - min_times_.push_back(report[0].run_name.min_time); ConsoleReporter::ReportRuns(report); }; + virtual void ReportRunsConfig(double min_time, bool has_explicit_iters, IterationCount iters) BENCHMARK_OVERRIDE { + min_times_.push_back(min_time); + } + TestReporter() {} virtual ~TestReporter() {} - const std::vector& GetMinTimes() const { return min_times_; } + const std::vector& GetMinTimes() const { return min_times_; } private: - std::vector min_times_; + std::vector min_times_; }; -void DoTestHelper(int* argc, const char** argv, const std::string& expected) { +bool AlmostEqual(double a, double b) { + return std::fabs(a - b) < std::numeric_limits::epsilon(); +} + +void DoTestHelper(int* argc, const char** argv, double expected) { benchmark::Initialize(argc, const_cast(argv)); TestReporter test_reporter; @@ -43,8 +55,8 @@ void DoTestHelper(int* argc, const char** argv, const std::string& expected) { assert(returned_count == 1); // Check the min_time - const std::vector& min_times = test_reporter.GetMinTimes(); - assert(!min_times.empty() && min_times[0] == expected); + const std::vector& min_times = test_reporter.GetMinTimes(); + assert(!min_times.empty() && AlmostEqual(min_times[0], expected)); } } // end namespace @@ -64,7 +76,7 @@ int main(int argc, char** argv) { const char* no_suffix = "--benchmark_min_time=4"; const char* with_suffix = "--benchmark_min_time=4.0s"; - std::string expected = "min_time:4.000s"; + double expected = 4.0; fake_argv[argc] = no_suffix; DoTestHelper(&fake_argc, fake_argv, expected); From 7886a3cbcdd97dfc40ec9c7d35194093b34652a8 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Fri, 13 Jan 2023 11:04:40 -0500 Subject: [PATCH 24/34] fix shorten in warning --- include/benchmark/benchmark.h | 4 +++- src/benchmark.cc | 11 +++++------ src/benchmark_runner.cc | 12 +++++++----- src/benchmark_runner.h | 9 +++++---- test/benchmark_min_time_flag_time_test.cc | 5 +++-- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index cf8515b88a..8a134c9c15 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1726,7 +1726,9 @@ class BENCHMARK_EXPORT BenchmarkReporter { // Called once for each group of benchmark runs, gives information about // the configurations of the runs. - virtual void ReportRunsConfig(double /*min_time*/, bool /*has_explicit_iters*/, IterationCount /*iters*/) {} + virtual void ReportRunsConfig(double /*min_time*/, + bool /*has_explicit_iters*/, + IterationCount /*iters*/) {} // Called once for each group of benchmark runs, gives information about // cpu-time and heap memory usage during the benchmark run. If the group diff --git a/src/benchmark.cc b/src/benchmark.cc index 1baaa29592..86537a6f88 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -386,12 +386,11 @@ void RunBenchmarks(const std::vector& benchmarks, if (runner.HasRepeatsRemaining()) continue; // FIXME: report each repetition separately, not all of them in bulk. - display_reporter->ReportRunsConfig(runner.GetMinTime(), - runner.HasExplicitIters(), - runner.GetIters()); - if (file_reporter) file_reporter->ReportRunsConfig(runner.GetMinTime(), - runner.HasExplicitIters(), - runner.GetIters()); + display_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); + if (file_reporter) + file_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); RunResults run_results = runner.GetResults(); diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 7e89aa3421..2f1974348a 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -208,16 +208,17 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { } double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, - const BenchTimeType& iters_or_time) { + const BenchTimeType& iters_or_time) { if (!IsZero(b.min_time())) return b.min_time(); - // If the flag was used to specify number of iters, then return the default min_time. + // If the flag was used to specify number of iters, then return the default + // min_time. if (iters_or_time.tag == BenchTimeType::ITERS) return kDefaultMinTime; return iters_or_time.time; } IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, - const BenchTimeType& iters_or_time) { + const BenchTimeType& iters_or_time) { if (b.iterations() != 0) return b.iterations(); // We've already concluded that this flag is currently used to pass @@ -245,8 +246,9 @@ BenchmarkRunner::BenchmarkRunner( parsed_benchtime_flag.tag == BenchTimeType::ITERS), pool(b.threads() - 1), - iters(has_explicit_iteration_count ? ComputeIters(b_, parsed_benchtime_flag) - : 1), + iters(has_explicit_iteration_count + ? ComputeIters(b_, parsed_benchtime_flag) + : 1), perf_counters_measurement(StrSplit(FLAGS_benchmark_perf_counters, ',')), perf_counters_measurement_ptr(perf_counters_measurement.IsValid() ? &perf_counters_measurement diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 539599e984..ebb30e6c3a 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -47,7 +47,7 @@ struct RunResults { struct BenchTimeType { enum { ITERS, TIME } tag; union { - int iters; + IterationCount iters; double time; }; }; @@ -71,11 +71,12 @@ class BenchmarkRunner { return reports_for_family; } - double GetMinTime() const {return min_time;} + double GetMinTime() const { return min_time; } + + bool HasExplicitIters() const { return has_explicit_iteration_count; } - bool HasExplicitIters() const {return has_explicit_iteration_count;} + IterationCount GetIters() const { return iters; } - IterationCount GetIters() const {return iters;} private: RunResults run_results; diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index ae61bbb91f..bfbb4fbead 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -28,7 +28,8 @@ class TestReporter : public benchmark::ConsoleReporter { ConsoleReporter::ReportRuns(report); }; - virtual void ReportRunsConfig(double min_time, bool has_explicit_iters, IterationCount iters) BENCHMARK_OVERRIDE { + virtual void ReportRunsConfig(double min_time, bool has_explicit_iters, + IterationCount iters) BENCHMARK_OVERRIDE { min_times_.push_back(min_time); } @@ -43,7 +44,7 @@ class TestReporter : public benchmark::ConsoleReporter { }; bool AlmostEqual(double a, double b) { - return std::fabs(a - b) < std::numeric_limits::epsilon(); + return std::fabs(a - b) < std::numeric_limits::epsilon(); } void DoTestHelper(int* argc, const char** argv, double expected) { From 9d17cb33ff169d3a6242bbde0b81f48113884a7c Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Fri, 13 Jan 2023 11:27:52 -0500 Subject: [PATCH 25/34] use suffix for existing min_time specs to silent warnings in tests --- test/BUILD | 2 +- test/CMakeLists.txt | 54 ++++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/test/BUILD b/test/BUILD index a29abe5ae1..8262d080fa 100644 --- a/test/BUILD +++ b/test/BUILD @@ -25,7 +25,7 @@ PER_SRC_COPTS = { "donotoptimize_test.cc": ["-O3"], } -TEST_ARGS = ["--benchmark_min_time=0.01"] +TEST_ARGS = ["--benchmark_min_time=0.01s"] PER_SRC_TEST_ARGS = { "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ca21b9b277..10f1f7a46d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -61,7 +61,7 @@ endmacro(compile_output_test) # Demonstration executable compile_benchmark_test(benchmark_test) -add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) +add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(spec_arg_test) add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) @@ -74,7 +74,7 @@ add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) - add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) + add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01s --benchmark_filter=${filter} ${expect}) add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) @@ -104,19 +104,19 @@ add_filter_test(filter_regex_end ".*Ba$" 1) add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) -add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) +add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01s) compile_benchmark_test(basic_test) -add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) +add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01s) compile_output_test(repetitions_test) -add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01 --benchmark_repetitions=3) +add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01s --benchmark_repetitions=3) compile_benchmark_test(diagnostics_test) -add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) +add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01s) compile_benchmark_test(skip_with_error_test) -add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01) +add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01s) compile_benchmark_test(donotoptimize_test) # Some of the issues with DoNotOptimize only occur when optimization is enabled @@ -124,55 +124,55 @@ check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) if (BENCHMARK_HAS_O3_FLAG) set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") endif() -add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01) +add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01s) compile_benchmark_test(fixture_test) -add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01) +add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01s) compile_benchmark_test(register_benchmark_test) -add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01) +add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(map_test) -add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) +add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01s) compile_benchmark_test(multiple_ranges_test) -add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) +add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01s) compile_benchmark_test(args_product_test) -add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01) +add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01s) compile_benchmark_test_with_main(link_main_test) -add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) +add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01s) compile_output_test(reporter_output_test) -add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01) +add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01s) compile_output_test(templated_fixture_test) -add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01) +add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01s) compile_output_test(user_counters_test) -add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) +add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01s) compile_output_test(perf_counters_test) -add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01 --benchmark_perf_counters=CYCLES,BRANCHES) +add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,BRANCHES) compile_output_test(internal_threading_test) -add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) +add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01s) compile_output_test(report_aggregates_only_test) -add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01) +add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(display_aggregates_only_test) -add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01) +add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(user_counters_tabular_test) -add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01s) compile_output_test(user_counters_thousands_test) -add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01) +add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01s) compile_output_test(memory_manager_test) -add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01) +add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s) # MSVC does not allow to set the language standard to C++98/03. if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") @@ -197,14 +197,14 @@ if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-lto-type-mismatch") endif() set_target_properties(cxx03_test PROPERTIES LINK_FLAGS "${DISABLE_LTO_WARNINGS}") - add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01) + add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01s) endif() # Attempt to work around flaky test failures when running on Appveyor servers. if (DEFINED ENV{APPVEYOR}) - set(COMPLEXITY_MIN_TIME "0.5") + set(COMPLEXITY_MIN_TIME "0.5s") else() - set(COMPLEXITY_MIN_TIME "0.01") + set(COMPLEXITY_MIN_TIME "0.01s") endif() compile_output_test(complexity_test) add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) From 35481c5ec422f52bc5315ddec06a9dbf7325d46f Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Fri, 13 Jan 2023 12:46:24 -0500 Subject: [PATCH 26/34] fix leaks --- src/benchmark_runner.cc | 1 - test/benchmark_min_time_flag_iters_test.cc | 2 ++ test/benchmark_min_time_flag_time_test.cc | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 2f1974348a..f588f4dabc 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -41,7 +41,6 @@ #include #include - #include "check.h" #include "colorprint.h" #include "commandlineflags.h" diff --git a/test/benchmark_min_time_flag_iters_test.cc b/test/benchmark_min_time_flag_iters_test.cc index 9a36ece517..4ed768c736 100644 --- a/test/benchmark_min_time_flag_iters_test.cc +++ b/test/benchmark_min_time_flag_iters_test.cc @@ -58,5 +58,7 @@ int main(int argc, char** argv) { // Check the executed iters. const std::vector iters = test_reporter.GetIters(); assert(!iters.empty() && iters[0] == 4); + + delete[] fake_argv; return 0; } diff --git a/test/benchmark_min_time_flag_time_test.cc b/test/benchmark_min_time_flag_time_test.cc index bfbb4fbead..b602031a8d 100644 --- a/test/benchmark_min_time_flag_time_test.cc +++ b/test/benchmark_min_time_flag_time_test.cc @@ -85,5 +85,6 @@ int main(int argc, char** argv) { fake_argv[argc] = with_suffix; DoTestHelper(&fake_argc, fake_argv, expected); + delete[] fake_argv; return 0; } From 2664a17e55bf5dd8cbaf6a00a3ab4818145cd212 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Tue, 17 Jan 2023 10:47:58 -0500 Subject: [PATCH 27/34] use default min-time value in flag decl for consistency --- include/benchmark/benchmark.h | 2 ++ src/benchmark.cc | 2 +- src/benchmark_runner.cc | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 8a134c9c15..228eba817e 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -280,7 +280,9 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); namespace benchmark { class BenchmarkReporter; +// Default number of minimum benchmark running time in seconds. const double kDefaultMinTime = 0.5; +const char kDefaultMinTimeStr[] = "0.5s"; BENCHMARK_EXPORT void PrintDefaultHelp(); diff --git a/src/benchmark.cc b/src/benchmark.cc index 86537a6f88..c2d157c87b 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -79,7 +79,7 @@ BM_DEFINE_string(benchmark_filter, ""); // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -BM_DEFINE_string(benchmark_min_time, "0.5s"); +BM_DEFINE_string(benchmark_min_time, kDefaultMinTimeStr); // Minimum number of seconds a benchmark should be run before results should be // taken into account. This e.g can be necessary for benchmarks of code which diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index f588f4dabc..8bbe6141f2 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -65,7 +65,6 @@ MemoryManager* memory_manager = nullptr; namespace { static constexpr IterationCount kMaxIterations = 1000000000; -static constexpr double kDefaultMinTime = 0.5; BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, From e96b8c6a2ebd2bb6d8e52613c0fceaf4a14dc88e Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Tue, 17 Jan 2023 11:01:15 -0500 Subject: [PATCH 28/34] removed double kMinTimeDecl from benchmark.h --- include/benchmark/benchmark.h | 1 - src/benchmark_runner.cc | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 228eba817e..6e603fad86 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -281,7 +281,6 @@ namespace benchmark { class BenchmarkReporter; // Default number of minimum benchmark running time in seconds. -const double kDefaultMinTime = 0.5; const char kDefaultMinTimeStr[] = "0.5s"; BENCHMARK_EXPORT void PrintDefaultHelp(); diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 8bbe6141f2..416f4a9daf 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -65,6 +65,8 @@ MemoryManager* memory_manager = nullptr; namespace { static constexpr IterationCount kMaxIterations = 1000000000; +const double kDefaultMinTime = + std::strtod(::benchmark::kDefaultMinTimeStr, /*p_end*/ nullptr); BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, From f0c1285c513840397fe78cbfd0f2b34595486fb2 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Thu, 19 Jan 2023 13:54:28 -0500 Subject: [PATCH 29/34] dont need to preserve errno --- src/benchmark_runner.cc | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 416f4a9daf..e4fb71c909 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -157,18 +157,13 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { if (value.back() == 'x') { const char* iters_str = value.c_str(); char* p_end; - // Reset error since atol will change it. - const int old_errno = errno; + // Reset errno before it's changed by strtol. errno = 0; IterationCount num_iters = std::strtol(iters_str, &p_end, 10); - const int new_errno = errno; - errno = old_errno; - - (void)new_errno; // After a valid parse, p_end should have been set to // point to the 'x' suffix. - BM_CHECK(new_errno == 0 && p_end != nullptr && *p_end == 'x') + BM_CHECK(errno == 0 && p_end != nullptr && *p_end == 'x') << "Malformed iters value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; @@ -185,18 +180,13 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { } char* p_end; - // Reset error before it's changed by strtod. - const int old_errno = errno; + // Reset errno before it's changed by strtod. errno = 0; double min_time = std::strtod(time_str, &p_end); - const int new_errno = errno; - errno = old_errno; - - (void)new_errno; // After a successfull parse, p_end should point to the suffix 's' // or the end of the string, if the suffix was omitted. - BM_CHECK(new_errno == 0 && p_end != nullptr && + BM_CHECK(errno == 0 && p_end != nullptr && (has_suffix && *p_end == 's' || *p_end == '\0')) << "Malformed seconds value passed to --benchmark_min_time: `" << value << "`. Expected --benchmark_min_time=x."; From 4a22d680cb22c4d47151916ab8f848a9bfebe16b Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Tue, 24 Jan 2023 12:57:17 -0500 Subject: [PATCH 30/34] add death tests --- src/benchmark_runner.cc | 44 ++++++++++++++++++------------------ src/benchmark_runner.h | 2 ++ test/CMakeLists.txt | 1 + test/min_time_parse_gtest.cc | 20 ++++++++++++++++ 4 files changed, 45 insertions(+), 22 deletions(-) create mode 100644 test/min_time_parse_gtest.cc diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index e4fb71c909..eb0d9cbe79 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -145,6 +145,28 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } +double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (!IsZero(b.min_time())) return b.min_time(); + // If the flag was used to specify number of iters, then return the default + // min_time. + if (iters_or_time.tag == BenchTimeType::ITERS) return kDefaultMinTime; + + return iters_or_time.time; +} + +IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (b.iterations() != 0) return b.iterations(); + + // We've already concluded that this flag is currently used to pass + // iters but do a check here again anyway. + BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); + return iters_or_time.iters; +} + +} // end namespace + BenchTimeType ParseBenchMinTime(const std::string& value) { BenchTimeType ret; @@ -197,28 +219,6 @@ BenchTimeType ParseBenchMinTime(const std::string& value) { return ret; } -double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, - const BenchTimeType& iters_or_time) { - if (!IsZero(b.min_time())) return b.min_time(); - // If the flag was used to specify number of iters, then return the default - // min_time. - if (iters_or_time.tag == BenchTimeType::ITERS) return kDefaultMinTime; - - return iters_or_time.time; -} - -IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, - const BenchTimeType& iters_or_time) { - if (b.iterations() != 0) return b.iterations(); - - // We've already concluded that this flag is currently used to pass - // iters but do a check here again anyway. - BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); - return iters_or_time.iters; -} - -} // end namespace - BenchmarkRunner::BenchmarkRunner( const benchmark::internal::BenchmarkInstance& b_, BenchmarkReporter::PerFamilyRunReports* reports_for_family_) diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index ebb30e6c3a..1d3e6426c5 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -52,6 +52,8 @@ struct BenchTimeType { }; }; +BenchTimeType ParseBenchMinTime(const std::string& value); + class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 10f1f7a46d..cfef13bd77 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -233,6 +233,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(string_util_gtest) add_gtest(perf_counters_gtest) add_gtest(time_unit_gtest) + add_gtest(min_time_parse_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/test/min_time_parse_gtest.cc b/test/min_time_parse_gtest.cc new file mode 100644 index 0000000000..f8c37c61d6 --- /dev/null +++ b/test/min_time_parse_gtest.cc @@ -0,0 +1,20 @@ +#include "../src/benchmark_runner.h" +#include "gtest/gtest.h" + +namespace { + +TEST(ParseMinTimeTest, InvalidInput) { + ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("abc"); }, + "Malformed seconds value passed to --benchmark_min_time: `abc`"); + + ASSERT_DEATH( + { benchmark::internal::ParseBenchMinTime("123ms"); }, + "Malformed seconds value passed to --benchmark_min_time: `123ms`"); + + ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("1z"); }, + "Malformed seconds value passed to --benchmark_min_time: `1z`"); + + ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("1hs"); }, + "Malformed seconds value passed to --benchmark_min_time: `1hs`"); +} +} // namespace From ee9f1c8136d2d5e0dd7d8ea2111396daf2737d10 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Tue, 24 Jan 2023 13:50:42 -0500 Subject: [PATCH 31/34] Add BENCHMARK_EXPORT to hopefully fix missing def errors --- src/benchmark_runner.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 1d3e6426c5..9d80653728 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -44,7 +44,7 @@ struct RunResults { bool file_report_aggregates_only = false; }; -struct BenchTimeType { +struct BENCHMARK_EXPORT BenchTimeType { enum { ITERS, TIME } tag; union { IterationCount iters; @@ -52,6 +52,7 @@ struct BenchTimeType { }; }; +BENCHMARK_EXPORT BenchTimeType ParseBenchMinTime(const std::string& value); class BenchmarkRunner { From 245e1600c6731a026b1964e7ec9924182f092023 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Tue, 24 Jan 2023 14:10:31 -0500 Subject: [PATCH 32/34] only enable death tests in debug mode because bm_check is no-op in release mode --- test/min_time_parse_gtest.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/min_time_parse_gtest.cc b/test/min_time_parse_gtest.cc index f8c37c61d6..888377c8a7 100644 --- a/test/min_time_parse_gtest.cc +++ b/test/min_time_parse_gtest.cc @@ -4,6 +4,8 @@ namespace { TEST(ParseMinTimeTest, InvalidInput) { + // Tests only runnable in debug mode (when BM_CHECK is enabled). +#ifndef NDEBUG ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("abc"); }, "Malformed seconds value passed to --benchmark_min_time: `abc`"); @@ -16,5 +18,6 @@ TEST(ParseMinTimeTest, InvalidInput) { ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("1hs"); }, "Malformed seconds value passed to --benchmark_min_time: `1hs`"); +#endif } } // namespace From 340c65b3f437b323faae4388f4e99d4d46e94e17 Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Tue, 24 Jan 2023 14:28:33 -0500 Subject: [PATCH 33/34] guard death tests with additional support-check macros --- test/min_time_parse_gtest.cc | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/min_time_parse_gtest.cc b/test/min_time_parse_gtest.cc index 888377c8a7..5a349963e4 100644 --- a/test/min_time_parse_gtest.cc +++ b/test/min_time_parse_gtest.cc @@ -4,20 +4,25 @@ namespace { TEST(ParseMinTimeTest, InvalidInput) { +#if GTEST_HAS_DEATH_TEST // Tests only runnable in debug mode (when BM_CHECK is enabled). #ifndef NDEBUG - ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("abc"); }, - "Malformed seconds value passed to --benchmark_min_time: `abc`"); + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("abc"); }, + "Malformed seconds value passed to --benchmark_min_time: `abc`"); - ASSERT_DEATH( + ASSERT_DEATH_IF_SUPPORTED( { benchmark::internal::ParseBenchMinTime("123ms"); }, "Malformed seconds value passed to --benchmark_min_time: `123ms`"); - ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("1z"); }, - "Malformed seconds value passed to --benchmark_min_time: `1z`"); + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("1z"); }, + "Malformed seconds value passed to --benchmark_min_time: `1z`"); - ASSERT_DEATH({ benchmark::internal::ParseBenchMinTime("1hs"); }, - "Malformed seconds value passed to --benchmark_min_time: `1hs`"); + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("1hs"); }, + "Malformed seconds value passed to --benchmark_min_time: `1hs`"); +#endif #endif } } // namespace From a902c7faa852f643ef3e7e8910140d67894df42f Mon Sep 17 00:00:00 2001 From: Vy Nguyen Date: Tue, 24 Jan 2023 15:49:30 -0500 Subject: [PATCH 34/34] Add additional guard to prevent running in Release mode --- test/min_time_parse_gtest.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/min_time_parse_gtest.cc b/test/min_time_parse_gtest.cc index 5a349963e4..e2bdf67850 100644 --- a/test/min_time_parse_gtest.cc +++ b/test/min_time_parse_gtest.cc @@ -7,6 +7,7 @@ TEST(ParseMinTimeTest, InvalidInput) { #if GTEST_HAS_DEATH_TEST // Tests only runnable in debug mode (when BM_CHECK is enabled). #ifndef NDEBUG +#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS ASSERT_DEATH_IF_SUPPORTED( { benchmark::internal::ParseBenchMinTime("abc"); }, "Malformed seconds value passed to --benchmark_min_time: `abc`"); @@ -24,5 +25,6 @@ TEST(ParseMinTimeTest, InvalidInput) { "Malformed seconds value passed to --benchmark_min_time: `1hs`"); #endif #endif +#endif } } // namespace