From 92e2bfcd23eb592b823b1d1794d9f7679bd83032 Mon Sep 17 00:00:00 2001 From: Griffin Bassman Date: Fri, 10 Feb 2023 13:52:32 -0500 Subject: [PATCH 1/5] ci: [epsilon_decay] benchmarks --- test/benchmarks/CMakeLists.txt | 2 + test/benchmarks/benchmark_epsilon_decay.cc | 55 ++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 test/benchmarks/benchmark_epsilon_decay.cc diff --git a/test/benchmarks/CMakeLists.txt b/test/benchmarks/CMakeLists.txt index 1a5058cdd82..0a43419cd68 100644 --- a/test/benchmarks/CMakeLists.txt +++ b/test/benchmarks/CMakeLists.txt @@ -8,6 +8,8 @@ if (NOT BUILD_ONLY_STANDALONE_BENCHMARKS) set(all_sources ${all_sources} input_format_benchmarks.cc benchmark_funcs.cc + benchmark_epsilon_decay.cc + ../../vowpalwabbit/core/tests/simulator.cc # These are just for benchmarking specific standard library operations #benchmark_std_function.cc diff --git a/test/benchmarks/benchmark_epsilon_decay.cc b/test/benchmarks/benchmark_epsilon_decay.cc new file mode 100644 index 00000000000..14b75ce65aa --- /dev/null +++ b/test/benchmarks/benchmark_epsilon_decay.cc @@ -0,0 +1,55 @@ +#include "benchmarks_common.h" +#include "vw/cache_parser/parse_example_cache.h" +#include "vw/config/options_cli.h" +#include "vw/core/parser.h" +#include "vw/core/vw.h" +#include "vw/io/io_adapter.h" +#include "vw/text_parser/parse_example_text.h" + +#include + + +#include "vw/core/reductions/epsilon_decay.h" + +#include "../../vowpalwabbit/core/tests/simulator.h" +#include "vw/core/learner.h" +#include "vw/core/metric_sink.h" +#include "vw/core/setup_base.h" + + +template +static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_args) +{ + std::array res = {extra_args...}; + std::string use_decay = res[0]; + std::string model_count = res[1]; + std::string bit_size = res[2]; + std::string tolerance = res[3]; + + using callback_map = typename std::map>; + callback_map test_hooks; + + for (auto _ : state) + { + const size_t num_iterations = 1000; + const size_t seed = 99; + const std::vector swap_after = {500}; + if (use_decay == "decay") + { + simulator::_test_helper_hook(std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf", "--epsilon_decay", "--model_count", model_count, "-b", bit_size, "--tol_x", tolerance}, test_hooks, num_iterations, seed, swap_after); + } + else + { + simulator::_test_helper_hook(std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf"}, test_hooks, num_iterations, seed, swap_after); + } + benchmark::ClobberMemory(); + } +} + +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_1_model_big_tol, "decay", "1", "18", "1e-2"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_2_model_big_tol, "decay", "2", "19", "1e-2"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_4_model_big_tol, "decay", "4", "20", "1e-2"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_1_model_small_tol, "decay", "1", "18", "1e-6"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_2_model_small_tol, "decay", "2", "19", "1e-6"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_4_model_small_tol, "decay", "4", "20", "1e-6"); +BENCHMARK_CAPTURE(bench_epsilon_decay, without_epsilon_decay, "nondecay", "", "", ""); \ No newline at end of file From ddf8b0d5e8fb40716a6fc04957ece6541f5c8cfc Mon Sep 17 00:00:00 2001 From: Griffin Bassman Date: Fri, 10 Feb 2023 13:53:38 -0500 Subject: [PATCH 2/5] clang --- test/benchmarks/benchmark_epsilon_decay.cc | 25 +++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/benchmarks/benchmark_epsilon_decay.cc b/test/benchmarks/benchmark_epsilon_decay.cc index 14b75ce65aa..f0f7705fa50 100644 --- a/test/benchmarks/benchmark_epsilon_decay.cc +++ b/test/benchmarks/benchmark_epsilon_decay.cc @@ -1,22 +1,18 @@ +#include "../../vowpalwabbit/core/tests/simulator.h" #include "benchmarks_common.h" #include "vw/cache_parser/parse_example_cache.h" #include "vw/config/options_cli.h" +#include "vw/core/learner.h" +#include "vw/core/metric_sink.h" #include "vw/core/parser.h" +#include "vw/core/reductions/epsilon_decay.h" +#include "vw/core/setup_base.h" #include "vw/core/vw.h" #include "vw/io/io_adapter.h" #include "vw/text_parser/parse_example_text.h" #include - -#include "vw/core/reductions/epsilon_decay.h" - -#include "../../vowpalwabbit/core/tests/simulator.h" -#include "vw/core/learner.h" -#include "vw/core/metric_sink.h" -#include "vw/core/setup_base.h" - - template static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_args) { @@ -26,7 +22,8 @@ static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_ar std::string bit_size = res[2]; std::string tolerance = res[3]; - using callback_map = typename std::map>; + using callback_map = + typename std::map>; callback_map test_hooks; for (auto _ : state) @@ -36,11 +33,15 @@ static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_ar const std::vector swap_after = {500}; if (use_decay == "decay") { - simulator::_test_helper_hook(std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf", "--epsilon_decay", "--model_count", model_count, "-b", bit_size, "--tol_x", tolerance}, test_hooks, num_iterations, seed, swap_after); + simulator::_test_helper_hook( + std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf", "--epsilon_decay", + "--model_count", model_count, "-b", bit_size, "--tol_x", tolerance}, + test_hooks, num_iterations, seed, swap_after); } else { - simulator::_test_helper_hook(std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf"}, test_hooks, num_iterations, seed, swap_after); + simulator::_test_helper_hook(std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf"}, + test_hooks, num_iterations, seed, swap_after); } benchmark::ClobberMemory(); } From c198a9d30a0fad6c92c4687bffa71f169bdd58dc Mon Sep 17 00:00:00 2001 From: Griffin Bassman Date: Fri, 10 Feb 2023 14:16:43 -0500 Subject: [PATCH 3/5] quiet --- test/benchmarks/benchmark_epsilon_decay.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/benchmarks/benchmark_epsilon_decay.cc b/test/benchmarks/benchmark_epsilon_decay.cc index f0f7705fa50..177568e268d 100644 --- a/test/benchmarks/benchmark_epsilon_decay.cc +++ b/test/benchmarks/benchmark_epsilon_decay.cc @@ -35,13 +35,14 @@ static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_ar { simulator::_test_helper_hook( std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf", "--epsilon_decay", - "--model_count", model_count, "-b", bit_size, "--tol_x", tolerance}, + "--model_count", model_count, "-b", bit_size, "--tol_x", "--quiet", tolerance}, test_hooks, num_iterations, seed, swap_after); } else { - simulator::_test_helper_hook(std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf"}, - test_hooks, num_iterations, seed, swap_after); + simulator::_test_helper_hook( + std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf", "--quiet"}, test_hooks, + num_iterations, seed, swap_after); } benchmark::ClobberMemory(); } From 1e131c2f82fd54a4f2bd334c21722107de063b26 Mon Sep 17 00:00:00 2001 From: Griffin Bassman Date: Fri, 10 Feb 2023 15:42:32 -0500 Subject: [PATCH 4/5] fix --- test/benchmarks/benchmark_epsilon_decay.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmarks/benchmark_epsilon_decay.cc b/test/benchmarks/benchmark_epsilon_decay.cc index 177568e268d..51f519ac6da 100644 --- a/test/benchmarks/benchmark_epsilon_decay.cc +++ b/test/benchmarks/benchmark_epsilon_decay.cc @@ -35,7 +35,7 @@ static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_ar { simulator::_test_helper_hook( std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf", "--epsilon_decay", - "--model_count", model_count, "-b", bit_size, "--tol_x", "--quiet", tolerance}, + "--model_count", model_count, "-b", bit_size, "--tol_x", tolerance, "--quiet"}, test_hooks, num_iterations, seed, swap_after); } else From d28f39fd302e65bcb4c113b7b220eae37e99273f Mon Sep 17 00:00:00 2001 From: Griffin Bassman Date: Mon, 13 Feb 2023 11:36:02 -0500 Subject: [PATCH 5/5] comment --- test/benchmarks/benchmark_epsilon_decay.cc | 25 +++++++++++----------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/test/benchmarks/benchmark_epsilon_decay.cc b/test/benchmarks/benchmark_epsilon_decay.cc index 51f519ac6da..34608056352 100644 --- a/test/benchmarks/benchmark_epsilon_decay.cc +++ b/test/benchmarks/benchmark_epsilon_decay.cc @@ -14,13 +14,12 @@ #include template -static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_args) +static void bench_epsilon_decay(benchmark::State& state, bool use_decay, ExtraArgs&&... extra_args) { std::array res = {extra_args...}; - std::string use_decay = res[0]; - std::string model_count = res[1]; - std::string bit_size = res[2]; - std::string tolerance = res[3]; + std::string model_count = res[0]; + std::string bit_size = res[1]; + std::string tolerance = res[2]; using callback_map = typename std::map>; @@ -31,7 +30,7 @@ static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_ar const size_t num_iterations = 1000; const size_t seed = 99; const std::vector swap_after = {500}; - if (use_decay == "decay") + if (use_decay) { simulator::_test_helper_hook( std::vector{"-l", "1e-3", "--power_t", "0", "-q::", "--cb_explore_adf", "--epsilon_decay", @@ -48,10 +47,10 @@ static void bench_epsilon_decay(benchmark::State& state, ExtraArgs&&... extra_ar } } -BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_1_model_big_tol, "decay", "1", "18", "1e-2"); -BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_2_model_big_tol, "decay", "2", "19", "1e-2"); -BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_4_model_big_tol, "decay", "4", "20", "1e-2"); -BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_1_model_small_tol, "decay", "1", "18", "1e-6"); -BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_2_model_small_tol, "decay", "2", "19", "1e-6"); -BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_4_model_small_tol, "decay", "4", "20", "1e-6"); -BENCHMARK_CAPTURE(bench_epsilon_decay, without_epsilon_decay, "nondecay", "", "", ""); \ No newline at end of file +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_1_model_big_tol, true, "1", "18", "1e-2"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_2_model_big_tol, true, "2", "19", "1e-2"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_4_model_big_tol, true, "4", "20", "1e-2"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_1_model_small_tol, true, "1", "18", "1e-6"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_2_model_small_tol, true, "2", "19", "1e-6"); +BENCHMARK_CAPTURE(bench_epsilon_decay, epsilon_decay_4_model_small_tol, true, "4", "20", "1e-6"); +BENCHMARK_CAPTURE(bench_epsilon_decay, without_epsilon_decay, false, "", "", ""); \ No newline at end of file