feat: integrate confidence sequences in automl and epsilon_decay

.scripts/linux/unit-tests-valgrind.sh

@@ -7,4 +7,5 @@ REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
 cd build
-ctest --verbose --output-on-failure --label-regex VWTestList
+export BOOST_TEST_LOG_LEVEL=unit_scope
+ctest --verbose --output-on-failure --label-regex VWTestList --timeout 2400

python/pylibvw.cc

@@ -307,6 +307,13 @@ struct python_dict_writer : VW::metric_sink_visitor
   void float_metric(const std::string& key, float value) override { _dest_dict[key] = value; }
   void string_metric(const std::string& key, const std::string& value) override { _dest_dict[key] = value; }
   void bool_metric(const std::string& key, bool value) override { _dest_dict[key] = value; }
+  void sink_metric(const std::string& key, const VW::metric_sink& value)
+  {
+    py::dict nested;
+    auto nested_py = python_dict_writer(nested);
+    value.visit(nested_py);
+    _dest_dict[key] = nested;
+  }
 
 private:
   py::dict& _dest_dict;

python/tests/confidence_sequence.py

@@ -1,7 +1,7 @@
 from cmath import sqrt
 
 
-class IncrementalFsum:
+class incremental_f_sum:
     """Incremental version of https://en.wikipedia.org/wiki/Kahan_summation_algorithm"""
 
     def __init__(self):
@@ -22,7 +22,7 @@ def __iadd__(self, x):
         return self
 
     def __add__(self, other):
-        result = IncrementalFsum()
+        result = incremental_f_sum()
         result.partials = deepcopy(self.partials)
         for y in other.partials:
             result += y
@@ -32,7 +32,7 @@ def __float__(self):
         return sum(self.partials, 0.0)
 
 
-class ConfidenceSequence(object):
+class confidence_sequence(object):
     def __init__(self, rmin=0, rmax=1, adjust=True, eta=1.1, s=1.1):
         super().__init__()
 
@@ -47,17 +47,17 @@ def __init__(self, rmin=0, rmax=1, adjust=True, eta=1.1, s=1.1):
 
         self.t = 0
 
-        self.sumwsqrsq = IncrementalFsum()
-        self.sumwsqr = IncrementalFsum()
-        self.sumwsq = IncrementalFsum()
-        self.sumwr = IncrementalFsum()
-        self.sumw = IncrementalFsum()
-        self.sumwrxhatlow = IncrementalFsum()
-        self.sumwxhatlow = IncrementalFsum()
-        self.sumxhatlowsq = IncrementalFsum()
-        self.sumwrxhathigh = IncrementalFsum()
-        self.sumwxhathigh = IncrementalFsum()
-        self.sumxhathighsq = IncrementalFsum()
+        self.sumwsqrsq = incremental_f_sum()
+        self.sumwsqr = incremental_f_sum()
+        self.sumwsq = incremental_f_sum()
+        self.sumwr = incremental_f_sum()
+        self.sumw = incremental_f_sum()
+        self.sumwrxhatlow = incremental_f_sum()
+        self.sumwxhatlow = incremental_f_sum()
+        self.sumxhatlowsq = incremental_f_sum()
+        self.sumwrxhathigh = incremental_f_sum()
+        self.sumwxhathigh = incremental_f_sum()
+        self.sumxhathighsq = incremental_f_sum()
 
     def addobs(self, w, r, p_drop=0, n_drop=None):
         assert w >= 0

python/tests/test_confidence_sequence.py

@@ -1,10 +1,10 @@
-from confidence_sequence import ConfidenceSequence
+from confidence_sequence import confidence_sequence
 import numpy as np
 
 
 def test_confidence_sequence():
     rs = [0.5, 0.6, 0.7, 0.8] * 1000
-    cs = ConfidenceSequence()
+    cs = confidence_sequence()
     for r, w in zip(rs, rs):
         cs.addobs(w, r)
     alpha = 0.05

test/run_tests_model_gen_and_load.py

@@ -104,8 +104,13 @@ def load_model(
     print(
         f"{color_enum.LIGHT_PURPLE}id: {test_id}, command: {command}{color_enum.ENDC}"
     )
-    vw = vowpalwabbit.Workspace(command)
-    vw.finish()
+
+    try:
+        vw = vowpalwabbit.Workspace(command)
+        vw.finish()
+    except Exception as e:
+        print(f"{color_enum.LIGHT_RED} FAILURE!! id: {test_id} {str(e)}")
+        raise e
 
 
 def get_tests(

test/test-sets/ref/aml_ccb_metrics.json

@@ -1 +1 @@
-{"cbea_label_first_action":1,"cbea_label_not_first":3,"cbea_labeled_ex":4,"cbea_max_actions":2,"cbea_min_actions":2,"cbea_non_zero_cost":0,"cbea_predict_in_learn":0,"conf_idx_amls_0":0,"conf_idx_amls_1":6,"conf_idx_amls_2":5,"current_champ":0,"sfm_count_learn_example_with_shared":4,"test_county":4,"total_champ_switches":0,"total_learn_calls":4,"total_log_calls":0,"total_predict_calls":0,"upcnt_amls_0":0,"upcnt_amls_1":4,"upcnt_amls_2":4,"upcnt_sc_0":0,"upcnt_sc_1":4,"upcnt_sc_2":4,"bound_amls_0":0.0,"bound_amls_1":0.0,"bound_amls_2":0.0,"bound_sc_0":0.0,"bound_sc_1":0.0,"bound_sc_2":0.0,"cbea_avg_actions_per_event":2.0,"cbea_avg_feat_per_action":0.0,"cbea_avg_feat_per_event":0.0,"cbea_avg_ns_per_action":0.0,"cbea_avg_ns_per_event":0.0,"cbea_sum_cost":0.0,"cbea_sum_cost_baseline":0.0,"ips_amls_0":0.0,"ips_amls_1":0.0,"ips_amls_2":0.0,"ips_sc_0":0.0,"ips_sc_1":0.0,"ips_sc_2":0.0,"r_amls_0":0.0,"r_amls_1":-0.0,"r_amls_2":-0.0,"r_sc_0":0.0,"r_sc_1":-0.0,"r_sc_2":-0.0,"w_amls_0":0.0,"w_amls_1":0.0,"w_amls_2":0.0,"w_sc_0":0.0,"w_sc_1":0.0,"w_sc_2":0.0,"exclusionc_0":"{}","exclusionc_1":"{[\"U\", \"U\"]}","exclusionc_2":"{[\"S\", \"U\"]}","interactions_amls_0":"-q AA -q AS -q AU -q SS -q SU -q UU -q AA[ccbid] -q AA -q AS[ccbid] -q AS -q AU[ccbid] -q AU -q SS[ccbid] -q SS -q SU[ccbid] -q SU -q UU[ccbid] -q UU -q [wild][ccbid] ","interactions_amls_1":"-q AA -q AS -q AU -q SS -q SU -q AA[ccbid] -q AA -q AS[ccbid] -q AS -q AU[ccbid] -q AU -q SS[ccbid] -q SS -q SU[ccbid] -q SU -q [wild][ccbid] ","interactions_amls_2":"-q AA -q AS -q AU -q SS -q UU -q AA[ccbid] -q AA -q AS[ccbid] -q AS -q AU[ccbid] -q AU -q SS[ccbid] -q SS -q UU[ccbid] -q UU -q [wild][ccbid] "}
+{"cbea_label_first_action":1,"cbea_label_not_first":3,"cbea_labeled_ex":4,"cbea_max_actions":2,"cbea_min_actions":2,"cbea_non_zero_cost":0,"cbea_predict_in_learn":0,"current_champ":0,"sfm_count_learn_example_with_shared":4,"test_county":4,"total_champ_switches":0,"total_learn_calls":4,"total_log_calls":0,"total_predict_calls":0,"cbea_avg_actions_per_event":2.0,"cbea_avg_feat_per_action":0.0,"cbea_avg_feat_per_event":0.0,"cbea_avg_ns_per_action":0.0,"cbea_avg_ns_per_event":0.0,"cbea_sum_cost":0.0,"cbea_sum_cost_baseline":0.0,"estimator_0":{"config_type":"exclusion","elements":"{}","self":{"conf_idx":0,"upcnt":0,"last_r":0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q AA -q AS -q AU -q SS -q SU -q UU -q AA[ccbid] -q AA -q AS[ccbid] -q AS -q AU[ccbid] -q AU -q SS[ccbid] -q SS -q SU[ccbid] -q SU -q UU[ccbid] -q UU -q [wild][ccbid] "}},"estimator_1":{"config_type":"exclusion","elements":"{[\"U\", \"U\"]}","self":{"conf_idx":6,"upcnt":4,"last_r":-0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q AA -q AS -q AU -q SS -q SU -q AA[ccbid] -q AA -q AS[ccbid] -q AS -q AU[ccbid] -q AU -q SS[ccbid] -q SS -q SU[ccbid] -q SU -q [wild][ccbid] "},"sync_champ":{"upcnt":4,"last_r":-0.0,"last_w":1.0,"lb":0.0,"ub":1.0}},"estimator_2":{"config_type":"exclusion","elements":"{[\"S\", \"U\"]}","self":{"conf_idx":5,"upcnt":4,"last_r":-0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q AA -q AS -q AU -q SS -q UU -q AA[ccbid] -q AA -q AS[ccbid] -q AS -q AU[ccbid] -q AU -q SS[ccbid] -q SS -q UU[ccbid] -q UU -q [wild][ccbid] "},"sync_champ":{"upcnt":4,"last_r":-0.0,"last_w":1.0,"lb":0.0,"ub":1.0}}}

test/test-sets/ref/metrics_.json

@@ -1 +1 @@
-{"cbea_label_first_action":0,"cbea_label_not_first":3,"cbea_labeled_ex":3,"cbea_max_actions":12,"cbea_min_actions":12,"cbea_non_zero_cost":2,"cbea_predict_in_learn":0,"conf_idx_amls_0":0,"conf_idx_amls_1":3,"conf_idx_amls_2":2,"current_champ":0,"line_parse_error":0,"number_events_zero_actions":0,"number_skipped_events":0,"sfm_count_learn_example_with_shared":3,"test_county":3,"total_champ_switches":0,"total_learn_calls":3,"total_log_calls":1,"total_predict_calls":0,"upcnt_amls_0":0,"upcnt_amls_1":3,"upcnt_amls_2":3,"upcnt_sc_0":0,"upcnt_sc_1":3,"upcnt_sc_2":3,"bound_amls_0":0.0,"bound_amls_1":0.0,"bound_amls_2":0.0,"bound_sc_0":0.0,"bound_sc_1":0.0,"bound_sc_2":0.0,"cbea_avg_actions_per_event":12.0,"cbea_avg_feat_per_action":835.0,"cbea_avg_feat_per_event":10030.0,"cbea_avg_ns_per_action":9.0,"cbea_avg_ns_per_event":117.0,"cbea_sum_cost":-2.0,"cbea_sum_cost_baseline":0.0,"dsjson_sum_cost_original":0.0,"dsjson_sum_cost_original_baseline":0.0,"ips_amls_0":0.0,"ips_amls_1":0.0,"ips_amls_2":0.0,"ips_sc_0":0.0,"ips_sc_1":0.0,"ips_sc_2":0.0,"r_amls_0":0.0,"r_amls_1":1.0,"r_amls_2":1.0,"r_sc_0":0.0,"r_sc_1":1.0,"r_sc_2":1.0,"w_amls_0":0.0,"w_amls_1":0.0,"w_amls_2":0.0,"w_sc_0":0.0,"w_sc_1":0.0,"w_sc_2":0.0,"exclusionc_0":"{}","exclusionc_1":"{[\"E\", \"G\"]}","exclusionc_2":"{[\"E\", \"M\"]}","first_event_id":"0074434d3a3a46529f65de8a59631939","first_event_time":"","interactions_amls_0":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj ","interactions_amls_1":"-q EE -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj ","interactions_amls_2":"-q EE -q EG -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj ","last_event_id":"9077f996581148978a0ebe2484260dab","last_event_time":""}
+{"cbea_label_first_action":0,"cbea_label_not_first":3,"cbea_labeled_ex":3,"cbea_max_actions":12,"cbea_min_actions":12,"cbea_non_zero_cost":2,"cbea_predict_in_learn":0,"current_champ":0,"line_parse_error":0,"number_events_zero_actions":0,"number_skipped_events":0,"sfm_count_learn_example_with_shared":3,"test_county":3,"total_champ_switches":0,"total_learn_calls":3,"total_log_calls":1,"total_predict_calls":0,"cbea_avg_actions_per_event":12.0,"cbea_avg_feat_per_action":835.0,"cbea_avg_feat_per_event":10030.0,"cbea_avg_ns_per_action":9.0,"cbea_avg_ns_per_event":117.0,"cbea_sum_cost":-2.0,"cbea_sum_cost_baseline":0.0,"dsjson_sum_cost_original":0.0,"dsjson_sum_cost_original_baseline":0.0,"first_event_id":"0074434d3a3a46529f65de8a59631939","first_event_time":"","last_event_id":"9077f996581148978a0ebe2484260dab","last_event_time":"","estimator_0":{"config_type":"exclusion","elements":"{}","self":{"conf_idx":0,"upcnt":0,"last_r":0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "}},"estimator_1":{"config_type":"exclusion","elements":"{[\"E\", \"G\"]}","self":{"conf_idx":3,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0}},"estimator_2":{"config_type":"exclusion","elements":"{[\"E\", \"M\"]}","self":{"conf_idx":2,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0}}}

test/test-sets/ref/metrics_cbadfautoml.json

@@ -1 +1 @@
-{"conf_idx_amls_0":0,"conf_idx_amls_1":3,"conf_idx_amls_2":2,"current_champ":0,"line_parse_error":0,"number_events_zero_actions":0,"number_skipped_events":0,"sfm_count_learn_example_with_shared":3,"test_county":3,"total_champ_switches":0,"total_learn_calls":3,"total_log_calls":1,"total_predict_calls":0,"upcnt_amls_0":0,"upcnt_amls_1":3,"upcnt_amls_2":3,"upcnt_sc_0":0,"upcnt_sc_1":3,"upcnt_sc_2":3,"bound_amls_0":0.0,"bound_amls_1":0.0,"bound_amls_2":0.0,"bound_sc_0":0.0,"bound_sc_1":0.0,"bound_sc_2":0.0,"dsjson_sum_cost_original":0.0,"dsjson_sum_cost_original_baseline":0.0,"ips_amls_0":0.0,"ips_amls_1":0.0,"ips_amls_2":0.0,"ips_sc_0":0.0,"ips_sc_1":0.0,"ips_sc_2":0.0,"r_amls_0":0.0,"r_amls_1":1.0,"r_amls_2":1.0,"r_sc_0":0.0,"r_sc_1":1.0,"r_sc_2":1.0,"w_amls_0":0.0,"w_amls_1":0.0,"w_amls_2":0.0,"w_sc_0":0.0,"w_sc_1":0.0,"w_sc_2":0.0,"exclusionc_0":"{}","exclusionc_1":"{[\"E\", \"G\"]}","exclusionc_2":"{[\"E\", \"M\"]}","first_event_id":"0074434d3a3a46529f65de8a59631939","first_event_time":"","interactions_amls_0":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj ","interactions_amls_1":"-q EE -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj ","interactions_amls_2":"-q EE -q EG -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj ","last_event_id":"9077f996581148978a0ebe2484260dab","last_event_time":""}
+{"current_champ":0,"line_parse_error":0,"number_events_zero_actions":0,"number_skipped_events":0,"sfm_count_learn_example_with_shared":3,"test_county":3,"total_champ_switches":0,"total_learn_calls":3,"total_log_calls":1,"total_predict_calls":0,"dsjson_sum_cost_original":0.0,"dsjson_sum_cost_original_baseline":0.0,"first_event_id":"0074434d3a3a46529f65de8a59631939","first_event_time":"","last_event_id":"9077f996581148978a0ebe2484260dab","last_event_time":"","estimator_0":{"config_type":"exclusion","elements":"{}","self":{"conf_idx":0,"upcnt":0,"last_r":0.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "}},"estimator_1":{"config_type":"exclusion","elements":"{[\"E\", \"G\"]}","self":{"conf_idx":3,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EM -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0}},"estimator_2":{"config_type":"exclusion","elements":"{[\"E\", \"M\"]}","self":{"conf_idx":2,"upcnt":3,"last_r":1.0,"last_w":0.0,"lb":0.0,"ub":1.0,"interactions":"-q EE -q EG -q EO -q ER -q ES -q ET -q Ei -q Ej -q GG -q GM -q GO -q GR -q GS -q GT -q Gi -q Gj -q MM -q MO -q MR -q MS -q MT -q Mi -q Mj -q OO -q OR -q OS -q OT -q Oi -q Oj -q RR -q RS -q RT -q Ri -q Rj -q SS -q ST -q Si -q Sj -q TT -q Ti -q Tj -q ii -q ij -q jj "},"sync_champ":{"upcnt":3,"last_r":1.0,"last_w":1.0,"lb":0.0,"ub":1.0}}}