Bug fixes for performance over time calculations

Makefile

@@ -65,9 +65,9 @@ pre-commit:
 	$(PRECOMMIT) run --all-files
 
 format-black:
-	$(BLACK) autosklearn/.*
-	$(BLACK) test/.*
-	$(BLACK) examples/.*
+	$(BLACK) autosklearn
+	$(BLACK) test
+	$(BLACK) examples
 
 format-isort:
 	$(ISORT) autosklearn

autosklearn/automl.py

@@ -1825,7 +1825,15 @@ def _get_runhistory_models_performance(self):
     @property
     def performance_over_time_(self):
         check_is_fitted(self)
-        individual_performance_frame = self._get_runhistory_models_performance()
+        individual_performance_frame = (
+            self._get_runhistory_models_performance().sort_values(
+                by=["Timestamp", "single_best_optimization_score"]
+            )
+        )
+
+        metric = self._metrics[0]
+        individual_performance_frame["single_best_optimization_score"] *= metric._sign
+
         best_values = pd.Series(
             {
                 "single_best_optimization_score": -np.inf,
@@ -1841,6 +1849,8 @@ def performance_over_time_(self):
                 best_values = individual_performance_frame.loc[idx]
             individual_performance_frame.loc[idx] = best_values
 
+        individual_performance_frame["single_best_optimization_score"] *= metric._sign
+
         performance_over_time = individual_performance_frame
 
         if self._ensemble_class is not None:
@@ -1856,6 +1866,10 @@ def performance_over_time_(self):
                     best_values = ensemble_performance_frame.loc[idx]
                 ensemble_performance_frame.loc[idx] = best_values
 
+            for c in ensemble_performance_frame.columns:
+                if c != "Timestamp":
+                    ensemble_performance_frame[c] *= metric._sign
+
             performance_over_time = (
                 pd.merge(
                     ensemble_performance_frame,
@@ -1867,7 +1881,7 @@ def performance_over_time_(self):
                 .fillna(method="ffill")
             )
 
-        return performance_over_time
+        return performance_over_time.drop_duplicates()
 
     @property
     def cv_results_(self):

autosklearn/estimators.py

@@ -530,7 +530,6 @@ def build_automl(self):
         return automl
 
     def fit(self, **kwargs):
-
         # Automatically set the cutoff time per task
         if self.per_run_time_limit is None:
             self.per_run_time_limit = self._n_jobs * self.time_left_for_this_task // 10

scripts/02_retrieve_metadata.py

@@ -195,6 +195,7 @@ def __init__(self, info, feat_type=None):
         self._info = info
         self.feat_type = feat_type
 
+
 def main():
     parser = ArgumentParser()
 
@@ -239,7 +240,7 @@ def main():
             configuration_space = pipeline.get_configuration_space(
                 DummyDatamanager(
                     info={"is_sparse": sparse, "task": task},
-                    feat_type={"A": "numerical", "B": "categorical"}
+                    feat_type={"A": "numerical", "B": "categorical"},
                 )
             )