Change returned Objectives

Author: PhMueller

hpobench/benchmarks/mo/cnn_benchmark.py

@@ -258,9 +258,8 @@ def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace:
         ConfigSpace.ConfigurationSpace
         """
         fidelity_space = CS.ConfigurationSpace(seed=seed)
-        fidelity_space.add_hyperparameters([CS.UniformIntegerHyperparameter(
-            'budget', lower=1, upper=25, default_value=25, log=False
-        )
+        fidelity_space.add_hyperparameters([
+            CS.UniformIntegerHyperparameter('budget', lower=1, upper=25, default_value=25, log=False)
         ])
         return fidelity_space
 
@@ -298,6 +297,7 @@ def init_model(self, config: Union[CS.Configuration, Dict]) -> Net:
     def __seed_everything(self):
         """Helperfunction: Make the benchmark deterministic by setting the correct seeds"""
         seed = self.rng.randint(0, 100000)
+        logger.debug(f'Generate seed: {seed}')
         random.seed(seed)
         np.random.seed(seed)
         torch.manual_seed(seed)
@@ -361,8 +361,10 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict],
         -------
         Dict -
             function_value : Dict
-                validation_accuracy: float
-                model_size: float
+                negative_accuracy: float
+                    -100 * validation accuracy
+                log_model_size: float
+                    log10 of the number of parameters
             cost : time to train the network
             info : Dict
                 train_accuracy : float,
@@ -433,8 +435,8 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict],
 
         elapsed_time = time.time() - time_in
 
-        return {'function_value': {'accuracy': val_accuracy,
-                                   'model_size': num_params},
+        return {'function_value': {'negative_accuracy': -100 * val_accuracy,
+                                   'log_model_size': float(np.log10(num_params))},
                 'cost': float(training_runtime),
                 'info': {'train_accuracy': train_accuracy,
                          'training_cost': training_runtime,
@@ -477,8 +479,10 @@ def objective_function_test(self, configuration: Union[CS.Configuration, Dict],
         -------
         Dict -
             function_value : Dict
-                validation_accuracy: float
-                model_size: float
+                negative_accuracy: float
+                    -100 * test accuracy
+                log_model_size: float
+                    log10 of the number of parameters
             cost : time to train the network
             info : Dict
                 train_accuracy : float,
@@ -542,8 +546,8 @@ def objective_function_test(self, configuration: Union[CS.Configuration, Dict],
 
         elapsed_time = time.time() - time_in
 
-        return {'function_value': {'accuracy': test_accuracy,
-                                   'model_size': num_params},
+        return {'function_value': {'negative_accuracy': -100 * test_accuracy,
+                                   'log_model_size': float(np.log10(num_params))},
                 'cost': training_runtime,
                 'info': {'train_accuracy': train_accuracy,
                          'training_cost': training_runtime,

tests/test_mo_cnn.py

@@ -4,6 +4,21 @@
 logging.basicConfig(level=logging.DEBUG)
 
 
+def test_mo_cnn_seeding():
+    from hpobench.benchmarks.mo.cnn_benchmark import FlowerCNNBenchmark
+    b1 = FlowerCNNBenchmark(rng=0)
+    b2 = FlowerCNNBenchmark(rng=0)
+    test_config = {
+        'batch_norm': True, 'batch_size': 71, 'conv_layer_0': 194,  'conv_layer_1': 152,
+        'conv_layer_2': 92, 'fc_layer_0': 65, 'fc_layer_1': 19, 'fc_layer_2': 273,
+        'global_avg_pooling': True, 'kernel_size': 5, 'learning_rate_init': 0.09091283280651452,
+        'n_conv_layers': 2, 'n_fc_layers': 2
+    }
+    result_1 = b1.objective_function(test_config, rng=1, fidelity={'budget': 3})
+    result_2 = b2.objective_function(test_config, rng=1, fidelity={'budget': 3})
+    assert result_1 == result_2
+
+
 def test_mo_cnn_benchmark():
     from hpobench.container.benchmarks.mo.cnn_benchmark import FlowerCNNBenchmark
 
@@ -28,6 +43,6 @@ def test_mo_cnn_benchmark():
     result_2 = benchmark.objective_function(test_config, rng=1, fidelity={'budget': 3})
 
     assert result_1['info']['valid_accuracy'] == pytest.approx(0.1029, rel=0.001)
-    assert result_1['info']['valid_accuracy'] == result_1['function_value']['accuracy']
+    assert result_1['info']['valid_accuracy'] == pytest.approx(-0.01 * result_1['function_value']['negative_accuracy'], abs=0.001)
     assert result_1['info']['train_accuracy'] == pytest.approx(0.1044, rel=0.001)
     assert result_1['info']['train_accuracy'] == result_2['info']['train_accuracy']