Speed up CI testing (#163)

* Speed up tests by disabling slow tests.

Author: PhMueller

hpobench/util/test_utils.py

@@ -0,0 +1,24 @@
+import os
+
+CONST_RUN_ALL_TESTS_ENV_VAR = 'HPOBENCH_RUN_EXPENSIVE_TESTS'
+DEFAULT_SKIP_MSG = 'Skip this test due to time limitations'
+
+
+def check_run_all_tests():
+    """ Helper function: Check if all tests should run. """
+    return os.environ.get(CONST_RUN_ALL_TESTS_ENV_VAR, 'false').lower() == 'true'
+
+
+def enable_all_tests():
+    """
+    Some tests are quite expensive. We control if all runs should be executed by this
+    environment variable.
+    """
+    os.environ[CONST_RUN_ALL_TESTS_ENV_VAR] = 'true'
+
+
+def disable_all_tests():
+    """
+    This function disables the evaluation of all test functions.
+    """
+    os.environ[CONST_RUN_ALL_TESTS_ENV_VAR] = 'false'

tests/test_data_manager.py

@@ -1,22 +1,21 @@
 import shutil
-from multiprocessing import Pool
-
 import pytest
+from multiprocessing import Pool
 
 import hpobench
 from hpobench.util.data_manager import NASBench_201Data, YearPredictionMSDData, ProteinStructureData, BostonHousingData
-skip_message = 'We currently skip this test because it takes too much time.'
+from hpobench.util.test_utils import DEFAULT_SKIP_MSG, check_run_all_tests
 
 
-@pytest.mark.skip(reason=skip_message)
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_nasbench_201_load_thread_safe():
     shutil.rmtree(hpobench.config_file.data_dir / "nasbench_201", ignore_errors=True)
     function = lambda: NASBench_201Data(dataset='cifar100').load()
     with Pool(3) as pool:
         pool.map(function, [])
 
 
-@pytest.mark.skip(reason=skip_message)
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_nasbench_201_init():
 
     data_manager = NASBench_201Data(dataset='cifar100')
@@ -30,7 +29,7 @@ def test_nasbench_201_init():
     assert data_manager._save_dir.exists()
 
 
-@pytest.mark.skip(reason=skip_message)
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_nasbench_201_load():
 
     shutil.rmtree(hpobench.config_file.data_dir / "nasbench_201", ignore_errors=True)

tests/test_mo_cnn.py

@@ -1,6 +1,8 @@
 import pytest
+from hpobench.util.test_utils import DEFAULT_SKIP_MSG, check_run_all_tests
 
 
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_mo_cnn_seeding():
     from hpobench.container.benchmarks.mo.cnn_benchmark import FlowerCNNBenchmark
     b1 = FlowerCNNBenchmark(rng=0)

tests/test_nasbench_201.py

@@ -1,14 +1,11 @@
-import logging
-logging.basicConfig(level=logging.DEBUG)
 import pytest
 
 from hpobench.container.benchmarks.nas.nasbench_201 import ImageNetNasBench201Benchmark, Cifar100NasBench201Benchmark, \
     Cifar10ValidNasBench201Benchmark
 from hpobench.benchmarks.nas.nasbench_201 import \
     Cifar10ValidNasBench201MOBenchmark as LocalCifar10ValidNasBench201MOBenchmark
 from hpobench.util.container_utils import disable_container_debug, enable_container_debug
-
-skip_message = 'We currently skip this test because it takes too much time.'
+from hpobench.util.test_utils import DEFAULT_SKIP_MSG, check_run_all_tests
 
 
 @pytest.fixture(scope='module')
@@ -18,7 +15,7 @@ def enable_debug():
     disable_container_debug()
 
 
-@pytest.mark.skip(reason=skip_message)
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_nasbench201_cifar10valid(enable_debug):
 
     b = Cifar10ValidNasBench201Benchmark(rng=0)
@@ -53,7 +50,7 @@ def test_nasbench201_cifar10valid(enable_debug):
         result = b.objective_function_test(configuration=config, fidelity={'epoch': 10})
 
 
-@pytest.mark.skip(reason=skip_message)
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_nasbench201_cifar100(enable_debug):
     b = Cifar100NasBench201Benchmark(rng=0)
 
@@ -73,7 +70,7 @@ def test_nasbench201_cifar100(enable_debug):
     assert result['info']['valid_cost'] == result['cost']
 
 
-@pytest.mark.skip(reason=skip_message)
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_nasbench201_Image(enable_debug):
     b = ImageNetNasBench201Benchmark(rng=0)
     config = {'1<-0': 'nor_conv_1x1',

tests/test_pybnn.py

@@ -1,14 +1,19 @@
+import sys
 import pytest
 
 from hpobench.container.benchmarks.ml.pybnn import BNNOnToyFunction, BNNOnBostonHousing, BNNOnProteinStructure, \
     BNNOnYearPrediction
 
-import logging
-logging.basicConfig(level=logging.DEBUG)
 from hpobench.util.container_utils import enable_container_debug
+from hpobench.util.test_utils import check_run_all_tests, DEFAULT_SKIP_MSG
+
 enable_container_debug()
+MSG = 'Skip this test for new (>3.9) python versions. ' \
+      'The paramnet benchmarks require an specific old scikit learn version. This version however does not work under ' \
+      'python 3.10. Therefore we skip this test. The containerized version does still work under 3.10.'
 
 
+@pytest.mark.skipif(sys.version_info > (3, 9), reason=MSG)
 def test_bnn_init():
     benchmark = BNNOnToyFunction(rng=1)
 
@@ -58,6 +63,7 @@ def test_bnn_boston_housing():
     assert test_result['info']['fidelity']['budget'] == 1000
 
 
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_bnn_protein():
     benchmark = BNNOnProteinStructure(rng=1)
     test_result = simple_call(benchmark)
@@ -66,6 +72,7 @@ def test_bnn_protein():
     assert test_result['info']['fidelity']['budget'] == 1000
 
 
+@pytest.mark.skipif(not check_run_all_tests(), reason=DEFAULT_SKIP_MSG)
 def test_year_pred():
     benchmark = BNNOnYearPrediction(rng=1)
     test_result = simple_call(benchmark)

tests/test_utils.py

@@ -105,3 +105,15 @@ def test_debug_level():
 
     disable_container_debug()
     assert os.environ['HPOBENCH_DEBUG'] == 'false'
+
+
+def test_test_utils():
+    from hpobench.util.test_utils import DEFAULT_SKIP_MSG, enable_all_tests, disable_all_tests, check_run_all_tests
+
+    assert isinstance(DEFAULT_SKIP_MSG, str)
+
+    enable_all_tests()
+    assert check_run_all_tests()
+
+    disable_all_tests()
+    assert not check_run_all_tests()

tests/test_whitebox.py

@@ -63,6 +63,7 @@ def test_whitebox_with_container():
     assert np.isclose(test_loss, 0.43636, atol=0.001)
 
 
+@pytest.mark.skipif(skip_container_test, reason="Requires singularity and flask")
 def test_cartpole():
     from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark
     b = Benchmark(container_name='cartpole',