Merge pull request numbbo#1888 from numbbo/development

Merge development into master for release 2.3.1

.circleci/config.yml

@@ -63,7 +63,7 @@ jobs:
       - run:
           name: Install octave for octave tests
           command: |
-            apt-get install -y liboctave-dev
+            DEBIAN_FRONTEND=noninteractive apt-get install -y liboctave-dev
             updatedb
       - run:
           name: Run coco octave tests
@@ -72,7 +72,7 @@ jobs:
       - run:
           name: Install jdk for java tests
           command: |
-            apt-get install -y openjdk-8-jdk
+            apt-get install -y openjdk-11-jdk
             updatedb
       - run:
           name: Run coco octave tests
@@ -163,7 +163,7 @@ jobs:
       - run:
           name: Install jdk for java tests
           command: |
-            apt-get install -y openjdk-8-jdk
+            apt-get install -y openjdk-11-jdk
             updatedb
       - run:
           name: Run coco java tests

code-experiments/build/c/example_experiment.c

@@ -120,7 +120,7 @@ int main(void) {
    * example one of the following:
    *   bbob                 24 unconstrained noiseless single-objective functions
    *   bbob-biobj           55 unconstrained noiseless bi-objective functions
-   *   bbob-biobj-ext       92 unconstrained noiseless bi-objective functions
+   *   [bbob-biobj-ext       92 unconstrained noiseless bi-objective functions]
    *   [bbob-constrained*   48 constrained noiseless single-objective functions]
    *   bbob-largescale      24 unconstrained noiseless single-objective functions in large dimension
    *   bbob-mixint          24 unconstrained noiseless single-objective functions with mixed-integer variables

code-experiments/build/java/ExampleExperiment.java

@@ -69,7 +69,7 @@ public static void main(String[] args) {
          *
          *   bbob                 24 unconstrained noiseless single-objective functions
          *   bbob-biobj           55 unconstrained noiseless bi-objective functions
-         *   bbob-biobj-ext       92 unconstrained noiseless bi-objective functions
+         *   [bbob-biobj-ext       92 unconstrained noiseless bi-objective functions]
          *   bbob-largescale      24 unconstrained noiseless single-objective functions in large dimension
          *   [bbob-constrained*   48 constrained noiseless single-objective functions]
          *   bbob-mixint          24 unconstrained noiseless single-objective functions with mixed-integer variables

code-experiments/build/matlab/cocoProblemGetLargestFValuesOfInterest.m

@@ -0,0 +1,8 @@
+% For multi-objective problems, returns a vector of largest values of 
+% interest in each objective. Currently, this equals the nadir point. 
+% For single-objective problems it raises an error..
+%
+% Parameters:
+%    problem  The given problem.
+function ub = cocoProblemGetLargestFValuesOfInterest(problem)
+ub = cocoCall('cocoProblemGetLargestFValuesOfInterest', problem);

code-experiments/build/matlab/exampleexperiment.m

@@ -28,7 +28,7 @@
 %
 % bbob               24 unconstrained noiseless single-objective functions
 % bbob-biobj         55 unconstrained noiseless bi-objective functions
-% bbob-biobj-ext     92 unconstrained noiseless bi-objective functions
+% [bbob-biobj-ext     92 unconstrained noiseless bi-objective functions]
 % bbob-largescale    24 unconstrained noiseless single-objective functions in large dimensions
 % [bbob-constrained* 48 constrained noiseless single-objective functions]
 % bbob-mixint        24 unconstrained noiseless single-objective functions with mixed-integer variables

code-experiments/build/python/example_experiment2.py

@@ -6,7 +6,7 @@
 features restarts, timings and recording termination conditions.
 
 To benchmark a different solver, `fmin` must be re-assigned and another
-`elif` block added around line 136 to account for the solver-specific
+`elif` block added around line 119 to account for the solver-specific
 call.
 
 When calling the script, previously assigned variables can be re-assigned
@@ -38,7 +38,7 @@
 import time  # output some timings per evaluation
 from collections import defaultdict
 import os, webbrowser  # to show post-processed results in the browser
-import numpy as np  # for np.median
+import numpy as np  # for median, zeros, random, asarray
 import cocoex  # experimentation module
 try: import cocopp  # post-processing module
 except: pass
@@ -63,6 +63,7 @@ def random_search(f, lbounds, ubounds, evals):
 suite_name = "bbob"  # see cocoex.known_suite_names
 budget_multiplier = 2  # times dimension, increase to 10, 100, ...
 suite_filter_options = (# "dimensions: 2,3,5,10,20 " +  # skip dimension 40
+                        # "year:2019 " +  # select instances by year
                         # "instance_indices: 1-5 " +  # relative to suite instances
                         "")  # without filtering a suite has instance_indices 1-15
 batches = 1  # number of batches, batch=3/32 works to set both, current_batch and batches
@@ -114,8 +115,10 @@ def random_search(f, lbounds, ubounds, evals):
     while evalsleft() > 0 and not problem.final_target_hit:
         irestart += 1
 
-        # here we assume that `fmin` evaluates the final/returned solution:
-        if fmin is scipy.optimize.fmin:
+        # here we assume that `fmin` evaluates the final/returned solution
+        if 11 < 3:  # add solver to investigate here
+            pass
+        elif fmin is scipy.optimize.fmin:
             output = fmin(problem, propose_x0(), maxfun=evalsleft(), disp=False, full_output=True)
             stoppings[problem.index].append(output[4])
         elif fmin is scipy.optimize.fmin_slsqp:
@@ -132,8 +135,6 @@ def random_search(f, lbounds, ubounds, evals):
         elif fmin is scipy.optimize.fmin_cobyla:
             fmin(problem, propose_x0(), lambda x: -problem.constraint(x), maxfun=evalsleft(),
                  disp=0, rhoend=1e-9)
-        else: # add another solver here
-            raise NotImplementedError
 
     timings[problem.dimension].append((time.time() - time1) / problem.evaluations
                                       if problem.evaluations else 0)

code-experiments/tools/cocoutils.py

@@ -9,6 +9,7 @@
 
 import sys
 import os
+from distutils.spawn import find_executable
 from shutil import copyfile, copytree, rmtree
 from subprocess import CalledProcessError, call, STDOUT
 
@@ -225,3 +226,10 @@ def expand_file(source, destination, dictionary):
         content = Template(fd.read())
         with open(destination, "w") as outfd:
             outfd.write(content.safe_substitute(dictionary))
+
+
+def executable_path(name):
+    p = find_executable(name)
+    if p:
+        return os.path.realpath(p)
+

code-postprocessing/cocopp/archiving.py

@@ -27,7 +27,7 @@
 publish it together with the URL, such that everyone can use the archive
 on the fly like
 
->>> remote_def = 'http://my-coco-online-archive/archive_definition.txt'
+>>> remote_def = 'http://my-coco-online-archive/coco_archive_definition.txt'
 >>> new_archive = create_from_remote('~/.cocopp/new-archives/unique-name',
 ...                                  remote_def)  # doctest:+SKIP
 
@@ -59,7 +59,7 @@
 except ImportError:
     from urllib import urlretrieve
 
-default_definition_filename = 'archive_definition.txt'
+default_definition_filename = 'coco_archive_definition.txt'
 
 
 def _abs_path(path):
@@ -133,7 +133,7 @@ def read_definition_file(local_path_or_definition_file):
         return ast.literal_eval(file_.read())
 
 def create_from_remote(local_path, url_definition_file):
-    """copy a definition file from url to ``local_path/archive_definition.txt``.
+    """copy a definition file from url to ``local_path/coco_archive_definition.txt``.
 
     `local_path` is the storage location and should (better) be empty.
     It is created if it does not exist.

code-postprocessing/cocopp/bbob-biobj-benchmarkshortinfos.txt

@@ -1,58 +1,58 @@
 1 Sphere/Sphere
 2 Sphere/sep. Ellipsoid
-3 Sphere/Attractive sector
+3 Sphere/Attr. sector
 4 Sphere/Rosenbrock
 5 Sphere/Sharp ridge
 6 Sphere/Different Powers
 7 Sphere/Rastrigin
 8 Sphere/Schaffer F7
 9 Sphere/Schwefel
 10 Sphere/Gallagher 101
-11 sep. Ellipsoid/sep. Ellipsoid
-12 sep. Ellipsoid/Attractive sector
+11 sep. Ellipsoid/sep. Elli.
+12 sep. Ellipsoid/Attr. sector
 13 sep. Ellipsoid/Rosenbrock
 14 sep. Ellipsoid/Sharp ridge
-15 sep. Ellipsoid/Different Powers
+15 sep. Ellipsoid/Diff. Powers
 16 sep. Ellipsoid/Rastrigin
 17 sep. Ellipsoid/Schaffer F7
 18 sep. Ellipsoid/Schwefel
-19 sep. Ellipsoid/Gallagher 101
-20 Attractive sector/Attractive sector
-21 Attractive sector/Rosenbrock
-22 Attractive sector/Sharp ridge
-23 Attractive sector/Different Powers
-24 Attractive sector/Rastrigin
-25 Attractive sector/Schaffer F7
-26 Attractive sector/Schwefel
-27 Attractive sector/Gallagher 101
+19 sep. Elli./Gallagher 101
+20 Attr. sector/Attr. sector
+21 Attr. sector/Rosenbrock
+22 Attr. sector/Sharp ridge
+23 Attr. sector/Diff. Powers
+24 Attr. sector/Rastrigin
+25 Attr. sector/Schaffer F7
+26 Attr. sector/Schwefel
+27 Attr. sector/Gallagher 101
 28 Rosenbrock/Rosenbrock
 29 Rosenbrock/Sharp ridge
-30 Rosenbrock/Different Powers
+30 Rosenbrock/Diff. Powers
 31 Rosenbrock/Rastrigin
 32 Rosenbrock/Schaffer F7
 33 Rosenbrock/Schwefel
 34 Rosenbrock/Gallagher 101
 35 Sharp ridge/Sharp ridge
-36 Sharp ridge/Different Powers
+36 Sharp ridge/Diff. Powers
 37 Sharp ridge/Rastrigin
 38 Sharp ridge/Schaffer F7
 39 Sharp ridge/Schwefel
 40 Sharp ridge/Gallagher 101
-41 Different Powers/Different Powers
-42 Different Powers/Rastrigin
-43 Different Powers/Schaffer F7
-44 Different Powers/Schwefel
-45 Different Powers/Gallagher 101
+41 Diff. Powers/Diff. Powers
+42 Diff. Powers/Rastrigin
+43 Diff. Powers/Schaffer F7
+44 Diff. Powers/Schwefel
+45 Diff. Powers/Gallagher 101
 46 Rastrigin/Rastrigin
 47 Rastrigin/Schaffer F7
 48 Rastrigin/Schwefel
 49 Rastrigin/Gallagher 101
 50 Schaffer F7/Schaffer F7
 51 Schaffer F7/Schwefel
-52 Schaffer F7/Gallagher 101
+52 Schaffer F7/Gallagh. 101
 53 Schwefel/Schwefel
 54 Schwefel/Gallagher 101
-55 Gallagher 101/Gallagher 101
+55 Gallagher 101/Gallagh. 101
 56 Sphere/sep. Rastrigin
 57 Sphere/Skew Rastrigin-Bueche
 58 Sphere/Linear slope
@@ -70,12 +70,12 @@
 70 Ellipsoid/Discus
 71 Ellipsoid/Bent cigar
 72 Ellipsoid/Sharp ridge
-73 Ellipsoid/Different Powers
+73 Ellipsoid/Diff. Powers
 74 Discus/Bent cigar
 75 Discus/Sharp ridge
-76 Discus/Different Powers
+76 Discus/Diff. Powers
 77 Bent cigar/Sharp ridge
-78 Bent cigar/Different Powers
+78 Bent cigar/Diff. Powers
 79 Rastrigin/Schaffer F7 cond 1000
 80 Rastrigin/Griewank-Rosenbrock
 81 Schaffer F7/Schaffer F7 cond 1000

code-postprocessing/cocopp/compall/pprldmany.py

@@ -420,10 +420,19 @@ def plot(dsList, targets=None, craftingeffort=0., **kwargs):
             evals = entry.detEvals([t])[0]
             runlengthsucc = evals[np.isnan(evals) == False] / divisor
             runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
-            if len(runlengthsucc) > 0:
-                x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
-                                      percentiles=[50],
-                                      samplesize=perfprofsamplesize)[1]
+            if len(runlengthsucc) > 0:  # else x == [inf, inf,...]
+                if testbedsettings.current_testbed.instances_are_uniform:
+                    x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
+                                          percentiles=[50],
+                                          samplesize=perfprofsamplesize)[1]
+                else:
+                    nruns = len(runlengthsucc) + len(runlengthunsucc)
+                    if perfprofsamplesize % nruns:
+                        warnings.warn("without simulated restarts nbsamples=%d"
+                                      " should be a multiple of nbruns=%d"
+                                      % (perfprofsamplesize, nruns))
+                    idx = toolsstats.randint_derandomized(nruns, size=perfprofsamplesize)
+                    x = np.hstack((runlengthsucc, len(runlengthunsucc) * [np.inf]))[idx]
             data.extend(x)
             maxevals.extend(runlengthunsucc)
 
@@ -636,10 +645,19 @@ def main(dictAlg, order=None, outputdir='.', info='default',
                         assert entry.dim == dim
                         runlengthsucc = evals[np.isnan(evals) == False] / divisor
                         runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
-                        if len(runlengthsucc) > 0:
-                            x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
-                                                  percentiles=[50],
-                                                  samplesize=perfprofsamplesize)[1]
+                        if len(runlengthsucc) > 0:  # else x == [inf, inf,...]
+                            if testbedsettings.current_testbed.instances_are_uniform:
+                                x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
+                                                      percentiles=[50],
+                                                      samplesize=perfprofsamplesize)[1]
+                            else:
+                                nruns = len(runlengthsucc) + len(runlengthunsucc)
+                                if perfprofsamplesize % nruns:
+                                    warnings.warn("without simulated restarts nbsamples=%d"
+                                                  " should be a multiple of nbruns=%d"
+                                                  % (perfprofsamplesize, nruns))
+                                idx = toolsstats.randint_derandomized(nruns, size=perfprofsamplesize)
+                                x = np.hstack((runlengthsucc, len(runlengthunsucc) * [np.inf]))[idx]
                     except (KeyError, IndexError):
                         # set_trace()
                         warntxt = ('Data for algorithm %s on function %d in %d-D '

code-postprocessing/cocopp/findfiles.py

@@ -97,7 +97,7 @@ def get_directory(directory, extract_files):
                     print('    archive extracted to folder', dir_name, '...')
             directory = dir_name
         else: # i.e. either directory or .tar or zipped .tar
-            head, tail = os.path.split(directory[:directory.find('.t')])
+            head, tail = os.path.split(directory[:directory.rfind('.t')])
             dir_name = os.path.join(head, genericsettings.extraction_folder_prefix + tail)
             # extract only if extracted folder does not exist yet or if it was
             # extracted earlier than last change of archive:

code-postprocessing/cocopp/ppfig.py

@@ -266,15 +266,15 @@ def get_rld_link(current_dir):
 
     file_name = '%s.html' % genericsettings.pprldmany_file_name
     links += add_link(current_dir, folder, file_name,
-                      pprldmany_per_func_dim_header, dimension=20)
+                      pprldmany_per_func_dim_header, dimension=testbedsettings.current_testbed.goto_dimension)
 
     file_name = '%s.html' % genericsettings.pprldmany_group_file_name
     links += add_link(current_dir, folder, file_name,
-                      pprldmany_per_group_dim_header, dimension=20)
+                      pprldmany_per_group_dim_header, dimension=testbedsettings.current_testbed.goto_dimension)
 
     file_name = '%s.html' % genericsettings.pprldmany_file_name
     links += add_link(current_dir, '', file_name,
-                      pprldmany_per_group_dim_header, dimension=20)
+                      pprldmany_per_group_dim_header, dimension=testbedsettings.current_testbed.goto_dimension)
 
     return links
 

code-postprocessing/cocopp/rungenericmany.py

@@ -289,6 +289,11 @@ def main(argv=None):
         print("  loading data...")
         dsList, sortedAlgs, dictAlg = processInputArgs(args, True)
 
+        # filter data set lists to be compliant with all suites
+        # (useful right now only for bbob-biobj and bbob-biobj-ext data)
+        dsList = testbedsettings.current_testbed.filter(dsList)
+        dictAlg = testbedsettings.current_testbed.filter(dictAlg)
+
         if not dsList:
             sys.exit()
 

code-postprocessing/cocopp/test.py

@@ -356,6 +356,7 @@ def main(arguments):
             print('**  subtest 10 finished in ', time.time() - t0, ' seconds')
             assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob`.'
 
+        with InfolderGoneWithTheWind():
             t0 = time.time()
             result = os.system(python + command +
                                join_path(recent_data_path, 'RS-bi'))
@@ -369,24 +370,28 @@ def main(arguments):
             # assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob-constrained`.'
             # delete_files(all_files=True)
 
+        with InfolderGoneWithTheWind():
             t0 = time.time()
             result = os.system(python + command + data_archive_get(
                 'test/RS-4.zip'))
             print('**  subtest 13 finished in ', time.time() - t0, ' seconds')
             assert result == 0, 'Test failed: rungeneric on RS-4.zip.'
 
+        with InfolderGoneWithTheWind():
             t0 = time.time()
             result = os.system(python + command +
                                join_path(recent_data_path, 'RS-la'))
             print('**  subtest 14 finished in ', time.time() - t0, ' seconds')
             assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob-largescale`.'
 
+        with InfolderGoneWithTheWind():
             t0 = time.time()
             result = os.system(python + command +
                                join_path(recent_data_path, 'RS-mi'))
             print('**  subtest 15 finished in ', time.time() - t0, ' seconds')
             assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob-mixint`.'
 
+        with InfolderGoneWithTheWind():
             t0 = time.time()
             result = os.system(python + command +
                                join_path(recent_data_path, 'RS-bi-mi'))