diff --git a/.circleci/config.yml b/.circleci/config.yml index ef4313010..c70ac9ead 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,7 +63,7 @@ jobs: - run: name: Install octave for octave tests command: | - apt-get install -y liboctave-dev + DEBIAN_FRONTEND=noninteractive apt-get install -y liboctave-dev updatedb - run: name: Run coco octave tests @@ -72,7 +72,7 @@ jobs: - run: name: Install jdk for java tests command: | - apt-get install -y openjdk-8-jdk + apt-get install -y openjdk-11-jdk updatedb - run: name: Run coco octave tests @@ -163,7 +163,7 @@ jobs: - run: name: Install jdk for java tests command: | - apt-get install -y openjdk-8-jdk + apt-get install -y openjdk-11-jdk updatedb - run: name: Run coco java tests diff --git a/code-experiments/build/c/example_experiment.c b/code-experiments/build/c/example_experiment.c index 18f64f330..0c69e65a1 100644 --- a/code-experiments/build/c/example_experiment.c +++ b/code-experiments/build/c/example_experiment.c @@ -120,7 +120,7 @@ int main(void) { * example one of the following: * bbob 24 unconstrained noiseless single-objective functions * bbob-biobj 55 unconstrained noiseless bi-objective functions - * bbob-biobj-ext 92 unconstrained noiseless bi-objective functions + * [bbob-biobj-ext 92 unconstrained noiseless bi-objective functions] * [bbob-constrained* 48 constrained noiseless single-objective functions] * bbob-largescale 24 unconstrained noiseless single-objective functions in large dimension * bbob-mixint 24 unconstrained noiseless single-objective functions with mixed-integer variables diff --git a/code-experiments/build/java/ExampleExperiment.java b/code-experiments/build/java/ExampleExperiment.java index b2869ce7b..d28c1f67c 100644 --- a/code-experiments/build/java/ExampleExperiment.java +++ b/code-experiments/build/java/ExampleExperiment.java @@ -69,7 +69,7 @@ public static void main(String[] args) { * * bbob 24 unconstrained noiseless single-objective functions * bbob-biobj 55 unconstrained noiseless bi-objective functions - * bbob-biobj-ext 92 unconstrained noiseless bi-objective functions + * [bbob-biobj-ext 92 unconstrained noiseless bi-objective functions] * bbob-largescale 24 unconstrained noiseless single-objective functions in large dimension * [bbob-constrained* 48 constrained noiseless single-objective functions] * bbob-mixint 24 unconstrained noiseless single-objective functions with mixed-integer variables diff --git a/code-experiments/build/matlab/cocoProblemGetLargestFValuesOfInterest.m b/code-experiments/build/matlab/cocoProblemGetLargestFValuesOfInterest.m new file mode 100644 index 000000000..3cb6eee72 --- /dev/null +++ b/code-experiments/build/matlab/cocoProblemGetLargestFValuesOfInterest.m @@ -0,0 +1,8 @@ +% For multi-objective problems, returns a vector of largest values of +% interest in each objective. Currently, this equals the nadir point. +% For single-objective problems it raises an error.. +% +% Parameters: +% problem The given problem. +function ub = cocoProblemGetLargestFValuesOfInterest(problem) +ub = cocoCall('cocoProblemGetLargestFValuesOfInterest', problem); \ No newline at end of file diff --git a/code-experiments/build/matlab/exampleexperiment.m b/code-experiments/build/matlab/exampleexperiment.m index 384404b5b..978e0473d 100644 --- a/code-experiments/build/matlab/exampleexperiment.m +++ b/code-experiments/build/matlab/exampleexperiment.m @@ -28,7 +28,7 @@ % % bbob 24 unconstrained noiseless single-objective functions % bbob-biobj 55 unconstrained noiseless bi-objective functions -% bbob-biobj-ext 92 unconstrained noiseless bi-objective functions +% [bbob-biobj-ext 92 unconstrained noiseless bi-objective functions] % bbob-largescale 24 unconstrained noiseless single-objective functions in large dimensions % [bbob-constrained* 48 constrained noiseless single-objective functions] % bbob-mixint 24 unconstrained noiseless single-objective functions with mixed-integer variables diff --git a/code-experiments/build/python/example_experiment2.py b/code-experiments/build/python/example_experiment2.py index ca519e6a1..c6d6d69c3 100755 --- a/code-experiments/build/python/example_experiment2.py +++ b/code-experiments/build/python/example_experiment2.py @@ -6,7 +6,7 @@ features restarts, timings and recording termination conditions. To benchmark a different solver, `fmin` must be re-assigned and another -`elif` block added around line 136 to account for the solver-specific +`elif` block added around line 119 to account for the solver-specific call. When calling the script, previously assigned variables can be re-assigned @@ -38,7 +38,7 @@ import time # output some timings per evaluation from collections import defaultdict import os, webbrowser # to show post-processed results in the browser -import numpy as np # for np.median +import numpy as np # for median, zeros, random, asarray import cocoex # experimentation module try: import cocopp # post-processing module except: pass @@ -63,6 +63,7 @@ def random_search(f, lbounds, ubounds, evals): suite_name = "bbob" # see cocoex.known_suite_names budget_multiplier = 2 # times dimension, increase to 10, 100, ... suite_filter_options = (# "dimensions: 2,3,5,10,20 " + # skip dimension 40 + # "year:2019 " + # select instances by year # "instance_indices: 1-5 " + # relative to suite instances "") # without filtering a suite has instance_indices 1-15 batches = 1 # number of batches, batch=3/32 works to set both, current_batch and batches @@ -114,8 +115,10 @@ def random_search(f, lbounds, ubounds, evals): while evalsleft() > 0 and not problem.final_target_hit: irestart += 1 - # here we assume that `fmin` evaluates the final/returned solution: - if fmin is scipy.optimize.fmin: + # here we assume that `fmin` evaluates the final/returned solution + if 11 < 3: # add solver to investigate here + pass + elif fmin is scipy.optimize.fmin: output = fmin(problem, propose_x0(), maxfun=evalsleft(), disp=False, full_output=True) stoppings[problem.index].append(output[4]) elif fmin is scipy.optimize.fmin_slsqp: @@ -132,8 +135,6 @@ def random_search(f, lbounds, ubounds, evals): elif fmin is scipy.optimize.fmin_cobyla: fmin(problem, propose_x0(), lambda x: -problem.constraint(x), maxfun=evalsleft(), disp=0, rhoend=1e-9) - else: # add another solver here - raise NotImplementedError timings[problem.dimension].append((time.time() - time1) / problem.evaluations if problem.evaluations else 0) diff --git a/code-experiments/tools/cocoutils.py b/code-experiments/tools/cocoutils.py index b3a94baa9..348e68c9b 100644 --- a/code-experiments/tools/cocoutils.py +++ b/code-experiments/tools/cocoutils.py @@ -9,6 +9,7 @@ import sys import os +from distutils.spawn import find_executable from shutil import copyfile, copytree, rmtree from subprocess import CalledProcessError, call, STDOUT @@ -225,3 +226,10 @@ def expand_file(source, destination, dictionary): content = Template(fd.read()) with open(destination, "w") as outfd: outfd.write(content.safe_substitute(dictionary)) + + +def executable_path(name): + p = find_executable(name) + if p: + return os.path.realpath(p) + diff --git a/code-postprocessing/cocopp/archiving.py b/code-postprocessing/cocopp/archiving.py index 9ddeb8b1b..569806082 100644 --- a/code-postprocessing/cocopp/archiving.py +++ b/code-postprocessing/cocopp/archiving.py @@ -27,7 +27,7 @@ publish it together with the URL, such that everyone can use the archive on the fly like ->>> remote_def = 'http://my-coco-online-archive/archive_definition.txt' +>>> remote_def = 'http://my-coco-online-archive/coco_archive_definition.txt' >>> new_archive = create_from_remote('~/.cocopp/new-archives/unique-name', ... remote_def) # doctest:+SKIP @@ -59,7 +59,7 @@ except ImportError: from urllib import urlretrieve -default_definition_filename = 'archive_definition.txt' +default_definition_filename = 'coco_archive_definition.txt' def _abs_path(path): @@ -133,7 +133,7 @@ def read_definition_file(local_path_or_definition_file): return ast.literal_eval(file_.read()) def create_from_remote(local_path, url_definition_file): - """copy a definition file from url to ``local_path/archive_definition.txt``. + """copy a definition file from url to ``local_path/coco_archive_definition.txt``. `local_path` is the storage location and should (better) be empty. It is created if it does not exist. diff --git a/code-postprocessing/cocopp/bbob-biobj-benchmarkshortinfos.txt b/code-postprocessing/cocopp/bbob-biobj-benchmarkshortinfos.txt index 730b182d1..63bdfd426 100644 --- a/code-postprocessing/cocopp/bbob-biobj-benchmarkshortinfos.txt +++ b/code-postprocessing/cocopp/bbob-biobj-benchmarkshortinfos.txt @@ -1,6 +1,6 @@ 1 Sphere/Sphere 2 Sphere/sep. Ellipsoid -3 Sphere/Attractive sector +3 Sphere/Attr. sector 4 Sphere/Rosenbrock 5 Sphere/Sharp ridge 6 Sphere/Different Powers @@ -8,51 +8,51 @@ 8 Sphere/Schaffer F7 9 Sphere/Schwefel 10 Sphere/Gallagher 101 -11 sep. Ellipsoid/sep. Ellipsoid -12 sep. Ellipsoid/Attractive sector +11 sep. Ellipsoid/sep. Elli. +12 sep. Ellipsoid/Attr. sector 13 sep. Ellipsoid/Rosenbrock 14 sep. Ellipsoid/Sharp ridge -15 sep. Ellipsoid/Different Powers +15 sep. Ellipsoid/Diff. Powers 16 sep. Ellipsoid/Rastrigin 17 sep. Ellipsoid/Schaffer F7 18 sep. Ellipsoid/Schwefel -19 sep. Ellipsoid/Gallagher 101 -20 Attractive sector/Attractive sector -21 Attractive sector/Rosenbrock -22 Attractive sector/Sharp ridge -23 Attractive sector/Different Powers -24 Attractive sector/Rastrigin -25 Attractive sector/Schaffer F7 -26 Attractive sector/Schwefel -27 Attractive sector/Gallagher 101 +19 sep. Elli./Gallagher 101 +20 Attr. sector/Attr. sector +21 Attr. sector/Rosenbrock +22 Attr. sector/Sharp ridge +23 Attr. sector/Diff. Powers +24 Attr. sector/Rastrigin +25 Attr. sector/Schaffer F7 +26 Attr. sector/Schwefel +27 Attr. sector/Gallagher 101 28 Rosenbrock/Rosenbrock 29 Rosenbrock/Sharp ridge -30 Rosenbrock/Different Powers +30 Rosenbrock/Diff. Powers 31 Rosenbrock/Rastrigin 32 Rosenbrock/Schaffer F7 33 Rosenbrock/Schwefel 34 Rosenbrock/Gallagher 101 35 Sharp ridge/Sharp ridge -36 Sharp ridge/Different Powers +36 Sharp ridge/Diff. Powers 37 Sharp ridge/Rastrigin 38 Sharp ridge/Schaffer F7 39 Sharp ridge/Schwefel 40 Sharp ridge/Gallagher 101 -41 Different Powers/Different Powers -42 Different Powers/Rastrigin -43 Different Powers/Schaffer F7 -44 Different Powers/Schwefel -45 Different Powers/Gallagher 101 +41 Diff. Powers/Diff. Powers +42 Diff. Powers/Rastrigin +43 Diff. Powers/Schaffer F7 +44 Diff. Powers/Schwefel +45 Diff. Powers/Gallagher 101 46 Rastrigin/Rastrigin 47 Rastrigin/Schaffer F7 48 Rastrigin/Schwefel 49 Rastrigin/Gallagher 101 50 Schaffer F7/Schaffer F7 51 Schaffer F7/Schwefel -52 Schaffer F7/Gallagher 101 +52 Schaffer F7/Gallagh. 101 53 Schwefel/Schwefel 54 Schwefel/Gallagher 101 -55 Gallagher 101/Gallagher 101 +55 Gallagher 101/Gallagh. 101 56 Sphere/sep. Rastrigin 57 Sphere/Skew Rastrigin-Bueche 58 Sphere/Linear slope @@ -70,12 +70,12 @@ 70 Ellipsoid/Discus 71 Ellipsoid/Bent cigar 72 Ellipsoid/Sharp ridge -73 Ellipsoid/Different Powers +73 Ellipsoid/Diff. Powers 74 Discus/Bent cigar 75 Discus/Sharp ridge -76 Discus/Different Powers +76 Discus/Diff. Powers 77 Bent cigar/Sharp ridge -78 Bent cigar/Different Powers +78 Bent cigar/Diff. Powers 79 Rastrigin/Schaffer F7 cond 1000 80 Rastrigin/Griewank-Rosenbrock 81 Schaffer F7/Schaffer F7 cond 1000 diff --git a/code-postprocessing/cocopp/compall/pprldmany.py b/code-postprocessing/cocopp/compall/pprldmany.py index 079dbfcfd..95322a895 100644 --- a/code-postprocessing/cocopp/compall/pprldmany.py +++ b/code-postprocessing/cocopp/compall/pprldmany.py @@ -420,10 +420,19 @@ def plot(dsList, targets=None, craftingeffort=0., **kwargs): evals = entry.detEvals([t])[0] runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor - if len(runlengthsucc) > 0: - x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, - percentiles=[50], - samplesize=perfprofsamplesize)[1] + if len(runlengthsucc) > 0: # else x == [inf, inf,...] + if testbedsettings.current_testbed.instances_are_uniform: + x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, + percentiles=[50], + samplesize=perfprofsamplesize)[1] + else: + nruns = len(runlengthsucc) + len(runlengthunsucc) + if perfprofsamplesize % nruns: + warnings.warn("without simulated restarts nbsamples=%d" + " should be a multiple of nbruns=%d" + % (perfprofsamplesize, nruns)) + idx = toolsstats.randint_derandomized(nruns, size=perfprofsamplesize) + x = np.hstack((runlengthsucc, len(runlengthunsucc) * [np.inf]))[idx] data.extend(x) maxevals.extend(runlengthunsucc) @@ -636,10 +645,19 @@ def main(dictAlg, order=None, outputdir='.', info='default', assert entry.dim == dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor - if len(runlengthsucc) > 0: - x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, - percentiles=[50], - samplesize=perfprofsamplesize)[1] + if len(runlengthsucc) > 0: # else x == [inf, inf,...] + if testbedsettings.current_testbed.instances_are_uniform: + x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, + percentiles=[50], + samplesize=perfprofsamplesize)[1] + else: + nruns = len(runlengthsucc) + len(runlengthunsucc) + if perfprofsamplesize % nruns: + warnings.warn("without simulated restarts nbsamples=%d" + " should be a multiple of nbruns=%d" + % (perfprofsamplesize, nruns)) + idx = toolsstats.randint_derandomized(nruns, size=perfprofsamplesize) + x = np.hstack((runlengthsucc, len(runlengthunsucc) * [np.inf]))[idx] except (KeyError, IndexError): # set_trace() warntxt = ('Data for algorithm %s on function %d in %d-D ' diff --git a/code-postprocessing/cocopp/findfiles.py b/code-postprocessing/cocopp/findfiles.py index c6526820e..1b13106c6 100644 --- a/code-postprocessing/cocopp/findfiles.py +++ b/code-postprocessing/cocopp/findfiles.py @@ -97,7 +97,7 @@ def get_directory(directory, extract_files): print(' archive extracted to folder', dir_name, '...') directory = dir_name else: # i.e. either directory or .tar or zipped .tar - head, tail = os.path.split(directory[:directory.find('.t')]) + head, tail = os.path.split(directory[:directory.rfind('.t')]) dir_name = os.path.join(head, genericsettings.extraction_folder_prefix + tail) # extract only if extracted folder does not exist yet or if it was # extracted earlier than last change of archive: diff --git a/code-postprocessing/cocopp/ppfig.py b/code-postprocessing/cocopp/ppfig.py index 11a30aa1e..5890c3ecd 100644 --- a/code-postprocessing/cocopp/ppfig.py +++ b/code-postprocessing/cocopp/ppfig.py @@ -266,15 +266,15 @@ def get_rld_link(current_dir): file_name = '%s.html' % genericsettings.pprldmany_file_name links += add_link(current_dir, folder, file_name, - pprldmany_per_func_dim_header, dimension=20) + pprldmany_per_func_dim_header, dimension=testbedsettings.current_testbed.goto_dimension) file_name = '%s.html' % genericsettings.pprldmany_group_file_name links += add_link(current_dir, folder, file_name, - pprldmany_per_group_dim_header, dimension=20) + pprldmany_per_group_dim_header, dimension=testbedsettings.current_testbed.goto_dimension) file_name = '%s.html' % genericsettings.pprldmany_file_name links += add_link(current_dir, '', file_name, - pprldmany_per_group_dim_header, dimension=20) + pprldmany_per_group_dim_header, dimension=testbedsettings.current_testbed.goto_dimension) return links diff --git a/code-postprocessing/cocopp/rungenericmany.py b/code-postprocessing/cocopp/rungenericmany.py index f92ddf97c..11a4e70bf 100644 --- a/code-postprocessing/cocopp/rungenericmany.py +++ b/code-postprocessing/cocopp/rungenericmany.py @@ -289,6 +289,11 @@ def main(argv=None): print(" loading data...") dsList, sortedAlgs, dictAlg = processInputArgs(args, True) + # filter data set lists to be compliant with all suites + # (useful right now only for bbob-biobj and bbob-biobj-ext data) + dsList = testbedsettings.current_testbed.filter(dsList) + dictAlg = testbedsettings.current_testbed.filter(dictAlg) + if not dsList: sys.exit() diff --git a/code-postprocessing/cocopp/test.py b/code-postprocessing/cocopp/test.py index d4759e33b..0d649343f 100644 --- a/code-postprocessing/cocopp/test.py +++ b/code-postprocessing/cocopp/test.py @@ -356,6 +356,7 @@ def main(arguments): print('** subtest 10 finished in ', time.time() - t0, ' seconds') assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob`.' + with InfolderGoneWithTheWind(): t0 = time.time() result = os.system(python + command + join_path(recent_data_path, 'RS-bi')) @@ -369,24 +370,28 @@ def main(arguments): # assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob-constrained`.' # delete_files(all_files=True) + with InfolderGoneWithTheWind(): t0 = time.time() result = os.system(python + command + data_archive_get( 'test/RS-4.zip')) print('** subtest 13 finished in ', time.time() - t0, ' seconds') assert result == 0, 'Test failed: rungeneric on RS-4.zip.' + with InfolderGoneWithTheWind(): t0 = time.time() result = os.system(python + command + join_path(recent_data_path, 'RS-la')) print('** subtest 14 finished in ', time.time() - t0, ' seconds') assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob-largescale`.' + with InfolderGoneWithTheWind(): t0 = time.time() result = os.system(python + command + join_path(recent_data_path, 'RS-mi')) print('** subtest 15 finished in ', time.time() - t0, ' seconds') assert result == 0, 'Test failed: rungeneric on newly generated random search data on `bbob-mixint`.' + with InfolderGoneWithTheWind(): t0 = time.time() result = os.system(python + command + join_path(recent_data_path, 'RS-bi-mi')) diff --git a/code-postprocessing/cocopp/testbedsettings.py b/code-postprocessing/cocopp/testbedsettings.py index 5257be3d1..8d45c0f55 100644 --- a/code-postprocessing/cocopp/testbedsettings.py +++ b/code-postprocessing/cocopp/testbedsettings.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import os import numpy as np import warnings @@ -165,6 +167,8 @@ class Testbed(object): """ reference_algorithm_displayname = None + instances_are_uniform = True + "False for biobjective suites, used (so far only) for simulated restarts in pprldmany" def info(self, fun_number=None): """info on the testbed if ``fun_number is None`` or one-line info @@ -193,6 +197,19 @@ def instantiate_attributes(self, class_, suffix_list=['target_values', 'targetsO if name.endswith(suffix): setattr(self, name, class_(getattr(self, name))) + def filter(self, dsl): + """ + Returns a new DataSetList from DataSetList dsl that is + consistent with the Testbed class. Works in the same manner + for a dictionary dsl of DataSetLists by directly removing + the corresponding entries from the DataSetLists. + + Right now only used for making bbob-biobj and bbob-biobj-ext suites + consistent. Here implemented as a stub. + """ + return dsl + + class GECCOBBOBTestbed(Testbed): """Testbed used in the GECCO BBOB workshops 2009, 2010, 2012, 2013, 2015, @@ -210,6 +227,7 @@ class GECCOBBOBTestbed(Testbed): name=testbed_name_single, short_names=get_short_names(shortinfo_filename), dimensions_to_display=(2, 3, 5, 10, 20, 40), + goto_dimension=20, # auto-focus on this dimension in html rldDimsOfInterest=dimsOfInterest, tabDimsOfInterest=dimsOfInterest, hardesttargetlatex='10^{-8}', # used for ppfigs, pptable and pptables @@ -369,8 +387,6 @@ def __init__(self, target_values): self.instantiate_attributes(target_values, [key]) - - class GECCOBiObjBBOBTestbed(Testbed): """Testbed used in the BBOB workshops to display data sets run on the `bbob-biobj` test suite. @@ -387,6 +403,7 @@ class GECCOBiObjBBOBTestbed(Testbed): name=testbed_name_bi, short_names=get_short_names(shortinfo_filename), dimensions_to_display=(2, 3, 5, 10, 20, 40), + goto_dimension=20, # auto-focus on this dimension in html rldDimsOfInterest=dimsOfInterest, tabDimsOfInterest=dimsOfInterest, hardesttargetlatex='10^{-5}', # used for ppfigs, pptable and pptables @@ -396,6 +413,7 @@ class GECCOBiObjBBOBTestbed(Testbed): pprldistr_target_values=(1e-1, 1e-2, 1e-3, 1e-5), pprldmany_target_values= np.append(np.append(10 ** np.arange(0, -5.1, -0.1), [0]), -10 ** np.arange(-5, -3.9, 0.2)), + instances_are_uniform = False, pprldmany_target_range_latex='$\{-10^{-4}, -10^{-4.2}, $ $-10^{-4.4}, -10^{-4.6}, -10^{-4.8}, -10^{-5}, 0, 10^{-5}, 10^{-4.9}, 10^{-4.8}, \dots, 10^{-0.1}, 10^0\}$', ppscatter_target_values=np.logspace(-5, 1, 21), # 21 was 51 rldValsOfInterest=(1e-1, 1e-2, 1e-3, 1e-4, 1e-5), @@ -438,6 +456,40 @@ def __init__(self, targetValues): # self.short_names = get_short_names(self.shortinfo_filename) self.instancesOfInterest = {1: 1, 2: 1, 3: 1, 4: 1, 5: 1} + def filter(self, dsl): + """ Returns a new DataSetList from DataSetList dsl that does only have the + first 55 functions, in case the original data is from either + the bbob-biobj or the bbob-biobj-ext suite. Filters in a similar + manner by removing directly from dsl if dsl is an algorithm dictionary. + + Gives an error if the data is not compatible. + """ + + if type(dsl) is list: + new_dsl = [] + for ds in dsl: + if not ds.get_suite() in ['bbob-biobj', 'bbob-biobj-ext']: + raise ValueError("Data from %s suite is not " + "compatible with other data from " + "the bbob-biobj and/or bbob-biobj-ext " + "suites" % str(ds.suite)) + else: + if ds.funcId <= 55: + new_dsl.append(ds) + return new_dsl + elif type(dsl) is dict: + for algname in dsl: + for i in range(len(dsl[algname]) - 1, -1, -1): + if not (dsl[algname])[i].get_suite() in ['bbob-biobj', 'bbob-biobj-ext']: + raise ValueError("Data from %s suite is not " + "compatible with other data from " + "the bbob-biobj and/or bbob-biobj-ext " + "suites" % str((dsl[algname])[i].suite)) + else: + if ((dsl[algname])[i]).funcId > 55: + dsl[algname].pop(i) + return dsl + class GECCOBiObjExtBBOBTestbed(GECCOBiObjBBOBTestbed): """Biobjective testbed to display data sets run on the `bbob-biobj-ext` @@ -458,7 +510,7 @@ class GECCOBiObjExtBBOBTestbed(GECCOBiObjBBOBTestbed): reference_algorithm_filename='', # TODO produce correct best2017 algo and delete this line reference_algorithm_displayname='', # TODO: should be read in from data set in reference_algorithm_filename instancesOfInterest={1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1, 10: 1}, # None:consider all instances - ) + ) def __init__(self, targetValues): super(GECCOBiObjExtBBOBTestbed, self).__init__(targetValues) @@ -491,6 +543,7 @@ class BBOBLargeScaleTestbed(GECCOBBOBTestbed): name=testbed_name_ls, short_names=get_short_names(shortinfo_filename), dimensions_to_display=(20, 40, 80, 160, 320, 640), + goto_dimension=160, # auto-focus on this dimension in html tabDimsOfInterest=dimsOfInterest, rldDimsOfInterest=dimsOfInterest, hardesttargetlatex='10^{-8}', # used for ppfigs, pptable and pptables @@ -550,6 +603,7 @@ class GECCOBBOBMixintTestbed(GECCOBBOBTestbed): name=testbed_name_mixint, first_dimension=5, dimensions_to_display=[5, 10, 20, 40, 80, 160], + goto_dimension=40, # auto-focus on this dimension in html tabDimsOfInterest=dimsOfInterest, rldDimsOfInterest=dimsOfInterest, reference_algorithm_filename=None, # TODO produce correct reference algo and update this line @@ -578,6 +632,8 @@ class GECCOBBOBBiObjMixintTestbed(GECCOBiObjExtBBOBTestbed): name=testbed_name_bi_mixint, first_dimension=5, dimensions_to_display=[5, 10, 20, 40, 80, 160], + goto_dimension=40, # auto-focus on this dimension in html + instances_are_uniform=False, tabDimsOfInterest=dimsOfInterest, rldDimsOfInterest=dimsOfInterest, reference_algorithm_filename=None, # TODO produce correct reference algo and update this line diff --git a/code-postprocessing/latex-templates/bbob.bib b/code-postprocessing/latex-templates/bbob.bib index 8e1fedf7f..8b5432bd0 100644 --- a/code-postprocessing/latex-templates/bbob.bib +++ b/code-postprocessing/latex-templates/bbob.bib @@ -1,12 +1,16 @@ % This file was created with JabRef 2.7.2. % Encoding: MacRoman -@ARTICLE{bbobls2019func, - author = {}, - title = {}, - journal = {}, - volume = {}, - year = 2019 +@unpublished{elhara:hal-02068407, + TITLE = {{COCO: The Large Scale Black-Box Optimization Benchmarking (bbob-largescale) Test Suite}}, + AUTHOR = {Elhara, Ouassim Ait and Varelas, Konstantinos and Nguyen, Duc Hung and Tusar, Tea and Brockhoff, Dimo and Hansen, Nikolaus and Auger, Anne}, + URL = {https://hal.inria.fr/hal-02068407}, + NOTE = {working paper or preprint}, + YEAR = {2019}, + MONTH = Mar, + PDF = {https://hal.inria.fr/hal-02068407/file/bbob-largescale-functions.pdf}, + HAL_ID = {hal-02068407}, + HAL_VERSION = {v1}, } @ARTICLE{hansen2016perfass, diff --git a/code-postprocessing/latex-templates/templateBBOBLSarticle.tex b/code-postprocessing/latex-templates/templateBBOBLSarticle.tex index 3a00bf6ad..aab296477 100644 --- a/code-postprocessing/latex-templates/templateBBOBLSarticle.tex +++ b/code-postprocessing/latex-templates/templateBBOBLSarticle.tex @@ -227,7 +227,7 @@ \section{CPU Timing} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % note that the following text is just a proposal and can/should be changed to your needs: -In order to evaluate the CPU timing of the algorithm, we have run the \change{\algorithmA} with restarts on the entire \bbobls test suite \cite{bbobls2019func} for $2 D$ function evaluations according to \cite{hansen2016exp}. The \change{C/Java/Matlab/Octave/Python} code was run on a \change{Mac Intel(R) Core(TM) i5-2400S CPU @ 2.50GHz} with \change{1} processor and \change{4} cores \change{and (compile) options xxx}. The time per function evaluation for dimensions 20, 40, 80, 160, 320\change{, 640} equals \change{$x.x$}, \change{$x.x$}, \change{$x.x$}, \change{$xx$}, \change{$xxx$}\change{, and $xxx$} seconds respectively. +In order to evaluate the CPU timing of the algorithm, we have run the \change{\algorithmA} with restarts on the entire \bbobls test suite \cite{elhara:hal-02068407} for $2 D$ function evaluations according to \cite{hansen2016exp}. The \change{C/Java/Matlab/Octave/Python} code was run on a \change{Mac Intel(R) Core(TM) i5-2400S CPU @ 2.50GHz} with \change{1} processor and \change{4} cores \change{and (compile) options xxx}. The time per function evaluation for dimensions 20, 40, 80, 160, 320\change{, 640} equals \change{$x.x$}, \change{$x.x$}, \change{$x.x$}, \change{$xx$}, \change{$xxx$}\change{, and $xxx$} seconds respectively. \ifthenelse{\equal{\numofalgs}{1}}{}{ \change{repeat the above for any algorithm tested} @@ -238,7 +238,7 @@ \section{Results} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Results from experiments according to \cite{hansen2016exp} and \cite{hansen2016perfass} on the -benchmark functions given in \cite{bbobls2019func} are +benchmark functions given in \cite{elhara:hal-02068407} are presented in %% \ifthenelse{\equal{\numofalgs}{1}}{ diff --git a/do.py b/do.py index 0850763b0..e731d04bf 100755 --- a/do.py +++ b/do.py @@ -6,7 +6,7 @@ import sys import os -from os.path import join +from os.path import join, abspath, realpath import shutil import tempfile import subprocess @@ -14,7 +14,6 @@ import platform import time import glob -import stat ## Change to the root directory of repository and add our tools/ @@ -25,6 +24,7 @@ from amalgamate import amalgamate from cocoutils import make, run, python, check_output from cocoutils import copy_file, expand_file, write_file +from cocoutils import executable_path from cocoutils import git_version, git_revision CORE_FILES = ['code-experiments/src/coco_random.c', @@ -619,8 +619,17 @@ def build_java(): {'COCO_VERSION': git_version(pep440=True)}) write_file(git_revision(), "code-experiments/build/java/REVISION") write_file(git_version(), "code-experiments/build/java/VERSION") - run('code-experiments/build/java', ['javac', '-classpath', '.', 'CocoJNI.java'], verbose=_verbosity) - run('code-experiments/build/java', ['javah', '-classpath', '.', 'CocoJNI'], verbose=_verbosity) + + javacpath = executable_path('javac') + javahpath = executable_path('javah') + if javacpath and javahpath: + run('code-experiments/build/java', ['javac', '-classpath', '.', 'CocoJNI.java'], verbose=_verbosity) + run('code-experiments/build/java', ['javah', '-classpath', '.', 'CocoJNI'], verbose=_verbosity) + elif javacpath: + run('code-experiments/build/java', ['javac', '-h', '.', 'CocoJNI.java'], verbose=_verbosity) + else: + raise RuntimeError('Can not find javac path!') + # Finds the path to the headers jni.h and jni_md.h (platform-dependent) # and compiles the CocoJNI library (compiler-dependent). So far, only @@ -659,9 +668,18 @@ def build_java(): # 4. Linux elif 'linux' in sys.platform: - jdkpath = check_output(['locate', 'jni.h'], stderr=STDOUT, - env=os.environ, universal_newlines=True) - jdkpath1 = jdkpath.split("jni.h")[0] + # bad bad bad... + #jdkpath = check_output(['locate', 'jni.h'], stderr=STDOUT, + # env=os.environ, universal_newlines=True) + #jdkpath1 = jdkpath.split("jni.h")[0] + # better + javapath = executable_path('java') + if not javapath: + raise RuntimeError('Can not find Java executable') + jdkhome = abspath(join(javapath, os.pardir, os.pardir)) + if os.path.basename(jdkhome) == 'jre': + jdkhome = join(jdkhome, os.pardir) + jdkpath1 = join(jdkhome, 'include') jdkpath2 = jdkpath1 + '/linux' run('code-experiments/build/java', ['gcc', '-I', jdkpath1, '-I', jdkpath2, '-c', 'CocoJNI.c'], @@ -687,11 +705,11 @@ def build_java(): ['gcc', '-dynamiclib', '-o', 'libCocoJNI.jnilib', 'CocoJNI.o'], verbose=_verbosity) - run('code-experiments/build/java', ['javac', '-classpath', '.', 'Problem.java'], verbose=_verbosity) - run('code-experiments/build/java', ['javac', '-classpath', '.', 'Benchmark.java'], verbose=_verbosity) - run('code-experiments/build/java', ['javac', '-classpath', '.', 'Observer.java'], verbose=_verbosity) - run('code-experiments/build/java', ['javac', '-classpath', '.', 'Suite.java'], verbose=_verbosity) - run('code-experiments/build/java', ['javac', '-classpath', '.', 'ExampleExperiment.java'], verbose=_verbosity) + run('code-experiments/build/java', ['javac', 'Problem.java'], verbose=_verbosity) + run('code-experiments/build/java', ['javac', 'Benchmark.java'], verbose=_verbosity) + run('code-experiments/build/java', ['javac', 'Observer.java'], verbose=_verbosity) + run('code-experiments/build/java', ['javac', 'Suite.java'], verbose=_verbosity) + run('code-experiments/build/java', ['javac', 'ExampleExperiment.java'], verbose=_verbosity) def run_java(): @@ -699,7 +717,7 @@ def run_java(): build_java() try: run('code-experiments/build/java', - ['java', '-Djava.library.path=.', '-classpath', '.', 'ExampleExperiment'], + ['java', '-Djava.library.path=.', 'ExampleExperiment'], verbose=_verbosity) except subprocess.CalledProcessError: sys.exit(-1) @@ -710,7 +728,7 @@ def test_java(): build_java() try: run('code-experiments/build/java', - ['java', '-Djava.library.path=.', '-classpath', '.', 'ExampleExperiment'], + ['java', '-Djava.library.path=.', 'ExampleExperiment'], verbose=_verbosity) except subprocess.CalledProcessError: sys.exit(-1) @@ -763,7 +781,6 @@ def test_postprocessing(all_tests=False, package_install_option=[]): shutil.rmtree('code-experiments/build/python/exdata/RS-' + s, ignore_errors=True) - def verify_postprocessing(package_install_option = []): install_postprocessing(package_install_option = package_install_option) # This is not affected by the _verbosity value. Verbose should always be True. @@ -793,7 +810,6 @@ def test_preprocessing(package_install_option = []): python('code-preprocessing/archive-update', ['-m', 'pytest'], verbose=_verbosity) python('code-preprocessing/log-reconstruction', ['-m', 'pytest'], verbose=_verbosity) - ################################################################################ ## Global def build(package_install_option = []): diff --git a/howtos/release-howto.md b/howtos/release-howto.md index 10ab19abc..60b3506d9 100644 --- a/howtos/release-howto.md +++ b/howtos/release-howto.md @@ -18,7 +18,8 @@ Simple Version (without documentation) Advanced Version (with documentation) ------------------------------------- - 1. Draft a release (under the Code / releases tabs). Consider previous releases to see what to write. + 1. Draft a release (under the Code / releases tabs). Consider previous releases to see what to write. + 2. check that instance numbers are up-to-date and `"year: this-year"` is implemented as suite option 2. check that README.md is up-to-date 3. clean and test the development branch 4. Run tests by pushing the development branch to the `devel-test1` and `test-nightly` branches, which