Merge pull request #1945 from numbbo/development

Merge into master branch for 2.3.3 release

code-postprocessing/cocopp/bestalg.py

@@ -88,7 +88,7 @@ class BestAlgSet(DataSet):
     numbers of function evaluations for evals or function values for
     funvals.
 
-    Known bug: algorithms where the aRT is NaN or Inf are not taken into
+    Known bug: algorithms where the ERT is NaN or Inf are not taken into
     account!?
 
     """
@@ -148,7 +148,7 @@ def __init__(self, dict_alg, algId='Virtual Best Algorithm'):
         sortedAlgs = list(dict_alg.keys())
         # algorithms will be sorted along sortedAlgs which is now a fixed list
 
-        # Align aRT
+        # Align ERT
         erts = list(np.transpose(np.vstack([dict_alg[i].target, dict_alg[i].ert]))
                     for i in sortedAlgs)
         res = readalign.alignArrayData(readalign.HArrayMultiReader(erts))
@@ -168,7 +168,7 @@ def __init__(self, dict_alg, algId='Virtual Best Algorithm'):
                     continue  # TODO: don't disregard these entries
                 if tmpert == currentbestert:
                     # TODO: what do we do in case of ties?
-                    # look at function values corresponding to the aRT?
+                    # look at function values corresponding to the ERT?
                     # Look at the function evaluations? the success ratio?
                     pass
                 elif tmpert < currentbestert:
@@ -308,11 +308,11 @@ def createDictInstance(self):
         return dictinstance
 
     def detERT(self, targets):
-        """Determine the average running time to reach target values.
+        """Determine the expected running time to reach target values.
 
         :keyword list targets: target function values of interest
 
-        :returns: list of average running times corresponding to the
+        :returns: list of expected running times corresponding to the
                   targets.
 
         """
@@ -644,7 +644,7 @@ def getAllContributingAlgorithmsToBest(algnamelist, target_lb=1e-8,
                                        target_ub=1e2):
     """Computes first the artificial best algorithm from given algorithm list
        algnamelist, constructed by extracting for each target/function pair
-       thalgorithm with best aRT among the given ones. Returns then the list
+       thalgorithm with best ERT among the given ones. Returns then the list
        of algorithms that are contributing to the definition of the best
        algorithm, separated by dimension, and sorted by importance (i.e. with
        respect to the number of target/function pairs where each algorithm is
@@ -713,9 +713,9 @@ def extractBestAlgorithms(args=algs2009, f_factor=2,
     """Returns (and prints) per dimension a list of algorithms within
     algorithm list args that contains an algorithm if for any
         dimension/target/function pair this algorithm:
-        - is the best algorithm wrt aRT
-        - its own aRT lies within a factor f_factor of the best aRT
-        - there is no algorithm within a factor of f_factor of the best aRT
+        - is the best algorithm wrt ERT
+        - its own ERT lies within a factor f_factor of the best ERT
+        - there is no algorithm within a factor of f_factor of the best ERT
           and the current algorithm is the second best.
 
     """
@@ -750,7 +750,7 @@ def extractBestAlgorithms(args=algs2009, f_factor=2,
                     # add best for this target:
                     selectedAlgsPerProblemDF.append(best.algs[i])
 
-                    # add second best or all algorithms that have an aRT
+                    # add second best or all algorithms that have an ERT
                     # within a factor of f_factor of the best:
                     secondbest_ERT = np.infty
                     secondbest_str = ''

code-postprocessing/cocopp/captions.py

@@ -81,7 +81,7 @@ def get_reference_algorithm_text(best_algorithm_mandatory=True):
 
     return text
 
-def get_best_art_text():
+def get_best_ert_text():
     text = ''
     testbed = testbedsettings.current_testbed
     if testbed.reference_algorithm_filename:
@@ -90,19 +90,19 @@ def get_best_art_text():
                 or testbed.name == testbedsettings.testbed_name_bi):
             if testbed.reference_algorithm_displayname:
                 if "best 2009" in testbed.reference_algorithm_displayname:
-                    text = "best \\aRT\ measured during BBOB-2009"
+                    text = "best \\ERT\ measured during BBOB-2009"
                 elif "best 2010" in testbed.reference_algorithm_displayname:
-                    text = "best \\aRT\ measured during BBOB-2010"
+                    text = "best \\ERT\ measured during BBOB-2010"
                 elif "best 2012" in testbed.reference_algorithm_displayname:
-                    text = "best \\aRT\ measured during BBOB-2012"
+                    text = "best \\ERT\ measured during BBOB-2012"
                 elif "best 2013" in testbed.reference_algorithm_displayname:
-                    text = "best \\aRT\ measured during BBOB-2013"
+                    text = "best \\ERT\ measured during BBOB-2013"
                 elif "best 2016" in testbed.reference_algorithm_displayname:
-                    text = "best \\aRT\ measured during BBOB-2016"
+                    text = "best \\ERT\ measured during BBOB-2016"
                 elif "best 2009-16" in testbed.reference_algorithm_displayname:
-                    text = "best \\aRT\ measured during BBOB-2009-16"
+                    text = "best \\ERT\ measured during BBOB-2009-16"
                 else:
-                    text = "the \\aRT\ of the reference algorithm"
+                    text = "the \\ERT\ of the reference algorithm"
         else:
             raise NotImplementedError('reference algorithm not supported for this testbed')
     else:
@@ -161,7 +161,7 @@ def get_light_brown_line_text(testbedname):
         '!!NUM-OF-TARGETS-IN-ECDF!!': lambda: str(len(testbedsettings.current_testbed.pprldmany_target_values)),
         '!!TARGET-RANGES-IN-ECDF!!': lambda: str(testbedsettings.current_testbed.pprldmany_target_range_latex),
         '!!TOTAL-NUM-OF-FUNCTIONS!!': lambda: str(testbedsettings.current_testbed.last_function_number - testbedsettings.current_testbed.first_function_number + 1),
-        '!!BEST-ART!!': lambda: get_best_art_text(),
+        '!!BEST-ERT!!': lambda: get_best_ert_text(),
         '!!NBTARGETS-SCATTER!!': lambda: str(len(testbedsettings.current_testbed.ppscatter_target_values)),
         '!!NBLOW!!': lambda: get_nblow(),
         '!!NBUP!!': lambda: get_nbup()

code-postprocessing/cocopp/cococommands.py

@@ -47,7 +47,7 @@
     Number of trials: 15
     Final target Df: 1e-08
     min / max number of evals per trial: 224 / 333
-       evals/DIM:  best     15%     50%     85%     max |  aRT/DIM  nsucc
+       evals/DIM:  best     15%     50%     85%     max |  ERT/DIM  nsucc
       ---Df---|-----------------------------------------|----------------
       1.0e+03 |       0       0       0       0       0 |      0.5  15
       1.0e+01 |       0       0       2       8      10 |      2.9  15

code-postprocessing/cocopp/comp2/ppfig2.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 
-"""Creates aRT-ratio comparison figures (ECDF) and convergence figures for the comparison of 2 algorithms.
+"""Creates ERT-ratio comparison figures (ECDF) and convergence figures for the comparison of 2 algorithms.
 
 Scale up figures for two algorithms can be done with compall/ppfigs.py
 
@@ -130,7 +130,7 @@ def beautify(xmin=None):
 
     # Annotate figure
     ax.set_xlabel('log10(Delta ftarget)')
-    ax.set_ylabel(r'log10(aRT1/aRT0) or ~#succ')  # TODO: replace hard-coded 15
+    ax.set_ylabel(r'log10(ERT1/ERT0) or ~#succ')  # TODO: replace hard-coded 15
     ax.grid(True)
 
     #Tick label handling
@@ -243,7 +243,7 @@ def annotate(entry0, entry1, dim, minfvalue=1e-8, nbtests=1):
                      transform=trans, clip_on=False)
 
 def main(dsList0, dsList1, minfvalue=1e-8, outputdir=''):
-    """Returns aRT1/aRT0 comparison figure."""
+    """Returns ERT1/ERT0 comparison figure."""
 
     #plt.rc("axes", labelsize=20, titlesize=24)
     #plt.rc("xtick", labelsize=20)

code-postprocessing/cocopp/comp2/pprldistr2.py

@@ -38,13 +38,14 @@ def beautify(handles):
 
     axisHandle = plt.gca()
     axisHandle.set_xscale('log')
-    plt.axvline(1, ls='-', color='k');  # symmetry line for aRT1/aRT0 = 1
-    xlim = min(max(numpy.abs(numpy.log10(plt.xlim()))),
-               numpy.ceil(numpy.log10(sys.float_info.max))-1) # correction of
-                                                              # numerical issues
-                                                              # with bbob-biobj
-                                                              # test
-    xlim = (min(0.1, 10.**(-xlim)), max(10., 10.**(xlim)))
+    plt.axvline(1, ls='-', color='k');  # symmetry line for ERT1/ERT0 = 1
+
+    xlim = plt.xlim()
+    xlim = numpy.maximum(xlim[0], 1e-9), numpy.minimum(xlim[1], 1e9)
+    xlim = numpy.minimum(xlim[0], 1. / 10.01), numpy.maximum(xlim[1], 10.01)
+    xlim = max(numpy.abs(numpy.log10(xlim)))
+    xlim = (10 ** (-xlim), 10 ** xlim)
+
     plt.axhline(0.5, ls=':', color='k', lw=2)  # symmetry line at y=0.5
     plt.xlim(xlim)
     plt.yticks(numpy.array((0., 0.25, 0.5, 0.75, 1.0)),
@@ -53,12 +54,7 @@ def beautify(handles):
     axisHandle.set_xlabel('log10 of FEvals ratio')
     axisHandle.set_ylabel('proportion of trials')
     axisHandle.grid(True)
-    xticks = axisHandle.get_xticks()
-    newxticks = []
-    for i in xticks:
-        if i > 0 and i < numpy.inf:
-            newxticks.append('%d' % round(numpy.log10(i)))
-    axisHandle.set_xticklabels(newxticks)
+
 
     # Prolong to the boundary...
     xmin, xmax = plt.xlim()
@@ -71,7 +67,7 @@ def beautify(handles):
         if len(xdata) == 0 or len(ydata) == 0:
             continue
         if not hasattr(xdata, 'dtype') or xdata.dtype != float:
-            xdata = numpy.array(xdata, dtype=float) 
+            xdata = numpy.array(xdata, dtype=float)
         xdata = numpy.insert(xdata, 0, xmin)
         xdata = numpy.insert(xdata, len(xdata), xmax)
         ydata = numpy.insert(ydata, 0, ydata[0])
@@ -80,12 +76,11 @@ def beautify(handles):
 
     toolsdivers.legend(loc='best')
 
-    # Inverted xticks
-    x = axisHandle.get_xticks()
-    # Operation for reverting the ticks for x < 1
-    x[x<1] = sorted(1/(x[x<1]*numpy.power(10, -2*numpy.floor(numpy.log10(x[x<1]))-1)))
-    x = x[(x<xmax) * (x>xmin)] # why?
+    x = numpy.asarray(axisHandle.get_xticks())
+    axisHandle.set_xticklabels([str(int(numpy.log10(xx))) for xx in x])
     axisHandle.set_xticks(x)
+    plt.xlim(xlim)
+
 
 def computeERT(fevals, maxevals):
     data = fevals.copy()

code-postprocessing/cocopp/comp2/ppscatter.py

@@ -2,9 +2,9 @@
 # -*- coding: utf-8 -*-
 """Scatter Plots.
 
-For two algorithms, this generates the scatter plot of log(aRT1(df)) vs.
-log(aRT0(df)), where aRT0(df) is the aRT of the reference algorithm,
-aRT1(df) is the aRT of the algorithm of concern, both for target
+For two algorithms, this generates the scatter plot of log(ERT1(df)) vs.
+log(ERT0(df)), where ERT0(df) is the ERT of the reference algorithm,
+ERT1(df) is the ERT of the algorithm of concern, both for target
 precision df.
 
 Different symbols are used for different dimension (see
@@ -20,8 +20,8 @@
 """
 from __future__ import absolute_import
 
-"""For two algorithms, aRTs(given target function value) can also be
-plotted in a scatter plot (log(aRT0) vs. log(aRT1)), which results in a
+"""For two algorithms, ERTs(given target function value) can also be
+plotted in a scatter plot (log(ERT0) vs. log(ERT1)), which results in a
 very attractive presentation, see the slides of Frank Hutter at
 http://www.msr-inria.inria.fr/events-news/first-search-biology-day. The
 advantage is that the absolute values do not get lost. The disadvantage
@@ -57,14 +57,14 @@
 
 def prepare_figure_caption():
 
-    caption_start_fixed = r"""Average running time (\aRT\ in $\log_{10}$ of number of function evaluations)
+    caption_start_fixed = r"""Expected running time (\ERT\ in $\log_{10}$ of number of function evaluations)
         of \algorithmA\ ($y$-axis) versus \algorithmB\ ($x$-axis) for $!!NBTARGETS-SCATTER!!$ target values
         $!!DF!! \in [!!NBLOW!!, !!NBUP!!]$ in each dimension on functions #1. """
 
-    caption_start_rlbased = r"""Average running time (\aRT\ in $\log_{10}$ of number of function evaluations)
+    caption_start_rlbased = r"""Expected running time (\ERT\ in $\log_{10}$ of number of function evaluations)
         of \algorithmA\ ($y$-axis) versus \algorithmB\ ($x$-axis) for $!!NBTARGETS-SCATTER!!$ runlength-based target
         values for budgets between $!!NBLOW!!$ and $!!NBUP!!$ evaluations.
-        Each runlength-based target $!!F!!$-value is chosen such that the \aRT{}s of 
+        Each runlength-based target $!!F!!$-value is chosen such that the \ERT{}s of 
         !!THE-REF-ALG!! for the given and a slightly easier
         target bracket the reference budget. """
 
@@ -177,14 +177,14 @@ def main(dsList0, dsList1, outputdir, settings):
                 entry1 = dictDim1[d][0] # should be only one element
             except (IndexError, KeyError):
                 continue
-            if linewidth:  # plot all reliable aRT values as a line
+            if linewidth:  # plot all reliable ERT values as a line
                 all_targets = np.array(sorted(set(entry0.target).union(entry1.target), reverse=True))
                 assert entry0.detSuccessRates([all_targets[0]]) == 1.0
                 assert entry1.detSuccessRates([all_targets[0]]) == 1.0
                 all_targets = all_targets[np.where(all_targets <= targets((f, d))[0])[0]]  # 
                 xdata_all = np.array(entry0.detERT(all_targets))
                 ydata_all = np.array(entry1.detERT(all_targets))
-                # idx of reliable targets: last index where success rate >= 1/2 and aRT <= maxevals
+                # idx of reliable targets: last index where success rate >= 1/2 and ERT <= maxevals
                 idx = []
                 for ari in (np.where(entry0.detSuccessRates(all_targets) >= 0.5)[0], 
                          np.where(entry1.detSuccessRates(all_targets) >= 0.5)[0], 

code-postprocessing/cocopp/compall/ppfigs.py

@@ -1,6 +1,6 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
-"""Creates aRTs and convergence figures for multiple algorithms."""
+"""Creates ERTs and convergence figures for multiple algorithms."""
 from __future__ import absolute_import, print_function
 import os
 import matplotlib.pyplot as plt
@@ -49,15 +49,15 @@ def fix_styles(plotting_styles, line_styles):
 
 def prepare_scaling_figure_caption():
 
-    scaling_figure_caption_start_fixed = (r"""Average running time (\aRT\ in number of $f$-evaluations
+    scaling_figure_caption_start_fixed = (r"""Expected running time (\ERT\ in number of $f$-evaluations
                     as $\log_{10}$ value), divided by dimension for target function value $!!PPFIGS-FTARGET!!$
                     versus dimension. Slanted grid lines indicate quadratic scaling with the dimension. """
                                           )
 
-    scaling_figure_caption_start_rlbased = (r"""Average running time (\aRT\ in number of $f$-evaluations
+    scaling_figure_caption_start_rlbased = (r"""Expected running time (\ERT\ in number of $f$-evaluations
                         as $\log_{10}$ value) divided by dimension versus dimension. The target function value
                         is chosen such that !!THE-REF-ALG!! just failed to achieve
-                        an \aRT\ of $!!PPFIGS-FTARGET!!\times\DIM$. """
+                        an \ERT\ of $!!PPFIGS-FTARGET!!\times\DIM$. """
                                             )
 
     scaling_figure_caption_end = (
@@ -434,7 +434,7 @@ def generateData(dataSet, target):
 
 
 def main(dictAlg, html_file_prefix, sorted_algorithms=None, output_dir='ppdata', latex_commands_file=''):
-    """From a DataSetList, returns figures showing the scaling: aRT/dim vs dim.
+    """From a DataSetList, returns figures showing the scaling: ERT/dim vs dim.
     
     One function and one target per figure.
     

code-postprocessing/cocopp/compall/ppperfprof.py

@@ -82,7 +82,7 @@ def beautify():
     a.set_xscale('log')
     #Tick label handling
 
-    plt.xlabel('log10 of (aRT / aRTref)')
+    plt.xlabel('log10 of (ERT / ERTref)')
     plt.ylabel('Proportion of functions')
     logxticks()
     beautifyECDF()
@@ -123,9 +123,9 @@ def plot(dsList, dsref, targets=defaulttargets, istoolsstats=False, **kwargs):
     """Generates a graph showing the performance profile of an algorithm.
 
     We display the empirical cumulative distribution function ECDF of
-    the bootstrapped distribution of the average running time (aRT)
+    the bootstrapped distribution of the expected running time (ERT)
     for an algorithm to reach the function value :py:data:`targets`
-    normalized by the aRT of the reference algorithm for these
+    normalized by the ERT of the reference algorithm for these
     targets.
 
     :param DataSetList dsList: data set for one algorithm
@@ -142,7 +142,7 @@ def plot(dsList, dsref, targets=defaulttargets, istoolsstats=False, **kwargs):
     for entry in dsList:
         for t in targets:
             # TODO: alternative: min(dsref[(entry.dim, entry.funcId)].detEvals((t,))[0])
-            #       is the min from the alg with the best aRT
+            #       is the min from the alg with the best ERT
             flg_ert = 1
             if flg_ert:
                 normalizer = dsref[(entry.dim, entry.funcId)].detERT((t,))[0]

code-postprocessing/cocopp/compall/pprldmany.py

@@ -1,11 +1,11 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 
-"""Generates figure of the bootstrap distribution of aRT.
+"""Generates figure of the bootstrap distribution of ERT.
     
 The main method in this module generates figures of Empirical
 Cumulative Distribution Functions of the bootstrap distribution of
-the Average Running Time (aRT) divided by the dimension for many
+the Expected Running Time (ERT) divided by the dimension for many
 algorithms.
 
 The outputs show the ECDFs of the running times of the simulated runs
@@ -20,7 +20,7 @@
 
     import cocopp
 
-    # Empirical cumulative distribution function of bootstrapped aRT figure
+    # Empirical cumulative distribution function of bootstrapped ERT figure
     ds = cocopp.load(cocopp.bbob.get('2009/BIPOP-CMA-ES'))
     figure()
     cocopp.compall.pprldmany.plot(ds) # must rather call main instead of plot?
@@ -555,7 +555,7 @@ def main(dictAlg, order=None, outputdir='.', info='default',
 
     From a dictionary of :py:class:`DataSetList` sorted by algorithms,
     generates the cumulative distribution function of the bootstrap
-    distribution of aRT for algorithms on multiple functions for
+    distribution of ERT for algorithms on multiple functions for
     multiple targets altogether.
 
     :param dict dictAlg: dictionary of :py:class:`DataSetList` instances