DEbugging automatic search strategy, tweaking nelder mead convergence…

… criteria.

src/apex/exhaustive.cpp

@@ -2,6 +2,7 @@
 #include <algorithm>
 #include <iostream>
 #include <fstream>
+#include "apex_options.hpp"
 
 namespace apex {
 
@@ -32,6 +33,12 @@ size_t Exhaustive::get_max_iterations() {
             }
         }
     }
+    // if we are only chosing between a couple of items, then
+    // increase our number of iterations so that we ensure good
+    // coverage.
+    if (vars.size() == 1 || max_iter < 10) {
+        max_iter = max_iter * std::max(10, apex_options::kokkos_tuning_window()+1);
+    }
     // want to see multiple values of each one
     //std::cout << "Max iterations: " << max_iter << std::endl;
     return max_iter;
@@ -67,10 +74,12 @@ void Exhaustive::evaluate(double new_cost) {
     if (new_cost < cost) {
         if (new_cost < best_cost) {
             best_cost = new_cost;
-            std::cout << "New best! " << new_cost << " k: " << k
+            std::cout << "Exhaustive search: New best! " << new_cost << " k: " << k
                       << " kmax: " << kmax;
-            for (auto& v : vars) { v.second.save_best(); }
-            for (auto& v : vars) { std::cout  << ", " << v.first << ": " << v.second.toString(); }
+            for (auto& v : vars) {
+                std::cout  << ", " << v.first << ": " << v.second.toString();
+                v.second.save_best();
+            }
             std::cout << std::endl;
         }
         cost = new_cost;

src/apex/genetic_search.cpp

@@ -167,10 +167,12 @@ void GeneticSearch::evaluate(double new_cost) {
     if (new_cost < cost) {
         if (new_cost < best_cost) {
             best_cost = new_cost;
-            std::cout << "New best! " << new_cost << " k: " << k
+            std::cout << "Genetic Search: New best! " << new_cost << " k: " << k
                       << " kmax: " << kmax;
-            for (auto& v : vars) { v.second.save_best(); }
-            for (auto& v : vars) { std::cout  << ", " << v.first << ": " << v.second.toString(); }
+            for (auto& v : vars) {
+                std::cout  << ", " << v.first << ": " << v.second.toString();
+                v.second.save_best();
+            }
             std::cout << std::endl;
         }
         cost = new_cost;

src/apex/nelder_mead.cpp

@@ -35,8 +35,12 @@ void NelderMead::start(void) {
         init_simplex.push_back(tmp);
     }
     searcher = new apex::internal::nelder_mead::Searcher<double>(init_point, init_simplex, lower_limit, upper_limit, true);
-    searcher->function_tolerance(100000);
-    searcher->point_tolerance(0.01);
+    searcher->function_tolerance(10000);
+    if (hasDiscrete) {
+        searcher->point_tolerance(1.0);
+    } else {
+        searcher->point_tolerance(0.01);
+    }
 }
 
 /*
@@ -77,12 +81,18 @@ void NelderMead::evaluate(double new_cost) {
     if (new_cost < cost) {
         if (new_cost < best_cost) {
             best_cost = new_cost;
-            std::cout << "New best! " << new_cost << " k: " << k << std::endl;
-            for (auto& v : vars) { v.second.save_best(); }
-            for (auto& v : vars) { std::cout  << ", " << v.first << ": " << v.second.toString(); }
+            std::cout << "Nelder Mead: New best! " << new_cost << " k: " << k;
+            for (auto& v : vars) {
+                std::cout  << ", " << v.first << ": " << v.second.toString();
+                v.second.save_best();
+            }
             std::cout << std::endl;
         }
         cost = new_cost;
+        // if the function evaluation takes a long time (in nanoseconds, remember), increase our tolerance.
+        auto tmp = std::max((new_cost / 50.0), 1000.0);
+        std::cout << "new function tolerance: " << tmp << std::endl;
+        searcher->function_tolerance(tmp);
     }
     k++;
     return;

src/apex/nelder_mead.hpp

@@ -129,10 +129,11 @@ class NelderMead {
     const size_t max_iterations{500};
     const size_t min_iterations{16};
     internal::nelder_mead::Searcher<double>* searcher;
+    bool hasDiscrete;
 public:
     void evaluate(double new_cost);
     NelderMead() :
-        kmax(0), k(1), searcher(nullptr) {
+        kmax(0), k(1), searcher(nullptr), hasDiscrete(false) {
         cost = std::numeric_limits<double>::max();
         best_cost = cost;
     }
@@ -160,6 +161,10 @@ class NelderMead {
     std::map<std::string, Variable>& get_vars() { return vars; }
     void add_var(std::string name, Variable var) {
         vars.insert(std::make_pair(name, var));
+        if (var.vtype == VariableType::longtype ||
+            var.vtype == VariableType::stringtype) {
+            hasDiscrete = true;
+        }
     }
     void start(void);
 };

src/apex/nelder_mead_internal.h

@@ -208,8 +208,8 @@ template <typename T = double> class Searcher {
              const std::vector<std::vector<T>> &initial_simplex = {},
              const std::vector<T> &_minimum_limits = {},
              const std::vector<T> &_maximum_limits = {}, bool _adaptive = true)
-        : adaptive(_adaptive), tol_fun(1e-8), tol_x(1e-8), max_iter(100000),
-          max_fun_evals(100000), current_simplex_index(0),
+        : adaptive(_adaptive), tol_fun(1e-8), tol_x(1e-8), max_iter(1000),
+          max_fun_evals(2000), current_simplex_index(0),
           minimum_limits(_minimum_limits), maximum_limits(_maximum_limits),
           _converged(false), verbose(false) {
         initialize(initial_point, initial_simplex);
@@ -435,7 +435,7 @@ template <typename T = double> class Searcher {
         }
         // have we converged? either by being within tolerance of the best -
         // worst or by being within the tolerance of a point distance?
-        if ((max_val_diff <= tol_fun or max_point_diff <= tol_x) or
+        if ((max_val_diff <= tol_fun and max_point_diff <= tol_x) or
             (func_evals_count >= max_fun_evals) or (niter >= max_iter)) {
             res = simplex[smallest_idx].vec();
             std::cout << "Converged after " << niter << " iterations."

src/apex/random.cpp

@@ -68,10 +68,12 @@ void Random::evaluate(double new_cost) {
     if (new_cost < cost) {
         if (new_cost < best_cost) {
             best_cost = new_cost;
-            std::cout << "New best! " << new_cost << " k: " << k
+            std::cout << "Random: New best! " << new_cost << " k: " << k
                       << " kmax: " << kmax;
-            for (auto& v : vars) { v.second.save_best(); }
-            for (auto& v : vars) { std::cout  << ", " << v.first << ": " << v.second.toString(); }
+            for (auto& v : vars) {
+                std::cout  << ", " << v.first << ": " << v.second.toString();
+                v.second.save_best();
+            }
             std::cout << std::endl;
         }
         cost = new_cost;

src/apex/simulated_annealing.cpp

@@ -58,10 +58,12 @@ void SimulatedAnnealing::evaluate(double new_cost) {
         for (auto& v : vars) { v.second.choose_neighbor(); }
         if (new_cost < best_cost) {
             best_cost = new_cost;
-            std::cout << "New best! " << new_cost << " k: " << k
+            std::cout << "Simulated Annealing: New best! " << new_cost << " k: " << k
                       << " kmax: " << kmax << " temp: " << temp;
-            for (auto& v : vars) { v.second.save_best(); }
-            for (auto& v : vars) { std::cout  << ", " << v.first << ": " << v.second.toString(); }
+            for (auto& v : vars) {
+                std::cout  << ", " << v.first << ": " << v.second.toString();
+                v.second.save_best();
+            }
             std::cout << std::endl;
             since_restart = 1;
         }