Debugging autotuning of kokkos kernels - problems with genetic search…

… and nested contexts should now be working.

src/apex/apex_kokkos_tuning.cpp

@@ -252,6 +252,7 @@ class KokkosSession {
     std::unordered_map<std::string, std::vector<int>> var_ids;
     std::map<size_t, Variable*> inputs;
     std::map<size_t, Variable*> outputs;
+    std::map<size_t, Variable*> all_vars;
     apex_policy_handle * start_policy_handle;
     apex_policy_handle * stop_policy_handle;
     std::unordered_map<size_t, std::string> active_requests;
@@ -300,30 +301,12 @@ bool KokkosSession::checkForCache() {
 void KokkosSession::saveInputVar(size_t id, Variable * var) {
     // insert the id given to us
     inputs.insert(std::make_pair(id, var));
-    // get a count of how many input variables we have seen
-    size_t count = inputs.size();
-    // if the id is equal to the size, there was a mingling of input and output ids.
-    // this might burn us later, so also insert the missing ID.
-    // For example, if the input ID is 4, and the new size is 3.
-    if (id > count) {
-        inputs.insert(std::make_pair(count, var));
-    }
-    /*
-    if (!use_history) {
-        cachedResults << "Input_" << id << ":" << std::endl;
-        cachedResults << var->toString();
-    }
-    */
+    all_vars.insert(std::make_pair(id, var));
 }
 
 void KokkosSession::saveOutputVar(size_t id, Variable * var) {
     outputs.insert(std::make_pair(id, var));
-    /*
-    if (!use_history) {
-        cachedResults << "Output_" << id << ":" << std::endl;
-        cachedResults << var->toString();
-    }
-    */
+    all_vars.insert(std::make_pair(id, var));
 }
 
 void KokkosSession::writeCache(void) {
@@ -741,36 +724,51 @@ std::string hashContext(size_t numVars,
     std::map<size_t, Variable*>& varmap, std::string tree_node) {
     std::stringstream ss;
     std::string d{"["};
+    /* This is REALLY annoying. sometimes the order of the variables
+     * can change, not sure how Kokkos is doing that but it may be a
+     * side effect of using an unordered map. regardless, we have to
+     * sort the variables by ID to make sure we generate the hash
+     * consistently. */
+    std::vector<std::pair<size_t,size_t>> reindex;
     for (size_t i = 0 ; i < numVars ; i++) {
-        auto id = values[i].type_id;
+        reindex.push_back(std::pair<size_t,size_t>(i,values[i].type_id));
+    }
+    sort(reindex.begin(), reindex.end(),
+            [](const auto& lhs, const auto& rhs) {
+                return lhs.second < rhs.second;
+            });
+    for (size_t i = 0 ; i < numVars ; i++) {
+        size_t ri = reindex[i].first;
+        auto id = values[ri].type_id;
         ss << d << id << ":";
         Variable* var{varmap[id]};
         switch (var->info.type) {
             case kokkos_value_double:
                 if (var->info.valueQuantity == kokkos_value_unbounded) {
-                    ss << var->getBin(values[i].value.double_value);
+                    ss << var->getBin(values[ri].value.double_value);
                 } else {
-                    ss << values[i].value.double_value;
+                    ss << values[ri].value.double_value;
                 }
                 break;
             case kokkos_value_int64:
                 if (var->info.valueQuantity == kokkos_value_unbounded) {
-                    ss << var->getBin(values[i].value.int_value);
+                    ss << var->getBin(values[ri].value.int_value);
                 } else {
-                    ss << values[i].value.int_value;
+                    ss << values[ri].value.int_value;
                 }
                 break;
             case kokkos_value_string:
-                ss << values[i].value.string_value;
+                ss << values[ri].value.string_value;
                 break;
             default:
                 break;
         }
         d = ",";
     }
     if(tree_node.size() > 0) {
-        ss << ",tree_node:" << tree_node << "]";
+        ss << ",tree_node:" << tree_node;
     }
+    ss << "]";
     std::string tmp{ss.str()};
     return tmp;
 }
@@ -1082,7 +1080,7 @@ void kokkosp_request_values(
     KokkosSession& session = KokkosSession::getSession();
     // create a unique name for this combination of input vars
     std::string name{hashContext(numContextVariables, contextVariableValues,
-        session.inputs, tree_node)};
+        session.all_vars, tree_node)};
     if (session.verbose) {
         std::cout << std::string(getDepth(), ' ');
         std::cout << __APEX_FUNCTION__ << " ctx: " << contextId << std::endl;

src/apex/exhaustive.hpp

@@ -99,7 +99,6 @@ class Exhaustive {
         best_cost = cost;
         //std::cout << "New Session!" << std::endl;
     }
-    double getEnergy() { return best_cost; }
     bool converged() { return (k >= kmax); }
     void getNewSettings() {
         /*   Increment neighbour */

src/apex/genetic_search.cpp

@@ -3,6 +3,7 @@
 #include <iostream>
 #include <fstream>
 #include <iomanip>
+#include <cmath>
 
 namespace apex {
 
@@ -78,41 +79,68 @@ class log_wrapper {
         crossover
 */
 
-auto get_random_number(const std::size_t min, const std::size_t max)
-{
+auto get_random_number(const std::size_t min, const std::size_t max) {
     const std::size_t values_count = max - min + 1;
     return rand() % values_count + min;
 }
 
+auto get_rank_order(const std::size_t min, const std::size_t max) {
+    // get 2^max minus 1
+    size_t powmax = pow(2,max);
+    size_t rando = get_random_number(min,powmax);
+    size_t index = 0;
+    for (size_t i = pow(2,max-1) ; i > min ; i = i / 2) {
+        if (rando > i) { break; }
+        index++;
+    }
+    return index;
+}
+
 void GeneticSearch::getNewSettings() {
-    static bool bootstrapping{true};
     if (bootstrapping) {
         if (population.size() >= population_size) {
             bootstrapping = false;
         } else {
             // we are still bootstrapping, so just get a random selection.
             for (auto& v : vars) { v.second.get_next_neighbor(); }
+            best_generation_cost = best_cost;
             return;
         }
     }
-    std::cout << "Have population of " << population.size() << " to evaluate!" << std::endl;
+    //std::cout << "Have population of " << population.size() << " to evaluate!" << std::endl;
     // time to cull the herd?
     if (population.size() >= population_size) {
-        std::cout << "Have population of " << population.size() << " to evaluate!" << std::endl;
+        if (best_cost < best_generation_cost) {
+            best_generation_cost = best_cost;
+        } else {
+            num_stable_generations++;
+        }
         // we need to sort the population...
         sort(population.begin(), population.end(),
             [](const individual& lhs, const individual& rhs) {
                 return lhs.cost < rhs.cost;
             });
         // ...then drop half of them - the "weakest" ones.
         population.erase(population.cbegin() + crossover, population.cend());
-        std::cout << "Now have population of " << population.size() << std::endl;
+        //std::cout << "Now have population of " << population.size() << std::endl;
     }
     // We want to generate a new individual using two "high quality" parents.
     // choose parent A
-    individual& A = population[get_random_number(0,crossover-1)];
+    auto indexA = get_rank_order(0,crossover-1);
+    individual& A = population[indexA];
+    /*
+    std::cout << "A: " << indexA;
+    for (auto& i : A.indexes) { std::cout << "," << i; }
+    std::cout << ", " << A.cost << std::endl;
+    */
     // choose parent B
-    individual& B = population[get_random_number(0,crossover-1)];
+    auto indexB = get_rank_order(0,crossover-1);
+    individual& B = population[indexB];
+    /*
+    std::cout << "B: " << indexB;
+    for (auto& i : B.indexes) { std::cout << "," << i; }
+    std::cout << ", " << B.cost << std::endl;
+    */
     // blend their variables into a new individual and maybe mutate?
     size_t i = 0;
     for (auto& v : vars) {
@@ -131,9 +159,7 @@ void GeneticSearch::getNewSettings() {
 
 void GeneticSearch::evaluate(double new_cost) {
     static log_wrapper log(vars);
-    static size_t count{0};
-    if (++count % 10000 == 0) { std::cout << count << std::endl; }
-    log.getstream() << count << ",";
+    log.getstream() << k << ",";
     for (auto& v : vars) { log.getstream() << v.second.toString() << ","; }
     log.getstream() << new_cost << std::endl;
     if (new_cost < cost) {
@@ -145,14 +171,18 @@ void GeneticSearch::evaluate(double new_cost) {
             std::cout << std::endl;
         }
         cost = new_cost;
+        /*
+    } else {
+        std::cout << "          " << new_cost << " k: " << k;
+        for (auto& v : vars) { std::cout  << ", value: " << v.second.toString(); }
+        std::cout << std::endl;
+        */
     }
     /* save our individual in the population */
     individual i;
     i.cost = new_cost;
     for (auto& v : vars) { i.indexes.push_back(v.second.current_index); }
     population.push_back(i);
-
-    //for (auto& v : vars) { v.second.choose_neighbor(); }
     k++;
     return;
 }

src/apex/genetic_search.hpp

@@ -51,7 +51,7 @@ class Variable {
     void save_best() { best_index = current_index; }
     void set_init() {
         maxlen = (std::max(std::max(dvalues.size(),
-            lvalues.size()), svalues.size())) - 1;
+            lvalues.size()), svalues.size()));
         current_index = 0;
         set_current_value();
     }
@@ -94,24 +94,35 @@ class GeneticSearch {
     size_t k;
     std::map<std::string, Variable> vars;
     const size_t max_iterations{1000};
-    const size_t min_iterations{100};
-    const size_t population_size{16};
-    const size_t crossover{8}; // half population
+    const size_t min_iterations{16};
+    const size_t population_size{32};
+    const size_t crossover{16}; // half population
     const size_t parent_ratio{50};
-    const size_t mutate_probability{5};
+    const size_t mutate_probability{15};
+    const size_t max_stable_generations{5};
+    size_t num_stable_generations;
     std::vector<individual> population;
+    bool bootstrapping;
+    double best_generation_cost;
 public:
     void evaluate(double new_cost);
     GeneticSearch() :
-        kmax(0), k(1) {
+        kmax(0), k(1), num_stable_generations(0), bootstrapping(true),
+        best_generation_cost(0.0) {
         cost = std::numeric_limits<double>::max();
         best_cost = cost;
         //std::cout << "New Session!" << std::endl;
         //srand (1);
         srand (time(NULL));
     }
-    double getEnergy() { return best_cost; }
-    bool converged() { return (k > kmax); }
+    bool converged() {
+        // if we haven't improved for X generations, quit
+        if (num_stable_generations >= max_stable_generations) {
+            return true;
+        }
+        // otherwise, just quit when we hit max iterations
+        return (k > kmax);
+    }
     void getNewSettings();
     void saveBestSettings() {
         for (auto& v : vars) { v.second.getBest(); }

src/apex/random.hpp

@@ -51,7 +51,7 @@ class Variable {
     void save_best() { best_index = current_index; }
     void set_init() {
         maxlen = (std::max(std::max(dvalues.size(),
-            lvalues.size()), svalues.size())) - 1;
+            lvalues.size()), svalues.size()));
         current_index = 0;
         set_current_value();
     }
@@ -100,7 +100,6 @@ class Random {
         //srand (1);
         srand (time(NULL));
     }
-    double getEnergy() { return best_cost; }
     bool converged() { return (k > kmax); }
     void getNewSettings() {
         /*   Increment neighbour */