Skip to content

Commit

Permalink
Adding native nelder-mead search.
Browse files Browse the repository at this point in the history
Due to ongoing technical difficulties with the Active Harmony
integration, we've implemented our own nelder mead search.
It still needs some debugging but appears to be working for
discretized input sets. Still needs refactoring to handle continuous
values from the Kokkos tuning interface. That's the next step.
  • Loading branch information
khuck committed Sep 16, 2024
1 parent 230e887 commit 3ceaea8
Show file tree
Hide file tree
Showing 12 changed files with 1,301 additions and 10 deletions.
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -688,6 +688,10 @@ if(APEX_WITH_KOKKOS)
if(APEX_BUILD_TESTS)
# Just for testing
SET(Kokkos_LIBRARY kokkoscore)
set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "" FORCE)
set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE)
set(Kokkos_ARCH_NATIVE ON CACHE BOOL "" FORCE)
set(Kokkos_ENABLE_TUNING ON CACHE BOOL "" FORCE)
add_subdirectory(kokkos)
endif(APEX_BUILD_TESTS)
endif()
Expand Down
4 changes: 4 additions & 0 deletions src/apex/CMakeLists_hpx.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -322,9 +322,11 @@ set(apex_headers
dependency_tree.hpp
event_listener.hpp
exhaustive.hpp
genetic_search.hpp
gzstream.hpp
handler.hpp
memory_wrapper.hpp
nelder_mead.hpp
policy_handler.hpp
profile.hpp
profiler.hpp
Expand Down Expand Up @@ -361,9 +363,11 @@ set(apex_sources
event_listener.cpp
event_filter.cpp
exhaustive.cpp
genetic_search.cpp
gzstream.cpp
handler.cpp
memory_wrapper.cpp
nelder_mead.cpp
nvtx_listener.cpp
policy_handler.cpp
profile_reducer.cpp
Expand Down
1 change: 1 addition & 0 deletions src/apex/CMakeLists_standalone.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ exhaustive.cpp
genetic_search.cpp
handler.cpp
memory_wrapper.cpp
nelder_mead.cpp
nvtx_listener.cpp
${OTF2_SOURCE}
${perfetto_sources}
Expand Down
3 changes: 3 additions & 0 deletions src/apex/apex_kokkos_tuning.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,9 @@ class KokkosSession {
} else if (strncmp(apex::apex_options::kokkos_tuning_policy(),
"genetic_search", strlen("genetic_search")) == 0) {
strategy = apex_ah_tuning_strategy::GENETIC_SEARCH;
} else if (strncmp(apex::apex_options::kokkos_tuning_policy(),
"nelder_mead", strlen("nelder_mead")) == 0) {
strategy = apex_ah_tuning_strategy::NELDER_MEAD_INTERNAL;
} else {
strategy = apex_ah_tuning_strategy::NELDER_MEAD;
}
Expand Down
107 changes: 107 additions & 0 deletions src/apex/apex_policies.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,42 @@ int apex_sa_policy(shared_ptr<apex_tuning_session> tuning_session,
return APEX_NOERROR;
}

int apex_nelder_mead_policy(shared_ptr<apex_tuning_session> tuning_session,
apex_context const context) {
APEX_UNUSED(context);
if (apex_final) return APEX_NOERROR; // we terminated
std::unique_lock<std::mutex> l{shutdown_mutex};
/* If we are doing nested search contexts, allow us to keep searching
* on outer contexts until all inner contexts have converged! */
bool force{true};
if (context.data != nullptr) {
// the context data is a pointer to a boolean value
force = *((bool*)(context.data));
}
if (tuning_session->nelder_mead_session.converged() && force) {
if (!tuning_session->converged_message) {
tuning_session->converged_message = true;
cout << "APEX: Tuning has converged for session " << tuning_session->id
<< "." << endl;
tuning_session->nelder_mead_session.saveBestSettings();
tuning_session->nelder_mead_session.printBestSettings();
}
tuning_session->nelder_mead_session.saveBestSettings();
return APEX_NOERROR;
}

// get a measurement of our current setting
double new_value = tuning_session->metric_of_interest();

/* Report the performance we've just measured. */
tuning_session->nelder_mead_session.evaluate(new_value);

/* Request new settings for next time */
tuning_session->nelder_mead_session.getNewSettings();

return APEX_NOERROR;
}

int apex_genetic_policy(shared_ptr<apex_tuning_session> tuning_session,
apex_context const context) {
APEX_UNUSED(context);
Expand Down Expand Up @@ -1572,6 +1608,67 @@ inline int __sa_setup(shared_ptr<apex_tuning_session>
return APEX_NOERROR;
}

inline int __nelder_mead_setup(shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request) {
APEX_UNUSED(tuning_session);
// set up the Simulated annealing!
// iterate over the parameters, and create variables.
using namespace apex::nelder_mead;
for(auto & kv : request.params) {
auto & param = kv.second;
const char * param_name = param->get_name().c_str();
switch(param->get_type()) {
case apex_param_type::LONG: {
auto param_long =
std::static_pointer_cast<apex_param_long>(param);
Variable v(VariableType::longtype, param_long->value.get());
long lvalue = param_long->min;
do {
v.lvalues.push_back(lvalue);
lvalue = lvalue + param_long->step;
} while (lvalue <= param_long->max);
v.set_init();
tuning_session->nelder_mead_session.add_var(param_name, std::move(v));
}
break;
case apex_param_type::DOUBLE: {
auto param_double =
std::static_pointer_cast<apex_param_double>(param);
Variable v(VariableType::doubletype, param_double->value.get());
double dvalue = param_double->min;
do {
v.dvalues.push_back(dvalue);
dvalue = dvalue + param_double->step;
} while (dvalue <= param_double->max);
v.set_init();
tuning_session->nelder_mead_session.add_var(param_name, std::move(v));
}
break;
case apex_param_type::ENUM: {
auto param_enum =
std::static_pointer_cast<apex_param_enum>(param);
Variable v(VariableType::stringtype, param_enum->value.get());
for(const std::string & possible_value :
param_enum->possible_values) {
v.svalues.push_back(possible_value);
}
v.set_init();
tuning_session->nelder_mead_session.add_var(param_name, std::move(v));
}
break;
default:
cerr <<
"ERROR: Attempted to register tuning parameter with unknown type."
<< endl;
return APEX_ERROR;
}
}
/* request initial settings */
tuning_session->nelder_mead_session.getNewSettings();

return APEX_NOERROR;
}

inline int __genetic_setup(shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request) {
APEX_UNUSED(tuning_session);
Expand Down Expand Up @@ -1848,6 +1945,16 @@ inline int __common_setup_custom_tuning(shared_ptr<apex_tuning_session>
}
);
}
} else if (request.strategy == apex_ah_tuning_strategy::NELDER_MEAD_INTERNAL) {
status = __nelder_mead_setup(tuning_session, request);
if(status == APEX_NOERROR) {
apex::register_policy(
request.trigger,
[=](apex_context const & context)->int {
return apex_nelder_mead_policy(tuning_session, context);
}
);
}
} else if (request.strategy == apex_ah_tuning_strategy::GENETIC_SEARCH) {
status = __genetic_setup(tuning_session, request);
if(status == APEX_NOERROR) {
Expand Down
16 changes: 15 additions & 1 deletion src/apex/apex_policies.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,12 @@
#include "random.hpp"
// include the genetic_search class
#include "genetic_search.hpp"
// include the nelder_mead class
#include "nelder_mead.hpp"

enum class apex_param_type : int {NONE, LONG, DOUBLE, ENUM};
enum class apex_ah_tuning_strategy : int {
EXHAUSTIVE, RANDOM, NELDER_MEAD,
EXHAUSTIVE, RANDOM, NELDER_MEAD, NELDER_MEAD_INTERNAL,
PARALLEL_RANK_ORDER, SIMULATED_ANNEALING,
APEX_EXHAUSTIVE, APEX_RANDOM,
GENETIC_SEARCH};
Expand Down Expand Up @@ -82,6 +84,8 @@ class apex_param {
tuning_session, apex_tuning_request & request);
friend int __genetic_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
friend int __nelder_mead_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
};

class apex_param_long : public apex_param {
Expand Down Expand Up @@ -121,6 +125,8 @@ class apex_param_long : public apex_param {
tuning_session, apex_tuning_request & request);
friend int __genetic_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
friend int __nelder_mead_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
};

class apex_param_double : public apex_param {
Expand Down Expand Up @@ -160,6 +166,8 @@ class apex_param_double : public apex_param {
tuning_session, apex_tuning_request & request);
friend int __genetic_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
friend int __nelder_mead_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
};

class apex_param_enum : public apex_param {
Expand Down Expand Up @@ -198,6 +206,8 @@ class apex_param_enum : public apex_param {
tuning_session, apex_tuning_request & request);
friend int __genetic_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
friend int __nelder_mead_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
};


Expand Down Expand Up @@ -340,6 +350,8 @@ class apex_tuning_request {
tuning_session, apex_tuning_request & request);
friend int __genetic_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
friend int __nelder_mead_setup(std::shared_ptr<apex_tuning_session>
tuning_session, apex_tuning_request & request);
};


Expand Down Expand Up @@ -368,6 +380,8 @@ struct apex_tuning_session {
apex::random::Random random_session;
// if using genetic, this is the request.
apex::genetic::GeneticSearch genetic_session;
// if using nelder mead, this is the request.
apex::nelder_mead::NelderMead nelder_mead_session;
bool converged_message = false;

// variables related to power throttling
Expand Down
95 changes: 95 additions & 0 deletions src/apex/nelder_mead.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#include "nelder_mead.hpp"
#include <algorithm>
#include <iostream>
#include <fstream>
#include <iomanip>
#include <cmath>

namespace apex {

namespace nelder_mead {

void NelderMead::start(void) {
// create a starting point
std::vector<double> init_point;
for (auto& v : vars) {
init_point.push_back(v.second.get_init());
}
// create a lower limit
std::vector<double> lower_limit;
std::vector<double> upper_limit;
for (auto& v : vars) {
auto& limits = v.second.get_limits();
lower_limit.push_back(limits[0]);
upper_limit.push_back(limits[1]);
}
// create a starting simplex - random values in the space, nvars+1 of them
std::vector<std::vector<double>> init_simplex;
for (size_t i = 0 ; i < (vars.size() + 1) ; i++) {
std::vector<double> tmp;
for (auto& v : vars) {
double r = ((double) std::rand() / (RAND_MAX));
auto& limits = v.second.get_limits();
tmp.push_back(r * ((limits[1] + limits[0]) / 2.0));
}
init_simplex.push_back(tmp);
}
searcher = new apex::internal::nelder_mead::Searcher<double>(init_point, init_simplex, lower_limit, upper_limit, true);
searcher->function_tolerance(100000);
searcher->point_tolerance(0.01);
}

/*
static std::string vector_to_string(std::vector<double> val) {
std::stringstream ss;
ss << "[";
for (size_t i = 0; i < val.size(); i++) {
ss << val[i];
ss << (i == val.size() - 1 ? "]" : ",");
}
return ss.str();
}
*/

void NelderMead::getNewSettings() {
if (searcher == nullptr) start();
// get the next point from the simplex search
auto point = searcher->get_next_point();
//std::cout << "Next point: " << vector_to_string(point) << std::endl;
size_t i{0};
for (auto& v : vars) {
// if continuous, we just get the value
if (v.second.vtype == VariableType::continuous) {
v.second.current_value = point[i];
} else {
// otherwise, we scale the value from [0:maxlen] to our discrete index
double tmp = point[i] * v.second.maxlen;
v.second.current_index = std::min((size_t)(std::trunc(tmp)),(v.second.maxlen-1));
}
v.second.set_current_value();
i++;
}
}

void NelderMead::evaluate(double new_cost) {
// report the result
searcher->report(new_cost);
if (new_cost < cost) {
if (new_cost < best_cost) {
best_cost = new_cost;
std::cout << "New best! " << new_cost << " k: " << k << std::endl;
for (auto& v : vars) { v.second.save_best(); }
for (auto& v : vars) { std::cout << ", " << v.first << ": " << v.second.toString(); }
std::cout << std::endl;
}
cost = new_cost;
}
k++;
return;
}

} // genetic

} // apex


Loading

0 comments on commit 3ceaea8

Please sign in to comment.