Skip to content

Commit 5063765

Browse files
costashatzcostashatz
committed
Adding a few gradient-based optimizers
1 parent 2bb2647 commit 5063765

File tree

4 files changed

+407
-0
lines changed

4 files changed

+407
-0
lines changed

‎src/limbo/opt.hpp

+2
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@
6161
#include <limbo/opt/nlopt_grad.hpp>
6262
#include <limbo/opt/nlopt_no_grad.hpp>
6363
#endif
64+
#include <limbo/opt/adam.hpp>
65+
#include <limbo/opt/gradient_ascent.hpp>
6466
#include <limbo/opt/parallel_repeater.hpp>
6567
#include <limbo/opt/random_point.hpp>
6668
#include <limbo/opt/rprop.hpp>

‎src/limbo/opt/adam.hpp

+156
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
//| Copyright Inria May 2015
2+
//| This project has received funding from the European Research Council (ERC) under
3+
//| the European Union's Horizon 2020 research and innovation programme (grant
4+
//| agreement No 637972) - see http://www.resibots.eu
5+
//|
6+
//| Contributor(s):
7+
//| - Jean-Baptiste Mouret ([email protected])
8+
//| - Antoine Cully ([email protected])
9+
//| - Konstantinos Chatzilygeroudis ([email protected])
10+
//| - Federico Allocati ([email protected])
11+
//| - Vaios Papaspyros ([email protected])
12+
//| - Roberto Rama ([email protected])
13+
//|
14+
//| This software is a computer library whose purpose is to optimize continuous,
15+
//| black-box functions. It mainly implements Gaussian processes and Bayesian
16+
//| optimization.
17+
//| Main repository: https://github.com/resibots/limbo
18+
//| Documentation: http://www.resibots.eu/limbo
19+
//|
20+
//| This software is governed by the CeCILL-C license under French law and
21+
//| abiding by the rules of distribution of free software. You can use,
22+
//| modify and/ or redistribute the software under the terms of the CeCILL-C
23+
//| license as circulated by CEA, CNRS and INRIA at the following URL
24+
//| "http://www.cecill.info".
25+
//|
26+
//| As a counterpart to the access to the source code and rights to copy,
27+
//| modify and redistribute granted by the license, users are provided only
28+
//| with a limited warranty and the software's author, the holder of the
29+
//| economic rights, and the successive licensors have only limited
30+
//| liability.
31+
//|
32+
//| In this respect, the user's attention is drawn to the risks associated
33+
//| with loading, using, modifying and/or developing or reproducing the
34+
//| software by the user in light of its specific status of free software,
35+
//| that may mean that it is complicated to manipulate, and that also
36+
//| therefore means that it is reserved for developers and experienced
37+
//| professionals having in-depth computer knowledge. Users are therefore
38+
//| encouraged to load and test the software's suitability as regards their
39+
//| requirements in conditions enabling the security of their systems and/or
40+
//| data to be ensured and, more generally, to use and operate it in the
41+
//| same conditions as regards security.
42+
//|
43+
//| The fact that you are presently reading this means that you have had
44+
//| knowledge of the CeCILL-C license and that you accept its terms.
45+
//|
46+
#ifndef LIMBO_OPT_ADAM_HPP
47+
#define LIMBO_OPT_ADAM_HPP
48+
49+
#include <algorithm>
50+
51+
#include <Eigen/Core>
52+
53+
#include <limbo/opt/optimizer.hpp>
54+
#include <limbo/tools/macros.hpp>
55+
#include <limbo/tools/math.hpp>
56+
57+
namespace limbo {
58+
namespace defaults {
59+
struct opt_adam {
60+
/// @ingroup opt_defaults
61+
/// number of max iterations
62+
BO_PARAM(int, iterations, 300);
63+
64+
/// alpha - learning rate
65+
BO_PARAM(double, alpha, 0.001);
66+
67+
/// β1
68+
BO_PARAM(double, b1, 0.9);
69+
70+
/// β2
71+
BO_PARAM(double, b2, 0.999);
72+
73+
/// norm epsilon for stopping
74+
BO_PARAM(double, eps_stop, 0.0);
75+
};
76+
} // namespace defaults
77+
namespace opt {
78+
/// @ingroup opt
79+
/// Adam optimizer
80+
/// Equations from: http://ruder.io/optimizing-gradient-descent/index.html#gradientdescentoptimizationalgorithms
81+
/// (I changed a bit the notation; η to α)
82+
///
83+
/// Parameters:
84+
/// - int iterations
85+
/// - double alpha
86+
/// - double b1
87+
/// - double b2
88+
/// - double eps_stop
89+
template <typename Params>
90+
struct Adam {
91+
template <typename F>
92+
Eigen::VectorXd operator()(const F& f, const Eigen::VectorXd& init, bool bounded) const
93+
{
94+
assert(Params::opt_adam::b1() >= 0. && Params::opt_adam::b1() < 1.);
95+
assert(Params::opt_adam::b2() >= 0. && Params::opt_adam::b2() < 1.);
96+
assert(Params::opt_adam::alpha() >= 0.);
97+
98+
size_t param_dim = init.size();
99+
double b1 = Params::opt_adam::b1();
100+
double b2 = Params::opt_adam::b2();
101+
double b1_t = b1;
102+
double b2_t = b2;
103+
double alpha = Params::opt_adam::alpha();
104+
double stop = Params::opt_adam::eps_stop();
105+
double epsilon = 1e-8;
106+
107+
Eigen::VectorXd m = Eigen::VectorXd::Zero(param_dim);
108+
Eigen::VectorXd v = Eigen::VectorXd::Zero(param_dim);
109+
110+
Eigen::VectorXd params = init;
111+
112+
if (bounded) {
113+
for (int j = 0; j < params.size(); j++) {
114+
if (params(j) < 0)
115+
params(j) = 0;
116+
if (params(j) > 1)
117+
params(j) = 1;
118+
}
119+
}
120+
121+
for (int i = 0; i < Params::opt_adam::iterations(); ++i) {
122+
Eigen::VectorXd prev_params = params;
123+
auto perf = opt::eval_grad(f, params);
124+
125+
Eigen::VectorXd grad = opt::grad(perf);
126+
m = b1 * m.array() + (1. - b1) * grad.array();
127+
v = b2 * v.array() + (1. - b2) * grad.array().square();
128+
129+
Eigen::VectorXd m_hat = m.array() / (1. - b1_t);
130+
Eigen::VectorXd v_hat = v.array() / (1. - b2_t);
131+
132+
params.array() += alpha * m_hat.array() / (v_hat.array().sqrt() + epsilon);
133+
134+
b1_t *= b1;
135+
b2_t *= b2;
136+
137+
if (bounded) {
138+
for (int j = 0; j < params.size(); j++) {
139+
if (params(j) < 0)
140+
params(j) = 0;
141+
if (params(j) > 1)
142+
params(j) = 1;
143+
}
144+
}
145+
146+
if ((prev_params - params).norm() < stop)
147+
break;
148+
}
149+
150+
return params;
151+
}
152+
};
153+
} // namespace opt
154+
} // namespace limbo
155+
156+
#endif

‎src/limbo/opt/gradient_ascent.hpp

+159
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
//| Copyright Inria May 2015
2+
//| This project has received funding from the European Research Council (ERC) under
3+
//| the European Union's Horizon 2020 research and innovation programme (grant
4+
//| agreement No 637972) - see http://www.resibots.eu
5+
//|
6+
//| Contributor(s):
7+
//| - Jean-Baptiste Mouret ([email protected])
8+
//| - Antoine Cully ([email protected])
9+
//| - Konstantinos Chatzilygeroudis ([email protected])
10+
//| - Federico Allocati ([email protected])
11+
//| - Vaios Papaspyros ([email protected])
12+
//| - Roberto Rama ([email protected])
13+
//|
14+
//| This software is a computer library whose purpose is to optimize continuous,
15+
//| black-box functions. It mainly implements Gaussian processes and Bayesian
16+
//| optimization.
17+
//| Main repository: https://github.com/resibots/limbo
18+
//| Documentation: http://www.resibots.eu/limbo
19+
//|
20+
//| This software is governed by the CeCILL-C license under French law and
21+
//| abiding by the rules of distribution of free software. You can use,
22+
//| modify and/ or redistribute the software under the terms of the CeCILL-C
23+
//| license as circulated by CEA, CNRS and INRIA at the following URL
24+
//| "http://www.cecill.info".
25+
//|
26+
//| As a counterpart to the access to the source code and rights to copy,
27+
//| modify and redistribute granted by the license, users are provided only
28+
//| with a limited warranty and the software's author, the holder of the
29+
//| economic rights, and the successive licensors have only limited
30+
//| liability.
31+
//|
32+
//| In this respect, the user's attention is drawn to the risks associated
33+
//| with loading, using, modifying and/or developing or reproducing the
34+
//| software by the user in light of its specific status of free software,
35+
//| that may mean that it is complicated to manipulate, and that also
36+
//| therefore means that it is reserved for developers and experienced
37+
//| professionals having in-depth computer knowledge. Users are therefore
38+
//| encouraged to load and test the software's suitability as regards their
39+
//| requirements in conditions enabling the security of their systems and/or
40+
//| data to be ensured and, more generally, to use and operate it in the
41+
//| same conditions as regards security.
42+
//|
43+
//| The fact that you are presently reading this means that you have had
44+
//| knowledge of the CeCILL-C license and that you accept its terms.
45+
//|
46+
#ifndef LIMBO_OPT_GRADIENT_ASCENT_HPP
47+
#define LIMBO_OPT_GRADIENT_ASCENT_HPP
48+
49+
#include <algorithm>
50+
51+
#include <Eigen/Core>
52+
53+
#include <limbo/opt/optimizer.hpp>
54+
#include <limbo/tools/macros.hpp>
55+
#include <limbo/tools/math.hpp>
56+
57+
namespace limbo {
58+
namespace defaults {
59+
struct opt_gradient_ascent {
60+
/// @ingroup opt_defaults
61+
/// number of max iterations
62+
BO_PARAM(int, iterations, 300);
63+
64+
/// alpha - learning rate
65+
BO_PARAM(double, alpha, 0.001);
66+
67+
/// gamma - for momentum
68+
BO_PARAM(double, gamma, 0.0);
69+
70+
/// nesterov momentum; turn on/off
71+
BO_PARAM(bool, nesterov, false);
72+
73+
/// norm epsilon for stopping
74+
BO_PARAM(double, eps_stop, 0.0);
75+
};
76+
} // namespace defaults
77+
namespace opt {
78+
/// @ingroup opt
79+
/// Gradient Ascent with or without momentum (Nesterov or simple)
80+
/// Equations from: http://ruder.io/optimizing-gradient-descent/index.html#gradientdescentoptimizationalgorithms
81+
/// (I changed a bit the notation; η to α)
82+
///
83+
/// Parameters:
84+
/// - int iterations
85+
/// - double alpha
86+
/// - double gamma
87+
/// - bool nesterov
88+
/// - double eps_stop
89+
template <typename Params>
90+
struct GradientAscent {
91+
template <typename F>
92+
Eigen::VectorXd operator()(const F& f, const Eigen::VectorXd& init, bool bounded) const
93+
{
94+
assert(Params::opt_gradient_ascent::gamma() >= 0. && Params::opt_gradient_ascent::gamma() < 1.);
95+
assert(Params::opt_gradient_ascent::alpha() >= 0.);
96+
97+
size_t param_dim = init.size();
98+
double gamma = Params::opt_gradient_ascent::gamma();
99+
double alpha = Params::opt_gradient_ascent::alpha();
100+
double stop = Params::opt_gradient_ascent::eps_stop();
101+
bool is_nesterov = Params::opt_gradient_ascent::nesterov();
102+
103+
Eigen::VectorXd v = Eigen::VectorXd::Zero(param_dim);
104+
105+
Eigen::VectorXd params = init;
106+
107+
if (bounded) {
108+
for (int j = 0; j < params.size(); j++) {
109+
if (params(j) < 0)
110+
params(j) = 0;
111+
if (params(j) > 1)
112+
params(j) = 1;
113+
}
114+
}
115+
116+
for (int i = 0; i < Params::opt_gradient_ascent::iterations(); ++i) {
117+
Eigen::VectorXd prev_params = params;
118+
Eigen::VectorXd query_params = params;
119+
// if Nesterov momentum, change query parameters
120+
if (is_nesterov) {
121+
query_params.array() += gamma * v.array();
122+
123+
// make sure that the parameters are still in bounds, if needed
124+
if (bounded) {
125+
for (int j = 0; j < query_params.size(); j++) {
126+
if (query_params(j) < 0)
127+
query_params(j) = 0;
128+
if (query_params(j) > 1)
129+
query_params(j) = 1;
130+
}
131+
}
132+
}
133+
auto perf = opt::eval_grad(f, query_params);
134+
135+
Eigen::VectorXd grad = opt::grad(perf);
136+
v = gamma * v.array() + alpha * grad.array();
137+
138+
params.array() += v.array();
139+
140+
if (bounded) {
141+
for (int j = 0; j < params.size(); j++) {
142+
if (params(j) < 0)
143+
params(j) = 0;
144+
if (params(j) > 1)
145+
params(j) = 1;
146+
}
147+
}
148+
149+
if ((prev_params - params).norm() < stop)
150+
break;
151+
}
152+
153+
return params;
154+
}
155+
};
156+
} // namespace opt
157+
} // namespace limbo
158+
159+
#endif

0 commit comments

Comments
 (0)