Skip to content
This repository has been archived by the owner on Jul 10, 2021. It is now read-only.

Improved documentation, tests and support for sklearn.grid_search #55

Merged
merged 1 commit into from
May 18, 2015
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Adding more tests for grid search and randomized search, using both c…
…lassifiers and regressors. Removed prefix from output layers. Adding example to advanced section of documentation.
  • Loading branch information
alexjc committed May 18, 2015
commit deb38ff7446e66643740a6c71339fe56a9f4b310
26 changes: 22 additions & 4 deletions docs/guide_advanced.rst
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,15 @@ Here's how to setup such a pipeline with a multi-layer perceptron as a classifie
You can then use the pipeline as you would the neural network, or any other standard API from scikit-learn.



Grid Search
-----------

In scikit-learn, you can use a ``GridSearchCV`` to optimize your neural network's parameters automatically, both the top-level parameters and the parameters within the layers. For example, assuming you have your MLP constructed as in the :ref:`Regression` example in the local variable called ``nn``, the layers are named automatically so you can refer to them as follows:
In scikit-learn, you can use a ``GridSearchCV`` to optimize your neural network's hyper-parameters automatically, both the top-level parameters and the parameters within the layers. For example, assuming you have your MLP constructed as in the :ref:`Regression` example in the local variable called ``nn``, the layers are named automatically so you can refer to them as follows:

* ``hidden0``
* ``hidden1``
* ...
* ``output2``
* ``output``

Keep in mind you can manually specify the ``name`` of any ``Layer`` in the constructor if you don't want the automatically assigned name. Then, you can use sklearn's hierarchical parameters to perform a grid search over those nested parameters too:

Expand All @@ -67,4 +66,23 @@ Keep in mind you can manually specify the ``name`` of any ``Layer`` in the const
'hidden0__type': ["Rectifier", "Sigmoid", "Tanh"]})
gs.fit(a_in, a_out)

This will search through the listed ``learning_rate`` values, the number of hidden units and the activation type for that layer too, and find the best combination of parameters.
This will search through the listed ``learning_rate`` values, the number of hidden units and the activation type for that layer too, and find the best combination of parameters.


Randomized Search
-----------------

In the cases when you have large numbers of hyper-parameters that you want to try automatically to find a good combination, you can use a randomized search as follows:

.. code:: python

from scipy import stats
from sklearn.grid_search import RandomizedSearchCV

rs = RandomizedSearchCV(nn, param_grid={
learning_rate: stats.uniform(0.001, 0.05),
'hidden0__units': stats.randint(4, 12),
'hidden0__type': ["Rectifier", "Sigmoid", "Tanh"]})
rs.fit(a_in, a_out)

This works for both :class:`sknn.mlp.Classifier` and :class:`sknn.mlp.Regressor`.
12 changes: 7 additions & 5 deletions sknn/mlp.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,12 @@ class Layer(object):

name: str, optional
You optionally can specify a name for this layer, and its parameters
will then be accessible to `scikit-learn` via a nested sub-object. For example,
if name is set to `hidden1`, then the parameter `hidden1__units` from the network
is bound to this layer's `units` variable.
will then be accessible to scikit-learn via a nested sub-object. For example,
if name is set to ``layer1``, then the parameter ``layer1__units`` from the network
is bound to this layer's ``units`` variable.

The name defaults to ``hiddenN`` where N is the integer index of that layer, and the
final layer is always ``output`` without an index.

units: int
The number of units (also known as neurons) in this layer. This applies to all
Expand Down Expand Up @@ -353,8 +356,7 @@ def __init__(

# Layer names are optional, if not specified then generate one.
if layer.name is None:
label = "hidden" if i < len(layers)-1 else "output"
layer.name = "%s%i" % (label, i)
layer.name = ("hidden%i" % i) if i < len(layers)-1 else "output"

# sklearn may pass layers in as additional named parameters, remove them.
if layer.name in params:
Expand Down
6 changes: 3 additions & 3 deletions sknn/tests/test_layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ class TestNestedParameters(unittest.TestCase):
def test_GetParamsIncludesLayers(self):
nn = MLPR(layers=[L("Linear", units=123)])
p = nn.get_params()
assert_in('output0', p)
assert_in('output', p)

def test_GetParamsMissingLayer(self):
nn = MLPR(layers=[L("Linear", units=123)])
Expand All @@ -19,9 +19,9 @@ def test_GetParamsMissingLayer(self):

def test_SetParamsDoubleUnderscore(self):
nn = MLPR(layers=[L("Linear", units=123)])
nn.set_params(output0__units=456)
nn.set_params(output__units=456)
assert_equal(nn.layers[0].units, 456)

def test_SetParamsValueError(self):
nn = MLPR(layers=[L("Linear")])
assert_raises(ValueError, nn.set_params, output0__range=1.0)
assert_raises(ValueError, nn.set_params, output__range=1.0)
61 changes: 46 additions & 15 deletions sknn/tests/test_sklearn.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,57 @@
from nose.tools import (assert_equal, assert_raises, assert_in, assert_not_in)

import numpy
from sklearn.grid_search import GridSearchCV
from scipy.stats import randint, uniform

from sklearn.grid_search import GridSearchCV, RandomizedSearchCV
from sklearn.cross_validation import cross_val_score

from sknn.mlp import Regressor as MLPR
from sknn.mlp import Regressor as MLPR, Classifier as MLPC
from sknn.mlp import Layer as L


class TestGridSearch(unittest.TestCase):

def test_RegressorGlobalParams(self):
a_in = numpy.random.uniform(0.0, 1.0, (64,16))
a_out = numpy.zeros((64,1))
class TestGridSearchRegressor(unittest.TestCase):

__estimator__ = MLPR

def setUp(self):
self.a_in = numpy.random.uniform(0.0, 1.0, (64,16))
self.a_out = numpy.zeros((64,1))

def test_GridGlobalParams(self):
clf = GridSearchCV(
MLPR(layers=[L("Linear")], n_iter=1),
self.__estimator__(layers=[L("Linear")], n_iter=1),
param_grid={'learning_rate': [0.01, 0.001]})
clf.fit(a_in, a_out)

def test_RegressorLayerParams(self):
a_in = numpy.random.uniform(0.0, 1.0, (64,16))
a_out = numpy.zeros((64,1))
clf.fit(self.a_in, self.a_out)

def test_GridLayerParams(self):
clf = GridSearchCV(
MLPR(layers=[L("Rectifier", units=12), L("Linear")], n_iter=1),
self.__estimator__(layers=[L("Rectifier", units=12), L("Linear")], n_iter=1),
param_grid={'hidden0__units': [4, 8, 12]})
clf.fit(a_in, a_out)
clf.fit(self.a_in, self.a_out)

def test_RandomGlobalParams(self):
clf = RandomizedSearchCV(
self.__estimator__(layers=[L("Softmax")], n_iter=1),
param_distributions={'learning_rate': uniform(0.001, 0.01)},
n_iter=2)
clf.fit(self.a_in, self.a_out)

def test_RandomLayerParams(self):
clf = RandomizedSearchCV(
self.__estimator__(layers=[L("Softmax", units=12), L("Linear")], n_iter=1),
param_distributions={'hidden0__units': randint(4, 12)},
n_iter=2)
clf.fit(self.a_in, self.a_out)


class TestGridSearchClassifier(TestGridSearchRegressor):

__estimator__ = MLPC

def setUp(self):
self.a_in = numpy.random.uniform(0.0, 1.0, (64,16))
self.a_out = numpy.random.randint(0, 4, (64,))


class TestCrossValidation(unittest.TestCase):
Expand All @@ -37,3 +62,9 @@ def test_Regressor(self):
a_out = numpy.zeros((64,1))

cross_val_score(MLPR(layers=[L("Linear")], n_iter=1), a_in, a_out, cv=5)

def test_Classifier(self):
a_in = numpy.random.uniform(0.0, 1.0, (64,16))
a_out = numpy.random.randint(0, 4, (64,))

cross_val_score(MLPC(layers=[L("Linear")], n_iter=1), a_in, a_out, cv=5)