diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 1b90bc53cb26..58e39efc2873 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1350,18 +1350,12 @@ nightly_scala_demo_test_cpu() { bash bin/run_im.sh } -nightly_estimator_gpu() { +nightly_estimator() { set -ex cd /work/mxnet/tests/nightly/estimator export PYTHONPATH=/work/mxnet/python/ python test_estimator_cnn.py --type gpu python test_sentiment_rnn.py --type gpu -} - -nightly_estimator_cpu() { - set -ex - cd /work/mxnet/tests/nightly/estimator - export PYTHONPATH=/work/mxnet/python/ python test_estimator_cnn.py --type cpu python test_sentiment_rnn.py --type cpu } diff --git a/python/mxnet/gluon/contrib/estimator/estimator.py b/python/mxnet/gluon/contrib/estimator/estimator.py index 78672d2f381a..d30595a6efb5 100644 --- a/python/mxnet/gluon/contrib/estimator/estimator.py +++ b/python/mxnet/gluon/contrib/estimator/estimator.py @@ -157,6 +157,8 @@ def prepare_loss_and_metrics(self): Based on loss functions and training metrics in estimator Create metric wrappers to record loss values, Create copies of train loss/metric objects to record validation values + Returns train_metrics and val_metrics + """ if any(not hasattr(self, attribute) for attribute in ['train_metrics', 'val_metrics']): @@ -165,8 +167,7 @@ def prepare_loss_and_metrics(self): self.train_metrics = [Accuracy()] self.val_metrics = [] for loss in self.loss: - self.train_metrics.append(Loss("Train " + ''.join([i for i in loss.name if not i.isdigit()]))) - self.val_metrics.append(Loss("Validation " + ''.join([i for i in loss.name if not i.isdigit()]))) + self.train_metrics.append(Loss(''.join([i for i in loss.name if not i.isdigit()]))) for metric in self.train_metrics: val_metric = copy.deepcopy(metric) metric.name = "Train " + metric.name @@ -231,21 +232,9 @@ def fit(self, train_data, from a data batch and load into contexts(devices) """ self.max_epochs = epochs - event_handlers = event_handlers or [] - # provide default logging handler - if not event_handlers: - train_metrics, val_metrics = self.prepare_loss_and_metrics() - event_handlers.append(MetricHandler(train_metrics=train_metrics)) - if val_data: - event_handlers.append(ValidationHandler(val_data=val_data, eval_fn=self.evaluate, - val_metrics=val_metrics)) - event_handlers.append(LoggingHandler(train_metrics=train_metrics, - val_metrics=val_metrics)) - warnings.warn("No Event Handler specified, default %s are used. " - "Please look at gluon.contrib.estimator.event_handler for more detail." % - ", ".join([handler.__class__.__name__ for handler in event_handlers])) - event_handlers.sort(key=lambda handler: getattr(handler, 'rank', 0), reverse=True) + # provide default handlers + event_handlers = self._prepare_default_handlers(val_data, event_handlers) train_begin, epoch_begin, batch_begin, \ batch_end, epoch_end, train_end = self._categorize_handlers(event_handlers) @@ -297,6 +286,54 @@ def fit(self, train_data, for handler in train_end: handler.train_end(estimator_ref) + def _prepare_default_handlers(self, val_data, event_handlers): + event_handlers = event_handlers or [] + default_handlers = [] + train_metrics, val_metrics = self.prepare_loss_and_metrics() + + if not any(isinstance(handler, MetricHandler) for handler in event_handlers): + event_handlers.append(MetricHandler(train_metrics=train_metrics)) + default_handlers.append("MetricHandler") + + if val_data and not any(isinstance(handler, ValidationHandler) for handler in event_handlers): + event_handlers.append(ValidationHandler(val_data=val_data, eval_fn=self.evaluate, + val_metrics=val_metrics)) + default_handlers.append("ValidationHandler") + + if not any(isinstance(handler, LoggingHandler) for handler in event_handlers): + event_handlers.append(LoggingHandler(train_metrics=train_metrics, + val_metrics=val_metrics)) + default_handlers.append("LoggingHandler") + + # if there is a mix of user defined event handlers and default event handlers + # they should have the save set of loss and metrics + if default_handlers: + msg = "You are training with the following default event handlers: %s. " \ + "They use loss and metrics from estimator.prepare_loss_and_metrics(). " \ + "Please use the same set of metrics for all your other handlers." % \ + ", ".join(default_handlers) + warnings.warn(msg) + references = [] + for handler in event_handlers: + for attribute in dir(handler): + if any(keyword in attribute for keyword in ['metric' or 'monitor']): + reference = getattr(handler, attribute) + if isinstance(reference, list): + references += reference + else: + references.append(reference) + for metric in references: + if metric and metric not in train_metrics + val_metrics: + msg = "We have added following default handlers for you: %s and used " \ + "estimator.prepare_loss_and_metrics() to pass metrics to " \ + "those handlers. Please use the same set of metrics " \ + "for all your handlers." % \ + ", ".join(default_handlers) + raise ValueError(msg) + + event_handlers.sort(key=lambda handler: getattr(handler, 'priority', 0)) + return event_handlers + def _categorize_handlers(self, event_handlers): """ categorize handlers into 6 event lists to avoid calling empty methods diff --git a/python/mxnet/gluon/contrib/estimator/event_handler.py b/python/mxnet/gluon/contrib/estimator/event_handler.py index 220aa31ab5dd..d8c3c6eaa6aa 100644 --- a/python/mxnet/gluon/contrib/estimator/event_handler.py +++ b/python/mxnet/gluon/contrib/estimator/event_handler.py @@ -299,7 +299,7 @@ def __init__(self, self.save_best_only = save_best_only if self.save_best_only and not isinstance(self.monitor, EvalMetric): raise ValueError("To save best model only, please provide one of the metric objects as monitor, " - "You can create these objects using estimator.prepare_loss_and_metric()") + "You can get these objects using estimator.prepare_loss_and_metric()") self.epoch_period = epoch_period self.batch_period = batch_period self.num_batches = 0 diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile index 1be084c9d3f5..758c86476f75 100755 --- a/tests/nightly/Jenkinsfile +++ b/tests/nightly/Jenkinsfile @@ -136,22 +136,6 @@ core_logic: { utils.docker_run('ubuntu_nightly_cpu', 'nightly_test_javascript', false) } } - }, - 'Gluon estimator: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/estimator-test-gpu') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator_gpu', true) - } - } - }, - 'Gluon estimator: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/estimator-test-cpu') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_nightly_cpu', 'nightly_estimator_cpu', false) - } - } } } } diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index ea6db1a20cbf..e4b9ff1acbb1 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -141,6 +141,14 @@ core_logic: { utils.docker_run('ubuntu_nightly_gpu', 'nightly_tutorial_test_ubuntu_python3_gpu', true, '1500m') } } + }, + 'Gluon estimator: GPU': { + node(NODE_LINUX_GPU) { + ws('workspace/estimator-test-gpu') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true) + } + } } } } diff --git a/tests/python/unittest/test_gluon_estimator.py b/tests/python/unittest/test_gluon_estimator.py index 6f19f435531b..643214212e3a 100644 --- a/tests/python/unittest/test_gluon_estimator.py +++ b/tests/python/unittest/test_gluon_estimator.py @@ -27,17 +27,28 @@ from nose.tools import assert_raises -def get_model(): +def _get_test_network(): net = nn.Sequential() net.add(nn.Dense(4, activation='relu', flatten=False)) return net +def _get_test_data(): + batch_size = 4 + in_data = mx.nd.random.uniform(shape=(10, 3)) + out_data = mx.nd.random.uniform(shape=(10, 4)) + # Input dataloader + dataset = gluon.data.dataset.ArrayDataset(in_data, out_data) + dataloader = gluon.data.DataLoader(dataset, batch_size=batch_size) + dataiter = mx.io.NDArrayIter(data=in_data, label=out_data, batch_size=batch_size) + return dataloader, dataiter + + def test_fit(): ''' test estimator with different train data types ''' - net = get_model() + net = _get_test_network() + dataloader, dataiter = _get_test_data() num_epochs = 1 - batch_size = 4 ctx = mx.cpu() loss = gluon.loss.L2Loss() acc = mx.metric.Accuracy() @@ -48,31 +59,25 @@ def test_fit(): metrics=acc, trainer=trainer, context=ctx) - in_data = mx.nd.random.uniform(shape=(10, 3)) - out_data = mx.nd.random.uniform(shape=(10, 4)) - # Input dataloader - dataset = gluon.data.dataset.ArrayDataset(in_data, out_data) - train_dataloader = gluon.data.DataLoader(dataset, batch_size=batch_size) - est.fit(train_data=train_dataloader, + + est.fit(train_data=dataloader, epochs=num_epochs) - # Input dataiter - train_dataiter = mx.io.NDArrayIter(data=in_data, label=out_data, batch_size=batch_size) with assert_raises(ValueError): - est.fit(train_data=train_dataiter, + est.fit(train_data=dataiter, epochs=num_epochs) # Input NDArray with assert_raises(ValueError): - est.fit(train_data=[in_data, out_data], + est.fit(train_data=[mx.nd.ones(shape=(10, 3))], epochs=num_epochs) def test_validation(): ''' test different validation data types''' - net = get_model() + net = _get_test_network() + dataloader, dataiter = _get_test_data() num_epochs = 1 - batch_size = 4 ctx = mx.cpu() loss = gluon.loss.L2Loss() acc = mx.metric.Accuracy() @@ -83,41 +88,35 @@ def test_validation(): metrics=acc, trainer=trainer, context=ctx) - in_data = mx.nd.random.uniform(shape=(10, 3)) - out_data = mx.nd.random.uniform(shape=(10, 4)) # Input dataloader - dataset = gluon.data.dataset.ArrayDataset(in_data, out_data) - train_dataloader = gluon.data.DataLoader(dataset, batch_size=batch_size) - val_dataloader = gluon.data.DataLoader(dataset, batch_size=batch_size) - est.fit(train_data=train_dataloader, - val_data=val_dataloader, + est.fit(train_data=dataloader, + val_data=dataloader, epochs=num_epochs) - # Input dataiter - train_dataiter = mx.io.NDArrayIter(data=in_data, label=out_data, batch_size=batch_size) - val_dataiter = mx.io.NDArrayIter(data=in_data, label=out_data, batch_size=batch_size) + # using validation handler + train_metrics, val_metrics = est.prepare_loss_and_metrics() + validation_handler = ValidationHandler(val_data=dataloader, eval_fn=est.evaluate, + val_metrics=val_metrics) + with assert_raises(ValueError): - est.fit(train_data=train_dataiter, - val_data=val_dataiter, + est.fit(train_data=dataiter, + val_data=dataiter, epochs=num_epochs) # Input NDArray with assert_raises(ValueError): - est.fit(train_data=[in_data, out_data], - val_data=[in_data, out_data], + est.fit(train_data=[mx.nd.ones(shape=(10, 3))], + val_data=[mx.nd.ones(shape=(10, 3))], epochs=num_epochs) @unittest.skipIf(sys.version_info.major < 3, 'Test on python 3') def test_initializer(): ''' test with no initializer, inconsistent initializer ''' - net = get_model() + net = _get_test_network() + train_data, _ = _get_test_data() num_epochs = 1 - batch_size = 4 ctx = mx.cpu() - in_data = mx.nd.random.uniform(shape=(10, 3)) - out_data = mx.nd.random.uniform(shape=(10, 4)) - dataset = gluon.data.dataset.ArrayDataset(in_data, out_data) - train_data = gluon.data.DataLoader(dataset, batch_size=batch_size) + loss = gluon.loss.L2Loss() acc = mx.metric.Accuracy() # no initializer @@ -129,7 +128,7 @@ def test_initializer(): epochs=num_epochs) # different initializer for net and estimator - net = get_model() + net = _get_test_network() net.initialize(mx.init.Xavier(), ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) # catch reinit warning @@ -148,14 +147,11 @@ def test_initializer(): @unittest.skipIf(sys.version_info.major < 3, 'Test on python 3') def test_trainer(): ''' test with no trainer and invalid trainer ''' - net = get_model() + net = _get_test_network() + train_data, _ = _get_test_data() num_epochs = 1 - batch_size = 4 ctx = mx.cpu() - in_data = mx.nd.random.uniform(shape=(10, 3)) - out_data = mx.nd.random.uniform(shape=(10, 4)) - dataset = gluon.data.dataset.ArrayDataset(in_data, out_data) - train_data = gluon.data.DataLoader(dataset, batch_size=batch_size) + loss = gluon.loss.L2Loss() acc = mx.metric.Accuracy() net.initialize(ctx=ctx) @@ -181,14 +177,11 @@ def test_trainer(): def test_metric(): ''' test with no metric, list of metrics, invalid metric ''' - net = get_model() + net = _get_test_network() + train_data, _ = _get_test_data() num_epochs = 1 - batch_size = 4 ctx = mx.cpu() - in_data = mx.nd.random.uniform(shape=(10, 3)) - out_data = mx.nd.random.uniform(shape=(10, 4)) - dataset = gluon.data.dataset.ArrayDataset(in_data, out_data) - train_data = gluon.data.DataLoader(dataset, batch_size=batch_size) + loss = gluon.loss.L2Loss() net.initialize(ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) @@ -227,7 +220,7 @@ def test_metric(): def test_loss(): ''' test with invalid loss ''' - net = get_model() + net = _get_test_network() ctx = mx.cpu() acc = mx.metric.Accuracy() net.initialize(ctx=ctx) @@ -243,7 +236,7 @@ def test_loss(): def test_context(): ''' test with no context, list of context, invalid context ''' - net = get_model() + net = _get_test_network() loss = gluon.loss.L2Loss() metrics = mx.metric.Accuracy() # input no context @@ -253,7 +246,7 @@ def test_context(): # input list of context gpus = mx.context.num_gpus() ctx = [mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()] - net = get_model() + net = _get_test_network() est = Estimator(net=net, loss=loss, metrics=metrics, @@ -309,3 +302,57 @@ def train_end(self): assert len(batch_begin) == 2 assert len(batch_end) == 1 assert len(train_end) == 2 + + +@unittest.skipIf(sys.version_info.major < 3, 'Test on python 3') +def test_default_handlers(): + net = _get_test_network() + train_data, _ = _get_test_data() + + num_epochs = 1 + ctx = mx.cpu() + + net.initialize(ctx=ctx) + trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) + + train_acc = mx.metric.RMSE() + loss = gluon.loss.L2Loss() + + est = Estimator(net=net, + loss=loss, + metrics=train_acc, + trainer=trainer, + context=ctx) + # no handler + with warnings.catch_warnings(record=True) as w: + est.fit(train_data=train_data, epochs=num_epochs) + assert 'You are training with the' in str(w[-1].message) + + # handler with prepared loss and metrics + train_metrics, val_metrics = est.prepare_loss_and_metrics() + logging = LoggingHandler(train_metrics=train_metrics, val_metrics=val_metrics) + with warnings.catch_warnings(record=True) as w: + est.fit(train_data=train_data, epochs=num_epochs, event_handlers=[logging]) + assert 'You are training with the' in str(w[-1].message) + # provide metric handler by default + assert 'MetricHandler' in str(w[-1].message) + + # handler with all user defined metrics + val_metrics = [mx.metric.RMSE("val acc")] + metric = MetricHandler(train_metrics=[train_acc]) + logging = LoggingHandler(train_metrics=train_metrics, val_metrics=val_metrics) + est.fit(train_data=train_data, epochs=num_epochs, event_handlers=[metric, logging]) + + # handler with mixed metrics, some handler use metrics prepared by estimator + # some handler use metrics user prepared + val_metrics = [mx.metric.RMSE("val acc")] + logging = LoggingHandler(train_metrics=train_metrics, val_metrics=val_metrics) + with assert_raises(ValueError): + est.fit(train_data=train_data, epochs=num_epochs, event_handlers=[logging]) + + # test handler order + early_stopping = EarlyStoppingHandler(monitor=val_metrics[0]) + handlers = est._prepare_default_handlers(val_data=None, event_handlers=[early_stopping]) + assert len(handlers) == 3 + assert isinstance(handlers[0], MetricHandler) + assert isinstance(handlers[2], LoggingHandler) diff --git a/tests/python/unittest/test_gluon_event_handler.py b/tests/python/unittest/test_gluon_event_handler.py index e151281ea9bd..cdb4264e18a0 100644 --- a/tests/python/unittest/test_gluon_event_handler.py +++ b/tests/python/unittest/test_gluon_event_handler.py @@ -19,10 +19,11 @@ import tempfile import mxnet as mx +from common import TemporaryDirectory from mxnet import nd from mxnet.gluon import nn, loss from mxnet.gluon.contrib.estimator import estimator, event_handler -from common import TemporaryDirectory + def _get_test_network(): net = nn.Sequential() @@ -92,10 +93,12 @@ def test_logging(): net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - ce_loss_metric = mx.metric.Loss(ce_loss.name) acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, metrics=acc) + train_metrics, val_metrics = est.prepare_loss_and_metrics() logging_handler = [event_handler.LoggingHandler(file_name=file_name, - file_location=tmpdir, train_metrics=[acc, ce_loss_metric])] + file_location=tmpdir, + train_metrics=train_metrics, + val_metrics=val_metrics)] est.fit(test_data, event_handlers=logging_handler, epochs=1) - assert os.path.isfile(output_dir) \ No newline at end of file + assert os.path.isfile(output_dir)