[MXNet-1349][Fit API]Add validation support and unit tests for fit() API

python/mxnet/gluon/estimator/estimator.py

@@ -19,6 +19,7 @@
 # pylint: disable=wildcard-import
 """Gluon Estimator"""
 
+import copy
 import warnings
 
 from .event_handler import LoggingHandler
@@ -64,17 +65,21 @@ def __init__(self, net,
             self.loss = [loss]
         else:
             self.loss = loss or []
+            if not self.loss:
+                raise ValueError("No loss specified, refer to gluon.loss.Loss")
             for l in self.loss:
-                if not isinstance(loss, gluon.loss.Loss):
+                if not isinstance(l, gluon.loss.Loss):
                     raise ValueError("loss must be a Loss or a list of Loss, refer to gluon.loss.Loss")
 
         if isinstance(metrics, EvalMetric):
-            self.metrics = [metrics]
+            self.train_metrics = [metrics]
         else:
-            self.metrics = metrics or []
-            for metric in self.metrics:
+            self.train_metrics = metrics or []
+            for metric in self.train_metrics:
                 if not isinstance(metric, EvalMetric):
                     raise ValueError("metrics must be a Metric or a list of Metric, refer to mxnet.metric.EvalMetric")
+        # Use same metrics for validation
+        self.test_metrics = copy.deepcopy(self.train_metrics)
 
         self.initializer = initializer
         # store training statistics
@@ -83,16 +88,21 @@ def __init__(self, net,
         self.train_stats['learning_rate'] = []
         # current step of the epoch
         self.train_stats['step'] = ''
-        for metric in self.metrics:
+        for metric in self.train_metrics:
             # record a history of metrics over each epoch
             self.train_stats['train_' + metric.name] = []
             # only record the latest metric numbers after each batch
             self.train_stats['batch_' + metric.name] = 0.
-        self.loss_metrics = []
+        for metric in self.test_metrics:
+            self.train_stats['val_' + metric.name] = []
+        self.train_loss_metrics = []
+        self.test_loss_metrics = []
         # using the metric wrapper for loss to record loss value
         for l in self.loss:
-            self.loss_metrics.append(Loss(l.name))
+            self.train_loss_metrics.append(Loss(l.name))
+            self.test_loss_metrics.append(Loss(l.name))
             self.train_stats['train_' + l.name] = []
+            self.train_stats['val_' + l.name] = []
             # only record the latest loss numbers after each batch
             self.train_stats['batch_' + l.name] = 0.
 
@@ -130,11 +140,13 @@ def __init__(self, net,
             self.trainers = [trainers]
         else:
             self.trainers = trainers or []
-        if not self.trainers:
-            warnings.warn("No trainer specified, default SGD optimizer "
-                          "with learning rate 0.001 is used.")
-            self.trainers = [gluon.Trainer(self.net.collect_params(),
-                                           'sgd', {'learning_rate': 0.001})]
+            if not self.trainers:
+                warnings.warn("No trainer specified, default SGD optimizer "
+                              "with learning rate 0.001 is used.")
+                self.trainers = [gluon.Trainer(self.net.collect_params(),
+                                               'sgd', {'learning_rate': 0.001})]
+            else:
+                raise ValueError("Invalid trainer specified, please provide a valid gluon.Trainer")
 
     def _is_initialized(self):
         param_dict = self.net.collect_params()
@@ -156,7 +168,33 @@ def _batch_fn(self, batch, ctx, is_iterator=False):
         label = gluon.utils.split_and_load(label, ctx_list=ctx, batch_axis=0)
         return data, label
 
+    def _evaluate(self, val_data, batch_fn=None):
+        for metric in self.test_metrics + self.test_loss_metrics:
+            metric.reset()
+
+        for _, batch in enumerate(val_data):
+            if not batch_fn:
+                if isinstance(val_data, gluon.data.DataLoader):
+                    data, label = self._batch_fn(batch, self.context)
+                elif isinstance(val_data, DataIter):
+                    data, label = self._batch_fn(batch, self.context, is_iterator=True)
+                else:
+                    raise ValueError("You are using a custom iteration, please also provide "
+                                     "batch_fn to extract data and label")
+            else:
+                data, label = batch_fn(batch, self.context)
+            pred = [self.net(x) for x in data]
+            losses = []
+            for loss in self.loss:
+                losses.append([loss(y_hat, y) for y_hat, y in zip(pred, label)])
+            # update metrics
+            for metric in self.test_metrics:
+                metric.update(label, pred)
+            for loss, loss_metric, in zip(losses, self.test_loss_metrics):
+                loss_metric.update(0, [l for l in loss])
+
     def fit(self, train_data,
+            val_data=None,
             epochs=1,
             batch_size=None,
             event_handlers=None,
@@ -192,6 +230,9 @@ def fit(self, train_data,
                 not any(isinstance(handler, LoggingHandler) for handler in event_handlers):
             event_handlers.append(LoggingHandler(self))
 
+        # Check for validation data
+        do_validation = True if val_data else False
+
         # training begin
         for handler in event_handlers:
             handler.train_begin()
@@ -204,7 +245,7 @@ def fit(self, train_data,
             for handler in event_handlers:
                 handler.epoch_begin()
 
-            for metric in self.metrics + self.loss_metrics:
+            for metric in self.train_metrics + self.train_loss_metrics:
                 metric.reset()
 
             for i, batch in enumerate(train_data):
@@ -233,11 +274,11 @@ def fit(self, train_data,
                     for l in loss:
                         l.backward()
 
-                # update metrics
-                for metric in self.metrics:
+                # update train metrics
+                for metric in self.train_metrics:
                     metric.update(label, pred)
                     self.train_stats['batch_' + metric.name] = metric.get()[1]
-                for loss, loss_metric, in zip(losses, self.loss_metrics):
+                for loss, loss_metric, in zip(losses, self.train_loss_metrics):
                     loss_metric.update(0, [l for l in loss])
                     self.train_stats['batch_' + loss_metric.name] = loss_metric.get()[1]
 
@@ -253,8 +294,14 @@ def fit(self, train_data,
                 for handler in event_handlers:
                     handler.batch_end()
 
-            for metric in self.metrics + self.loss_metrics:
+            if do_validation:
+                self._evaluate(val_data, batch_fn)
+
+            for metric in self.train_metrics + self.train_loss_metrics:
                 self.train_stats['train_' + metric.name].append(metric.get()[1])
+            for metric in self.test_metrics + self.test_loss_metrics:
+                self.train_stats['val_' + metric.name].append(metric.get()[1])
+
             # epoch end
             for handler in event_handlers:
                 handler.epoch_end()

python/mxnet/gluon/estimator/event_handler.py

@@ -118,7 +118,7 @@ def epoch_end(self):
         epoch = self._estimator.train_stats['epochs'][-1]
         msg = '\n[Epoch %d] finished in %.3fs: ' % (epoch, epoch_time)
         for key in self._estimator.train_stats.keys():
-            if key.startswith('train_') or key.startswith('test_'):
+            if key.startswith('train_') or key.startswith('val_'):
                 msg += key + ': ' + '%.4f ' % self._estimator.train_stats[key][epoch]
         self.logger.info(msg)