Merge branch 'deep_supervised_rnn' of https://github.com/kjw0612/rcn into deep_supervised_rnn

Author: kjw0612

rcn_train_dag.m

@@ -275,12 +275,16 @@
   net.params(i).der = min(max(net.params(i).der, -opts.gradRange/mult), opts.gradRange/mult);
   thisDecay = opts.weightDecay * net.params(i).weightDecay ;
 
+  momentum_prev = state.momentum{i};
   state.momentum{i} = opts.momentum * state.momentum{i} ...
     - lr * net.params(i).learningRate * ...
       thisDecay * net.params(i).value ...
     - lr * net.params(i).learningRate * (1 / batchSize) * net.params(i).der ;
 
-  net.params(i).value = net.params(i).value + state.momentum{i};
+  %Nesterov
+  net.params(i).value = net.params(i).value ...
+                        - opts.momentum * momentum_prev ...
+                        + (1 + opts.momentum) * state.momentum{i};
 end
 
 % -------------------------------------------------------------------------