diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
index 0bdc2cf8c92e..5e812c433b43 100644
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -29,6 +29,7 @@ COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
 COPY install/ubuntu_python.sh /work/
+COPY install/requirements /work/
 RUN /work/ubuntu_python.sh
 
 COPY install/ubuntu_scala.sh /work/
diff --git a/ci/other/pylintrc b/ci/other/pylintrc
index 841a3bea13fa..a1034cd50f9c 100644
--- a/ci/other/pylintrc
+++ b/ci/other/pylintrc
@@ -40,7 +40,7 @@ persistent=yes
 load-plugins=
 
 # Use multiple processes to speed up Pylint.
-jobs=8
+jobs=1
 
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
@@ -113,7 +113,10 @@ disable=
     too-many-instance-attributes,
     too-many-locals,
     too-many-public-methods,
-    too-many-statements
+    too-many-statements,
+    too-many-lines,
+    duplicate-code,
+    cyclic-import
 
 # disable=unicode-builtin,delslice-method,using-cmp-argument,setslice-method,dict-view-method,parameter-unpacking,range-builtin-not-iterating,print-statement,file-builtin,old-raise-syntax,basestring-builtin,execfile-builtin,indexing-exception,import-star-module-level,coerce-method,long-builtin,old-ne-operator,old-division,no-absolute-import,raw_input-builtin,old-octal-literal,oct-method,xrange-builtin,hex-method,unpacking-in-except,nonzero-method,raising-string,intern-builtin,reload-builtin,metaclass-assignment,cmp-method,filter-builtin-not-iterating,apply-builtin,map-builtin-not-iterating,next-method-called,unichr-builtin,buffer-builtin,dict-iter-method,input-builtin,coerce-builtin,getslice-method,useless-suppression,standarderror-builtin,zip-builtin-not-iterating,suppressed-message,cmp-builtin,backtick,long-suffix,reduce-builtin,round-builtin
 
diff --git a/docs/python_docs/_static/autodoc.js b/docs/python_docs/_static/autodoc.js
index 0825f54ea143..5b536ddcf6ae 100644
--- a/docs/python_docs/_static/autodoc.js
+++ b/docs/python_docs/_static/autodoc.js
@@ -20,30 +20,32 @@
 /* Customizations to the Sphinx auto module plugin output */
 function auto_index() {
     var targets = $("dl.class>dt,dl.function>dt");
-
     var li_node = $("li.current>span>a.current.reference.internal").parent().parent();
     var html = "<ul id='autodoc'>";
     if (li_node.length > 0) {
-        for (var i = 0; i < targets.length; ++i) {
-            var id = $(targets[i]).attr('id');
-            if (id) {
-                var paths = id.split('.')
-                if (paths.length >= 2) {
-                    var id_simple = paths.pop();
-                    id_simple = paths.pop() + "." + id_simple;
-                } else {
-                    var id_simple = id;
+        if (targets.length > 0) {
+            for (var i = 0; i < targets.length; ++i) {
+                var id = $(targets[i]).attr('id');
+                if (id) {
+                    var paths = id.split('.')
+                    if (paths.length >= 2) {
+                        var id_simple = paths.pop();
+                        id_simple = paths.pop() + "." + id_simple;
+                    } else {
+                        var id_simple = id;
+                    }
+                    html += "<li><span class='link-wrapper'><a class='reference internal' href='#";
+                    html += id;
+                    html += "'>" + id_simple + "</a></span</li>";
                 }
-                html += "<li><span class='link-wrapper'><a class='reference internal' href='#";
-                html += id;
-                html += "'>" + id_simple + "</a></span</li>";
             }
+            html += "</ul>";
+            li_node.append(html);
+            li_node.prepend("<a><span id='autodoc_toggle' onclick='$(\"#autodoc\").toggle()'>[toggle]</span></a>")
         }
-        html += "</ul>";
-        li_node.append(html);
-        li_node.prepend("<a><span id='autodoc_toggle' onclick='$(\"#autodoc\").toggle()'>[toggle]</span></a>")
     } else {
         setTimeout(auto_index, 500);
     }
+
 }
 $(document).ready(auto_index);
\ No newline at end of file
diff --git a/docs/static_site/src/_includes/get_started/get_started.html b/docs/static_site/src/_includes/get_started/get_started.html
index 614b4a409773..4905d28ce2d3 100644
--- a/docs/static_site/src/_includes/get_started/get_started.html
+++ b/docs/static_site/src/_includes/get_started/get_started.html
@@ -1,7 +1,7 @@
 <script>
     /** Defaults **/
     /** See options.js for the full ugly script **/
-    var versionSelect = defaultVersion = 'v1.5.0';
+    var versionSelect = defaultVersion = 'v1.5.1';
     var platformSelect = 'linux';
     var languageSelect = 'python';
     var processorSelect = 'cpu';
@@ -21,10 +21,10 @@ <h2>Installing MXNet</h2>
             </div>
             <div class="col-9 install-right">
                 <div class="dropdown">
-                    <button class="current-version dropbtn btn" type="button" data-toggle="dropdown">v1.5.0</button>
+                    <button class="current-version dropbtn btn" type="button" data-toggle="dropdown">v1.5.1</button>
                     <ul class="opt-group dropdown-content">
                         <li class="opt versions"><a href="#">master</a></li>
-                        <li class="opt active versions"><a href="#">v1.5.0</a></li>
+                        <li class="opt active versions"><a href="#">v1.5.1</a></li>
                         <li class="opt versions"><a href="#">v1.4.1</a></li>
                         <li class="opt versions"><a href="#">v1.3.1</a></li>
                         <li class="opt versions"><a href="#">v1.2.1</a></li>
diff --git a/docs/static_site/src/_includes/get_started/pip_snippet.md b/docs/static_site/src/_includes/get_started/pip_snippet.md
index 0ca854005ffd..f5cc4ea12803 100644
--- a/docs/static_site/src/_includes/get_started/pip_snippet.md
+++ b/docs/static_site/src/_includes/get_started/pip_snippet.md
@@ -3,7 +3,7 @@ Check the chart below for other options, refer to <a href="https://pypi.org/proj
 other MXNet pip packages</a>, or <a href="get_started/validate_mxnet.html">validate your MXNet installation</a>.
 
 <div style="text-align: center">
-    <img src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages.png"
+    <img src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages-1.5.1.png"
     alt="pip packages"/>
 </div>
 
diff --git a/docs/static_site/src/pages/get_started/download.md b/docs/static_site/src/pages/get_started/download.md
index 7d0176aef39c..b8f2e00024d4 100644
--- a/docs/static_site/src/pages/get_started/download.md
+++ b/docs/static_site/src/pages/get_started/download.md
@@ -29,6 +29,7 @@ These source archives are generated from tagged releases. Updates and patches wi
 
 | Version | Source                                                                                                      | PGP                                                                                                             | SHA                                                                                                                |
 |---------|-------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|
+| 1.5.1   | [Download](https://apache.org/dist/incubator/mxnet/1.5.1/apache-mxnet-src-1.5.1-incubating.tar.gz)                 | [Download](https://apache.org/dist/incubator/mxnet/1.5.1/apache-mxnet-src-1.5.1-incubating.tar.gz.asc)    |  [Download](https://apache.org/dist/incubator/mxnet/1.5.1/apache-mxnet-src-1.5.1-incubating.tar.gz.sha512)     |
 | 1.5.0   | [Download](https://apache.org/dist/incubator/mxnet/1.5.0/apache-mxnet-src-1.5.0-incubating.tar.gz)                 | [Download](https://apache.org/dist/incubator/mxnet/1.5.0/apache-mxnet-src-1.5.0-incubating.tar.gz.asc)    |  [Download](https://apache.org/dist/incubator/mxnet/1.5.0/apache-mxnet-src-1.5.0-incubating.tar.gz.sha512)     |
 | 1.4.1   | [Download](https://www.apache.org/dyn/closer.cgi/incubator/mxnet/1.4.1/apache-mxnet-src-1.4.1-incubating.tar.gz)   | [Download](https://apache.org/dist/incubator/mxnet/1.4.1/apache-mxnet-src-1.4.1-incubating.tar.gz.asc)    | [Download](https://apache.org/dist/incubator/mxnet/1.4.1/apache-mxnet-src-1.4.1-incubating.tar.gz.sha512)      |
 | 1.4.0   | [Download](https://www.apache.org/dyn/closer.cgi/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz)   | [Download](https://apache.org/dist/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz.asc)    | [Download](https://apache.org/dist/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz.sha512)      |
diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py
index 8ad5247fa263..667c3bddb71f 100644
--- a/python/mxnet/_numpy_op_doc.py
+++ b/python/mxnet/_numpy_op_doc.py
@@ -704,3 +704,47 @@ def _np_squeeze(a, axis=None, out=None):
     (1, 3)
     """
     pass
+
+
+def _np_moveaxis(a, source, destination):
+    """Move axes of an array to new positions.
+    Other axes remain in their original order.
+
+    Parameters
+    ----------
+    a : ndarray
+        The array whose axes should be reordered.
+        source : int or sequence of int
+        Original positions of the axes to move. These must be unique.
+        destination : int or sequence of int
+        Destination positions for each of the original axes. These must also be
+        unique.
+
+    Returns
+    -------
+    result : ndarray
+        Array with moved axes. This array is a view of the input array.
+
+    See Also
+    --------
+        transpose: Permute the dimensions of an array.
+        swapaxes: Interchange two axes of an array.
+
+    Examples
+    --------
+    >>> x = np.zeros((3, 4, 5))
+    >>> np.moveaxis(x, 0, -1).shape
+    (4, 5, 3)
+    >>> np.moveaxis(x, -1, 0).shape
+    (5, 3, 4)
+    These all achieve the same result:
+    >>> np.transpose(x).shape
+    (5, 4, 3)
+    >>> np.swapaxes(x, 0, -1).shape
+    (5, 4, 3)
+    >>> np.moveaxis(x, [0, 1], [-1, -2]).shape
+    (5, 4, 3)
+    >>> np.moveaxis(x, [0, 1, 2], [-1, -2, -3]).shape
+    (5, 4, 3)
+    """
+    pass
diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py
index dedd81496c2f..8800684ad0b4 100644
--- a/python/mxnet/gluon/parameter.py
+++ b/python/mxnet/gluon/parameter.py
@@ -24,9 +24,10 @@
            'ParameterDict', 'tensor_types']
 
 
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 import warnings
 import numpy as np
+import mxnet as mx
 
 from ..base import mx_real_t, MXNetError
 from .. import symbol, ndarray, initializer, context
@@ -887,8 +888,22 @@ def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False,
 
     def zero_grad(self):
         """Sets all Parameters' gradient buffer to 0."""
-        for i in self.values():
-            i.zero_grad()
+        # collect gradient arrays for each ctx
+        arrays = defaultdict(list)
+        for p in self.values():
+            if p.grad_req == 'null' or p._grad is None:
+                continue
+            for g in p.list_grad():
+                if g.stype == 'row_sparse':
+                    mx.ndarray.zeros_like(g, out=g)
+                else:
+                    arrays[g.context].append(g)
+
+        if len(arrays) == 0:
+            return
+
+        for arr in arrays.values():
+            mx.nd.reset_arrays(*arr, num_arrays=len(arr))
 
     def reset_ctx(self, ctx):
         """Re-assign all Parameters to other contexts.
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index e3cf09703533..fbdb8e98a15f 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -22,7 +22,7 @@
 from __future__ import absolute_import
 import numpy as _np
 from ...base import numeric_types
-from ...util import set_module
+from ...util import _sanity_check_params, set_module
 from ...context import current_context
 from . import _internal as _npi
 from ..ndarray import NDArray
@@ -31,12 +31,13 @@
            'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs',
            'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2',
            'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor',
-           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'tensordot', 'histogram',
-           'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean',
-           'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign',
-           'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'hypot', 'rad2deg', 'deg2rad',
-           'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',
-           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal']
+           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'tensordot', 'histogram', 'eye',
+           'linspace', 'logspace', 'expand_dims', 'tile', 'arange', 'split', 'vsplit', 'concatenate',
+           'stack', 'vstack', 'dstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax',
+           'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip',
+           'around', 'hypot', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril', 'identity', 'take',
+           'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal',
+           'hsplit', 'rot90']
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -788,11 +789,41 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, density=None):
     raise ValueError("np.histogram fails with", locals())
 
 
+@set_module('mxnet.ndarray.numpy')
+def eye(N, M=None, k=0, dtype=_np.float32, **kwargs):
+    """
+    Return a 2-D array with ones on the diagonal and zeros elsewhere.
+
+    Parameters
+    ----------
+    N : int
+        Number of rows in the output.
+    M : int, optional
+        Number of columns in the output. If None, defaults to N.
+    k : int, optional
+        Index of the diagonal: 0 (the default) refers to the main diagonal,
+        a positive value refers to an upper diagonal,
+        and a negative value to a lower diagonal.
+    dtype : data-type, optional
+        Data-type of the returned array.
+
+    Returns
+    -------
+    I : ndarray of shape (N,M)
+        An array where all elements are equal to zero,
+        except for the k-th diagonal, whose values are equal to one.
+    """
+    _sanity_check_params('eye', ['order'], kwargs)
+    ctx = kwargs.pop('ctx', current_context())
+    if ctx is None:
+        ctx = current_context()
+    return _npi.eye(N, M, k, ctx, dtype)
+
+
 @set_module('mxnet.ndarray.numpy')
 def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0, ctx=None):  # pylint: disable=too-many-arguments
     r"""
     Return evenly spaced numbers over a specified interval.
-
     Returns num evenly spaced samples, calculated over the interval [start, stop].
     The endpoint of the interval can optionally be excluded.
 
@@ -886,6 +917,89 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis
         return _npi.linspace(start=start, stop=stop, num=num, endpoint=endpoint, ctx=ctx, dtype=dtype)
 
 
+@set_module('mxnet.ndarray.numpy')
+def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0, ctx=None):  # pylint: disable=too-many-arguments
+    r"""Return numbers spaced evenly on a log scale.
+
+    In linear space, the sequence starts at ``base ** start``
+    (`base` to the power of `start`) and ends with ``base ** stop``
+    (see `endpoint` below).
+
+        Non-scalar `start` and `stop` are now supported.
+
+    Parameters
+    ----------
+    start : int or float
+        ``base ** start`` is the starting value of the sequence.
+    stop : int or float
+        ``base ** stop`` is the final value of the sequence, unless `endpoint`
+        is False.  In that case, ``num + 1`` values are spaced over the
+        interval in log-space, of which all but the last (a sequence of
+        length `num`) are returned.
+    num : integer, optional
+        Number of samples to generate.  Default is 50.
+    endpoint : boolean, optional
+        If true, `stop` is the last sample. Otherwise, it is not included.
+        Default is True.
+    base : float, optional
+        The base of the log space. The step size between the elements in
+        ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform.
+        Default is 10.0.
+    dtype : dtype
+        The type of the output array.  If `dtype` is not given, infer the data
+        type from the other input arguments.
+    axis : int, optional
+        The axis in the result to store the samples.  Relevant only if start
+        or stop are array-like.  By default (0), the samples will be along a
+        new axis inserted at the beginning. Now, axis only support axis = 0.
+    ctx : Context, optional
+        An optional device context (default is the current default context).
+
+    Returns
+    -------
+    samples : ndarray
+        `num` samples, equally spaced on a log scale.
+
+    See Also
+    --------
+    arange : Similar to linspace, with the step size specified instead of the
+             number of samples. Note that, when used with a float endpoint, the
+             endpoint may or may not be included.
+    linspace : Similar to logspace, but with the samples uniformly distributed
+               in linear space, instead of log space.
+
+    Notes
+    -----
+    Logspace is equivalent to the code. Now wo only support axis = 0.
+
+    >>> y = np.linspace(start, stop, num=num, endpoint=endpoint)
+    ...
+    >>> power(base, y).astype(dtype)
+    ...
+
+    Examples
+    --------
+    >>> np.logspace(2.0, 3.0, num=4)
+    array([ 100.     ,  215.44347,  464.15887, 1000.     ])
+    >>> np.logspace(2.0, 3.0, num=4, endpoint=False)
+    array([100.     , 177.82794, 316.22775, 562.3413 ])
+    >>> np.logspace(2.0, 3.0, num=4, base=2.0)
+    array([4.       , 5.0396843, 6.349604 , 8.       ])
+    >>> np.logspace(2.0, 3.0, num=4, base=2.0, dtype=np.int32)
+    array([4, 5, 6, 8], dtype=int32)
+    >>> np.logspace(2.0, 3.0, num=4, ctx=npx.gpu(0))
+    array([ 100.     ,  215.44347,  464.15887, 1000.     ], ctx=gpu(0))
+    """
+    if isinstance(start, (list, tuple, _np.ndarray, NDArray)) or \
+       isinstance(stop, (list, tuple, _np.ndarray, NDArray)):
+        raise NotImplementedError('start and stop only support int and float')
+    if axis != 0:
+        raise NotImplementedError("the function only support axis 0")
+    if ctx is None:
+        ctx = current_context()
+    return _npi.logspace(start=start, stop=stop, num=num, endpoint=endpoint, base=base, ctx=ctx, dtype=dtype)
+
+
 @set_module('mxnet.ndarray.numpy')
 def expand_dims(a, axis):
     """Expand the shape of an array.
@@ -2323,7 +2437,7 @@ def split(ary, indices_or_sections, axis=0):
     ----------
     ary : ndarray
         Array to be divided into sub-arrays.
-    indices_or_sections : int or 1-D array
+    indices_or_sections : int or 1-D python tuple, list or set.
         If `indices_or_sections` is an integer, N, the array will be divided
         into N equal arrays along `axis`.  If such a split is not possible,
         an error is raised.
@@ -2355,10 +2469,10 @@ def split(ary, indices_or_sections, axis=0):
             raise ValueError('array split does not result in an equal division')
         section_size = int(axis_size / sections)
         indices = [i * section_size for i in range(sections)]
-    elif isinstance(indices_or_sections, tuple):
+    elif isinstance(indices_or_sections, (list, set, tuple)):
         indices = [0] + list(indices_or_sections)
     else:
-        raise ValueError('indices_or_sections must either int or tuple of ints')
+        raise ValueError('indices_or_sections must either int, or tuple / list / set of ints')
     ret = _npi.split(ary, indices, axis, False)
     if not isinstance(ret, list):
         return [ret]
@@ -2366,6 +2480,198 @@ def split(ary, indices_or_sections, axis=0):
 # pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
+@set_module('mxnet.ndarray.numpy')
+def hsplit(ary, indices_or_sections):
+    """Split an array into multiple sub-arrays horizontally (column-wise).
+
+    This is equivalent to ``split`` with ``axis=0`` if ``ary`` has one
+    dimension, and otherwise that with ``axis=1``.
+
+    Parameters
+    ----------
+    ary : ndarray
+        Array to be divided into sub-arrays.
+    indices_or_sections : int, list of ints or tuple of ints.
+        If `indices_or_sections` is an integer, N, the array will be divided
+        into N equal arrays along `axis`.  If such a split is not possible,
+        an error is raised.
+
+        If `indices_or_sections` is a list of sorted integers, the entries
+        indicate where along `axis` the array is split.
+
+        If an index exceeds the dimension of the array along `axis`,
+        it will raises errors. so index must less than or euqal to
+        the dimension of the array along axis.
+
+    Returns
+    -------
+    sub-arrays : list of ndarrays
+        A list of sub-arrays.
+
+    Notes
+    ------
+    - If `indices_or_sections` is given as an integer, but a split
+      does not result in equal division.It will raises ValueErrors.
+
+    - If indices_or_sections is an integer, and the number is 1, it will
+      raises an error. Because single output from split is not supported yet...
+
+    See Also
+    --------
+    split : Split an array into multiple sub-arrays of equal size.
+
+    Examples
+    --------
+    >>> x = np.arange(16.0).reshape(4, 4)
+    >>> x
+    array([[ 0.,  1.,  2.,  3.],
+           [ 4.,  5.,  6.,  7.],
+           [ 8.,  9., 10., 11.],
+           [12., 13., 14., 15.]])
+    >>> np.hsplit(x, 2)
+    [array([[ 0.,  1.],
+           [ 4.,  5.],
+           [ 8.,  9.],
+           [12., 13.]]),
+    array([[ 2.,  3.],
+           [ 6.,  7.],
+           [10., 11.],
+           [14., 15.]])]
+    >>> np.hsplit(x, [3, 6])
+    [array([[ 0.,  1.,  2.],
+           [ 4.,  5.,  6.],
+           [ 8.,  9., 10.],
+           [12., 13., 14.]]),
+    array([[ 3.],
+           [ 7.],
+           [11.],
+           [15.]]),
+    array([], shape=(4, 0), dtype=float32)]
+
+    With a higher dimensional array the split is still along the second axis.
+
+    >>> x = np.arange(8.0).reshape(2, 2, 2)
+    >>> x
+    array([[[ 0.,  1.],
+            [ 2.,  3.]],
+           [[ 4.,  5.],
+            [ 6.,  7.]]])
+    >>> np.hsplit(x, 2)
+    [array([[[ 0.,  1.]],
+            [[ 4.,  5.]]]),
+     array([[[ 2.,  3.]],
+            [[ 6.,  7.]]])]
+
+    If ``ary`` has one dimension, 'axis' = 0.
+    >>> x = np.arange(4)
+    array([0., 1., 2., 3.])
+    >>> np.hsplit(x, 2)
+    [array([0., 1.]), array([2., 3.])]
+
+    If you want to produce an empty sub-array, you can see an example.
+    >>> np.hsplit(x, [2, 2])
+    [array([0., 1.]), array([], dtype=float32), array([2., 3.])]
+    """
+    indices = []
+    axis = 1
+    if (len(ary.shape) == 1):
+        axis = 0
+    axis_size = ary.shape[axis]
+    if isinstance(indices_or_sections, int):
+        sections = indices_or_sections
+        if axis_size % sections:
+            raise ValueError('array hsplit does not result in an equal division')
+        section_size = int(axis_size / sections)
+        indices = [i * section_size for i in range(sections)]
+    elif isinstance(indices_or_sections, (list, set, tuple)):
+        indices = [0] + list(indices_or_sections)
+    else:
+        raise ValueError('indices_or_sections must either int or tuple of ints')
+    ret = _npi.hsplit(ary, indices, axis, False)
+    if not isinstance(ret, list):
+        raise NotImplementedError('single output from hsplit is not supported yet...')
+    return ret
+# pylint: enable=redefined-outer-name
+
+
+@set_module('mxnet.ndarray.numpy')
+def vsplit(ary, indices_or_sections):
+    r"""
+    vsplit(ary, indices_or_sections)
+
+    Split an array into multiple sub-arrays vertically (row-wise).
+
+    ``vsplit`` is equivalent to ``split`` with `axis=0` (default): the array is always split
+    along the first axis regardless of the array dimension.
+
+    Parameters
+    ----------
+    ary : ndarray
+        Array to be divided into sub-arrays.
+    indices_or_sections : int or 1 - D Python tuple, list or set.
+        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays
+        along axis 0.  If such a split is not possible, an error is raised.
+
+        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where
+        along axis 0 the array is split.  For example, ``[2, 3]`` would result in
+
+          - ary[:2]
+          - ary[2:3]
+          - ary[3:]
+
+        If an index exceeds the dimension of the array along axis 0, an error will be thrown.
+
+    Returns
+    -------
+    sub-arrays : list of ndarrays
+        A list of sub-arrays.
+
+    See Also
+    --------
+    split : Split an array into multiple sub-arrays of equal size.
+
+    Notes
+    -------
+    This function differs from the original `numpy.degrees
+    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in
+    the following aspects:
+
+    - Currently parameter ``indices_or_sections`` does not support ndarray, but supports scalar,
+    tuple and list.
+    - In ``indices_or_sections``, if an index exceeds the dimension of the array along axis 0,
+    an error will be thrown.
+
+    Examples
+    --------
+    >>> x = np.arange(16.0).reshape(4, 4)
+    >>> x
+    array([[  0.,   1.,   2.,   3.],
+           [  4.,   5.,   6.,   7.],
+           [  8.,   9.,  10.,  11.],
+           [ 12.,  13.,  14.,  15.]])
+    >>> np.vsplit(x, 2)
+    [array([[0., 1., 2., 3.],
+            [4., 5., 6., 7.]]), array([[ 8.,  9., 10., 11.],
+            [12., 13., 14., 15.]])]
+
+    With a higher dimensional array the split is still along the first axis.
+
+    >>> x = np.arange(8.0).reshape(2, 2, 2)
+    >>> x
+    array([[[ 0.,  1.],
+            [ 2.,  3.]],
+           [[ 4.,  5.],
+            [ 6.,  7.]]])
+    >>> np.vsplit(x, 2)
+    [array([[[0., 1.],
+            [2., 3.]]]), array([[[4., 5.],
+            [6., 7.]]])]
+
+    """
+    return split(ary, indices_or_sections, 0)
+
+
 @set_module('mxnet.ndarray.numpy')
 def concatenate(seq, axis=0, out=None):
     """Join a sequence of arrays along an existing axis.
@@ -2467,6 +2773,48 @@ def get_list(arrays):
     return _npi.vstack(*arrays)
 
 
+@set_module('mxnet.ndarray.numpy')
+def dstack(arrays):
+    """
+    Stack arrays in sequence depth wise (along third axis).
+    This is equivalent to concatenation along the third axis after 2-D arrays
+    of shape `(M,N)` have been reshaped to `(M,N,1)` and 1-D arrays of shape
+    `(N,)` have been reshaped to `(1,N,1)`. Rebuilds arrays divided by
+    `dsplit`.
+    This function makes most sense for arrays with up to 3 dimensions. For
+    instance, for pixel-data with a height (first axis), width (second axis),
+    and r/g/b channels (third axis). The functions `concatenate`, `stack` and
+    `block` provide more general stacking and concatenation operations.
+
+    Parameters
+    ----------
+    tup : sequence of arrays
+        The arrays must have the same shape along all but the third axis.
+        1-D or 2-D arrays must have the same shape.
+
+    Returns
+    -------
+    stacked : ndarray
+        The array formed by stacking the given arrays, will be at least 3-D.
+
+    Examples
+    --------
+    >>> a = np.array((1,2,3))
+    >>> b = np.array((2,3,4))
+    >>> np.dstack((a,b))
+    array([[[1, 2],
+            [2, 3],
+            [3, 4]]])
+    >>> a = np.array([[1],[2],[3]])
+    >>> b = np.array([[2],[3],[4]])
+    >>> np.dstack((a,b))
+    array([[[1, 2]],
+           [[2, 3]],
+           [[3, 4]]])
+    """
+    return _npi.dstack(*arrays)
+
+
 @set_module('mxnet.ndarray.numpy')
 def maximum(x1, x2, out=None):
     """Returns element-wise maximum of the input arrays with broadcasting.
@@ -3892,3 +4240,49 @@ def less_equal(x1, x2, out=None):
     """
     return _ufunc_helper(x1, x2, _npi.less_equal, _np.less_equal, _npi.less_equal_scalar,
                          _npi.greater_equal_scalar, out)
+
+
+@set_module('mxnet.ndarray.numpy')
+def rot90(m, k=1, axes=(0, 1)):
+    """
+    Rotate an array by 90 degrees in the plane specified by axes.
+    Rotation direction is from the first towards the second axis.
+    Parameters
+    ----------
+    m : ndarray
+        Array of two or more dimensions.
+    k : integer
+        Number of times the array is rotated by 90 degrees.
+    axes: (2,) array_like
+        The array is rotated in the plane defined by the axes.
+        Axes must be different.
+
+    Returns
+    -------
+    y : ndarray
+        A rotated view of `m`.
+
+    -----
+    rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1))
+    rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1))
+    Examples
+    --------
+    >>> m = np.array([[1,2],[3,4]], 'int')
+    >>> m
+    array([[1, 2],
+           [3, 4]], dtype=int64)
+    >>> np.rot90(m)
+    array([[2, 4],
+           [1, 3]], dtype=int64)
+    >>> np.rot90(m, 2)
+    array([[4, 3],
+           [2, 1]], dtype=int64)
+    >>> m = np.arange(8).reshape((2,2,2))
+    >>> np.rot90(m, 1, (1,2))
+    array([[[1., 3.],
+            [0., 2.]],
+
+           [[5., 7.],
+            [4., 6.]]])
+    """
+    return _npi.rot90(m, k=k, axes=axes)
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 5ee52f14bb16..70977c0b7f2d 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -50,12 +50,12 @@
            'sqrt', 'cbrt', 'abs', 'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log',
            'degrees', 'log2', 'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative',
            'fix', 'ceil', 'floor', 'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh',
-           'tensordot', 'histogram', 'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate',
-           'stack', 'vstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices',
-           'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'arctan2', 'hypot',
-           'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',
-           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal']
-
+           'tensordot', 'histogram', 'eye', 'linspace', 'logspace', 'expand_dims', 'tile', 'arange',
+           'split', 'vsplit', 'concatenate', 'stack', 'vstack', 'dstack', 'mean', 'maximum', 'minimum',
+           'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming',
+           'blackman', 'flip', 'around', 'arctan2', 'hypot', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril',
+           'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal', 'greater', 'less',
+           'greater_equal', 'less_equal', 'hsplit', 'rot90']
 
 # Return code for dispatching indexing function call
 _NDARRAY_UNSUPPORTED_INDEXING = -1
@@ -706,7 +706,7 @@ def __repr__(self):
             if dtype == _np.float32:
                 array_str = array_str[:array_str.rindex(',')] + ')'
         elif dtype not in (_np.float32, _np.bool_):
-            array_str = array_str[:-1] + ', dtype={})'.format(dtype.__name__)
+            array_str = array_str[:-1] + ', dtype={})'.format(dtype)
 
         context = self.context
         if context.device_type == 'cpu':
@@ -1680,6 +1680,26 @@ def size(self):
         """Number of elements in the array."""
         return super(ndarray, self).size
 
+    @property
+    def dtype(self):
+        """Data-type of the array's elements.
+
+        Returns
+        -------
+        numpy.dtype
+            This NDArray's data type.
+
+        Examples
+        --------
+        >>> x = np.zeros((2,3))
+        >>> x.dtype
+        dtype('float32')
+        >>> y = np.zeros((2,3), dtype='int32')
+        >>> y.dtype
+        dtype('int32')
+        """
+        return _np.dtype(super(ndarray, self).dtype)
+
     def tostype(self, stype):
         raise AttributeError('mxnet.numpy.ndarray object has no attribute tostype')
 
@@ -3645,6 +3665,33 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, density=None):
     return _mx_nd_np.histogram(a, bins=bins, range=range, normed=normed, weights=weights, density=density)
 
 
+@set_module('mxnet.numpy')
+def eye(N, M=None, k=0, dtype=_np.float32, **kwargs):
+    """
+    Return a 2-D array with ones on the diagonal and zeros elsewhere.
+
+    Parameters
+    ----------
+    N : int
+        Number of rows in the output.
+    M : int, optional
+        Number of columns in the output. If None, defaults to N.
+    k : int, optional
+        Index of the diagonal: 0 (the default) refers to the main diagonal,
+        a positive value refers to an upper diagonal,
+        and a negative value to a lower diagonal.
+    dtype : data-type, optional
+        Data-type of the returned array.
+
+    Returns
+    -------
+    I : ndarray of shape (N,M)
+        An array where all elements are equal to zero,
+        except for the k-th diagonal, whose values are equal to one.
+    """
+    return _mx_nd_np.eye(N, M, k, dtype, **kwargs)
+
+
 @set_module('mxnet.numpy')
 def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0, ctx=None):  # pylint: disable=too-many-arguments
     r"""
@@ -3732,6 +3779,82 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis
     return _mx_nd_np.linspace(start, stop, num, endpoint, retstep, dtype, axis, ctx)
 
 
+@set_module('mxnet.numpy')
+def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0, ctx=None):
+    r"""Return numbers spaced evenly on a log scale.
+
+    In linear space, the sequence starts at ``base ** start``
+    (`base` to the power of `start`) and ends with ``base ** stop``
+    (see `endpoint` below).
+
+        Non-scalar `start` and `stop` are now supported.
+
+    Parameters
+    ----------
+    start : int or float
+        ``base ** start`` is the starting value of the sequence.
+    stop : int or float
+        ``base ** stop`` is the final value of the sequence, unless `endpoint`
+        is False.  In that case, ``num + 1`` values are spaced over the
+        interval in log-space, of which all but the last (a sequence of
+        length `num`) are returned.
+    num : integer, optional
+        Number of samples to generate.  Default is 50.
+    endpoint : boolean, optional
+        If true, `stop` is the last sample. Otherwise, it is not included.
+        Default is True.
+    base : float, optional
+        The base of the log space. The step size between the elements in
+        ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform.
+        Default is 10.0.
+    dtype : dtype
+        The type of the output array.  If `dtype` is not given, infer the data
+        type from the other input arguments.
+    axis : int, optional
+        The axis in the result to store the samples.  Relevant only if start
+        or stop are array-like.  By default (0), the samples will be along a
+        new axis inserted at the beginning. Now, axis only support axis = 0.
+    ctx : Context, optional
+        An optional device context (default is the current default context).
+
+    Returns
+    -------
+    samples : ndarray
+        `num` samples, equally spaced on a log scale.
+
+    See Also
+    --------
+    arange : Similar to linspace, with the step size specified instead of the
+             number of samples. Note that, when used with a float endpoint, the
+             endpoint may or may not be included.
+    linspace : Similar to logspace, but with the samples uniformly distributed
+               in linear space, instead of log space.
+
+    Notes
+    -----
+    Logspace is equivalent to the code
+
+    >>> y = np.linspace(start, stop, num=num, endpoint=endpoint)
+    ...
+    >>> power(base, y).astype(dtype)
+    ...
+
+    Examples
+    --------
+    >>> np.logspace(2.0, 3.0, num=4)
+    array([ 100.     ,  215.44347,  464.15887, 1000.     ])
+    >>> np.logspace(2.0, 3.0, num=4, endpoint=False)
+    array([100.     , 177.82794, 316.22775, 562.3413 ])
+    >>> np.logspace(2.0, 3.0, num=4, base=2.0)
+    array([4.       , 5.0396843, 6.349604 , 8.       ])
+    >>> np.logspace(2.0, 3.0, num=4, base=2.0, dtype=np.int32)
+    array([4, 5, 6, 8], dtype=int32)
+    >>> np.logspace(2.0, 3.0, num=4, ctx=npx.gpu(0))
+    array([ 100.     ,  215.44347,  464.15887, 1000.     ], ctx=gpu(0))
+    """
+    return _mx_nd_np.logspace(start, stop, num, endpoint, base, dtype, axis, ctx=ctx)
+
+
 @set_module('mxnet.numpy')
 def expand_dims(a, axis):
     """Expand the shape of an array.
@@ -3904,7 +4027,7 @@ def split(ary, indices_or_sections, axis=0):
     ----------
     ary : ndarray
         Array to be divided into sub-arrays.
-    indices_or_sections : int or 1-D array
+    indices_or_sections : int or 1-D Python tuple, list or set.
         If `indices_or_sections` is an integer, N, the array will be divided
         into N equal arrays along `axis`.  If such a split is not possible,
         an error is raised.
@@ -3930,6 +4053,82 @@ def split(ary, indices_or_sections, axis=0):
     return _mx_nd_np.split(ary, indices_or_sections, axis=axis)
 
 
+@set_module('mxnet.numpy')
+def vsplit(ary, indices_or_sections):
+    r"""
+    vsplit(ary, indices_or_sections)
+
+    Split an array into multiple sub-arrays vertically (row-wise).
+
+    ``vsplit`` is equivalent to ``split`` with `axis=0` (default): the array is always split
+    along the first axis regardless of the array dimension.
+
+    Parameters
+    ----------
+    ary : ndarray
+        Array to be divided into sub-arrays.
+    indices_or_sections : int or 1 - D Python tuple, list or set.
+        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays
+        along axis 0.  If such a split is not possible, an error is raised.
+
+        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where
+        along axis 0 the array is split.  For example, ``[2, 3]`` would result in
+
+          - ary[:2]
+          - ary[2:3]
+          - ary[3:]
+
+        If an index exceeds the dimension of the array along axis 0, an error will be thrown.
+
+    Returns
+    -------
+    sub-arrays : list of ndarrays
+        A list of sub-arrays.
+
+    See Also
+    --------
+    split : Split an array into multiple sub-arrays of equal size.
+
+    Notes
+    -------
+    This function differs from the original `numpy.degrees
+    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in
+    the following aspects:
+
+    - Currently parameter ``indices_or_sections`` does not support ndarray, but supports scalar,
+    tuple and list.
+    - In ``indices_or_sections``, if an index exceeds the dimension of the array along axis 0,
+    an error will be thrown.
+
+    Examples
+    --------
+    >>> x = np.arange(16.0).reshape(4, 4)
+    >>> x
+    array([[  0.,   1.,   2.,   3.],
+           [  4.,   5.,   6.,   7.],
+           [  8.,   9.,  10.,  11.],
+           [ 12.,  13.,  14.,  15.]])
+    >>> np.vsplit(x, 2)
+    [array([[0., 1., 2., 3.],
+            [4., 5., 6., 7.]]), array([[ 8.,  9., 10., 11.],
+            [12., 13., 14., 15.]])]
+
+    >>> # With a higher dimensional array the split is still along the first axis.
+    >>> x = np.arange(8.0).reshape(2, 2, 2)
+    >>> x
+    array([[[ 0.,  1.],
+            [ 2.,  3.]],
+           [[ 4.,  5.],
+            [ 6.,  7.]]])
+    >>> np.vsplit(x, 2)
+    [array([[[0., 1.],
+            [2., 3.]]]), array([[[4., 5.],
+            [6., 7.]]])]
+
+    """
+    return split(ary, indices_or_sections, 0)
+
+
 @set_module('mxnet.numpy')
 def concatenate(seq, axis=0, out=None):
     """Join a sequence of arrays along an existing axis.
@@ -4019,6 +4218,50 @@ def vstack(arrays, out=None):
     return _mx_nd_np.vstack(arrays)
 
 
+@set_module('mxnet.numpy')
+def dstack(arrays):
+    """
+    Stack arrays in sequence depth wise (along third axis).
+
+    This is equivalent to concatenation along the third axis after 2-D arrays
+    of shape `(M,N)` have been reshaped to `(M,N,1)` and 1-D arrays of shape
+    `(N,)` have been reshaped to `(1,N,1)`. Rebuilds arrays divided by
+    `dsplit`.
+
+    This function makes most sense for arrays with up to 3 dimensions. For
+    instance, for pixel-data with a height (first axis), width (second axis),
+    and r/g/b channels (third axis). The functions `concatenate`, `stack` and
+    `block` provide more general stacking and concatenation operations.
+
+    Parameters
+    ----------
+    tup : sequence of arrays
+        The arrays must have the same shape along all but the third axis.
+        1-D or 2-D arrays must have the same shape.
+
+    Returns
+    -------
+    stacked : ndarray
+        The array formed by stacking the given arrays, will be at least 3-D.
+
+    Examples
+    --------
+    >>> a = np.array((1,2,3))
+    >>> b = np.array((2,3,4))
+    >>> np.dstack((a,b))
+    array([[[1, 2],
+            [2, 3],
+            [3, 4]]])
+    >>> a = np.array([[1],[2],[3]])
+    >>> b = np.array([[2],[3],[4]])
+    >>> np.dstack((a,b))
+    array([[[1, 2]],
+           [[2, 3]],
+           [[3, 4]]])
+    """
+    return _npi.dstack(*arrays)
+
+
 @set_module('mxnet.numpy')
 def maximum(x1, x2, out=None):
     """Returns element-wise maximum of the input arrays with broadcasting.
@@ -5406,3 +5649,136 @@ def less_equal(x1, x2, out=None):
     array([True])
     """
     return _mx_nd_np.less_equal(x1, x2, out)
+
+
+@set_module('mxnet.numpy')
+def rot90(m, k=1, axes=(0, 1)):
+    """
+    Rotate an array by 90 degrees in the plane specified by axes.
+    Rotation direction is from the first towards the second axis.
+    Parameters
+    ----------
+    m : ndarray
+        Array of two or more dimensions.
+    k : integer
+        Number of times the array is rotated by 90 degrees.
+    axes: (2,) array_like
+        The array is rotated in the plane defined by the axes.
+        Axes must be different.
+
+    Returns
+    -------
+    y : ndarray
+        A rotated view of `m`.
+
+    -----
+    rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1))
+    rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1))
+    Examples
+    --------
+    >>> m = np.array([[1,2],[3,4]], 'int')
+    >>> m
+    array([[1, 2],
+           [3, 4]], dtype=int64)
+    >>> np.rot90(m)
+    array([[2, 4],
+           [1, 3]], dtype=int64)
+    >>> np.rot90(m, 2)
+    array([[4, 3],
+           [2, 1]], dtype=int64)
+    >>> m = np.arange(8).reshape((2,2,2))
+    >>> np.rot90(m, 1, (1,2))
+    array([[[1., 3.],
+            [0., 2.]],
+
+           [[5., 7.],
+            [4., 6.]]])
+    """
+    return _mx_nd_np.rot90(m, k=k, axes=axes)
+
+
+@set_module('mxnet.numpy')
+def hsplit(ary, indices_or_sections):
+    """Split an array into multiple sub-arrays horizontally (column-wise).
+    This is equivalent to ``split`` with ``axis=0`` if ``ary`` has one
+    dimension, and otherwise that with ``axis=1``.
+
+    Parameters
+    ----------
+    ary : ndarray
+        Array to be divided into sub-arrays.
+    indices_or_sections : int, list of ints or tuple of ints.
+        If `indices_or_sections` is an integer, N, the array will be divided
+        into N equal arrays along `axis`.  If such a split is not possible,
+        an error is raised.
+        If `indices_or_sections` is a list of sorted integers, the entries
+        indicate where along `axis` the array is split.
+        If an index exceeds the dimension of the array along `axis`,
+        it will raises errors. so index must less than or euqal to
+        the dimension of the array along axis.
+
+    Returns
+    -------
+    sub-arrays : list of ndarrays
+        A list of sub-arrays.
+
+    Notes
+    ------
+    - If `indices_or_sections` is given as an integer, but a split
+      does not result in equal division.It will raises ValueErrors.
+    - If indices_or_sections is an integer, and the number is 1, it will
+      raises an error. Because single output from split is not supported yet...
+
+    See Also
+    --------
+    split : Split an array into multiple sub-arrays of equal size.
+
+    Examples
+    --------
+    >>> x = np.arange(16.0).reshape(4, 4)
+    >>> x
+    array([[ 0.,  1.,  2.,  3.],
+           [ 4.,  5.,  6.,  7.],
+           [ 8.,  9., 10., 11.],
+           [12., 13., 14., 15.]])
+    >>> np.hsplit(x, 2)
+    [array([[ 0.,  1.],
+           [ 4.,  5.],
+           [ 8.,  9.],
+           [12., 13.]]),
+    array([[ 2.,  3.],
+           [ 6.,  7.],
+           [10., 11.],
+           [14., 15.]])]
+    >>> np.hsplit(x, [3, 6])
+    [array([[ 0.,  1.,  2.],
+           [ 4.,  5.,  6.],
+           [ 8.,  9., 10.],
+           [12., 13., 14.]]),
+    array([[ 3.],
+           [ 7.],
+           [11.],
+           [15.]]),
+    array([], shape=(4, 0), dtype=float32)]
+    With a higher dimensional array the split is still along the second axis.
+    >>> x = np.arange(8.0).reshape(2, 2, 2)
+    >>> x
+    array([[[ 0.,  1.],
+            [ 2.,  3.]],
+           [[ 4.,  5.],
+            [ 6.,  7.]]])
+    >>> np.hsplit(x, 2)
+    [array([[[ 0.,  1.]],
+            [[ 4.,  5.]]]),
+     array([[[ 2.,  3.]],
+            [[ 6.,  7.]]])]
+    If ``ary`` has one dimension, 'axis' = 0.
+    >>> x = np.arange(4)
+    array([0., 1., 2., 3.])
+    >>> np.hsplit(x, 2)
+    [array([0., 1.]), array([2., 3.])]
+    If you want to produce an empty sub-array, you can see an example.
+    >>> np.hsplit(x, [2, 2])
+    [array([0., 1.]), array([], dtype=float32), array([2., 3.])]
+    """
+    return _mx_nd_np.hsplit(ary, indices_or_sections)
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index f483e299ffed..b02a7e53cfbf 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -98,8 +98,10 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'min',
     'ones_like',
     'prod',
+    'ravel',
     'repeat',
     'reshape',
+    'roll',
     'split',
     'squeeze',
     'stack',
diff --git a/python/mxnet/numpy_op_signature.py b/python/mxnet/numpy_op_signature.py
index 314de302109d..3e8b6d578537 100644
--- a/python/mxnet/numpy_op_signature.py
+++ b/python/mxnet/numpy_op_signature.py
@@ -19,6 +19,7 @@
 
 import sys
 import warnings
+import inspect
 from . import _numpy_op_doc
 from . import numpy as mx_np
 from . import numpy_extension as mx_npx
@@ -38,13 +39,12 @@ def _get_builtin_op(op_name):
         return None
 
     submodule_name = _get_op_submodule_name(op_name, op_name_prefix, submodule_name_list)
+    op_module = root_module
     if len(submodule_name) > 0:
         op_module = getattr(root_module, submodule_name[1:-1], None)
         if op_module is None:
             raise ValueError('Cannot find submodule {} in module {}'
                              .format(submodule_name[1:-1], root_module.__name__))
-    else:
-        op_module = root_module
 
     op = getattr(op_module, op_name[(len(op_name_prefix)+len(submodule_name)):], None)
     if op is None:
@@ -61,7 +61,6 @@ def _register_op_signatures():
                       .format(str(sys.version)))
         return
 
-    import inspect
     for op_name in dir(_numpy_op_doc):
         op = _get_builtin_op(op_name)
         if op is not None:
diff --git a/python/mxnet/optimizer/optimizer.py b/python/mxnet/optimizer/optimizer.py
index c3a1f3374a94..bc87777e40fc 100644
--- a/python/mxnet/optimizer/optimizer.py
+++ b/python/mxnet/optimizer/optimizer.py
@@ -1942,7 +1942,7 @@ def __init__(self, optimizer):
 
     def __call__(self, index, grad, weight):
         """Updates weight given gradient and index."""
-        allow_np = self.optimizer.allow_np_array
+        allow_np = self.optimizer.allow_np_array if hasattr(self.optimizer, "allow_np_array") else is_np_array()
         if not isinstance(index, (list, tuple)):
             indices = [index]
             grads = [_as_classic(grad, allow_np)]
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 338c70d1e53d..66b2aa2ac3fb 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -33,12 +33,13 @@
            'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'absolute', 'exp',
            'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'log1p',
            'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor',
-           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'tensordot', 'histogram',
-           'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean',
-           'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign',
-           'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'hypot', 'rad2deg', 'deg2rad',
-           'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',
-           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal']
+           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'tensordot', 'histogram', 'eye',
+           'linspace', 'logspace', 'expand_dims', 'tile', 'arange', 'split', 'vsplit', 'concatenate',
+           'stack', 'vstack', 'dstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax',
+           'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip',
+           'around', 'hypot', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril', 'identity', 'take',
+           'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal', 'greater', 'less', 'greater_equal',
+           'less_equal', 'hsplit', 'rot90']
 
 
 def _num_outputs(sym):
@@ -1305,6 +1306,37 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, density=None):
     raise ValueError("histogram fails with", locals())
 
 
+@set_module('mxnet.symbol.numpy')
+def eye(N, M=None, k=0, dtype=_np.float32, **kwargs):
+    """
+    Return a 2-D array with ones on the diagonal and zeros elsewhere.
+
+    Parameters
+    ----------
+    N : int
+        Number of rows in the output.
+    M : int, optional
+        Number of columns in the output. If None, defaults to N.
+    k : int, optional
+        Index of the diagonal: 0 (the default) refers to the main diagonal,
+        a positive value refers to an upper diagonal,
+        and a negative value to a lower diagonal.
+    dtype : data-type, optional
+        Data-type of the returned array.
+
+    Returns
+    -------
+    I : ndarray of shape (N,M)
+        An array where all elements are equal to zero,
+        except for the k-th diagonal, whose values are equal to one.
+    """
+    _sanity_check_params('eye', ['order'], kwargs)
+    ctx = kwargs.pop('ctx', current_context())
+    if ctx is None:
+        ctx = current_context()
+    return _npi.eye(N, M, k, ctx, dtype)
+
+
 @set_module('mxnet.symbol.numpy')
 def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0, ctx=None): # pylint: disable=too-many-arguments
     r"""
@@ -1378,6 +1410,89 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis
         return _npi.linspace(start=start, stop=stop, num=num, endpoint=endpoint, ctx=ctx, dtype=dtype)
 
 
+@set_module('mxnet.symbol.numpy')
+def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0, ctx=None): # pylint: disable=too-many-arguments
+    r"""Return numbers spaced evenly on a log scale.
+
+    In linear space, the sequence starts at ``base ** start``
+    (`base` to the power of `start`) and ends with ``base ** stop``
+    (see `endpoint` below).
+
+        Non-scalar `start` and `stop` are now supported.
+
+    Parameters
+    ----------
+    start : scalar
+        ``base ** start`` is the starting value of the sequence.
+    stop : scalar
+        ``base ** stop`` is the final value of the sequence, unless `endpoint`
+        is False.  In that case, ``num + 1`` values are spaced over the
+        interval in log-space, of which all but the last (a sequence of
+        length `num`) are returned.
+    num : scalar, optional
+        Number of samples to generate.  Default is 50.
+    endpoint : boolean, optional
+        If true, `stop` is the last sample. Otherwise, it is not included.
+        Default is True.
+    base : scalar, optional
+        The base of the log space. The step size between the elements in
+        ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform.
+        Default is 10.0.
+    dtype : dtype
+        The type of the output array.  If `dtype` is not given, infer the data
+        type from the other input arguments.
+    axis : scalar, optional
+        The axis in the result to store the samples.  Relevant only if start
+        or stop are array-like.  By default (0), the samples will be along a
+        new axis inserted at the beginning. Now, axis only support axis = 0.
+    ctx : Context, optional
+        An optional device context (default is the current default context).
+
+    Returns
+    -------
+    samples : _Symbol
+        `num` samples, equally spaced on a log scale.
+
+    See Also
+    --------
+    arange : Similar to linspace, with the step size specified instead of the
+             number of samples. Note that, when used with a float endpoint, the
+             endpoint may or may not be included.
+    linspace : Similar to logspace, but with the samples uniformly distributed
+               in linear space, instead of log space.
+
+    Notes
+    -----
+    Logspace is equivalent to the code
+
+    >>> y = np.linspace(start, stop, num=num, endpoint=endpoint)
+    ...
+    >>> power(base, y).astype(dtype)
+    ...
+
+    Examples
+    --------
+    >>> np.logspace(2.0, 3.0, num=4)
+    array([ 100.     ,  215.44347,  464.15887, 1000.     ])
+    >>> np.logspace(2.0, 3.0, num=4, endpoint=False)
+    array([100.     , 177.82794, 316.22775, 562.3413 ])
+    >>> np.logspace(2.0, 3.0, num=4, base=2.0)
+    array([4.       , 5.0396843, 6.349604 , 8.       ])
+    >>> np.logspace(2.0, 3.0, num=4, base=2.0, dtype=np.int32)
+    array([4, 5, 6, 8], dtype=int32)
+    >>> np.logspace(2.0, 3.0, num=4, ctx=npx.gpu(0))
+    array([ 100.     ,  215.44347,  464.15887, 1000.     ], ctx=gpu(0))
+    """
+    if isinstance(start, (list, _np.ndarray)) or \
+       isinstance(stop, (list, _np.ndarray)):
+        raise NotImplementedError('start and stop only support int')
+    if axis != 0:
+        raise NotImplementedError("the function only support axis 0")
+    if ctx is None:
+        ctx = current_context()
+    return _npi.logspace(start=start, stop=stop, num=num, endpoint=endpoint, base=base, ctx=ctx, dtype=dtype)
+
+
 @set_module('mxnet.symbol.numpy')
 def expand_dims(a, axis):
     """Expand the shape of an array.
@@ -2570,7 +2685,7 @@ def split(ary, indices_or_sections, axis=0):
     ----------
     ary : ndarray
         Array to be divided into sub-arrays.
-    indices_or_sections : int or 1-D array
+    indices_or_sections : int or 1-D python tuple, list or set.
         If `indices_or_sections` is an integer, N, the array will be divided
         into N equal arrays along `axis`.  If such a split is not possible,
         an error is raised.
@@ -2597,15 +2712,172 @@ def split(ary, indices_or_sections, axis=0):
     sections = 0
     if isinstance(indices_or_sections, int):
         sections = indices_or_sections
-    elif isinstance(indices_or_sections, tuple):
+    elif isinstance(indices_or_sections, (list, set, tuple)):
         indices = [0] + list(indices_or_sections)
     else:
-        raise ValueError('indices_or_sections must either int or tuple of ints')
+        raise ValueError('indices_or_sections must either int or tuple / list / set of ints')
     ret = _npi.split(ary, indices, axis, False, sections)
     return ret
 # pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
+@set_module('mxnet.ndarray.numpy')
+def hsplit(ary, indices_or_sections):
+    """Split an array into multiple sub-arrays horizontally (column-wise).
+
+    This is equivalent to ``split`` with ``axis=0`` if ``ary`` has one
+    dimension, and otherwise that with ``axis=1``.
+
+    Parameters
+    ----------
+    ary : _Symbol
+        Array to be divided into sub-arrays.
+    indices_or_sections : int, list of ints or tuple of ints.
+        If `indices_or_sections` is an integer, N, the array will be divided
+        into N equal arrays along `axis`.  If such a split is not possible,
+        an error is raised.
+
+        If `indices_or_sections` is a list of sorted integers, the entries
+        indicate where along `axis` the array is split.
+
+        If an index exceeds the dimension of the array along `axis`,
+        it will raises errors. so index must less than or euqal to
+        the dimension of the array along axis.
+
+    Returns
+    -------
+    sub-arrays : _Symbol
+        A list of sub-arrays.
+
+    Notes
+    ------
+    - If `indices_or_sections` is given as an integer, but a split
+      does not result in equal division.It will raises ValueErrors.
+
+    - If indices_or_sections is an integer, and the number is 1, it will
+      raises an error. Because single output from split is not supported yet...
+
+    See Also
+    --------
+    split : Split an array into multiple sub-arrays of equal size.
+
+    Examples
+    --------
+    >>> x = np.arange(16.0).reshape(4, 4)
+    >>> x
+    array([[ 0.,  1.,  2.,  3.],
+           [ 4.,  5.,  6.,  7.],
+           [ 8.,  9., 10., 11.],
+           [12., 13., 14., 15.]])
+    >>> np.hsplit(x, 2)
+    [array([[ 0.,  1.],
+           [ 4.,  5.],
+           [ 8.,  9.],
+           [12., 13.]]),
+    array([[ 2.,  3.],
+           [ 6.,  7.],
+           [10., 11.],
+           [14., 15.]])]
+    >>> np.hsplit(x, [3, 6])
+    [array([[ 0.,  1.,  2.],
+           [ 4.,  5.,  6.],
+           [ 8.,  9., 10.],
+           [12., 13., 14.]]),
+    array([[ 3.],
+           [ 7.],
+           [11.],
+           [15.]]),
+    array([], shape=(4, 0), dtype=float32)]
+
+    With a higher dimensional array the split is still along the second axis.
+
+    >>> x = np.arange(8.0).reshape(2, 2, 2)
+    >>> x
+    array([[[ 0.,  1.],
+            [ 2.,  3.]],
+           [[ 4.,  5.],
+            [ 6.,  7.]]])
+    >>> np.hsplit(x, 2)
+    [array([[[ 0.,  1.]],
+            [[ 4.,  5.]]]),
+     array([[[ 2.,  3.]],
+            [[ 6.,  7.]]])]
+
+    If ``ary`` has one dimension, 'axis' = 0.
+    >>> x = np.arange(4)
+    array([0., 1., 2., 3.])
+    >>> np.hsplit(x, 2)
+    [array([0., 1.]), array([2., 3.])]
+
+    If you want to produce an empty sub-array, you can see an example.
+    >>> np.hsplit(x, [2, 2])
+    [array([0., 1.]), array([], dtype=float32), array([2., 3.])]
+    """
+    indices = []
+    sections = 0
+    if isinstance(indices_or_sections, int):
+        sections = indices_or_sections
+    elif isinstance(indices_or_sections, (list, set, tuple)):
+        indices = [0] + list(indices_or_sections)
+    else:
+        raise ValueError('indices_or_sections must either int or tuple of ints')
+    ret = _npi.hsplit(ary, indices, 1, False, sections)
+    return ret
+# pylint: enable=redefined-outer-name
+
+
+@set_module('mxnet.symbol.numpy')
+def vsplit(ary, indices_or_sections):
+    r"""
+    vsplit(ary, indices_or_sections)
+
+    Split an array into multiple sub-arrays vertically (row-wise).
+
+    ``vsplit`` is equivalent to ``split`` with `axis=0` (default): the array is always split
+    along the first axis regardless of the array dimension.
+
+    Parameters
+    ----------
+    ary : _Symbol
+        Array to be divided into sub-arrays.
+    indices_or_sections : int or 1 - D Python tuple, list or set.
+        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays
+        along axis 0.  If such a split is not possible, an error is raised.
+
+        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where
+        along axis 0 the array is split.  For example, ``[2, 3]`` would result in
+
+          - ary[:2]
+          - ary[2:3]
+          - ary[3:]
+
+        If an index exceeds the dimension of the array along axis 0, an error will be thrown.
+
+    Returns
+    -------
+    sub-arrays : list of _Symbols
+        A list of sub-arrays.
+
+    See Also
+    --------
+    split : Split an array into multiple sub-arrays of equal size.
+
+    Notes
+    -------
+    This function differs from the original `numpy.degrees
+    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in
+    the following aspects:
+
+    - Currently parameter ``indices_or_sections`` does not support ndarray, but supports scalar,
+    tuple and list
+    - In ``indices_or_sections``, if an index exceeds the dimension of the array along axis 0,
+    an error will be thrown.
+
+    """
+    return split(ary, indices_or_sections, 0)
+
+
 @set_module('mxnet.symbol.numpy')
 def concatenate(seq, axis=0, out=None):
     """Join a sequence of arrays along an existing axis.
@@ -2689,6 +2961,35 @@ def get_list(arrays):
     return _npi.vstack(*arrays)
 
 
+@set_module('mxnet.symbol.numpy')
+def dstack(arrays):
+    """
+    Stack arrays in sequence depth wise (along third axis).
+
+    This is equivalent to concatenation along the third axis after 2-D arrays
+    of shape `(M,N)` have been reshaped to `(M,N,1)` and 1-D arrays of shape
+    `(N,)` have been reshaped to `(1,N,1)`. Rebuilds arrays divided by
+    `dsplit`.
+
+    This function makes most sense for arrays with up to 3 dimensions. For
+    instance, for pixel-data with a height (first axis), width (second axis),
+    and r/g/b channels (third axis). The functions `concatenate`, `stack` and
+    `block` provide more general stacking and concatenation operations.
+
+    Parameters
+    ----------
+    tup : sequence of _Symbol
+        The arrays must have the same shape along all but the first axis.
+        1-D arrays must have the same length.
+
+    Returns
+    -------
+    stacked : _Symbol
+        The array formed by stacking the given arrays, will be at least 2-D.
+    """
+    return _npi.dstack(*arrays)
+
+
 @set_module('mxnet.symbol.numpy')
 def maximum(x1, x2, out=None):
     return _ufunc_helper(x1, x2, _npi.maximum, _np.maximum, _npi.maximum_scalar, None, out)
@@ -3967,4 +4268,47 @@ def less_equal(x1, x2, out=None):
                          _npi.greater_equal_scalar, out)
 
 
+@set_module('mxnet.symbol.numpy')
+def rot90(m, k=1, axes=(0, 1)):
+    """
+    Rotate an array by 90 degrees in the plane specified by axes.
+    Rotation direction is from the first towards the second axis.
+    Parameters
+    ----------
+    m : _Symbol
+        Array of two or more dimensions.
+    k : integer
+        Number of times the array is rotated by 90 degrees.
+    axes: (2,) array_like
+        The array is rotated in the plane defined by the axes.
+        Axes must be different.
+    Returns
+    -------
+    y : _Symbol
+        A rotated view of `m`.
+    -----
+    rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1))
+    rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1))
+    Examples
+    --------
+    >>> m = np.array([[1,2],[3,4]], 'int')
+    >>> m
+    array([[1, 2],
+           [3, 4]], dtype=int64)
+    >>> np.rot90(m)
+    array([[2, 4],
+           [1, 3]], dtype=int64)
+    >>> np.rot90(m, 2)
+    array([[4, 3],
+           [2, 1]], dtype=int64)
+    >>> m = np.arange(8).reshape((2,2,2))
+    >>> np.rot90(m, 1, (1,2))
+    array([[[1., 3.],
+            [0., 2.]],
+           [[5., 7.],
+            [4., 6.]]])
+    """
+    return _npi.rot90(m, k=k, axes=axes)
+
+
 _set_np_symbol_class(_Symbol)
diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py
index e5f2c6c02089..6ee37982a124 100644
--- a/python/mxnet/test_utils.py
+++ b/python/mxnet/test_utils.py
@@ -31,6 +31,7 @@
 import zipfile
 import json
 from contextlib import contextmanager
+from collections import OrderedDict
 import numpy as np
 import numpy.testing as npt
 import numpy.random as rnd
@@ -109,6 +110,16 @@ def random_sample(population, k):
     return population_copy[0:k]
 
 
+def _sorted_items(d):
+    """Return (key, value) pairs of dict 'd' in a deterministic order (sorted by key)."""
+    return sorted(d.items(), key=lambda t: t[0])
+
+
+def _sorted_dict(d):
+    """Return ordered dictionary containing items ordered by their keys."""
+    return OrderedDict(_sorted_items(d))
+
+
 def _validate_csr_generation_inputs(num_rows, num_cols, density,
                                     distribution="uniform"):
     """Validates inputs for csr generation helper functions
@@ -482,9 +493,10 @@ def find_max_violation(a, b, rtol=None, atol=None):
     """Finds and returns the location of maximum violation."""
     rtol = get_rtol(rtol)
     atol = get_atol(atol)
-    diff = np.abs(a-b)
+    # 'smart' absdiff that considers inf's as equals (to match np.allclose)
+    absdiff = np.where(np.equal(a, b), 0, np.abs(a-b))
     tol = atol + rtol*np.abs(b)
-    violation = diff/(tol+1e-20)
+    violation = absdiff/(tol+1e-20)
     loc = np.argmax(violation)
     idx = np.unravel_index(loc, violation.shape)
     return idx, np.max(violation)
@@ -500,40 +512,122 @@ def same(a, b):
     """
     return np.array_equal(a, b)
 
-def almost_equal(a, b, rtol=None, atol=None, equal_nan=False, use_broadcast=True):
-    """Test if two numpy arrays are almost equal."""
-    # pylint: disable=unexpected-keyword-arg
-    if (not use_broadcast) and a.shape != b.shape:
+
+def checkShapes(a, b):
+    if a.shape != b.shape:
         msg = npt.build_err_msg([a, b],
                                 err_msg="a.shape = {} and b.shape = {} are not equal"
                                 .format(str(a.shape), str(b.shape)))
         raise AssertionError(msg)
+
+
+def almost_equal(a, b, rtol=None, atol=None, equal_nan=False, use_broadcast=True):
+    """Test if two numpy arrays are almost equal."""
+    # pylint: disable=unexpected-keyword-arg
+    if not use_broadcast:
+        checkShapes(a, b)
+
     return np.allclose(a, b, rtol=get_rtol(rtol), atol=get_atol(atol), equal_nan=equal_nan)
     # pylint: enable=unexpected-keyword-arg
 
-def assert_almost_equal(a, b, rtol=None, atol=None, names=('a', 'b'), equal_nan=False, use_broadcast=True):
+
+def assert_almost_equal(a, b, rtol=None, atol=None, names=('a', 'b'), equal_nan=False,
+                        use_broadcast=True, mismatches=(10, 10)):
     """Test that two numpy arrays are almost equal. Raise exception message if not.
 
     Parameters
     ----------
-    a : np.ndarray
-    b : np.ndarray
-    threshold : None or float
-        The checking threshold. Default threshold will be used if set to ``None``.
+    a : np.ndarray or mx.nd.array
+    b : np.ndarray or mx.nd.array
+    rtol : None or float
+        The relative threshold. Default threshold will be used if set to ``None``.
+    atol : None or float
+        The absolute threshold. Default threshold will be used if set to ``None``.
+    names : tuple of names, optional
+        The names used in error message when an exception occurs
+    equal_nan : boolean, optional
+        The flag determining how to treat NAN values in comparison
+    mismatches : tuple of mismatches
+        Maximum number of mismatches to be printed (mismatches[0]) and determine (mismatches[1])
     """
+    if not use_broadcast:
+        checkShapes(a, b)
+
     rtol = get_rtol(rtol)
     atol = get_atol(atol)
-    if almost_equal(a, b, rtol, atol, equal_nan=equal_nan, use_broadcast=use_broadcast):
-        return
+    use_np_allclose = isinstance(a, np.ndarray) and isinstance(b, np.ndarray)
+    if not use_np_allclose:
+        if not (hasattr(a, 'context') and hasattr(b, 'context') and a.context == b.context and a.dtype == b.dtype):
+            use_np_allclose = True
+            if isinstance(a, mx.nd.NDArray):
+                a = a.asnumpy()
+            if isinstance(b, mx.nd.NDArray):
+                b = b.asnumpy()
+
+    if use_np_allclose:
+        if almost_equal(a, b, rtol, atol, equal_nan=equal_nan):
+            return
+    else:
+        output = mx.nd.contrib.allclose(a, b, rtol, atol, equal_nan)
+        if output.asnumpy() == 1:
+            return
+
+        a = a.asnumpy()
+        b = b.asnumpy()
+
+    def locationError(a, b, index, names, maxError=False):
+        """Create element mismatch comment
+
+        Parameters
+        ----------
+        a, b : compared np.ndarray's
+        index : tuple of coordinate arrays
+            Location of violation
+        names : tuple of names
+            The names of compared arrays.
+        maxError: boolean, optional
+            Flag indicating that maximum error is reporting.
+        """
+        maximum = "maximum " if maxError else ""
+        return "Location of %serror: %s, %s=%.8f, %s=%.8f" \
+               % (maximum, str(index), names[0], a[index], names[1], b[index])
+
     index, rel = find_max_violation(a, b, rtol, atol)
+    indexErr = index
+    relErr = rel
+
+    print('\n*** Maximum errors for vector of size {}:  rtol={}, atol={}\n'.format(a.size, rtol, atol))
+    aTmp = a.copy()
+    bTmp = b.copy()
+    i = 1
+    while i <= a.size:
+        if i <= mismatches[0]:
+            print("%3d: Error %f  %s" %(i, rel, locationError(a, b, index, names)))
+
+        aTmp[index] = bTmp[index] = 0
+        if almost_equal(aTmp, bTmp, rtol, atol, equal_nan=equal_nan):
+            break
+
+        i += 1
+        if i <= mismatches[1] or mismatches[1] <= 0:
+            index, rel = find_max_violation(aTmp, bTmp, rtol, atol)
+        else:
+            break
+
+    mismatchDegree = "at least " if mismatches[1] > 0 and i > mismatches[1] else ""
+    errMsg = "Error %f exceeds tolerance rtol=%e, atol=%e (mismatch %s%f%%).\n%s" % \
+             (relErr, rtol, atol, mismatchDegree, 100*i/a.size, \
+             locationError(a, b, indexErr, names, maxError=True))
     np.set_printoptions(threshold=4, suppress=True)
-    msg = npt.build_err_msg([a, b],
-                            err_msg="Error %f exceeds tolerance rtol=%f, atol=%f. "
-                                    " Location of maximum error:%s, a=%f, b=%f"
-                            % (rel, rtol, atol, str(index), a[index], b[index]),
-                            names=names)
+    msg = npt.build_err_msg([a, b], err_msg=errMsg)
+
     raise AssertionError(msg)
 
+
+def assert_allclose(a, b, rtol=1e-07, atol=0, equal_nan=True):
+    assert_almost_equal(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan)
+
+
 def assert_almost_equal_with_err(a, b, rtol=None, atol=None, etol=None, names=('a', 'b'), equal_nan=False):
     """Test that two numpy arrays are almost equal within given error rate. Raise exception message if not.
 
@@ -554,7 +648,6 @@ def assert_almost_equal_with_err(a, b, rtol=None, atol=None, etol=None, names=('
         equals = np.isclose(a, b, rtol=rtol, atol=atol)
         err = 1 - np.count_nonzero(equals) / equals.size
         if err > etol:
-            #if True:
             index, rel = find_max_violation(a, b, rtol, atol)
             np.set_printoptions(threshold=4, suppress=True)
             msg = npt.build_err_msg([a, b],
@@ -684,7 +777,7 @@ def simple_forward(sym, ctx=None, is_train=False, **inputs):
 
 
 def _parse_location(sym, location, ctx, dtype=default_dtype()):
-    """Parses the given location to a dictionary.
+    """Parses the given location to a ordered dictionary.
 
     Arguments of the provided op `sym` are used as dictionary keys
     and elements of `location` are used as values.
@@ -740,7 +833,7 @@ def _parse_location(sym, location, ctx, dtype=default_dtype()):
         location = {k: v for k, v in zip(sym.list_arguments(), location)}
     location = {k: mx.nd.array(v, ctx=ctx, dtype=v.dtype if dtype == "asnumpy" else dtype) \
                if isinstance(v, np.ndarray) else v for k, v in location.items()}
-    return location
+    return _sorted_dict(location)
 
 
 def _parse_aux_states(sym, aux_states, ctx, dtype=default_dtype()):
@@ -1177,7 +1270,8 @@ def check_symbolic_backward(sym, location, out_grads, expected, rtol=1e-5, atol=
     if isinstance(expected, (list, tuple)):
         expected = {k:v for k, v in zip(sym.list_arguments(), expected)}
 
-    args_grad_npy = {k:np.random.normal(size=v.shape) for k, v in expected.items()}
+    # Dirty the output buffer deterministically, for reproducibility.
+    args_grad_npy = {k:np.random.normal(size=v.shape) for k, v in _sorted_items(expected)}
     args_grad_data = {}
     for k, v in args_grad_npy.items():
         nd = mx.nd.array(v, ctx=ctx, dtype=expected[k].dtype if dtype == "asnumpy" else dtype)
@@ -1313,6 +1407,15 @@ def check_speed(sym, location=None, ctx=None, N=20, grad_req=None, typ="whole",
     else:
         raise ValueError('typ can only be "whole" or "forward".')
 
+
+def get_tolerance(rtol, ctx):
+    if 'atol' in ctx:
+        return ctx['atol']
+    if 'atol_mult' in ctx:
+        return ctx['atol_mult'] * rtol
+    return rtol
+
+
 def check_consistency(sym, ctx_list, scale=1.0, grad_req='write',
                       arg_params=None, aux_params=None, tol=None,
                       raise_on_err=True, ground_truth=None, equal_nan=False,
@@ -1431,12 +1534,15 @@ def check_consistency(sym, ctx_list, scale=1.0, grad_req='write',
     for i, exe in enumerate(exe_list):
         if i == max_idx:
             continue
+
+        rtol = tol[dtypes[i]]
+        atol = get_tolerance(rtol, ctx_list[i])
         for name, arr in zip(output_names, exe.outputs):
-            gtarr = gt[name].astype(dtypes[i]).asnumpy()
-            arr = arr.asnumpy()
+            # Previously, the cast was to dtypes[i], but symbol may be mixed-precision,
+            # so casting the ground truth to the actual output type seems more correct.
+            gtarr = gt[name].astype(arr.dtype)
             try:
-                assert_almost_equal(arr, gtarr, rtol=tol[dtypes[i]], atol=tol[dtypes[i]],
-                                    equal_nan=equal_nan)
+                assert_almost_equal(arr, gtarr, rtol=rtol, atol=atol, equal_nan=equal_nan)
             except AssertionError as e:
                 print('Predict Err: ctx %d vs ctx %d at %s'%(i, max_idx, name))
                 traceback.print_exc()
@@ -1454,16 +1560,20 @@ def check_consistency(sym, ctx_list, scale=1.0, grad_req='write',
         for i, exe in enumerate(exe_list):
             if i == max_idx:
                 continue
+
+            rtol = tol[dtypes[i]]
+            atol = get_tolerance(rtol, ctx_list[i])
             curr = zip(output_names + arg_names, exe.outputs + exe.grad_arrays)
             for name, arr in curr:
                 if gt[name] is None:
                     assert arr is None
                     continue
-                gtarr = gt[name].astype(dtypes[i]).asnumpy()
-                arr = arr.asnumpy()
+
+                # Previous cast was to dtypes[i], but symbol may be mixed-precision,
+                # so casting the ground truth to the actual output type seems more correct.
+                gtarr = gt[name].astype(arr.dtype)
                 try:
-                    assert_almost_equal(arr, gtarr, rtol=tol[dtypes[i]], atol=tol[dtypes[i]],
-                                        equal_nan=equal_nan)
+                    assert_almost_equal(arr, gtarr, rtol=rtol, atol=atol, equal_nan=equal_nan)
                 except AssertionError as e:
                     print('Train Err: ctx %d vs ctx %d at %s'%(i, max_idx, name))
                     traceback.print_exc()
@@ -1694,7 +1804,7 @@ def get_mnist_iterator(batch_size, input_shape, num_parts=1, part_index=0):
     """
 
     get_mnist_ubyte()
-    flat = not bool(len(input_shape) == 3)
+    flat = len(input_shape) != 3
 
     train_dataiter = mx.io.MNISTIter(
         image="data/train-images-idx3-ubyte",
@@ -2134,12 +2244,14 @@ def verify_generator(generator, buckets, probs, nsamples=1000000, nrepeat=5, suc
 
 def compare_ndarray_tuple(t1, t2, rtol=None, atol=None):
     """Compare ndarray tuple."""
-    if t1 is not None and t2 is not None:
-        if isinstance(t1, tuple):
-            for s1, s2 in zip(t1, t2):
-                compare_ndarray_tuple(s1, s2, rtol, atol)
-        else:
-            assert_almost_equal(t1.asnumpy(), t2.asnumpy(), rtol=rtol, atol=atol)
+    if t1 is None or t2 is None:
+        return
+
+    if isinstance(t1, tuple):
+        for s1, s2 in zip(t1, t2):
+            compare_ndarray_tuple(s1, s2, rtol, atol)
+    else:
+        assert_almost_equal(t1, t2, rtol=rtol, atol=atol)
 
 
 def compare_optimizer(opt1, opt2, shape, dtype, w_stype='default', g_stype='default',
@@ -2171,7 +2283,7 @@ def compare_optimizer(opt1, opt2, shape, dtype, w_stype='default', g_stype='defa
     opt2.update_multi_precision(0, w2, g2, state2)
     if compare_states:
         compare_ndarray_tuple(state1, state2, rtol=rtol, atol=atol)
-    assert_almost_equal(w1.asnumpy(), w2.asnumpy(), rtol=rtol, atol=atol)
+    assert_almost_equal(w1, w2, rtol=rtol, atol=atol)
 
 
 def same_symbol_structure(sym1, sym2):
diff --git a/src/operator/contrib/allclose_op-inl.h b/src/operator/contrib/allclose_op-inl.h
new file mode 100644
index 000000000000..a858450f0007
--- /dev/null
+++ b/src/operator/contrib/allclose_op-inl.h
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file allclose-inl.h
+ * \brief Operator implementing numpy.allclose function.
+ * \author Andrei Ivanov
+ */
+#ifndef MXNET_OPERATOR_CONTRIB_ALLCLOSE_OP_INL_H_
+#define MXNET_OPERATOR_CONTRIB_ALLCLOSE_OP_INL_H_
+
+#include <mxnet/operator_util.h>
+#include <vector>
+#include "../mshadow_op.h"
+#include "../mxnet_op.h"
+#include "../operator_common.h"
+#include "../elemwise_op_common.h"
+#include "../tensor/init_op.h"
+
+namespace mxnet {
+namespace op {
+
+// Intermediate and Output data types could be integers OR unsigned characters
+#define USE_INTEGER   0
+#if USE_INTEGER
+  #define INTERM_DATA_TYPE int32_t
+  #define OUT_DATA_TYPE    mshadow::kInt32
+#else
+  #define INTERM_DATA_TYPE uint8_t
+  #define OUT_DATA_TYPE    mshadow::kUint8
+#endif
+
+struct AllCloseParam : public dmlc::Parameter<AllCloseParam> {
+  float rtol, atol;
+  bool equal_nan;
+  DMLC_DECLARE_PARAMETER(AllCloseParam) {
+    DMLC_DECLARE_FIELD(rtol)
+      .set_default(1e-05)
+      .describe("Relative tolerance.");
+    DMLC_DECLARE_FIELD(atol)
+      .set_default(1e-08)
+      .describe("Absolute tolerance.");
+    DMLC_DECLARE_FIELD(equal_nan)
+      .set_default(true)
+      .describe("Whether to compare NaN’s as equal. If True, NaN’s in A will be considered equal "
+                "to NaN’s in B in the output array.");
+  }
+};
+
+inline bool AllCloseShape(const nnvm::NodeAttrs& attrs,
+                          std::vector<TShape>* in_attrs,
+                          std::vector<TShape>* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 2U) << "Input:[array1, array2]";
+  CHECK_EQ(out_attrs->size(), 1U);
+
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape(0, -1));
+  return in_attrs->at(0) == in_attrs->at(1);
+}
+
+inline bool AllCloseType(const nnvm::NodeAttrs& attrs,
+                         std::vector<int> *in_attrs,
+                         std::vector<int> *out_attrs) {
+  CHECK_EQ(in_attrs->size(), 2U);
+  CHECK_EQ(out_attrs->size(), 1U);
+
+  // The output will be boolean stored as an OUT_DATA_TYPE format
+  TYPE_ASSIGN_CHECK(*out_attrs, 0, OUT_DATA_TYPE);
+  return (*out_attrs)[0] != -1;
+}
+
+using namespace mshadow_op::isnan_typed;
+
+template<int req>
+struct allclose_forward {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, INTERM_DATA_TYPE *out_data,
+                                  const DType *in_a, const DType *in_b,
+                                  const float rtol, const float atol, bool equal_nan) {
+      const DType a = in_a[i], b = in_b[i];
+      bool val;
+      if (IsNan(a) || IsNan(b))
+        val = equal_nan && IsNan(a) == IsNan(b);
+      else
+        val = a == b || (a > b?  a - b : b - a) <= atol + (b > 0? rtol * b :  (-rtol) * b);
+
+      KERNEL_ASSIGN(out_data[i], req, val? 1 : 0);
+  }
+};
+
+template<typename xpu>
+size_t GetAdditionalMemoryLogical(mshadow::Stream<xpu> *s, const int num_items);
+
+template<typename xpu>
+INTERM_DATA_TYPE *GetAdditionalMemoryLogical(const OpContext& ctx,
+                                             int num_items, size_t *pExtraStorageBytes) {
+// Get length of the additional memory (which is used only by DeviceReduce::Min(...) on gpu)
+  *pExtraStorageBytes = GetAdditionalMemoryLogical<xpu>(ctx.get_stream<xpu>(), num_items);
+  const size_t workspace_total_bytes_ = num_items * sizeof(INTERM_DATA_TYPE) + *pExtraStorageBytes;
+  mshadow::Tensor<xpu, 1, uint8_t> workspace =
+    ctx.requested[0].get_space_typed<xpu, 1, uint8_t>(
+      mshadow::Shape1(workspace_total_bytes_), ctx.get_stream<xpu>());
+
+  return reinterpret_cast<INTERM_DATA_TYPE *>(workspace.dptr_);
+}
+
+template<typename xpu>
+void GetResultLogical(mshadow::Stream<xpu> *s, INTERM_DATA_TYPE *workMem, size_t extraStorageBytes,
+                      int num_items, INTERM_DATA_TYPE *outPntr);
+
+template<typename xpu>
+void AllClose(const nnvm::NodeAttrs& attrs,
+              const OpContext& ctx,
+              const std::vector<TBlob>& inputs,
+              const std::vector<OpReqType>& req,
+              const std::vector<TBlob>& outputs) {
+  CHECK_EQ(inputs.size(), 2U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(req.size(), 1U);
+
+  const TBlob& in0 = inputs[0];
+  const TBlob& in1 = inputs[1];
+  const int num_items = in0.Size();
+
+  size_t extraStorageBytes;
+  auto workspaceMem = GetAdditionalMemoryLogical<xpu>(ctx, num_items, &extraStorageBytes);
+  auto s = ctx.get_stream<xpu>();
+  const AllCloseParam& param = nnvm::get<AllCloseParam>(attrs.parsed);
+  using namespace mxnet_op;
+  MSHADOW_TYPE_SWITCH(in0.type_flag_, DType, {
+    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+      Kernel<allclose_forward<req_type>, xpu>::Launch(
+        s, num_items, workspaceMem, in0.dptr<DType>(), in1.dptr<DType>(),
+        param.rtol, param.atol, param.equal_nan);
+    });
+  });
+
+  auto *pOut = outputs[0].dptr<INTERM_DATA_TYPE>();
+  GetResultLogical<xpu>(s, workspaceMem, extraStorageBytes, num_items, pOut);
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_CONTRIB_ALLCLOSE_OP_INL_H_
diff --git a/src/operator/contrib/allclose_op.cc b/src/operator/contrib/allclose_op.cc
new file mode 100644
index 000000000000..6b301ad6519f
--- /dev/null
+++ b/src/operator/contrib/allclose_op.cc
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file allclose_op.cc
+ * \brief CPU Implementation of allclose op
+ * \author Andrei Ivanov
+ */
+#include "./allclose_op-inl.h"
+
+namespace mxnet {
+namespace op {
+
+DMLC_REGISTER_PARAMETER(AllCloseParam);
+
+NNVM_REGISTER_OP(_contrib_allclose)
+.describe(R"code(This operators implements the numpy.allclose(a, b, rtol=1e-05, atol=1e-08, equal_nan=False)
+
+.. math::
+
+    f(x) = |a−b|≤atol+rtol|b|
+
+where
+:math:`a, b` are the input tensors of equal types an shapes
+:math:`atol, rtol` the values of absolute and relative tolerance (by default, rtol=1e-05, atol=1e-08)
+
+Examples::
+
+  a = [1e10, 1e-7],
+  b = [1.00001e10, 1e-8]
+  y = allclose(a, b)
+  y = False
+
+  a = [1e10, 1e-8],
+  b = [1.00001e10, 1e-9]
+  y = allclose(a, b)
+  y = True
+
+)code" ADD_FILELINE)
+.set_attr_parser(ParamParser<AllCloseParam>)
+.set_num_inputs(2)
+.set_num_outputs(1)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    return std::vector<std::string>{"a", "b"};
+  })
+.set_attr<mxnet::FInferShape>("FInferShape", AllCloseShape)
+.set_attr<nnvm::FInferType>("FInferType", AllCloseType)
+.set_attr<FCompute>("FCompute<cpu>", AllClose<cpu>)
+.set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& n) {
+  return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+})
+.add_argument("a", "NDArray-or-Symbol", "Input array a")
+.add_argument("b", "NDArray-or-Symbol", "Input array b")
+.add_arguments(AllCloseParam::__FIELDS__());
+
+template<>
+size_t GetAdditionalMemoryLogical<cpu>(mshadow::Stream<cpu> *s, const int num_items) {
+  return 0;
+}
+
+template<>
+void GetResultLogical<cpu>(mshadow::Stream<cpu> *s, INTERM_DATA_TYPE *workMem,
+                           size_t extraStorageBytes, int num_items, INTERM_DATA_TYPE *outPntr) {
+  while (num_items-- > 0 && workMem[num_items]) {}
+  outPntr[0] = num_items >= 0? 0 : 1;
+}
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/contrib/allclose_op.cu b/src/operator/contrib/allclose_op.cu
new file mode 100644
index 000000000000..f923ab060813
--- /dev/null
+++ b/src/operator/contrib/allclose_op.cu
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file allclose_op.cu
+ * \brief GPU Implementation of allclose op
+ * \author Andrei Ivanov
+ */
+#include "./allclose_op-inl.h"
+#include <cub/cub.cuh>
+
+namespace mxnet {
+namespace op {
+
+template<typename T>
+size_t GetAdditionalMemory(mshadow::Stream<gpu> *s, const int num_items) {
+  T *d_in = nullptr;
+  T *d_out = nullptr;
+  size_t temp_storage_bytes = 0;
+  cudaStream_t stream = mshadow::Stream<gpu>::GetStream(s);
+  cub::DeviceReduce::Min(nullptr, temp_storage_bytes, d_in, d_out, num_items, stream);
+  return temp_storage_bytes;
+}
+
+template<>
+size_t GetAdditionalMemoryLogical<gpu>(mshadow::Stream<gpu> *s, const int num_items) {
+  return GetAdditionalMemory<INTERM_DATA_TYPE>(s, num_items);
+}
+
+template<>
+void GetResultLogical<gpu>(mshadow::Stream<gpu> *s, INTERM_DATA_TYPE *workMem,
+                           size_t extraStorageBytes, int num_items, INTERM_DATA_TYPE *outPntr) {
+  cudaStream_t stream = mshadow::Stream<gpu>::GetStream(s);
+  cub::DeviceReduce::Min(workMem + num_items, extraStorageBytes,
+                         workMem, outPntr, num_items, stream);
+}
+
+NNVM_REGISTER_OP(_contrib_allclose)
+.set_attr<FCompute>("FCompute<gpu>", AllClose<gpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/contrib/mrcnn_mask_target-inl.h b/src/operator/contrib/mrcnn_mask_target-inl.h
new file mode 100644
index 000000000000..891453369089
--- /dev/null
+++ b/src/operator/contrib/mrcnn_mask_target-inl.h
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ *  Copyright (c) 2019 by Contributors
+ * \file mrcnn_mask_target-inl.h
+ * \brief Mask-RCNN target generator
+ * \author Serge Panev
+ */
+
+
+#ifndef MXNET_OPERATOR_CONTRIB_MRCNN_MASK_TARGET_INL_H_
+#define MXNET_OPERATOR_CONTRIB_MRCNN_MASK_TARGET_INL_H_
+
+#include <mxnet/operator.h>
+#include <vector>
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+#include "../tensor/init_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace mrcnn_index {
+  enum ROIAlignOpInputs {kRoi, kGtMask, kMatches, kClasses};
+  enum ROIAlignOpOutputs {kMask, kMaskClasses};
+}  // namespace mrcnn_index
+
+struct MRCNNMaskTargetParam : public dmlc::Parameter<MRCNNMaskTargetParam> {
+  int num_rois;
+  int num_classes;
+  int mask_size;
+  int sample_ratio;
+
+  DMLC_DECLARE_PARAMETER(MRCNNMaskTargetParam) {
+    DMLC_DECLARE_FIELD(num_rois)
+    .describe("Number of sampled RoIs.");
+    DMLC_DECLARE_FIELD(num_classes)
+    .describe("Number of classes.");
+    DMLC_DECLARE_FIELD(mask_size)
+    .describe("Size of the pooled masks.");
+    DMLC_DECLARE_FIELD(sample_ratio).set_default(2)
+    .describe("Sampling ratio of ROI align. Set to -1 to use adaptative size.");
+  }
+};
+
+inline bool MRCNNMaskTargetShape(const NodeAttrs& attrs,
+                                 std::vector<mxnet::TShape>* in_shape,
+                                 std::vector<mxnet::TShape>* out_shape) {
+  using namespace mshadow;
+  const MRCNNMaskTargetParam& param = nnvm::get<MRCNNMaskTargetParam>(attrs.parsed);
+
+  CHECK_EQ(in_shape->size(), 4) << "Input:[rois, gt_masks, matches, cls_targets]";
+
+  // (B, N, 4)
+  mxnet::TShape tshape = in_shape->at(mrcnn_index::kRoi);
+  CHECK_EQ(tshape.ndim(), 3) << "rois should be a 2D tensor of shape [batch, rois, 4]";
+  CHECK_EQ(tshape[2], 4) << "rois should be a 2D tensor of shape [batch, rois, 4]";
+  auto batch_size = tshape[0];
+  auto num_rois = tshape[1];
+
+  // (B, M, H, W)
+  tshape = in_shape->at(mrcnn_index::kGtMask);
+  CHECK_EQ(tshape.ndim(), 4) << "gt_masks should be a 4D tensor";
+  CHECK_EQ(tshape[0], batch_size) << " batch size should be the same for all the inputs.";
+
+  // (B, N)
+  tshape = in_shape->at(mrcnn_index::kMatches);
+  CHECK_EQ(tshape.ndim(), 2) << "matches should be a 2D tensor";
+  CHECK_EQ(tshape[0], batch_size) << " batch size should be the same for all the inputs.";
+
+  // (B, N)
+  tshape = in_shape->at(mrcnn_index::kClasses);
+  CHECK_EQ(tshape.ndim(), 2) << "matches should be a 2D tensor";
+  CHECK_EQ(tshape[0], batch_size) << " batch size should be the same for all the inputs.";
+
+  // out: 2 * (B, N, C, MS, MS)
+  auto oshape = Shape5(batch_size, num_rois, param.num_classes, param.mask_size, param.mask_size);
+  out_shape->clear();
+  out_shape->push_back(oshape);
+  out_shape->push_back(oshape);
+  return true;
+}
+
+inline bool MRCNNMaskTargetType(const NodeAttrs& attrs,
+                                std::vector<int>* in_type,
+                                std::vector<int>* out_type) {
+  CHECK_EQ(in_type->size(), 4);
+  int dtype = (*in_type)[1];
+  CHECK_NE(dtype, -1) << "Input must have specified type";
+
+  out_type->clear();
+  out_type->push_back(dtype);
+  out_type->push_back(dtype);
+  return true;
+}
+
+template<typename xpu>
+void MRCNNMaskTargetRun(const MRCNNMaskTargetParam& param, const std::vector<TBlob> &inputs,
+                        const std::vector<TBlob> &outputs, mshadow::Stream<xpu> *s);
+
+template<typename xpu>
+void MRCNNMaskTargetCompute(const nnvm::NodeAttrs& attrs,
+                            const OpContext &ctx,
+                            const std::vector<TBlob> &inputs,
+                            const std::vector<OpReqType> &req,
+                            const std::vector<TBlob> &outputs) {
+  auto s = ctx.get_stream<xpu>();
+  const auto& p = dmlc::get<MRCNNMaskTargetParam>(attrs.parsed);
+  MRCNNMaskTargetRun<xpu>(p, inputs, outputs, s);
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_CONTRIB_MRCNN_MASK_TARGET_INL_H_
diff --git a/src/operator/contrib/mrcnn_mask_target.cu b/src/operator/contrib/mrcnn_mask_target.cu
new file mode 100644
index 000000000000..0170e739d437
--- /dev/null
+++ b/src/operator/contrib/mrcnn_mask_target.cu
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ *  Copyright (c) 2019 by Contributors
+ * \file mrcnn_mask_target.cu
+ * \brief Mask-RCNN target generator
+ * \author Serge Panev
+ */
+
+#include "./mrcnn_mask_target-inl.h"
+
+namespace mxnet {
+namespace op {
+
+using namespace mshadow::cuda;
+
+// The maximum number of blocks to use in the default kernel call.
+constexpr int MAXIMUM_NUM_BLOCKS = 4096;
+
+inline int CUDA_GET_BLOCKS(const int N) {
+  return std::max(
+      std::min(
+          (N + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock,
+          MAXIMUM_NUM_BLOCKS),
+      // Use at least 1 block, since CUDA does not allow empty block.
+      1);
+}
+
+// Kernels
+
+template <typename T>
+__device__ T bilinear_interpolate(
+    const T* in_data,
+    const int height,
+    const int width,
+    T y,
+    T x,
+    const int index /* index for debug only*/) {
+  // deal with cases that inverse elements are out of feature map boundary
+  if (y < -1.0 || y > height || x < -1.0 || x > width) {
+    // empty
+    return 0;
+  }
+
+  if (y <= 0) {
+    y = 0;
+  }
+  if (x <= 0) {
+    x = 0;
+  }
+
+  int y_low = static_cast<int>(y);
+  int x_low = static_cast<int>(x);
+  int y_high;
+  int x_high;
+
+  if (y_low >= height - 1) {
+    y_high = y_low = height - 1;
+    y = (T)y_low;
+  } else {
+    y_high = y_low + 1;
+  }
+
+  if (x_low >= width - 1) {
+    x_high = x_low = width - 1;
+    x = (T)x_low;
+  } else {
+    x_high = x_low + 1;
+  }
+
+  T ly = y - y_low;
+  T lx = x - x_low;
+  T hy = 1. - ly, hx = 1. - lx;
+  // do bilinear interpolation
+  T v1 = in_data[y_low * width + x_low];
+  T v2 = in_data[y_low * width + x_high];
+  T v3 = in_data[y_high * width + x_low];
+  T v4 = in_data[y_high * width + x_high];
+  T w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx;
+
+  T val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4);
+
+  return val;
+}
+
+// Modified version of RoIAlignForwardKernel from Caffe (in roi_align.cu)
+// Main modifications:
+// - We don't need position_sensitive neither spatial_scale from the original RoIAlign kernel.
+// - We replace `channels` by `num_classes` and modify the logic consequently (e.g. offset_in_data
+//   does not use `c` anymore).
+template <typename T>
+__device__ void RoIAlignForward(
+    const T* in_data,  // (B, M, H, W)
+    const T* rois,  // (B, N, 4)
+    const T* matches,  // (B, N)
+    const int num_el,
+    const int num_classes,
+    const int height,
+    const int width,
+    const int pooled_height,
+    const int pooled_width,
+    const int sampling_ratio,
+    const int num_rois,
+    const int num_gtmasks,
+    T* out_data) {  // (B, N, C, H, W)
+  // Update kernel
+  for (size_t index = blockIdx.x * blockDim.x + threadIdx.x;
+       index < num_el;
+       index += blockDim.x * gridDim.x) {
+    // (n, c, ph, pw) is an element in the pooled output
+    int pw = index % pooled_width;
+    int ph = (index / pooled_width) % pooled_height;
+    // int c = (index / pooled_width / pooled_height) % num_classes;
+    int n = (index / pooled_width / pooled_height / num_classes) % num_rois;
+    int batch_idx = (index / pooled_width / pooled_height / num_classes / num_rois);
+
+    int roi_batch_ind = matches[batch_idx * num_rois + n];
+
+    const T* offset_rois = rois + batch_idx * (4 * num_rois) + n * 4;
+    // Do not using rounding; this implementation detail is critical
+    T roi_start_w = offset_rois[0];
+    T roi_start_h = offset_rois[1];
+    T roi_end_w = offset_rois[2];
+    T roi_end_h = offset_rois[3];
+
+    // Force malformed ROIs to be 1x1
+    T roi_width = max(roi_end_w - roi_start_w, (T)1.);
+    T roi_height = max(roi_end_h - roi_start_h, (T)1.);
+    T bin_size_h = static_cast<T>(roi_height) / static_cast<T>(pooled_height);
+    T bin_size_w = static_cast<T>(roi_width) / static_cast<T>(pooled_width);
+
+    const T* offset_in_data =
+        in_data + batch_idx * num_gtmasks * height * width
+        + roi_batch_ind * height * width;
+
+    // We use roi_bin_grid to sample the grid and mimic integral
+    int roi_bin_grid_h = (sampling_ratio > 0)
+        ? sampling_ratio
+        : ceil(roi_height / pooled_height);  // e.g., = 2
+    int roi_bin_grid_w =
+        (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width);
+
+    // We do average (integral) pooling inside a bin
+    const T count = roi_bin_grid_h * roi_bin_grid_w;  // e.g. = 4
+
+    T output_val = 0.;
+    for (int iy = 0; iy < roi_bin_grid_h; iy++) {  // e.g., iy = 0, 1
+      const T y = roi_start_h + ph * bin_size_h +
+          static_cast<T>(iy + .5f) * bin_size_h /
+              static_cast<T>(roi_bin_grid_h);  // e.g., 0.5, 1.5
+      for (int ix = 0; ix < roi_bin_grid_w; ix++) {
+        const T x = roi_start_w + pw * bin_size_w +
+            static_cast<T>(ix + .5f) * bin_size_w /
+                static_cast<T>(roi_bin_grid_w);
+
+        T val = bilinear_interpolate(
+            offset_in_data, height, width, y, x, index);
+        output_val += val;
+      }
+    }
+    output_val /= count;
+
+    out_data[index] = output_val;
+  }
+}
+
+
+template<typename DType>
+__global__ void MRCNNMaskTargetKernel(const DType *rois,
+                                      const DType *gt_masks,
+                                      const DType *matches,
+                                      const DType *cls_targets,
+                                      DType* sampled_masks,
+                                      DType* mask_cls,
+                                      const int total_out_el,
+                                      int batch_size,
+                                      int num_classes,
+                                      int num_rois,
+                                      int num_gtmasks,
+                                      int gt_height,
+                                      int gt_width,
+                                      int mask_size,
+                                      int sample_ratio) {
+  // computing sampled_masks
+  RoIAlignForward(gt_masks, rois, matches, total_out_el,
+                  num_classes, gt_height, gt_width, mask_size, mask_size,
+                  sample_ratio, num_rois, num_gtmasks, sampled_masks);
+  // computing mask_cls
+  int num_masks = batch_size * num_rois * num_classes;
+  int mask_vol = mask_size * mask_size;
+  for (int mask_idx = blockIdx.x; mask_idx < num_masks; mask_idx += gridDim.x) {
+    int cls_idx = mask_idx % num_classes;
+    int roi_idx = (mask_idx / num_classes) % num_rois;
+    int batch_idx = (mask_idx / num_classes / num_rois);
+
+    DType* mask_cls_out = mask_cls + mask_idx * mask_vol;
+
+    DType cls_target = cls_targets[batch_idx * num_rois + roi_idx];
+    DType out_val = (cls_target == cls_idx);
+    for (int mask_pixel = threadIdx.x; mask_pixel < mask_vol; mask_pixel += blockDim.x) {
+      mask_cls_out[mask_pixel] = out_val;
+    }
+  }
+}
+
+template<>
+void MRCNNMaskTargetRun<gpu>(const MRCNNMaskTargetParam& param, const std::vector<TBlob> &inputs,
+                             const std::vector<TBlob> &outputs, mshadow::Stream<gpu> *s) {
+  const int block_dim_size = kMaxThreadsPerBlock;
+  using namespace mxnet_op;
+  using mshadow::Tensor;
+
+  auto stream = mshadow::Stream<gpu>::GetStream(s);
+
+  MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, {
+    auto rois = inputs[mrcnn_index::kRoi].FlatToKD<gpu, 3, DType>(s);
+    auto gt_masks = inputs[mrcnn_index::kGtMask].FlatToKD<gpu, 4, DType>(s);
+    auto matches = inputs[mrcnn_index::kMatches].FlatTo2D<gpu, DType>(s);
+    auto cls_targets = inputs[mrcnn_index::kClasses].FlatTo2D<gpu, DType>(s);
+
+    auto out_masks = outputs[mrcnn_index::kMask].FlatToKD<gpu, 5, DType>(s);
+    auto out_mask_cls = outputs[mrcnn_index::kMaskClasses].FlatToKD<gpu, 5, DType>(s);
+
+    int batch_size = gt_masks.shape_[0];
+    int num_gtmasks = gt_masks.shape_[1];
+    int gt_height = gt_masks.shape_[2];
+    int gt_width = gt_masks.shape_[3];
+
+    int num_el = outputs[mrcnn_index::kMask].Size();
+
+    dim3 dimGrid = dim3(CUDA_GET_BLOCKS(num_el));
+    dim3 dimBlock = dim3(block_dim_size);
+
+    MRCNNMaskTargetKernel<<<dimGrid, dimBlock, 0, stream>>>
+    (rois.dptr_, gt_masks.dptr_, matches.dptr_, cls_targets.dptr_,
+    out_masks.dptr_, out_mask_cls.dptr_,
+    num_el, batch_size, param.num_classes, param.num_rois,
+    num_gtmasks, gt_height, gt_width, param.mask_size, param.sample_ratio);
+    MSHADOW_CUDA_POST_KERNEL_CHECK(MRCNNMaskTargetKernel);
+  });
+}
+
+DMLC_REGISTER_PARAMETER(MRCNNMaskTargetParam);
+
+NNVM_REGISTER_OP(_contrib_mrcnn_mask_target)
+.describe("Generate mask targets for Mask-RCNN.")
+.set_num_inputs(4)
+.set_num_outputs(2)
+.set_attr_parser(ParamParser<MRCNNMaskTargetParam>)
+.set_attr<mxnet::FInferShape>("FInferShape", MRCNNMaskTargetShape)
+.set_attr<nnvm::FInferType>("FInferType", MRCNNMaskTargetType)
+.set_attr<FCompute>("FCompute<gpu>", MRCNNMaskTargetCompute<gpu>)
+.add_argument("rois", "NDArray-or-Symbol", "Bounding box coordinates, a 3D array")
+.add_argument("gt_masks", "NDArray-or-Symbol", "Input masks of full image size, a 4D array")
+.add_argument("matches", "NDArray-or-Symbol", "Index to a gt_mask, a 2D array")
+.add_argument("cls_targets", "NDArray-or-Symbol",
+              "Value [0, num_class), excluding background class, a 2D array")
+.add_arguments(MRCNNMaskTargetParam::__FIELDS__());
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/contrib/reset_arrays-inl.h b/src/operator/contrib/reset_arrays-inl.h
new file mode 100644
index 000000000000..3559a21879e4
--- /dev/null
+++ b/src/operator/contrib/reset_arrays-inl.h
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ *  Copyright (c) 2019 by Contributors
+ * \file reset_arrays-inl.h
+ * \brief setting all array element values to zeros
+ * \author Moises Hernandez-Fernandez, Andrei Ivanov
+ */
+
+#ifndef MXNET_OPERATOR_CONTRIB_RESET_ARRAYS_INL_H_
+#define MXNET_OPERATOR_CONTRIB_RESET_ARRAYS_INL_H_
+
+#include <vector>
+#include "../tensor/init_op.h"
+
+namespace mxnet {
+namespace op {
+
+struct ResetArraysParam : public dmlc::Parameter<ResetArraysParam> {
+  int num_arrays;
+  DMLC_DECLARE_PARAMETER(ResetArraysParam) {
+    DMLC_DECLARE_FIELD(num_arrays)
+    .describe("number of input arrays.");
+  }
+};
+
+inline bool ResetArraysShape(const NodeAttrs& attrs,
+                            std::vector<mxnet::TShape>* in_shape,
+                            std::vector<mxnet::TShape>* out_shape) {
+  const auto& param = dmlc::get<ResetArraysParam>(attrs.parsed);
+  CHECK_EQ(in_shape->size(), param.num_arrays);
+  for (auto s : *in_shape) {
+    if (s.ndim() == 0)
+      return false;
+  }
+
+  return true;
+}
+
+inline bool ResetArraysType(const NodeAttrs& attrs,
+                           std::vector<int>* in_type,
+                           std::vector<int>* out_type) {
+  const auto& param = dmlc::get<ResetArraysParam>(attrs.parsed);
+  CHECK_EQ(in_type->size(), param.num_arrays);
+  for (size_t i = 0; i < in_type->size(); ++i) {
+    if ((*in_type)[i] == -1)
+      return false;
+  }
+
+  return true;
+}
+
+template<typename xpu>
+void ResetMemory(void *pntr, size_t len, mshadow::Stream<xpu> *s);
+
+template<typename xpu>
+void ResetArrays(const nnvm::NodeAttrs& attrs,
+                 const OpContext &ctx,
+                 const std::vector<TBlob> &inputs,
+                 const std::vector<OpReqType> &req,
+                 const std::vector<TBlob> &outputs) {
+  auto s = ctx.get_stream<xpu>();
+  const auto& param = nnvm::get<ResetArraysParam>(attrs.parsed);
+  for (int i = 0; i < param.num_arrays; i++) {  // array index in inputs
+    const size_t size = inputs[i].shape_.Size();
+    MSHADOW_REAL_TYPE_SWITCH(inputs[i].type_flag_, DType,
+      ResetMemory(inputs[i].FlatTo2D<xpu, DType>(s).dptr_, size * sizeof(DType), s);
+    )
+  }
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_CONTRIB_RESET_ARRAYS_INL_H_
diff --git a/src/operator/contrib/reset_arrays.cc b/src/operator/contrib/reset_arrays.cc
new file mode 100644
index 000000000000..f67e0098cd11
--- /dev/null
+++ b/src/operator/contrib/reset_arrays.cc
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ *  Copyright (c) 2019 by Contributors
+ * \file reset_arrays.cc
+ * \brief setting all array element values to zeros
+ * \author Moises Hernandez-Fernandez, Andrei Ivanov
+ */
+
+#include "./reset_arrays-inl.h"
+
+namespace mxnet {
+namespace op {
+
+DMLC_REGISTER_PARAMETER(ResetArraysParam);
+
+NNVM_REGISTER_OP(reset_arrays)
+.describe(R"code(Set to zero multiple arrays
+)code" ADD_FILELINE)
+.set_num_inputs([](const nnvm::NodeAttrs& attrs) {
+    return static_cast<uint32_t>(dmlc::get<ResetArraysParam>(attrs.parsed).num_arrays);
+  })
+.set_attr<nnvm::FMutateInputs>("FMutateInputs",
+  [](const nnvm::NodeAttrs& attrs) {
+    const uint32_t num_args = dmlc::get<ResetArraysParam>(attrs.parsed).num_arrays;
+    std::vector<uint32_t> ret;
+    for (uint32_t i = 0; i < num_args; ++i) {
+      ret.push_back(i);
+    }
+    return ret;
+  })
+.set_num_outputs(0)
+.set_attr_parser(ParamParser<ResetArraysParam>)
+.set_attr<mxnet::FInferShape>("FInferShape", ResetArraysShape)
+.set_attr<nnvm::FInferType>("FInferType", ResetArraysType)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    const uint32_t num_args = dmlc::get<ResetArraysParam>(attrs.parsed).num_arrays;
+    std::vector<std::string> ret;
+    for (uint32_t i = 0; i < num_args; ++i) {
+      ret.push_back(std::string("array_") + std::to_string(i));
+    }
+    return ret;
+  })
+.add_argument("data", "NDArray-or-Symbol[]", "Arrays")
+.add_arguments(ResetArraysParam::__FIELDS__());
+
+NNVM_REGISTER_OP(reset_arrays)
+.set_attr<FCompute>("FCompute<cpu>", ResetArrays<cpu>);
+
+template<>
+void ResetMemory<cpu>(void *pntr, size_t len, mshadow::Stream<cpu> *s) {
+  memset(pntr, 0, len);
+}
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/contrib/reset_arrays.cu b/src/operator/contrib/reset_arrays.cu
new file mode 100644
index 000000000000..f7a9d0034665
--- /dev/null
+++ b/src/operator/contrib/reset_arrays.cu
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ *  Copyright (c) 2019 by Contributors
+ * \file reset_arrays.cu
+ * \brief setting all array element values to zeros
+ * \author Moises Hernandez-Fernandez, Andrei Ivanov
+ */
+#include "./reset_arrays-inl.h"
+
+namespace mxnet {
+namespace op {
+
+template<>
+void ResetMemory<gpu>(void *pntr, size_t len, mshadow::Stream<gpu> *s) {
+  CUDA_CALL(cudaMemsetAsync(pntr, 0, len, mshadow::Stream<gpu>::GetStream(s)));
+}
+
+NNVM_REGISTER_OP(reset_arrays)
+.set_attr<FCompute>("FCompute<gpu>", ResetArrays<gpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h
index 471b6f395a05..e14c8dbb0b78 100644
--- a/src/operator/mshadow_op.h
+++ b/src/operator/mshadow_op.h
@@ -383,6 +383,8 @@ MXNET_BINARY_MATH_OP(logical_or, a || b ? DType(1) : DType(0));
 
 MXNET_BINARY_MATH_OP(logical_xor, (a || b) && !(a && b) ? DType(1) : DType(0));
 
+MXNET_BINARY_MATH_OP(bitwise_xor, static_cast<int64_t>(a) ^ static_cast<int64_t>(b));
+
 MXNET_UNARY_MATH_OP(square_root, math::sqrt(a));
 
 MXNET_UNARY_MATH_OP(square_root_grad, 0.5f / math::id(a));
diff --git a/src/operator/nn/concat-inl.h b/src/operator/nn/concat-inl.h
index 7a58ae6f0ccc..1fb20ac65473 100644
--- a/src/operator/nn/concat-inl.h
+++ b/src/operator/nn/concat-inl.h
@@ -141,6 +141,37 @@ void ConcatCompute(const nnvm::NodeAttrs& attrs, const OpContext& ctx,
   });
 }
 
+template<typename xpu>
+void DStackCompute(const nnvm::NodeAttrs& attrs, const OpContext& ctx,
+                   const std::vector<TBlob>& inputs,
+                   const std::vector<OpReqType>& req,
+                   const std::vector<TBlob>& outputs) {
+  ConcatParam param = nnvm::get<ConcatParam>(attrs.parsed);
+  param.dim = 2;
+  std::vector<TBlob> modified_inputs(inputs.size());
+  for (int i = 0; i < param.num_args; ++i) {
+    if (inputs[i].shape_.ndim() == 0) {
+      modified_inputs[i] = inputs[i].reshape(TShape(3, 1));
+    } else if (inputs[i].shape_.ndim() == 1) {
+      TShape t = TShape(3, 1);
+      t[1] = inputs[i].shape_[0];
+      modified_inputs[i] = inputs[i].reshape(t);
+    } else if (inputs[i].shape_.ndim() == 2) {
+      TShape t = TShape(3, 1);
+      t[0] = inputs[i].shape_[0];
+      t[1] = inputs[i].shape_[1];
+      modified_inputs[i] = inputs[i].reshape(t);
+    } else {
+      modified_inputs[i] = inputs[i];
+    }
+  }
+  MSHADOW_TYPE_SWITCH(inputs[concat_enum::kData0].type_flag_, DType, {
+    ConcatOp<xpu, DType> op;
+    op.Init(param);
+    op.Forward(ctx, modified_inputs, req, outputs);
+  });
+}
+
 template<typename xpu>
 void ConcatGradCompute(const nnvm::NodeAttrs& attrs, const OpContext& ctx,
                        const std::vector<TBlob>& inputs,
@@ -154,6 +185,37 @@ void ConcatGradCompute(const nnvm::NodeAttrs& attrs, const OpContext& ctx,
   });
 }
 
+template<typename xpu>
+void DStackGradCompute(const nnvm::NodeAttrs& attrs, const OpContext& ctx,
+                       const std::vector<TBlob>& inputs,
+                       const std::vector<OpReqType>& req,
+                       const std::vector<TBlob>& outputs) {
+  ConcatParam param = nnvm::get<ConcatParam>(attrs.parsed);
+  param.dim = 2;
+  std::vector<TBlob> modified_outputs(outputs.size());
+  for (int i = 0; i < param.num_args; ++i) {
+    if (outputs[i].shape_.ndim() == 0) {
+      modified_outputs[i] = outputs[i].reshape(TShape(3, 1));
+    } else if (outputs[i].shape_.ndim() == 1) {
+      TShape t  = TShape(3, 1);
+      t[1] = outputs[i].shape_[0];
+      modified_outputs[i] = outputs[i].reshape(t);
+    } else if (outputs[i].shape_.ndim() == 2) {
+      TShape t  = TShape(3, 1);
+      t[0] = outputs[i].shape_[0];
+      t[1] = outputs[i].shape_[1];
+      modified_outputs[i] = outputs[i].reshape(t);
+    } else {
+      modified_outputs[i] = outputs[i];
+    }
+  }
+  MSHADOW_TYPE_SWITCH(inputs[concat_enum::kOut].type_flag_, DType, {
+    ConcatOp<xpu, DType> op;
+    op.Init(param);
+    op.Backward(ctx, inputs[concat_enum::kOut], req, modified_outputs);
+  });
+}
+
 /*!
  * \brief concat CSRNDArray on the first dimension.
  */
diff --git a/src/operator/numpy/np_elemwise_broadcast_op.cu b/src/operator/numpy/np_elemwise_broadcast_op.cu
index b48c0b0ace6d..3b780325227a 100644
--- a/src/operator/numpy/np_elemwise_broadcast_op.cu
+++ b/src/operator/numpy/np_elemwise_broadcast_op.cu
@@ -27,6 +27,7 @@
 
 namespace mxnet {
 namespace op {
+
 NNVM_REGISTER_OP(_npi_add)
 .set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastCompute<gpu, op::mshadow_op::plus>);
 
@@ -137,5 +138,8 @@ NNVM_REGISTER_OP(_backward_npi_ldexp_scalar)
 NNVM_REGISTER_OP(_backward_npi_rldexp_scalar)
 .set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Backward<gpu, mshadow_op::rldexp_grad>);
 
+NNVM_REGISTER_OP(_np_bitwise_xor)
+.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastCompute<gpu, mshadow_op::bitwise_xor>);
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_init_op.cc b/src/operator/numpy/np_init_op.cc
index 2477573c2413..f2446c85b182 100644
--- a/src/operator/numpy/np_init_op.cc
+++ b/src/operator/numpy/np_init_op.cc
@@ -22,6 +22,7 @@
  * \file np_init_op.cc
  * \brief CPU Implementation of numpy init op
  */
+
 #include "../tensor/init_op.h"
 #include "../tensor/elemwise_unary_op.h"
 #include "./np_init_op.h"
@@ -29,7 +30,10 @@
 namespace mxnet {
 namespace op {
 
+
+DMLC_REGISTER_PARAMETER(NumpyEyeParam);
 DMLC_REGISTER_PARAMETER(IndicesOpParam);
+DMLC_REGISTER_PARAMETER(LogspaceParam);
 
 inline bool NumpyIndicesShape(const nnvm::NodeAttrs& attrs,
                               mxnet::ShapeVector* in_shapes,
@@ -51,6 +55,18 @@ inline bool NumpyIndicesShape(const nnvm::NodeAttrs& attrs,
   return shape_is_known(out_shapes->at(0));
 }
 
+inline bool LogspaceShape(const nnvm::NodeAttrs& attrs,
+                          mxnet::ShapeVector *in_attrs,
+                          mxnet::ShapeVector *out_attrs) {
+  const LogspaceParam& param = nnvm::get<LogspaceParam>(attrs.parsed);
+  CHECK_EQ(in_attrs->size(), 0U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  CHECK_GE(param.num, 0)
+    << "Number of sequence should be non-negative, received " << param.num;
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape({static_cast<nnvm::dim_t>(param.num)}));
+  return true;
+}
+
 NNVM_REGISTER_OP(_npi_zeros)
 .set_num_inputs(0)
 .set_num_outputs(1)
@@ -117,23 +133,6 @@ NNVM_REGISTER_OP(_np_ones_like)
 .add_argument("a", "NDArray-or-Symbol",
               "The shape and data-type of a define these same attributes of the returned array.");
 
-bool NumpyRangeShape(const nnvm::NodeAttrs& attrs,
-                     mxnet::ShapeVector* in_shapes,
-                     mxnet::ShapeVector* out_shapes) {
-  const RangeParam& param = nnvm::get<RangeParam>(attrs.parsed);
-  CHECK_EQ(in_shapes->size(), 0U);
-  CHECK_EQ(out_shapes->size(), 1U);
-  CHECK_NE(param.step, 0) << "_npi_arange does not support step=0";
-  CHECK_EQ(param.repeat, 1) << "_npi_arange only supports repeat=1, received " << param.repeat;
-  CHECK(param.stop.has_value()) << "_npi_arange requires stop to have a value";
-  double out_size = std::ceil((param.stop.value() - param.start) / param.step);
-  if (out_size < 0) {
-    out_size = 0;
-  }
-  SHAPE_ASSIGN_CHECK(*out_shapes, 0, mxnet::TShape({static_cast<nnvm::dim_t>(out_size)}));
-  return true;
-}
-
 NNVM_REGISTER_OP(_npi_arange)
 .set_num_inputs(0)
 .set_num_outputs(1)
@@ -143,6 +142,16 @@ NNVM_REGISTER_OP(_npi_arange)
 .set_attr<FCompute>("FCompute<cpu>", RangeCompute<cpu, RangeParam>)
 .add_arguments(RangeParam::__FIELDS__());
 
+NNVM_REGISTER_OP(_npi_eye)
+.describe("Return a 2-D array with ones on the diagonal and zeros elsewhere.")
+.set_num_inputs(0)
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<NumpyEyeParam>)
+.set_attr<mxnet::FInferShape>("FInferShape", NumpyEyeShape)
+.set_attr<nnvm::FInferType>("FInferType", InitType<NumpyEyeParam>)
+.set_attr<FCompute>("FCompute<cpu>", NumpyEyeFill<cpu>)
+.add_arguments(NumpyEyeParam::__FIELDS__());
+
 NNVM_REGISTER_OP(_npi_indices)
 .describe("Return an array representing the indices of a grid.")
 .set_num_inputs(0)
@@ -153,5 +162,15 @@ NNVM_REGISTER_OP(_npi_indices)
 .set_attr<FCompute>("FCompute<cpu>", IndicesCompute<cpu>)
 .add_arguments(IndicesOpParam::__FIELDS__());
 
+NNVM_REGISTER_OP(_npi_logspace)
+.describe("Return numbers spaced evenly on a log scale.")
+.set_num_inputs(0)
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<LogspaceParam>)
+.set_attr<mxnet::FInferShape>("FInferShape", LogspaceShape)
+.set_attr<nnvm::FInferType>("FInferType", InitType<LogspaceParam>)
+.set_attr<FCompute>("FCompute<cpu>", LogspaceCompute<cpu>)
+.add_arguments(LogspaceParam::__FIELDS__());
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_init_op.cu b/src/operator/numpy/np_init_op.cu
index e68dd9ad36a1..b6e2f93e8d3c 100644
--- a/src/operator/numpy/np_init_op.cu
+++ b/src/operator/numpy/np_init_op.cu
@@ -47,8 +47,14 @@ NNVM_REGISTER_OP(_np_ones_like)
 NNVM_REGISTER_OP(_npi_arange)
 .set_attr<FCompute>("FCompute<gpu>", RangeCompute<gpu, RangeParam>);
 
+NNVM_REGISTER_OP(_npi_eye)
+.set_attr<FCompute>("FCompute<gpu>", NumpyEyeFill<gpu>);
+
 NNVM_REGISTER_OP(_npi_indices)
 .set_attr<FCompute>("FCompute<gpu>", IndicesCompute<gpu>);
 
+NNVM_REGISTER_OP(_npi_logspace)
+.set_attr<FCompute>("FCompute<gpu>", LogspaceCompute<gpu>);
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_init_op.h b/src/operator/numpy/np_init_op.h
index 3e1c345d59c3..69999ae8710e 100644
--- a/src/operator/numpy/np_init_op.h
+++ b/src/operator/numpy/np_init_op.h
@@ -26,6 +26,7 @@
 #ifndef MXNET_OPERATOR_NUMPY_NP_INIT_OP_H_
 #define MXNET_OPERATOR_NUMPY_NP_INIT_OP_H_
 
+#include <algorithm>
 #include <vector>
 #include <string>
 #include "../tensor/init_op.h"
@@ -35,6 +36,34 @@
 namespace mxnet {
 namespace op {
 
+struct NumpyEyeParam : public dmlc::Parameter<NumpyEyeParam> {
+  nnvm::dim_t N;
+  dmlc::optional<nnvm::dim_t> M;
+  nnvm::dim_t k;
+  std::string ctx;
+  int dtype;
+  DMLC_DECLARE_PARAMETER(NumpyEyeParam) {
+    DMLC_DECLARE_FIELD(N)
+    .describe("Number of rows in the output.");
+    DMLC_DECLARE_FIELD(M)
+    .set_default(dmlc::optional<nnvm::dim_t>())
+    .describe("Number of columns in the output. If None, defaults to N.");
+    DMLC_DECLARE_FIELD(k)
+    .set_default(0)
+    .describe("Index of the diagonal. 0 (the default) refers to the main diagonal,"
+              "a positive value refers to an upper diagonal."
+              "and a negative value to a lower diagonal.");
+    DMLC_DECLARE_FIELD(ctx)
+    .set_default("")
+    .describe("Context of output, in format [cpu|gpu|cpu_pinned](n)."
+              "Only used for imperative calls.");
+    DMLC_DECLARE_FIELD(dtype)
+    .set_default(mshadow::kFloat32)
+    MXNET_ADD_ALL_TYPES
+    .describe("Data-type of the returned array.");
+  }
+};
+
 struct IndicesOpParam : public dmlc::Parameter<IndicesOpParam> {
   mxnet::TShape dimensions;
   int dtype;
@@ -52,6 +81,50 @@ struct IndicesOpParam : public dmlc::Parameter<IndicesOpParam> {
   }
 };
 
+inline bool NumpyRangeShape(const nnvm::NodeAttrs& attrs,
+                            mxnet::ShapeVector* in_shapes,
+                            mxnet::ShapeVector* out_shapes) {
+  const RangeParam& param = nnvm::get<RangeParam>(attrs.parsed);
+  CHECK_EQ(in_shapes->size(), 0U);
+  CHECK_EQ(out_shapes->size(), 1U);
+  CHECK_NE(param.step, 0) << "_npi_arange does not support step=0";
+  CHECK_EQ(param.repeat, 1) << "_npi_arange only supports repeat=1, received " << param.repeat;
+  CHECK(param.stop.has_value()) << "_npi_arange requires stop to have a value";
+  double out_size = std::ceil((param.stop.value() - param.start) / param.step);
+  if (out_size < 0) {
+    out_size = 0;
+  }
+  SHAPE_ASSIGN_CHECK(*out_shapes, 0, mxnet::TShape({static_cast<nnvm::dim_t>(out_size)}));
+  return true;
+}
+
+inline bool NumpyEyeShape(const nnvm::NodeAttrs& attrs,
+                          mxnet::ShapeVector *in_attrs,
+                          mxnet::ShapeVector *out_attrs) {
+  const NumpyEyeParam& param = nnvm::get<NumpyEyeParam>(attrs.parsed);
+  CHECK_EQ(in_attrs->size(), 0U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  nnvm::dim_t M = param.M.has_value() ? param.M.value() : param.N;
+  CHECK(param.N >= 0) << "negative dimensions are not allowed. N is " << param.N;
+  CHECK(M >= 0) << "negative dimensions are not allowed. M is " << M;
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::Shape2(param.N, M));
+
+  return out_attrs->at(0).ndim() != 0U;
+}
+template<typename xpu>
+void NumpyEyeFill(const nnvm::NodeAttrs& attrs,
+                  const OpContext& ctx,
+                  const std::vector<TBlob>& inputs,
+                  const std::vector<OpReqType>& req,
+                  const std::vector<TBlob>& outputs) {
+  CHECK_EQ(inputs.size(), 0U);
+  CHECK_EQ(outputs.size(), 1U);
+  if (outputs[0].shape_.Size() == 0) return;  // zero-size tensor
+  const NumpyEyeParam& param = nnvm::get<NumpyEyeParam>(attrs.parsed);
+  const nnvm::dim_t num_cols = param.M.has_value() ? param.M.value() : param.N;
+  EyeFillImpl<xpu>(outputs[0], ctx, req, num_cols, param.N, param.k);
+}
+
 template<int req>
 struct indices_fwd {
   template<typename DType>
@@ -140,6 +213,65 @@ void IdentityCompute(const nnvm::NodeAttrs& attrs,
   });
 }
 
+struct LogspaceParam : public dmlc::Parameter<LogspaceParam> {
+  double start;
+  double stop;
+  int num;
+  bool endpoint;
+  double base;
+  std::string ctx;
+  int dtype;
+  DMLC_DECLARE_PARAMETER(LogspaceParam) {
+    DMLC_DECLARE_FIELD(start)
+    .describe("The starting value of the sequence.");
+    DMLC_DECLARE_FIELD(stop)
+    .describe("The ending value of the sequence");
+    DMLC_DECLARE_FIELD(num)
+    .describe("Number of samples to generate. Must be non-negative.");
+    DMLC_DECLARE_FIELD(endpoint)
+    .set_default(true)
+    .describe("If True, stop is the last sample. Otherwise, it is not included.");
+    DMLC_DECLARE_FIELD(base)
+    .set_default(10.0)
+    .describe("The base of the log space. The step size between the elements in "
+    "ln(samples) / ln(base) (or log_base(samples)) is uniform.");
+    DMLC_DECLARE_FIELD(ctx)
+    .set_default("")
+    .describe("Context of output, in format [cpu|gpu|cpu_pinned](n)."
+    "Only used for imperative calls.");
+    DMLC_DECLARE_FIELD(dtype).set_default(mshadow::kFloat32)
+    MXNET_ADD_ALL_TYPES
+    .describe("Target data type.");
+  }
+};
+
+struct logspace_fwd {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, double start, double stop, double base,
+                                  double step, int req, DType* out) {
+    KERNEL_ASSIGN(out[i], req,
+                  static_cast<DType>(math::pow(base, static_cast<double>(start + step * i))));
+  }
+};
+
+template<typename xpu>
+void LogspaceCompute(const nnvm::NodeAttrs& attrs,
+                     const OpContext& ctx,
+                     const std::vector<TBlob>& inputs,
+                     const std::vector<OpReqType>& req,
+                     const std::vector<TBlob>& outputs) {
+  using namespace mxnet_op;
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const LogspaceParam& param = nnvm::get<LogspaceParam>(attrs.parsed);
+  if (param.num == 0) return;
+  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, {
+      int step_num = param.endpoint ? param.num - 1 : param.num;
+      double step = step_num > 0 ? (param.stop - param.start) / step_num : 0.0f;
+      Kernel<logspace_fwd, xpu>::Launch(s, outputs[0].Size(), param.start, param.stop, param.base,
+          step, req[0], outputs[0].dptr<DType>());
+  });
+}
+
 }  // namespace op
 }  // namespace mxnet
 
diff --git a/src/operator/numpy/np_matrix_op-inl.h b/src/operator/numpy/np_matrix_op-inl.h
index 0eefeb9cdb5d..b3206bf4aa75 100644
--- a/src/operator/numpy/np_matrix_op-inl.h
+++ b/src/operator/numpy/np_matrix_op-inl.h
@@ -365,6 +365,373 @@ void NumpyFlipForward(const nnvm::NodeAttrs& attrs,
   NumpyFlipForwardImpl<xpu>(ctx, inputs, outputs, stride_, trailing_, flip_index);
 }
 
+struct NumpyMoveaxisParam : public dmlc::Parameter<NumpyMoveaxisParam> {
+  mxnet::TShape source;
+  mxnet::TShape destination;
+  DMLC_DECLARE_PARAMETER(NumpyMoveaxisParam) {
+    DMLC_DECLARE_FIELD(source)
+    .describe("Original positions of the axes to move. These must be unique.");
+    DMLC_DECLARE_FIELD(destination)
+    .describe("Destination positions for each of the original axes. "
+              "These must also be unique.");
+  }
+};
+
+inline mxnet::TShape NumpyMoveaxisShapeImpl(const nnvm::NodeAttrs& attrs,
+                                            const int& ndim) {
+  const NumpyMoveaxisParam& param = nnvm::get<NumpyMoveaxisParam>(attrs.parsed);
+  mxnet::TShape axes(ndim, -1);
+  std::vector<bool> state_axes(ndim, false);
+  mxnet::TShape real_src(param.source.ndim(), -1);
+  mxnet::TShape real_des(param.destination.ndim(), -1);
+  for (int i = 0; i < param.source.ndim(); ++i) {
+    if (param.source[i] >= 0) {
+      CHECK_LT(static_cast<size_t>(param.source[i]), ndim);
+      real_src[i] = param.source[i];
+    } else {
+      CHECK_LT(param.source[i] + ndim, ndim);
+      real_src[i] = param.source[i] + ndim;
+    }
+    if (param.destination[i] >= 0) {
+      CHECK_LT(static_cast<size_t>(param.destination[i]), ndim);
+      real_des[i] = param.destination[i];
+    } else {
+      CHECK_LT(param.destination[i] + ndim, ndim);
+      real_des[i] = param.destination[i] + ndim;
+    }
+  }
+  if (ndim > 1) {
+    for (int i = 0; i < param.source.ndim() - 1; ++i) {
+      for (int j = i + 1; j < param.source.ndim(); ++j) {
+        CHECK_NE(real_src[i], real_src[j])
+          << "repeated axis in `source` argument";
+        CHECK_NE(real_des[i], real_des[j])
+          << "repeated axis in `destination` argument";
+      }
+    }
+  }
+  for (int i = 0; i < param.source.ndim(); ++i) {
+    axes[real_des[i]] = real_src[i];
+    state_axes[real_src[i]] = true;
+  }
+  for (int i = 0; i < axes.ndim(); ++i) {
+    if (axes[i] < 0) {
+      for (int j = 0; j < axes.ndim(); ++j) {
+        if (state_axes[j] == false) {
+          axes[i] = j;
+          state_axes[j] = true;
+          break;
+        }
+      }
+    }
+  }
+  return axes;
+}
+
+template<typename xpu>
+void NumpyMoveaxisCompute(const nnvm::NodeAttrs& attrs,
+                          const OpContext& ctx,
+                          const std::vector<TBlob>& inputs,
+                          const std::vector<OpReqType>& req,
+                          const std::vector<TBlob>& outputs) {
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  const NumpyMoveaxisParam& param = nnvm::get<NumpyMoveaxisParam>(attrs.parsed);
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(req[0], kWriteTo) << "Moveaxis does not support inplace";
+  CHECK_EQ(param.source.ndim(), param.destination.ndim())
+    << "source and destination not equal.";
+  mxnet::TShape axes;
+  axes = NumpyMoveaxisShapeImpl(attrs, inputs[0].ndim());
+  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, Dtype, {
+    TransposeImpl<xpu>(ctx.run_ctx, inputs[0], outputs[0], axes);
+  })
+}
+
+struct NumpyRot90Param : public dmlc::Parameter<NumpyRot90Param> {
+  int k;
+  dmlc::optional<mxnet::TShape> axes;
+  DMLC_DECLARE_PARAMETER(NumpyRot90Param) {
+    DMLC_DECLARE_FIELD(k)
+    .set_default(1)
+    .describe("Number of times the array is rotated by 90 degrees.");
+    DMLC_DECLARE_FIELD(axes)
+    .set_default(dmlc::optional<mxnet::TShape>())
+    .describe(" The array is rotated in the plane defined by the axes. Axes must be different.");
+  }
+};
+
+struct rot90reverse {
+  MSHADOW_XINLINE static index_t ReverseIndex(index_t idx,
+                                              index_t nreversedim,
+                                              const index_t * stride_,
+                                              const index_t * trailing_) {
+    index_t outputIndex = idx;
+    for (index_t i = 0; i < nreversedim; ++i) {
+      const index_t low = outputIndex % trailing_[i];
+      index_t high = outputIndex / trailing_[i];
+      const index_t x = high % stride_[i];
+      high /= stride_[i];
+      outputIndex = (high * stride_[i] + stride_[i] - 1 - x) * trailing_[i] + low;
+    }
+    return outputIndex;
+  }
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t index, index_t nreversedim, const DType *src, DType *dst,
+                                  const index_t * stride_,
+                                  const index_t * trailing_) {
+    index_t new_idx = ReverseIndex(index, nreversedim, stride_, trailing_);
+    dst[new_idx] = src[index];
+  }
+};
+
+template<typename xpu>
+void NumpyRot90ComputeFlipIml(const OpContext& ctx,
+                              const std::vector<TBlob>& inputs,
+                              const std::vector<OpReqType>& req,
+                              const std::vector<TBlob>& outputs,
+                              const index_t axis0, const index_t axis1) {
+  using namespace mshadow;
+  using namespace mxnet_op;
+
+  const mxnet::TShape& ishape = inputs[0].shape_;
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+
+  std::vector<index_t> stride_(2);
+  std::vector<index_t>  trailing_(2);
+  index_t reverse_index = 0;
+  std::vector<index_t> temp{axis0, axis1};
+  for (int axis : temp) {
+    stride_[reverse_index] = ishape[axis];
+    trailing_[reverse_index] = 1;
+    for (int i2 = axis + 1; i2 < ishape.ndim(); ++i2) {
+      trailing_[reverse_index] *= ishape[i2];
+    }
+    reverse_index++;
+  }
+
+  index_t workspace_size = 2 * sizeof(index_t);
+  Tensor<xpu, 1, char> workspace =
+      ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(2 * workspace_size), s);
+  Tensor<cpu, 1, index_t> stride_cpu_tensor(stride_.data(), Shape1(stride_.size()));
+  Tensor<xpu, 1, index_t> stride_xpu_tensor(
+      reinterpret_cast<index_t*>(workspace.dptr_), Shape1(stride_.size()));
+  Tensor<cpu, 1, index_t> trailing_cpu_tensor(trailing_.data(), Shape1(trailing_.size()));
+  Tensor<xpu, 1, index_t> trailing_xpu_tensor(
+      reinterpret_cast<index_t*>(workspace.dptr_ + workspace_size), Shape1(trailing_.size()));
+
+  mshadow::Copy(stride_xpu_tensor, stride_cpu_tensor, s);
+  mshadow::Copy(trailing_xpu_tensor, trailing_cpu_tensor, s);
+  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, {
+    Kernel<rot90reverse, xpu>::Launch(s, inputs[0].Size(), reverse_index,
+                                      inputs[0].dptr<DType>(), outputs[0].dptr<DType>(),
+                                      stride_xpu_tensor.dptr_, trailing_xpu_tensor.dptr_);
+  });
+}
+
+struct rot90Transreverse {
+  MSHADOW_XINLINE static index_t ReverseIndex(index_t idx,
+                                              const index_t stride_,
+                                              const index_t trailing_) {
+    index_t outputIndex = idx;
+    const index_t low = outputIndex % trailing_;
+    index_t high = outputIndex / trailing_;
+    const index_t x = high % stride_;
+    high /= stride_;
+    outputIndex = (high * stride_ + stride_ - 1 - x) * trailing_ + low;
+
+    return outputIndex;
+  }
+  template<typename DType>
+  MSHADOW_XINLINE  static void Map(index_t index, const DType *src, DType *dst,
+                                   const index_t  stride_,
+                                   const index_t  trailing_) {
+    index_t new_idx = ReverseIndex(index, stride_, trailing_);
+    dst[new_idx] = src[index];
+  }
+};
+
+template<typename xpu>
+void NumpyRot90ComputeFlipTransposeIml(const OpContext& ctx,
+                                       const std::vector<TBlob>& inputs,
+                                       const std::vector<OpReqType>& req,
+                                       const std::vector<TBlob>& outputs,
+                                       const mxnet::TShape axes_list,
+                                       const index_t axis) {
+  using namespace mshadow;
+  using namespace mxnet_op;
+
+  const mxnet::TShape& ishape = inputs[0].shape_;
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+
+  index_t stride_;
+  index_t trailing_;
+
+  stride_ = ishape[axis];
+  trailing_ = 1;
+  for (int i2 = axis + 1; i2 < ishape.ndim(); ++i2) {
+    trailing_ *= ishape[i2];
+  }
+
+  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, {
+    index_t workspace_size = inputs[0].Size() * sizeof(DType);
+    Tensor<xpu, 1, char> workspace =
+        ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(workspace_size), s);
+    DType* data_ptr = reinterpret_cast<DType*>(workspace.dptr_);
+    TBlob mid_data = TBlob(data_ptr, inputs[0].shape_, xpu::kDevMask);
+    Kernel<rot90Transreverse, xpu>::Launch(s, inputs[0].Size(), inputs[0].dptr<DType>(),
+                                           mid_data.dptr<DType>(),
+                                           stride_, trailing_);
+    mxnet::op::TransposeImpl<xpu>(ctx.run_ctx, mid_data, outputs[0], axes_list);
+  });
+}
+
+
+template<typename xpu>
+void NumpyRot90ComputeTransposeFlipIml(const OpContext& ctx,
+                                       const std::vector<TBlob>& inputs,
+                                       const std::vector<OpReqType>& req,
+                                       const std::vector<TBlob>& outputs,
+                                       const mxnet::TShape axes_list,
+                                       const index_t axis) {
+  using namespace mshadow;
+  using namespace mxnet_op;
+
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+
+  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, {
+    index_t workspace_size = inputs[0].Size() * sizeof(DType);
+    Tensor<xpu, 1, char> workspace =
+        ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(workspace_size), s);
+    DType* data_ptr = reinterpret_cast<DType*>(workspace.dptr_);
+    mxnet::TShape mid_shape(outputs[0].shape_);
+    TBlob mid_data = TBlob(data_ptr, mid_shape, xpu::kDevMask);
+    mxnet::op::TransposeImpl<xpu>(ctx.run_ctx, inputs[0], mid_data, axes_list);
+
+    index_t stride_;
+    index_t trailing_;
+    stride_ = mid_shape[axis];
+    trailing_ = 1;
+    for (int i2 = axis + 1; i2 < mid_shape.ndim(); ++i2) {
+      trailing_ *= mid_shape[i2];
+    }
+    Kernel<rot90Transreverse, xpu>::Launch(s, mid_data.Size(), mid_data.dptr<DType>(),
+                                           outputs[0].dptr<DType>(),
+                                           stride_, trailing_);
+  });
+}
+
+template<int req>
+struct rot90 {
+  template<typename DType>
+  MSHADOW_XINLINE  static void Map(index_t i, const DType *in_data, DType *out_data) {
+    KERNEL_ASSIGN(out_data[i], req, in_data[i]);
+  }
+};
+
+template<typename xpu>
+void NumpyRot90Compute(const nnvm::NodeAttrs& attrs,
+                       const OpContext& ctx,
+                       const std::vector<TBlob>& inputs,
+                       const std::vector<OpReqType>& req,
+                       const std::vector<TBlob>& outputs) {
+  using namespace mshadow;
+  using namespace mxnet_op;
+  const NumpyRot90Param& param = nnvm::get<NumpyRot90Param>(attrs.parsed);
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(inputs[0].type_flag_, outputs[0].type_flag_);
+  if (outputs[0].Size() == 0) return;
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+
+  int real_k(param.k);
+  real_k = real_k % 4;
+  if (real_k < 0) {
+    real_k += 4;
+  }
+
+  // axis has value
+  mxnet::TShape real_axes(param.axes.value());
+  for (index_t i = 0; i < real_axes.ndim(); i++) {
+    if (real_axes[i] < 0) {
+      real_axes[i] += inputs[0].shape_.ndim();
+    }
+  }
+  if (real_k == 0) {
+    MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, {
+      MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+        Kernel<rot90<req_type>, xpu>::Launch(s, inputs[0].Size(), inputs[0].dptr<DType>(),
+                                             outputs[0].dptr<DType>());
+      });
+    });
+  } else if (real_k == 2) {
+    NumpyRot90ComputeFlipIml<xpu>(ctx, inputs, req, outputs, real_axes[0], real_axes[1]);
+  } else if (real_k == 1) {
+    mxnet::TShape axes_list(inputs[0].shape_.ndim(), -1);
+    for (int i = 0; i < inputs[0].shape_.ndim(); ++i) {
+      axes_list[i] = i;
+    }
+    axes_list[real_axes[0]] += axes_list[real_axes[1]];
+    axes_list[real_axes[1]] = axes_list[real_axes[0]] - axes_list[real_axes[1]];
+    axes_list[real_axes[0]] -= axes_list[real_axes[1]];
+    NumpyRot90ComputeFlipTransposeIml<xpu>(ctx, inputs, req, outputs, axes_list, real_axes[1]);
+  } else if (real_k == 3) {
+    mxnet::TShape axes_list(inputs[0].shape_.ndim(), -1);
+    for (int i = 0; i < inputs[0].shape_.ndim(); ++i) {
+      axes_list[i] = i;
+    }
+    axes_list[real_axes[0]] += axes_list[real_axes[1]];
+    axes_list[real_axes[1]] = axes_list[real_axes[0]] - axes_list[real_axes[1]];
+    axes_list[real_axes[0]] -= axes_list[real_axes[1]];
+    NumpyRot90ComputeTransposeFlipIml<xpu>(ctx, inputs, req, outputs, axes_list, real_axes[1]);
+  }
+}
+
+template<typename xpu>
+inline void HSplitOpForward(const nnvm::NodeAttrs &attrs,
+                            const OpContext &ctx,
+                            const std::vector<TBlob> &inputs,
+                            const std::vector<OpReqType> &req,
+                            const std::vector<TBlob> &outputs) {
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  using namespace mxnet_op;
+  const SplitParam &param = nnvm::get<SplitParam>(attrs.parsed);
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), param.sections > 0 ? param.sections : param.indices.ndim());
+  const TBlob &input_data = inputs[split_enum::kData];
+  int real_axis;
+  if (input_data.ndim() > 1) {
+    real_axis = 1;
+  } else {
+    real_axis = 0;
+  }
+  SplitOpForwardImpl<xpu>(attrs, ctx, inputs, req, outputs, real_axis);
+}
+
+template<typename xpu>
+inline void HSplitOpBackward(const nnvm::NodeAttrs &attrs,
+                             const OpContext &ctx,
+                             const std::vector<TBlob> &inputs,
+                             const std::vector<OpReqType> &req,
+                             const std::vector<TBlob> &outputs) {
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  using namespace mxnet_op;
+  const SplitParam &param = nnvm::get<SplitParam>(attrs.parsed);
+  CHECK_EQ(inputs.size(), (param.sections > 0) ? param.sections : param.indices.ndim())
+    << "out grad vector size mush match the output size";
+  CHECK_EQ(outputs.size(), 1U);
+  int real_axis;
+  if (outputs[split_enum::kData].ndim() > 1) {
+    real_axis = 1;
+  } else {
+    real_axis = 0;
+  }
+  SplitOpBackwardImpl<xpu>(attrs, ctx, inputs, req, outputs, real_axis);
+}
+
 }  // namespace op
 }  // namespace mxnet
 
diff --git a/src/operator/numpy/np_matrix_op.cc b/src/operator/numpy/np_matrix_op.cc
index 83f0b1aae9b0..7bcd6ad27b52 100644
--- a/src/operator/numpy/np_matrix_op.cc
+++ b/src/operator/numpy/np_matrix_op.cc
@@ -32,6 +32,8 @@ namespace op {
 
 DMLC_REGISTER_PARAMETER(NumpyTransposeParam);
 DMLC_REGISTER_PARAMETER(NumpyRollParam);
+DMLC_REGISTER_PARAMETER(NumpyMoveaxisParam);
+DMLC_REGISTER_PARAMETER(NumpyRot90Param);
 
 bool NumpyTransposeShape(const nnvm::NodeAttrs& attrs,
                          mxnet::ShapeVector *in_attrs,
@@ -292,6 +294,67 @@ bool ConcatShape(const nnvm::NodeAttrs& attrs,
                  mxnet::ShapeVector *in_shape,
                  mxnet::ShapeVector *out_shape);
 
+bool DStackShape(const nnvm::NodeAttrs& attrs,
+                 mxnet::ShapeVector *in_shape,
+                 mxnet::ShapeVector *out_shape) {
+  using namespace mshadow;
+  ConcatParam param_ = nnvm::get<ConcatParam>(attrs.parsed);
+  CHECK_EQ(in_shape->size(), static_cast<size_t>(param_.num_args));
+  mxnet::TShape dshape;
+  dim_t size = 0;
+  bool has_unknown_dim_size = false;
+  int axis = 2;
+  param_.dim = axis;
+  for (int i = 0; i < param_.num_args; ++i) {
+    if ((*in_shape)[i].ndim() == 0) {
+      (*in_shape)[i] = mxnet::TShape(3, 1);
+    } else if ((*in_shape)[i].ndim() == 1) {
+      mxnet::TShape t = mxnet::TShape(3, 1);
+      t[1] = (*in_shape)[i][0];
+      (*in_shape)[i] = t;
+    } else if ((*in_shape)[i].ndim() == 2) {
+      mxnet::TShape t = mxnet::TShape(3, 1);
+      t[0] = (*in_shape)[i][0];
+      t[1] = (*in_shape)[i][1];
+      (*in_shape)[i] = t;
+    }
+    mxnet::TShape &tmp = (*in_shape)[i];
+    if (tmp.ndim() > 0) {
+      CheckAxis(axis, tmp.ndim());
+      if (!mxnet::dim_size_is_known(tmp, axis)) {
+        has_unknown_dim_size = true;
+      } else {
+        size += tmp[axis];
+      }
+      tmp[axis] = -1;
+      shape_assign(&dshape, tmp);
+    }
+  }
+
+  mxnet::TShape tmp = (*out_shape)[0];
+  if (tmp.ndim() > 0) {
+    axis = CheckAxis(param_.dim, tmp.ndim());
+    tmp[axis] = -1;
+    shape_assign(&dshape, tmp);
+  }
+
+  if (dshape.ndim() == -1) return false;
+  CHECK_NE(dshape.ndim(), 0) << "zero-dimensional arrays cannot be concatenated";
+
+  for (int i = 0; i < param_.num_args; ++i) {
+    CHECK(shape_assign(&(*in_shape)[i], dshape))
+      << "Incompatible input shape: expected " << dshape << ", got " << (*in_shape)[i];
+  }
+
+  if (!has_unknown_dim_size) {
+    dshape[axis] = size;
+  }
+  CHECK(shape_assign(&(*out_shape)[0], dshape))
+    << "Incompatible output shape: expected " << dshape << ", got " << (*out_shape)[0];
+
+  return shape_is_known(dshape);
+}
+
 bool ConcatType(const nnvm::NodeAttrs& attrs,
                 std::vector<int> *in_type,
                 std::vector<int> *out_type);
@@ -306,7 +369,6 @@ struct NumpyConcatGrad {
   }
 };
 
-
 NNVM_REGISTER_OP(_npi_concatenate)
 .describe(R"code(Join a sequence of arrays along an existing axis.)code" ADD_FILELINE)
 .set_num_inputs([](const NodeAttrs& attrs) {
@@ -527,6 +589,44 @@ NNVM_REGISTER_OP(_backward_np_vstack)
 .set_attr<nnvm::TIsBackward>("TIsBackward", true)
 .set_attr<FCompute>("FCompute<cpu>", NumpyVstackBackward<cpu>);
 
+NNVM_REGISTER_OP(_npi_dstack)
+.describe(R"code(Stack tensors in sequence depthwise (in third dimension))code" ADD_FILELINE)
+.set_num_inputs([](const NodeAttrs& attrs) {
+  const ConcatParam& params = nnvm::get<ConcatParam>(attrs.parsed);
+  return params.num_args;
+})
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<ConcatParam>)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    const ConcatParam& params = nnvm::get<ConcatParam>(attrs.parsed);
+    std::vector<std::string> ret;
+    for (int i = 0; i < params.num_args; ++i) {
+      ret.push_back(std::string("data") + std::to_string(i));
+    }
+    return ret;
+})
+.set_attr<nnvm::FListOutputNames>("FListOutputNames",
+  [](const NodeAttrs& attrs) {
+    return std::vector<std::string>{"out"};
+})
+.set_attr<std::string>("key_var_num_args", "num_args")
+.set_attr<nnvm::FInferType>("FInferType", ConcatType)
+.set_attr<mxnet::FInferShape>("FInferShape", DStackShape)
+.set_attr<FCompute>("FCompute<cpu>", DStackCompute<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", NumpyConcatGrad{"_backward_np_dstack"})
+.add_argument("data", "NDArray-or-Symbol[]", "List of arrays to concatenate")
+.add_arguments(ConcatParam::__FIELDS__());
+
+NNVM_REGISTER_OP(_backward_np_dstack)
+.set_num_outputs([](const NodeAttrs& attrs) {
+  const ConcatParam& params = nnvm::get<ConcatParam>(attrs.parsed);
+  return params.num_args;
+})
+.set_attr_parser(ParamParser<ConcatParam>)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<FCompute>("FCompute<cpu>", DStackGradCompute<cpu>);
+
 inline bool NumpyRollShape(const nnvm::NodeAttrs& attrs,
                            mxnet::ShapeVector *in_attrs,
                            mxnet::ShapeVector *out_attrs) {
@@ -649,5 +749,171 @@ NNVM_REGISTER_OP(_backward_npi_flip)
 })
 .set_attr<FCompute>("FCompute<cpu>", NumpyFlipForward<cpu>);
 
+bool NumpyMoveaxisShape(const nnvm::NodeAttrs& attrs,
+                        mxnet::ShapeVector *in_attrs,
+                        mxnet::ShapeVector *out_attrs) {
+  const NumpyMoveaxisParam& param = nnvm::get<NumpyMoveaxisParam>(attrs.parsed);
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  mxnet::TShape& shp = (*in_attrs)[0];
+  CHECK_LE(shp.ndim(), 6) << "Transpose support at most 6 dimensions";
+  CHECK_EQ(param.source.ndim(), param.destination.ndim())
+    << "source and destination not equal.";
+  mxnet::TShape ret(shp.ndim(), -1);
+  mxnet::TShape axes;
+  axes = NumpyMoveaxisShapeImpl(attrs, shp.ndim());
+  for (int i = 0; i < shp.ndim(); ++i) {
+    CHECK(axes[i] < static_cast<int64_t>(shp.ndim()));
+    ret[i] = shp[axes[i]];
+  }
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, ret);
+  return shape_is_known(ret);
+}
+
+NNVM_REGISTER_OP(_np_moveaxis)
+.describe(R"code(Move axes of an array to new positions.
+Other axes remain in their original order.
+)code" ADD_FILELINE)
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<NumpyMoveaxisParam>)
+.set_attr<mxnet::FInferShape>("FInferShape", NumpyMoveaxisShape)
+.set_attr<nnvm::FInferType>("FInferType", ElemwiseType<1, 1>)
+.set_attr<nnvm::FGradient>("FGradient",
+  [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
+     const NumpyMoveaxisParam& param = nnvm::get<NumpyMoveaxisParam>(n->attrs.parsed);
+     std::ostringstream os1;
+     os1 << param.source;
+     std::ostringstream os2;
+     os2 << param.destination;
+     return MakeNonlossGradNode("_np_moveaxis", n, ograds, {},
+                                {{"source", os2.str()}, {"destination", os1.str()}});
+})
+.set_attr<FCompute>("FCompute<cpu>", NumpyMoveaxisCompute<cpu>)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) { return std::vector<std::string>{"a"};
+})
+.add_argument("a", "NDArray-or-Symbol", "Source input")
+.add_arguments(NumpyMoveaxisParam::__FIELDS__());
+
+inline bool NumpyRot90Shape(const nnvm::NodeAttrs& attrs,
+                            mxnet::ShapeVector *in_attrs,
+                            mxnet::ShapeVector *out_attrs) {
+  using namespace mshadow;
+  const NumpyRot90Param& param = nnvm::get<NumpyRot90Param>(attrs.parsed);
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  mxnet::TShape& shp = (*in_attrs)[0];
+  if (!param.axes.has_value() || (param.axes.has_value() && param.axes.value().ndim() != 2)) {
+    LOG(FATAL) << "The length of axes must be 2.";
+  }
+  int real_k(param.k);
+  real_k = real_k % 4;
+  if (real_k < 0) {
+    real_k += 4;
+  }
+
+  mxnet::TShape res(shp);
+  mxnet::TShape real_axes(param.axes.value());
+  for (index_t i = 0; i < real_axes.ndim(); i++) {
+    if (real_axes[i] < 0) {
+      real_axes[i] += shp.ndim();
+    }
+  }
+
+  CHECK_NE(real_axes[0], real_axes[1])
+      << "axes have duplicates "
+      << real_axes;
+  if (real_axes[0] > shp.ndim() || real_axes[1] > shp.ndim() ||
+      real_axes[0] < 0 || real_axes[1] < 0) {
+    LOG(FATAL) << "Axes out of range for array of ndim";
+  }
+
+  if (real_k % 2 == 0) {
+    SHAPE_ASSIGN_CHECK(*out_attrs, 0, res);
+    return shape_is_known(res);
+  }
+
+  res[real_axes[0]] += res[real_axes[1]];
+  res[real_axes[1]] = res[real_axes[0]] - res[real_axes[1]];
+  res[real_axes[0]] -= res[real_axes[1]];
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, res);
+  return shape_is_known(res);
+}
+
+NNVM_REGISTER_OP(_npi_rot90)
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<NumpyRot90Param>)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+     return std::vector<std::string>{"data"};
+})
+.set_attr<mxnet::FInferShape>("FInferShape", NumpyRot90Shape)
+.set_attr<nnvm::FInferType>("FInferType", ElemwiseType<1, 1>)
+.set_attr<mxnet::FCompute>("FCompute<cpu>", NumpyRot90Compute<cpu>)
+.set_attr<nnvm::FGradient>("FGradient",
+  [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
+     const NumpyRot90Param& param = nnvm::get<NumpyRot90Param>(n->attrs.parsed);
+     std::ostringstream os1;
+     os1 << param.k;
+     std::ostringstream os2;
+     os2 << param.axes;
+     return MakeNonlossGradNode("_npi_rot90", n, ograds, {},
+                                {{"k", os1.str()}, {"axes", os2.str()}});
+})
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& n) {
+     return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+})
+.add_argument("data", "NDArray-or-Symbol", "Input ndarray")
+.add_arguments(NumpyRot90Param::__FIELDS__());
+
+inline bool HSplitOpShape(const nnvm::NodeAttrs& attrs,
+                          mxnet::ShapeVector* in_attrs,
+                          mxnet::ShapeVector* out_attrs) {
+  using namespace mshadow;
+  CHECK_EQ(in_attrs->size(), 1U);
+  mxnet::TShape dshape = in_attrs->at(split_enum::kData);
+  if (!mxnet::ndim_is_known(dshape)) return false;
+  int real_axis;
+  if (dshape.ndim() > 1) {
+    real_axis = 1;
+  } else {
+    real_axis = 0;
+  }
+  return SplitOpShapeImpl(attrs, in_attrs, out_attrs, real_axis);
+}
+
+NNVM_REGISTER_OP(_npi_hsplit)
+.set_attr_parser(ParamParser<SplitParam>)
+.set_num_inputs(1)
+.set_num_outputs(SplitNumOutputs)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+     return std::vector<std::string>{"data"};
+})
+.set_attr<mxnet::FInferShape>("FInferShape", HSplitOpShape)
+.set_attr<nnvm::FInferType>("FInferType", SplitOpType)
+.set_attr<FCompute>("FCompute<cpu>", HSplitOpForward<cpu>)
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& n) {
+     return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+})
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_npi_hsplit_backward"})
+.add_argument("data", "NDArray-or-Symbol", "The input")
+.add_arguments(SplitParam::__FIELDS__());
+
+NNVM_REGISTER_OP(_npi_hsplit_backward)
+.set_attr_parser(ParamParser<SplitParam>)
+.set_num_inputs(SplitNumOutputs)
+.set_num_outputs(1)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& n) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+})
+.set_attr<FCompute>("FCompute<cpu>", HSplitOpBackward<cpu>);
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_matrix_op.cu b/src/operator/numpy/np_matrix_op.cu
index caab4108b40e..8c8301bb3bbf 100644
--- a/src/operator/numpy/np_matrix_op.cu
+++ b/src/operator/numpy/np_matrix_op.cu
@@ -53,6 +53,12 @@ NNVM_REGISTER_OP(_npi_vstack)
 NNVM_REGISTER_OP(_backward_np_vstack)
 .set_attr<FCompute>("FCompute<gpu>", NumpyVstackBackward<gpu>);
 
+NNVM_REGISTER_OP(_npi_dstack)
+.set_attr<FCompute>("FCompute<gpu>", DStackCompute<gpu>);
+
+NNVM_REGISTER_OP(_backward_np_dstack)
+.set_attr<FCompute>("FCompute<gpu>", DStackGradCompute<gpu>);
+
 NNVM_REGISTER_OP(_np_roll)
 .set_attr<FCompute>("FCompute<gpu>", NumpyRollCompute<gpu>);
 
@@ -90,5 +96,18 @@ NNVM_REGISTER_OP(_npi_flip)
 
 NNVM_REGISTER_OP(_backward_npi_flip)
 .set_attr<FCompute>("FCompute<gpu>", NumpyFlipForward<gpu>);
+
+NNVM_REGISTER_OP(_np_moveaxis)
+.set_attr<FCompute>("FCompute<gpu>", NumpyMoveaxisCompute<gpu>);
+
+NNVM_REGISTER_OP(_npi_rot90)
+.set_attr<FCompute>("FCompute<gpu>", NumpyRot90Compute<gpu>);
+
+NNVM_REGISTER_OP(_npi_hsplit)
+.set_attr<FCompute>("FCompute<gpu>", HSplitOpForward<gpu>);
+
+NNVM_REGISTER_OP(_npi_hsplit_backward)
+.set_attr<FCompute>("FCompute<gpu>", HSplitOpBackward<gpu>);
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/operator_tune.cc b/src/operator/operator_tune.cc
index 398d5a714bd1..50a6b41701cf 100644
--- a/src/operator/operator_tune.cc
+++ b/src/operator/operator_tune.cc
@@ -394,6 +394,7 @@ IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::logical_or);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::logical_or);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::logical_xor);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::logical_xor);  // NOLINT()
+IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::bitwise_xor);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::smooth_l1_loss);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::smooth_l1_gradient);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::lcm);  // NOLINT()
diff --git a/src/operator/tensor/init_op.h b/src/operator/tensor/init_op.h
index 8e8896e5c014..d2107a1406e2 100644
--- a/src/operator/tensor/init_op.h
+++ b/src/operator/tensor/init_op.h
@@ -483,6 +483,29 @@ void FillComputeZerosEx(const nnvm::NodeAttrs& attrs,
   }
 }
 
+template<typename xpu>
+inline void EyeFillImpl(const TBlob& out_data,
+                        const OpContext& ctx,
+                        const std::vector<OpReqType>& req,
+                        const nnvm::dim_t num_cols,
+                        const nnvm::dim_t N,
+                        const nnvm::dim_t k) {
+  using namespace mxnet_op;
+  const nnvm::dim_t cnnz = std::max(num_cols - std::abs(k), (nnvm::dim_t)0);
+  const nnvm::dim_t rnnz = std::max(N - std::abs(k), (nnvm::dim_t)0);
+  const nnvm::dim_t nnz = k > 0 ? std::min(cnnz, N) :
+                          std::min(rnnz, num_cols);
+  mshadow::Stream<xpu> *s = ctx.get_stream<xpu>();
+  MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, {
+      MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+        Fill(s, out_data, req[0], static_cast<DType>(0));
+        if (nnz > 0) {
+          Kernel<eye_dns_fill<req_type>, xpu>::Launch(s, nnz, out_data.dptr<DType>(),
+            std::max(static_cast<nnvm::dim_t>(0), k), k, num_cols);
+        }
+      });
+  });
+}
 
 template<typename xpu>
 void EyeFill(const nnvm::NodeAttrs& attrs,
@@ -493,25 +516,10 @@ void EyeFill(const nnvm::NodeAttrs& attrs,
   CHECK_EQ(inputs.size(), 0U);
   CHECK_EQ(outputs.size(), 1U);
   CHECK_EQ(req.size(), 1U);
-  mshadow::Stream<xpu> *s = ctx.get_stream<xpu>();
   const EyeParam& param = nnvm::get<EyeParam>(attrs.parsed);
   const TBlob& out_data = outputs[0];
   const nnvm::dim_t num_cols = param.M > 0 ? param.M : param.N;
-
-  const nnvm::dim_t cnnz = std::max(num_cols - std::abs(param.k), (nnvm::dim_t)0);
-  const nnvm::dim_t rnnz = std::max(param.N - std::abs(param.k), (nnvm::dim_t)0);
-  const nnvm::dim_t nnz = param.k > 0 ? std::min(cnnz, param.N) :
-                                        std::min(rnnz, num_cols);
-  using namespace mxnet_op;
-  MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, {
-    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
-      Fill(s, out_data, req[0], static_cast<DType>(0));
-      if (nnz > 0) {
-        Kernel<eye_dns_fill<req_type>, xpu>::Launch(s, nnz, out_data.dptr<DType>(),
-          std::max(static_cast<nnvm::dim_t>(0), param.k), param.k, num_cols);
-      }
-    });
-  });
+  EyeFillImpl<xpu>(out_data, ctx, req, num_cols, param.N, param.k);
 }
 
 
diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h
index 2c1bc2b2098f..801e4e7126b4 100644
--- a/src/operator/tensor/matrix_op-inl.h
+++ b/src/operator/tensor/matrix_op-inl.h
@@ -2715,24 +2715,14 @@ inline bool SplitOpType(const nnvm::NodeAttrs& attrs,
   return true;
 }
 
-inline bool SplitOpShape(const nnvm::NodeAttrs& attrs,
-                         mxnet::ShapeVector* in_attrs,
-                         mxnet::ShapeVector* out_attrs) {
+inline bool SplitOpShapeImpl(const nnvm::NodeAttrs& attrs,
+                             mxnet::ShapeVector* in_attrs,
+                             mxnet::ShapeVector* out_attrs,
+                             const int real_axis) {
   using namespace mshadow;
   const SplitParam& param = nnvm::get<SplitParam>(attrs.parsed);
-  CHECK_EQ(in_attrs->size(), 1U);
   mxnet::TShape dshape = in_attrs->at(split_enum::kData);
   mxnet::TShape ishape = in_attrs->at(split_enum::kData);
-  if (!mxnet::ndim_is_known(dshape)) return false;
-  if (param.axis >= 0) {
-    CHECK_LT(param.axis, dshape.ndim());
-  } else {
-    CHECK_LT(param.axis + dshape.ndim(), dshape.ndim());
-  }
-  int real_axis = param.axis;
-  if (real_axis < 0) {
-    real_axis += dshape.ndim();
-  }
   const mxnet::TShape indices =
     (param.sections > 0) ? GetSplitIndices(ishape, real_axis, param.sections) : param.indices;
   int num_outputs = (param.sections > 0) ? indices.ndim() - 1 : indices.ndim();
@@ -2749,7 +2739,7 @@ inline bool SplitOpShape(const nnvm::NodeAttrs& attrs,
     if (ishape[real_axis] == 0U) {
       end = start;
     } else {
-      CHECK(start < end)
+      CHECK(start <= end)
         << "start " << start << " is not less than end " << end << "for subarray " << i;
       CHECK(end <= ishape[real_axis])
         << "end " << end << " is no less than the size of the axis " << ishape[real_axis];
@@ -2785,6 +2775,26 @@ inline bool SplitOpShape(const nnvm::NodeAttrs& attrs,
   return true;
 }
 
+inline bool SplitOpShape(const nnvm::NodeAttrs& attrs,
+                         mxnet::ShapeVector* in_attrs,
+                         mxnet::ShapeVector* out_attrs) {
+  using namespace mshadow;
+  const SplitParam& param = nnvm::get<SplitParam>(attrs.parsed);
+  CHECK_EQ(in_attrs->size(), 1U);
+  mxnet::TShape dshape = in_attrs->at(split_enum::kData);
+  if (!mxnet::ndim_is_known(dshape)) return false;
+  if (param.axis >= 0) {
+    CHECK_LT(param.axis, dshape.ndim());
+  } else {
+    CHECK_LT(param.axis + dshape.ndim(), dshape.ndim());
+  }
+  int real_axis = param.axis;
+  if (real_axis < 0) {
+    real_axis += dshape.ndim();
+  }
+  return SplitOpShapeImpl(attrs, in_attrs, out_attrs, real_axis);
+}
+
 struct SplitKernel {
   /*!
    * \brief Map function for forward split_v2 operator
@@ -2850,24 +2860,19 @@ struct ConcatenateKernel {
 };
 
 template<typename xpu>
-inline void SplitOpForward(const nnvm::NodeAttrs& attrs,
-                           const OpContext& ctx,
-                           const std::vector<TBlob>& inputs,
-                           const std::vector<OpReqType>& req,
-                           const std::vector<TBlob>& outputs) {
+inline void SplitOpForwardImpl(const nnvm::NodeAttrs& attrs,
+                               const OpContext& ctx,
+                               const std::vector<TBlob>& inputs,
+                               const std::vector<OpReqType>& req,
+                               const std::vector<TBlob>& outputs,
+                               const int real_axis) {
   using namespace mshadow;
   using namespace mshadow::expr;
   using namespace mxnet_op;
   const SplitParam& param = nnvm::get<SplitParam>(attrs.parsed);
-  CHECK_EQ(inputs.size(), 1U);
-  CHECK_EQ(outputs.size(), (param.sections > 0) ? param.sections : param.indices.ndim());
   Stream<xpu> *s = ctx.get_stream<xpu>();
   const TBlob& input_data = inputs[split_enum::kData];
   size_t leading = 1, trailing = 1;
-  int real_axis = param.axis;
-  if (real_axis < 0) {
-    real_axis += input_data.ndim();
-  }
   CHECK_LT(real_axis, input_data.ndim());
   size_t mid = input_data.shape_[real_axis];
   for (int i = 0; i < real_axis; ++i) {
@@ -2913,25 +2918,39 @@ inline void SplitOpForward(const nnvm::NodeAttrs& attrs,
 }
 
 template<typename xpu>
-inline void SplitOpBackward(const nnvm::NodeAttrs& attrs,
-                            const OpContext& ctx,
-                            const std::vector<TBlob>& inputs,
-                            const std::vector<OpReqType>& req,
-                            const std::vector<TBlob>& outputs) {
+inline void SplitOpForward(const nnvm::NodeAttrs& attrs,
+                           const OpContext& ctx,
+                           const std::vector<TBlob>& inputs,
+                           const std::vector<OpReqType>& req,
+                           const std::vector<TBlob>& outputs) {
   using namespace mshadow;
   using namespace mshadow::expr;
   using namespace mxnet_op;
   const SplitParam& param = nnvm::get<SplitParam>(attrs.parsed);
-  CHECK_EQ(inputs.size(), (param.sections > 0) ? param.sections : param.indices.ndim())
-    << "out grad vector size mush match the output size";
-  CHECK_EQ(outputs.size(), 1U);
-  Stream<xpu> *s = ctx.get_stream<xpu>();
-  TBlob input_grad = outputs[split_enum::kData];
-  size_t leading = 1, trailing = 1;
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), (param.sections > 0) ? param.sections : param.indices.ndim());
+  const TBlob& input_data = inputs[split_enum::kData];
   int real_axis = param.axis;
   if (real_axis < 0) {
-      real_axis += input_grad.ndim();
+    real_axis += input_data.ndim();
   }
+  SplitOpForwardImpl<xpu>(attrs, ctx, inputs, req, outputs, real_axis);
+}
+
+template<typename xpu>
+inline void SplitOpBackwardImpl(const nnvm::NodeAttrs& attrs,
+                                const OpContext& ctx,
+                                const std::vector<TBlob>& inputs,
+                                const std::vector<OpReqType>& req,
+                                const std::vector<TBlob>& outputs,
+                                const int real_axis) {
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  using namespace mxnet_op;
+  const SplitParam& param = nnvm::get<SplitParam>(attrs.parsed);
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  TBlob input_grad = outputs[split_enum::kData];
+  size_t leading = 1, trailing = 1;
   CHECK_LT(real_axis, input_grad.ndim());
   size_t mid = input_grad.shape_[real_axis];
   for (int i = 0; i < real_axis; ++i) {
@@ -2976,6 +2995,26 @@ inline void SplitOpBackward(const nnvm::NodeAttrs& attrs,
   });
 }
 
+template<typename xpu>
+inline void SplitOpBackward(const nnvm::NodeAttrs& attrs,
+                            const OpContext& ctx,
+                            const std::vector<TBlob>& inputs,
+                            const std::vector<OpReqType>& req,
+                            const std::vector<TBlob>& outputs) {
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  using namespace mxnet_op;
+  const SplitParam& param = nnvm::get<SplitParam>(attrs.parsed);
+  CHECK_EQ(inputs.size(), (param.sections > 0) ? param.sections : param.indices.ndim())
+    << "out grad vector size mush match the output size";
+  CHECK_EQ(outputs.size(), 1U);
+  int real_axis = param.axis;
+  if (real_axis < 0) {
+      real_axis += outputs[split_enum::kData].ndim();
+  }
+  SplitOpBackwardImpl<xpu>(attrs, ctx, inputs, req, outputs, real_axis);
+}
+
 inline uint32_t SplitNumOutputs(const NodeAttrs& attrs) {
   const SplitParam& param = nnvm::get<SplitParam>(attrs.parsed);
   return (param.sections > 0) ? param.sections : param.indices.ndim();
diff --git a/tests/python-pytest/onnx/mxnet_export_test.py b/tests/python-pytest/onnx/mxnet_export_test.py
index 6c81198a8bca..90e92cccee06 100644
--- a/tests/python-pytest/onnx/mxnet_export_test.py
+++ b/tests/python-pytest/onnx/mxnet_export_test.py
@@ -74,7 +74,7 @@ def _check_onnx_export(net, group_outputs=False, shape_type=tuple, extra_params=
         # Confirm network outputs are the same
         imported_net_output = _force_list(imported_net(data))
         for out, imp_out in zip(output, imported_net_output):
-            mx.test_utils.assert_almost_equal(out.asnumpy(), imp_out.asnumpy())
+            mx.test_utils.assert_almost_equal(out, imp_out)
 
 
 class TestExport(unittest.TestCase):
diff --git a/tests/python/gpu/test_gluon_gpu.py b/tests/python/gpu/test_gluon_gpu.py
index fc650294a538..48fff7810c2e 100644
--- a/tests/python/gpu/test_gluon_gpu.py
+++ b/tests/python/gpu/test_gluon_gpu.py
@@ -20,26 +20,17 @@
 import os
 import tempfile
 import time
-import multiprocessing as mp
-import unittest
-import random
 import mxnet as mx
+import multiprocessing as mp
+from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal, rand_ndarray
 import mxnet.ndarray as nd
 import numpy as np
-import unittest
 import math
-from nose.tools import assert_raises
-from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal
-from mxnet.base import MXNetError
 from mxnet import autograd
-from numpy.testing import assert_allclose
-from mxnet.test_utils import rand_ndarray
-
 
 curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
 sys.path.insert(0, os.path.join(curr_path, '../unittest'))
-from common import setup_module, with_seed, teardown, assert_raises_cudnn_not_satisfied
-from common import run_in_spawned_process
+from common import setup_module, with_seed, teardown, assert_raises_cudnn_not_satisfied, run_in_spawned_process
 from test_gluon import *
 from test_loss import *
 from test_gluon_rnn import *
@@ -60,9 +51,9 @@ def check_rnn_layer(layer):
         co, cs = layer(x, states)
 
     # atol of 1e-6 required, as exposed by seed 2124685726
-    assert_almost_equal(go.asnumpy(), co.asnumpy(), rtol=1e-2, atol=1e-6)
+    assert_almost_equal(go, co, rtol=1e-2, atol=1e-6)
     for g, c in zip(gs, cs):
-        assert_almost_equal(g.asnumpy(), c.asnumpy(), rtol=1e-2, atol=1e-6)
+        assert_almost_equal(g, c, rtol=1e-2, atol=1e-6)
 
 
 @with_seed()
@@ -79,9 +70,9 @@ def check_rnn_layer_w_rand_inputs(layer):
         states = layer.begin_state(16)
         co, cs = layer(x, states)
 
-    assert_almost_equal(go.asnumpy(), co.asnumpy(), rtol=1e-2, atol=1e-6)
+    assert_almost_equal(go, co, rtol=1e-2, atol=1e-6)
     for g, c in zip(gs, cs):
-        assert_almost_equal(g.asnumpy(), c.asnumpy(), rtol=1e-2, atol=1e-6)
+        assert_almost_equal(g, c, rtol=1e-2, atol=1e-6)
 
 
 @with_seed()
@@ -117,21 +108,19 @@ def test_lstmp():
         layer_output = lstm_layer(lstm_input.copy())
         cell_output = lstm_cell.unroll(seq_len, lstm_input.copy(), layout='TNC',
                                        merge_outputs=True)[0]
-    assert_almost_equal(layer_output.asnumpy(),
-                        cell_output.asnumpy(), rtol=rtol, atol=atol)
+
+    assert_almost_equal(layer_output, cell_output, rtol=rtol, atol=atol)
     layer_output.backward()
     cell_output.backward()
     for k, v in weights.items():
         layer_grad = layer_params['lstm0_l0_' + k].grad()
         cell_grad = cell_params['lstm0_l0_' + k].grad()
         print('checking gradient for {}'.format('lstm0_l0_' + k))
-        assert_almost_equal(layer_grad.asnumpy(), cell_grad.asnumpy(),
-                            rtol=rtol, atol=atol)
+        assert_almost_equal(layer_grad, cell_grad, rtol=rtol, atol=atol)
     check_rnn_layer_forward(gluon.rnn.LSTM(
         10, 2, projection_size=5), mx.nd.ones((8, 3, 20)), ctx=ctx)
     check_rnn_layer_forward(gluon.rnn.LSTM(10, 2, projection_size=5, bidirectional=True), mx.nd.ones(
         (8, 3, 20)), [mx.nd.ones((4, 3, 5)), mx.nd.ones((4, 3, 10))], ctx=ctx)
-
     check_rnn_layer_forward(gluon.rnn.LSTM(10, 2, dropout=0.5, projection_size=5), mx.nd.ones((8, 3, 20)),
                             run_only=True, ctx=ctx)
     check_rnn_layer_forward(gluon.rnn.LSTM(10, 2, bidirectional=True, dropout=0.5, projection_size=5),
@@ -223,7 +212,8 @@ def forward(self, inpt):
             'r0', 'r0l0')].set_data(weights[k])
 
     data = mx.random.uniform(shape=(11, 10, in_size))
-    assert_allclose(net(data).asnumpy(), ref_net(data).asnumpy())
+    mx.test_utils.assert_allclose(net(data), ref_net(data), rtol=1e-6)
+
 
 
 def check_layer_bidirectional_varseqlen(size, in_size):
@@ -336,8 +326,7 @@ def test_gluon_ctc_consistency():
         l_gpu = loss(gpu_data, gpu_label)
         l_gpu.backward()
 
-    assert_almost_equal(cpu_data.grad.asnumpy(),
-                        gpu_data.grad.asnumpy(), atol=1e-3, rtol=1e-3)
+    assert_almost_equal(cpu_data.grad, gpu_data.grad, atol=1e-3, rtol=1e-3)
 
 
 @with_seed()
@@ -351,9 +340,88 @@ def test_global_norm_clip_multi_device():
             assert norm == 5.0
         else:
             assert norm.asscalar() == 5.0
-        assert_almost_equal(x1.asnumpy(), np.ones((3, 3)) / 5)
-        assert_almost_equal(x2.asnumpy(), np.ones((4, 4)) / 5)
+        assert_almost_equal(x1, np.ones((3, 3)) / 5)
+        assert_almost_equal(x2, np.ones((4, 4)) / 5)
+
+
+def _check_batchnorm_result(input, num_devices=1, cuda=False):
+    from mxnet.gluon.utils import split_and_load
+    def _find_bn(module):
+        if isinstance(module, (mx.gluon.nn.BatchNorm, mx.gluon.contrib.nn.SyncBatchNorm)):
+            return module
+        elif isinstance(module.module, (mx.gluon.nn.BatchNorm, mx.gluon.contrib.nn.SyncBatchNorm)):
+            return module.module
+
+        raise RuntimeError('BN not found')
+
+    def _syncParameters(bn1, bn2, ctx):
+        ctx = input.context
+        bn2.gamma.set_data(bn1.gamma.data(ctx))
+        bn2.beta.set_data(bn1.beta.data(ctx))
+        bn2.running_mean.set_data(bn1.running_mean.data(ctx))
+        bn2.running_var.set_data(bn1.running_var.data(ctx))
+
+    input1 = input.copy()
+    input2 = input.copy()
+
+    if cuda:
+        input1 = input.as_in_context(mx.gpu(0))
+        ctx_list = [mx.gpu(i) for i in range(num_devices)]
+    else:
+        ctx_list = [mx.cpu(0) for _ in range(num_devices)]
+
+    nch = input.shape[1]
+    bn1 = mx.gluon.nn.BatchNorm(in_channels=nch)
+    bn2 = mx.gluon.contrib.nn.SyncBatchNorm(in_channels=nch, num_devices=num_devices)
 
+    bn1.initialize(ctx=ctx_list[0])
+    bn2.initialize(ctx=ctx_list)
+
+    # using the same values for gamma and beta
+    #_syncParameters(_find_bn(bn1), _find_bn(bn2), ctx_list[0])
+
+    input1.attach_grad()
+    inputs2 = split_and_load(input2, ctx_list, batch_axis=0)
+    for xi in inputs2:
+        xi.attach_grad()
+
+    with mx.autograd.record():
+        output1 = bn1(input1)
+        output2  = [bn2(xi) for xi in inputs2]
+        loss1 = (output1 ** 2).sum()
+        loss2 = [(output ** 2).sum() for output in output2]
+        mx.autograd.backward(loss1)
+        mx.autograd.backward(loss2)
+
+    output2 = mx.nd.concat(*[output.as_in_context(input.context) for output in output2], dim=0)
+    # assert forwarding
+    assert_almost_equal(input1, input2, atol=1e-3, rtol=1e-3)
+    assert_almost_equal(output1, output2, atol=1e-3, rtol=1e-3)
+    assert_almost_equal(_find_bn(bn1).running_mean.data(ctx_list[0]),
+                        _find_bn(bn2).running_mean.data(ctx_list[0]),
+                        atol=1e-3, rtol=1e-3)
+    assert_almost_equal(_find_bn(bn1).running_var.data(ctx_list[0]),
+                        _find_bn(bn2).running_var.data(ctx_list[0]),
+                        atol=1e-3, rtol=1e-3)
+    input2grad = mx.nd.concat(*[output.grad.as_in_context(input.context) for output in inputs2], dim=0)
+    assert_almost_equal(input1.grad, input2grad, atol=1e-3, rtol=1e-3)
+
+@with_seed()
+def test_sync_batchnorm():
+    def get_num_devices():
+        for i in range(100):
+            try:
+                mx.nd.zeros((1,), ctx=mx.gpu(i))
+            except:
+                return i
+    # no need to use SyncBN with 1 gpu
+    if get_num_devices() < 2:
+        return
+    ndev = 2
+    # check with unsync version
+    for i in range(10):
+        _check_batchnorm_result(mx.nd.random.uniform(shape=(4, 1, 4, 4)),
+                                num_devices=ndev, cuda=True)
 
 @with_seed()
 def test_symbol_block_fp16():
@@ -468,10 +536,7 @@ def get_net(num_ops):
 
     time_per_iteration.value = (time.time() - start) / num_iterations
 
-
-@with_seed()
-@unittest.skip('skippping temporarily, tracked by https://github.com/apache/incubator-mxnet/issues/14970')
-def test_bulking():
+def _test_bulking(test_bulking_func):
     # test case format: (max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training)
     test_cases = [(0, 0, True), (1, 1, True), (15, 15, False),
                   (15, 0, True), (0, 15, True), (15, 15, True)]
@@ -480,7 +545,8 @@ def test_bulking():
     for seg_sizes in test_cases:
         # Create shared variable to return measured time from test process
         time_per_iteration = mp.Manager().Value('d', 0.0)
-        if not run_in_spawned_process(_test_bulking_in_process,
+
+        if not run_in_spawned_process(test_bulking_func,
                                       {'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD': seg_sizes[0],
                                        'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD': seg_sizes[1],
                                        'MXNET_EXEC_BULK_EXEC_TRAIN': seg_sizes[2]},
@@ -492,8 +558,7 @@ def test_bulking():
             '\n    runtime of (fwd,bwd,enable) op seg setting ({},{},{}) =\t{:.1f} msec'.format(
                 seg_sizes[0], seg_sizes[1], seg_sizes[2], 1000.0 * times[seg_sizes])
 
-    fastest_non_bulked_time = min(
-        times[(0, 0, True)], times[(1, 1, True)], times[(15, 15, False)])
+    fastest_non_bulked_time = min(times[(0, 0, True)], times[(1, 1, True)], times[(15, 15, False)])
     slowest_half_bulked_time = max(times[(0, 15, True)], times[(15, 0, True)])
     fastest_half_bulked_time = min(times[(0, 15, True)], times[(15, 0, True)])
     fully_bulked_time = times[(15, 15, True)]
@@ -509,6 +574,11 @@ def test_bulking():
         'The fully-bulked exec time is slower than a half-bulked time by {} secs! {}' \
         .format(fully_bulked_time - fastest_half_bulked_time, times_str)
 
+@with_seed()
+@unittest.skip('skippping temporarily, tracked by https://github.com/apache/incubator-mxnet/issues/14970')
+def test_bulking_gluon_gpu():
+    _test_bulking(_test_bulking_in_process)
+
 
 if __name__ == '__main__':
     import nose
diff --git a/tests/python/gpu/test_gluon_model_zoo_gpu.py b/tests/python/gpu/test_gluon_model_zoo_gpu.py
index d4f6f31a30e7..6f559db62808 100644
--- a/tests/python/gpu/test_gluon_model_zoo_gpu.py
+++ b/tests/python/gpu/test_gluon_model_zoo_gpu.py
@@ -81,16 +81,17 @@ def test_inference():
             gpu_param = gpu_params.get(k)
             gpu_param.set_data(cpu_param.data().as_in_context(mx.gpu()))
 
+        cpu_data = mx.nd.array(data, ctx=mx.cpu())
         for i in range(5):
             # Run inference.
             with autograd.record(train_mode=False):
-                cpu_out = cpu_model(mx.nd.array(data, ctx=mx.cpu()))
+                cpu_out = cpu_model(cpu_data)
                 gpu_out = gpu_model(gpu_data)
-            out = cpu_out.asnumpy()
-            max_val = np.max(np.abs(out))
+
+            max_val = np.max(np.abs(cpu_out.asnumpy()))
             gpu_max_val = np.max(np.abs(gpu_out.asnumpy()))
             eprint(model_name + ": CPU " + str(max_val) + ", GPU " + str(gpu_max_val))
-            assert_almost_equal(out / max_val, gpu_out.asnumpy() / max_val, rtol=1e-3, atol=1e-3)
+            assert_almost_equal(cpu_out / max_val, gpu_out / gpu_max_val, rtol=1e-3, atol=1e-3)
 
 def get_nn_model(name):
     if "densenet" in name:
@@ -161,7 +162,7 @@ def test_training():
         max_val = np.max(np.abs(cpu_out.asnumpy()))
         gpu_max_val = np.max(np.abs(gpu_out.asnumpy()))
         eprint(model_name + ": CPU " + str(max_val) + ", GPU " + str(gpu_max_val))
-        assert_almost_equal(cpu_out.asnumpy() / max_val, gpu_out.asnumpy() / max_val, rtol=1e-3, atol=1e-3)
+        assert_almost_equal(cpu_out / max_val, gpu_out / max_val, rtol=1e-3, atol=1e-3)
         cpu_loss.backward()
         gpu_loss.backward()
         cpu_trainer.step(batch_size)
@@ -177,8 +178,7 @@ def test_training():
                 k = k.replace(cpu_params.prefix, '')
                 cpu_param = cpu_params.get(k)
                 gpu_param = gpu_params.get(k)
-                assert_almost_equal(cpu_param.data().asnumpy(), gpu_param.data().asnumpy(),
-                        rtol=1e-3, atol=1e-3)
+                assert_almost_equal(cpu_param.data(), gpu_param.data(), rtol=1e-3, atol=1e-3)
 
 if __name__ == '__main__':
     import nose
diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py
index 0f1cd93755c3..b79b08219221 100644
--- a/tests/python/gpu/test_operator_gpu.py
+++ b/tests/python/gpu/test_operator_gpu.py
@@ -20,15 +20,13 @@
 import os
 import time
 import multiprocessing as mp
-import unittest
 import mxnet as mx
 import numpy as np
 import unittest
 from nose.tools import assert_raises
-from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal
+from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal, assert_allclose
 from mxnet.base import MXNetError
 from mxnet import autograd
-from numpy.testing import assert_allclose
 
 curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
 sys.path.insert(0, os.path.join(curr_path, '../unittest'))
@@ -46,6 +44,7 @@
 from test_sparse_operator import *
 from test_ndarray import *
 from test_subgraph_op import *
+from test_gluon_gpu import _test_bulking
 from test_contrib_operator import test_multibox_target_op
 from test_tvm_op import *
 from test_library_loading import *
@@ -230,7 +229,7 @@ def check_fft(shape):
                     a[i,j,:,p+1] = out2[i,j+out1[0].shape[1],:,k]
                     p = p+2
 
-    assert_almost_equal(a, out1[0],rtol=1e-3, atol=1e-5)
+    assert_almost_equal(a, out1[0], rtol=1e-3, atol=1e-5)
 
     # backward
     if len(shape) == 2:
@@ -244,7 +243,7 @@ def check_fft(shape):
         for exe in exe_list:
             exe.backward([out_grad])
         a = np.fft.ifft(out_grad_complex, n=None, axis=-1, norm=None)
-        assert_almost_equal(a.real, exe.grad_arrays[0].asnumpy()/shape[1],rtol=1e-3, atol=1e-5)
+        assert_almost_equal(a.real, exe.grad_arrays[0]/shape[1],rtol=1e-3, atol=1e-5)
 
     if len(shape) == 4:
         out_grad = mx.nd.empty(out1[0].shape)
@@ -257,7 +256,7 @@ def check_fft(shape):
         for exe in exe_list:
             exe.backward([out_grad])
         a = np.fft.ifft(out_grad_complex, n=None, axis=-1, norm=None)
-        assert_almost_equal(a.real, exe.grad_arrays[0].asnumpy()/shape[3],rtol=1e-3, atol=1e-5)
+        assert_almost_equal(a.real, exe.grad_arrays[0]/shape[3],rtol=1e-3, atol=1e-5)
 
 @with_seed()
 def test_fft():
@@ -1682,7 +1681,7 @@ def check_rnn_consistency(cell1, cell2):
     mod1.forward(batch, is_train=False)
     mod2.forward(batch, is_train=False)
 
-    assert_allclose(mod1.get_outputs()[0].asnumpy(), mod2.get_outputs()[0].asnumpy(), rtol=1e-2, atol=1e-4)
+    mx.test_utils.assert_allclose(mod1.get_outputs()[0], mod2.get_outputs()[0], rtol=1e-2, atol=1e-4)
 
 @with_seed()
 @assert_raises_cudnn_not_satisfied(min_version='5.1.10')
@@ -1716,7 +1715,7 @@ def test_lstm_forget_bias():
 
     bias_name = next(x for x in args if x.endswith('f_bias'))
     expected_bias = forget_bias * np.ones(10, )
-    assert_allclose(args[bias_name].asnumpy(), expected_bias)
+    mx.test_utils.assert_allclose(args[bias_name], expected_bias)
 
 @with_seed()
 @assert_raises_cudnn_not_satisfied(min_version='5.1.10')
@@ -2008,9 +2007,9 @@ def check_rnn_layer(layer):
         co, cs = layer(x, states)
 
     # atol of 1e-6 required, as exposed by seed 2124685726
-    assert_almost_equal(go.asnumpy(), co.asnumpy(), rtol=1e-2, atol=1e-6)
+    assert_almost_equal(go, co, rtol=1e-2, atol=1e-6)
     for g, c in zip(gs, cs):
-        assert_almost_equal(g.asnumpy(), c.asnumpy(), rtol=1e-2, atol=1e-6)
+        assert_almost_equal(g, c, rtol=1e-2, atol=1e-6)
 
 def check_rnn_layer_w_rand_inputs(layer):
     layer.collect_params().initialize(ctx=[mx.cpu(0), mx.gpu(0)])
@@ -2025,9 +2024,9 @@ def check_rnn_layer_w_rand_inputs(layer):
         states = layer.begin_state(16)
         co, cs = layer(x, states)
 
-    assert_almost_equal(go.asnumpy(), co.asnumpy(), rtol=1e-2, atol=1e-6)
+    assert_almost_equal(go, co, rtol=1e-2, atol=1e-6)
     for g, c in zip(gs, cs):
-        assert_almost_equal(g.asnumpy(), c.asnumpy(), rtol=1e-2, atol=1e-6)
+        assert_almost_equal(g, c, rtol=1e-2, atol=1e-6)
 
 @with_seed()
 def test_sequence_reverse():
@@ -2092,7 +2091,7 @@ def test_cross_device_autograd():
 
         y.backward()
 
-    dx = x.grad.asnumpy()
+    dx = x.grad.copy()
     x.grad[:] = 0
 
     with mx.autograd.record():
@@ -2101,7 +2100,7 @@ def test_cross_device_autograd():
             y = mx.nd.tanh(y)
         y.backward()
 
-    assert_almost_equal(dx, x.grad.asnumpy())
+    assert_almost_equal(dx, x.grad)
 
 @with_seed()
 def test_multi_proposal_op():
@@ -2285,12 +2284,11 @@ def test_softmax_activation():
     with mx.autograd.record():
         gpu_y = mx.nd.SoftmaxActivation(data = gpu_a)
         cpu_y = mx.nd.SoftmaxActivation(data = cpu_a)
-        assert_almost_equal(cpu_y.asnumpy(), gpu_y.asnumpy(), atol = 1e-3, rtol = 1e-3)
+        assert_almost_equal(cpu_y, gpu_y, atol = 1e-3, rtol = 1e-3)
 
         gpu_y.backward()
         cpu_y.backward()
-        assert_almost_equal(cpu_a.grad.asnumpy(), gpu_a.grad.asnumpy(),
-                atol = 1e-3, rtol = 1e-3)
+        assert_almost_equal(cpu_a.grad, gpu_a.grad, atol = 1e-3, rtol = 1e-3)
 
 
 @with_seed()
@@ -2320,13 +2318,13 @@ def test_bilinear_sampler_versions():
             exe.arg_dict['data'][:] = test_data
             exe.arg_dict['grid'][:] = test_grid
             exe.forward(is_train=True)
-            assert_almost_equal(exe_list[ref_idx].outputs[0].asnumpy(), exe.outputs[0].asnumpy(), rtol=1e-3, atol=1e-5)
+            mx.test_utils.assert_almost_equal(exe_list[ref_idx].outputs[0], exe.outputs[0], rtol=1e-3, atol=1e-5)
 
         out_grad = np.random.uniform(low=-0.01, high=0.01,size=data_shape[:2] + grid_shape[2:]).astype(np.float32)
         for exe in exe_list:
             exe.backward(mx.nd.array(out_grad))
-            assert_almost_equal(exe.grad_dict['data'].asnumpy(), exe_list[ref_idx].grad_dict['data'].asnumpy(), rtol=1e-3, atol=1e-5)
-            assert_almost_equal(exe.grad_dict['grid'].asnumpy(), exe_list[ref_idx].grad_dict['grid'].asnumpy(), rtol=1e-3, atol=1e-5)
+            assert_almost_equal(exe.grad_dict['data'], exe_list[ref_idx].grad_dict['data'], rtol=1e-3, atol=1e-5)
+            assert_almost_equal(exe.grad_dict['grid'], exe_list[ref_idx].grad_dict['grid'], rtol=1e-3, atol=1e-5)
 
         data_grad = exe_list[ref_idx].grad_dict['data'].asnumpy()
         grid_grad = exe_list[ref_idx].grad_dict['grid'].asnumpy()
@@ -2345,10 +2343,10 @@ def test_bilinear_sampler_versions():
             exe.grad_dict['grid'][:] = grid_initial_grad
             exe.forward(is_train=True)
             exe.backward(mx.nd.array(out_grad))
-            assert_almost_equal(exe.grad_dict['data'].asnumpy(), exe_list[ref_idx].grad_dict['data'].asnumpy(), rtol=1e-3, atol=1e-5)
-            assert_almost_equal(exe.grad_dict['grid'].asnumpy(), exe_list[ref_idx].grad_dict['grid'].asnumpy(), rtol=1e-3, atol=1e-5)
-        assert_almost_equal(exe_list[ref_idx].grad_dict['data'].asnumpy(), data_grad + data_initial_grad, rtol=1e-3, atol=1e-5)
-        assert_almost_equal(exe_list[ref_idx].grad_dict['grid'].asnumpy(), grid_grad + grid_initial_grad, rtol=1e-3, atol=1e-5)
+            assert_almost_equal(exe.grad_dict['data'], exe_list[ref_idx].grad_dict['data'], rtol=1e-3, atol=1e-5)
+            assert_almost_equal(exe.grad_dict['grid'], exe_list[ref_idx].grad_dict['grid'], rtol=1e-3, atol=1e-5)
+        assert_almost_equal(exe_list[ref_idx].grad_dict['data'], data_grad + data_initial_grad, rtol=1e-3, atol=1e-5)
+        assert_almost_equal(exe_list[ref_idx].grad_dict['grid'], grid_grad + grid_initial_grad, rtol=1e-3, atol=1e-5)
 
         for req_dict in [{'data' : 'null', 'grid' : 'write'}, {'data' : 'write', 'grid' : 'null'}]:
             # Mixture of kWriteTo and kNullOp
@@ -2362,9 +2360,9 @@ def test_bilinear_sampler_versions():
                 exe.forward(is_train=True)
                 exe.backward(mx.nd.array(out_grad))
                 if req_dict['data'] is 'write':
-                    assert_almost_equal(exe.grad_dict['data'].asnumpy(), exe_list[ref_idx].grad_dict['data'].asnumpy(), rtol=1e-3, atol=1e-5)
+                    assert_almost_equal(exe.grad_dict['data'], exe_list[ref_idx].grad_dict['data'], rtol=1e-3, atol=1e-5)
                 if req_dict['grid'] is 'write':
-                    assert_almost_equal(exe.grad_dict['grid'].asnumpy(), exe_list[ref_idx].grad_dict['grid'].asnumpy(), rtol=1e-3, atol=1e-5)
+                    assert_almost_equal(exe.grad_dict['grid'], exe_list[ref_idx].grad_dict['grid'], rtol=1e-3, atol=1e-5)
 
 
 # isolated execution bulking test function to be invoked with different env var settings
@@ -2394,7 +2392,12 @@ def _test_bulking_in_process(seed, time_per_iteration):
         dx.wait_to_read()
     time_per_iteration.value = (time.time() - start) / num_iterations
 
+
 @with_seed()
+def test_bulking_operator_gpu():
+    _test_bulking(_test_bulking_in_process)
+
+
 @unittest.skip('skippping temporarily, tracked by https://github.com/apache/incubator-mxnet/issues/14970')
 def test_bulking():
     # test case format: (max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training)
@@ -2433,6 +2436,10 @@ def test_bulking():
             .format(fully_bulked_time - fastest_half_bulked_time, times_str)
 
 
+@with_seed()
+def test_allclose_function_gpu():
+    allclose_function([mx.cpu(), mx.gpu(0)])
+
 def test_context_num_gpus():
     # Test that num_gpus reports at least one GPU, as the test is run on a GPU host.
     assert mx.context.num_gpus() > 0
diff --git a/tests/python/mkl/test_mkldnn.py b/tests/python/mkl/test_mkldnn.py
index 496b2a65517d..e43daf12c464 100644
--- a/tests/python/mkl/test_mkldnn.py
+++ b/tests/python/mkl/test_mkldnn.py
@@ -161,8 +161,8 @@ def hybrid_forward(self, F, x, *args, **kwargs):
     with mx.autograd.record():
         out2 = net(x)
     out2.backward()
-    mx.test_utils.assert_almost_equal(dx1.asnumpy(), x.grad.asnumpy(), rtol=1e-5, atol=1e-6)
-    mx.test_utils.assert_almost_equal(out1.asnumpy(), out2.asnumpy(), rtol=1e-5, atol=1e-6)
+    assert_almost_equal(dx1, x.grad, rtol=1e-5, atol=1e-6)
+    assert_almost_equal(out1, out2, rtol=1e-5, atol=1e-6)
 
 
 @with_seed()
@@ -195,8 +195,8 @@ def hybrid_forward(self, F, x, *args, **kwargs):
     with mx.autograd.record():
         out2 = net(x)
     out2.backward()
-    mx.test_utils.assert_almost_equal(dx1.asnumpy(), x.grad.asnumpy(), rtol=1e-5, atol=1e-6)
-    mx.test_utils.assert_almost_equal(out1.asnumpy(), out2.asnumpy(), rtol=1e-5, atol=1e-6)
+    assert_almost_equal(dx1, x.grad, rtol=1e-5, atol=1e-6)
+    assert_almost_equal(out1, out2, rtol=1e-5, atol=1e-6)
 
 
 @with_seed()
@@ -229,8 +229,8 @@ def hybrid_forward(self, F, x, *args, **kwargs):
     with mx.autograd.record():
         out2 = net(x)
     out2.backward()
-    mx.test_utils.assert_almost_equal(dx1.asnumpy(), x.grad.asnumpy(), rtol=1e-5, atol=1e-6)
-    mx.test_utils.assert_almost_equal(out1.asnumpy(), out2.asnumpy(), rtol=1e-5, atol=1e-6)
+    assert_almost_equal(dx1, x.grad, rtol=1e-5, atol=1e-6)
+    assert_almost_equal(out1, out2, rtol=1e-5, atol=1e-6)
 
 
 @with_seed()
diff --git a/tests/python/unittest/test_contrib_operator.py b/tests/python/unittest/test_contrib_operator.py
index 43ba9d8b6318..c23be6c47783 100644
--- a/tests/python/unittest/test_contrib_operator.py
+++ b/tests/python/unittest/test_contrib_operator.py
@@ -23,6 +23,7 @@
 import itertools
 from numpy.testing import assert_allclose, assert_array_equal
 from mxnet.test_utils import *
+from common import with_seed
 import unittest
 
 def test_box_nms_op():
@@ -351,6 +352,63 @@ def test_box_decode_op():
     assert_allclose(Y.asnumpy(), np.array([[[-0.0562755, -0.00865743, 0.26227552, 0.42465743], \
         [0.13240421, 0.17859563, 0.93759584, 1.1174043 ]]]), atol=1e-5, rtol=1e-5)
 
+@with_seed()
+def test_op_mrcnn_mask_target():
+    if default_context().device_type != 'gpu':
+        return
+
+    num_rois = 2
+    num_classes = 4
+    mask_size = 3
+    ctx = mx.gpu(0)
+    # (B, N, 4)
+    rois = mx.nd.array([[[2.3, 4.3, 2.2, 3.3],
+                        [3.5, 5.5, 0.9, 2.4]]], ctx=ctx)
+    gt_masks = mx.nd.arange(0, 4*32*32, ctx=ctx).reshape(1, 4, 32, 32)
+
+    # (B, N)
+    matches = mx.nd.array([[2, 0]], ctx=ctx)
+    # (B, N)
+    cls_targets = mx.nd.array([[2, 1]], ctx=ctx)
+
+    mask_targets, mask_cls = mx.nd.contrib.mrcnn_mask_target(rois, gt_masks, matches, cls_targets,
+                                                             num_rois=num_rois,
+                                                             num_classes=num_classes,
+                                                             mask_size=mask_size)
+
+    # Ground truth outputs were generated with GluonCV's target generator
+    # gluoncv.model_zoo.mask_rcnn.MaskTargetGenerator(1, num_rois, num_classes, mask_size)
+    gt_mask_targets = mx.nd.array([[[[[2193.4    , 2193.7332 , 2194.0667 ],
+                                      [2204.0667 , 2204.4    , 2204.7334 ],
+                                      [2214.7334 , 2215.0667 , 2215.4    ]],
+                                     [[2193.4    , 2193.7332 , 2194.0667 ],
+                                      [2204.0667 , 2204.4    , 2204.7334 ],
+                                      [2214.7334 , 2215.0667 , 2215.4    ]],
+                                     [[2193.4    , 2193.7332 , 2194.0667 ],
+                                      [2204.0667 , 2204.4    , 2204.7334 ],
+                                      [2214.7334 , 2215.0667 , 2215.4    ]],
+                                     [[2193.4    , 2193.7332 , 2194.0667 ],
+                                      [2204.0667 , 2204.4    , 2204.7334 ],
+                                      [2214.7334 , 2215.0667 , 2215.4    ]]],
+                                    [[[ 185.     ,  185.33334,  185.66667],
+                                      [ 195.66667,  196.00002,  196.33334],
+                                      [ 206.33333,  206.66666,  207.     ]],
+                                     [[ 185.     ,  185.33334,  185.66667],
+                                      [ 195.66667,  196.00002,  196.33334],
+                                      [ 206.33333,  206.66666,  207.     ]],
+                                     [[ 185.     ,  185.33334,  185.66667],
+                                      [ 195.66667,  196.00002,  196.33334],
+                                      [ 206.33333,  206.66666,  207.     ]],
+                                     [[ 185.     ,  185.33334,  185.66667],
+                                      [ 195.66667,  196.00002,  196.33334],
+                                      [  206.33333,  206.66666,  207.     ]]]]])
+
+    gt_mask_cls = mx.nd.array([[0,0,1,0], [0,1,0,0]])
+    gt_mask_cls = gt_mask_cls.reshape(1,2,4,1,1).broadcast_axes(axis=(3,4), size=(3,3))
+
+    assert_almost_equal(mask_targets.asnumpy(), gt_mask_targets.asnumpy())
+    assert_almost_equal(mask_cls.asnumpy(), gt_mask_cls.asnumpy())
+
 if __name__ == '__main__':
     import nose
     nose.runmodule()
diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py
index 380ce762a9f7..c6da0f4d9d96 100644
--- a/tests/python/unittest/test_gluon.py
+++ b/tests/python/unittest/test_gluon.py
@@ -33,6 +33,7 @@
 import warnings
 import json
 import unittest
+import random
 
 @with_seed()
 def test_parameter():
@@ -1504,15 +1505,62 @@ def test_hybrid_multi_context():
 
 @with_seed()
 def test_zero_grad():
-    data = mx.nd.random.uniform(shape=(3,3))
-    net = nn.Embedding(3, 4, sparse_grad=True, prefix='test_zero_grad_')
-    net.initialize()
-    with mx.autograd.record():
-        l = net(data)
-        l.backward()
-    net.collect_params().zero_grad()
-    grad = net.collect_params()['test_zero_grad_weight'].grad()
-    assert_almost_equal(grad.asnumpy(), grad.asnumpy() * 0)
+    def _test_grad_reset(ctx, dtype='float32', sparse=False, embeddingType=None):
+        data = mx.nd.random.uniform(shape=(3,3), dtype=dtype, ctx=ctx)
+        if embeddingType is None:
+            embeddingType = dtype
+        net = nn.Embedding(3, 4, sparse_grad=sparse, prefix='test_zero_grad_', dtype=embeddingType)
+        net.initialize(ctx=ctx)
+        with mx.autograd.record():
+            l = net(data)
+            l.backward()
+        net.collect_params().zero_grad()
+        grad = net.collect_params()['test_zero_grad_weight'].grad()
+        assert_almost_equal(grad.asnumpy(), grad.asnumpy() * 0)
+
+    def _test_multi_reset(nArrays, dtype, ctx):
+        # Construct the list of non-zeros arrays with random shapes
+        arr = []
+        for _ in range(nArrays):
+            arrType = random.choice(dtype) if isinstance(dtype, list) else dtype
+            shape = ()
+            for _ in range(np.random.randint(1, 5)):
+                shape = shape + (np.random.randint(1, 10),)
+            arr.append(mx.nd.random.uniform(shape=shape, dtype=arrType, ctx=ctx))
+
+        # Reset all arrays
+        mx.nd.reset_arrays(*arr, num_arrays=len(arr))
+
+        # Check results
+        for i in range(nArrays):
+            grad = arr[i].asnumpy()
+            assert_almost_equal(grad, grad * 0)
+
+
+    # Setting context for current test
+    ctx = mx.context.current_context()
+
+    # Launching _test_multi_reset 10 times with different types & randomly chosen nArrays
+    testedTypes = ['float16', 'float32', 'float64']
+    for _ in range(10):
+        for type in [testedTypes] + testedTypes:
+            _test_multi_reset(np.random.randint(1, 50), type, ctx)
+
+    # Saving value of environment variable, if it was defined
+    envVarKey = 'MXNET_STORAGE_FALLBACK_LOG_VERBOSE'
+    envVarValue = os.environ[envVarKey] if envVarKey in os.environ else None
+    # Changing value of environment variable
+    os.environ[envVarKey] = '0'
+    for type in ['float16', 'float32', 'float64']:
+        for embType in ['float32', 'float64']:
+            for sparse in [True, False]:
+                _test_grad_reset(ctx, dtype=type, sparse=sparse, embeddingType=embType)
+
+    # Remove or restore the value of environment variable
+    if envVarValue is None:
+        del os.environ[envVarKey]
+    else:
+        os.environ[envVarKey] = envVarValue
 
 def check_hybrid_static_memory(**kwargs):
     x = mx.nd.random.uniform(shape=(2, 3, 32, 32))
diff --git a/tests/python/unittest/test_gluon_contrib.py b/tests/python/unittest/test_gluon_contrib.py
index 1e0555900f17..123db085e817 100644
--- a/tests/python/unittest/test_gluon_contrib.py
+++ b/tests/python/unittest/test_gluon_contrib.py
@@ -24,10 +24,9 @@
 from mxnet.gluon.contrib.nn import (
     Concurrent, HybridConcurrent, Identity, SparseEmbedding, PixelShuffle1D,
     PixelShuffle2D, PixelShuffle3D)
-from mxnet.test_utils import almost_equal, default_context, assert_almost_equal
+from mxnet.test_utils import almost_equal, default_context, assert_almost_equal, assert_allclose
 from common import setup_module, with_seed, teardown
 import numpy as np
-from numpy.testing import assert_allclose
 
 
 def check_rnn_cell(cell, prefix, in_shape=(10, 50), out_shape=(10, 100), begin_state=None):
@@ -112,6 +111,7 @@ def test_conv_fill_shape():
     check_rnn_forward(cell, mx.nd.ones((8, 3, 5, 7)))
     assert cell.i2h_weight.shape[1] == 5, cell.i2h_weight.shape[1]
 
+
 @with_seed()
 def test_lstmp():
     nhid = 100
@@ -193,8 +193,7 @@ def test_concurrent():
 def test_identity():
     model = Identity()
     x = mx.nd.random.uniform(shape=(128, 33, 64))
-    mx.test_utils.assert_almost_equal(model(x).asnumpy(),
-                                      x.asnumpy())
+    assert_almost_equal(model(x), x)
 
 @with_seed()
 def test_sparse_embedding():
@@ -219,7 +218,7 @@ def test_pixelshuffle1d():
     y = layer(x)
     assert y.shape == shape_after
     assert_allclose(
-        y.asnumpy(),
+        y,
         [[[0, 3, 1, 4, 2, 5],
           [6, 9, 7, 10, 8, 11]]]
     )
@@ -242,7 +241,7 @@ def test_pixelshuffle2d():
     # - Increasing the block index adds an offset of 1
     # - Increasing the channel index adds an offset of `nx * up_x * ny * up_y`
     assert_allclose(
-        y.asnumpy(),
+        y,
         [[[[ 0,  6, 12,  1,  7, 13,  2,  8, 14],
            [18, 24, 30, 19, 25, 31, 20, 26, 32],
            [ 3,  9, 15,  4, 10, 16,  5, 11, 17],
@@ -273,7 +272,7 @@ def test_pixelshuffle3d():
     #   column index by 1, e.g. the block [[[ 0, 24]], [[48, 72]]]
     # - Increasing the block index adds an offset of 1
     assert_allclose(
-        y.asnumpy(),
+        y,
         [[[[[ 0, 24,  1, 25,  2, 26,  3, 27],
             [ 4, 28,  5, 29,  6, 30,  7, 31],
             [ 8, 32,  9, 33, 10, 34, 11, 35]],
@@ -382,19 +381,17 @@ def check_unroll(cell_type, num_states, layout):
         trainer = gluon.Trainer(params2, 'sgd', {'learning_rate' : 0.03})
         with mx.autograd.record():
             res2, states2 = layer(rnn_data, states, valid_length)
-        assert_almost_equal(res1.asnumpy(), res2.asnumpy(), rtol=0.001, atol=0.0001)
+        assert_almost_equal(res1, res2, rtol=0.001, atol=0.0001)
         assert len(states1) == len(states2)
         for i in range(len(states1)):
-            assert_almost_equal(states1[i].asnumpy(), states2[i].asnumpy(),
-                                rtol=0.001, atol=0.0001)
+            assert_almost_equal(states1[i], states2[i], rtol=0.001, atol=0.0001)
         res2.backward()
         trainer.step(batch_size)
 
         for key, val in params1.items():
             weight1 = val.data()
             weight2 = params2[key].data()
-            assert_almost_equal(weight1.asnumpy(), weight2.asnumpy(),
-                    rtol=0.001, atol=0.0001)
+            assert_almost_equal(weight1, weight2, rtol=0.001, atol=0.0001)
 
 
 @with_seed()
diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py
index 3b9b46b16f93..af7c4838d5a9 100644
--- a/tests/python/unittest/test_loss.py
+++ b/tests/python/unittest/test_loss.py
@@ -49,10 +49,10 @@ def test_loss_ndarray():
 
     loss = gluon.loss.SoftmaxCrossEntropyLoss()
     L = loss(output, label).asnumpy()
-    mx.test_utils.assert_almost_equal(L, np.array([ 2.12692809,  0.04858733]))
+    assert_almost_equal(L, np.array([ 2.12692809,  0.04858733]))
 
     L = loss(output, label, weighting).asnumpy()
-    mx.test_utils.assert_almost_equal(L, np.array([ 1.06346405,  0.04858733]))
+    assert_almost_equal(L, np.array([ 1.06346405,  0.04858733]))
 
 
 def get_net(num_hidden, flatten=True):
@@ -126,8 +126,8 @@ def test_logistic_loss_equal_bce():
     loss_bce = gluon.loss.SigmoidBCELoss(from_sigmoid=False)
     data = mx.random.uniform(-10, 10, shape=(N, 1))
     label = mx.nd.round(mx.random.uniform(0, 1, shape=(N, 1)))
-    assert_almost_equal(loss_binary(data, label).asnumpy(), loss_bce(data, label).asnumpy(), atol=1e-6)
-    assert_almost_equal(loss_signed(data, 2 * label - 1).asnumpy(), loss_bce(data, label).asnumpy(), atol=1e-6)
+    assert_almost_equal(loss_binary(data, label), loss_bce(data, label), atol=1e-6)
+    assert_almost_equal(loss_signed(data, 2 * label - 1), loss_bce(data, label), atol=1e-6)
 
 @with_seed()
 def test_kl_loss():
@@ -186,27 +186,27 @@ def test_l1_loss():
 def test_ctc_loss():
     loss = gluon.loss.CTCLoss()
     l = loss(mx.nd.ones((2,20,4)), mx.nd.array([[1,0,-1,-1],[2,1,1,-1]]))
-    mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741]))
+    assert_almost_equal(l, np.array([18.82820702, 16.50581741]))
 
     loss = gluon.loss.CTCLoss(layout='TNC')
     l = loss(mx.nd.ones((20,2,4)), mx.nd.array([[1,0,-1,-1],[2,1,1,-1]]))
-    mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741]))
+    assert_almost_equal(l, np.array([18.82820702, 16.50581741]))
 
     loss = gluon.loss.CTCLoss(layout='TNC', label_layout='TN')
     l = loss(mx.nd.ones((20,2,4)), mx.nd.array([[1,0,-1,-1],[2,1,1,-1]]).T)
-    mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741]))
+    assert_almost_equal(l, np.array([18.82820702, 16.50581741]))
 
     loss = gluon.loss.CTCLoss()
     l = loss(mx.nd.ones((2,20,4)), mx.nd.array([[2,1,2,2],[3,2,2,2]]), None, mx.nd.array([2,3]))
-    mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741]))
+    assert_almost_equal(l, np.array([18.82820702, 16.50581741]))
 
     loss = gluon.loss.CTCLoss()
     l = loss(mx.nd.ones((2,25,4)), mx.nd.array([[2,1,-1,-1],[3,2,2,-1]]), mx.nd.array([20,20]))
-    mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741]))
+    assert_almost_equal(l, np.array([18.82820702, 16.50581741]))
 
     loss = gluon.loss.CTCLoss()
     l = loss(mx.nd.ones((2,25,4)), mx.nd.array([[2,1,3,3],[3,2,2,3]]), mx.nd.array([20,20]), mx.nd.array([2,3]))
-    mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741]))
+    assert_almost_equal(l, np.array([18.82820702, 16.50581741]))
 
 
 @with_seed()
@@ -367,12 +367,14 @@ def test_cosine_loss():
     assert_almost_equal(loss.asnumpy(), numpy_loss.asnumpy(), rtol=1e-3, atol=1e-5)
 
 def test_poisson_nllloss():
-    pred = mx.nd.random.normal(shape=(3, 4))
+    shape=(3, 4)
+    not_axis0 = tuple(range(1, len(shape)))
+    pred = mx.nd.random.normal(shape=shape)
     min_pred = mx.nd.min(pred)
     #This is necessary to ensure only positive random values are generated for prediction,
     # to avoid ivalid log calculation
     pred[:] = pred + mx.nd.abs(min_pred)
-    target = mx.nd.random.normal(shape=(3, 4))
+    target = mx.nd.random.normal(shape=shape)
     min_target = mx.nd.min(target)
     #This is necessary to ensure only positive random values are generated for prediction,
     # to avoid ivalid log calculation
@@ -396,8 +398,9 @@ def test_poisson_nllloss():
         assert_almost_equal(np_loss_no_logits, loss_no_logits.asscalar())
 
     #3) Testing for Sterling approximation
-    np_pred = np.random.uniform(1, 5, (2, 3))
-    np_target = np.random.uniform(1, 5, (2, 3))
+    shape=(2, 3)
+    np_pred = np.random.uniform(1, 5, shape)
+    np_target = np.random.uniform(1, 5, shape)
     np_compute_full = np.mean((np_pred - np_target * np.log(np_pred + 1e-08)) + ((np_target * np.log(np_target)-\
      np_target + 0.5 * np.log(2 * np_target * np.pi))*(np_target > 1)))
     Loss_compute_full = gluon.loss.PoissonNLLLoss(from_logits=False, compute_full=True)
diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py
index 7be59df6efda..bee4bff0f7c0 100644
--- a/tests/python/unittest/test_ndarray.py
+++ b/tests/python/unittest/test_ndarray.py
@@ -41,6 +41,10 @@ def check_with_uniform(uf, arg_shapes, dim=None, npuf=None, rmin=-10, type_list=
         assert dim
         shape = tuple(np.random.randint(1, int(1000**(1.0/dim)), size=dim))
         arg_shapes = [shape] * arg_shapes
+
+    if npuf is None:
+        npuf = uf
+
     for dtype in type_list:
         ndarray_arg = []
         numpy_arg = []
@@ -50,10 +54,7 @@ def check_with_uniform(uf, arg_shapes, dim=None, npuf=None, rmin=-10, type_list=
             ndarray_arg.append(narr)
             numpy_arg.append(npy)
         out1 = uf(*ndarray_arg)
-        if npuf is None:
-            out2 = uf(*numpy_arg).astype(dtype)
-        else:
-            out2 = npuf(*numpy_arg).astype(dtype)
+        out2 = npuf(*numpy_arg).astype(dtype)
 
         assert out1.shape == out2.shape
         if isinstance(out1, mx.nd.NDArray):
@@ -1712,13 +1713,13 @@ def l2norm(input_data, axis=0, keepdims=False):
                         np_arr, i, keep_dims)
                     mx_out = mx.nd.norm(mx_arr, ord=ord, axis=i, keepdims=keep_dims)
                     assert npy_out.shape == mx_out.shape
-                    mx.test_utils.assert_almost_equal(npy_out, mx_out.asnumpy())
+                    assert_almost_equal(npy_out, mx_out)
                     if (i < 3):
                         npy_out = l1norm(np_arr, (i, i + 1), keep_dims) if ord == 1 else l2norm(
                             np_arr, (i, i + 1), keep_dims)
                         mx_out = mx.nd.norm(mx_arr, ord=ord, axis=(i, i + 1), keepdims=keep_dims)
                         assert npy_out.shape == mx_out.shape
-                        mx.test_utils.assert_almost_equal(npy_out, mx_out.asnumpy())
+                        assert_almost_equal(npy_out, mx_out)
 
 
 @with_seed()
@@ -1732,7 +1733,7 @@ def test_dlpack():
     for dtype in [np.float32, np.int32]:
         for shape in [(3, 4, 5, 6), (2, 10), (15,)]:
             a = mx.nd.random.uniform(shape = shape)
-            a_np = a.asnumpy()
+            a_np = a.copy()
 
             pack = a.to_dlpack_for_read()
             b = mx.nd.from_dlpack(pack)
@@ -1750,14 +1751,10 @@ def test_dlpack():
 
             del a, pack, pack2, pack3, pack4
 
-            b_np = b.asnumpy()
-            c_np = c.asnumpy()
-            d_np = d.asnumpy()
-            e_np = e.asnumpy()
-            mx.test_utils.assert_almost_equal(a_np, b_np)
-            mx.test_utils.assert_almost_equal(a_np, c_np)
-            mx.test_utils.assert_almost_equal(a_np, d_np)
-            mx.test_utils.assert_almost_equal(a_np, e_np)
+            assert_almost_equal(a_np, b)
+            assert_almost_equal(a_np, c)
+            assert_almost_equal(a_np, d)
+            assert_almost_equal(a_np, e)
 
 @with_seed()
 def test_ndarray_is_inf():
diff --git a/tests/python/unittest/test_numpy_gluon.py b/tests/python/unittest/test_numpy_gluon.py
index f24eb6a325bf..af5425336699 100644
--- a/tests/python/unittest/test_numpy_gluon.py
+++ b/tests/python/unittest/test_numpy_gluon.py
@@ -113,6 +113,15 @@ def hybrid_forward(self, F, pred, label):
         trainer.step(1)
 
 
+@with_seed()
+@use_np
+def test_optimizer_backward_compat():
+    optimizer = mx.optimizer.SGD()
+    delattr(optimizer, "allow_np_array")
+    updater = mx.optimizer.Updater(optimizer)
+    updater(0, np.ones((0, 0)), np.zeros((0, 0)))
+
+
 @with_seed()
 @use_np
 def test_np_loss_ndarray():
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index f0c205f72dc4..af5991381bd4 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -25,6 +25,7 @@
 from mxnet.numpy_dispatch_protocol import with_array_function_protocol, with_array_ufunc_protocol
 from mxnet.numpy_dispatch_protocol import _NUMPY_ARRAY_FUNCTION_LIST, _NUMPY_ARRAY_UFUNC_LIST
 
+import itertools
 
 class OpArgMngr(object):
     """Operator argument manager for storing operator workloads."""
@@ -49,6 +50,10 @@ def _prepare_workloads():
         '1x1x0': np.array([[[]]])
     }
 
+    dt_int = [np.int8, np.int32, np.int64, np.uint8]
+    dt_float = [np.float16, np.float32, np.float64]
+    dt = dt_int + dt_float
+
     # workloads for array function protocol
     OpArgMngr.add_workload('argmax', array_pool['4x1'])
     OpArgMngr.add_workload('broadcast_arrays', array_pool['4x1'], array_pool['1x2'])
@@ -57,8 +62,14 @@ def _prepare_workloads():
     OpArgMngr.add_workload('concatenate', [array_pool['4x1'], array_pool['4x1']])
     OpArgMngr.add_workload('concatenate', [array_pool['4x1'], array_pool['4x1']], axis=1)
     OpArgMngr.add_workload('copy', array_pool['4x1'])
-    OpArgMngr.add_workload('cumsum', array_pool['4x1'])
-    OpArgMngr.add_workload('cumsum', array_pool['4x1'], axis=1)
+
+    for ctype in dt:
+        OpArgMngr.add_workload('cumsum', np.array([1, 2, 10, 11, 6, 5, 4], dtype=ctype))
+        OpArgMngr.add_workload('cumsum', np.array([[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]], dtype=ctype), axis=0)
+        OpArgMngr.add_workload('cumsum', np.array([[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]], dtype=ctype), axis=1)
+
+    OpArgMngr.add_workload('ravel', np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]))
+
     OpArgMngr.add_workload('dot', array_pool['4x1'], array_pool['4x1'].T)
     OpArgMngr.add_workload('expand_dims', array_pool['4x1'], -1)
     OpArgMngr.add_workload('fix', array_pool['4x1'])
@@ -68,8 +79,72 @@ def _prepare_workloads():
     OpArgMngr.add_workload('mean', array_pool['4x1'], axis=0, keepdims=True)
     OpArgMngr.add_workload('ones_like', array_pool['4x1'])
     OpArgMngr.add_workload('prod', array_pool['4x1'])
+
     OpArgMngr.add_workload('repeat', array_pool['4x1'], 3)
-    OpArgMngr.add_workload('reshape', array_pool['4x1'], -1)
+    OpArgMngr.add_workload('repeat', np.array(_np.arange(12).reshape(4, 3)[:, 2]), 3)
+
+    m = _np.array([1, 2, 3, 4, 5, 6])
+    m_rect = m.reshape((2, 3))
+
+    # OpArgMngr.add_workload('repeat', np.array(m), [1, 3, 2, 1, 1, 2]) # Argument "repeats" only supports int
+    OpArgMngr.add_workload('repeat', np.array(m), 2)
+    B = np.array(m_rect)
+    # OpArgMngr.add_workload('repeat', B, [2, 1], axis=0)  # Argument "repeats" only supports int
+    # OpArgMngr.add_workload('repeat', B, [1, 3, 2], axis=1)  # Argument "repeats" only supports int
+    OpArgMngr.add_workload('repeat', B, 2, axis=0)
+    OpArgMngr.add_workload('repeat', B, 2, axis=1)
+
+    # test_repeat_broadcasting
+    a = _np.arange(60).reshape(3, 4, 5)
+    for axis in itertools.chain(range(-a.ndim, a.ndim), [None]):
+        OpArgMngr.add_workload('repeat', np.array(a), 2, axis=axis)
+    #    OpArgMngr.add_workload('repeat', np.array(a), [2], axis=axis)   # Argument "repeats" only supports int
+
+    arr = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]])
+    OpArgMngr.add_workload('reshape', arr, (2, 6))
+    OpArgMngr.add_workload('reshape', arr, (3, 4))
+    # OpArgMngr.add_workload('reshape', arr, (3, 4), order='F') # Items are not equal with order='F'
+    OpArgMngr.add_workload('reshape', arr, (3, 4), order='C')
+    OpArgMngr.add_workload('reshape', np.array(_np.ones(100)), (100, 1, 1))
+    
+    # test_reshape_order
+    a = np.array(_np.arange(6))
+    # OpArgMngr.add_workload('reshape', a, (2, 3), order='F')  # Items are not equal with order='F'
+    a = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
+    b = a[:, 1]
+    # OpArgMngr.add_workload('reshape', b, (2, 2), order='F')  # Items are not equal with order='F'
+
+    a = np.array(_np.ones((0, 2)))
+    OpArgMngr.add_workload('reshape', a, -1, 2)
+
+    OpArgMngr.add_workload('rint', np.array(4607998452777363968))
+    OpArgMngr.add_workload('rint', array_pool['4x1'])
+
+    # test_roll1d(self)
+    OpArgMngr.add_workload('roll', np.array(_np.arange(10)), 2)
+
+    # test_roll2d(self)
+    x2 = np.array(_np.reshape(_np.arange(10), (2, 5)))
+    OpArgMngr.add_workload('roll', x2, 1)
+    OpArgMngr.add_workload('roll', x2, 1, axis=0)
+    OpArgMngr.add_workload('roll', x2, 1, axis=1)
+    # # Roll multiple axes at once.
+    OpArgMngr.add_workload('roll', x2, 1, axis=(0, 1))
+    OpArgMngr.add_workload('roll', x2, (1, 0), axis=(0, 1))
+    OpArgMngr.add_workload('roll', x2, (-1, 0), axis=(0, 1))
+    OpArgMngr.add_workload('roll', x2, (0, 1), axis=(0, 1))
+    OpArgMngr.add_workload('roll', x2, (0, -1), axis=(0, 1))
+    OpArgMngr.add_workload('roll', x2, (1, 1), axis=(0, 1))
+    OpArgMngr.add_workload('roll', x2, (-1, -1), axis=(0, 1))
+    # # Roll the same axis multiple times.
+    # OpArgMngr.add_workload('roll', x2, 1, axis=(0, 0)) # Check failed: axes[i - 1] < axes[i] (0 vs. 0) : axes have duplicates [0,0]
+    # OpArgMngr.add_workload('roll', x2, 1, axis=(1, 1)) # Check failed: axes[i - 1] < axes[i] (1 vs. 1) : axes have duplicates [1,1]
+    # # Roll more than one turn in either direction.
+    OpArgMngr.add_workload('roll', x2, 6, axis=1)
+    OpArgMngr.add_workload('roll', x2, -4, axis=1)
+    # # test_roll_empty
+    OpArgMngr.add_workload('roll', np.array([]), 1)
+    
     OpArgMngr.add_workload('split', array_pool['4x1'], 2)
     OpArgMngr.add_workload('squeeze', array_pool['4x1'])
     OpArgMngr.add_workload('stack', [array_pool['4x1']] * 2)
@@ -104,10 +179,64 @@ def _prepare_workloads():
     OpArgMngr.add_workload('mod', array_pool['4x1'], 2)
     OpArgMngr.add_workload('mod', 2, array_pool['4x1'])
     OpArgMngr.add_workload('mod', array_pool['4x1'], array_pool['1x1x0'])
-    OpArgMngr.add_workload('remainder', array_pool['4x1'], array_pool['1x2'])
-    OpArgMngr.add_workload('remainder', array_pool['4x1'], 2)
-    OpArgMngr.add_workload('remainder', 2, array_pool['4x1'])
-    OpArgMngr.add_workload('remainder', array_pool['4x1'], array_pool['1x1x0'])
+
+    # test remainder basic
+    OpArgMngr.add_workload('remainder', np.array([0, 1, 2, 4, 2], dtype=np.float16),
+                            np.array([-2, 5, 1, 4, 3], dtype=np.float16))
+
+    def _signs(dt):
+        if dt in [np.uint8]:
+            return (+1,)
+        else:
+            return (+1, -1)
+
+    for ct in dt:
+        for sg1, sg2 in itertools.product(_signs(ct), _signs(ct)):
+            a = np.array(sg1*71, dtype=ct)
+            b = np.array(sg2*19, dtype=ct)
+            OpArgMngr.add_workload('remainder', a, b)
+
+    # test remainder exact
+    nlst = list(range(-127, 0))
+    plst = list(range(1, 128))
+    dividend = nlst + [0] + plst
+    divisor = nlst + plst
+    arg = list(itertools.product(dividend, divisor))
+    tgt = list(divmod(*t) for t in arg)
+    a, b = np.array(arg, dtype=int).T
+    # convert exact integer results from Python to float so that
+    # signed zero can be used, it is checked.
+    for dt in [np.float16, np.float32, np.float64]:
+        fa = a.astype(dt)
+        fb = b.astype(dt)
+        OpArgMngr.add_workload('remainder', fa, fb)
+    
+    # test_float_remainder_roundoff
+    for ct in dt_float:
+        for sg1, sg2 in itertools.product((+1, -1), (+1, -1)):
+            a = np.array(sg1*78*6e-8, dtype=ct)
+            b = np.array(sg2*6e-8, dtype=ct)
+            OpArgMngr.add_workload('remainder', a, b)
+
+    # test_float_remainder_corner_cases
+    # Check remainder magnitude.
+    for ct in dt_float:
+        b = _np.array(1.0)
+        a = np.array(_np.nextafter(_np.array(0.0), -b), dtype=ct)
+        b = np.array(b, dtype=ct)
+        OpArgMngr.add_workload('remainder', a, b)
+        OpArgMngr.add_workload('remainder', -a, -b)
+
+        # Check nans, inf
+        for ct in [np.float16, np.float32, np.float64]:
+            fone = np.array(1.0, dtype=ct)
+            fzer = np.array(0.0, dtype=ct)
+            finf = np.array(np.inf, dtype=ct)
+            fnan = np.array(np.nan, dtype=ct)
+            # OpArgMngr.add_workload('remainder', fone, fzer) # failed
+            OpArgMngr.add_workload('remainder', fone, fnan)
+            OpArgMngr.add_workload('remainder', finf, fone)
+
     OpArgMngr.add_workload('maximum', array_pool['4x1'], array_pool['1x2'])
     OpArgMngr.add_workload('maximum', array_pool['4x1'], 2)
     OpArgMngr.add_workload('maximum', 2, array_pool['4x1'])
@@ -118,8 +247,12 @@ def _prepare_workloads():
     OpArgMngr.add_workload('minimum', array_pool['4x1'], array_pool['1x1x0'])
     OpArgMngr.add_workload('negative', array_pool['4x1'])
     OpArgMngr.add_workload('absolute', array_pool['4x1'])
-    OpArgMngr.add_workload('rint', array_pool['4x1'])
+    
     OpArgMngr.add_workload('sign', array_pool['4x1'])
+    OpArgMngr.add_workload('sign', np.array([-2, 5, 1, 4, 3], dtype=np.float16))
+    OpArgMngr.add_workload('sign', np.array([-.1, 0, .1]))
+    # OpArgMngr.add_workload('sign', np.array(_np.array([_np.nan]))) # failed
+
     OpArgMngr.add_workload('exp', array_pool['4x1'])
     OpArgMngr.add_workload('log', array_pool['4x1'])
     OpArgMngr.add_workload('log2', array_pool['4x1'])
@@ -128,7 +261,12 @@ def _prepare_workloads():
     OpArgMngr.add_workload('sqrt', array_pool['4x1'])
     OpArgMngr.add_workload('square', array_pool['4x1'])
     OpArgMngr.add_workload('cbrt', array_pool['4x1'])
-    OpArgMngr.add_workload('reciprocal', array_pool['4x1'])
+
+    for ctype in [np.float16, np.float32, np.float64]:
+        OpArgMngr.add_workload('reciprocal', np.array([-2, 5, 1, 4, 3], dtype=ctype))
+        OpArgMngr.add_workload('reciprocal', np.array([-2, 0, 1, 0, 3], dtype=ctype))
+        OpArgMngr.add_workload('reciprocal', np.array([0], dtype=ctype))
+
     OpArgMngr.add_workload('sin', array_pool['4x1'])
     OpArgMngr.add_workload('cos', array_pool['4x1'])
     OpArgMngr.add_workload('tan', array_pool['4x1'])
@@ -176,10 +314,10 @@ def _check_interoperability_helper(op_name, *args, **kwargs):
             assert isinstance(arr, np.ndarray)
         for arr, expected_arr in zip(out, expected_out):
             assert isinstance(arr, np.ndarray)
-            assert_almost_equal(arr.asnumpy(), expected_arr, rtol=1e-3, atol=1e-4, use_broadcast=False)
+            assert_almost_equal(arr.asnumpy(), expected_arr, rtol=1e-3, atol=1e-4, use_broadcast=False, equal_nan=True)
     else:
         assert isinstance(out, np.ndarray)
-        assert_almost_equal(out.asnumpy(), expected_out, rtol=1e-3, atol=1e-4, use_broadcast=False)
+        assert_almost_equal(out.asnumpy(), expected_out, rtol=1e-3, atol=1e-4, use_broadcast=False, equal_nan=True)
 
 
 def check_interoperability(op_list):
diff --git a/tests/python/unittest/test_numpy_ndarray.py b/tests/python/unittest/test_numpy_ndarray.py
index 53a8076f1303..b6692c009c18 100644
--- a/tests/python/unittest/test_numpy_ndarray.py
+++ b/tests/python/unittest/test_numpy_ndarray.py
@@ -1075,6 +1075,29 @@ def test_boolean_indexing_autograd():
     test_boolean_indexing_autograd()
 
 
+@with_seed()
+@use_np
+def test_np_get_dtype():
+    dtypes = [_np.int8, _np.int32, _np.float16, _np.float32, _np.float64, _np.bool, _np.bool_,
+              'int8', 'int32', 'float16', 'float32', 'float64', 'bool', None]
+    objects = [
+        [],
+        (),
+        [[1, 2], [3, 4]],
+        _np.random.uniform(size=rand_shape_nd(3)),
+        _np.random.uniform(size=(3, 0, 4))
+    ]
+    for dtype in dtypes:
+        for src in objects:
+            mx_arr = np.array(src, dtype=dtype)
+            assert mx_arr.ctx == mx.current_context()
+            if isinstance(src, mx.nd.NDArray):
+                np_arr = _np.array(src.asnumpy(), dtype=dtype if dtype is not None else _np.float32)
+            else:
+                np_arr = _np.array(src, dtype=dtype if dtype is not None else _np.float32)
+            assert type(mx_arr.dtype) == type(np_arr.dtype)
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule()
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 0848b0861a76..28edc2e05306 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -853,6 +853,56 @@ def hybrid_forward(self, F, x):
                     assert_almost_equal(mx_out.asnumpy(), np_out, atol=1e-3, rtol=1e-5)
 
 
+@with_seed()
+@use_np
+def test_np_logspace():
+    class TestLogspace(HybridBlock):
+        def __init__(self, start, stop, num=50, endpoint=None, base=50.0, dtype=None, axis=0):
+            super(TestLogspace, self).__init__()
+            self._start = start
+            self._stop = stop
+            self._num = num
+            self._endpoint = endpoint
+            self._base = base
+            self._dtype = dtype
+            self.axis = axis
+
+        def hybrid_forward(self, F, x):
+            return x + F.np.logspace(self._start, self._stop, self._num, self._endpoint, self._base, self._dtype, self.axis)
+
+    configs = [
+        (0.0, 1.0, 20),
+        (2, 8, 0),
+        (22, 11, 1),
+        (2.22, 9.99, 11),
+        (4.99999, 12.11111111, 111)
+    ]
+    base_configs = [0, 1, 5, 8, 10, 33]
+    dtypes = ['float32', 'float64', None]
+
+    for config in configs:
+        for dtype in dtypes:
+            for endpoint in [False, True]:
+                for hybridize in [False, True]:
+                    for base in base_configs:
+                        x = np.zeros(shape=(), dtype=dtype)
+                        net = TestLogspace(*config, endpoint=endpoint, base=base, dtype=dtype)
+                        np_out = _np.logspace(*config, endpoint=endpoint, base=base, dtype=dtype)
+                        if hybridize:
+                            net.hybridize()
+                        mx_out = net(x)
+                        assert_almost_equal(mx_out.asnumpy(), np_out, atol=1e-3, rtol=1e-5)
+                        if dtype is not None:
+                            assert mx_out.dtype == np_out.dtype
+
+                        # Test imperative once again
+                        mx_ret = np.logspace(*config, endpoint=endpoint, base=base, dtype=dtype)
+                        np_ret = _np.logspace(*config, endpoint=endpoint, base=base, dtype=dtype)
+                        assert_almost_equal(mx_ret.asnumpy(), np_ret, atol=1e-3, rtol=1e-5)
+                        if dtype is not None:
+                            assert mx_out.dtype == np_out.dtype
+
+
 @with_seed()
 @use_np
 def test_npx_slice():
@@ -1591,6 +1641,61 @@ def get_indices(axis_size):
                     assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
 
 
+@with_seed()
+@use_np
+def test_np_vsplit():
+    class TestVsplit(HybridBlock):
+        def __init__(self, indices_or_sections):
+            super(TestVsplit, self).__init__()
+            self._indices_or_sections = indices_or_sections
+
+        def hybrid_forward(self, F, a, *args, **kwargs):
+            return F.np.vsplit(a, indices_or_sections=self._indices_or_sections)
+
+    def get_indices(axis_size):
+        if axis_size is 0:
+            axis_size = random.randint(3, 6)
+        samples = random.randint(1, axis_size - 1)
+        indices = sorted(random.sample([i for i in range(1, axis_size)], samples))
+        indices = tuple(indices)
+        return indices
+
+    shapes = [
+        (2, 1, 2, 9),
+        (4, 3, 3),
+        (4, 0, 2),  # zero-size shape
+        (0, 3), # first dim being zero
+    ]
+    for hybridize in [True, False]:
+        for shape in shapes:
+            axis_size = shape[0]
+            indices = get_indices(axis_size)
+            sections = 7 if axis_size is 0 else axis_size
+            for indices_or_sections in [indices, sections]:
+                # test gluon
+                test_vsplit = TestVsplit(indices_or_sections=indices_or_sections)
+                if hybridize:
+                    test_vsplit.hybridize()
+                a = rand_ndarray(shape).as_np_ndarray() # TODO: check type
+                a.attach_grad()
+                expected_ret = _np.vsplit(a.asnumpy(), indices_or_sections=indices_or_sections)
+                with mx.autograd.record():
+                    y = test_vsplit(a)
+                assert len(y) == len(expected_ret)
+                for mx_out, np_out in zip(y, expected_ret):
+                    assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+
+                mx.autograd.backward(y)
+
+                assert_almost_equal(a.grad.asnumpy(), _np.ones(a.shape), rtol=1e-3, atol=1e-5)
+
+                # test imperative
+                mx_outs = np.vsplit(a, indices_or_sections=indices_or_sections)
+                np_outs = _np.vsplit(a.asnumpy(), indices_or_sections=indices_or_sections)
+                for mx_out, np_out in zip(mx_outs, np_outs):
+                    assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+
+
 @with_seed()
 @use_np
 def test_np_concat():
@@ -1693,6 +1798,67 @@ def hybrid_forward(self, F, a, *args):
                 assert same(mx_out.asnumpy(), np_out)
 
 
+@with_seed()
+@use_np
+def test_np_dstack():
+    class TestDStack(HybridBlock):
+        def __init__(self):
+            super(TestDStack, self).__init__()
+
+        def hybrid_forward(self, F, a, *args):
+            return F.np.dstack([a] + list(args))
+
+    def get_new_shape(shape):
+        if len(shape) < 3:
+            return shape
+        axis = 2
+        shape_lst = list(shape)
+        shape_lst[axis] = random.randint(0, 5)
+        return tuple(shape_lst)
+
+    shapes = [
+        (),
+        (1,),
+        (2,1),
+        (2,2,4),
+        (2,0,0),
+        (0,1,3),
+        (2,0,3),
+        (2,3,4,5)
+    ]
+    for hybridize in [True, False]:
+        for shape in shapes:
+            test_dstack = TestDStack()
+            if hybridize:
+                test_dstack.hybridize()
+            # test symbolic forward
+            a = mx.nd.random.uniform(shape=get_new_shape(shape)).as_np_ndarray()
+            a.attach_grad()
+            b = mx.nd.random.uniform(shape=get_new_shape(shape)).as_np_ndarray()
+            b.attach_grad()
+            c = mx.nd.random.uniform(shape=get_new_shape(shape)).as_np_ndarray()
+            c.attach_grad()
+            d = mx.nd.random.uniform(shape=get_new_shape(shape)).as_np_ndarray()
+            d.attach_grad()
+            with mx.autograd.record():
+                mx_out = test_dstack(a, b, c, d)
+            np_out = _np.dstack((a.asnumpy(), b.asnumpy(), c.asnumpy(), d.asnumpy()))
+            assert mx_out.shape == np_out.shape
+            assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+
+            # test symbolic backward
+            mx_out.backward()
+            assert_almost_equal(a.grad.asnumpy(), _np.ones(a.shape), rtol=1e-3, atol=1e-5)
+            assert_almost_equal(b.grad.asnumpy(), _np.ones(b.shape), rtol=1e-3, atol=1e-5)
+            assert_almost_equal(c.grad.asnumpy(), _np.ones(c.shape), rtol=1e-3, atol=1e-5)
+            assert_almost_equal(d.grad.asnumpy(), _np.ones(d.shape), rtol=1e-3, atol=1e-5)
+
+            # test imperative
+            mx_out = np.dstack((a, b, c, d))
+            np_out = _np.dstack((a.asnumpy(),b.asnumpy(), c.asnumpy(), d.asnumpy()))
+            assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+
+
 @with_seed()
 @use_np
 def test_np_ravel():
@@ -2161,6 +2327,74 @@ def test_indexing_mode(sampler, set_size, samples_size, replace, weight=None):
             test_indexing_mode(test_choice_weighted, num_classes, num_classes // 2, replace, weight)
 
 
+@with_seed()
+@use_np
+def test_np_eye():
+    configs = [
+        4,
+        1000,
+        (4, 3),
+        (5, None),
+        (4, None, 1),
+        (2, 2, 1),
+        (4, 6, 1),
+        (7, 3, -3),
+        (3, 2, -2),
+        (4, 0),
+        (0, 0),
+        (0, 3),
+        (0, 0, -2)
+    ]
+    exception_configs = [
+        -1,
+        -1000,
+        (-2, None),
+        (1, -1)
+    ]
+    dtypes = ['int32', 'float16', 'float32', 'float64', None]
+    for config in configs:
+        for dtype in dtypes:
+            if isinstance(config, tuple):
+                mx_ret = np.eye(*config, dtype=dtype)
+                np_ret = _np.eye(*config, dtype=dtype)
+            else:
+                mx_ret = np.eye(config, dtype=dtype)
+                np_ret = _np.eye(config, dtype=dtype)
+            assert same(mx_ret.asnumpy(), np_ret)
+    # check for exception input
+    for config in exception_configs:
+        if isinstance(config, tuple):
+            assertRaises(MXNetError, np.eye, *config)
+        else:
+            assertRaises(MXNetError, np.eye, config)
+
+    class TestEye(HybridBlock):
+        def __init__(self, N, M=None, k=0, dtype=None):
+            super(TestEye, self).__init__()
+            self._N = N
+            self._M = M
+            self._k = k
+            self._dtype = dtype
+
+        def hybrid_forward(self, F, x):
+            return x + F.np.eye(self._N, self._M, self._k, dtype=self._dtype)
+
+    for dtype in dtypes:
+        x = np.zeros(shape=(), dtype=dtype)
+        for config in configs:
+            for hybridize in [False, True]:
+                if isinstance(config, tuple):
+                    net = TestEye(*config, dtype=dtype)
+                    np_out = _np.eye(*config, dtype=dtype)
+                else:
+                    net = TestEye(config, dtype=dtype)
+                    np_out = _np.eye(config, dtype=dtype)
+                if hybridize:
+                    net.hybridize()
+                mx_out = net(x)
+                assert same(mx_out.asnumpy(), np_out)
+
+
 @with_seed()
 @use_np
 def test_np_indices():
@@ -3151,6 +3385,154 @@ def test_np_builtin_op_signature():
             assert str(op.__signature__) == str(inspect.signature(getattr(_numpy_op_doc, op_name)))
 
 
+@with_seed()
+@use_np
+def test_np_moveaxis():
+    class TestMoveaxis(HybridBlock):
+        def __init__(self, source=None, destination=None):
+            super(TestMoveaxis, self).__init__()
+            self._source = source
+            self._destination= destination
+
+        def hybrid_forward(self, F, x):
+            return F.np.moveaxis(x, source=self._source, destination=self._destination)
+
+    dtypes = ['int32', 'int64', 'float16', 'float32', 'float64']
+    for hybridize in [False, True]:
+        for dtype in dtypes:
+            for ndim in [0, 1, 2, 3, 4, 5, 6]:
+                shape = rand_shape_nd(ndim, dim=5, allow_zero_size=True)
+                np_data = _np.random.uniform(low=-100, high=100, size=shape).astype(dtype)
+                mx_data = np.array(np_data, dtype=dtype)
+                axis = [i for i in range(ndim)]
+                random.shuffle(axis)
+                for i in range(ndim):
+                    source = random.sample(axis, i)
+                    destination = random.sample(axis, i)
+
+                    # test gluon
+                    test_moveaxis = TestMoveaxis(source,destination)
+                    if hybridize:
+                        test_moveaxis.hybridize()
+                    np_out = _np.moveaxis(np_data, source=source, destination=destination)
+                    mx_data.attach_grad()
+                    with mx.autograd.record():
+                        mx_out = test_moveaxis(mx_data)
+                    assert mx_out.shape == np_out.shape
+                    mx_out.backward()
+                    assert same(mx_data.grad.shape, mx_data.shape)
+                    assert same(mx_data.grad.asnumpy(), _np.ones(shape))
+                    # test imperative
+                    np_out = _np.moveaxis(np_data, source=source, destination=destination)
+                    mx_out = np.moveaxis(mx_data, source=source, destination= destination)
+                    assert np_out.dtype == mx_out.dtype
+                    assert same(mx_out.asnumpy(), np_out)
+
+
+@with_seed()
+@use_np
+def test_np_rot90():
+    class TestTRot90(HybridBlock):
+        def __init__(self, k=1, axes=(0, 1)):
+            super(TestTRot90, self).__init__()
+            self._k = k
+            self._axes = axes
+
+        def hybrid_forward(self, F, a, *args):
+            return F.np.rot90(a, self._k, self._axes)
+
+    configs = [
+        ((2, 3), 1, (0, 1)),
+        ((2, 3), 3, (0, 1)),
+        ((2, 3), 1, (1, 0)),
+        ((2, 3), 2, (1, 0)),
+        ((2, 3), 3, (1, 0)),
+        ((2, 3), 0, (1, 0)),
+        ((2, 3, 4, 5), 3, (1, 2)),
+        ((2, 3, 4, 5), -3, (2, 3)),
+        ((2, 3, 0, 5), -2, (2, 3)),
+        ((2, 0, 0, 5), -3, (2, 3)),
+        ((2, 3, 0, 5), 0, (2, 1)),
+    ]
+    dtypes = ['uint8', 'int8', 'int32', 'int64', 'float16', 'float32', 'float64']
+
+    for config in configs:
+        for dtype in dtypes:
+            for hybridize in [True, False]:
+                shape, k, axes = config[0], config[1], config[2]
+                x = rand_ndarray(shape=shape, dtype=dtype).as_np_ndarray()
+                net = TestTRot90(k=k, axes=axes)
+                if hybridize:
+                    net.hybridize()
+
+                x.attach_grad()
+                np_out = _np.rot90(x.asnumpy(), k=k, axes=axes)
+                with mx.autograd.record():
+                    mx_out = net(x)
+                assert mx_out.shape == np_out.shape
+                assert same(mx_out.asnumpy(), np_out)
+                mx_out.backward()
+                np_backward = _np.ones(shape, dtype)
+
+                assert same(x.grad.asnumpy().shape, np_backward.shape)
+                assert same(x.grad.asnumpy(), np_backward)
+
+                np_out = _np.rot90(x.asnumpy(), k=k, axes=axes)
+                mx_out = np.rot90(x, k=k, axes=axes)
+                assert same(mx_out.asnumpy(), np_out)
+
+
+@with_seed()
+@use_np
+def test_np_hsplit():
+    class TestHSplit(HybridBlock):
+        def __init__(self, indices_or_sections):
+            super(TestHSplit, self).__init__()
+            self._indices_or_sections = indices_or_sections
+
+        def hybrid_forward(self, F, a, *args, **kwargs):
+            return F.np.hsplit(a, indices_or_sections=self._indices_or_sections)
+
+    shapes = [
+        (10,),
+        (3, 8, 5),
+        (3, 0, 5),
+        (3, 8, 5, 6),
+        (3, 0, 5, 6),
+    ]
+    indices_or_sections_num = [
+        (2, 4),
+        (3, 3),
+        (3,),
+        (1,),
+        2,
+    ]
+    for hybridize in [True, False]:
+        for shape in shapes:
+            for indices_or_sections in indices_or_sections_num:
+                # test gluon
+                test_hsplit = TestHSplit(indices_or_sections=indices_or_sections)
+                if hybridize:
+                    test_hsplit.hybridize()
+
+                a = mx.nd.random.uniform(-1.0, 1.0, shape=shape).as_np_ndarray()
+                a.attach_grad()
+                expected_ret = _np.hsplit(a.asnumpy(), indices_or_sections=indices_or_sections)
+                with mx.autograd.record():
+                    y = test_hsplit(a)
+                assert len(y) == len(expected_ret)
+                for mx_out, np_out in zip(y, expected_ret):
+                    assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+                mx.autograd.backward(y)
+                assert_almost_equal(a.grad.asnumpy(), _np.ones(a.shape), rtol=1e-3, atol=1e-5)
+
+                # test imperative
+                mx_outs = np.hsplit(a, indices_or_sections=indices_or_sections)
+                np_outs = _np.hsplit(a.asnumpy(), indices_or_sections=indices_or_sections)
+                for mx_out, np_out in zip(mx_outs, np_outs):
+                    assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule()
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index 41e48242b2cb..91e47ca61896 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -398,9 +398,9 @@ def check_elementwise_sum_with_shape(shape, n):
     exec1 = out.bind(default_context(),
                      args=arr,
                      args_grad=arr_grad)
-    out1 = exec1.outputs[0].asnumpy()
+
     exec1.forward(is_train=True)
-    out1 = exec1.outputs[0].asnumpy()
+    out1 = exec1.outputs[0]
     out = sum(a.asnumpy() for a  in arr)
     assert_almost_equal(out, out1, rtol=1e-5, atol=1e-5)
 
@@ -409,7 +409,7 @@ def check_elementwise_sum_with_shape(shape, n):
     # backward
     exec1.backward([out_grad])
     for a in arr_grad:
-        assert_almost_equal(a.asnumpy(), out_grad.asnumpy(), rtol=1e-5, atol=1e-5)
+        assert_almost_equal(a, out_grad, rtol=1e-5, atol=1e-5)
 
 
 @with_seed()
@@ -454,7 +454,7 @@ def check_concat_with_shape(shapes, dimension, skip_second):
     exec1.forward(is_train=True)
     out1 = exec1.outputs[0]
     ret = np.concatenate([narray.asnumpy() for narray in arr], axis=dimension)
-    assert_almost_equal(out1.asnumpy(), ret)
+    assert_almost_equal(out1, ret)
     # backward
     out1.copyto(out_grad)
     out_grad[:] += 1
@@ -464,7 +464,7 @@ def check_concat_with_shape(shapes, dimension, skip_second):
         if not skip_second or name != 'arg1':
             grad = dict_grad[name]
             np_grad = arr_np[i]
-            assert_almost_equal(grad.asnumpy(), np_grad + 1)
+            assert_almost_equal(grad, np_grad + 1)
 
 
 @with_seed()
@@ -549,18 +549,17 @@ def check_slice_channel(data_ndim, axis, num_outputs, squeeze_axis):
             gt = data_npy.take(np.arange(i * shape[axis]/num_outputs,
                                          (i+1) * shape[axis]/num_outputs).astype(np.int), axis=axis)
             if squeeze_axis:
-
-                assert_almost_equal(outputs[i].asnumpy(), gt.reshape(outputs[i].shape))
+                assert_almost_equal(outputs[i], gt.reshape(outputs[i].shape))
             else:
-                assert_almost_equal(outputs[i].asnumpy(), gt)
+                assert_almost_equal(outputs[i], gt)
         # test backward
         exe.backward(out_grads=[mx.nd.array(ele, ctx=default_context()) for ele in out_grads_npy])
         if squeeze_axis:
-            assert_almost_equal(exe.grad_arrays[0].asnumpy(),
+            assert_almost_equal(exe.grad_arrays[0],
                                 np.concatenate([np.expand_dims(ele, axis=axis) for ele in out_grads_npy],
                                                axis=axis))
         else:
-            assert_almost_equal(exe.grad_arrays[0].asnumpy(),
+            assert_almost_equal(exe.grad_arrays[0],
                                 np.concatenate(out_grads_npy, axis=axis))
     check_slice_channel(data_ndim=2, axis=1, num_outputs=3, squeeze_axis=True)
     check_slice_channel(data_ndim=4, axis=2, num_outputs=3, squeeze_axis=False)
@@ -592,8 +591,8 @@ def check_regression(symbol, forward, backward, shape, stype='default', densitie
             out_exec.backward()
             np_out = forward(arr_data.asnumpy())
             out_grad = backward(np_out, arr_label.asnumpy().reshape(np_out.shape)) / shape[1]
-            assert_almost_equal(out_exec.outputs[0].asnumpy(), np_out, atol=atol)
-            assert_almost_equal(grad_map["data"].asnumpy(), out_grad, atol=atol)
+            assert_almost_equal(out_exec.outputs[0], np_out, atol=atol)
+            assert_almost_equal(grad_map["data"], out_grad, atol=atol)
 
     shape = (50, 30)
 
@@ -716,7 +715,7 @@ def check_softmax_with_shape(shape, xpu, preserve_shape=False):
     atol = 1e-6
     assert_almost_equal(out, np_softmax(x.asnumpy()), rtol=rtol, atol=atol)
     exec1.backward()
-    assert_almost_equal(grad.asnumpy(), np_softmax(x.asnumpy()) - l.asnumpy(), rtol=rtol, atol=atol)
+    assert_almost_equal(grad, np_softmax(x.asnumpy()) - l.asnumpy(), rtol=rtol, atol=atol)
 
 
 def test_python_op():
@@ -729,9 +728,9 @@ def test_python_op():
     dy = mx.ndarray.ones((10))
     exec1 = s.bind(default_context(), args=[x], args_grad = {'X': dx})
     exec1.forward(is_train=True)
-    assert_almost_equal(x.asnumpy(), exec1.outputs[0].asnumpy())
+    assert_almost_equal(x, exec1.outputs[0])
     exec1.backward(dy)
-    assert_almost_equal(dy.asnumpy(), dx.asnumpy())
+    assert_almost_equal(dy, dx)
 
 
 def test_swapaxes():
@@ -745,7 +744,7 @@ def test_swapaxes():
     swap = mx.symbol.SwapAxis(data=swap0, dim1=1, dim2=2)
     exe_c = swap.bind(default_context(), args=[arr_data])
     exe_c.forward(is_train=True)
-    out = exe_c.outputs[0].asnumpy()
+    out = exe_c.outputs[0]
 
     swap0_ = np.swapaxes(data_tmp, 0, 2)
     swap_ = np.swapaxes(swap0_, 1, 2)
@@ -1056,7 +1055,7 @@ def test_shape_array():
         exe.backward([yg])
         yo = exe.outputs[0].asnumpy()
         same(yo, ya)
-        assert_almost_equal(xg.asnumpy(), np.zeros_like(xg.asnumpy()))
+        assert_almost_equal(xg, np.zeros_like(xg.asnumpy()))
 
 @with_seed()
 def test_size_array():
@@ -1074,7 +1073,7 @@ def test_size_array():
         exe.backward([yg])
         yo = exe.outputs[0].asnumpy()
         same(yo, ya)
-        assert_almost_equal(xg.asnumpy(), np.zeros_like(xg.asnumpy()))
+        assert_almost_equal(xg, np.zeros_like(xg.asnumpy()))
 
 @with_seed()
 def test_hard_sigmoid():
@@ -1133,7 +1132,7 @@ def _inner_test(forward_gt, logic_sym, x_shape, y_shape, test_scalar=True):
         x_npy = np.random.randint(0, 4, size=x_shape).astype(np.float32)
         y_npy = np.random.randint(0, 4, size=y_shape).astype(np.float32)
         exe = z.simple_bind(ctx=default_context(), x=x_shape, y=y_shape)
-        mx_out = exe.forward(is_train=True, x=x_npy, y=y_npy)[0].asnumpy()
+        mx_out = exe.forward(is_train=True, x=x_npy, y=y_npy)[0]
         assert_almost_equal(mx_out, forward_gt(x_npy, y_npy))
         exe.backward()
         if test_scalar:
@@ -1141,8 +1140,8 @@ def _inner_test(forward_gt, logic_sym, x_shape, y_shape, test_scalar=True):
             z_rscalar = logic_sym(x, 1)
             exe_lscalar = z_lscalar.simple_bind(ctx=default_context(), y=y_shape)
             exe_rscalar = z_rscalar.simple_bind(ctx=default_context(), x=x_shape)
-            mx_lscalar_out = exe_lscalar.forward(is_train=True, y=y_npy)[0].asnumpy()
-            mx_rscalar_out = exe_rscalar.forward(is_train=True, x=x_npy)[0].asnumpy()
+            mx_lscalar_out = exe_lscalar.forward(is_train=True, y=y_npy)[0]
+            mx_rscalar_out = exe_rscalar.forward(is_train=True, x=x_npy)[0]
             assert_almost_equal(mx_lscalar_out, forward_gt(1, y_npy))
             assert_almost_equal(mx_rscalar_out, forward_gt(x_npy, 1))
             exe_lscalar.backward()
@@ -1189,12 +1188,12 @@ def reference(a, dtype):
     xa = np.random.randint(-2, 2, size=shape).astype(np.float32)
     mx_xa = mx.nd.array(xa)
     mx_out = mx.nd.logical_not(mx_xa)
-    assert_almost_equal(mx_out.asnumpy(), reference(xa, dtype=xa.dtype))
+    assert_almost_equal(mx_out, reference(xa, dtype=xa.dtype))
     x = mx.sym.Variable('x')
     y = mx.sym.logical_not(data=x)
     exe = y.simple_bind(ctx=default_context(), x=shape)
     sym_out = exe.forward(is_train=True, x=mx_xa)[0]
-    assert_almost_equal(sym_out.asnumpy(), reference(xa, dtype=xa.dtype))
+    assert_almost_equal(sym_out, reference(xa, dtype=xa.dtype))
 
 
 @with_seed()
@@ -1219,13 +1218,13 @@ def test_embedding():
     # Non-zero atol required, as exposed by seed 781663739
     rtol = 1e-5
     atol = 1e-5
-    assert_almost_equal(exe_test.outputs[0].asnumpy(), np.dot(np_onehot, np_weight), rtol=rtol, atol=atol)
+    assert_almost_equal(exe_test.outputs[0], np.dot(np_onehot, np_weight), rtol=rtol, atol=atol)
     # backward
     np_grad = np.random.uniform(-1, 1, exe_test.outputs[0].shape)
     grad = mx.nd.zeros(np_grad.shape)
     grad[:] = np_grad
     exe_test.backward([grad])
-    assert_almost_equal(grad_map["embed_weight"].asnumpy(), np.dot(np_onehot.T, np_grad), rtol=rtol, atol=atol)
+    assert_almost_equal(grad_map["embed_weight"], np.dot(np_onehot.T, np_grad), rtol=rtol, atol=atol)
 
 
 # check ops handle duplicate input correctly.
@@ -1243,9 +1242,9 @@ def test_binary_op_duplicate_input():
     square = data * data
     exe_square = square.bind(default_context(), args=[arr_data], args_grad=[arr_grad])
     exe_square.forward(is_train=True)
-    assert_almost_equal(exe_square.outputs[0].asnumpy(), data_tmp * data_tmp)
+    assert_almost_equal(exe_square.outputs[0], data_tmp * data_tmp)
     exe_square.backward(out_grad)
-    assert_almost_equal(arr_grad.asnumpy(), 2.0 * data_tmp)
+    assert_almost_equal(arr_grad, 2.0 * data_tmp)
 
 
 @with_seed()
@@ -1261,7 +1260,7 @@ def test_sign():
     test = mx.sym.sign(data)
     exe_test = test.bind(default_context(), args=[arr_data], args_grad=[arr_grad])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout = np.sign(data_tmp)
     assert_almost_equal(out, npout)
 
@@ -1270,7 +1269,7 @@ def test_sign():
     npout_grad = out_grad.asnumpy()
     npout_grad = 0;
     exe_test.backward(out_grad)
-    assert_almost_equal(arr_grad.asnumpy(), npout_grad)
+    assert_almost_equal(arr_grad, npout_grad)
 
 
 @with_seed()
@@ -1286,7 +1285,7 @@ def test_round_ceil_floor():
     test = mx.sym.round(data) + mx.sym.ceil(data) +  mx.sym.floor(data)
     exe_test = test.bind(default_context(), args=[arr_data])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout = np.round(data_tmp) + np.ceil(data_tmp) + np.floor(data_tmp)
     assert_almost_equal(out, npout)
 
@@ -1300,7 +1299,7 @@ def test_trunc():
 
     exe_test = test.bind(default_context(), args=[arr_data])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     # 'trunc' is sensitive to the precision of the calculation.  Force numpy to match mxnet's float32.
     # Repro issue with seed 1660190454
     npout = np.trunc(np.float32(data_tmp))
@@ -1321,16 +1320,16 @@ def test_rsqrt_cos_sin():
     test =  mx.sym.rsqrt(data) + mx.sym.cos(data) + mx.sym.sin(data)
     exe_test = test.bind(default_context(), args=[arr_data], args_grad=[arr_grad])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout =  1/ np.sqrt(data_tmp) + np.cos(data_tmp) + np.sin(data_tmp)
     assert_almost_equal(out, npout)
 
     out_grad = mx.nd.empty(shape)
-    out_grad[:] = 2;
+    out_grad[:] = 2
     npout_grad = out_grad.asnumpy()
     npout_grad = npout_grad * -(1.0 / (2.0 * data_tmp * np.sqrt(data_tmp))) + npout_grad * -1 * np.sin(data_tmp) + npout_grad * np.cos(data_tmp)
     exe_test.backward(out_grad)
-    assert_almost_equal(arr_grad.asnumpy(), npout_grad)
+    assert_almost_equal(arr_grad, npout_grad)
 
 
 @with_seed()
@@ -1352,7 +1351,7 @@ def test_maximum_minimum():
     test =  mx.sym.maximum(data1,data2) + mx.sym.minimum(data1,data2)
     exe_test = test.bind(default_context(), args=[arr_data1,arr_data2], args_grad=[arr_grad1,arr_grad2])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout =  np.maximum(data_tmp1,data_tmp2) + np.minimum(data_tmp1,data_tmp2)
     assert_almost_equal(out, npout)
 
@@ -1367,8 +1366,8 @@ def test_maximum_minimum():
     npout_grad1 = npout_grad * mask1 + npout_grad * mask2
     npout_grad2 = (npout_grad - npout_grad * mask1) + (npout_grad - npout_grad * mask2)
 
-    assert_almost_equal(arr_grad1.asnumpy(), npout_grad1)
-    assert_almost_equal(arr_grad2.asnumpy(), npout_grad2)
+    assert_almost_equal(arr_grad1, npout_grad1)
+    assert_almost_equal(arr_grad2, npout_grad2)
 
 
 @with_seed()
@@ -1384,7 +1383,7 @@ def test_maximum_minimum_scalar():
     test =  mx.sym.maximum(data1,3) + mx.sym.maximum(9,data1) + mx.sym.minimum(5,data1) + mx.sym.minimum(data1,4)
     exe_test = test.bind(default_context(), args=[arr_data1], args_grad=[arr_grad1])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout =  np.maximum(data_tmp1,3) + np.maximum(9,data_tmp1) + np.minimum(5,data_tmp1) + np.minimum(data_tmp1,4)
     assert_almost_equal(out, npout)
 
@@ -1400,7 +1399,7 @@ def test_maximum_minimum_scalar():
     mask4 = (data_tmp1 < 4).astype('float')
     npout_grad1 = npout_grad * mask1 + (npout_grad - npout_grad * mask2) + (npout_grad - npout_grad * mask3) + npout_grad * mask4
 
-    assert_almost_equal(arr_grad1.asnumpy(), npout_grad1)
+    assert_almost_equal(arr_grad1, npout_grad1)
 
 
 @with_seed()
@@ -1416,7 +1415,7 @@ def test_abs():
     test = mx.sym.abs(data)
     exe_test = test.bind(default_context(), args=[arr_data], args_grad=[arr_grad])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout = abs(data_tmp)
     assert_almost_equal(out, npout)
 
@@ -1425,7 +1424,7 @@ def test_abs():
     npout_grad = out_grad.asnumpy()
     npout_grad = npout_grad * np.sign(data_tmp)
     exe_test.backward(out_grad)
-    assert_almost_equal(arr_grad.asnumpy(), npout_grad)
+    assert_almost_equal(arr_grad, npout_grad)
 
 
 def check_deconvolution_forward_backward(input_shape, num_filter, kernel, stride, pad):
@@ -1457,9 +1456,9 @@ def check_deconvolution_forward_backward(input_shape, num_filter, kernel, stride
 
     exe = deconv.bind(default_context(), args=args, args_grad=args_grad)
     exe.forward(is_train=True)
-    out = exe.outputs[0].asnumpy()
+    out = exe.outputs[0]
     exe.backward(out_grad)
-    assert_almost_equal(out, args_grad[0].asnumpy(), rtol=1E-3, atol=1e-3)
+    assert_almost_equal(out, args_grad[0], rtol=1E-3, atol=1e-3)
 
     args_grad_addto_npy = [np.random.normal(size=s) for s in arg_shapes]
     args_grad_addto = [mx.nd.array(ele) for ele in args_grad_addto_npy]
@@ -1516,7 +1515,7 @@ def check_deconvolution_gradient(input_shape, num_filter, pad):
     exe_deconv.forward(is_train=True)
     deconv_out_grad = conv_data[:]
     exe_deconv.backward(deconv_out_grad)
-    assert_almost_equal(conv_args_grad[1].asnumpy(), deconv_args_grad[1].asnumpy(), rtol=1e-3, atol=1e-2)
+    assert_almost_equal(conv_args_grad[1], deconv_args_grad[1], rtol=1e-3, atol=1e-2)
     # Test AddTo
     exe_deconv_addto = deconv.bind(default_context(), args=deconv_args,
                                    args_grad=deconv_addto_args_grad,
@@ -2104,7 +2103,7 @@ def test_depthwise_convolution():
                             exe2.backward(exe2.outputs[0])
 
                             for arr1, arr2 in zip(exe1.outputs + exe1.grad_arrays, exe2.outputs + exe2.grad_arrays):
-                                np.testing.assert_allclose(arr1.asnumpy(), arr2.asnumpy(), rtol=1e-3, atol=1e-3)
+                                assert_allclose(arr1, arr2, rtol=1e-3, atol=1e-3)
 
 
 @with_seed()
@@ -2885,9 +2884,9 @@ def test_broadcasting_ele(sym_bcast):
                                  args_grad={'a': grad_nd})
             net.forward(is_train=True)
             assert (net.outputs[0].shape == target_shape).all()
-            assert_almost_equal(net.outputs[0].asnumpy(), groundtruth, rtol=1e-4)
+            assert_almost_equal(net.outputs[0], groundtruth, rtol=1e-4)
             net.backward(out_grads=mx.nd.array(outgrad_npy))
-            assert_almost_equal(grad_nd.asnumpy(), grad_groundtruth, rtol=1e-4)
+            assert_almost_equal(grad_nd, grad_groundtruth, rtol=1e-4)
         test_broadcasting_ele(sym_bcast_axis)
         test_broadcasting_ele(sym_bcast_to)
         test_broadcasting_ele(sym_bcast_to_with_zero)
@@ -3155,13 +3154,13 @@ def test_stn():
                     grad_grad = [mx.nd.zeros(shape, ctx=dev) for shape in arg_shapes]
                     exe = stn.bind(dev, args=args, args_grad=grad_grad)
                     exe.forward(is_train=True)
-                    out = exe.outputs[0].asnumpy()
+                    out = exe.outputs[0]
                     # check forward
                     assert_almost_equal(out, args['data'].asnumpy()[:, :, h//4:h-h//4, w//4:w-w//4], rtol=1e-2, atol=1e-4)
                     out_grad = mx.nd.ones(out.shape, ctx=dev)
                     exe.backward([out_grad])
                     # check backward
-                    assert_almost_equal(out_grad.asnumpy(), grad_grad[0].asnumpy()[:, :, h//4:h-h//4, w//4:w-w//4], rtol=1e-2, atol=1e-4)
+                    assert_almost_equal(out_grad, grad_grad[0].asnumpy()[:, :, h//4:h-h//4, w//4:w-w//4], rtol=1e-2, atol=1e-4)
 
 
 def test_stn_valid_sampling():
@@ -3223,6 +3222,7 @@ def test_dot():
     # Test normal dot.
     for ndim in ndims:
         for data_type in dtypes:
+            tol = 1e-2 if data_type == 'float16' else 1e-3
             for m in range(1, 5):
                 for k in range(1, 5):
                     if ndim == 1 and k != 1:
@@ -3247,16 +3247,10 @@ def test_dot():
                         c = mx.sym.dot(a, b)
                         exe = c.simple_bind(ctx=ctx, a=a_npy.shape, b=b_npy.shape)
                         outputs = exe.forward(is_train=True, a=a_npy, b=b_npy)
-                        assert_almost_equal(outputs[0].asnumpy(), c_npy,
-                                            rtol=1e-2 if data_type == 'float16' else 1e-3,
-                                            atol=1e-2 if data_type == 'float16' else 1e-3)
+                        assert_almost_equal(outputs[0], c_npy, rtol=tol, atol=tol)
                         exe.backward(out_grads=[mx.nd.array(ograd_npy, mx.cpu()).astype(data_type)])
-                        assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy,
-                                            rtol=1e-2 if data_type == 'float16' else 1e-3,
-                                            atol=1e-2 if data_type == 'float16' else 1e-3)
-                        assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy,
-                                            rtol=1e-2 if data_type == 'float16' else 1e-3,
-                                            atol=1e-2 if data_type == 'float16' else 1e-3)
+                        assert_almost_equal(exe.grad_dict['a'], agrad_npy, rtol=tol, atol=tol)
+                        assert_almost_equal(exe.grad_dict['b'], bgrad_npy, rtol=tol, atol=tol)
 
     # Test dot with transpose flag using gradient checker.
     def dot_sym(data_type):
@@ -3293,8 +3287,9 @@ def dot_sym_xT_yT(data_type):
 
 @with_seed()
 def test_batch_dot():
+    ctx = default_context()
     dtypes = ['float32', 'float64']
-    if default_context().device_type == 'gpu':
+    if ctx.device_type == 'gpu':
         dtypes += ['float16']
 
     for data_type in dtypes:
@@ -3332,30 +3327,30 @@ def test_batch_dot():
                             b_npy = np.transpose(b_npy, axes=(0, 2, 1))
                             bgrad_npy = np.transpose(bgrad_npy, axes=(0, 2, 1))
                             b_init_grad_npy = np.transpose(b_init_grad_npy, axes=(0, 2, 1))
-                        exe = c.simple_bind(ctx=default_context(),
+                        exe = c.simple_bind(ctx=ctx,
                             a=a_npy.shape, b=b_npy.shape, grad_req='write')
-                        exe_add = c.simple_bind(ctx=default_context(),
+                        exe_add = c.simple_bind(ctx=ctx,
                             a=a_npy.shape, b=b_npy.shape, grad_req='add')
                         exe_add.grad_dict['a'][:] = a_init_grad_npy
                         exe_add.grad_dict['b'][:] = b_init_grad_npy
                         outputs = exe.forward(is_train=True, a=a_npy, b=b_npy)
-                        assert_almost_equal(outputs[0].asnumpy(), c_npy,
+                        assert_almost_equal(outputs[0], c_npy,
                                             rtol=1e-2 if data_type == 'float16' else 1e-3,
                                             atol=1e-2 if data_type == 'float16' else 1e-4)
                         exe.backward(out_grads=[mx.nd.array(ograd_npy, ctx=exe._ctx)])
-                        assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy,
+                        assert_almost_equal(exe.grad_dict['a'], agrad_npy,
                                             rtol=1e-2 if data_type == 'float16' else 1e-3,
                                             atol=1e-2 if data_type == 'float16' else 1e-4)
-                        assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy,
+                        assert_almost_equal(exe.grad_dict['b'], bgrad_npy,
                                             rtol=1e-2 if data_type == 'float16' else 1e-3,
                                             atol=1e-2 if data_type == 'float16' else 1e-4)
                         exe_add.forward(is_train=True, a=a_npy, b=b_npy)
                         exe_add.backward(out_grads=[mx.nd.array(ograd_npy, ctx=exe._ctx)])
-                        assert_almost_equal(exe_add.grad_dict['a'].asnumpy(),
+                        assert_almost_equal(exe_add.grad_dict['a'],
                                             agrad_npy + a_init_grad_npy,
                                             rtol=1e-2 if data_type == 'float16' else 1e-3,
                                             atol=1e-2 if data_type == 'float16' else 1e-4)
-                        assert_almost_equal(exe_add.grad_dict['b'].asnumpy(),
+                        assert_almost_equal(exe_add.grad_dict['b'],
                                             bgrad_npy + b_init_grad_npy,
                                             rtol=1e-2 if data_type == 'float16' else 1e-3,
                                             atol=1e-2 if data_type == 'float16' else 1e-4)
@@ -3487,7 +3482,7 @@ def unittest_correlation(data_shape,kernel_size,max_displacement,stride1,stride2
     forward_result,tmp1,tmp2 = correlation_forward(img1,img2,pad_size,kernel_size,stride1,stride2,max_displacement,is_multiply)
 
     # forward error
-    assert_almost_equal(exe1.outputs[0].asnumpy(), forward_result, rtol=1e-4, atol=1e-4)
+    assert_almost_equal(exe1.outputs[0], forward_result, rtol=1e-4, atol=1e-4)
 
     # out_grad
     a = np.ones(forward_result.shape)
@@ -3498,8 +3493,8 @@ def unittest_correlation(data_shape,kernel_size,max_displacement,stride1,stride2
     grad1,grad2 = correlation_backward(a,tmp1,tmp2,img1,img2,pad_size,kernel_size,stride1,stride2,max_displacement,is_multiply)
 
     # backward error
-    assert_almost_equal(exe1.grad_dict['img1'].asnumpy(), grad1, rtol=1e-3, atol=1e-4)
-    assert_almost_equal(exe1.grad_dict['img2'].asnumpy(), grad2, rtol=1e-3, atol=1e-4)
+    assert_almost_equal(exe1.grad_dict['img1'], grad1, rtol=1e-3, atol=1e-4)
+    assert_almost_equal(exe1.grad_dict['img2'], grad2, rtol=1e-3, atol=1e-4)
 
 
 @with_seed()
@@ -3557,7 +3552,7 @@ def test_support_vector_machine_l1_svm():
     exec1 = Y.bind(xpu, args = [x, l], args_grad = {'X': grad})
     exec1.forward(is_train=True)
 
-    assert_almost_equal(x_np, exec1.outputs[0].asnumpy())
+    assert_almost_equal(x_np, exec1.outputs[0])
 
     exec1.backward()
 
@@ -3565,7 +3560,7 @@ def test_support_vector_machine_l1_svm():
     l_mask = np.array(l_mask, dtype=np.float32)*2 -1
     grad_np = (-1) * l_mask * np.greater(1 - l_mask * x_np, 0)
 
-    assert_almost_equal(grad_np, grad.asnumpy())
+    assert_almost_equal(grad_np, grad)
 
 
 @with_seed()
@@ -3588,7 +3583,7 @@ def test_support_vector_machine_l2_svm():
     exec1 = Y.bind(xpu, args = [x, l], args_grad = {'X': grad})
     exec1.forward(is_train=True)
 
-    assert_almost_equal(x_np, exec1.outputs[0].asnumpy())
+    assert_almost_equal(x_np, exec1.outputs[0])
 
     exec1.backward()
 
@@ -3596,7 +3591,7 @@ def test_support_vector_machine_l2_svm():
     l_mask = np.array(l_mask, dtype=np.float32)*2 -1
     grad_np = (-2)*l_mask*np.maximum(1-l_mask*x_np,0)
     grad_np = grad_np.astype(np.float32)
-    assert_almost_equal(grad_np, grad.asnumpy())
+    assert_almost_equal(grad_np, grad)
 
 
 # Seed set because the test is not robust enough to operate on random data
@@ -3630,7 +3625,7 @@ def check_pad_with_shape(shape, xpu, pad_width, mode, dtype="float64"):
     grad = mx.nd.empty(shape, ctx = xpu, dtype=dtype)
     exec1 = Y.bind(xpu, args = [x], args_grad = {'X': grad})
     exec1.forward(is_train=True)
-    out = exec1.outputs[0].asnumpy()
+    out = exec1.outputs[0]
     # compare numpy + mxnet
     assert_almost_equal(out, np_out)
     # grad check
@@ -3687,7 +3682,7 @@ def check_instance_norm_with_shape(shape, xpu):
     np_out = np_instance_norm(x.asnumpy(), gamma.asnumpy(), beta.asnumpy(), eps)
     exec1 = Y.bind(xpu, args = {'X':x, 'G':gamma, 'B':beta})
     exec1.forward(is_train=False)
-    out = exec1.outputs[0].asnumpy()
+    out = exec1.outputs[0]
     assert_almost_equal(out, np_out, rtol=1e-4, atol=1e-4)
     check_numeric_gradient(Y, {'X':x.asnumpy(), 'G':gamma.asnumpy(), 'B':beta.asnumpy()},
                            numeric_eps=1e-2, rtol=1e-2, atol=1e-2)
@@ -3729,7 +3724,7 @@ def check_l2_normalization(in_shape, mode, dtype, norm_eps=1e-10):
     exe = out.simple_bind(ctx=ctx, data=in_data.shape)
     output = exe.forward(is_train=True, data=in_data)
     # compare numpy + mxnet
-    assert_almost_equal(exe.outputs[0].asnumpy(), np_out, rtol=1e-2 if dtype is 'float16' else 1e-5, atol=1e-5)
+    assert_almost_equal(exe.outputs[0], np_out, rtol=1e-2 if dtype is 'float16' else 1e-5, atol=1e-5)
     # check gradient
     check_numeric_gradient(out, [in_data], numeric_eps=1e-3, rtol=1e-2, atol=5e-3)
 
@@ -3793,7 +3788,7 @@ def npy_layer_norm_grad(data, gamma, out_grad, axis, eps):
     exe.arg_dict['beta'][:] = beta
     out_nd = exe.forward()[0]
     out = npy_layer_norm(data, gamma, beta, axis, eps)
-    assert_almost_equal(out, out_nd.asnumpy(), forward_check_eps, forward_check_eps)
+    assert_almost_equal(out, out_nd, forward_check_eps, forward_check_eps)
 
     if finite_grad_check:
         for req in ['write', 'add']:
@@ -4133,7 +4128,7 @@ def mathematical_core_binary(name,
     test = forward_mxnet_call(data1, data2)
     exe_test = test.bind(default_context(), args=[arr_data1, arr_data2], args_grad=[arr_grad1, arr_grad2])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout = forward_numpy_call(data_tmp1, data_tmp2)
     assert_almost_equal(out, npout)
 
@@ -4146,8 +4141,6 @@ def mathematical_core_binary(name,
 
     npout_grad1 = npout_grad * backward_numpy_call1(data_tmp1, data_tmp2)
     npout_grad2 = npout_grad * backward_numpy_call2(data_tmp1, data_tmp2)
-    arr_grad1 = arr_grad1.asnumpy()
-    arr_grad2 = arr_grad2.asnumpy()
 
     assert_almost_equal(arr_grad1, npout_grad1)
     assert_almost_equal(arr_grad2, npout_grad2)
@@ -4165,7 +4158,7 @@ def mathematical_core(name, forward_mxnet_call, forward_numpy_call, backward_num
     test = forward_mxnet_call(data)
     exe_test = test.bind(default_context(), args=[arr_data], args_grad=[arr_grad])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout = forward_numpy_call(data_tmp)
     assert_almost_equal(out, npout)
 
@@ -4175,10 +4168,6 @@ def mathematical_core(name, forward_mxnet_call, forward_numpy_call, backward_num
     temp = backward_numpy_call(data_tmp)
     npout_grad = npout_grad * temp
     exe_test.backward(out_grad)
-    arr_grad = arr_grad.asnumpy()
-    # print(name)
-    # print(arr_grad)
-    # print(npout_grad)
     assert_almost_equal(arr_grad, npout_grad)
 
 
@@ -4217,7 +4206,7 @@ def rounding(name, forward_mxnet_call, forward_numpy_call, data_init=5., grad_in
     test = forward_mxnet_call(data)
     exe_test = test.bind(default_context(), args=[arr_data])
     exe_test.forward(is_train=True)
-    out = exe_test.outputs[0].asnumpy()
+    out = exe_test.outputs[0]
     npout = forward_numpy_call(data_tmp)
     assert_almost_equal(out, npout)
 
@@ -4355,7 +4344,7 @@ def test_basic_val_init(sym_func, np_func, shape, dtype):
         x = sym_func(shape=shape, dtype=dtype)
         exe = x.bind(default_context(), args=[], args_grad=[])
         exe.forward(is_train=True)
-        assert_almost_equal(exe.outputs[0].asnumpy(), np_func(shape=shape, dtype=dtype))
+        assert_almost_equal(exe.outputs[0], np_func(shape=shape, dtype=dtype))
         assert exe.outputs[0].asnumpy().dtype == dtype
 
     def test_arange():
@@ -4372,14 +4361,14 @@ def test_arange():
                 repeats = random.choice([1, 3])
                 np_out = np.repeat(np.arange(*config, dtype=dtype), repeats)
                 nd_out = mx.nd.arange(*config, repeat=repeats, dtype=dtype)
-                assert_almost_equal(np_out, nd_out.asnumpy())
+                assert_almost_equal(np_out, nd_out)
 
     def test_arange_inferstop():
         s = mx.sym.arange(start=0, stop=None, infer_range=True)
         s = mx.sym.elemwise_add(s, mx.sym.zeros(shape=[5]))
         exe = s.bind(ctx=mx.cpu(), args={})
         exe.forward()
-        assert_almost_equal(exe.outputs[0].asnumpy(), np.array([0,1,2,3,4]))
+        assert_almost_equal(exe.outputs[0], np.array([0,1,2,3,4]))
 
     def test_arange_like():
         shape_list = [(10,), (10, 20), (10, 20, 30), (10, 20, 30, 40)]
@@ -4549,7 +4538,7 @@ def test_blockgrad():
     exe = b.simple_bind(ctx=default_context(), a=(10, 10))
     a_npy = np.random.rand(10, 10)
     exe.forward(is_train=True, a=a_npy)
-    assert_almost_equal(exe.outputs[0].asnumpy(), a_npy)
+    assert_almost_equal(exe.outputs[0], a_npy)
     exe.backward()  # No error if BlockGrad works
 
 
@@ -4614,7 +4603,7 @@ def check_output_n_grad(data_shape, idx_shape, axis, mode, out_of_range=True):
                 # Did not raise exception
                 assert False, "did not raise %s" % MXNetError.__name__
 
-        assert_almost_equal(exe.outputs[0].asnumpy(), np.take(data_real, idx_real, axis=axis, mode=mode))
+        assert_almost_equal(exe.outputs[0], np.take(data_real, idx_real, axis=axis, mode=mode))
 
         for i in np.nditer(idx_real):
             if mode == 'clip':
@@ -4622,7 +4611,7 @@ def check_output_n_grad(data_shape, idx_shape, axis, mode, out_of_range=True):
             grad_helper(grad_in, axis, i)
 
         exe.backward([mx.nd.array(grad_out)])
-        assert_almost_equal(exe.grad_dict['a'].asnumpy(), grad_in)
+        assert_almost_equal(exe.grad_dict['a'], grad_in)
 
     def check_autograd_req():
         row_len = 2
@@ -4647,7 +4636,7 @@ def check_autograd_req():
             x = x.sum()
 
         x.backward()
-        assert_almost_equal(np.ones(sc.grad.shape), sc.grad.asnumpy())
+        assert_almost_equal(np.ones(sc.grad.shape), sc.grad)
 
     for mode in ['clip', 'wrap', 'raise']:
         for data_ndim in range(1, 5):
@@ -4693,7 +4682,7 @@ def test_grid_generator():
         tmp[1] = -1.0 + (np.arange(target_shape[0]*target_shape[1]) // target_shape[1]) * (2.0 / (target_shape[0]-1))
         tmp[2] = 1
         grad_est = np.dot(out_grad[0].reshape(2,target_shape[0]*target_shape[1]),tmp.T).reshape(1,6)
-        assert_almost_equal(exe.grad_dict['affine'].asnumpy(), grad_est, rtol=1e-3, atol=1e-5)
+        assert_almost_equal(exe.grad_dict['affine'], grad_est, rtol=1e-3, atol=1e-5)
         # check addto
         exe = grid.simple_bind(ctx=default_context(), affine=(1,6), grad_req='add')
         grid_grad_npy = np.random.normal(size=exe.grad_dict['affine'].shape)
@@ -4701,7 +4690,7 @@ def test_grid_generator():
         exe.arg_dict['affine'][:] = np.array([[1.0, 0, 0, 0, 1.0, 0]])
         exe.forward(is_train=True)
         exe.backward(mx.nd.array(out_grad))
-        assert_almost_equal(exe.grad_dict['affine'].asnumpy(), grad_est + grid_grad_npy, rtol=1e-2, atol=1e-5)
+        assert_almost_equal(exe.grad_dict['affine'], grad_est + grid_grad_npy, rtol=1e-2, atol=1e-5)
 
     # transform_type = warp
     test_case = [(12,21),(4,3),(6,12)]
@@ -4724,7 +4713,7 @@ def test_grid_generator():
         grad_est = np.zeros((1,2)+target_shape)
         grad_est[0,0] = out_grad[0,0] / ((target_shape[1]-1.0) / 2.0)
         grad_est[0,1] = out_grad[0,1] / ((target_shape[0]-1.0) / 2.0)
-        assert_almost_equal(exe.grad_dict['flow'].asnumpy(), grad_est, rtol=1e-3)
+        assert_almost_equal(exe.grad_dict['flow'], grad_est, rtol=1e-3)
         # check addto
         exe_add = grid.simple_bind(ctx=default_context(), flow=(1, 2) + target_shape, grad_req='add')
         flow_grad_npy = np.random.normal(size=exe_add.grad_dict['flow'].shape)
@@ -4732,7 +4721,7 @@ def test_grid_generator():
         exe_add.grad_dict['flow'][:] = flow_grad_npy
         exe_add.forward(is_train=True)
         exe_add.backward(mx.nd.array(out_grad))
-        assert_almost_equal(exe_add.grad_dict['flow'].asnumpy(), grad_est + flow_grad_npy, rtol=1e-3, atol=1e-5)
+        assert_almost_equal(exe_add.grad_dict['flow'], grad_est + flow_grad_npy, rtol=1e-3, atol=1e-5)
 
 
 @with_seed()
@@ -4743,7 +4732,7 @@ def test_index2d():
         data = mx.random.uniform(-1, 1, shape=(n, m), ctx=default_context())
         x = mx.nd.array(np.random.randint(0, m, size=n), ctx=default_context(), dtype='int32')
         r = mx.nd.batch_take(data, x)
-        assert_almost_equal(r.asnumpy(), data.asnumpy()[np.arange(n), x.asnumpy()])
+        assert_almost_equal(r, data.asnumpy()[np.arange(n), x.asnumpy()])
 
 
 @with_seed()
@@ -4759,8 +4748,8 @@ def test_cast():
             exe.arg_arrays[0][:] = X
             exe.forward(is_train=True)
             exe.backward(mx.nd.array(X, dtype=dsttype, ctx=default_context()))
-            assert_almost_equal(exe.outputs[0].asnumpy(), X.astype(srctype).astype(dsttype), rtol=1e-3, atol=1e-5)
-            assert_almost_equal(exe.grad_arrays[0].asnumpy(), X.astype(dsttype).astype(srctype), rtol=1e-3, atol=1e-5)
+            assert_almost_equal(exe.outputs[0], X.astype(srctype).astype(dsttype), rtol=1e-3, atol=1e-5)
+            assert_almost_equal(exe.grad_arrays[0], X.astype(dsttype).astype(srctype), rtol=1e-3, atol=1e-5)
 
 def get_cast_op_data():
     FP16_FRACTION_BITS = 10
@@ -4888,12 +4877,12 @@ def test_repeat_forward():
             a = np.random.random_sample(size=shape)
             aa = np.repeat(a, repeats)
             b = mx.nd.array(a, ctx=default_context())
-            bb = mx.nd.repeat(b, repeats).asnumpy()
+            bb = mx.nd.repeat(b, repeats)
             assert_almost_equal(aa, bb)
 
             for axis in range(0, ndim):
                 aa = np.repeat(a, repeats, axis)
-                bb = mx.nd.repeat(b, repeats, axis).asnumpy()
+                bb = mx.nd.repeat(b, repeats, axis)
                 assert_almost_equal(aa, bb)
 
     def test_repeat_backward(axis):
@@ -4931,7 +4920,7 @@ def test_repeat_backward(axis):
         else:
             raise RuntimeError("Invalid axis value")
 
-        assert_almost_equal(expected_grad, arr_grad.asnumpy(), rtol=1e-3)
+        assert_almost_equal(expected_grad, arr_grad, rtol=1e-3)
 
     def test_repeat_numeric_gradient():
         data = mx.sym.Variable('data')
@@ -5028,7 +5017,7 @@ def test_tile_backward():
             for j in range(shape[1]):
                 expected_grad[i][j] += sum(sum(npout_grad[i:(n1 * reps1):reps1, j:(n2 * reps2):reps2]))
 
-        assert_almost_equal(expected_grad, arr_grad.asnumpy(), rtol=1e-3)
+        assert_almost_equal(expected_grad, arr_grad, rtol=1e-3)
 
     def test_tile_numeric_gradient():
         data = mx.sym.Variable('data')
@@ -5344,8 +5333,7 @@ def softmax_forward(input_data, true_output):
         exec1 = out1.bind(default_context(), args={'data': input_data})
         exec1.forward()[0].wait_to_read()
         ndarr = exec1.outputs[0][0][0][0]
-        nparr = ndarr.asnumpy()
-        assert_almost_equal(nparr, true_output, rtol=1e-5, atol=1e-5)
+        assert_almost_equal(ndarr, true_output, rtol=1e-5, atol=1e-5)
 
     softmax_forward(mx.nd.array([[[[-1e30,-1e30]]]]), np.array([1.0,1.0]))
     softmax_forward(mx.nd.array([[[[1e30,1e30]]]]), np.array([1.0,1.0]))
@@ -5367,14 +5355,10 @@ def check_dtypes_almost_equal(op_name,
         with mx.autograd.record():
             dtype_softmax = op(dtype_input, axis=-1, dtype=odtype)
             ref_softmax = op(ref_input, axis=-1, dtype=odtype)
-        dtype_softmax_np = dtype_softmax.asnumpy()
-        ref_softmax_np = ref_softmax.asnumpy()
-        assert_almost_equal(dtype_softmax_np, ref_softmax_np, rtol=rtol, atol=atol)
+        assert_almost_equal(dtype_softmax, ref_softmax, rtol=rtol, atol=atol)
         dtype_softmax.backward()
         ref_softmax.backward()
-        dtype_grad_np = dtype_input.grad.asnumpy()
-        ref_grad_np = ref_input.grad.asnumpy()
-        assert_almost_equal(dtype_grad_np, ref_grad_np, rtol=grad_rtol, atol=grad_atol)
+        assert_almost_equal(dtype_input.grad, ref_input.grad, rtol=grad_rtol, atol=grad_atol)
 
     import sys
     is_windows = sys.platform.startswith('win')
@@ -5474,48 +5458,28 @@ def test_pick_helper(index_type=np.int32):
     test_pick_helper(np.float32)
 
 
-def check_ctc_loss(acts, labels, loss_truth):
+def check_ctc_loss(acts, labels, loss_truth, contrib=False):
     in_var = mx.sym.Variable('input')
     labels_var = mx.sym.Variable('labels')
-    ctc = mx.sym.ctc_loss(in_var, labels_var)
-    acts_nd = mx.nd.array(acts, ctx=default_context())
-    labels_nd = mx.nd.array(labels, ctx=default_context())
-    exe = ctc.bind(ctx=default_context(), args=[acts_nd, labels_nd])
-    # test forward with grad calc
-    exe.forward(is_train=True)
-    outTest = exe.outputs[0]
-    # test forward without grad calc
-    exe.forward(is_train=False)
-    outTrain = exe.outputs[0]
-    # make sure losses calculated with both modes are the same
-    assert_almost_equal(outTest.asnumpy(), outTrain.asnumpy())
-
-    # test against ground truth, if available
-    if loss_truth is not None:
-        assert_almost_equal(outTest.asnumpy(), loss_truth)
-    # test grad
-    check_numeric_gradient(ctc, [acts, labels], grad_nodes=['input'], rtol=0.05, atol=1e-3)
-
-# check contrib operator for backward compatibility
-def check_contrib_ctc_loss(acts, labels, loss_truth):
-    in_var = mx.sym.Variable('input')
-    labels_var = mx.sym.Variable('labels')
-    ctc = mx.sym.contrib.ctc_loss(in_var, labels_var)
+    if contrib:
+        ctc = mx.sym.contrib.ctc_loss(in_var, labels_var)
+    else:
+        ctc = mx.sym.ctc_loss(in_var, labels_var)
     acts_nd = mx.nd.array(acts, ctx=default_context())
     labels_nd = mx.nd.array(labels, ctx=default_context())
     exe = ctc.bind(ctx=default_context(), args=[acts_nd, labels_nd])
     # test forward with grad calc
     exe.forward(is_train=True)
-    outTest = exe.outputs[0]
+    outTest = exe.outputs[0].copy()
     # test forward without grad calc
     exe.forward(is_train=False)
     outTrain = exe.outputs[0]
     # make sure losses calculated with both modes are the same
-    assert_almost_equal(outTest.asnumpy(), outTrain.asnumpy())
+    assert_almost_equal(outTest, outTrain)
 
     # test against ground truth, if available
     if loss_truth is not None:
-        assert_almost_equal(outTest.asnumpy(), loss_truth)
+        assert_almost_equal(outTest, loss_truth)
     # test grad
     check_numeric_gradient(ctc, [acts, labels], grad_nodes=['input'], rtol=0.05, atol=1e-3)
 
@@ -5529,8 +5493,9 @@ def test_ctc_loss():
                     dtype=np.float32)
     labels = np.array([[2, 3, 0], [2, 3, 0]])
     true_loss = np.array([4.04789, 4.04789], dtype=np.float32) # from Torch
-    check_ctc_loss(acts, labels, true_loss)
-    check_contrib_ctc_loss(acts, labels, true_loss)
+    for contrib in [False, True]:
+        check_ctc_loss(acts, labels, true_loss, contrib=contrib)
+
 
     # Test 2:
     acts2 = np.array([
@@ -5539,14 +5504,14 @@ def test_ctc_loss():
         [[-15, -14, -13, -12, -11], [-15, -14.2, -13.5, -12.2, -11.22]]], dtype=np.float32)
     labels2 = np.array([[2, 3, 1], [2, 0, 0]], dtype=np.float32)
     true_loss = np.array([7.3557, 5.4091], dtype=np.float32) # from Torch
-    check_ctc_loss(acts2, labels2, true_loss)
-    check_contrib_ctc_loss(acts2, labels2, true_loss)
+    for contrib in [False, True]:
+        check_ctc_loss(acts2, labels2, true_loss, contrib=contrib)
 
     # Test 3: check use integer type as label
     labels3 = np.array([[2, 3, 1], [2, 0, 0]], dtype=np.int32)
     true_loss = np.array([7.3557, 5.4091], dtype=np.float32) # from Torch
-    check_ctc_loss(acts2, labels3, true_loss)
-    check_contrib_ctc_loss(acts2, labels3, true_loss)
+    for contrib in [False, True]:
+        check_ctc_loss(acts2, labels3, true_loss, contrib=contrib)
 
 @with_seed()
 def test_ctc_loss_with_large_classes():
@@ -5566,90 +5531,11 @@ def test_ctc_loss_with_large_classes():
     nd_label = mx.nd.array(label)
     loss = mx.nd.ctc_loss(data=nd_data, label=nd_label)
     expected_loss = np.array([688.02826, 145.34462])
-    assert_almost_equal(loss.asnumpy(), expected_loss)
+    assert_almost_equal(loss, expected_loss)
 
 @with_seed()
 def test_ctc_loss_grad():
-    def check_ctc_loss_grad(blank_label): # from tf
-        vocab_size = 5
-        max_label_len = 5
-        padding_mask = -1+ (blank_label=='first')
-
-        targets_0 = [0, 1, 2, 1, 0]
-        loss_log_prob_0 = -3.34211
-        input_prob_matrix_0 = np.asarray(
-            [[0.633766, 0.221185, 0.0917319, 0.0129757, 0.0142857, 0.0260553],
-             [0.111121, 0.588392, 0.278779, 0.0055756, 0.00569609, 0.010436],
-             [0.0357786, 0.633813, 0.321418, 0.00249248, 0.00272882, 0.0037688],
-             [0.0663296, 0.643849, 0.280111, 0.00283995, 0.0035545, 0.00331533],
-             [0.458235, 0.396634, 0.123377, 0.00648837, 0.00903441, 0.00623107]],
-            dtype=np.float32)
-        gradient_log_prob_0 = np.asarray(
-            [[-0.366234, 0.221185, 0.0917319, 0.0129757, 0.0142857, 0.0260553],
-             [0.111121, -0.411608, 0.278779, 0.0055756, 0.00569609, 0.010436],
-             [0.0357786, 0.633813, -0.678582, 0.00249248, 0.00272882, 0.0037688],
-             [0.0663296, -0.356151, 0.280111, 0.00283995, 0.0035545, 0.00331533],
-             [-0.541765, 0.396634, 0.123377, 0.00648837, 0.00903441, 0.00623107]],
-            dtype=np.float32)
-
-        targets_1 = [0, 1, 1, 0]
-        loss_log_prob_1 = -5.42262
-        input_prob_matrix_1 = np.asarray(
-            [[0.30176, 0.28562, 0.0831517, 0.0862751, 0.0816851, 0.161508],
-             [0.24082, 0.397533, 0.0557226, 0.0546814, 0.0557528, 0.19549],
-             [0.230246, 0.450868, 0.0389607, 0.038309, 0.0391602, 0.202456],
-             [0.280884, 0.429522, 0.0326593, 0.0339046, 0.0326856, 0.190345],
-             [0.423286, 0.315517, 0.0338439, 0.0393744, 0.0339315, 0.154046]],
-            dtype=np.float32)
-        gradient_log_prob_1 = np.asarray(
-            [[-0.69824, 0.28562, 0.0831517, 0.0862751, 0.0816851, 0.161508],
-             [0.24082, -0.602467, 0.0557226, 0.0546814, 0.0557528, 0.19549],
-             [0.230246, 0.450868, 0.0389607, 0.038309, 0.0391602, -0.797544],
-             [0.280884, -0.570478, 0.0326593, 0.0339046, 0.0326856, 0.190345],
-             [-0.576714, 0.315517, 0.0338439, 0.0393744, 0.0339315, 0.154046]],
-            dtype=np.float32)
-
-        inputs = [
-            np.vstack(
-                [input_prob_matrix_0[t, :], input_prob_matrix_1[t, :]])
-            for t in range(5)
-        ] + 2 * [np.nan * np.ones((2, vocab_size+1), np.float32)]
-        inputs = np.log(np.asarray(inputs, dtype=np.float32))
-
-        grad_truth = np.array([
-            np.vstack(
-                [gradient_log_prob_0[t, :], gradient_log_prob_1[t, :]])
-            for t in range(5)
-        ] + 2 * [np.zeros((2, vocab_size+1), np.float32)])
-
-        if blank_label == 'first':
-            inputs = np.roll(inputs, 1, axis=2)
-            grad_truth = np.roll(grad_truth, 1, axis=2)
-
-        labels = (np.asarray([x + [padding_mask]*(max_label_len-len(x))
-                             for x in [targets_0, targets_1]])+(blank_label == 'first'))
-
-        seq_lens = np.array([5, 5], dtype=np.int32)
-        label_lens = np.array([5, 4], dtype=np.int32)
-        loss_truth = np.array([-loss_log_prob_0, -loss_log_prob_1], np.float32)
-
-        with default_context():
-            data = mx.nd.array(inputs)
-            label = mx.nd.array(labels)
-            data.attach_grad()
-            with mx.autograd.record():
-                l = mx.ndarray.CTCLoss(data, label,
-                                       use_data_lengths=True,
-                                       use_label_lengths=True,
-                                       data_lengths=mx.nd.array(seq_lens),
-                                       label_lengths=mx.nd.array(label_lens),
-                                       blank_label=blank_label)
-                l.backward()
-            assert_almost_equal(l.asnumpy(), loss_truth, atol=1e-5, rtol=1e-5)
-            assert_almost_equal(data.grad.asnumpy(), grad_truth, atol=1e-5, rtol=1e-5)
-
-    # check contrib operator for backward compatibility
-    def check_contrib_ctc_loss_grad(blank_label): # from tf
+    def check_ctc_loss_grad(blank_label, contrib=False): # from tf
         vocab_size = 5
         max_label_len = 5
         padding_mask = -1+ (blank_label=='first')
@@ -5717,22 +5603,28 @@ def check_contrib_ctc_loss_grad(blank_label): # from tf
             label = mx.nd.array(labels)
             data.attach_grad()
             with mx.autograd.record():
-                l = mx.contrib.ndarray.CTCLoss(data, label,
-                                               use_data_lengths=True,
-                                               use_label_lengths=True,
-                                               data_lengths=mx.nd.array(seq_lens),
-                                               label_lengths=mx.nd.array(label_lens),
-                                               blank_label=blank_label)
+                if contrib:
+                    l = mx.contrib.ndarray.CTCLoss(data, label,
+                                           use_data_lengths=True,
+                                           use_label_lengths=True,
+                                           data_lengths=mx.nd.array(seq_lens),
+                                           label_lengths=mx.nd.array(label_lens),
+                                           blank_label=blank_label)
+                else:
+                    l = mx.ndarray.CTCLoss(data, label,
+                                           use_data_lengths=True,
+                                           use_label_lengths=True,
+                                           data_lengths=mx.nd.array(seq_lens),
+                                           label_lengths=mx.nd.array(label_lens),
+                                           blank_label=blank_label)
                 l.backward()
-            assert_almost_equal(l.asnumpy(), loss_truth, atol=1e-5, rtol=1e-5)
-            assert_almost_equal(data.grad.asnumpy(), grad_truth, atol=1e-5, rtol=1e-5)
 
+            assert_almost_equal(l, loss_truth, atol=1e-5, rtol=1e-5)
+            assert_almost_equal(data.grad, grad_truth, atol=1e-5, rtol=1e-5)
 
-    check_ctc_loss_grad('first')
-    check_ctc_loss_grad('last')
-    check_contrib_ctc_loss_grad('first')
-    check_contrib_ctc_loss_grad('last')
-
+    for contrib in [False, True]:
+        for label in ['first', 'last']:
+            check_ctc_loss_grad(label, contrib=contrib)
 
 @with_seed()
 def test_quantization_op():
@@ -5971,8 +5863,8 @@ def create_operator(self, ctx, shapes, dtypes):
     expected_grad = 2 * x2
     rtol = 1e-4
     atol = 1e-6
-    assert_almost_equal(output.asnumpy(), expected_output.asnumpy(), rtol=rtol, atol=atol)
-    assert_almost_equal(x2.grad.asnumpy(), expected_grad.asnumpy(), rtol=rtol, atol=atol)
+    assert_almost_equal(output, expected_output, rtol=rtol, atol=atol)
+    assert_almost_equal(x2.grad, expected_grad, rtol=rtol, atol=atol)
 
 
     # test for backward compatibility, i.e. the correctness of default implementation of
@@ -6009,8 +5901,8 @@ def create_operator(self, ctx, shapes, dtypes):
     with mx.autograd.record():
         y = mx.nd.Custom(lhs, rhs, name='mult', op_type='mult')
         y.backward()
-    assert_almost_equal(rhs.asnumpy(), lhs.grad.asnumpy(), rtol=rtol, atol=atol)
-    assert_almost_equal(lhs.asnumpy(), rhs.grad.asnumpy(), rtol=rtol, atol=atol)
+    assert_almost_equal(rhs, lhs.grad, rtol=rtol, atol=atol)
+    assert_almost_equal(lhs, rhs.grad, rtol=rtol, atol=atol)
 
     class MultNoGrad(mx.operator.CustomOp):
         def forward(self, is_train, req, in_data, out_data, aux):
@@ -6043,8 +5935,8 @@ def infer_storage_type_backward(self, ograd_stype, in_stype, out_stype, igrad_st
     with mx.autograd.record():
         y2 = mx.nd.Custom(lhs, rhs, name="mult_no_grad", op_type="mult_no_grad")
         y2.backward()
-    assert_almost_equal(rhs.asnumpy(), lhs.grad.asnumpy(), rtol=rtol, atol=atol)
-    assert_almost_equal(lhs.asnumpy(), rhs.grad.asnumpy(), rtol=rtol, atol=atol)
+    assert_almost_equal(rhs, lhs.grad, rtol=rtol, atol=atol)
+    assert_almost_equal(lhs, rhs.grad, rtol=rtol, atol=atol)
 
     class NoInputOp(mx.operator.CustomOp):
         def __init__(self, length, depth):
@@ -6081,7 +5973,7 @@ def create_operator(self, ctx, shapes, dtypes):
 
     with mx.autograd.record():
         x = mx.nd.Custom(length=10, depth=10, op_type="no_input_op")
-    assert_almost_equal(x.asnumpy(), np.ones(shape=(10, 10), dtype=np.float32))
+    assert_almost_equal(x, np.ones(shape=(10, 10), dtype=np.float32))
 
 
 @with_seed()
@@ -6558,95 +6450,83 @@ def test_laop():
     dtype = np.float64
     rtol_fw = 1e-7
     atol_fw = 1e-9
-    num_eps = 1e-6
+    num_eps = 2e-6
     rtol_bw = 1e-5
-    atol_bw = 1e-6
+    atol_bw = 1e-5
     # enable numerical checking of gradients
     grad_check = 1
 
     data1 = mx.symbol.Variable('data1')
     data2 = mx.symbol.Variable('data2')
-    data3 = mx.symbol.Variable('data3')
 
-    check_fw = lambda sym, location, expected :\
-        check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
-                               atol=atol_fw, dtype=dtype)
-    check_grad = lambda sym, location:\
-        check_numeric_gradient(sym, location, numeric_eps=num_eps, rtol=rtol_bw,
-                               atol=atol_bw, dtype=dtype)
     rep_3x = lambda a, m, n :\
         np.reshape(np.tile(np.array(a).flatten(), 3), (3, 1, m, n))
 
+    def check_fw_grad(sym, location, expected):
+        check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
+                               atol=atol_fw, dtype=dtype)
+        if grad_check == 1:
+            check_numeric_gradient(sym, location, numeric_eps=num_eps, rtol=rtol_bw,
+                                   atol=atol_bw, dtype=dtype)
+
+    matrix = np.array([[9., 3., -6., 12.],
+                       [3., 26., -7., -11.],
+                       [-6., -7., 9., 7.],
+                       [12., -11., 7., 65.]])
+    trian  = np.array([[3., 0., 0., 0.],
+                       [1., 5., 0., 0.],
+                       [-2., -1., 2., 0.],
+                       [4., -3., 6., 2.]])
+    pow    = np.array([[2., 1., 1., 1.],
+                       [1., 4., 1., 1.],
+                       [1., 1., 8., 1.],
+                       [1., 1., 1., 16.]])
+    inv    = np.array([[8.95/3., 0.05/3., 2.65, -2.5/3.],
+                       [0.05/3., 0.05, 0.05, 0.],
+                       [2.65, 0.05, 2.5, -0.75],
+                       [-2.5/3., 0., -0.75, 0.25]])
+    ident  = np.eye(4)
+    shape = (4, 4, 1, 1)
+    ones = mx.nd.ones(shape).asnumpy()
+
     for lower in [True, False]:
         upper = not lower
 
         # Tests with trivial 1x1 matrices.
-        shape = (4, 4, 1, 1)
         data_in = np.random.uniform(1, 10, shape)
         # test potrf
         # Note: Have to symmetrize input, for gradient test to work
         res_potrf = np.sqrt(data_in)
         test_potrf = mx.sym.linalg.potrf(data1, lower=lower)
-        check_fw(test_potrf, [data_in], [res_potrf])
-        if grad_check == 1:
-            check_grad(test_potrf, [data_in])
+        check_fw_grad(test_potrf, [data_in], [res_potrf])
         # test potri
-        ones = mx.nd.ones(shape).asnumpy()
         res_potri = np.divide(ones, data_in * data_in)
         test_potri = mx.sym.linalg.potri(data1, lower=lower)
-        check_fw(test_potri, [data_in], [res_potri])
-        if grad_check == 1:
-            check_grad(test_potri, [data_in])
+        check_fw_grad(test_potri, [data_in], [res_potri])
         # test trsm
         trian_in = data_in * 7.
         test_trsm = mx.sym.linalg.trsm(data1, data2, alpha=7., lower=lower)
-        check_fw(test_trsm, [trian_in, data_in], [ones])
-        if grad_check == 1:
-            check_grad(test_trsm, [trian_in,data_in])
+        check_fw_grad(test_trsm, [trian_in, data_in], [ones])
         # test trmm
         trian_in = np.divide(ones, trian_in)
         test_trmm = mx.sym.linalg.trmm(data1, data2, alpha=7., transpose=True,
                                        rightside=True, lower=lower)
-        check_fw(test_trmm, [trian_in, data_in], [ones])
-        if grad_check == 1:
-            check_grad(test_trmm, [trian_in, data_in])
+        check_fw_grad(test_trmm, [trian_in, data_in], [ones])
         # test sumlogdiag
         res_sumlogdiag = np.reshape(np.log(data_in), (4, 4))
         test_sumlogdiag = mx.sym.linalg.sumlogdiag(data1)
-        check_fw(test_sumlogdiag, [data_in], [res_sumlogdiag])
-        if grad_check == 1:
-            check_grad(test_sumlogdiag, [data_in])
+        check_fw_grad(test_sumlogdiag, [data_in], [res_sumlogdiag])
 
         # more elaborate example of Cholesky factorization
-        matrix = np.array([[9., 3., -6., 12.],
-                           [3., 26., -7., -11.],
-                           [-6., -7., 9., 7.],
-                           [12., -11., 7., 65.]])
-        trian  = np.array([[3., 0., 0., 0.],
-                           [1., 5., 0., 0.],
-                           [-2., -1., 2., 0.],
-                           [4., -3., 6., 2.]])
-        pow    = np.array([[2., 1., 1., 1.],
-                           [1., 4., 1., 1.],
-                           [1., 1., 8., 1.],
-                           [1., 1., 1., 16.]])
-        inv    = np.array([[8.95/3., 0.05/3., 2.65, -2.5/3.],
-                           [0.05/3., 0.05, 0.05, 0.],
-                           [2.65, 0.05, 2.5, -0.75],
-                           [-2.5/3., 0., -0.75, 0.25]])
-        ident  = np.eye(4)
-
         low_trian = trian
-        if not lower:
+        if upper:
             trian = np.transpose(trian)
 
         # test potrf
         test_potrf = mx.sym.linalg.potrf(_make_symm_symbol(data1, ndims=4), lower=lower)
         a = rep_3x(matrix, 4, 4)
         r = rep_3x(trian, 4, 4)
-        check_fw(test_potrf, [a], [r])
-        if grad_check == 1:
-            check_grad(test_potrf, [a])
+        check_fw_grad(test_potrf, [a], [r])
 
         #test potri
         data1_ltri = _make_triangle_symm(
@@ -6654,77 +6534,54 @@ def test_laop():
         test_potri = mx.sym.linalg.potri(data1_ltri, lower=lower)
         a = rep_3x(trian, 4, 4)
         r = rep_3x(inv, 4, 4)
-        check_fw(test_potri, [a], [r])
-        if grad_check == 1:
-            check_grad(test_potri, [a])
+        check_fw_grad(test_potri, [a], [r])
 
         # test trsm
         test_trsm = mx.sym.linalg.trsm(data1_ltri, data2, alpha=7., transpose=upper, lower=lower)
-        a = rep_3x(trian, 4, 4)
         b = rep_3x(matrix, 4, 4)
         r = rep_3x(7. * np.transpose(low_trian), 4, 4)
-        check_fw(test_trsm, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trsm, [a, b])
+        check_fw_grad(test_trsm, [a, b], [r])
 
         test_trsm2 = mx.sym.linalg.trsm(
             data1_ltri, data2, alpha=-2., rightside=True, transpose=lower, lower=lower)
         r = rep_3x(-2. * low_trian, 4, 4)
-        check_fw(test_trsm2, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trsm2, [a, b])
+        check_fw_grad(test_trsm2, [a, b], [r])
 
         test_trsm3 = mx.sym.linalg.trsm(
             data1_ltri, data2, alpha=0.5, transpose=lower, lower=lower)
         b = rep_3x(np.transpose(low_trian), 4, 4)
         r = rep_3x(0.5 * ident, 4, 4)
-        check_fw(test_trsm3, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trsm3, [a, b])
+        check_fw_grad(test_trsm3, [a, b], [r])
 
         test_trsm4 = mx.sym.linalg.trsm(
             data1_ltri, data2, alpha=-0.5, rightside=True, transpose=upper, lower=lower)
         b = rep_3x(low_trian, 4, 4)
         r = rep_3x(-0.5 * ident, 4, 4)
-        check_fw(test_trsm4, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trsm4, [a, b])
+        check_fw_grad(test_trsm4, [a, b], [r])
 
         # test trmm
         test_trmm = mx.sym.linalg.trmm(
             data1_ltri, data2, alpha=7., transpose=True, rightside=True, lower=lower)
-        a = rep_3x(trian, 4, 4)
-        b = rep_3x(matrix, 4, 4)
+        a = [a, rep_3x(matrix, 4, 4)]
         r = rep_3x(7. * np.dot(matrix, trian.T), 4, 4)
-        check_fw(test_trmm, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trmm, [a, b])
+        check_fw_grad(test_trmm, a, [r])
 
         test_trmm2 = mx.sym.linalg.trmm(data1_ltri, data2, alpha=-2., lower=lower)
         r = rep_3x(-2. * np.dot(trian, matrix), 4, 4)
-        check_fw(test_trmm2, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trmm2, [a, b])
+        check_fw_grad(test_trmm2, a, [r])
 
         test_trmm3 = mx.sym.linalg.trmm(data1_ltri, data2, rightside=True, lower=lower)
         r = rep_3x(np.dot(matrix, trian), 4, 4)
-        check_fw(test_trmm3, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trmm3, [a, b])
+        check_fw_grad(test_trmm3, a, [r])
 
         test_trmm4 = mx.sym.linalg.trmm(
             data1_ltri, data2, alpha=1.2, transpose=True, lower=lower)
         r = rep_3x(1.2 * np.dot(trian.T, matrix), 4, 4)
-        check_fw(test_trmm4, [a, b], [r])
-        if grad_check == 1:
-            check_grad(test_trmm4, [a, b])
+        check_fw_grad(test_trmm4, a, [r])
 
-    # test sumlogdiag
-    a = rep_3x(pow, 4, 4)
-    r = np.reshape(np.tile(10. * np.log(np.array([2.])), 3), (3,))
-    check_fw(test_sumlogdiag, [a], [r])
-    if grad_check == 1:
-        check_grad(test_sumlogdiag, [a])
+        # test sumlogdiag
+        r = np.reshape(np.tile(10. * np.log(np.array([2.])), 3), (3,))
+        check_fw_grad(test_sumlogdiag, [rep_3x(pow, 4, 4)], [r])
 
 
 # Tests for operators linalg.syrk, linalg.gelqf
@@ -7929,12 +7786,12 @@ def py_bilinear_resize_backward(x, incoming_grads, mode='size'):
     def check_bilinear_resize_op(shape, height, width):
         x = mx.nd.random.uniform(shape=shape)
         y = mx.nd.contrib.BilinearResize2D(x, height=height, width=width)
-        assert_almost_equal(y.asnumpy(), py_bilinear_resize(x.asnumpy(), height, width))
+        assert_almost_equal(y, py_bilinear_resize(x.asnumpy(), height, width))
 
         x_scale = width / shape[-1]
         y_scale = height / shape[-2]
         y = mx.nd.contrib.BilinearResize2D(x, scale_height=y_scale, scale_width=x_scale)
-        assert_almost_equal(y.asnumpy(), py_bilinear_resize(x.asnumpy(), height, width))
+        assert_almost_equal(y, py_bilinear_resize(x.asnumpy(), height, width))
     def check_bilinear_resize_modes_op(shape, scale_height=None, scale_width=None, shape_1=None, mode=None):
         x = mx.nd.random.uniform(shape=shape)
         original_h = shape[2]
@@ -8150,6 +8007,7 @@ def f(x, a, b, c):
     data = mx.symbol.Variable('data')
     quad_sym = mx.sym.contrib.quadratic(data=data, a=a, b=b, c=c)
     for dtype in [np.float16, np.float32, np.float64]:
+        tol = 1e-2 if dtype is np.float16 else 1e-5
         for ndim in range(1, 6):
             shape = rand_shape_nd(ndim, 5)
             data_np = np.random.randn(*shape).astype(dtype)
@@ -8158,21 +8016,92 @@ def f(x, a, b, c):
 
             # check imperative forward
             output = mx.nd.contrib.quadratic(mx.nd.array(data_np), a=a, b=b, c=c)
-            assert_almost_equal(output.asnumpy(),expected,
-                                rtol=1e-2 if dtype is np.float16 else 1e-5,
-                                atol=1e-2 if dtype is np.float16 else 1e-5)
+            assert_almost_equal(output, expected, rtol=tol, atol=tol)
             # check forward
-            check_symbolic_forward(quad_sym, [data_np], [expected],
-                                   rtol=1e-2 if dtype is np.float16 else 1e-5,
-                                   atol=1e-2 if dtype is np.float16 else 1e-5)
+            check_symbolic_forward(quad_sym, [data_np], [expected], rtol=tol, atol=tol)
             # check backward
             check_symbolic_backward(quad_sym, [data_np], [np.ones(expected.shape)],
-                                    [backward_expected],
-                                    rtol=1e-2 if dtype is np.float16 else 1e-5,
-                                    atol=1e-2 if dtype is np.float16 else 1e-5)
+                                    [backward_expected], rtol=tol, atol=tol)
             # check backward using finite difference
             check_numeric_gradient(quad_sym, [data_np], atol=0.001)
 
+def allclose_function(contexts):
+    def getRandom(base, percent = 1.):
+        return base * (1 + percent * (2 * np.random.random_sample() - 1.) / 100)
+
+    title = 'exp'
+    for ctx in contexts:
+        title += ' cpu' if ctx == mx.cpu() else ' gpu'
+
+    title += '        nElem     shape'
+    num_ctx = len(contexts)
+    result = [False, False]
+    for dtype in [np.float16, np.float32, np.float64]:
+        rtol = getRandom(1e-2 if dtype is np.float16 else 1e-5)
+        atol = getRandom(1e-4 if dtype is np.float16 else 1e-7)
+        print('\nnumpy.{}:  atol = {}  rtol = {}'.format(dtype.__name__, atol, rtol))
+        print(title)
+        for ndim in range(1, 10):
+            shape = rand_shape_nd(ndim, 8)
+            a_np = np.random.randn(*shape).astype(dtype)
+            b_np = (a_np + np.random.randn(*shape).astype(dtype) / 10000000).astype(dtype)
+            expected = np.allclose(a_np, b_np, rtol, atol)
+
+            for n, ctx in enumerate(contexts):
+                a_ctx = mx.nd.array(a_np, dtype = dtype, ctx=ctx)
+                b_ctx = mx.nd.array(b_np, dtype = dtype, ctx=ctx)
+                output = mx.nd.contrib.allclose(a_ctx, b_ctx, rtol=rtol, atol=atol)
+                result[n] = output.asnumpy() == 1
+                if expected != result[n]:
+                    # Preparing the output of elements of the array, which are considered as "not close" AND
+                    # corresponding elements of comparison CPU/GPU/Python vectors, which are considered as "close"
+                    v_ctx = 'CPU' if ctx == mx.cpu() else 'GPU'
+                    if expected:
+                        v_cmp = 'Python'
+                        a_b = a_ctx.asnumpy()
+                        b_b = b_ctx.asnumpy()
+                        a_g = np.asarray(a_np)
+                        b_g = np.asarray(b_np)
+
+                    else:
+                        v_cmp = v_ctx
+                        v_ctx = 'Python'
+                        a_b = np.asarray(a_np)
+                        b_b = np.asarray(b_np)
+                        a_g = a_ctx.asnumpy()
+                        b_g = b_ctx.asnumpy()
+
+                    print('\n *** Violations found on %s, but not on %s side  ***' % (v_ctx, v_cmp))
+                    frmt = "                 a[{0:d}]:                 b[{0:d}]:"  \
+                           "          abs(a[{0:d}]-b[{0:d}]) - atol + rtol*abs(b[{0:d}]):"
+
+                    # Define the indices of all violations and corresponding values of coordinates
+                    bad_indexes = np.abs(a_b - b_b) >= atol + rtol * abs(b_b)
+                    a_values = [a_b[bad_indexes], a_g[bad_indexes]]
+                    b_values = [b_b[bad_indexes], b_g[bad_indexes]]
+                    idx = np.asarray(np.where(bad_indexes == True))
+                    idx = idx.reshape(1, idx.size)
+                    idx_flat = np.asarray(np.where(bad_indexes.flatten() == True)).flatten()
+                    for i in range(len(a_values[0])):
+                        flat_idx = idx_flat[i]
+                        print('{}:  index = {}   flat_index = {}'.format('%4d'%i, idx[i], flat_idx))
+                        print(frmt.format(flat_idx))
+                        for j in range(2):
+                            diff = np.abs(a_values[j][i]-b_values[j][i]) - atol + rtol*abs(b_values[j][i])
+                            print('{}:  {}  {}              {}'.format('%6s'%v_ctx, a_values[j][i], b_values[j][i], diff))
+
+
+            if num_ctx == 1:
+                print(' {0:d}   {1:d}    {2:10d}   {3:}'.format(expected, result[0], np.prod(shape), shape))
+            else:
+                print(' {0:d}   {1:d}   {2:d}    {3:10d}   {4:}'.format(expected, result[0], result[1], np.prod(shape), shape))
+
+            if expected != result[0] or num_ctx > 1 and expected != result[1]:
+                assert False
+
+@with_seed()
+def test_allclose_function():
+    allclose_function([default_context()])
 
 @with_seed()
 def test_histogram():
@@ -8188,12 +8117,12 @@ def f(x, bins=10, range=None):
         bin_range = (-2.5, 2.5)
         mx_histo1, mx_bins1 = mx.nd.histogram(x, bins=bin_cnt, range=bin_range)
         np_histo1, np_bins1 = f(x.asnumpy(), bins=bin_cnt, range=bin_range)
-        assert_almost_equal(mx_bins1.asnumpy(), np_bins1)
-        assert_almost_equal(mx_histo1.asnumpy(), np_histo1, rtol=1e-3, atol=1e-5)
+        assert_almost_equal(mx_bins1, np_bins1)
+        assert_almost_equal(mx_histo1, np_histo1, rtol=1e-3, atol=1e-5)
         mx_histo2, mx_bins2 = mx.nd.histogram(x, bins=mx_bins)
         np_histo2, np_bins2 = f(x.asnumpy(), bins=np_bins)
-        assert_almost_equal(mx_histo2.asnumpy(), np_histo2, rtol=1e-3, atol=1e-5)
-        assert_almost_equal(mx_bins2.asnumpy(), np_bins2, rtol=1e-3, atol=1e-5)
+        assert_almost_equal(mx_histo2, np_histo2, rtol=1e-3, atol=1e-5)
+        assert_almost_equal(mx_bins2, np_bins2, rtol=1e-3, atol=1e-5)
 
         data = mx.sym.Variable("data")
 
@@ -8555,9 +8484,9 @@ def test_roi_align_value(sampling_ratio=0, position_sensitive=False):
                                                              spatial_scale, sampling_ratio,
                                                              position_sensitive, dy.asnumpy())
 
-        assert_almost_equal(output.asnumpy(), real_output, atol=1e-3)
-        assert_almost_equal(data.grad.asnumpy(), dx, atol=1e-3)
-        assert_almost_equal(rois.grad.asnumpy(), drois, atol=1e-3)
+        assert_almost_equal(output, real_output, atol=1e-3)
+        assert_almost_equal(data.grad, dx, atol=1e-3)
+        assert_almost_equal(rois.grad, drois, atol=1e-3)
 
     # modified from test_roipooling()
     def test_roi_align_autograd(sampling_ratio=0):
@@ -8730,24 +8659,8 @@ def test_diag():
     a_np = np.random.random((h, w)).astype(np.float32)
     a = mx.nd.array(a_np).astype('float32')
 
-    # k == 0
-    r = mx.nd.diag(a)
-    assert_almost_equal(r.asnumpy(), np.diag(a_np))
-
-    # k == 1
-    k = 1
-    r = mx.nd.diag(a, k=k)
-    assert_almost_equal(r.asnumpy(), np.diag(a_np, k=k))
-
-    # k == -1
-    k = -1
-    r = mx.nd.diag(a, k=k)
-    assert_almost_equal(r.asnumpy(), np.diag(a_np, k=k))
-
-    # random k
-    k = np.random.randint(-min(h,w) + 1, min(h,w))
-    r = mx.nd.diag(a, k=k)
-    assert_almost_equal(r.asnumpy(), np.diag(a_np, k=k))
+    for k in [0, 1, -1, np.random.randint(-min(h,w) + 1, min(h,w))]:
+        assert_almost_equal(mx.nd.diag(a, k=k), np.diag(a_np, k=k))
 
     # invalid k
     k = max(h,w) + 1
@@ -8775,9 +8688,7 @@ def test_diag():
 
     # k is random
     k = np.random.randint(-d,d)
-    r = mx.nd.diag(a, k=k)
-
-    assert_almost_equal(r.asnumpy(), np.diag(a_np, k=k))
+    assert_almost_equal(mx.nd.diag(a, k=k), np.diag(a_np, k=k))
 
     # Test 2d backward, k=0
     data = mx.sym.Variable('data')
@@ -8804,19 +8715,19 @@ def test_diag():
 
     # k = 0, axis1=0, axis2=1
     r = mx.nd.diag(data=a, k=0, axis1=0, axis2=1)
-    assert_almost_equal(r.asnumpy(), np.diagonal(a_np, offset=0, axis1=0, axis2=1))
+    assert_almost_equal(r, np.diagonal(a_np, offset=0, axis1=0, axis2=1))
 
     # k = 1, axis1=1, axis2=0
     r = mx.nd.diag(data=a, k=1, axis1=1, axis2=0)
-    assert_almost_equal(r.asnumpy(), np.diagonal(a_np, offset=1, axis1=1, axis2=0))
+    assert_almost_equal(r, np.diagonal(a_np, offset=1, axis1=1, axis2=0))
 
     # k = -1 axis1=1, axis3=3
     r = mx.nd.diag(data=a, k=-1, axis1=1, axis2=3)
-    assert_almost_equal(r.asnumpy(), np.diagonal(a_np, offset=-1, axis1=1, axis2=3))
+    assert_almost_equal(r, np.diagonal(a_np, offset=-1, axis1=1, axis2=3))
 
     # k = 2, axis1=-2, axis2=0
     r = mx.nd.diag(data=a, k=2, axis1=-2, axis2=0)
-    assert_almost_equal(r.asnumpy(), np.diagonal(a_np, offset=2, axis1=-2, axis2=0))
+    assert_almost_equal(r, np.diagonal(a_np, offset=2, axis1=-2, axis2=0))
 
     # Test 4d backward, k=0, axis1=3, axis2=0
     data = mx.sym.Variable('data')
@@ -8858,7 +8769,7 @@ def f(x, blocksize):
     data_np = data.asnumpy()
     expected = f(data_np, block)
     output = mx.nd.depth_to_space(data, block)
-    assert_almost_equal(output.asnumpy(), expected, atol=1e-3, rtol=1e-3)
+    assert_almost_equal(output, expected, atol=1e-3, rtol=1e-3)
 
     shape_out = (n, c // (block ** 2), h * block, w * block)
     data = mx.sym.Variable('data')
@@ -8909,7 +8820,7 @@ def f(x, blocksize):
     data_np = data.asnumpy()
     expected = f(data_np, block)
     output = mx.nd.space_to_depth(data, block)
-    assert_almost_equal(output.asnumpy(), expected, atol=1e-3, rtol=1e-3)
+    assert_almost_equal(output, expected, atol=1e-3, rtol=1e-3)
 
     shape_out = (n, c * (block ** 2), h // block, w // block)
     data = mx.sym.Variable('data')
diff --git a/tests/python/unittest/test_random.py b/tests/python/unittest/test_random.py
index fe276685bfe3..efcf16dc78da 100644
--- a/tests/python/unittest/test_random.py
+++ b/tests/python/unittest/test_random.py
@@ -19,7 +19,7 @@
 import math
 import itertools
 import mxnet as mx
-from mxnet.test_utils import verify_generator, gen_buckets_probs_with_ppf, retry
+from mxnet.test_utils import verify_generator, gen_buckets_probs_with_ppf, retry, assert_almost_equal
 import numpy as np
 import random as rnd
 from common import setup_module, with_seed, random_seed, teardown
@@ -583,14 +583,14 @@ def test_sample_multinomial():
                 prob = prob.reshape((1, prob.shape[0]))
             for i in range(x.shape[0]):
                 freq = np.bincount(y[i,:].astype('int32'), minlength=5)/np.float32(samples)*x[i,:].sum()
-                mx.test_utils.assert_almost_equal(freq, x[i], rtol=0.20, atol=1e-1)
+                assert_almost_equal(freq, x[i], rtol=0.20, atol=1e-1)
                 rprob = x[i][y[i].astype('int32')]/x[i].sum()
-                mx.test_utils.assert_almost_equal(np.log(rprob), prob.asnumpy()[i], atol=1e-5)
+                assert_almost_equal(np.log(rprob), prob.asnumpy()[i], atol=1e-5)
 
                 real_dx = np.zeros((5,))
                 for j in range(samples):
                     real_dx[int(y[i][j])] += 5.0 / rprob[j]
-                mx.test_utils.assert_almost_equal(real_dx, dx[i, :], rtol=1e-4, atol=1e-5)
+                assert_almost_equal(real_dx, dx[i, :], rtol=1e-4, atol=1e-5)
     for dtype in ['uint8', 'float16', 'float32']:
         # Bound check for the output data types. 'int32' and 'float64' require large memory so are skipped.
         x = mx.nd.zeros(2 ** 25)  # Larger than the max integer in float32 without precision loss.
@@ -883,8 +883,8 @@ def compute_expected_prob():
     # test ndarray
     true_classes = mx.nd.random.uniform(0, range_max, shape=(num_true,)).astype('int32')
     sampled_classes, exp_cnt_true, exp_cnt_sampled = mx.nd.contrib.rand_zipfian(true_classes, num_sampled, range_max)
-    mx.test_utils.assert_almost_equal(exp_cnt_sampled.asnumpy(), exp_cnt[sampled_classes].asnumpy(), rtol=1e-1, atol=1e-2)
-    mx.test_utils.assert_almost_equal(exp_cnt_true.asnumpy(), exp_cnt[true_classes].asnumpy(), rtol=1e-1, atol=1e-2)
+    assert_almost_equal(exp_cnt_sampled, exp_cnt[sampled_classes], rtol=1e-1, atol=1e-2)
+    assert_almost_equal(exp_cnt_true, exp_cnt[true_classes], rtol=1e-1, atol=1e-2)
 
     # test symbol
     true_classes_var = mx.sym.var('true_classes')
@@ -893,8 +893,8 @@ def compute_expected_prob():
     executor = outputs.bind(mx.context.current_context(), {'true_classes' : true_classes})
     executor.forward()
     sampled_classes, exp_cnt_true, exp_cnt_sampled = executor.outputs
-    mx.test_utils.assert_almost_equal(exp_cnt_sampled.asnumpy(), exp_cnt[sampled_classes].asnumpy(), rtol=1e-1, atol=1e-2)
-    mx.test_utils.assert_almost_equal(exp_cnt_true.asnumpy(), exp_cnt[true_classes].asnumpy(), rtol=1e-1, atol=1e-2)
+    assert_almost_equal(exp_cnt_sampled, exp_cnt[sampled_classes], rtol=1e-1, atol=1e-2)
+    assert_almost_equal(exp_cnt_true, exp_cnt[true_classes], rtol=1e-1, atol=1e-2)
 
 # Issue #10277 (https://github.com/apache/incubator-mxnet/issues/10277) discusses this test.
 @with_seed()
diff --git a/tests/python/unittest/test_sparse_operator.py b/tests/python/unittest/test_sparse_operator.py
index 91194c562a57..4c4e3dbdfc51 100644
--- a/tests/python/unittest/test_sparse_operator.py
+++ b/tests/python/unittest/test_sparse_operator.py
@@ -732,11 +732,9 @@ def check_sparse_mathematical_core(name, stype,
 
         assert arr_grad.stype == expected_grad_result_type
 
-        arr_grad = arr_grad.asnumpy()
-
         if verbose is True:
             print(name)
-            print("arr_grad", arr_grad)
+            print("arr_grad", arr_grad.asnumpy())
             print("input_grad", input_grad)
 
         assert_almost_equal(arr_grad, input_grad, equal_nan=True)
diff --git a/tests/python/unittest/test_subgraph.py b/tests/python/unittest/test_subgraph.py
index 4c13f9c70dfc..3da125a946bc 100644
--- a/tests/python/unittest/test_subgraph.py
+++ b/tests/python/unittest/test_subgraph.py
@@ -20,15 +20,8 @@
 import numpy as np
 import mxnet as mx
 import copy
-import math
-import ctypes
-import random
-import itertools
-from numpy.testing import assert_allclose, assert_array_equal
 from mxnet.test_utils import *
-from mxnet.base import py_str, MXNetError, _as_list, SymbolHandle, check_call, _LIB, c_handle_array, mx_uint
 from common import setup_module, with_seed, teardown
-import unittest
 from mxnet.gluon.model_zoo.vision import get_model
 
 def make_subgraph(subg, *args):
diff --git a/tests/utils/notebook_test/__init__.py b/tests/utils/notebook_test/__init__.py
index 0a794cd13619..3cbc8211b17f 100644
--- a/tests/utils/notebook_test/__init__.py
+++ b/tests/utils/notebook_test/__init__.py
@@ -111,7 +111,7 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir='
             nbformat.write(notebook, output_file)
             output_nb = io.open(output_file, mode='r', encoding='utf-8')
             for line in output_nb:
-                if "Warning:" in line:
+                if "Warning:" in line and "numpy operator signatures" not in line:
                     errors.append("Warning:\n" + line)
         if len(errors) > 0:
             logging.error('\n'.join(errors))