[NumPy] Add NumPy support for norm (#17014)

* norm

* full test

* add default behaviour

* add col norm forward

* add matrix col row norms forward

* row col norm backward

* improve tests

* beautify cpp

* update broadcast_op

* C1002

* billie holiday even told it even better

* probing for windows unittest numpy error

* update test

* update test

* fix style

* retrigger unix ci

* update according to reviews

* fix backward set_num_input

* fix CI

3rdparty/mshadow/mshadow/base.h

@@ -713,6 +713,11 @@ template<>
 MSHADOW_XINLINE bool MinValue<bool>(void) {
   return false;
 }
+/*! \brief minimum value of unsigned int */
+template<>
+MSHADOW_XINLINE unsigned int MinValue<unsigned int>(void) {
+  return 0;
+}
 
 /*!
  * \brief negative infinity of certain types
@@ -785,6 +790,11 @@ template<>
 MSHADOW_XINLINE bool MaxValue<bool>(void) {
   return true;
 }
+/*! \brief maximum value of uint32_t */
+template<>
+MSHADOW_XINLINE uint32_t MaxValue<uint32_t>(void) {
+  return -1;
+}
 
 /*!
  * \brief positive infinity of certain types

python/mxnet/ndarray/numpy/linalg.py

@@ -96,49 +96,166 @@ def pinv(a, rcond=1e-15, hermitian=False):
     return _npi.pinv(a, rcond, hermitian)
 
 
+# pylint: disable=too-many-return-statements
 def norm(x, ord=None, axis=None, keepdims=False):
     r"""Matrix or vector norm.
-
-    This function can only support Frobenius norm for now.
-    The Frobenius norm is given by [1]_:
-
-        :math:`||A||_F = [\sum_{i,j} abs(a_{i,j})^2]^{1/2}`
-
+    This function is able to return one of eight different matrix norms,
+    or one of an infinite number of vector norms (described below), depending
+    on the value of the ``ord`` parameter.
     Parameters
     ----------
     x : ndarray
-        Input array.
-    ord : {'fro'}, optional
-        Order of the norm.
+        Input array.  If `axis` is None, `x` must be 1-D or 2-D.
+    ord : {non-zero int, inf, -inf, 'fro', 'nuc'}, optional
+        Order of the norm (see table under ``Notes``). inf means numpy's
+        `inf` object.
     axis : {int, 2-tuple of ints, None}, optional
         If `axis` is an integer, it specifies the axis of `x` along which to
         compute the vector norms.  If `axis` is a 2-tuple, it specifies the
         axes that hold 2-D matrices, and the matrix norms of these matrices
-        are computed.  If `axis` is None, the norm of the whole ndarray is
-        returned.
-
+        are computed.  If `axis` is None then either a vector norm (when `x`
+        is 1-D) or a matrix norm (when `x` is 2-D) is returned.
     keepdims : bool, optional
         If this is set to True, the axes which are normed over are left in the
         result as dimensions with size one.  With this option the result will
         broadcast correctly against the original `x`.
-
     Returns
     -------
-    n : float or ndarray
+    n : ndarray
         Norm of the matrix or vector(s).
-
+    Notes
+    -----
+    For values of ``ord <= 0``, the result is, strictly speaking, not a
+    mathematical 'norm', but it may still be useful for various numerical
+    purposes.
+    The following norms can be calculated:
+    =====  ============================  ==========================
+    ord    norm for matrices             norm for vectors
+    =====  ============================  ==========================
+    None   Frobenius norm                2-norm
+    'fro'  Frobenius norm                --
+    'nuc'  --                            --
+    inf    max(sum(abs(x), axis=1))      max(abs(x))
+    -inf   min(sum(abs(x), axis=1))      min(abs(x))
+    0      --                            sum(x != 0)
+    1      max(sum(abs(x), axis=0))      as below
+    -1     min(sum(abs(x), axis=0))      as below
+    2      --                            as below
+    -2     --                            as below
+    other  --                            sum(abs(x)**ord)**(1./ord)
+    =====  ============================  ==========================
+    The Frobenius norm is given by [1]_:
+        :math:`||A||_F = [\sum_{i,j} abs(a_{i,j})^2]^{1/2}`
+    The nuclear norm is the sum of the singular values.
+    When you want to operate norm for matrices,if you ord is (-1, 1, inf, -inf),
+    you must give you axis, it is not support default axis.
     References
     ----------
     .. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*,
            Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15
+    Examples
+    --------
+    >>> from mxnet import np
+    >>> a = np.arange(9) - 4
+    >>> a
+    array([-4., -3., -2., -1.,  0.,  1.,  2.,  3.,  4.])
+    >>> b = a.reshape((3, 3))
+    >>> b
+    array([[-4., -3., -2.],
+           [-1.,  0.,  1.],
+           [ 2.,  3.,  4.]])
+    >>> np.linalg.norm(a)
+    array(7.745967)
+    >>> np.linalg.norm(b)
+    array(7.745967)
+    >>> np.linalg.norm(b, 'fro')
+    array(7.745967)
+    >>> np.linalg.norm(a, 'inf')
+    array(4.)
+    >>> np.linalg.norm(b, 'inf', axis=(0, 1))
+    array(9.)
+    >>> np.linalg.norm(a, '-inf')
+    array(0.)
+    >>> np.linalg.norm(b, '-inf', axis=(0, 1))
+    array(2.)
+    >>> np.linalg.norm(a, 1)
+    array(20.)
+    >>> np.linalg.norm(b, 1, axis=(0, 1))
+    array(7.)
+    >>> np.linalg.norm(a, -1)
+    array(0.)
+    >>> np.linalg.norm(b, -1, axis=(0, 1))
+    array(6.)
+    >>> np.linalg.norm(a, 2)
+    array(7.745967)
+    >>> np.linalg.norm(a, -2)
+    array(0.)
+    >>> np.linalg.norm(a, 3)
+    array(5.8480353)
+    >>> np.linalg.norm(a, -3)
+    array(0.)
+    Using the `axis` argument to compute vector norms:
+    >>> c = np.array([[ 1, 2, 3],
+    ...               [-1, 1, 4]])
+    >>> np.linalg.norm(c, axis=0)
+    array([1.4142135, 2.236068 , 5.       ])
+    >>> np.linalg.norm(c, axis=1)
+    array([3.7416573, 4.2426405])
+    >>> np.linalg.norm(c, ord=1, axis=1)
+    array([6., 6.])
+    Using the `axis` argument to compute matrix norms:
+    >>> m = np.arange(8).reshape(2,2,2)
+    >>> np.linalg.norm(m, axis=(1,2))
+    array([ 3.7416573, 11.224973 ])
+    >>> np.linalg.norm(m[0, :, :]), np.linalg.norm(m[1, :, :])
+    (array(3.7416573), array(11.224973))
     """
-    if ord is not None and ord != 'fro':
-        raise ValueError('only support Frobenius norm for now, received ord={}'.format(str(ord)))
-    if isinstance(axis, tuple) and len(axis) > 2:
-        raise ValueError('Improper number of dimensions to norm')
-    if ord == 'fro' and x.ndim > 2 and axis is None:
-        raise ValueError('Improper number of dimensions to norm')
-    return _mx_nd_np.sqrt(_mx_nd_np.sum(x * x, axis=axis, keepdims=keepdims))
+    if axis is None and ord is None:
+        return _npi.norm(x, ord=2, axis=None, keepdims=keepdims, flag=-2)
+    if axis is None or isinstance(axis, (int, tuple)):  # pylint: disable=too-many-nested-blocks
+        if axis is not None:
+            if isinstance(axis, int):
+                axis = (axis, )
+            if len(axis) == 2:
+                if ord in ['inf', '-inf']:
+                    row_axis, col_axis = axis
+                    if not keepdims:
+                        if row_axis > col_axis:
+                            row_axis -= 1
+                    if ord == 'inf':
+                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=col_axis, keepdims=keepdims).max(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                    else:
+                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=col_axis, keepdims=keepdims).min(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                if ord in [1, -1]:
+                    row_axis, col_axis = axis
+                    if not keepdims:
+                        if row_axis < col_axis:
+                            col_axis -= 1
+                    if ord == 1:
+                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=row_axis, keepdims=keepdims).max(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                    elif ord == -1:
+                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=row_axis, keepdims=keepdims).min(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                if ord in [2, -2]:
+                    return _npi.norm(x, ord=ord, axis=axis, keepdims=keepdims, flag=0)
+                if ord is None:
+                    return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=1)
+        if ord == 'inf':
+            return _mx_nd_np.max(_mx_nd_np.abs(x), axis=axis, keepdims=keepdims)
+        elif ord == '-inf':
+            return _mx_nd_np.min(_mx_nd_np.abs(x), axis=axis, keepdims=keepdims)
+        elif ord is None:
+            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=1)
+        elif ord == 2:
+            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=-1)
+        elif ord == 'nuc':
+            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=2)
+        elif ord in ['fro', 'f']:
+            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=1)
+        else:
+            return _npi.norm(x, ord=ord, axis=axis, keepdims=keepdims, flag=-1)
+    else:
+        raise TypeError("'axis' must be None, an integer or a tuple of integers.")
+# pylint: enable=too-many-return-statements
 
 
 def svd(a):