From 713d962359cfe5b014ca24c0f9b3676d86b28569 Mon Sep 17 00:00:00 2001 From: Minghao Liu <40382964+Tommliu@users.noreply.github.com> Date: Wed, 11 Mar 2020 06:02:00 +0800 Subject: [PATCH] [Numpy] FFI: Bincount, Percentile/Quantile, All/Any (#17717) * ffi_bincount percentile/quantile all/any * new ffi --- python/mxnet/_numpy_op_doc.py | 101 -------------- python/mxnet/ndarray/numpy/_op.py | 124 +++++++++++++++--- python/mxnet/numpy/multiarray.py | 116 +++++++++++++++- python/mxnet/symbol/numpy/_symbol.py | 63 ++++++++- src/api/operator/numpy/np_bincount_op.cc | 61 +++++++++ .../numpy/np_broadcast_reduce_op_boolean.cc | 93 +++++++++++++ src/api/operator/numpy/np_percentile_op.cc | 98 ++++++++++++++ src/api/operator/op_utils.cc | 21 +++ src/api/operator/op_utils.h | 1 + src/operator/numpy/np_bincount_op-inl.h | 8 ++ src/operator/numpy/np_broadcast_reduce_op.h | 7 + .../numpy/np_broadcast_reduce_op_boolean.cc | 4 +- .../numpy/np_broadcast_reduce_op_boolean.cu | 4 +- src/operator/numpy/np_percentile_op-inl.h | 12 ++ tests/python/unittest/test_numpy_op.py | 2 - 15 files changed, 586 insertions(+), 129 deletions(-) create mode 100644 src/api/operator/numpy/np_bincount_op.cc create mode 100644 src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc create mode 100644 src/api/operator/numpy/np_percentile_op.cc diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py index 279501d385f8..8dfc0867cdb2 100644 --- a/python/mxnet/_numpy_op_doc.py +++ b/python/mxnet/_numpy_op_doc.py @@ -20,107 +20,6 @@ """Doc placeholder for numpy ops with prefix _np.""" -def _np_all(a, axis=None, keepdims=False, out=None): - """ - Test whether all array elements along a given axis evaluate to True. - - Parameters - ---------- - a : array_like - Input array or object that can be converted to an array. - axis : None or int or tuple of ints, optional - Axis or axes along which a logical AND reduction is performed. - The default (axis = None) is to perform a logical AND over - all the dimensions of the input array. - keepdims : bool, optional - If this is set to True, the axes which are reduced are left in - the result as dimensions with size one. With this option, - the result will broadcast correctly against the input array. - out : ndarray, optional - Alternate output array in which to place the result. It must have - the same shape as the expected output and its type is preserved - - Returns - -------- - all : ndarray, bool - A new boolean or array is returned unless out is specified, - in which case a reference to out is returned. - - Examples: - --------- - >>> np.all([[True,False],[True,True]]) - False - - >>> np.all([[True,False],[True,True]], axis=0) - array([ True, False]) - - >>> np.all([-1, 4, 5]) - True - - >>> np.all([1.0, np.nan]) - True - - >>> o=np.array(False) - >>> z=np.all([-1, 4, 5], out=o) - >>> id(z), id(o), z - (28293632, 28293632, array(True)) # may vary - """ - pass - -def _np_any(a, axis=None, keepdims=False, out=None): - """ - Test whether any array element along a given axis evaluates to True. - Returns single boolean unless axis is not None - - Parameters - ---------- - a : array_like - Input array or object that can be converted to an array. - axis : None or int or tuple of ints, optional - Axis or axes along which a logical AND reduction is performed. - The default (axis = None) is to perform a logical AND over - all the dimensions of the input array. - keepdims : bool, optional - If this is set to True, the axes which are reduced are left in - the result as dimensions with size one. With this option, - the result will broadcast correctly against the input array. - out : ndarray, optional - Alternate output array in which to place the result. It must have - the same shape as the expected output and its type is preserved - - Returns - -------- - any : bool or ndarray - A new boolean or ndarray is returned unless out is specified, - in which case a reference to out is returned. - - Examples: - --------- - >>> np.any([[True, False], [True, True]]) - True - - >>> np.any([[True, False], [False, False]], axis=0) - array([ True, False]) - - >>> np.any([-1, 0, 5]) - True - - >>> np.any(np.nan) - True - - >>> o=np.array(False) - >>> z=np.any([-1, 4, 5], out=o) - >>> z, o - (array(True), array(True)) - >>> # Check now that z is a reference to o - >>> z is o - True - >>> id(z), id(o) # identity of z and o # doctest: +SKIP - (191614240, 191614240) - """ - pass - - def _np_sometrue(a, axis=None, keepdims=False, out=None): """ Check whether some values are true. diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py index 3371b419e328..3d30333d6da2 100644 --- a/python/mxnet/ndarray/numpy/_op.py +++ b/python/mxnet/ndarray/numpy/_op.py @@ -34,7 +34,7 @@ 'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'insert', 'fabs', 'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'matmul', 'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram', - 'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort', + 'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'all', 'any', 'sort', 'tensordot', 'eye', 'linspace', 'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit', 'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack', @@ -1380,6 +1380,110 @@ def power(x1, x2, out=None, **kwargs): return _ufunc_helper(x1, x2, _npi.power, _np.power, _npi.power_scalar, _npi.rpower_scalar, out) +@set_module('mxnet.ndarray.numpy') +def all(a, axis=None, out=None, keepdims=False): + """ + Test whether all array elements along a given axis evaluate to True. + + Parameters + ---------- + a : ndarray + Input array or object that can be converted to an array. + axis : None or int or tuple of ints, optional + Axis or axes along which a logical AND reduction is performed. + The default (axis = None) is to perform a logical AND over + all the dimensions of the input array. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left in + the result as dimensions with size one. With this option, + the result will broadcast correctly against the input array. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + + Returns + -------- + all : ndarray, bool + A new boolean or array is returned unless out is specified, + in which case a reference to out is returned. + + Examples: + --------- + >>> np.all([[True,False],[True,True]]) + False + + >>> np.all([[True,False],[True,True]], axis=0) + array([ True, False]) + + >>> np.all([-1, 4, 5]) + True + + >>> np.all([1.0, np.nan]) + True + + >>> o=np.array(False) + >>> z=np.all([-1, 4, 5], out=o) + >>> id(z), id(o), z + (28293632, 28293632, array(True)) # may vary + """ + return _api_internal.all(a, axis, keepdims, out) + + +@set_module('mxnet.ndarray.numpy') +def any(a, axis=None, out=None, keepdims=False): + """ + Test whether any array element along a given axis evaluates to True. + Returns single boolean unless axis is not None + + Parameters + ---------- + a : ndarray + Input array or object that can be converted to an array. + axis : None or int or tuple of ints, optional + Axis or axes along which a logical AND reduction is performed. + The default (axis = None) is to perform a logical AND over + all the dimensions of the input array. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left in + the result as dimensions with size one. With this option, + the result will broadcast correctly against the input array. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + + Returns + -------- + any : bool or ndarray + A new boolean or ndarray is returned unless out is specified, + in which case a reference to out is returned. + + Examples: + --------- + >>> np.any([[True, False], [True, True]]) + True + + >>> np.any([[True, False], [False, False]], axis=0) + array([ True, False]) + + >>> np.any([-1, 0, 5]) + True + + >>> np.any(np.nan) + True + + >>> o=np.array(False) + >>> z=np.any([-1, 4, 5], out=o) + >>> z, o + (array(True), array(True)) + >>> # Check now that z is a reference to o + >>> z is o + True + >>> id(z), id(o) # identity of z and o # doctest: +SKIP + (191614240, 191614240) + """ + return _api_internal.any(a, axis, keepdims, out) + + @set_module('mxnet.ndarray.numpy') def argsort(a, axis=-1, kind=None, order=None): """ @@ -6637,11 +6741,7 @@ def percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='l """ if overwrite_input is not None: raise NotImplementedError('overwrite_input is not supported yet') - if isinstance(q, numeric_types): - return _npi.percentile(a, axis=axis, interpolation=interpolation, - keepdims=keepdims, q_scalar=q, out=out) - return _npi.percentile(a, q, axis=axis, interpolation=interpolation, - keepdims=keepdims, q_scalar=None, out=out) + return _api_internal.percentile(a, q, axis, interpolation, keepdims, out) @set_module('mxnet.ndarray.numpy') @@ -6722,11 +6822,7 @@ def quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='lin """ if overwrite_input is not None: raise NotImplementedError('overwrite_input is not supported yet') - if isinstance(q, numeric_types): - return _npi.percentile(a, axis=axis, interpolation=interpolation, - keepdims=keepdims, q_scalar=q * 100, out=out) - return _npi.percentile(a, q * 100, axis=axis, interpolation=interpolation, - keepdims=keepdims, q_scalar=None, out=out) + return _api_internal.percentile(a, q * 100, axis, interpolation, keepdims, out) @set_module('mxnet.ndarray.numpy') @@ -7498,13 +7594,9 @@ def bincount(x, weights=None, minlength=0): >>> np.bincount(x, weights=w) array([ 0.3, 0.7, 1.1]) """ - if not isinstance(x, NDArray): - raise TypeError("Input data should be NDarray") if minlength < 0: raise ValueError("Minlength value should greater than 0") - if weights is None: - return _npi.bincount(x, minlength=minlength, has_weights=False) - return _npi.bincount(x, weights=weights, minlength=minlength, has_weights=True) + return _api_internal.bincount(x, weights, minlength) @set_module('mxnet.ndarray.numpy') diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py index ea0d51c65d0e..4396cfa5207f 100644 --- a/python/mxnet/numpy/multiarray.py +++ b/python/mxnet/numpy/multiarray.py @@ -22,8 +22,10 @@ try: + from __builtin__ import all as py_all from __builtin__ import slice as py_slice except ImportError: + from builtins import all as py_all from builtins import slice as py_slice from array import array as native_array @@ -50,7 +52,7 @@ __all__ = ['ndarray', 'empty', 'empty_like', 'array', 'shape', - 'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'broadcast_to', + 'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'all', 'any', 'broadcast_to', 'add', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'power', 'bitwise_not', 'delete', 'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'invert', 'sqrt', 'cbrt', 'abs', 'absolute', 'fabs', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', @@ -310,7 +312,7 @@ def __array_function__(self, func, types, args, kwargs): # pylint: disable=bad- else: # Note: this allows subclasses that don't override # __array_function__ to handle mxnet.numpy.ndarray objects - if not all(issubclass(t, ndarray) for t in types): + if not py_all(issubclass(t, ndarray) for t in types): return NotImplemented return mx_np_func(*args, **kwargs) @@ -626,7 +628,7 @@ def __getitem__(self, key): if isinstance(key, tuple) and len(key) == 0: return self if isinstance(key, tuple) and len(key) == ndim\ - and all(isinstance(idx, integer_types) for idx in key): + and py_all(isinstance(idx, integer_types) for idx in key): out = self for idx in key: out = out[idx] @@ -1091,10 +1093,10 @@ def T(self): # pylint: enable= invalid-name, undefined-variable def all(self, axis=None, out=None, keepdims=False): - return _mx_nd_np.all(self, axis=axis, keepdims=keepdims, out=out) + return _mx_nd_np.all(self, axis=axis, out=out, keepdims=keepdims) def any(self, axis=None, out=None, keepdims=False): - return _mx_nd_np.any(self, axis=axis, keepdims=keepdims, out=out) + return _mx_nd_np.any(self, axis=axis, out=out, keepdims=keepdims) def as_nd_ndarray(self): """Convert mxnet.numpy.ndarray to mxnet.ndarray.NDArray to use its fluent methods.""" @@ -2572,6 +2574,110 @@ def empty_like(prototype, dtype=None, order='C', subok=False, shape=None): # pyl # pylint: disable=redefined-outer-name +@set_module('mxnet.numpy') +def all(a, axis=None, out=None, keepdims=False): + """ + Test whether all array elements along a given axis evaluate to True. + + Parameters + ---------- + a : ndarray + Input array or object that can be converted to an array. + axis : None or int or tuple of ints, optional + Axis or axes along which a logical AND reduction is performed. + The default (axis = None) is to perform a logical AND over + all the dimensions of the input array. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left in + the result as dimensions with size one. With this option, + the result will broadcast correctly against the input array. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + + Returns + -------- + all : ndarray, bool + A new boolean or array is returned unless out is specified, + in which case a reference to out is returned. + + Examples: + --------- + >>> np.all([[True,False],[True,True]]) + False + + >>> np.all([[True,False],[True,True]], axis=0) + array([ True, False]) + + >>> np.all([-1, 4, 5]) + True + + >>> np.all([1.0, np.nan]) + True + + >>> o=np.array(False) + >>> z=np.all([-1, 4, 5], out=o) + >>> id(z), id(o), z + (28293632, 28293632, array(True)) # may vary + """ + return _mx_nd_np.all(a, axis=axis, out=out, keepdims=keepdims) + + +@set_module('mxnet.numpy') +def any(a, axis=None, out=None, keepdims=False): + """ + Test whether any array element along a given axis evaluates to True. + Returns single boolean unless axis is not None + + Parameters + ---------- + a : ndarray + Input array or object that can be converted to an array. + axis : None or int or tuple of ints, optional + Axis or axes along which a logical AND reduction is performed. + The default (axis = None) is to perform a logical AND over + all the dimensions of the input array. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left in + the result as dimensions with size one. With this option, + the result will broadcast correctly against the input array. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + + Returns + -------- + any : bool or ndarray + A new boolean or ndarray is returned unless out is specified, + in which case a reference to out is returned. + + Examples: + --------- + >>> np.any([[True, False], [True, True]]) + True + + >>> np.any([[True, False], [False, False]], axis=0) + array([ True, False]) + + >>> np.any([-1, 0, 5]) + True + + >>> np.any(np.nan) + True + + >>> o=np.array(False) + >>> z=np.any([-1, 4, 5], out=o) + >>> z, o + (array(True), array(True)) + >>> # Check now that z is a reference to o + >>> z is o + True + >>> id(z), id(o) # identity of z and o # doctest: +SKIP + (191614240, 191614240) + """ + return _mx_nd_np.any(a, axis=axis, out=out, keepdims=keepdims) + + @set_module('mxnet.numpy') def identity(n, dtype=None, ctx=None): """ diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py index 18474581d991..8e0087665075 100644 --- a/python/mxnet/symbol/numpy/_symbol.py +++ b/python/mxnet/symbol/numpy/_symbol.py @@ -43,7 +43,7 @@ 'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort', 'tensordot', 'eye', 'linspace', 'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit', 'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack', - 'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_', 'flatnonzero', + 'average', 'mean', 'maximum', 'minimum', 'any', 'all', 'around', 'round', 'round_', 'flatnonzero', 'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index', 'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr', 'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm', @@ -4097,6 +4097,67 @@ def minimum(x1, x2, out=None, **kwargs): return _ufunc_helper(x1, x2, _npi.minimum, _np.minimum, _npi.minimum_scalar, None, out) +@set_module('mxnet.symbol.numpy') +def all(a, axis=None, out=None, keepdims=False): + """ + Test whether all array elements along a given axis evaluate to True. + + Parameters + ---------- + a : _Symbol + Input array or object that can be converted to an array. + axis : None or int or tuple of ints, optional + Axis or axes along which a logical AND reduction is performed. + The default (axis = None) is to perform a logical AND over + all the dimensions of the input array. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left in + the result as dimensions with size one. With this option, + the result will broadcast correctly against the input array. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + + Returns + -------- + all : _Symbol, bool + A new boolean or array is returned unless out is specified, + in which case a reference to out is returned. + """ + return _npi.all(a, axis=axis, keepdims=keepdims, out=out) + + +@set_module('mxnet.symbol.numpy') +def any(a, axis=None, out=None, keepdims=False): + """ + Test whether any array element along a given axis evaluates to True. + Returns single boolean unless axis is not None + + Parameters + ---------- + a : _Symbol + Input array or object that can be converted to an array. + axis : None or int or tuple of ints, optional + Axis or axes along which a logical AND reduction is performed. + The default (axis = None) is to perform a logical AND over + all the dimensions of the input array. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left in + the result as dimensions with size one. With this option, + the result will broadcast correctly against the input array. + out : ndarray, optional + Alternate output array in which to place the result. It must have + the same shape as the expected output and its type is preserved + + Returns + -------- + any : bool or _Symbol + A new boolean or ndarray is returned unless out is specified, + in which case a reference to out is returned. + """ + return _npi.any(a, axis=axis, keepdims=keepdims, out=out) + + @set_module('mxnet.symbol.numpy') def clip(a, a_min, a_max, out=None): """clip(a, a_min, a_max, out=None) diff --git a/src/api/operator/numpy/np_bincount_op.cc b/src/api/operator/numpy/np_bincount_op.cc new file mode 100644 index 000000000000..afa3278c24e4 --- /dev/null +++ b/src/api/operator/numpy/np_bincount_op.cc @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file np_bincount_op.cc + * \brief Implementation of the API of functions in src/operator/numpy/np_bincount_op.cc + */ +#include +#include "../utils.h" +#include "../../../operator/numpy/np_bincount_op-inl.h" + +namespace mxnet { + +MXNET_REGISTER_API("_npi.bincount") +.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) { + using namespace runtime; + const nnvm::Op* op = Op::Get("_npi_bincount"); + nnvm::NodeAttrs attrs; + op::NumpyBincountParam param; + + int num_outputs = 0; + if (args[1].type_code() == kNull) { + param.minlength = args[2].operator int64_t(); + param.has_weights = false; + NDArray* inputs[] = {args[0].operator mxnet::NDArray*()}; + int num_inputs = 1; + attrs.parsed = param; + attrs.op = op; + SetAttrDict(&attrs); + auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr); + *ret = reinterpret_cast(ndoutputs[0]); + } else { + param.minlength = args[2].operator int64_t(); + param.has_weights = true; + NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()}; + int num_inputs = 2; + attrs.parsed = param; + attrs.op = op; + SetAttrDict(&attrs); + auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr); + *ret = reinterpret_cast(ndoutputs[0]); + } +}); + +} // namespace mxnet diff --git a/src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc b/src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc new file mode 100644 index 000000000000..dea510a41608 --- /dev/null +++ b/src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file np_broadcast_reduce_op_boolean.cc + * \brief Implementation of the API of functions in src/operator/numpy/np_broadcast_reduce_op_boolean.cc + */ +#include +#include +#include "../utils.h" +#include "../../../operator/numpy/np_broadcast_reduce_op.h" + +namespace mxnet { + +MXNET_REGISTER_API("_npi.all") +.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) { + using namespace runtime; + const nnvm::Op* op = Op::Get("_npi_all"); + nnvm::NodeAttrs attrs; + op::NumpyReduceAxesBoolParam param; + + NDArray* out = args[3].operator mxnet::NDArray*(); + NDArray** outputs = out == nullptr ? nullptr : &out; + int num_outputs = out != nullptr; + if (args[1].type_code() == kNull) { + param.axis = dmlc::nullopt; + } else if (args[1].type_code() == kDLInt) { + param.axis = Tuple(1, args[1].operator int64_t()); + } else { + param.axis = Tuple(args[1].operator ObjectRef()); + } + param.keepdims = args[2].operator bool(); + NDArray* inputs[] = {args[0].operator mxnet::NDArray*()}; + int num_inputs = 1; + attrs.parsed = std::move(param); + attrs.op = op; + SetAttrDict(&attrs); + auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs); + if (out) { + *ret = PythonArg(3); + } else { + *ret = reinterpret_cast(ndoutputs[0]); + } +}); + +MXNET_REGISTER_API("_npi.any") +.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) { + using namespace runtime; + const nnvm::Op* op = Op::Get("_npi_any"); + nnvm::NodeAttrs attrs; + op::NumpyReduceAxesBoolParam param; + + NDArray* out = args[3].operator mxnet::NDArray*(); + NDArray** outputs = out == nullptr ? nullptr : &out; + int num_outputs = out != nullptr; + if (args[1].type_code() == kNull) { + param.axis = dmlc::nullopt; + } else if (args[1].type_code() == kDLInt) { + param.axis = Tuple(1, args[1].operator int64_t()); + } else { + param.axis = Tuple(args[1].operator ObjectRef()); + } + param.keepdims = args[2].operator bool(); + NDArray* inputs[] = {args[0].operator mxnet::NDArray*()}; + int num_inputs = 1; + attrs.parsed = std::move(param); + attrs.op = op; + SetAttrDict(&attrs); + auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs); + if (out) { + *ret = PythonArg(3); + } else { + *ret = reinterpret_cast(ndoutputs[0]); + } +}); + +} // namespace mxnet diff --git a/src/api/operator/numpy/np_percentile_op.cc b/src/api/operator/numpy/np_percentile_op.cc new file mode 100644 index 000000000000..634ee092c64d --- /dev/null +++ b/src/api/operator/numpy/np_percentile_op.cc @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file np_percentile_op.cc + * \brief Implementation of the API of functions in src/operator/numpy/np_percentile_op.cc + */ +#include +#include +#include "../utils.h" +#include "../../../operator/numpy/np_percentile_op-inl.h" + +namespace mxnet { + +inline int String2MXNetPercentileType(const std::string& s) { + using namespace op; + if (s == "linear") { + return percentile_enum::kLinear; + } else if (s == "lower") { + return percentile_enum::kLower; + } else if (s == "higher") { + return percentile_enum::kHigher; + } else if (s == "midpoint") { + return percentile_enum::kMidpoint; + } else if (s== "nearest") { + return percentile_enum::kNearest; + } else { + LOG(FATAL) << "unknown type " << s; + } + LOG(FATAL) << "should not reach here "; + return 0; +} + +MXNET_REGISTER_API("_npi.percentile") +.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) { + using namespace runtime; + const nnvm::Op* op = Op::Get("_npi_percentile"); + nnvm::NodeAttrs attrs; + op::NumpyPercentileParam param; + + NDArray* out = args[5].operator mxnet::NDArray*(); + NDArray** outputs = out == nullptr ? nullptr : &out; + int num_outputs = out != nullptr; + if (args[2].type_code() == kNull) { + param.axis = dmlc::nullopt; + } else if (args[2].type_code() == kDLInt) { + param.axis = Tuple(1, args[2].operator int64_t()); + } else { + param.axis = Tuple(args[2].operator ObjectRef()); + } + param.interpolation = String2MXNetPercentileType(args[3].operator std::string()); + param.keepdims = args[4].operator bool(); + if (args[1].type_code() == kDLInt || args[1].type_code() == kDLFloat) { + param.q_scalar = args[1].operator double(); + NDArray* inputs[] = {args[0].operator mxnet::NDArray*()}; + int num_inputs = 1; + attrs.parsed = std::move(param); + attrs.op = op; + SetAttrDict(&attrs); + auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs); + if (out) { + *ret = PythonArg(5); + } else { + *ret = reinterpret_cast(ndoutputs[0]); + } + } else { + param.q_scalar = dmlc::nullopt; + NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()}; + int num_inputs = 2; + attrs.parsed = std::move(param); + attrs.op = op; + SetAttrDict(&attrs); + auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs); + if (out) { + *ret = PythonArg(5); + } else { + *ret = reinterpret_cast(ndoutputs[0]); + } + } +}); + +} // namespace mxnet diff --git a/src/api/operator/op_utils.cc b/src/api/operator/op_utils.cc index 220a880336db..bb54662e7a62 100644 --- a/src/api/operator/op_utils.cc +++ b/src/api/operator/op_utils.cc @@ -24,6 +24,7 @@ #include "op_utils.h" #include +#include "../../operator/numpy/np_percentile_op-inl.h" namespace mxnet { @@ -52,4 +53,24 @@ std::string String2MXNetTypeWithBool(int dtype) { return ""; } +std::string MXNetPercentileType2String(int interpolation) { + using namespace op; + switch (interpolation) { + case percentile_enum::kLinear: + return "linear"; + case percentile_enum::kLower: + return "lower"; + case percentile_enum::kHigher: + return "higher"; + case percentile_enum::kMidpoint: + return "midpoint"; + case percentile_enum::kNearest: + return "nearest"; + default: + LOG(FATAL) << "Unknown type enum " << interpolation; + } + LOG(FATAL) << "should not reach here "; + return ""; +} + } // namespace mxnet diff --git a/src/api/operator/op_utils.h b/src/api/operator/op_utils.h index 4c577983c405..f41680df6fd6 100644 --- a/src/api/operator/op_utils.h +++ b/src/api/operator/op_utils.h @@ -29,6 +29,7 @@ namespace mxnet { std::string String2MXNetTypeWithBool(int dtype); +std::string MXNetPercentileType2String(int interpolation); } // namespace mxnet diff --git a/src/operator/numpy/np_bincount_op-inl.h b/src/operator/numpy/np_bincount_op-inl.h index 254ea8fdec22..a2e758f36059 100644 --- a/src/operator/numpy/np_bincount_op-inl.h +++ b/src/operator/numpy/np_bincount_op-inl.h @@ -28,6 +28,7 @@ #include #include #include +#include #include "../mshadow_op.h" #include "../mxnet_op.h" #include "../operator_common.h" @@ -50,6 +51,13 @@ struct NumpyBincountParam : public dmlc::Parameter { .set_default(false) .describe("Determine whether Bincount has weights."); } + void SetAttrDict(std::unordered_map* dict) { + std::ostringstream minlength_s, has_weights_s; + minlength_s << minlength; + has_weights_s << has_weights; + (*dict)["minlength"] = minlength_s.str(); + (*dict)["has_weights"] = has_weights_s.str(); + } }; inline bool NumpyBincountType(const nnvm::NodeAttrs& attrs, diff --git a/src/operator/numpy/np_broadcast_reduce_op.h b/src/operator/numpy/np_broadcast_reduce_op.h index 791c896600c9..dfa48440b848 100644 --- a/src/operator/numpy/np_broadcast_reduce_op.h +++ b/src/operator/numpy/np_broadcast_reduce_op.h @@ -99,6 +99,13 @@ struct NumpyReduceAxesBoolParam : public dmlc::Parameter* dict) { + std::ostringstream axis_s, keepdims_s; + axis_s << axis; + keepdims_s << keepdims; + (*dict)["axis"] = axis_s.str(); + (*dict)["keepdims"] = keepdims_s.str(); + } }; inline TShape NumpyReduceAxesShapeImpl(const TShape& ishape, diff --git a/src/operator/numpy/np_broadcast_reduce_op_boolean.cc b/src/operator/numpy/np_broadcast_reduce_op_boolean.cc index b137975ca427..0c2f02b0d578 100644 --- a/src/operator/numpy/np_broadcast_reduce_op_boolean.cc +++ b/src/operator/numpy/np_broadcast_reduce_op_boolean.cc @@ -39,7 +39,7 @@ inline bool NumpyReduceAxesBoolType(const nnvm::NodeAttrs& attrs, DMLC_REGISTER_PARAMETER(NumpyReduceAxesBoolParam); -NNVM_REGISTER_OP(_np_any) +NNVM_REGISTER_OP(_npi_any) .add_alias("_np_sometrue") .set_attr_parser(ParamParser) .set_num_inputs(1) @@ -61,7 +61,7 @@ NNVM_REGISTER_OP(_np_any) .add_argument("data", "NDArray-or-Symbol", "Input ndarray") .add_arguments(NumpyReduceAxesBoolParam::__FIELDS__()); -NNVM_REGISTER_OP(_np_all) +NNVM_REGISTER_OP(_npi_all) .set_attr_parser(ParamParser) .set_num_inputs(1) .set_num_outputs(1) diff --git a/src/operator/numpy/np_broadcast_reduce_op_boolean.cu b/src/operator/numpy/np_broadcast_reduce_op_boolean.cu index 636c25b505ff..d3247b743bc5 100644 --- a/src/operator/numpy/np_broadcast_reduce_op_boolean.cu +++ b/src/operator/numpy/np_broadcast_reduce_op_boolean.cu @@ -28,11 +28,11 @@ namespace mxnet { namespace op { -NNVM_REGISTER_OP(_np_any) +NNVM_REGISTER_OP(_npi_any) .set_attr("FCompute", NumpyReduceAxesBoolCompute); -NNVM_REGISTER_OP(_np_all) +NNVM_REGISTER_OP(_npi_all) .set_attr("FCompute", NumpyReduceAxesBoolCompute); diff --git a/src/operator/numpy/np_percentile_op-inl.h b/src/operator/numpy/np_percentile_op-inl.h index 5383adb70ca2..80d275f8872c 100644 --- a/src/operator/numpy/np_percentile_op-inl.h +++ b/src/operator/numpy/np_percentile_op-inl.h @@ -25,6 +25,7 @@ #define MXNET_OPERATOR_NUMPY_NP_PERCENTILE_OP_INL_H_ #include +#include #include "../tensor/ordering_op-inl.h" #include "../tensor/matrix_op-inl.h" #include "../../common/utils.h" @@ -32,6 +33,7 @@ #include "../operator_common.h" #include "../elemwise_op_common.h" #include "np_broadcast_reduce_op.h" +#include "../../api/operator/op_utils.h" namespace mxnet { namespace op { @@ -65,6 +67,16 @@ struct NumpyPercentileParam : public dmlc::Parameter { DMLC_DECLARE_FIELD(q_scalar).set_default(dmlc::optional()) .describe("inqut q is a scalar"); } + void SetAttrDict(std::unordered_map* dict) { + std::ostringstream axis_s, keepdims_s, q_scalar_s; + axis_s << axis; + keepdims_s << keepdims; + q_scalar_s << q_scalar; + (*dict)["axis"] = axis_s.str(); + (*dict)["interpolation"] = MXNetPercentileType2String(interpolation); + (*dict)["keepdims"] = keepdims_s.str(); + (*dict)["q_scalar"] = q_scalar_s.str(); + } }; template diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py index d1b91c1efd80..b0ee3e5ef459 100644 --- a/tests/python/unittest/test_numpy_op.py +++ b/tests/python/unittest/test_numpy_op.py @@ -7644,10 +7644,8 @@ def hybrid_forward(self, F, a): test_diag = TestDiag(k) if hybridize: test_diag.hybridize() - x = np.random.uniform(-2.0, 2.0, size=shape).astype(dtype) if len(shape) != 0 else np.array(()) x.attach_grad() - np_out = _np.diag(x.asnumpy(), k) with mx.autograd.record(): mx_out = test_diag(x)