From 6f905574553d5903a7456f79551401d09d83facb Mon Sep 17 00:00:00 2001 From: vexilligera Date: Wed, 16 Oct 2019 06:00:03 +0000 Subject: [PATCH] Tests of NumPy interoperability (#16469) * cumsum test * ravel test * test reciprocal remainder repeat reshape * test rint roll sign * print test * print debug * life is short use pytorch --- .../python/tutorials/deploy/export/onnx.md | 12 +- python/mxnet/numpy_dispatch_protocol.py | 2 + .../unittest/test_numpy_interoperability.py | 160 ++++++++++++++++-- 3 files changed, 157 insertions(+), 17 deletions(-) diff --git a/docs/python_docs/python/tutorials/deploy/export/onnx.md b/docs/python_docs/python/tutorials/deploy/export/onnx.md index f3ba3b979fe3..7f28a6818f52 100644 --- a/docs/python_docs/python/tutorials/deploy/export/onnx.md +++ b/docs/python_docs/python/tutorials/deploy/export/onnx.md @@ -28,7 +28,7 @@ In this tutorial, we will learn how to use MXNet to ONNX exporter on pre-trained ## Prerequisites To run the tutorial you will need to have installed the following python modules: -- [MXNet >= 1.3.0]() +- [MXNet >= 1.3.0](/get_started) - [onnx]( https://github.com/onnx/onnx#installation) v1.2.1 (follow the install guide) *Note:* MXNet-ONNX importer and exporter follows version 7 of ONNX operator set which comes with ONNX v1.2.1. @@ -59,7 +59,7 @@ Now, we have downloaded ResNet-18 symbol, params and synset file on the disk. ## MXNet to ONNX exporter API -Let us describe the MXNet's `export_model` API. +Let us describe the MXNet's `export_model` API. ```python help(onnx_mxnet.export_model) @@ -74,7 +74,7 @@ export_model(sym, params, input_shape, input_type=, onnx_f Exports the MXNet model file, passed as a parameter, into ONNX model. Accepts both symbol,parameter objects as well as json and params filepaths as input. Operator support and coverage - https://cwiki.apache.org/confluence/display/MXNET/MXNet-ONNX+Integration - + Parameters ---------- sym : str or symbol object @@ -89,7 +89,7 @@ export_model(sym, params, input_shape, input_type=, onnx_f Path where to save the generated onnx file verbose : Boolean If true will print logs of the model conversion - + Returns ------- onnx_file_path : str @@ -145,6 +145,6 @@ model_proto = onnx.load_model(converted_model_path) checker.check_graph(model_proto.graph) ``` -If the converted protobuf format doesn't qualify to ONNX proto specifications, the checker will throw errors, but in this case it successfully passes. +If the converted protobuf format doesn't qualify to ONNX proto specifications, the checker will throw errors, but in this case it successfully passes. -This method confirms exported model protobuf is valid. Now, the model is ready to be imported in other frameworks for inference! \ No newline at end of file +This method confirms exported model protobuf is valid. Now, the model is ready to be imported in other frameworks for inference! diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py index f483e299ffed..b02a7e53cfbf 100644 --- a/python/mxnet/numpy_dispatch_protocol.py +++ b/python/mxnet/numpy_dispatch_protocol.py @@ -98,8 +98,10 @@ def _run_with_array_ufunc_proto(*args, **kwargs): 'min', 'ones_like', 'prod', + 'ravel', 'repeat', 'reshape', + 'roll', 'split', 'squeeze', 'stack', diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py index f0c205f72dc4..af5991381bd4 100644 --- a/tests/python/unittest/test_numpy_interoperability.py +++ b/tests/python/unittest/test_numpy_interoperability.py @@ -25,6 +25,7 @@ from mxnet.numpy_dispatch_protocol import with_array_function_protocol, with_array_ufunc_protocol from mxnet.numpy_dispatch_protocol import _NUMPY_ARRAY_FUNCTION_LIST, _NUMPY_ARRAY_UFUNC_LIST +import itertools class OpArgMngr(object): """Operator argument manager for storing operator workloads.""" @@ -49,6 +50,10 @@ def _prepare_workloads(): '1x1x0': np.array([[[]]]) } + dt_int = [np.int8, np.int32, np.int64, np.uint8] + dt_float = [np.float16, np.float32, np.float64] + dt = dt_int + dt_float + # workloads for array function protocol OpArgMngr.add_workload('argmax', array_pool['4x1']) OpArgMngr.add_workload('broadcast_arrays', array_pool['4x1'], array_pool['1x2']) @@ -57,8 +62,14 @@ def _prepare_workloads(): OpArgMngr.add_workload('concatenate', [array_pool['4x1'], array_pool['4x1']]) OpArgMngr.add_workload('concatenate', [array_pool['4x1'], array_pool['4x1']], axis=1) OpArgMngr.add_workload('copy', array_pool['4x1']) - OpArgMngr.add_workload('cumsum', array_pool['4x1']) - OpArgMngr.add_workload('cumsum', array_pool['4x1'], axis=1) + + for ctype in dt: + OpArgMngr.add_workload('cumsum', np.array([1, 2, 10, 11, 6, 5, 4], dtype=ctype)) + OpArgMngr.add_workload('cumsum', np.array([[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]], dtype=ctype), axis=0) + OpArgMngr.add_workload('cumsum', np.array([[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]], dtype=ctype), axis=1) + + OpArgMngr.add_workload('ravel', np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]])) + OpArgMngr.add_workload('dot', array_pool['4x1'], array_pool['4x1'].T) OpArgMngr.add_workload('expand_dims', array_pool['4x1'], -1) OpArgMngr.add_workload('fix', array_pool['4x1']) @@ -68,8 +79,72 @@ def _prepare_workloads(): OpArgMngr.add_workload('mean', array_pool['4x1'], axis=0, keepdims=True) OpArgMngr.add_workload('ones_like', array_pool['4x1']) OpArgMngr.add_workload('prod', array_pool['4x1']) + OpArgMngr.add_workload('repeat', array_pool['4x1'], 3) - OpArgMngr.add_workload('reshape', array_pool['4x1'], -1) + OpArgMngr.add_workload('repeat', np.array(_np.arange(12).reshape(4, 3)[:, 2]), 3) + + m = _np.array([1, 2, 3, 4, 5, 6]) + m_rect = m.reshape((2, 3)) + + # OpArgMngr.add_workload('repeat', np.array(m), [1, 3, 2, 1, 1, 2]) # Argument "repeats" only supports int + OpArgMngr.add_workload('repeat', np.array(m), 2) + B = np.array(m_rect) + # OpArgMngr.add_workload('repeat', B, [2, 1], axis=0) # Argument "repeats" only supports int + # OpArgMngr.add_workload('repeat', B, [1, 3, 2], axis=1) # Argument "repeats" only supports int + OpArgMngr.add_workload('repeat', B, 2, axis=0) + OpArgMngr.add_workload('repeat', B, 2, axis=1) + + # test_repeat_broadcasting + a = _np.arange(60).reshape(3, 4, 5) + for axis in itertools.chain(range(-a.ndim, a.ndim), [None]): + OpArgMngr.add_workload('repeat', np.array(a), 2, axis=axis) + # OpArgMngr.add_workload('repeat', np.array(a), [2], axis=axis) # Argument "repeats" only supports int + + arr = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]) + OpArgMngr.add_workload('reshape', arr, (2, 6)) + OpArgMngr.add_workload('reshape', arr, (3, 4)) + # OpArgMngr.add_workload('reshape', arr, (3, 4), order='F') # Items are not equal with order='F' + OpArgMngr.add_workload('reshape', arr, (3, 4), order='C') + OpArgMngr.add_workload('reshape', np.array(_np.ones(100)), (100, 1, 1)) + + # test_reshape_order + a = np.array(_np.arange(6)) + # OpArgMngr.add_workload('reshape', a, (2, 3), order='F') # Items are not equal with order='F' + a = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) + b = a[:, 1] + # OpArgMngr.add_workload('reshape', b, (2, 2), order='F') # Items are not equal with order='F' + + a = np.array(_np.ones((0, 2))) + OpArgMngr.add_workload('reshape', a, -1, 2) + + OpArgMngr.add_workload('rint', np.array(4607998452777363968)) + OpArgMngr.add_workload('rint', array_pool['4x1']) + + # test_roll1d(self) + OpArgMngr.add_workload('roll', np.array(_np.arange(10)), 2) + + # test_roll2d(self) + x2 = np.array(_np.reshape(_np.arange(10), (2, 5))) + OpArgMngr.add_workload('roll', x2, 1) + OpArgMngr.add_workload('roll', x2, 1, axis=0) + OpArgMngr.add_workload('roll', x2, 1, axis=1) + # # Roll multiple axes at once. + OpArgMngr.add_workload('roll', x2, 1, axis=(0, 1)) + OpArgMngr.add_workload('roll', x2, (1, 0), axis=(0, 1)) + OpArgMngr.add_workload('roll', x2, (-1, 0), axis=(0, 1)) + OpArgMngr.add_workload('roll', x2, (0, 1), axis=(0, 1)) + OpArgMngr.add_workload('roll', x2, (0, -1), axis=(0, 1)) + OpArgMngr.add_workload('roll', x2, (1, 1), axis=(0, 1)) + OpArgMngr.add_workload('roll', x2, (-1, -1), axis=(0, 1)) + # # Roll the same axis multiple times. + # OpArgMngr.add_workload('roll', x2, 1, axis=(0, 0)) # Check failed: axes[i - 1] < axes[i] (0 vs. 0) : axes have duplicates [0,0] + # OpArgMngr.add_workload('roll', x2, 1, axis=(1, 1)) # Check failed: axes[i - 1] < axes[i] (1 vs. 1) : axes have duplicates [1,1] + # # Roll more than one turn in either direction. + OpArgMngr.add_workload('roll', x2, 6, axis=1) + OpArgMngr.add_workload('roll', x2, -4, axis=1) + # # test_roll_empty + OpArgMngr.add_workload('roll', np.array([]), 1) + OpArgMngr.add_workload('split', array_pool['4x1'], 2) OpArgMngr.add_workload('squeeze', array_pool['4x1']) OpArgMngr.add_workload('stack', [array_pool['4x1']] * 2) @@ -104,10 +179,64 @@ def _prepare_workloads(): OpArgMngr.add_workload('mod', array_pool['4x1'], 2) OpArgMngr.add_workload('mod', 2, array_pool['4x1']) OpArgMngr.add_workload('mod', array_pool['4x1'], array_pool['1x1x0']) - OpArgMngr.add_workload('remainder', array_pool['4x1'], array_pool['1x2']) - OpArgMngr.add_workload('remainder', array_pool['4x1'], 2) - OpArgMngr.add_workload('remainder', 2, array_pool['4x1']) - OpArgMngr.add_workload('remainder', array_pool['4x1'], array_pool['1x1x0']) + + # test remainder basic + OpArgMngr.add_workload('remainder', np.array([0, 1, 2, 4, 2], dtype=np.float16), + np.array([-2, 5, 1, 4, 3], dtype=np.float16)) + + def _signs(dt): + if dt in [np.uint8]: + return (+1,) + else: + return (+1, -1) + + for ct in dt: + for sg1, sg2 in itertools.product(_signs(ct), _signs(ct)): + a = np.array(sg1*71, dtype=ct) + b = np.array(sg2*19, dtype=ct) + OpArgMngr.add_workload('remainder', a, b) + + # test remainder exact + nlst = list(range(-127, 0)) + plst = list(range(1, 128)) + dividend = nlst + [0] + plst + divisor = nlst + plst + arg = list(itertools.product(dividend, divisor)) + tgt = list(divmod(*t) for t in arg) + a, b = np.array(arg, dtype=int).T + # convert exact integer results from Python to float so that + # signed zero can be used, it is checked. + for dt in [np.float16, np.float32, np.float64]: + fa = a.astype(dt) + fb = b.astype(dt) + OpArgMngr.add_workload('remainder', fa, fb) + + # test_float_remainder_roundoff + for ct in dt_float: + for sg1, sg2 in itertools.product((+1, -1), (+1, -1)): + a = np.array(sg1*78*6e-8, dtype=ct) + b = np.array(sg2*6e-8, dtype=ct) + OpArgMngr.add_workload('remainder', a, b) + + # test_float_remainder_corner_cases + # Check remainder magnitude. + for ct in dt_float: + b = _np.array(1.0) + a = np.array(_np.nextafter(_np.array(0.0), -b), dtype=ct) + b = np.array(b, dtype=ct) + OpArgMngr.add_workload('remainder', a, b) + OpArgMngr.add_workload('remainder', -a, -b) + + # Check nans, inf + for ct in [np.float16, np.float32, np.float64]: + fone = np.array(1.0, dtype=ct) + fzer = np.array(0.0, dtype=ct) + finf = np.array(np.inf, dtype=ct) + fnan = np.array(np.nan, dtype=ct) + # OpArgMngr.add_workload('remainder', fone, fzer) # failed + OpArgMngr.add_workload('remainder', fone, fnan) + OpArgMngr.add_workload('remainder', finf, fone) + OpArgMngr.add_workload('maximum', array_pool['4x1'], array_pool['1x2']) OpArgMngr.add_workload('maximum', array_pool['4x1'], 2) OpArgMngr.add_workload('maximum', 2, array_pool['4x1']) @@ -118,8 +247,12 @@ def _prepare_workloads(): OpArgMngr.add_workload('minimum', array_pool['4x1'], array_pool['1x1x0']) OpArgMngr.add_workload('negative', array_pool['4x1']) OpArgMngr.add_workload('absolute', array_pool['4x1']) - OpArgMngr.add_workload('rint', array_pool['4x1']) + OpArgMngr.add_workload('sign', array_pool['4x1']) + OpArgMngr.add_workload('sign', np.array([-2, 5, 1, 4, 3], dtype=np.float16)) + OpArgMngr.add_workload('sign', np.array([-.1, 0, .1])) + # OpArgMngr.add_workload('sign', np.array(_np.array([_np.nan]))) # failed + OpArgMngr.add_workload('exp', array_pool['4x1']) OpArgMngr.add_workload('log', array_pool['4x1']) OpArgMngr.add_workload('log2', array_pool['4x1']) @@ -128,7 +261,12 @@ def _prepare_workloads(): OpArgMngr.add_workload('sqrt', array_pool['4x1']) OpArgMngr.add_workload('square', array_pool['4x1']) OpArgMngr.add_workload('cbrt', array_pool['4x1']) - OpArgMngr.add_workload('reciprocal', array_pool['4x1']) + + for ctype in [np.float16, np.float32, np.float64]: + OpArgMngr.add_workload('reciprocal', np.array([-2, 5, 1, 4, 3], dtype=ctype)) + OpArgMngr.add_workload('reciprocal', np.array([-2, 0, 1, 0, 3], dtype=ctype)) + OpArgMngr.add_workload('reciprocal', np.array([0], dtype=ctype)) + OpArgMngr.add_workload('sin', array_pool['4x1']) OpArgMngr.add_workload('cos', array_pool['4x1']) OpArgMngr.add_workload('tan', array_pool['4x1']) @@ -176,10 +314,10 @@ def _check_interoperability_helper(op_name, *args, **kwargs): assert isinstance(arr, np.ndarray) for arr, expected_arr in zip(out, expected_out): assert isinstance(arr, np.ndarray) - assert_almost_equal(arr.asnumpy(), expected_arr, rtol=1e-3, atol=1e-4, use_broadcast=False) + assert_almost_equal(arr.asnumpy(), expected_arr, rtol=1e-3, atol=1e-4, use_broadcast=False, equal_nan=True) else: assert isinstance(out, np.ndarray) - assert_almost_equal(out.asnumpy(), expected_out, rtol=1e-3, atol=1e-4, use_broadcast=False) + assert_almost_equal(out.asnumpy(), expected_out, rtol=1e-3, atol=1e-4, use_broadcast=False, equal_nan=True) def check_interoperability(op_list):