From 810517474e57d3277f64438618ed82df10be4090 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Thu, 14 Jan 2021 00:55:10 +0000 Subject: [PATCH 1/6] fix multiple output bug --- .../contrib/onnx/mx2onnx/_op_translations.py | 40 +++++++++++++++++++ .../mxnet/contrib/onnx/mx2onnx/export_onnx.py | 5 ++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index ce20b7da4418..b85443345d24 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -2902,3 +2902,43 @@ def convert_repeat(node, **kwargs): ] return nodes + + +@mx_op.register('_contrib_box_nms') +def convert_contrib_box_nms(node, **kwargs): + """Map MXNet's _contrib_box_nms operator to ONNX + """ + from onnx.helper import make_node + from onnx import TensorProto + name, input_nodes, attrs = get_inputs(node, kwargs) + + opset_version = kwargs['opset_version'] + if opset_version < 11: + raise AttributeError('ONNX opset 11 or greater is required to export this operator') + + overlap_thresh = float(attrs.get('overlap_thresh', '0.5')) + valid_thresh = float(attrs.get('valid_thresh', '0.5')) + topk = int(attrs.get('topk', '-1')) + coord_start = int(attrs.get('coord_start', '2')) + score_start = int(attrs.get('score_start', '1')) + id_index = int(attrs.get('id_index', '-1')) + background_id = int(attrs.get('background_id', '-1')) + force_suppress = attrs.get('force_suppress', 'False') + in_format = attrs.get('in_format', 'corner') + out_format = attrs.get('in_format', 'corner') + + print(input_nodes) + nodes = [ + #create_tensor([coord_start], name+'_cs', kwargs['initializer']), + #create_tensor([coord_start+4], name+'_cs_p4', kwargs['initializer']), + #create_tensor([-1], name+'_m1', kwargs['initializer']), + #make_node('Slice', [input_nodes[0], name+'_cs', name+'_cs_p4', name+'_m1'], + # [name+'_slice']), + make_node('Identity', [input_nodes[0]], [name], name=name), + #make_node('Range', [name+'_cs_p4', name+'_cs', name+'_m1'], [name]) + ] + print(nodes) + print(name) + + return nodes + diff --git a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py index 817a019e0c60..077e04cd2283 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py +++ b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py @@ -128,6 +128,7 @@ def get_outputs(sym, params, in_shape, in_label, in_type): :rtype: dict of (str, tuple(int, ...)) """ from onnx import mapping + import re # remove any input listed in params from sym.list_inputs() and bind them to the input shapes provided # by user. Also remove in_label, which is the name of the label symbol that may have been used # as the label for loss during training. @@ -140,8 +141,10 @@ def get_outputs(sym, params, in_shape, in_label, in_type): out_names = list() for name in sym.list_outputs(): - if name.endswith('_output'): + if re.search('.*_output$', name): out_names.append(name[:-len('_output')]) + elif re.search('.*_output[0-9]$', name): + out_names.append(name[:-len('_output0')]) else: logging.info("output '%s' does not end with '_output'", name) out_names.append(name) From 79c7d7bda22fc583be54c082118edc550395fbe1 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Thu, 14 Jan 2021 22:47:33 +0000 Subject: [PATCH 2/6] basic implementation of nms conversion --- .../contrib/onnx/mx2onnx/_op_translations.py | 79 ++++++++++++++++--- tests/python-pytest/onnx/test_operators.py | 60 ++++++++++++++ 2 files changed, 126 insertions(+), 13 deletions(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index b85443345d24..8105bc566b6e 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -2916,29 +2916,82 @@ def convert_contrib_box_nms(node, **kwargs): if opset_version < 11: raise AttributeError('ONNX opset 11 or greater is required to export this operator') + input_type = kwargs['in_type'] + dtype = onnx.mapping.TENSOR_TYPE_TO_NP_TYPE[input_type] + overlap_thresh = float(attrs.get('overlap_thresh', '0.5')) - valid_thresh = float(attrs.get('valid_thresh', '0.5')) + valid_thresh = float(attrs.get('valid_thresh', '0')) topk = int(attrs.get('topk', '-1')) coord_start = int(attrs.get('coord_start', '2')) - score_start = int(attrs.get('score_start', '1')) + score_index = int(attrs.get('score_index', '1')) id_index = int(attrs.get('id_index', '-1')) background_id = int(attrs.get('background_id', '-1')) force_suppress = attrs.get('force_suppress', 'False') in_format = attrs.get('in_format', 'corner') - out_format = attrs.get('in_format', 'corner') + out_format = attrs.get('out_format', 'corner') + + center_point_box = 0 if in_format == 'corner' else 1 - print(input_nodes) + if in_format != out_format: + raise NotImplementedError('box_nms does not currently support in_fomat != out_format') + + if background_id != -1: + raise NotImplementedError('box_nms does not currently support background_id != -1') + + if id_index != -1: + raise NotImplementedError('box_nms does not currently support id_index != -1') + nodes = [ - #create_tensor([coord_start], name+'_cs', kwargs['initializer']), - #create_tensor([coord_start+4], name+'_cs_p4', kwargs['initializer']), - #create_tensor([-1], name+'_m1', kwargs['initializer']), - #make_node('Slice', [input_nodes[0], name+'_cs', name+'_cs_p4', name+'_m1'], - # [name+'_slice']), - make_node('Identity', [input_nodes[0]], [name], name=name), - #make_node('Range', [name+'_cs_p4', name+'_cs', name+'_m1'], [name]) + create_tensor([coord_start], name+'_cs', kwargs['initializer']), + create_tensor([coord_start+4], name+'_cs_p4', kwargs['initializer']), + create_tensor([score_index], name+'_si', kwargs['initializer']), + create_tensor([score_index+1], name+'_si_p1', kwargs['initializer']), + create_tensor([topk], name+'_topk', kwargs['initializer']), + create_tensor([overlap_thresh], name+'_ot', kwargs['initializer'], dtype=np.float32), + create_tensor([valid_thresh], name+'_vt', kwargs['initializer'], dtype=np.float32), + create_tensor([-1], name+'_m1', kwargs['initializer']), + create_tensor([-1], name+'_m1_f', kwargs['initializer'], dtype=dtype), + create_tensor([0], name+'_0', kwargs['initializer']), + create_tensor([1], name+'_1', kwargs['initializer']), + create_tensor([2], name+'_2', kwargs['initializer']), + create_tensor([3], name+'_3', kwargs['initializer']), + create_tensor([], name+'_void', kwargs['initializer']), + create_tensor([0, 1, -1], name+'_scores_shape', kwargs['initializer']), + create_tensor([0, 0, 1, 0], name+'_pad', kwargs['initializer']), + create_tensor([0, -1], name+'_bat_spat_helper', kwargs['initializer']), + make_node('Shape', [input_nodes[0]], [name+'_shape']), + make_node('Shape', [name+'_shape'], [name+'_dim']), + make_node('Sub', [name+'_dim', name+'_2'], [name+'_dim_m2']), + make_node('Slice', [name+'_shape', name+'_dim_m2', name+'_dim'], [name+'_shape_last2']), + make_node('Concat', [name+'_m1', name+'_shape_last2'], [name+'_shape_3d'], axis=0), + make_node('Reshape', [input_nodes[0], name+'_shape_3d'], [name+'_data_3d']), + make_node('Slice', [name+'_data_3d', name+'_cs', name+'_cs_p4', name+'_m1'], + [name+'_boxes']), + make_node('Slice', [name+'_data_3d', name+'_si', name+'_si_p1', name+'_m1'], + [name+'_scores_raw']), + make_node('Reshape', [name+'_scores_raw', name+'_scores_shape'], [name+'_scores']), + make_node('Shape', [name+'_scores'], [name+'_scores_shape_actual']), + make_node('NonMaxSuppression', [name+'_boxes', name+'_scores', name+'_topk', name+'_ot', + name+'_vt'], [name+'_nms'], center_point_box=center_point_box), + make_node('Slice', [name+'_nms', name+'_0', name+'_3', name+'_m1', name+'_2'], + [name+'_nms_sliced']), + make_node('GatherND', [name+'_data_3d', name+'_nms_sliced'], [name+'_candidates']), + make_node('Pad', [name+'_candidates', name+'_pad', name+'_m1_f'], [name+'_cand_padded']), + make_node('Shape', [name+'_nms'], [name+'_nms_shape']), + make_node('Slice', [name+'_nms_shape', name+'_0', name+'_1'], [name+'_cand_cnt']), + make_node('Reshape', [name+'_cand_cnt', name+'_void'], [name+'_cc_s']), + make_node('Range', [name+'_0', name+'_cc_s', name+'_1'], [name+'_cand_indices']), + make_node('Slice', [name+'_scores_shape_actual', name+'_0', name+'_3', name+'_m1', + name+'_2'], [name+'_shape_bat_spat']), + make_node('Slice', [name+'_shape_bat_spat', name+'_1', name+'_2'], [name+'_spat_dim']), + make_node('Expand', [name+'_cand_cnt', name+'_shape_bat_spat'], [name+'_base_indices']), + make_node('ScatterND', [name+'_base_indices', name+'_nms_sliced', name+'_cand_indices'], + [name+'_indices']), + make_node('TopK', [name+'_indices', name+'_spat_dim'], [name+'_indices_sorted', name+'__'], + largest=0, axis=-1, sorted=1), + make_node('Gather', [name+'_cand_padded', name+'_indices_sorted'], [name+'_gather']), + make_node('Reshape', [name+'_gather', name+'_shape'], [name]) ] - print(nodes) - print(name) return nodes diff --git a/tests/python-pytest/onnx/test_operators.py b/tests/python-pytest/onnx/test_operators.py index 2efa582d4c95..766067c975cf 100644 --- a/tests/python-pytest/onnx/test_operators.py +++ b/tests/python-pytest/onnx/test_operators.py @@ -386,3 +386,63 @@ def test_onnx_export_contrib_BilinearResize2D(tmp_path, dtype, params): x = mx.nd.arange(0, 160).reshape((2, 2, 5, 8)) M = def_model('contrib.BilinearResize2D', **params) op_export_test('contrib_BilinearResize2D', M, [x], tmp_path) + + + +@pytest.mark.parametrize('topk', [2, 3, 4]) +@pytest.mark.parametrize('valid_thresh', [0.3, 0.4, 0.8]) +@pytest.mark.parametrize('overlap_thresh', [0.4, 0.7, 1.0]) + +#@pytest.mark.parametrize('topk', [3]) +#@pytest.mark.parametrize('valid_thresh', [0.3]) +#@pytest.mark.parametrize('overlap_thresh', [0.4]) +def test_onnx_export_contrib_box_nms_manual(tmp_path, topk, valid_thresh, overlap_thresh): + # Note that ONNX NMS op only supports float32 + + # Also note that onnxruntime's nms has slightly different implementation in handling + # overlaps and score ordering when certain boxes are suppressed than that of mxnet + # the following test tensors are manually tweaked to avoid such diferences + # The purpose of theses tests cases are to show that the high level conversion logic is + # laid out correctly + + A = mx.nd.array([[ + [[[[0.5, 0.1, 0.1, 0.2, 0.2], + [0.4, 0.1, 0.1, 0.2, 0.2], + [0.7, 0.5, 0.5, 0.9, 0.9], + [0.8, 0.1, 0.9, 0.11, 0.91], + [0.001, 0.01, 0.01, 0.02, 0.02]]]], + + [[[[0.5, 0.1, 0.1, 0.2, 0.2], + [0.4, 0.1, 0.1, 0.2, 0.2], + [0.7, 0.5, 0.5, 0.9, 0.9], + [0.8, 0.1, 0.9, 0.11, 0.91], + [0.001, 0.01, 0.01, 0.02, 0.02]]]], + + [[[[0.4, 0.1, 0.1, 0.2, 0.2], + [0.3, 0.1, 0.1, 0.2, 0.2], + [0.7, 0.5, 0.5, 0.9, 0.9], + [0.8, 0.1, 0.9, 0.11, 0.91], + [0.001, 0.01, 0.01, 0.02, 0.02]]]], + ]]) + M = def_model('contrib.box_nms', coord_start=1, force_suppress=True, + overlap_thresh=overlap_thresh, valid_thresh=valid_thresh, score_index=0, + topk=topk, in_format='corner', out_format='corner') + op_export_test('contrib_nms_manual_coner', M, [A], tmp_path) + + B = mx.nd.array([ + [[[[0.7, 0.5, 0.5, 0.2, 0.2], + [0.6, 0.48, 0.48, 0.2, 0.2], + [0.8, 0.76, 0.76, 0.2, 0.2], + [0.9, 0.7, 0.7, 0.2, 0.2], + [0.001, 0.5, 0.1, 0.02, 0.02]]]], + + [[[[0.5, 0.2, 0.2, 0.2, 0.2], + [0.6, 0.4, 0.4, 0.21, 0.21], + [0.7, 0.5, 0.5, 0.9, 0.9], + [0.8, 0.1, 0.9, 0.01, 0.01], + [0.001, 0.6, 0.1, 0.02, 0.02]]]], + ]) + M = def_model('contrib.box_nms', coord_start=1, force_suppress=True, + overlap_thresh=overlap_thresh, valid_thresh=valid_thresh, score_index=0, + topk=topk, in_format='center', out_format='center') + op_export_test('contrib_nms_manual_center', M, [B], tmp_path) From 89cbb717e6ba045a77efc1dce16dc9a22fa18927 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Fri, 15 Jan 2021 00:41:57 +0000 Subject: [PATCH 3/6] tweak --- python/mxnet/contrib/onnx/mx2onnx/_op_translations.py | 2 +- python/mxnet/contrib/onnx/mx2onnx/export_onnx.py | 2 +- tests/python-pytest/onnx/test_operators.py | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index 8105bc566b6e..0e24734f32f6 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -2990,7 +2990,7 @@ def convert_contrib_box_nms(node, **kwargs): make_node('TopK', [name+'_indices', name+'_spat_dim'], [name+'_indices_sorted', name+'__'], largest=0, axis=-1, sorted=1), make_node('Gather', [name+'_cand_padded', name+'_indices_sorted'], [name+'_gather']), - make_node('Reshape', [name+'_gather', name+'_shape'], [name]) + make_node('Reshape', [name+'_gather', name+'_shape'], [name+'0']) ] return nodes diff --git a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py index 077e04cd2283..c319476313c2 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py +++ b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py @@ -144,7 +144,7 @@ def get_outputs(sym, params, in_shape, in_label, in_type): if re.search('.*_output$', name): out_names.append(name[:-len('_output')]) elif re.search('.*_output[0-9]$', name): - out_names.append(name[:-len('_output0')]) + out_names.append(name[:-len('_output0')]+name[-1]) else: logging.info("output '%s' does not end with '_output'", name) out_names.append(name) diff --git a/tests/python-pytest/onnx/test_operators.py b/tests/python-pytest/onnx/test_operators.py index 766067c975cf..dce0e4f8eeb3 100644 --- a/tests/python-pytest/onnx/test_operators.py +++ b/tests/python-pytest/onnx/test_operators.py @@ -392,10 +392,6 @@ def test_onnx_export_contrib_BilinearResize2D(tmp_path, dtype, params): @pytest.mark.parametrize('topk', [2, 3, 4]) @pytest.mark.parametrize('valid_thresh', [0.3, 0.4, 0.8]) @pytest.mark.parametrize('overlap_thresh', [0.4, 0.7, 1.0]) - -#@pytest.mark.parametrize('topk', [3]) -#@pytest.mark.parametrize('valid_thresh', [0.3]) -#@pytest.mark.parametrize('overlap_thresh', [0.4]) def test_onnx_export_contrib_box_nms_manual(tmp_path, topk, valid_thresh, overlap_thresh): # Note that ONNX NMS op only supports float32 From 040da7512d1c22b438cae2a2b7b192927608d1d5 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Fri, 15 Jan 2021 10:49:25 -0800 Subject: [PATCH 4/6] Update _op_translations.py --- python/mxnet/contrib/onnx/mx2onnx/_op_translations.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index 870638eaef8f..bef912e55531 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -2909,7 +2909,6 @@ def convert_contrib_box_nms(node, **kwargs): """Map MXNet's _contrib_box_nms operator to ONNX """ from onnx.helper import make_node - from onnx import TensorProto name, input_nodes, attrs = get_inputs(node, kwargs) opset_version = kwargs['opset_version'] @@ -2931,7 +2930,7 @@ def convert_contrib_box_nms(node, **kwargs): out_format = attrs.get('out_format', 'corner') center_point_box = 0 if in_format == 'corner' else 1 - + if in_format != out_format: raise NotImplementedError('box_nms does not currently support in_fomat != out_format') @@ -2941,6 +2940,8 @@ def convert_contrib_box_nms(node, **kwargs): if id_index != -1: raise NotImplementedError('box_nms does not currently support id_index != -1') + force_suppress = True + nodes = [ create_tensor([coord_start], name+'_cs', kwargs['initializer']), create_tensor([coord_start+4], name+'_cs_p4', kwargs['initializer']), @@ -2971,8 +2972,9 @@ def convert_contrib_box_nms(node, **kwargs): [name+'_scores_raw']), make_node('Reshape', [name+'_scores_raw', name+'_scores_shape'], [name+'_scores']), make_node('Shape', [name+'_scores'], [name+'_scores_shape_actual']), - make_node('NonMaxSuppression', [name+'_boxes', name+'_scores', name+'_topk', name+'_ot', - name+'_vt'], [name+'_nms'], center_point_box=center_point_box), + make_node('NonMaxSuppression', + [name+'_boxes', name+'_scores', name+'_topk', name+'_ot', name+'_vt'], + [name+'_nms'], center_point_box=center_point_box), make_node('Slice', [name+'_nms', name+'_0', name+'_3', name+'_m1', name+'_2'], [name+'_nms_sliced']), make_node('GatherND', [name+'_data_3d', name+'_nms_sliced'], [name+'_candidates']), From d7bba6088ecb7ca31c4a6df913ee7f6c8d9469df Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Fri, 15 Jan 2021 14:16:10 -0800 Subject: [PATCH 5/6] Update _op_translations.py --- python/mxnet/contrib/onnx/mx2onnx/_op_translations.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index bef912e55531..f851c0304a12 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -2925,7 +2925,6 @@ def convert_contrib_box_nms(node, **kwargs): score_index = int(attrs.get('score_index', '1')) id_index = int(attrs.get('id_index', '-1')) background_id = int(attrs.get('background_id', '-1')) - force_suppress = attrs.get('force_suppress', 'False') in_format = attrs.get('in_format', 'corner') out_format = attrs.get('out_format', 'corner') @@ -2940,8 +2939,6 @@ def convert_contrib_box_nms(node, **kwargs): if id_index != -1: raise NotImplementedError('box_nms does not currently support id_index != -1') - force_suppress = True - nodes = [ create_tensor([coord_start], name+'_cs', kwargs['initializer']), create_tensor([coord_start+4], name+'_cs_p4', kwargs['initializer']), From 2f8d5923dac909d64aed3f5bc829957a204044c0 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Fri, 22 Jan 2021 21:32:22 +0000 Subject: [PATCH 6/6] fix --- python/mxnet/contrib/onnx/mx2onnx/export_onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py index 024256529ea6..ec5ea2d4a273 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py +++ b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py @@ -141,7 +141,7 @@ def get_outputs(sym, params, in_shape, in_label, in_type): out_names = list() for name in sym.list_outputs(): - if re.search('.*_output$', name): + if name.endswith('_output'): out_names.append(name[:-len('_output')]) elif re.search('.*_output[0-9]$', name): out_names.append(name[:-len('_output0')]+name[-1])