From b9aa3654b4ffab4528f1c3ddb2018588281d0cad Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Fri, 5 Feb 2021 02:00:16 +0000 Subject: [PATCH 01/13] Enable end-to-end tests for all models we currently support exporting to onnx. --- tests/python-pytest/onnx/test_onnxruntime.py | 39 +++++++++----------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 60e84d047439..151c5bebd9b2 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -151,13 +151,12 @@ def test_bert_inference_onnxruntime(tmp_path, model): 'mobilenetv2_0.25', 'mobilenetv3_large', 'mobilenetv3_small', - # failing due to accuracy - #'resnest14', - #'resnest26', - #'resnest50', - #'resnest101', - #'resnest200', - #'resnest269', + 'resnest14', + 'resnest26', + 'resnest50', + 'resnest101', + 'resnest200', + 'resnest269', 'resnet18_v1', 'resnet18_v1b_0.89', 'resnet18_v2', @@ -246,7 +245,7 @@ def normalize_image(imgfile): 'center_net_resnet50_v1b_voc', 'center_net_resnet101_v1b_voc', 'center_net_resnet18_v1b_coco', - #'center_net_resnet50_v1b_coco', + 'center_net_resnet50_v1b_coco', 'center_net_resnet101_v1b_coco' ]) def test_obj_detection_model_inference_onnxruntime(tmp_path, model): @@ -294,11 +293,10 @@ def normalize_image(imgfile): 'fcn_resnet101_ade', 'deeplab_resnet50_ade', 'deeplab_resnet101_ade', - # the 4 models below are failing due to an accuracy issue - #'deeplab_resnest50_ade', - #'deeplab_resnest101_ade', - #'deeplab_resnest200_ade', - #'deeplab_resnest269_ade', + 'deeplab_resnest50_ade', + 'deeplab_resnest101_ade', + 'deeplab_resnest200_ade', + 'deeplab_resnest269_ade', 'fcn_resnet101_coco', 'deeplab_resnet101_coco', 'fcn_resnet101_voc', @@ -358,13 +356,13 @@ def normalize_image(imgfile): 'simple_pose_resnet101_v1d', 'simple_pose_resnet152_v1b', 'simple_pose_resnet152_v1d', + 'alpha_pose_resnet101_v1b_coco', #'mobile_pose_resnet18_v1b', #'mobile_pose_resnet50_v1b', #'mobile_pose_mobilenet1.0', #'mobile_pose_mobilenetv2_1.0', #'mobile_pose_mobilenetv3_large', #'mobile_pose_mobilenetv3_small', - #'alpha_pose_resnet101_v1b_coco', ]) def test_pose_estimation_model_inference_onnxruntime(tmp_path, model): def normalize_image(imgfile): @@ -407,15 +405,14 @@ def normalize_image(imgfile): 'inceptionv1_kinetics400', 'resnet18_v1b_kinetics400', 'resnet34_v1b_kinetics400', + 'resnet50_v1b_kinetics400', + 'resnet101_v1b_kinetics400', + 'resnet152_v1b_kinetics400', 'resnet50_v1b_hmdb51', 'resnet50_v1b_sthsthv2', 'vgg16_ucf101', - # the following models are failing due to an accuracy issue - #'resnet50_v1b_kinetics400', - #'resnet101_v1b_kinetics400', - #'resnet152_v1b_kinetics400', - #'inceptionv3_kinetics400', - #'inceptionv3_ucf101', + 'inceptionv3_kinetics400', + 'inceptionv3_ucf101', ]) def test_action_recognition_model_inference_onnxruntime(tmp_path, model): batch_size = 64 @@ -449,7 +446,7 @@ def load_video(filepath): data = load_video(video) mx_result = M.predict(data) onnx_result = session.run([], {input_name: data.asnumpy()})[0] - assert_almost_equal(mx_result, onnx_result) + assert_almost_equal(mx_result, onnx_result, rtol=0.001, atol=0.01) finally: shutil.rmtree(tmp_path) From e066217cbe64b16ef5767d7d3f85768cf94c9fe7 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Fri, 5 Feb 2021 17:48:59 +0000 Subject: [PATCH 02/13] Replace deprecated ConstantFill onnx operator with Identity, add commented out models that work with onnx export but fail with onnxruntime. --- .../contrib/onnx/mx2onnx/_op_translations.py | 8 ++++---- tests/python-pytest/onnx/test_onnxruntime.py | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index 778678e6b5d4..a2d9a6bf1c2c 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -787,10 +787,10 @@ def convert_copy(node, **kwargs): @mx_op.register("identity") def convert_identity(node, **kwargs): - """Map MXNet's identity operator attributes to onnx's ConstantFill operator + """Map MXNet's identity operator attributes to onnx's Identity operator and return the created node. """ - return create_basic_op_node('ConstantFill', node, kwargs) + return create_basic_op_node('Identity', node, kwargs) @mx_op.register("InstanceNorm") def convert_instancenorm(node, **kwargs): @@ -1017,12 +1017,12 @@ def convert_logistic_regression_output(node, **kwargs): @mx_op.register("BlockGrad") def convert_blockgrad(node, **kwargs): """ Skip operator """ - return create_basic_op_node('ConstantFill', node, kwargs) + return create_basic_op_node('Identity', node, kwargs) @mx_op.register("MakeLoss") def convert_makeloss(node, **kwargs): """ Skip operator """ - return create_basic_op_node('ConstantFill', node, kwargs) + return create_basic_op_node('Identity', node, kwargs) @mx_op.register("Concat") def convert_concat(node, **kwargs): diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 151c5bebd9b2..319d65348bb8 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -246,7 +246,24 @@ def normalize_image(imgfile): 'center_net_resnet101_v1b_voc', 'center_net_resnet18_v1b_coco', 'center_net_resnet50_v1b_coco', - 'center_net_resnet101_v1b_coco' + 'center_net_resnet101_v1b_coco', + # the following models are failing due to onnxruntime errors + #'ssd_300_vgg16_atrous_voc', + #'ssd_512_vgg16_atrous_voc', + #'ssd_512_resnet50_v1_voc', + #'ssd_512_mobilenet1.0_voc', + #'faster_rcnn_resnet50_v1b_voc', + #'yolo3_darknet53_voc', + #'yolo3_mobilenet1.0_voc', + #'ssd_300_vgg16_atrous_coco', + #'ssd_512_vgg16_atrous_coco', + #'ssd_300_resnet34_v1b_coco', + #'ssd_512_resnet50_v1_coco', + #'ssd_512_mobilenet1.0_coco', + #'faster_rcnn_resnet50_v1b_coco', + #'faster_rcnn_resnet101_v1d_coco', + #'yolo3_darknet53_coco', + #'yolo3_mobilenet1.0_coco', ]) def test_obj_detection_model_inference_onnxruntime(tmp_path, model): def normalize_image(imgfile): From 752ce9ee4ce2e74cbfbafb85f835f1e677a8bfc8 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Fri, 5 Feb 2021 21:58:47 +0000 Subject: [PATCH 03/13] Enable xception end-to-end test. --- tests/python-pytest/onnx/test_onnxruntime.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 319d65348bb8..0dea22dd06f7 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -190,7 +190,8 @@ def test_bert_inference_onnxruntime(tmp_path, model): 'vgg16', 'vgg16_bn', 'vgg19', - 'vgg19_bn' + 'vgg19_bn', + 'xception' ]) def test_obj_class_model_inference_onnxruntime(tmp_path, model): def normalize_image(imgfile): From 9976f76460caf9003a20b91557f5ec0f25010ec9 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Fri, 5 Feb 2021 22:50:34 +0000 Subject: [PATCH 04/13] Add inceptionv3 to object classification model tests and adapt to support different image sizes. --- tests/python-pytest/onnx/test_onnxruntime.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 0dea22dd06f7..4dd034b67c40 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -191,23 +191,25 @@ def test_bert_inference_onnxruntime(tmp_path, model): 'vgg16_bn', 'vgg19', 'vgg19_bn', - 'xception' + 'xception', + 'inceptionv3' ]) def test_obj_class_model_inference_onnxruntime(tmp_path, model): + inlen = 299 if 'inceptionv3' == model else 224 def normalize_image(imgfile): img_data = mx.image.imread(imgfile) - img_data = mx.image.imresize(img_data, 224, 224) + img_data = mx.image.imresize(img_data, inlen, inlen) img_data = img_data.transpose([2, 0, 1]).astype('float32') mean_vec = mx.nd.array([0.485, 0.456, 0.406]) stddev_vec = mx.nd.array([0.229, 0.224, 0.225]) norm_img_data = mx.nd.zeros(img_data.shape).astype('float32') for i in range(img_data.shape[0]): norm_img_data[i,:,:] = (img_data[i,:,:]/255 - mean_vec[i]) / stddev_vec[i] - return norm_img_data.reshape(1, 3, 224, 224).astype('float32') + return norm_img_data.reshape(1, 3, inlen, inlen).astype('float32') try: tmp_path = str(tmp_path) - M = GluonModel(model, (1,3,224,224), 'float32', tmp_path) + M = GluonModel(model, (1,3,inlen,inlen), 'float32', tmp_path) onnx_file = M.export_onnx() # create onnxruntime session using the generated onnx file From 4f13946494e9ddddbc6dfc3f1ee2098d5dcabfd0 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 8 Feb 2021 23:24:53 +0000 Subject: [PATCH 05/13] Add supported models to end-to-end tests. --- tests/python-pytest/onnx/test_onnxruntime.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 4dd034b67c40..a101c68fef50 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -377,12 +377,12 @@ def normalize_image(imgfile): 'simple_pose_resnet152_v1b', 'simple_pose_resnet152_v1d', 'alpha_pose_resnet101_v1b_coco', - #'mobile_pose_resnet18_v1b', - #'mobile_pose_resnet50_v1b', - #'mobile_pose_mobilenet1.0', - #'mobile_pose_mobilenetv2_1.0', - #'mobile_pose_mobilenetv3_large', - #'mobile_pose_mobilenetv3_small', + 'mobile_pose_resnet18_v1b', + 'mobile_pose_resnet50_v1b', + 'mobile_pose_mobilenet1.0', + 'mobile_pose_mobilenetv2_1.0', + 'mobile_pose_mobilenetv3_large', + 'mobile_pose_mobilenetv3_small', ]) def test_pose_estimation_model_inference_onnxruntime(tmp_path, model): def normalize_image(imgfile): From b93fe7874c62fe271fe65ba278bc78999b38c3c9 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Tue, 9 Feb 2021 03:18:44 +0000 Subject: [PATCH 06/13] Split onnx tests into different pipelines for now. --- ci/docker/runtime_functions.sh | 55 ++++++++++++++++++++- ci/jenkins/Jenkins_steps.groovy | 86 ++++++++++++++++++++++++++++++++- ci/jenkins/Jenkinsfile_unix_cpu | 6 +++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index c389cf92913e..c24d004f1d89 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1258,7 +1258,60 @@ integrationtest_ubuntu_cpu_onnx() { pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py - pytest -n 24 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py +} + +integrationtest_ubuntu_cpu_onnx_export_nlp_models() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime +} + +integrationtest_ubuntu_cpu_onnx_export_obj_class_models() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime +} + +integrationtest_ubuntu_cpu_onnx_export_obj_det_models() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime +} + +integrationtest_ubuntu_cpu_onnx_export_img_seg_models() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime +} + +integrationtest_ubuntu_cpu_onnx_export_pose_est_models() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime +} + +integrationtest_ubuntu_cpu_onnx_export_act_rec_models() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime } integrationtest_ubuntu_gpu_python() { diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index a40e66a407d5..3a74c197947e 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1184,7 +1184,7 @@ def test_unix_julia10_cpu() { } def test_unix_onnx_cpu() { - return ['Onnx CPU': { + return ['Onnx CPU - Unit Tests': { node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { @@ -1197,6 +1197,90 @@ def test_unix_onnx_cpu() { }] } +def test_unix_onnx_cpu_nlp_export_tests() { + return ['Onnx CPU - NLP Model Export Tests': { + node(NODE_LINUX_CPU) { + ws('workspace/it-onnx-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_nlp_models', false) + utils.publish_test_coverage() + } + } + } + }] +} + +def test_unix_onnx_cpu_obj_class_export_tests() { + return ['Onnx CPU - Obj Class Export Tests': { + node(NODE_LINUX_CPU) { + ws('workspace/it-onnx-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_obj_class_models', false) + utils.publish_test_coverage() + } + } + } + }] +} + +def test_unix_onnx_cpu_obj_det_export_tests() { + return ['Onnx CPU - Obj Det Export Tests': { + node(NODE_LINUX_CPU) { + ws('workspace/it-onnx-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_obj_det_models', false) + utils.publish_test_coverage() + } + } + } + }] +} + +def test_unix_onnx_cpu_img_seg_export_tests() { + return ['Onnx CPU - Img Seg Export Tests': { + node(NODE_LINUX_CPU) { + ws('workspace/it-onnx-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_img_seg_models', false) + utils.publish_test_coverage() + } + } + } + }] +} + +def test_unix_onnx_cpu_pose_est_export_tests() { + return ['Onnx CPU - Pose Est Export Tests': { + node(NODE_LINUX_CPU) { + ws('workspace/it-onnx-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_pose_est_models', false) + utils.publish_test_coverage() + } + } + } + }] +} + +def test_unix_onnx_cpu_act_rec_export_tests() { + return ['Onnx CPU - Act Rec Export Tests': { + node(NODE_LINUX_CPU) { + ws('workspace/it-onnx-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_act_rec_models', false) + utils.publish_test_coverage() + } + } + } + }] +} + def test_unix_distributed_kvstore_cpu() { return ['dist-kvstore tests CPU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 71917de58e82..61401ffe2572 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -59,6 +59,12 @@ core_logic: { custom_steps.test_unix_julia07_cpu(), custom_steps.test_unix_julia10_cpu(), custom_steps.test_unix_onnx_cpu(), + custom_steps.test_unix_onnx_cpu_nlp_export_tests(), + custom_steps.test_unix_onnx_cpu_obj_class_export_tests(), + custom_steps.test_unix_onnx_cpu_obj_det_export_tests(), + custom_steps.test_unix_onnx_cpu_img_seg_export_tests(), + custom_steps.test_unix_onnx_cpu_pose_est_export_tests(), + custom_steps.test_unix_onnx_cpu_act_rec_export_tests(), custom_steps.test_unix_cpp_cpu(), custom_steps.test_static_scala_cpu(), custom_steps.test_static_python_cpu(), From 0d425dac2391a2252e43809a31db24fa831c0653 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 10 Feb 2021 05:15:28 +0000 Subject: [PATCH 07/13] Split onnx test stage into 4 different stages to reduce load and pipeline duration. --- ci/docker/runtime_functions.sh | 40 +++++------------------- ci/jenkins/Jenkins_steps.groovy | 54 ++++----------------------------- ci/jenkins/Jenkinsfile_unix_cpu | 7 ++--- 3 files changed, 16 insertions(+), 85 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index c24d004f1d89..226c93ba46a0 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1254,10 +1254,10 @@ integrationtest_ubuntu_cpu_onnx() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py pytest $COV_ARG --verbose tests/python-pytest/onnx/mxnet_export_test.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py } integrationtest_ubuntu_cpu_onnx_export_nlp_models() { @@ -1269,49 +1269,25 @@ integrationtest_ubuntu_cpu_onnx_export_nlp_models() { pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime } -integrationtest_ubuntu_cpu_onnx_export_obj_class_models() { - set -ex - export PYTHONPATH=./python/ - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime -} - -integrationtest_ubuntu_cpu_onnx_export_obj_det_models() { - set -ex - export PYTHONPATH=./python/ - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime -} - -integrationtest_ubuntu_cpu_onnx_export_img_seg_models() { - set -ex - export PYTHONPATH=./python/ - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime -} - -integrationtest_ubuntu_cpu_onnx_export_pose_est_models() { +integrationtest_ubuntu_cpu_onnx_export_cv_models1() { set -ex export PYTHONPATH=./python/ export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime + pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime } -integrationtest_ubuntu_cpu_onnx_export_act_rec_models() { +integrationtest_ubuntu_cpu_onnx_export_cv_models2() { set -ex export PYTHONPATH=./python/ export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime + pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime + pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime + pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime + pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime } integrationtest_ubuntu_gpu_python() { diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 3a74c197947e..aee7e1f8d224 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1211,13 +1211,13 @@ def test_unix_onnx_cpu_nlp_export_tests() { }] } -def test_unix_onnx_cpu_obj_class_export_tests() { - return ['Onnx CPU - Obj Class Export Tests': { +def test_unix_onnx_cpu_cv_export_tests1() { + return ['Onnx CPU - CV Model Export Tests 1': { node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_obj_class_models', false) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_cv_models1', false) utils.publish_test_coverage() } } @@ -1225,55 +1225,13 @@ def test_unix_onnx_cpu_obj_class_export_tests() { }] } -def test_unix_onnx_cpu_obj_det_export_tests() { - return ['Onnx CPU - Obj Det Export Tests': { +def test_unix_onnx_cpu_cv_export_tests2() { + return ['Onnx CPU - CV Model Export Tests 2': { node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_obj_det_models', false) - utils.publish_test_coverage() - } - } - } - }] -} - -def test_unix_onnx_cpu_img_seg_export_tests() { - return ['Onnx CPU - Img Seg Export Tests': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_img_seg_models', false) - utils.publish_test_coverage() - } - } - } - }] -} - -def test_unix_onnx_cpu_pose_est_export_tests() { - return ['Onnx CPU - Pose Est Export Tests': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_pose_est_models', false) - utils.publish_test_coverage() - } - } - } - }] -} - -def test_unix_onnx_cpu_act_rec_export_tests() { - return ['Onnx CPU - Act Rec Export Tests': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_act_rec_models', false) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_cv_models2', false) utils.publish_test_coverage() } } diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 61401ffe2572..c16d591a551d 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -60,11 +60,8 @@ core_logic: { custom_steps.test_unix_julia10_cpu(), custom_steps.test_unix_onnx_cpu(), custom_steps.test_unix_onnx_cpu_nlp_export_tests(), - custom_steps.test_unix_onnx_cpu_obj_class_export_tests(), - custom_steps.test_unix_onnx_cpu_obj_det_export_tests(), - custom_steps.test_unix_onnx_cpu_img_seg_export_tests(), - custom_steps.test_unix_onnx_cpu_pose_est_export_tests(), - custom_steps.test_unix_onnx_cpu_act_rec_export_tests(), + custom_steps.test_unix_onnx_cpu_cv_export_tests1(), + custom_steps.test_unix_onnx_cpu_cv_export_tests2(), custom_steps.test_unix_cpp_cpu(), custom_steps.test_static_scala_cpu(), custom_steps.test_static_python_cpu(), From 752745e4766e8cd6fd4cb67b8e692e05355e9edb Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 10 Feb 2021 17:18:16 +0000 Subject: [PATCH 08/13] Further divide onnx runtime tests to prevent a stage from taking too long. --- ci/docker/runtime_functions.sh | 8 ++++++++ ci/jenkins/Jenkins_steps.groovy | 14 ++++++++++++++ ci/jenkins/Jenkinsfile_unix_cpu | 1 + 3 files changed, 23 insertions(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 226c93ba46a0..193ad9dd0ace 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1286,6 +1286,14 @@ integrationtest_ubuntu_cpu_onnx_export_cv_models2() { COV_ARG="--cov=./ --cov-report=xml --cov-append" pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime +} + +integrationtest_ubuntu_cpu_onnx_export_cv_models3() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index aee7e1f8d224..fedc214721cc 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1239,6 +1239,20 @@ def test_unix_onnx_cpu_cv_export_tests2() { }] } +def test_unix_onnx_cpu_cv_export_tests3() { + return ['Onnx CPU - CV Model Export Tests 3': { + node(NODE_LINUX_CPU) { + ws('workspace/it-onnx-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_cv_models3', false) + utils.publish_test_coverage() + } + } + } + }] +} + def test_unix_distributed_kvstore_cpu() { return ['dist-kvstore tests CPU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index c16d591a551d..8d7a54991517 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -62,6 +62,7 @@ core_logic: { custom_steps.test_unix_onnx_cpu_nlp_export_tests(), custom_steps.test_unix_onnx_cpu_cv_export_tests1(), custom_steps.test_unix_onnx_cpu_cv_export_tests2(), + custom_steps.test_unix_onnx_cpu_cv_export_tests3(), custom_steps.test_unix_cpp_cpu(), custom_steps.test_static_scala_cpu(), custom_steps.test_static_python_cpu(), From ed7e4eaa95b613832622d249e70d41f00a3efcf0 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 10 Feb 2021 21:23:03 +0000 Subject: [PATCH 09/13] Increase process count for onnx tests to decrease runtime. --- ci/docker/runtime_functions.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 193ad9dd0ace..272cdb479a33 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1275,7 +1275,7 @@ integrationtest_ubuntu_cpu_onnx_export_cv_models1() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime } integrationtest_ubuntu_cpu_onnx_export_cv_models2() { @@ -1284,8 +1284,8 @@ integrationtest_ubuntu_cpu_onnx_export_cv_models2() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime - pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime } integrationtest_ubuntu_cpu_onnx_export_cv_models3() { @@ -1294,8 +1294,8 @@ integrationtest_ubuntu_cpu_onnx_export_cv_models3() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime - pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime + pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime } integrationtest_ubuntu_gpu_python() { From bc2f2b28be908e34c5972c45bab3f72c342f0ad8 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 10 Feb 2021 22:40:47 +0000 Subject: [PATCH 10/13] It turns out it's faster to just run a single pytest process. --- ci/docker/runtime_functions.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 272cdb479a33..42b248bae9b1 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1266,7 +1266,7 @@ integrationtest_ubuntu_cpu_onnx_export_nlp_models() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime } integrationtest_ubuntu_cpu_onnx_export_cv_models1() { @@ -1275,7 +1275,7 @@ integrationtest_ubuntu_cpu_onnx_export_cv_models1() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime } integrationtest_ubuntu_cpu_onnx_export_cv_models2() { @@ -1284,8 +1284,8 @@ integrationtest_ubuntu_cpu_onnx_export_cv_models2() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime } integrationtest_ubuntu_cpu_onnx_export_cv_models3() { @@ -1294,8 +1294,8 @@ integrationtest_ubuntu_cpu_onnx_export_cv_models3() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime - pytest -n 8 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime } integrationtest_ubuntu_gpu_python() { From 12559eaa6c672568f0c341309b45cbb16c85a41a Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 10 Feb 2021 22:43:59 +0000 Subject: [PATCH 11/13] Move test data to test fixtures so the data can be reused across tests. --- tests/python-pytest/onnx/test_onnxruntime.py | 182 +++++++++++-------- 1 file changed, 111 insertions(+), 71 deletions(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index a101c68fef50..ce372c78e250 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -59,15 +59,6 @@ def predict(self, data): return self.model(data) -def download_test_images(image_urls, tmpdir): - from urllib.parse import urlparse - paths = [] - for url in image_urls: - filename = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) - mx.test_utils.download(url, fname=filename) - paths.append(filename) - return paths - @with_seed() @pytest.mark.parametrize('model', ['bert_12_768_12']) @@ -123,6 +114,30 @@ def test_bert_inference_onnxruntime(tmp_path, model): shutil.rmtree(tmp_path) + +@pytest.fixture(scope="session") +def obj_class_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("obj_class_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths + @pytest.mark.parametrize('model', [ 'alexnet', 'cifar_resnet20_v1', @@ -194,7 +209,7 @@ def test_bert_inference_onnxruntime(tmp_path, model): 'xception', 'inceptionv3' ]) -def test_obj_class_model_inference_onnxruntime(tmp_path, model): +def test_obj_class_model_inference_onnxruntime(tmp_path, model, obj_class_test_images): inlen = 299 if 'inceptionv3' == model else 224 def normalize_image(imgfile): img_data = mx.image.imread(imgfile) @@ -218,20 +233,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in obj_class_test_images: img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()})[0] @@ -241,6 +243,26 @@ def normalize_image(imgfile): shutil.rmtree(tmp_path) +@pytest.fixture(scope="session") +def obj_detection_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("obj_det_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths @pytest.mark.parametrize('model', [ @@ -268,7 +290,7 @@ def normalize_image(imgfile): #'yolo3_darknet53_coco', #'yolo3_mobilenet1.0_coco', ]) -def test_obj_detection_model_inference_onnxruntime(tmp_path, model): +def test_obj_detection_model_inference_onnxruntime(tmp_path, model, obj_detection_test_images): def normalize_image(imgfile): img = mx.image.imread(imgfile) img, _ = mx.image.center_crop(img, size=(512, 512)) @@ -285,18 +307,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in obj_detection_test_images: img_data = normalize_image(img) mx_class_ids, mx_scores, mx_boxes = M.predict(img_data) onnx_scores, onnx_class_ids, onnx_boxes = session.run([], {input_name: img_data.asnumpy()}) @@ -307,6 +318,28 @@ def normalize_image(imgfile): finally: shutil.rmtree(tmp_path) +@pytest.fixture(scope="session") +def img_segmentation_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("img_seg_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths @pytest.mark.parametrize('model', [ 'fcn_resnet50_ade', @@ -326,7 +359,7 @@ def normalize_image(imgfile): 'deeplab_resnet101_citys', 'deeplab_v3b_plus_wideresnet_citys' ]) -def test_img_segmentation_model_inference_onnxruntime(tmp_path, model): +def test_img_segmentation_model_inference_onnxruntime(tmp_path, model, img_segmentation_test_images): def normalize_image(imgfile): img = mx.image.imread(imgfile).astype('float32') img, _ = mx.image.center_crop(img, size=(480, 480)) @@ -344,20 +377,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in img_segmentation_test_images: img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()}) @@ -368,6 +388,25 @@ def normalize_image(imgfile): finally: shutil.rmtree(tmp_path) + +@pytest.fixture(scope="session") +def pose_estimation_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("pose_est_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths + @pytest.mark.parametrize('model', [ 'simple_pose_resnet18_v1b', 'simple_pose_resnet50_v1b', @@ -384,7 +423,7 @@ def normalize_image(imgfile): 'mobile_pose_mobilenetv3_large', 'mobile_pose_mobilenetv3_small', ]) -def test_pose_estimation_model_inference_onnxruntime(tmp_path, model): +def test_pose_estimation_model_inference_onnxruntime(tmp_path, model, pose_estimation_test_images): def normalize_image(imgfile): img = mx.image.imread(imgfile).astype('float32') img, _ = mx.image.center_crop(img, size=(512, 512)) @@ -401,15 +440,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in pose_estimation_test_images: img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()}) @@ -420,6 +451,22 @@ def normalize_image(imgfile): finally: shutil.rmtree(tmp_path) +@pytest.fixture(scope="session") +def act_recognition_test_data(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("act_rec_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/biking.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/diving.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/golfing.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/sledding.rec', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths @pytest.mark.parametrize('model', [ 'inceptionv1_kinetics400', @@ -434,7 +481,7 @@ def normalize_image(imgfile): 'inceptionv3_kinetics400', 'inceptionv3_ucf101', ]) -def test_action_recognition_model_inference_onnxruntime(tmp_path, model): +def test_action_recognition_model_inference_onnxruntime(tmp_path, model, act_recognition_test_data): batch_size = 64 input_len = 224 if 'inceptionv3' in model: @@ -455,14 +502,7 @@ def load_video(filepath): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_video_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/biking.rec', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/diving.rec', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/golfing.rec', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/sledding.rec', - ] - - for video in download_test_images(test_video_urls, tmp_path): + for video in act_recognition_test_data: data = load_video(video) mx_result = M.predict(data) onnx_result = session.run([], {input_name: data.asnumpy()})[0] From 659c0c11c639d9965881ea4a974a4c8ea022dde6 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 11 Feb 2021 05:51:02 +0000 Subject: [PATCH 12/13] Move all onnxruntime tests to nightly test pipeline, only test a few CV models for active pipeline due to resource and time limits. --- ci/docker/runtime_functions.sh | 59 +++++++++++----------------- ci/jenkins/Jenkins_steps.groovy | 56 -------------------------- ci/jenkins/Jenkinsfile_unix_cpu | 4 -- tests/nightly/JenkinsfileForBinaries | 8 ++++ 4 files changed, 30 insertions(+), 97 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 42b248bae9b1..ac488c4a9cfc 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1258,44 +1258,15 @@ integrationtest_ubuntu_cpu_onnx() { pytest $COV_ARG --verbose tests/python-pytest/onnx/mxnet_export_test.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py -} - -integrationtest_ubuntu_cpu_onnx_export_nlp_models() { - set -ex - export PYTHONPATH=./python/ - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - COV_ARG="--cov=./ --cov-report=xml --cov-append" pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime -} - -integrationtest_ubuntu_cpu_onnx_export_cv_models1() { - set -ex - export PYTHONPATH=./python/ - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime -} - -integrationtest_ubuntu_cpu_onnx_export_cv_models2() { - set -ex - export PYTHONPATH=./python/ - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime -} - -integrationtest_ubuntu_cpu_onnx_export_cv_models3() { - set -ex - export PYTHONPATH=./python/ - export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=10 - COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[mobilenetv3_large] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnest200] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnet50_v2] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[vgg19_bn] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime[center_net_resnet101_v1b_voc] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime[deeplab_resnet50_citys] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime[mobile_pose_mobilenet1.0] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime[inceptionv3_kinetics400] } integrationtest_ubuntu_gpu_python() { @@ -1624,6 +1595,20 @@ nightly_estimator() { nosetests test_sentiment_rnn.py } +nightly_onnx_tests() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime +} + # For testing PRs deploy_docs() { set -ex diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index fedc214721cc..c985a182bb28 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1197,62 +1197,6 @@ def test_unix_onnx_cpu() { }] } -def test_unix_onnx_cpu_nlp_export_tests() { - return ['Onnx CPU - NLP Model Export Tests': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_nlp_models', false) - utils.publish_test_coverage() - } - } - } - }] -} - -def test_unix_onnx_cpu_cv_export_tests1() { - return ['Onnx CPU - CV Model Export Tests 1': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_cv_models1', false) - utils.publish_test_coverage() - } - } - } - }] -} - -def test_unix_onnx_cpu_cv_export_tests2() { - return ['Onnx CPU - CV Model Export Tests 2': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_cv_models2', false) - utils.publish_test_coverage() - } - } - } - }] -} - -def test_unix_onnx_cpu_cv_export_tests3() { - return ['Onnx CPU - CV Model Export Tests 3': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx_export_cv_models3', false) - utils.publish_test_coverage() - } - } - } - }] -} - def test_unix_distributed_kvstore_cpu() { return ['dist-kvstore tests CPU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 8d7a54991517..71917de58e82 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -59,10 +59,6 @@ core_logic: { custom_steps.test_unix_julia07_cpu(), custom_steps.test_unix_julia10_cpu(), custom_steps.test_unix_onnx_cpu(), - custom_steps.test_unix_onnx_cpu_nlp_export_tests(), - custom_steps.test_unix_onnx_cpu_cv_export_tests1(), - custom_steps.test_unix_onnx_cpu_cv_export_tests2(), - custom_steps.test_unix_onnx_cpu_cv_export_tests3(), custom_steps.test_unix_cpp_cpu(), custom_steps.test_static_scala_cpu(), custom_steps.test_static_python_cpu(), diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 1a7ef1aaaa05..00f543fb58cb 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -93,6 +93,14 @@ core_logic: { utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true) } } + }, + 'ONNX: CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/onnx-test-cpu') { + utils.unpack_and_init('cpu_int64', mx_cmake_lib) + utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_tests', false) + } + } } } } From cc4da398550018a09903ef3565a7a3834d64b207 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Fri, 12 Feb 2021 00:11:48 +0000 Subject: [PATCH 13/13] Just run all tests in test_onnxruntime.py. --- ci/docker/runtime_functions.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index ac488c4a9cfc..5db09f322b0e 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1601,12 +1601,7 @@ nightly_onnx_tests() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py } # For testing PRs