[v1.x] Move majority of ONNX model tests to nightly, only test a few …

…models in PR pipeline (#19848)

* Enable end-to-end tests for all models we currently support exporting to onnx.

* Replace deprecated ConstantFill onnx operator with Identity, add commented out models that work with onnx export but fail with onnxruntime.

* Enable xception end-to-end test.

* Add inceptionv3 to object classification model tests and adapt to support different image sizes.

* Add supported models to end-to-end tests.

* Split onnx tests into different pipelines for now.

* Split onnx test stage into 4 different stages to reduce load and pipeline duration.

* Further divide onnx runtime tests to prevent a stage from taking too long.

* Increase process count for onnx tests to decrease runtime.

* It turns out it's faster to just run a single pytest process.

* Move test data to test fixtures so the data can be reused across tests.

* Move all onnxruntime tests to nightly test pipeline, only test a few CV models for active pipeline due to resource and time limits.

* Just run all tests in test_onnxruntime.py.

Co-authored-by: Joe Evans <[email protected]>

ci/docker/runtime_functions.sh

@@ -1254,11 +1254,19 @@ integrationtest_ubuntu_cpu_onnx() {
     export MXNET_SUBGRAPH_VERBOSE=0
     export DMLC_LOG_STACK_TRACE_DEPTH=10
     COV_ARG="--cov=./ --cov-report=xml --cov-append"
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py
     pytest $COV_ARG --verbose tests/python-pytest/onnx/mxnet_export_test.py
     pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py
     pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py
-    pytest -n 24 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[mobilenetv3_large]
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnest200]
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnet50_v2]
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[vgg19_bn]
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime[center_net_resnet101_v1b_voc]
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime[deeplab_resnet50_citys]
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime[mobile_pose_mobilenet1.0]
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime[inceptionv3_kinetics400]
 }
 
 integrationtest_ubuntu_gpu_python() {
@@ -1587,6 +1595,15 @@ nightly_estimator() {
     nosetests test_sentiment_rnn.py
 }
 
+nightly_onnx_tests() {
+    set -ex
+    export PYTHONPATH=./python/
+    export MXNET_SUBGRAPH_VERBOSE=0
+    export DMLC_LOG_STACK_TRACE_DEPTH=10
+    COV_ARG="--cov=./ --cov-report=xml --cov-append"
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py
+}
+
 # For testing PRs
 deploy_docs() {
     set -ex

ci/jenkins/Jenkins_steps.groovy

@@ -1184,7 +1184,7 @@ def test_unix_julia10_cpu() {
 }
 
 def test_unix_onnx_cpu() {
-    return ['Onnx CPU': {
+    return ['Onnx CPU - Unit Tests': {
       node(NODE_LINUX_CPU) {
         ws('workspace/it-onnx-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {

python/mxnet/contrib/onnx/mx2onnx/_op_translations.py

@@ -787,10 +787,10 @@ def convert_copy(node, **kwargs):
 
 @mx_op.register("identity")
 def convert_identity(node, **kwargs):
-    """Map MXNet's identity operator attributes to onnx's ConstantFill operator
+    """Map MXNet's identity operator attributes to onnx's Identity operator
     and return the created node.
     """
-    return create_basic_op_node('ConstantFill', node, kwargs)
+    return create_basic_op_node('Identity', node, kwargs)
 
 @mx_op.register("InstanceNorm")
 def convert_instancenorm(node, **kwargs):
@@ -1017,12 +1017,12 @@ def convert_logistic_regression_output(node, **kwargs):
 @mx_op.register("BlockGrad")
 def convert_blockgrad(node, **kwargs):
     """ Skip operator  """
-    return create_basic_op_node('ConstantFill', node, kwargs)
+    return create_basic_op_node('Identity', node, kwargs)
 
 @mx_op.register("MakeLoss")
 def convert_makeloss(node, **kwargs):
     """ Skip operator  """
-    return create_basic_op_node('ConstantFill', node, kwargs)
+    return create_basic_op_node('Identity', node, kwargs)
 
 @mx_op.register("Concat")
 def convert_concat(node, **kwargs):

tests/nightly/JenkinsfileForBinaries

@@ -93,6 +93,14 @@ core_logic: {
           utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true)
         }
       }
+    },
+    'ONNX: CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/onnx-test-cpu') {
+          utils.unpack_and_init('cpu_int64', mx_cmake_lib)
+          utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_tests', false)
+        }
+      }
     }
   }
 }