diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 0ae9079e3026..f88e867b0d49 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -939,8 +939,7 @@ unittest_centos7_gpu() { integrationtest_ubuntu_cpu_onnx() { set -ex export PYTHONPATH=./python/ - pytest tests/python-pytest/onnx/gluon_backend_test.py - pytest tests/python-pytest/onnx/mxnet_backend_test.py + python tests/python-pytest/onnx/backend_test.py pytest tests/python-pytest/onnx/mxnet_export_test.py pytest tests/python-pytest/onnx/test_models.py pytest tests/python-pytest/onnx/test_node.py diff --git a/tests/python-pytest/onnx/backend_test.py b/tests/python-pytest/onnx/backend_test.py index 6c6c3d2d9c7d..048a6782c245 100644 --- a/tests/python-pytest/onnx/backend_test.py +++ b/tests/python-pytest/onnx/backend_test.py @@ -22,30 +22,51 @@ raise ImportError("Onnx and protobuf need to be installed") import test_cases +import unittest +import backend as mxnet_backend +import logging +operations = ['import', 'export'] +backends = ['mxnet', 'gluon'] +# This is a pytest magic variable to load extra plugins +pytest_plugins = "onnx.backend.test.report", -def prepare_tests(backend, operation): + +def test_suite(backend_tests): # type: () -> unittest.TestSuite + ''' + TestSuite that can be run by TestRunner + This has been borrowed from onnx/onnx/backend/test/runner/__init__.py, + since Python3 cannot sort objects of type 'Type' as Runner.test_suite() + expects. + ''' + suite = unittest.TestSuite() + for case in backend_tests.test_cases.values(): + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(case)) + return suite + + +def prepare_tests(backend, oper): """ Prepare the test list :param backend: mxnet/gluon backend - :param operation: str. export or import + :param oper: str. export or import :return: backend test list """ BACKEND_TESTS = onnx.backend.test.BackendTest(backend, __name__) implemented_ops = test_cases.IMPLEMENTED_OPERATORS_TEST.get('both', []) + \ - test_cases.IMPLEMENTED_OPERATORS_TEST.get(operation, []) + test_cases.IMPLEMENTED_OPERATORS_TEST.get(oper, []) for op_test in implemented_ops: BACKEND_TESTS.include(op_test) basic_models = test_cases.BASIC_MODEL_TESTS.get('both', []) + \ - test_cases.BASIC_MODEL_TESTS.get(operation, []) + test_cases.BASIC_MODEL_TESTS.get(oper, []) for basic_model_test in basic_models: BACKEND_TESTS.include(basic_model_test) std_models = test_cases.STANDARD_MODEL.get('both', []) + \ - test_cases.STANDARD_MODEL.get(operation, []) + test_cases.STANDARD_MODEL.get(oper, []) for std_model_test in std_models: BACKEND_TESTS.include(std_model_test) @@ -53,3 +74,15 @@ def prepare_tests(backend, operation): BACKEND_TESTS.exclude('.*bcast.*') return BACKEND_TESTS + + +for bkend in backends: + for operation in operations: + log = logging.getLogger(bkend + operation) + if bkend == 'gluon' and operation == 'export': + log.warning('Gluon->ONNX export not implemented. Skipping tests...') + continue + log.info('Executing tests for ' + bkend + ' backend: ' + operation) + mxnet_backend.MXNetBackend.set_params(bkend, operation) + BACKEND_TESTS = prepare_tests(mxnet_backend, operation) + unittest.TextTestRunner().run(test_suite(BACKEND_TESTS.enable_report())) diff --git a/tests/python-pytest/onnx/gluon_backend_test.py b/tests/python-pytest/onnx/gluon_backend_test.py deleted file mode 100644 index 0f320aeb0e08..000000000000 --- a/tests/python-pytest/onnx/gluon_backend_test.py +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""ONNX test backend wrapper""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import unittest -import backend as mxnet_backend -import backend_test - -try: - import onnx.backend.test -except ImportError: - raise ImportError("Onnx and protobuf need to be installed") - -operations = ['import'] # Gluon->ONNX exprot is not supported yet -# This is a pytest magic variable to load extra plugins -pytest_plugins = "onnx.backend.test.report", - - -for operation in operations: - mxnet_backend.MXNetBackend.set_params('gluon', operation) - BACKEND_TESTS = backend_test.prepare_tests(mxnet_backend, operation) - # import all test cases at global scope to make them visible to python.unittest - globals().update(BACKEND_TESTS.enable_report().test_cases) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/python-pytest/onnx/mxnet_backend_test.py b/tests/python-pytest/onnx/mxnet_backend_test.py deleted file mode 100644 index bf249fe687b8..000000000000 --- a/tests/python-pytest/onnx/mxnet_backend_test.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""ONNX test backend wrapper""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - - -import unittest -import backend as mxnet_backend -import backend_test - -try: - import onnx.backend.test -except ImportError: - raise ImportError("Onnx and protobuf need to be installed") - -operations = ['import', 'export'] -# This is a pytest magic variable to load extra plugins -pytest_plugins = "onnx.backend.test.report", - - -for operation in operations: - mxnet_backend.MXNetBackend.set_params('mxnet', operation) - BACKEND_TESTS = backend_test.prepare_tests(mxnet_backend, operation) - # import all test cases at global scope to make them visible to python.unittest - globals().update(BACKEND_TESTS.enable_report().test_cases) - -if __name__ == '__main__': - unittest.main()