diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index ad9bbe50f980..0ae9079e3026 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -939,11 +939,11 @@ unittest_centos7_gpu() { integrationtest_ubuntu_cpu_onnx() { set -ex export PYTHONPATH=./python/ - pytest tests/python-pytest/onnx/import/mxnet_backend_test.py - pytest tests/python-pytest/onnx/import/onnx_import_test.py - pytest tests/python-pytest/onnx/import/gluon_backend_test.py - pytest tests/python-pytest/onnx/export/onnx_backend_test.py - python tests/python-pytest/onnx/export/mxnet_export_test.py + pytest tests/python-pytest/onnx/gluon_backend_test.py + pytest tests/python-pytest/onnx/mxnet_backend_test.py + pytest tests/python-pytest/onnx/mxnet_export_test.py + pytest tests/python-pytest/onnx/test_models.py + pytest tests/python-pytest/onnx/test_node.py } integrationtest_ubuntu_gpu_python() { diff --git a/tests/python-pytest/onnx/README.md b/tests/python-pytest/onnx/README.md new file mode 100644 index 000000000000..d8f58cba3d5c --- /dev/null +++ b/tests/python-pytest/onnx/README.md @@ -0,0 +1,33 @@ +# ONNX tests + +## Directory structure: + +```bash +. +├── README.md +├── backend.py +├── backend_rep.py +├── backend_test.py +├── gluon_backend_test.py +├── mxnet_backend_test.py +├── mxnet_export_test.py +├── test_cases.py +├── test_models.py +└── test_node.py +``` + +* `backend.py` - MXNetBackend. This file contains prepare(). \ +This class can be used for both, MXNet and Gluon backend. +* `backend_rep.py` - MXNetBackendRep and GluonBackendRep for running inference +* `backend_test.py` - prepare tests by including tests from `test_cases.py` +* `gluon_backend_test.py` - Set backend as gluon and execute ONNX tests for ONNX->Gluon import. +* `mxnet_backend_test.py` - Set backend as gluon and add tests for ONNX->MXNet import/export. +Since MXNetBackend for export, tests both import and export, the test list in this file is +a union of tests that execute for import and export, export alone, and import alone. +* `mxnet_export_test.py` - Execute unit tests for testing MXNet export code - this is not specific to +any operator. +* `test_cases.py` - list of test cases for operators/models that are supported +for "both", import and export, "import" alone, or "export" alone. +* `test_models.py` - custom tests for models +* `test_node.py` - custom tests for operators. These tests are written independent of ONNX tests, in case +ONNX doesn't have tests yet or for MXNet specific operators. \ No newline at end of file diff --git a/tests/python-pytest/onnx/export/backend.py b/tests/python-pytest/onnx/backend.py similarity index 57% rename from tests/python-pytest/onnx/export/backend.py rename to tests/python-pytest/onnx/backend.py index 3ea1dafca255..2f9e2470d225 100644 --- a/tests/python-pytest/onnx/export/backend.py +++ b/tests/python-pytest/onnx/backend.py @@ -16,51 +16,57 @@ # under the License. # coding: utf-8 -"""backend wrapper for onnx test infrastructure""" -import os -import sys -import numpy as np +"""MXNet/Gluon backend wrapper for onnx test infrastructure""" + from mxnet.contrib.onnx.onnx2mx.import_onnx import GraphProto from mxnet.contrib.onnx.mx2onnx.export_onnx import MXNetGraph +import mxnet as mx +import numpy as np + try: from onnx import helper, TensorProto, mapping from onnx.backend.base import Backend except ImportError: - raise ImportError("Onnx and protobuf need to be installed") -CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -sys.path.insert(0, os.path.join(CURR_PATH, '../')) -from backend_rep import MXNetBackendRep + raise ImportError("Onnx and protobuf need to be installed. Instructions to" + + " install - https://github.com/onnx/onnx#installation") +from backend_rep import MXNetBackendRep, GluonBackendRep + -# Using these functions for onnx test infrastructure. -# Implemented by following onnx docs guide: -# https://github.com/onnx/onnx/blob/master/docs/Implementing%20an%20ONNX%20backend.md # MXNetBackend class will take an ONNX model with inputs, perform a computation, # and then return the output. +# Implemented by following onnx docs guide: +# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md class MXNetBackend(Backend): - """MXNet backend for ONNX""" + """MXNet/Gluon backend for ONNX""" + + backend = 'mxnet' + operation = 'import' + + @classmethod + def set_params(cls, backend, operation): + cls.backend = backend + cls.operation = operation @staticmethod - def perform_import_export(graph_proto, input_shape): + def perform_import_export(sym, arg_params, aux_params, input_shape): """ Import ONNX model to mxnet model and then export to ONNX model and then import it back to mxnet for verifying the result""" graph = GraphProto() - sym, arg_params, aux_params = graph.from_onnx(graph_proto) - params = {} params.update(arg_params) params.update(aux_params) # exporting to onnx graph proto format converter = MXNetGraph() - graph_proto = converter.create_onnx_graph_proto(sym, params, in_shape=input_shape, in_type=mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype('float32')]) + graph_proto = converter.create_onnx_graph_proto(sym, params, in_shape=input_shape, + in_type=mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype('float32')]) # importing back to MXNET for verifying result. sym, arg_params, aux_params = graph.from_onnx(graph_proto) return sym, arg_params, aux_params - @classmethod def prepare(cls, model, device='CPU', **kwargs): """For running end to end model(used for onnx test backend) @@ -80,13 +86,31 @@ def prepare(cls, model, device='CPU', **kwargs): Returns object of MXNetBackendRep class which will be in turn used to run inference on the input model and return the result for comparison. """ + backend = kwargs.get('backend', cls.backend) + operation = kwargs.get('operation', cls.operation) graph = GraphProto() - metadata = graph.get_graph_metadata(model.graph) - input_data = metadata['input_tensor_data'] - input_shape = [data[1] for data in input_data] - sym, arg_params, aux_params = MXNetBackend.perform_import_export(model.graph, input_shape) - return MXNetBackendRep(sym, arg_params, aux_params, device) + if device == 'CPU': + ctx = mx.cpu() + else: + raise NotImplementedError("ONNX tests are run only for CPU context.") + + if backend == 'mxnet': + sym, arg_params, aux_params = graph.from_onnx(model.graph) + if operation == 'export': + metadata = graph.get_graph_metadata(model.graph) + input_data = metadata['input_tensor_data'] + input_shape = [data[1] for data in input_data] + sym, arg_params, aux_params = MXNetBackend.perform_import_export(sym, arg_params, aux_params, + input_shape) + + return MXNetBackendRep(sym, arg_params, aux_params, device) + elif backend == 'gluon': + if operation == 'import': + net = graph.graph_to_gluon(model.graph, ctx) + return GluonBackendRep(net, device) + elif operation == 'export': + raise NotImplementedError("Gluon->ONNX export not implemented.") @classmethod def supports_device(cls, device): @@ -96,6 +120,4 @@ def supports_device(cls, device): prepare = MXNetBackend.prepare -run_node = MXNetBackend.run_node - supports_device = MXNetBackend.supports_device diff --git a/tests/python-pytest/onnx/backend_rep.py b/tests/python-pytest/onnx/backend_rep.py index 63836ac848df..bcb99ab025cc 100644 --- a/tests/python-pytest/onnx/backend_rep.py +++ b/tests/python-pytest/onnx/backend_rep.py @@ -22,7 +22,9 @@ except ImportError: raise ImportError("Onnx and protobuf need to be installed. Instructions to" + " install - https://github.com/onnx/onnx#installation") +import numpy as np import mxnet as mx +from mxnet import nd # Using these functions for onnx test infrastructure. # Implemented by following onnx docs guide: @@ -82,3 +84,47 @@ def run(self, inputs, **kwargs): exe.forward(is_train=False) result = exe.outputs[0].asnumpy() return [result] + + +# GluonBackendRep object will be returned by GluonBackend's prepare method which is used to +# execute a model repeatedly. +# Inputs will be passed to the run method of MXNetBackendRep class, it will perform computation and +# retrieve the corresponding results for comparison to the onnx backend. +# https://github.com/onnx/onnx/blob/master/onnx/backend/test/runner/__init__.py. +# Implemented by following onnx docs guide: +# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md + +class GluonBackendRep(BackendRep): + """Running model inference on gluon backend and return the result + to onnx test infrastructure for comparison.""" + def __init__(self, net, device): + self.net = net + self.device = device + + def run(self, inputs, **kwargs): + """Run model inference and return the result + + Parameters + ---------- + inputs : numpy array + input to run a layer on + + Returns + ------- + params : numpy array + result obtained after running the inference on mxnet + """ + # create module, passing cpu context + if self.device == 'CPU': + ctx = mx.cpu() + else: + raise NotImplementedError("ONNX tests are run only for CPU context.") + + # run inference + net_inputs = [nd.array(input_data, ctx=ctx) for input_data in inputs] + net_outputs = self.net(*net_inputs) + results = [] + results.extend([o for o in net_outputs.asnumpy()]) + result = np.array(results) + + return [result] diff --git a/tests/python-pytest/onnx/backend_test.py b/tests/python-pytest/onnx/backend_test.py new file mode 100644 index 000000000000..6c6c3d2d9c7d --- /dev/null +++ b/tests/python-pytest/onnx/backend_test.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""ONNX test backend wrapper""" +try: + import onnx.backend.test +except ImportError: + raise ImportError("Onnx and protobuf need to be installed") + +import test_cases + + +def prepare_tests(backend, operation): + """ + Prepare the test list + :param backend: mxnet/gluon backend + :param operation: str. export or import + :return: backend test list + """ + BACKEND_TESTS = onnx.backend.test.BackendTest(backend, __name__) + implemented_ops = test_cases.IMPLEMENTED_OPERATORS_TEST.get('both', []) + \ + test_cases.IMPLEMENTED_OPERATORS_TEST.get(operation, []) + + for op_test in implemented_ops: + BACKEND_TESTS.include(op_test) + + basic_models = test_cases.BASIC_MODEL_TESTS.get('both', []) + \ + test_cases.BASIC_MODEL_TESTS.get(operation, []) + + for basic_model_test in basic_models: + BACKEND_TESTS.include(basic_model_test) + + std_models = test_cases.STANDARD_MODEL.get('both', []) + \ + test_cases.STANDARD_MODEL.get(operation, []) + + for std_model_test in std_models: + BACKEND_TESTS.include(std_model_test) + + BACKEND_TESTS.exclude('.*bcast.*') + + return BACKEND_TESTS diff --git a/tests/python-pytest/onnx/export/mxnet_export_test.py b/tests/python-pytest/onnx/export/mxnet_export_test.py deleted file mode 100644 index b4fa4b12c781..000000000000 --- a/tests/python-pytest/onnx/export/mxnet_export_test.py +++ /dev/null @@ -1,495 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Tests for individual operators -This module contains operator tests which currently do not exist on -ONNX backend test framework. Once we have PRs on the ONNX repo and get -those PRs merged, this file will get EOL'ed. -""" -# pylint: disable=too-many-locals,wrong-import-position,import-error -from __future__ import absolute_import -import sys -import os -import unittest -import logging -import tarfile -import tempfile -from collections import namedtuple -import numpy as np -import numpy.testing as npt -from onnx import numpy_helper, helper -from onnx import TensorProto -from mxnet import nd, sym -from mxnet.gluon import nn -from mxnet.test_utils import download -from mxnet.contrib import onnx as onnx_mxnet -import mxnet as mx -CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -sys.path.insert(0, os.path.join(CURR_PATH, '../../../python/unittest')) -import backend -from common import with_seed - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) -URLS = { - 'bvlc_googlenet': - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/bvlc_googlenet.tar.gz', - 'bvlc_reference_caffenet': - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/bvlc_reference_caffenet.tar.gz', - 'bvlc_reference_rcnn_ilsvrc13': - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/bvlc_reference_rcnn_ilsvrc13.tar.gz', - 'inception_v1': - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/inception_v1.tar.gz', - 'inception_v2': - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/inception_v2.tar.gz' -} - -def get_test_files(name): - """Extract tar file and returns model path and input, output data""" - tar_name = download(URLS.get(name), dirname=CURR_PATH.__str__()) - # extract tar file - tar_path = os.path.join(CURR_PATH, tar_name) - tar = tarfile.open(tar_path.__str__(), "r:*") - tar.extractall(path=CURR_PATH.__str__()) - tar.close() - data_dir = os.path.join(CURR_PATH, name) - model_path = os.path.join(data_dir, 'model.onnx') - - inputs = [] - outputs = [] - # get test files - for test_file in os.listdir(data_dir): - case_dir = os.path.join(data_dir, test_file) - # skip the non-dir files - if not os.path.isdir(case_dir): - continue - input_file = os.path.join(case_dir, 'input_0.pb') - input_tensor = TensorProto() - with open(input_file, 'rb') as proto_file: - input_tensor.ParseFromString(proto_file.read()) - inputs.append(numpy_helper.to_array(input_tensor)) - - output_tensor = TensorProto() - output_file = os.path.join(case_dir, 'output_0.pb') - with open(output_file, 'rb') as proto_file: - output_tensor.ParseFromString(proto_file.read()) - outputs.append(numpy_helper.to_array(output_tensor)) - - return model_path, inputs, outputs - - -def forward_pass(sym, arg, aux, data_names, input_data): - """ Perform forward pass on given data - :param sym: Symbol - :param arg: Arg params - :param aux: Aux params - :param data_names: Input names (list) - :param input_data: Input data (list). If there is only one input, - pass it as a list. For example, if input is [1, 2], - pass input_data=[[1, 2]] - :return: result of forward pass - """ - # create module - mod = mx.mod.Module(symbol=sym, data_names=data_names, context=mx.cpu(), label_names=None) - - data_shapes = [] - data_forward = [] - for idx in range(len(data_names)): - val = input_data[idx] - data_shapes.append((data_names[idx], np.shape(val))) - data_forward.append(mx.nd.array(val)) - - mod.bind(for_training=False, data_shapes=data_shapes, label_shapes=None) - mod.set_params(arg_params=arg, aux_params=aux, - allow_missing=True, allow_extra=True) - - # run inference - batch = namedtuple('Batch', ['data']) - mod.forward(batch(data_forward), is_train=False) - - return mod.get_outputs()[0].asnumpy() - - -def test_models(model_name, input_shape, output_shape): - """ Tests Googlenet model for both onnx import and export""" - model_path, inputs, outputs = get_test_files(model_name) - logging.info("Translating model from ONNX model zoo to Mxnet") - sym, arg_params, aux_params = onnx_mxnet.import_model(model_path) - params = {} - params.update(arg_params) - params.update(aux_params) - - dir_path = os.path.dirname(model_path) - new_model_name = "exported_" + model_name + ".onnx" - onnx_file = os.path.join(dir_path, new_model_name) - - logging.info("Translating converted model from mxnet to ONNX") - converted_model_path = onnx_mxnet.export_model(sym, params, [input_shape], np.float32, onnx_file) - - sym, arg_params, aux_params = onnx_mxnet.import_model(converted_model_path) - - metadata = onnx_mxnet.get_model_metadata(converted_model_path) - assert len(metadata) == 2 - assert metadata.get('input_tensor_data') - assert metadata.get('input_tensor_data')[0][1] == input_shape - assert metadata.get('output_tensor_data') - assert metadata.get('output_tensor_data')[0][1] == output_shape - data_names = [input_name[0] for input_name in metadata.get('input_tensor_data')] - - logging.info("Running inference on onnx re-import model in mxnet") - # run test for each test file - for input_data, output_data in zip(inputs, outputs): - result = forward_pass(sym, arg_params, aux_params, data_names, [input_data]) - - # verify the results - npt.assert_equal(result.shape, output_data.shape) - npt.assert_almost_equal(output_data, result, decimal=3) - logging.info(model_name + " conversion successful") - - -def test_model_accuracy(model_name, input_shape): - """ Imports ONNX model, runs inference, exports and imports back - run inference, compare result with the previous inference result""" - model_path, inputs, outputs = get_test_files(model_name) - logging.info("Translating model from ONNX model zoo to Mxnet") - sym, arg_params, aux_params = onnx_mxnet.import_model(model_path) - - metadata = onnx_mxnet.get_model_metadata(model_path) - data_names = [input_name[0] for input_name in metadata.get('input_tensor_data')] - - expected_result= [] - for input_data, output_data in zip(inputs, outputs): - result = forward_pass(sym, arg_params, aux_params, data_names, [input_data]) - expected_result.append(result) - - params = {} - params.update(arg_params) - params.update(aux_params) - - dir_path = os.path.dirname(model_path) - new_model_name = "exported_" + model_name + ".onnx" - onnx_file = os.path.join(dir_path, new_model_name) - - logging.info("Translating converted model from mxnet to ONNX") - converted_model_path = onnx_mxnet.export_model(sym, params, [input_shape], np.float32, - onnx_file) - - sym, arg_params, aux_params = onnx_mxnet.import_model(converted_model_path) - - metadata = onnx_mxnet.get_model_metadata(converted_model_path) - data_names = [input_name[0] for input_name in metadata.get('input_tensor_data')] - - actual_result = [] - for input_data, output_data in zip(inputs, outputs): - result = forward_pass(sym, arg_params, aux_params, data_names, [input_data]) - actual_result.append(result) - - # verify the results - for expected, actual in zip(expected_result, actual_result): - npt.assert_equal(expected.shape, actual.shape) - npt.assert_almost_equal(expected, actual, decimal=3) - -@with_seed() -def test_spacetodepth(): - n, c, h, w = shape = (1, 1, 4, 6) - input1 = np.random.rand(n, c, h, w).astype("float32") - blocksize = 2 - inputs = [helper.make_tensor_value_info("input1", TensorProto.FLOAT, shape=shape)] - - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=(1, 4, 2, 3))] - - nodes = [helper.make_node("SpaceToDepth", ["input1"], ["output"], block_size=blocksize)] - - graph = helper.make_graph(nodes, - "spacetodepth_test", - inputs, - outputs) - - spacetodepth_model = helper.make_model(graph) - - bkd_rep = backend.prepare(spacetodepth_model) - output = bkd_rep.run([input1]) - - tmp = np.reshape(input1, [n, c, - h // blocksize, blocksize, - w // blocksize, blocksize]) - tmp = np.transpose(tmp, [0, 3, 5, 1, 2, 4]) - numpy_op = np.reshape(tmp, [n, c * (blocksize**2), - h // blocksize, - w // blocksize]) - - npt.assert_almost_equal(output[0], numpy_op) - -@with_seed() -def test_square(): - input1 = np.random.randint(1, 10, (2, 3)).astype("float32") - - ipsym = mx.sym.Variable("input1") - square = mx.sym.square(data=ipsym) - model = mx.mod.Module(symbol=square, data_names=['input1'], label_names=None) - model.bind(for_training=False, data_shapes=[('input1', np.shape(input1))], label_shapes=None) - model.init_params() - - args, auxs = model.get_params() - params = {} - params.update(args) - params.update(auxs) - - converted_model = onnx_mxnet.export_model(square, params, [np.shape(input1)], np.float32, "square.onnx") - - sym, arg_params, aux_params = onnx_mxnet.import_model(converted_model) - result = forward_pass(sym, arg_params, aux_params, ['input1'], [input1]) - - numpy_op = np.square(input1) - - npt.assert_almost_equal(result, numpy_op) - - -@with_seed() -def test_fully_connected(): - def random_arrays(*shapes): - """Generate some random numpy arrays.""" - arrays = [np.random.randn(*s).astype("float32") - for s in shapes] - if len(arrays) == 1: - return arrays[0] - return arrays - - data_names = ['x', 'w', 'b'] - - dim_in, dim_out = (3, 4) - input_data = random_arrays((4, dim_in), (dim_out, dim_in), (dim_out,)) - - ipsym = [] - data_shapes = [] - data_forward = [] - for idx in range(len(data_names)): - val = input_data[idx] - data_shapes.append((data_names[idx], np.shape(val))) - data_forward.append(mx.nd.array(val)) - ipsym.append(mx.sym.Variable(data_names[idx])) - - op = mx.sym.FullyConnected(data=ipsym[0], weight=ipsym[1], bias=ipsym[2], num_hidden=dim_out, name='FC') - - model = mx.mod.Module(op, data_names=data_names, label_names=None) - model.bind(for_training=False, data_shapes=data_shapes, label_shapes=None) - - model.init_params() - - args, auxs = model.get_params() - params = {} - params.update(args) - params.update(auxs) - - converted_model = onnx_mxnet.export_model(op, params, [shape[1] for shape in data_shapes], np.float32, "fc.onnx") - - sym, arg_params, aux_params = onnx_mxnet.import_model(converted_model) - result = forward_pass(sym, arg_params, aux_params, data_names, input_data) - - numpy_op = np.dot(input_data[0], input_data[1].T) + input_data[2] - - npt.assert_almost_equal(result, numpy_op) - - -def test_softmax(): - input1 = np.random.rand(1000, 1000).astype("float32") - label1 = np.random.rand(1000) - input_nd = mx.nd.array(input1) - label_nd = mx.nd.array(label1) - - ipsym = mx.sym.Variable("ipsym") - label = mx.sym.Variable('label') - sym = mx.sym.SoftmaxOutput(data=ipsym, label=label, ignore_label=0, use_ignore=False) - ex = sym.bind(ctx=mx.cpu(0), args={'ipsym': input_nd, 'label': label_nd}) - ex.forward(is_train=True) - softmax_out = ex.outputs[0].asnumpy() - - converted_model = onnx_mxnet.export_model(sym, {}, [(1000, 1000), (1000,)], np.float32, "softmaxop.onnx") - - sym, arg_params, aux_params = onnx_mxnet.import_model(converted_model) - result = forward_pass(sym, arg_params, aux_params, ['ipsym'], input1) - - # Comparing result of forward pass before using onnx export, import - npt.assert_almost_equal(result, softmax_out) - -@with_seed() -def test_comparison_ops(): - """Test greater, lesser, equal""" - def test_ops(op_name, inputs, input_tensors, numpy_op): - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=np.shape(inputs[0]))] - nodes = [helper.make_node(op_name, ["input"+str(i+1) for i in range(len(inputs))], ["output"])] - graph = helper.make_graph(nodes, - op_name + "_test", - input_tensors, - outputs) - model = helper.make_model(graph) - bkd_rep = backend.prepare(model) - output = bkd_rep.run(inputs) - npt.assert_almost_equal(output[0], numpy_op) - input_data = [np.random.rand(1, 3, 4, 5).astype("float32"), - np.random.rand(1, 5).astype("float32")] - input_tensor = [] - for idx, ip in enumerate(input_data): - input_tensor.append(helper.make_tensor_value_info("input" + str(idx + 1), - TensorProto.FLOAT, shape=np.shape(ip))) - test_ops("Greater", input_data, input_tensor, - np.greater(input_data[0], input_data[1]).astype(np.float32)) - test_ops("Less", input_data, input_tensor, - np.less(input_data[0], input_data[1]).astype(np.float32)) - test_ops("Equal", input_data, input_tensor, - np.equal(input_data[0], input_data[1]).astype(np.float32)) - - -def get_int_inputs(interval, shape): - """Helper to get integer input of given shape and range""" - assert len(interval) == len(shape) - inputs = [] - input_tensors = [] - for idx in range(len(interval)): - low, high = interval[idx] - inputs.append(np.random.randint(low, high, size=shape[idx]).astype("float32")) - input_tensors.append(helper.make_tensor_value_info("input"+str(idx+1), - TensorProto.FLOAT, shape=shape[idx])) - return inputs, input_tensors - - -@with_seed() -def test_logical_ops(): - """Test for logical and, or, not, xor operators""" - def test_ops(op_name, inputs, input_tensors, numpy_op): - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=np.shape(inputs[0]))] - nodes = [helper.make_node(op_name, ["input"+str(i+1) for i in range(len(inputs))], ["output"])] - graph = helper.make_graph(nodes, - op_name + "_test", - input_tensors, - outputs) - model = helper.make_model(graph) - bkd_rep = backend.prepare(model) - output = bkd_rep.run(inputs) - npt.assert_almost_equal(output[0], numpy_op) - input_data, input_tensor = get_int_inputs([(0, 2), (0, 2)], [(3, 4, 5), (3, 4, 5)]) - test_ops("And", input_data, input_tensor, - np.logical_and(input_data[0], input_data[1]).astype(np.float32)) - test_ops("Or", input_data, input_tensor, - np.logical_or(input_data[0], input_data[1]).astype(np.float32)) - test_ops("Xor", input_data, input_tensor, - np.logical_xor(input_data[0], input_data[1]).astype(np.float32)) - test_ops("Not", [input_data[0]], [input_tensor[0]], - np.logical_not(input_data[0]).astype(np.float32)) - - -def _assert_sym_equal(lhs, rhs): - assert lhs.list_inputs() == rhs.list_inputs() # input names must be identical - assert len(lhs.list_outputs()) == len(rhs.list_outputs()) # number of outputs must be identical - - -def _force_list(output): - if isinstance(output, nd.NDArray): - return [output] - return list(output) - - -def _optional_group(symbols, group=False): - if group: - return sym.Group(symbols) - else: - return symbols - - -def _check_onnx_export(net, group_outputs=False, shape_type=tuple, extra_params={}): - net.initialize() - data = nd.random.uniform(0, 1, (1, 1024)) - output = _force_list(net(data)) # initialize weights - net_sym = _optional_group(net(sym.Variable('data')), group_outputs) - net_params = {name:param._reduce() for name, param in net.collect_params().items()} - net_params.update(extra_params) - with tempfile.TemporaryDirectory() as tmpdirname: - onnx_file_path = os.path.join(tmpdirname, 'net.onnx') - export_path = onnx_mxnet.export_model( - sym=net_sym, - params=net_params, - input_shape=[shape_type(data.shape)], - onnx_file_path=onnx_file_path) - assert export_path == onnx_file_path - # Try importing the model to symbol - _assert_sym_equal(net_sym, onnx_mxnet.import_model(export_path)[0]) - - # Try importing the model to gluon - imported_net = onnx_mxnet.import_to_gluon(export_path, ctx=None) - _assert_sym_equal(net_sym, _optional_group(imported_net(sym.Variable('data')), group_outputs)) - - # Confirm network outputs are the same - imported_net_output = _force_list(imported_net(data)) - for out, imp_out in zip(output, imported_net_output): - mx.test_utils.assert_almost_equal(out.asnumpy(), imp_out.asnumpy()) - - -@with_seed() -def test_onnx_export_single_output(): - net = nn.HybridSequential(prefix='single_output_net') - with net.name_scope(): - net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) - _check_onnx_export(net) - - -@with_seed() -def test_onnx_export_multi_output(): - class MultiOutputBlock(nn.HybridBlock): - def __init__(self): - super(MultiOutputBlock, self).__init__() - with self.name_scope(): - self.net = nn.HybridSequential() - for i in range(10): - self.net.add(nn.Dense(100 + i * 10, activation='relu')) - - def hybrid_forward(self, F, x): - out = tuple(block(x) for block in self.net._children.values()) - return out - - net = MultiOutputBlock() - assert len(sym.Group(net(sym.Variable('data'))).list_outputs()) == 10 - _check_onnx_export(net, group_outputs=True) - - -@with_seed() -def test_onnx_export_list_shape(): - net = nn.HybridSequential(prefix='list_shape_net') - with net.name_scope(): - net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) - _check_onnx_export(net, shape_type=list) - - -@with_seed() -def test_onnx_export_extra_params(): - net = nn.HybridSequential(prefix='extra_params_net') - with net.name_scope(): - net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) - _check_onnx_export(net, extra_params={'extra_param': nd.array([1, 2])}) - - -if __name__ == '__main__': - test_models("bvlc_googlenet", (1, 3, 224, 224), (1, 1000)) - test_models("bvlc_reference_caffenet", (1, 3, 224, 224), (1, 1000)) - test_models("bvlc_reference_rcnn_ilsvrc13", (1, 3, 224, 224), (1, 200)) - - # Comparing MXNet inference result, since MXNet results don't match - # ONNX expected results due to AveragePool issue github issue(#10194) - test_model_accuracy("inception_v1", (1, 3, 224, 224)) - test_model_accuracy("inception_v2", (1, 3, 224, 224)) - - unittest.main() diff --git a/tests/python-pytest/onnx/export/onnx_backend_test.py b/tests/python-pytest/onnx/export/onnx_backend_test.py deleted file mode 100644 index c9926c4d5e15..000000000000 --- a/tests/python-pytest/onnx/export/onnx_backend_test.py +++ /dev/null @@ -1,151 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""ONNX test backend wrapper""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import unittest -try: - import onnx.backend.test -except ImportError: - raise ImportError("Onnx and protobuf need to be installed") - -import backend as mxnet_backend - -# This is a pytest magic variable to load extra plugins -pytest_plugins = "onnx.backend.test.report", - -BACKEND_TESTS = onnx.backend.test.BackendTest(mxnet_backend, __name__) - -IMPLEMENTED_OPERATORS_TEST = [ - 'test_random_uniform', - 'test_random_normal', - 'test_add', - 'test_sub', - 'test_mul', - 'test_div', - 'test_neg', - 'test_abs', - 'test_sum', - 'test_tanh', - 'test_cos', - 'test_sin', - 'test_tan', - 'test_acos', - 'test_asin', - 'test_atan' - 'test_ceil', - 'test_floor', - 'test_concat', - 'test_identity', - 'test_sigmoid', - 'test_relu', - 'test_constant_pad', - 'test_edge_pad', - 'test_reflect_pad', - 'test_reduce_min', - 'test_reduce_max', - 'test_reduce_mean', - 'test_reduce_prod', - 'test_reduce_sum_d', - 'test_reduce_sum_keepdims_random', - 'test_squeeze', - 'test_softmax_example', - 'test_softmax_large_number', - 'test_softmax_axis_2', - 'test_transpose', - 'test_globalmaxpool', - 'test_globalaveragepool', - # enabling partial test cases for matmul - 'test_matmul_3d', - 'test_matmul_4d', - 'test_slice_cpu', - 'test_slice_neg', - 'test_squeeze_', - 'test_reciprocal', - 'test_sqrt', - 'test_pow', - 'test_exp_', - 'test_argmax', - 'test_argmin', - 'test_min', - 'test_max' - #pytorch operator tests - 'test_operator_exp', - 'test_operator_maxpool', - 'test_operator_params', - 'test_operator_permute2', - 'test_clip' - 'test_cast', - 'test_depthtospace', - 'test_hardsigmoid', - 'test_instancenorm', - 'test_shape', - 'test_size' - ] - -BASIC_MODEL_TESTS = [ - 'test_AvgPool2D', - 'test_BatchNorm', - 'test_ConstantPad2d', - 'test_Conv2d', - 'test_ELU', - 'test_LeakyReLU', - 'test_MaxPool', - 'test_PReLU', - 'test_ReLU', - 'test_selu_default' - 'test_Sigmoid', - 'test_Softmax', - 'test_softmax_functional', - 'test_softmax_lastdim', - 'test_Tanh' - ] - -STANDARD_MODEL = [ - 'test_bvlc_alexnet', - 'test_densenet121', - # 'test_inception_v1', - # 'test_inception_v2', - 'test_resnet50', - # 'test_shufflenet', - 'test_squeezenet', - 'test_vgg16', - 'test_vgg19' - ] - -for op_test in IMPLEMENTED_OPERATORS_TEST: - BACKEND_TESTS.include(op_test) - -for basic_model_test in BASIC_MODEL_TESTS: - BACKEND_TESTS.include(basic_model_test) - -for std_model_test in STANDARD_MODEL: - BACKEND_TESTS.include(std_model_test) - -BACKEND_TESTS.exclude('.*broadcast.*') -BACKEND_TESTS.exclude('.*bcast.*') - - -# import all test cases at global scope to make them visible to python.unittest -globals().update(BACKEND_TESTS.enable_report().test_cases) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/python-pytest/onnx/import/gluon_backend_test.py b/tests/python-pytest/onnx/gluon_backend_test.py similarity index 60% rename from tests/python-pytest/onnx/import/gluon_backend_test.py rename to tests/python-pytest/onnx/gluon_backend_test.py index 6dd5f8a071c9..0f320aeb0e08 100644 --- a/tests/python-pytest/onnx/import/gluon_backend_test.py +++ b/tests/python-pytest/onnx/gluon_backend_test.py @@ -22,34 +22,24 @@ from __future__ import unicode_literals import unittest +import backend as mxnet_backend +import backend_test + try: import onnx.backend.test except ImportError: - raise ImportError("Onnx and protobuf need to be installed. Instructions to" - + " install - https://github.com/onnx/onnx#installation") - -import gluon_backend -import test_cases + raise ImportError("Onnx and protobuf need to be installed") +operations = ['import'] # Gluon->ONNX exprot is not supported yet # This is a pytest magic variable to load extra plugins pytest_plugins = "onnx.backend.test.report", -BACKEND_TESTS = onnx.backend.test.BackendTest(gluon_backend, __name__) - -for op_tests in test_cases.IMPLEMENTED_OPERATORS_TEST: - BACKEND_TESTS.include(op_tests) - -for std_model_test in test_cases.STANDARD_MODEL: - BACKEND_TESTS.include(std_model_test) - -for basic_model_test in test_cases.BASIC_MODEL_TESTS: - BACKEND_TESTS.include(basic_model_test) - -BACKEND_TESTS.exclude('.*broadcast.*') -BACKEND_TESTS.exclude('.*bcast.*') -# import all test cases at global scope to make them visible to python.unittest -globals().update(BACKEND_TESTS.enable_report().test_cases) +for operation in operations: + mxnet_backend.MXNetBackend.set_params('gluon', operation) + BACKEND_TESTS = backend_test.prepare_tests(mxnet_backend, operation) + # import all test cases at global scope to make them visible to python.unittest + globals().update(BACKEND_TESTS.enable_report().test_cases) if __name__ == '__main__': unittest.main() diff --git a/tests/python-pytest/onnx/import/gluon_backend.py b/tests/python-pytest/onnx/import/gluon_backend.py deleted file mode 100644 index 25be60b57dc6..000000000000 --- a/tests/python-pytest/onnx/import/gluon_backend.py +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# coding: utf-8 -"""Gluon backend wrapper for onnx test infrastructure""" -from mxnet.contrib.onnx.onnx2mx.import_onnx import GraphProto -import mxnet as mx - -try: - from onnx import helper, TensorProto - from onnx.backend.base import Backend -except ImportError: - raise ImportError("Onnx and protobuf need to be installed. Instructions to" - + " install - https://github.com/onnx/onnx#installation") -from gluon_backend_rep import GluonBackendRep - -# GluonBackend class will take an ONNX model with inputs, perform a computation, -# and then return the output. -# Implemented by following onnx docs guide: -# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md - -class GluonBackend(Backend): - """Gluon backend for ONNX""" - - @classmethod - def prepare(cls, model, device='CPU', **kwargs): - """For running end to end model(used for onnx test backend) - - Parameters - ---------- - model : onnx ModelProto object - loaded onnx graph - device : 'CPU' - specifying device to run test on - kwargs : - other arguments - - Returns - ------- - GluonBackendRep : object - Returns object of GluonBackendRep class which will be in turn - used to run inference on the input model and return the result for comparison. - """ - graph = GraphProto() - if device == 'CPU': - ctx = mx.cpu() - else: - raise NotImplementedError("ONNX tests are run only for CPU context.") - - net = graph.graph_to_gluon(model.graph, ctx) - return GluonBackendRep(net, device) - - @classmethod - def supports_device(cls, device): - """Supports only CPU for testing""" - return device == 'CPU' - - -prepare = GluonBackend.prepare - -supports_device = GluonBackend.supports_device diff --git a/tests/python-pytest/onnx/import/gluon_backend_rep.py b/tests/python-pytest/onnx/import/gluon_backend_rep.py deleted file mode 100644 index 04c6ddde63e9..000000000000 --- a/tests/python-pytest/onnx/import/gluon_backend_rep.py +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# coding: utf-8 -"""gluon backend rep for onnx test infrastructure""" -import numpy as np -try: - from onnx.backend.base import BackendRep -except ImportError: - raise ImportError("Onnx and protobuf need to be installed. Instructions to" - + " install - https://github.com/onnx/onnx#installation") -import mxnet as mx -from mxnet import nd - -# GluonBackendRep object will be returned by GluonBackend's prepare method which is used to -# execute a model repeatedly. -# Inputs will be passed to the run method of MXNetBackendRep class, it will perform computation and -# retrieve the corresponding results for comparison to the onnx backend. -# https://github.com/onnx/onnx/blob/master/onnx/backend/test/runner/__init__.py. -# Implemented by following onnx docs guide: -# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md - - -class GluonBackendRep(BackendRep): - """Running model inference on gluon backend and return the result - to onnx test infrastructure for comparison.""" - def __init__(self, net, device): - self.net = net - self.device = device - - def run(self, inputs, **kwargs): - """Run model inference and return the result - - Parameters - ---------- - inputs : numpy array - input to run a layer on - - Returns - ------- - params : numpy array - result obtained after running the inference on mxnet - """ - # create module, passing cpu context - if self.device == 'CPU': - ctx = mx.cpu() - else: - raise NotImplementedError("ONNX tests are run only for CPU context.") - - # run inference - net_inputs = [nd.array(input_data, ctx=ctx) for input_data in inputs] - net_outputs = self.net(*net_inputs) - results = [] - results.extend([o for o in net_outputs.asnumpy()]) - result = np.array(results) - - return [result] diff --git a/tests/python-pytest/onnx/import/mxnet_backend.py b/tests/python-pytest/onnx/import/mxnet_backend.py deleted file mode 100644 index bd4910b64f85..000000000000 --- a/tests/python-pytest/onnx/import/mxnet_backend.py +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# coding: utf-8 -"""MXNet backend wrapper for onnx test infrastructure""" -import os -import sys -from mxnet.contrib.onnx.onnx2mx.import_onnx import GraphProto -try: - from onnx import helper, TensorProto - from onnx.backend.base import Backend -except ImportError: - raise ImportError("Onnx and protobuf need to be installed. Instructions to" - + " install - https://github.com/onnx/onnx#installation") -CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -sys.path.insert(0, os.path.join(CURR_PATH, '../')) -from backend_rep import MXNetBackendRep - -# MXNetBackend class will take an ONNX model with inputs, perform a computation, -# and then return the output. -# Implemented by following onnx docs guide: -# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md - -class MXNetBackend(Backend): - """MXNet backend for ONNX""" - - @classmethod - def prepare(cls, model, device='CPU', **kwargs): - """For running end to end model(used for onnx test backend) - - Parameters - ---------- - model : onnx ModelProto object - loaded onnx graph - device : 'CPU' - specifying device to run test on - kwargs : - other arguments - - Returns - ------- - MXNetBackendRep : object - Returns object of MXNetBackendRep class which will be in turn - used to run inference on the input model and return the result for comparison. - """ - graph = GraphProto() - sym, arg_params, aux_params = graph.from_onnx(model.graph) - return MXNetBackendRep(sym, arg_params, aux_params, device) - - @classmethod - def supports_device(cls, device): - """Supports only CPU for testing""" - return device == 'CPU' - -prepare = MXNetBackend.prepare - -supports_device = MXNetBackend.supports_device diff --git a/tests/python-pytest/onnx/import/onnx_import_test.py b/tests/python-pytest/onnx/import/onnx_import_test.py deleted file mode 100644 index c2d1e9cb2d36..000000000000 --- a/tests/python-pytest/onnx/import/onnx_import_test.py +++ /dev/null @@ -1,275 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Tests for individual operators -This module contains operator tests which currently do not exist on -ONNX backend test framework. Once we have PRs on the ONNX repo and get -those PRs merged, this file will get EOL'ed. -""" -# pylint: disable=too-many-locals,wrong-import-position,import-error -from __future__ import absolute_import -import sys -import os -import unittest -import logging -import hashlib -import tarfile -from collections import namedtuple -import numpy as np -import numpy.testing as npt -from onnx import helper -from onnx import numpy_helper -from onnx import TensorProto -from mxnet.test_utils import download -from mxnet.contrib import onnx as onnx_mxnet -import mxnet as mx -CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -sys.path.insert(0, os.path.join(CURR_PATH, '../../../python/unittest')) -from common import with_seed -import mxnet_backend - - -URLS = { - 'bvlc_googlenet' : - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/opset7/bvlc_googlenet.tar.gz', - 'bvlc_reference_caffenet' : - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/opset7/bvlc_reference_caffenet.tar.gz', - 'bvlc_reference_rcnn_ilsvrc13' : - 'https://s3.amazonaws.com/onnx-mxnet/model-zoo/opset7/bvlc_reference_rcnn_ilsvrc13.tar.gz', -} - -@with_seed() -def test_broadcast(): - """Test for broadcasting in onnx operators.""" - input1 = np.random.rand(1, 3, 4, 5).astype("float32") - input2 = np.random.rand(1, 5).astype("float32") - inputs = [helper.make_tensor_value_info("input1", TensorProto.FLOAT, shape=(1, 3, 4, 5)), - helper.make_tensor_value_info("input2", TensorProto.FLOAT, shape=(1, 5))] - - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=(1, 3, 4, 5))] - - nodes = [helper.make_node("Add", ["input1", "input2"], ["output"])] - - graph = helper.make_graph(nodes, - "bcast_test", - inputs, - outputs) - - bcast_model = helper.make_model(graph) - - bkd_rep = mxnet_backend.prepare(bcast_model) - numpy_op = input1 + input2 - output = bkd_rep.run([input1, input2]) - npt.assert_almost_equal(output[0], numpy_op) - -@with_seed() -def test_greater(): - """Test for logical greater in onnx operators.""" - input1 = np.random.rand(1, 3, 4, 5).astype("float32") - input2 = np.random.rand(1, 5).astype("float32") - inputs = [helper.make_tensor_value_info("input1", TensorProto.FLOAT, shape=(1, 3, 4, 5)), - helper.make_tensor_value_info("input2", TensorProto.FLOAT, shape=(1, 5))] - - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=(1, 3, 4, 5))] - - nodes = [helper.make_node("Greater", ["input1", "input2"], ["output"])] - - graph = helper.make_graph(nodes, - "greater_test", - inputs, - outputs) - - greater_model = helper.make_model(graph) - - bkd_rep = mxnet_backend.prepare(greater_model) - numpy_op = np.greater(input1, input2).astype(np.float32) - output = bkd_rep.run([input1, input2]) - npt.assert_almost_equal(output[0], numpy_op) - -@with_seed() -def test_lesser(): - """Test for logical greater in onnx operators.""" - input1 = np.random.rand(1, 3, 4, 5).astype("float32") - input2 = np.random.rand(1, 5).astype("float32") - inputs = [helper.make_tensor_value_info("input1", TensorProto.FLOAT, shape=(1, 3, 4, 5)), - helper.make_tensor_value_info("input2", TensorProto.FLOAT, shape=(1, 5))] - - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=(1, 3, 4, 5))] - - nodes = [helper.make_node("Less", ["input1", "input2"], ["output"])] - - graph = helper.make_graph(nodes, - "lesser_test", - inputs, - outputs) - - greater_model = helper.make_model(graph) - - bkd_rep = mxnet_backend.prepare(greater_model) - numpy_op = np.less(input1, input2).astype(np.float32) - output = bkd_rep.run([input1, input2]) - npt.assert_almost_equal(output[0], numpy_op) - -@with_seed() -def test_equal(): - """Test for logical greater in onnx operators.""" - input1 = np.random.rand(1, 3, 4, 5).astype("float32") - input2 = np.random.rand(1, 5).astype("float32") - inputs = [helper.make_tensor_value_info("input1", TensorProto.FLOAT, shape=(1, 3, 4, 5)), - helper.make_tensor_value_info("input2", TensorProto.FLOAT, shape=(1, 5))] - - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=(1, 3, 4, 5))] - - nodes = [helper.make_node("Equal", ["input1", "input2"], ["output"])] - - graph = helper.make_graph(nodes, - "equal_test", - inputs, - outputs) - - greater_model = helper.make_model(graph) - - bkd_rep = mxnet_backend.prepare(greater_model) - numpy_op = np.equal(input1, input2).astype(np.float32) - output = bkd_rep.run([input1, input2]) - npt.assert_almost_equal(output[0], numpy_op) - - -def get_test_files(name): - """Extract tar file and returns model path and input, output data""" - tar_name = download(URLS.get(name), dirname=CURR_PATH.__str__()) - # extract tar file - tar_path = os.path.join(CURR_PATH, tar_name) - tar = tarfile.open(tar_path.__str__(), "r:*") - tar.extractall(path=CURR_PATH.__str__()) - tar.close() - data_dir = os.path.join(CURR_PATH, name) - model_path = os.path.join(data_dir, 'model.onnx') - - inputs = [] - outputs = [] - # get test files - for test_file in os.listdir(data_dir): - case_dir = os.path.join(data_dir, test_file) - # skip the non-dir files - if not os.path.isdir(case_dir): - continue - input_file = os.path.join(case_dir, 'input_0.pb') - input_tensor = TensorProto() - with open(input_file, 'rb') as proto_file: - input_tensor.ParseFromString(proto_file.read()) - inputs.append(numpy_helper.to_array(input_tensor)) - - output_tensor = TensorProto() - output_file = os.path.join(case_dir, 'output_0.pb') - with open(output_file, 'rb') as proto_file: - output_tensor.ParseFromString(proto_file.read()) - outputs.append(numpy_helper.to_array(output_tensor)) - - return model_path, inputs, outputs - -def test_bvlc_googlenet(): - """ Tests Googlenet model""" - model_path, inputs, outputs = get_test_files('bvlc_googlenet') - logging.info("Translating Googlenet model from ONNX to Mxnet") - sym, arg_params, aux_params = onnx_mxnet.import_model(model_path) - metadata = onnx_mxnet.get_model_metadata(model_path) - assert len(metadata) == 2 - assert metadata.get('input_tensor_data') - assert metadata.get('input_tensor_data') == [(u'data_0', (1, 3, 224, 224))] - assert metadata.get('output_tensor_data') - assert metadata.get('output_tensor_data') == [(u'prob_1', (1, 1000))] - data_names = [input_name[0] for input_name in metadata.get('input_tensor_data')] - - # run test for each test file - for input_data, output_data in zip(inputs, outputs): - # create module - mod = mx.mod.Module(symbol=sym, data_names=data_names, context=mx.cpu(), label_names=None) - mod.bind(for_training=False, data_shapes=[(data_names[0], input_data.shape)], label_shapes=None) - mod.set_params(arg_params=arg_params, aux_params=aux_params, - allow_missing=True, allow_extra=True) - # run inference - batch = namedtuple('Batch', ['data']) - mod.forward(batch([mx.nd.array(input_data)]), is_train=False) - - # verify the results - npt.assert_equal(mod.get_outputs()[0].shape, output_data.shape) - npt.assert_almost_equal(output_data, mod.get_outputs()[0].asnumpy(), decimal=3) - logging.info("Googlenet model conversion Successful") - -def test_bvlc_reference_caffenet(): - """Tests the bvlc cafenet model""" - model_path, inputs, outputs = get_test_files('bvlc_reference_caffenet') - logging.info("Translating Caffenet model from ONNX to Mxnet") - sym, arg_params, aux_params = onnx_mxnet.import_model(model_path) - metadata = onnx_mxnet.get_model_metadata(model_path) - assert len(metadata) == 2 - assert metadata.get('input_tensor_data') - assert metadata.get('input_tensor_data') == [(u'data_0', (1, 3, 224, 224))] - assert metadata.get('output_tensor_data') - assert metadata.get('output_tensor_data') == [(u'prob_1', (1, 1000))] - data_names = [input_name[0] for input_name in metadata.get('input_tensor_data')] - - # run test for each test file - for input_data, output_data in zip(inputs, outputs): - # create module - mod = mx.mod.Module(symbol=sym, data_names=data_names, context=mx.cpu(), label_names=None) - mod.bind(for_training=False, data_shapes=[(data_names[0], input_data.shape)], label_shapes=None) - mod.set_params(arg_params=arg_params, aux_params=aux_params, - allow_missing=True, allow_extra=True) - # run inference - batch = namedtuple('Batch', ['data']) - mod.forward(batch([mx.nd.array(input_data)]), is_train=False) - - # verify the results - npt.assert_equal(mod.get_outputs()[0].shape, output_data.shape) - npt.assert_almost_equal(output_data, mod.get_outputs()[0].asnumpy(), decimal=3) - logging.info("Caffenet model conversion Successful") - -def test_bvlc_rcnn_ilsvrc13(): - """Tests the bvlc rcnn model""" - model_path, inputs, outputs = get_test_files('bvlc_reference_rcnn_ilsvrc13') - logging.info("Translating rcnn_ilsvrc13 model from ONNX to Mxnet") - sym, arg_params, aux_params = onnx_mxnet.import_model(model_path) - metadata = onnx_mxnet.get_model_metadata(model_path) - assert len(metadata) == 2 - assert metadata.get('input_tensor_data') - assert metadata.get('input_tensor_data') == [(u'data_0', (1, 3, 224, 224))] - assert metadata.get('output_tensor_data') - assert metadata.get('output_tensor_data') == [(u'fc-rcnn_1', (1, 200))] - data_names = [input_name[0] for input_name in metadata.get('input_tensor_data')] - - # run test for each test file - for input_data, output_data in zip(inputs, outputs): - # create module - mod = mx.mod.Module(symbol=sym, data_names=data_names, context=mx.cpu(), label_names=None) - mod.bind(for_training=False, data_shapes=[(data_names[0], input_data.shape)], label_shapes=None) - mod.set_params(arg_params=arg_params, aux_params=aux_params, - allow_missing=True, allow_extra=True) - # run inference - batch = namedtuple('Batch', ['data']) - mod.forward(batch([mx.nd.array(input_data)]), is_train=False) - - # verify the results - npt.assert_equal(mod.get_outputs()[0].shape, output_data.shape) - npt.assert_almost_equal(output_data, mod.get_outputs()[0].asnumpy(), decimal=3) - logging.info("rcnn_ilsvrc13 model conversion Successful") - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/python-pytest/onnx/import/test_cases.py b/tests/python-pytest/onnx/import/test_cases.py deleted file mode 100644 index e0b26cc49830..000000000000 --- a/tests/python-pytest/onnx/import/test_cases.py +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Test Cases to be run for the import module""" - -IMPLEMENTED_OPERATORS_TEST = [ - 'test_split_equal' - 'test_random_', - 'test_add', - 'test_sub', - 'test_mul', - 'test_div', - 'test_neg', - 'test_abs', - 'test_sum', - 'test_tanh', - 'test_ceil', - 'test_floor', - 'test_concat', - 'test_identity', - 'test_sigmoid', - 'test_relu', - 'test_constant_pad', - 'test_edge_pad', - 'test_reflect_pad', - 'test_squeeze_', - 'test_unsqueeze', - 'test_softmax_example', - 'test_softmax_large_number', - 'test_softmax_axis_2', - 'test_transpose', - 'test_globalmaxpool', - 'test_globalaveragepool', - 'test_global_lppooling', - 'test_slice_cpu', - 'test_slice_neg', - 'test_reciprocal', - 'test_sqrt', - 'test_pow', - 'test_exp_', - 'test_argmax', - 'test_argmin', - 'test_min', - # enabling partial test cases for matmul - 'test_matmul_3d', - 'test_matmul_4d', - 'test_clip', - 'test_softsign', - 'test_reduce_', - 'test_softplus', - 'test_mean', - 'test_acos', - 'test_asin', - 'test_atan', - 'test_cos', - 'test_sin', - 'test_tan', - 'test_shape', - 'test_hardsigmoid', - 'test_averagepool_1d', - 'test_averagepool_2d_pads_count_include_pad', - 'test_averagepool_2d_precomputed_pads_count_include_pad', - 'test_averagepool_2d_precomputed_strides', - 'test_averagepool_2d_strides', - 'test_averagepool_3d', - 'test_LpPool_', - 'test_cast', - 'test_instancenorm', - #pytorch operator tests - 'test_operator_exp', - 'test_operator_maxpool', - 'test_operator_params', - 'test_operator_permute2', - 'test_depthtospace', - 'test_size' - ] - -BASIC_MODEL_TESTS = [ - 'test_AvgPool2D', - 'test_BatchNorm', - 'test_ConstantPad2d' - 'test_Conv2d', - 'test_ELU', - 'test_LeakyReLU', - 'test_MaxPool', - 'test_PReLU', - 'test_ReLU', - 'test_selu_default', - 'test_Sigmoid', - 'test_Softmax', - 'test_softmax_functional', - 'test_softmax_lastdim', - 'test_Tanh' - ] - -STANDARD_MODEL = [ - 'test_bvlc_alexnet', - 'test_densenet121', - #'test_inception_v1', - #'test_inception_v2', - 'test_resnet50', - #'test_shufflenet', - 'test_squeezenet', - 'test_zfnet512', - 'test_vgg19' - ] diff --git a/tests/python-pytest/onnx/import/mxnet_backend_test.py b/tests/python-pytest/onnx/mxnet_backend_test.py similarity index 60% rename from tests/python-pytest/onnx/import/mxnet_backend_test.py rename to tests/python-pytest/onnx/mxnet_backend_test.py index d9e4dccae24e..bf249fe687b8 100644 --- a/tests/python-pytest/onnx/import/mxnet_backend_test.py +++ b/tests/python-pytest/onnx/mxnet_backend_test.py @@ -21,35 +21,26 @@ from __future__ import print_function from __future__ import unicode_literals + import unittest +import backend as mxnet_backend +import backend_test + try: import onnx.backend.test except ImportError: - raise ImportError("Onnx and protobuf need to be installed. Instructions to" - + " install - https://github.com/onnx/onnx#installation") - -import mxnet_backend -import test_cases + raise ImportError("Onnx and protobuf need to be installed") +operations = ['import', 'export'] # This is a pytest magic variable to load extra plugins pytest_plugins = "onnx.backend.test.report", -BACKEND_TESTS = onnx.backend.test.BackendTest(mxnet_backend, __name__) - -for op_tests in test_cases.IMPLEMENTED_OPERATORS_TEST: - BACKEND_TESTS.include(op_tests) - -for basic_model_test in test_cases.BASIC_MODEL_TESTS: - BACKEND_TESTS.include(basic_model_test) - -for std_model_test in test_cases.STANDARD_MODEL: - BACKEND_TESTS.include(std_model_test) - -BACKEND_TESTS.exclude('.*broadcast.*') -BACKEND_TESTS.exclude('.*bcast.*') -# import all test cases at global scope to make them visible to python.unittest -globals().update(BACKEND_TESTS.enable_report().test_cases) +for operation in operations: + mxnet_backend.MXNetBackend.set_params('mxnet', operation) + BACKEND_TESTS = backend_test.prepare_tests(mxnet_backend, operation) + # import all test cases at global scope to make them visible to python.unittest + globals().update(BACKEND_TESTS.enable_report().test_cases) if __name__ == '__main__': unittest.main() diff --git a/tests/python-pytest/onnx/mxnet_export_test.py b/tests/python-pytest/onnx/mxnet_export_test.py new file mode 100644 index 000000000000..6c81198a8bca --- /dev/null +++ b/tests/python-pytest/onnx/mxnet_export_test.py @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# pylint: disable=too-many-locals,wrong-import-position,import-error +from __future__ import absolute_import +import os +import unittest +import logging +import tempfile +from mxnet import nd, sym +from mxnet.gluon import nn +from mxnet.contrib import onnx as onnx_mxnet +import mxnet as mx + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +def _assert_sym_equal(lhs, rhs): + assert lhs.list_inputs() == rhs.list_inputs() # input names must be identical + assert len(lhs.list_outputs()) == len(rhs.list_outputs()) # number of outputs must be identical + + +def _force_list(output): + if isinstance(output, nd.NDArray): + return [output] + return list(output) + + +def _optional_group(symbols, group=False): + if group: + return sym.Group(symbols) + else: + return symbols + + +def _check_onnx_export(net, group_outputs=False, shape_type=tuple, extra_params={}): + net.initialize() + data = nd.random.uniform(0, 1, (1, 1024)) + output = _force_list(net(data)) # initialize weights + net_sym = _optional_group(net(sym.Variable('data')), group_outputs) + net_params = {name: param._reduce() for name, param in net.collect_params().items()} + net_params.update(extra_params) + with tempfile.TemporaryDirectory() as tmpdirname: + onnx_file_path = os.path.join(tmpdirname, 'net.onnx') + export_path = onnx_mxnet.export_model( + sym=net_sym, + params=net_params, + input_shape=[shape_type(data.shape)], + onnx_file_path=onnx_file_path) + assert export_path == onnx_file_path + # Try importing the model to symbol + _assert_sym_equal(net_sym, onnx_mxnet.import_model(export_path)[0]) + + # Try importing the model to gluon + imported_net = onnx_mxnet.import_to_gluon(export_path, ctx=None) + _assert_sym_equal(net_sym, _optional_group(imported_net(sym.Variable('data')), group_outputs)) + + # Confirm network outputs are the same + imported_net_output = _force_list(imported_net(data)) + for out, imp_out in zip(output, imported_net_output): + mx.test_utils.assert_almost_equal(out.asnumpy(), imp_out.asnumpy()) + + +class TestExport(unittest.TestCase): + """ Tests ONNX export. + """ + + def test_onnx_export_single_output(self): + net = nn.HybridSequential(prefix='single_output_net') + with net.name_scope(): + net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) + _check_onnx_export(net) + + def test_onnx_export_multi_output(self): + class MultiOutputBlock(nn.HybridBlock): + def __init__(self): + super(MultiOutputBlock, self).__init__() + with self.name_scope(): + self.net = nn.HybridSequential() + for i in range(10): + self.net.add(nn.Dense(100 + i * 10, activation='relu')) + + def hybrid_forward(self, F, x): + out = tuple(block(x) for block in self.net._children.values()) + return out + + net = MultiOutputBlock() + assert len(sym.Group(net(sym.Variable('data'))).list_outputs()) == 10 + _check_onnx_export(net, group_outputs=True) + + def test_onnx_export_list_shape(self): + net = nn.HybridSequential(prefix='list_shape_net') + with net.name_scope(): + net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) + _check_onnx_export(net, shape_type=list) + + def test_onnx_export_extra_params(self): + net = nn.HybridSequential(prefix='extra_params_net') + with net.name_scope(): + net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) + _check_onnx_export(net, extra_params={'extra_param': nd.array([1, 2])}) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/python-pytest/onnx/test_cases.py b/tests/python-pytest/onnx/test_cases.py new file mode 100644 index 000000000000..1001836cc408 --- /dev/null +++ b/tests/python-pytest/onnx/test_cases.py @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +IMPLEMENTED_OPERATORS_TEST = { + 'both': ['test_add', + 'test_sub', + 'test_mul', + 'test_div', + 'test_neg', + 'test_abs', + 'test_sum', + 'test_tanh', + 'test_ceil', + 'test_floor', + 'test_concat', + 'test_identity', + 'test_sigmoid', + 'test_relu', + 'test_constant_pad', + 'test_edge_pad', + 'test_reflect_pad', + 'test_softmax_example', + 'test_softmax_large_number', + 'test_softmax_axis_2', + 'test_transpose', + 'test_globalmaxpool', + 'test_globalaveragepool', + 'test_slice_cpu', + 'test_slice_neg', + 'test_reciprocal', + 'test_sqrt', + 'test_pow', + 'test_exp_', + 'test_argmax', + 'test_argmin', + 'test_min', + # pytorch operator tests + 'test_operator_exp', + 'test_operator_maxpool', + 'test_operator_params', + 'test_operator_permute2', + 'test_cos', + 'test_sin', + 'test_tan', + 'test_acos', + 'test_asin', + 'test_atan', + 'test_squeeze', + 'test_matmul_3d', + 'test_matmul_4d', + 'test_depthtospace', + 'test_hardsigmoid', + 'test_instancenorm', + 'test_shape', + 'test_cast', + 'test_clip', + 'test_size' + ], + 'import': ['test_unsqueeze', + 'test_global_lppooling', + 'test_softsign', + 'test_reduce_', + 'test_softplus', + 'test_mean', + 'test_averagepool_1d', + 'test_averagepool_2d_pads_count_include_pad', + 'test_averagepool_2d_precomputed_pads_count_include_pad', + 'test_averagepool_2d_precomputed_strides', + 'test_averagepool_2d_strides', + 'test_averagepool_3d', + 'test_LpPool_', + 'test_split_equal' + 'test_random_', + ], + 'export': ['test_random_uniform', + 'test_random_normal', + 'test_reduce_min', + 'test_reduce_max', + 'test_squeeze', + 'test_reduce_mean', + 'test_reduce_prod', + 'test_reduce_sum_d', + 'test_reduce_sum_keepdims_random', + 'test_max_', + ] +} + +BASIC_MODEL_TESTS = { + 'both': ['test_AvgPool2D', + 'test_BatchNorm', + 'test_ConstantPad2d' + 'test_Conv2d', + 'test_ELU', + 'test_LeakyReLU', + 'test_MaxPool', + 'test_PReLU', + 'test_ReLU', + 'test_selu_default', + 'test_Sigmoid', + 'test_Softmax', + 'test_softmax_functional', + 'test_softmax_lastdim', + 'test_Tanh'] +} + +STANDARD_MODEL = { + 'both': ['test_bvlc_alexnet', + 'test_densenet121', + # 'test_inception_v1', + # 'test_inception_v2', + 'test_resnet50', + # 'test_shufflenet', + 'test_squeezenet', + 'test_vgg19' + ], + 'import': ['test_zfnet512'], + 'export': ['test_vgg16'] +} diff --git a/tests/python-pytest/onnx/test_models.py b/tests/python-pytest/onnx/test_models.py new file mode 100644 index 000000000000..12bc2711190d --- /dev/null +++ b/tests/python-pytest/onnx/test_models.py @@ -0,0 +1,167 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# pylint: disable=too-many-locals,wrong-import-position,import-error +from __future__ import absolute_import +import sys +import os +import unittest +import logging +import tarfile +from collections import namedtuple +import numpy as np +import numpy.testing as npt +from onnx import numpy_helper +from onnx import TensorProto +from mxnet.test_utils import download +from mxnet.contrib import onnx as onnx_mxnet +import mxnet as mx + +CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +sys.path.insert(0, os.path.join(CURR_PATH, '../../python/unittest')) + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) +URLS = { + 'bvlc_googlenet': + 'https://s3.amazonaws.com/download.onnx/models/opset_8/bvlc_googlenet.tar.gz', + 'bvlc_reference_caffenet': + 'https://s3.amazonaws.com/download.onnx/models/opset_8/bvlc_reference_caffenet.tar.gz', + 'bvlc_reference_rcnn_ilsvrc13': + 'https://s3.amazonaws.com/download.onnx/models/opset_8/bvlc_reference_rcnn_ilsvrc13.tar.gz', + 'inception_v1': + 'https://s3.amazonaws.com/download.onnx/models/opset_8/inception_v1.tar.gz', + 'inception_v2': + 'https://s3.amazonaws.com/download.onnx/models/opset_8/inception_v2.tar.gz' +} + + +def get_test_files(name): + """Extract tar file and returns model path and input, output data""" + tar_name = download(URLS.get(name), dirname=CURR_PATH.__str__()) + # extract tar file + tar_path = os.path.join(CURR_PATH, tar_name) + tar = tarfile.open(tar_path.__str__(), "r:*") + tar.extractall(path=CURR_PATH.__str__()) + tar.close() + data_dir = os.path.join(CURR_PATH, name) + model_path = os.path.join(data_dir, 'model.onnx') + + inputs = [] + outputs = [] + # get test files + for test_file in os.listdir(data_dir): + case_dir = os.path.join(data_dir, test_file) + # skip the non-dir files + if not os.path.isdir(case_dir): + continue + input_file = os.path.join(case_dir, 'input_0.pb') + input_tensor = TensorProto() + with open(input_file, 'rb') as proto_file: + input_tensor.ParseFromString(proto_file.read()) + inputs.append(numpy_helper.to_array(input_tensor)) + + output_tensor = TensorProto() + output_file = os.path.join(case_dir, 'output_0.pb') + with open(output_file, 'rb') as proto_file: + output_tensor.ParseFromString(proto_file.read()) + outputs.append(numpy_helper.to_array(output_tensor)) + + return model_path, inputs, outputs + + +def forward_pass(sym, arg, aux, data_names, input_data): + """ Perform forward pass on given data""" + # create module + mod = mx.mod.Module(symbol=sym, data_names=data_names, context=mx.cpu(), label_names=None) + mod.bind(for_training=False, data_shapes=[(data_names[0], input_data.shape)], label_shapes=None) + mod.set_params(arg_params=arg, aux_params=aux, + allow_missing=True, allow_extra=True) + # run inference + batch = namedtuple('Batch', ['data']) + mod.forward(batch([mx.nd.array(input_data)]), is_train=False) + + return mod.get_outputs()[0].asnumpy() + + +class TestModel(unittest.TestCase): + """ Tests for models. + Tests are dynamically added. + Therefore edit test_models to add more tests. + """ + def test_import_export(self): + def get_model_results(modelpath): + symbol, args, aux = onnx_mxnet.import_model(modelpath) + + data = onnx_mxnet.get_model_metadata(modelpath) + data_names = [input_name[0] for input_name in data.get('input_tensor_data')] + + result = [] + for input_data, output_data in zip(inputs, outputs): + output = forward_pass(symbol, args, aux, data_names, input_data) + result.append(output) + return symbol, args, aux, result, data + + for test in test_cases: + model_name, input_shape, output_shape = test + with self.subTest(model_name): + model_path, inputs, outputs = get_test_files(model_name) + logging.info("Translating " + model_name + " from ONNX model zoo to MXNet") + + sym, arg_params, aux_params, expected_result, _ = get_model_results(model_path) + + params = {} + params.update(arg_params) + params.update(aux_params) + + dir_path = os.path.dirname(model_path) + new_model_name = "exported_" + model_name + ".onnx" + onnx_file = os.path.join(dir_path, new_model_name) + + logging.info("Translating converted model from mxnet to ONNX") + converted_model_path = onnx_mxnet.export_model(sym, params, [input_shape], np.float32, onnx_file) + + sym, arg_params, aux_params, actual_result, metadata = get_model_results(converted_model_path) + + assert len(metadata) == 2 + assert metadata.get('input_tensor_data') + assert metadata.get('input_tensor_data')[0][1] == input_shape + assert metadata.get('output_tensor_data') + assert metadata.get('output_tensor_data')[0][1] == output_shape + + # verify the results + for expected, actual in zip(expected_result, actual_result): + npt.assert_equal(expected.shape, actual.shape) + npt.assert_almost_equal(expected, actual, decimal=3) + + logging.info(model_name + " conversion successful") + + +# test_case = ("model name", input shape, output shape) +test_cases = [ + ("bvlc_googlenet", (1, 3, 224, 224), (1, 1000)), + ("bvlc_reference_caffenet", (1, 3, 224, 224), (1, 1000)), + ("bvlc_reference_rcnn_ilsvrc13", (1, 3, 224, 224), (1, 200)), + ("inception_v1", (1, 3, 224, 224), (1, 1000)), + ("inception_v2", (1, 3, 224, 224), (1, 1000)) +] + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/python-pytest/onnx/test_node.py b/tests/python-pytest/onnx/test_node.py new file mode 100644 index 000000000000..07ae866b96cf --- /dev/null +++ b/tests/python-pytest/onnx/test_node.py @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Tests for individual operators +This module contains operator tests which currently do not exist on +ONNX backend test framework. Once we have PRs on the ONNX repo and get +those PRs merged, this file will get EOL'ed. +""" +# pylint: disable=too-many-locals,wrong-import-position,import-error +from __future__ import absolute_import +import sys +import os +import unittest +import logging +import tarfile +from collections import namedtuple +import numpy as np +import numpy.testing as npt +from onnx import numpy_helper, helper, load_model +from onnx import TensorProto +from mxnet.test_utils import download +from mxnet.contrib import onnx as onnx_mxnet +import mxnet as mx +import backend + +CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +sys.path.insert(0, os.path.join(CURR_PATH, '../../python/unittest')) + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +def get_rnd(shape, low=-1.0, high=1.0, dtype=np.float32): + if dtype == np.float32: + return (np.random.uniform(low, high, + np.prod(shape)).reshape(shape).astype(np.float32)) + elif dtype == np.int32: + return (np.random.randint(low, high, + np.prod(shape)).reshape(shape).astype(np.float32)) + elif dtype == np.bool_: + return np.random.choice(a=[False, True], size=shape).astype(np.float32) + + +def forward_pass(sym, arg, aux, data_names, input_data): + """ Perform forward pass on given data + :param sym: Symbol + :param arg: Arg params + :param aux: Aux params + :param data_names: Input names (list) + :param input_data: Input data (list). If there is only one input, + pass it as a list. For example, if input is [1, 2], + pass input_data=[[1, 2]] + :return: result of forward pass + """ + data_shapes = [] + data_forward = [] + for idx in range(len(data_names)): + val = input_data[idx] + data_shapes.append((data_names[idx], np.shape(val))) + data_forward.append(mx.nd.array(val)) + # create module + mod = mx.mod.Module(symbol=sym, data_names=data_names, context=mx.cpu(), label_names=None) + mod.bind(for_training=False, data_shapes=data_shapes, label_shapes=None) + if not arg and not aux: + mod.init_params() + else: + mod.set_params(arg_params=arg, aux_params=aux, + allow_missing=True, allow_extra=True) + # run inference + batch = namedtuple('Batch', ['data']) + mod.forward(batch(data_forward), is_train=False) + + return mod.get_outputs()[0].asnumpy() + + +class TestNode(unittest.TestCase): + """ Tests for models. + Tests are dynamically added. + Therefore edit test_models to add more tests. + """ + + def test_import_export(self): + def get_input_tensors(input_data): + input_tensor = [] + input_names = [] + input_sym = [] + for idx, ip in enumerate(input_data): + name = "input" + str(idx + 1) + input_sym.append(mx.sym.Variable(name)) + input_names.append(name) + input_tensor.append(helper.make_tensor_value_info(name, + TensorProto.FLOAT, shape=np.shape(ip))) + return input_names, input_tensor, input_sym + + def get_onnx_graph(testname, input_names, inputs, output_name, output_shape, attr): + outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=output_shape)] + + nodes = [helper.make_node(output_name, input_names, ["output"], **attr)] + + graph = helper.make_graph(nodes, testname, inputs, outputs) + + model = helper.make_model(graph) + return model + + for test in test_cases: + test_name, mxnet_op, onnx_name, inputs, attrs, mxnet_specific = test + with self.subTest(test_name): + names, input_tensors, inputsym = get_input_tensors(inputs) + test_op = mxnet_op(*inputsym, **attrs) + mxnet_output = forward_pass(test_op, None, None, names, inputs) + outputshape = np.shape(mxnet_output) + + if mxnet_specific: + onnxmodelfile = onnx_mxnet.export_model(test_op, {}, [np.shape(ip) for ip in inputs], + np.float32, + onnx_name + ".onnx") + onnxmodel = load_model(onnxmodelfile) + else: + onnxmodel = get_onnx_graph(test_name, names, input_tensors, onnx_name, outputshape, attrs) + + bkd_rep = backend.prepare(onnxmodel, operation='export') + output = bkd_rep.run(inputs) + + npt.assert_almost_equal(output[0], mxnet_output) + + +# test_case = ("test_case_name", mxnet op, "ONNX_op_name", [input_list], attribute map, MXNet_specific=True/False) +test_cases = [ + ("test_equal", mx.sym.broadcast_equal, "Equal", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False), + ("test_greater", mx.sym.broadcast_greater, "Greater", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False), + ("test_less", mx.sym.broadcast_lesser, "Less", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False), + ("test_and", mx.sym.broadcast_logical_and, "And", + [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False), + ("test_xor", mx.sym.broadcast_logical_xor, "Xor", + [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False), + ("test_or", mx.sym.broadcast_logical_or, "Or", + [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False), + ("test_not", mx.sym.logical_not, "Not", [get_rnd((3, 4, 5), dtype=np.bool_)], {}, False), + ("test_square", mx.sym.square, "Pow", [get_rnd((2, 3), dtype=np.int32)], {}, True), + ("test_spacetodepth", mx.sym.space_to_depth, "SpaceToDepth", [get_rnd((1, 1, 4, 6))], + {'block_size': 2}, False), + ("test_softmax", mx.sym.SoftmaxOutput, "Softmax", [get_rnd((1000, 1000)), get_rnd(1000)], + {'ignore_label': 0, 'use_ignore': False}, True), + ("test_fullyconnected", mx.sym.FullyConnected, "Gemm", [get_rnd((4,3)), get_rnd((4, 3)), get_rnd(4)], + {'num_hidden': 4, 'name': 'FC'}, True) +] + +if __name__ == '__main__': + unittest.main()