Skip to content

Update onnx_backend_test_series.py #146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
snnn merged 2 commits into from
Dec 13, 2018
Merged

Update onnx_backend_test_series.py #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cc42e32
Update onnx_backend_test_series.py
snnn Dec 11, 2018
b00be62
Update BUILD.md
snnn Dec 11, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions BUILD.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,21 @@ OS/Compiler Matrix:
ONNX Runtime python binding only supports Python 3.x. Please use python 3.5+.

## Build
Install cmake-3.11 or better from https://cmake.org/download/.
1. Checkout the source tree:
```
git clone --recursive https://github.com/Microsoft/onnxruntime
cd onnxruntime
```
2. Install cmake-3.11 or better from https://cmake.org/download/.
3. (optional) Install protobuf 3.6.1 from source code(cmake/external/protobuf). CMake flag protobuf\_BUILD\_SHARED\_LIBS must be turned off. After the installation, you should have the 'protoc' executable in your PATH.
4. (optional) Install onnx from source code(cmake/external/onnx)
```
export ONNX_ML=1
python3 setup.py bdist_wheel
pip3 install --upgrade dist/*.whl
```
5. Run './build.sh --config RelWithDebInfo --build\_wheel' for Linux (or './build.bat --config RelWithDebInfo --build\_wheel' for Windows)

Checkout the source tree:
```
git clone --recursive https://github.com/Microsoft/onnxruntime
cd onnxruntime
./build.sh for Linux (or ./build.bat for Windows)
```
The build script runs all unit tests by default.

The complete list of build options can be found by running `./build.sh (or ./build.bat) --help`
Expand Down
108 changes: 69 additions & 39 deletions onnxruntime/test/python/onnx_backend_test_series.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,49 +12,79 @@

backend_test = onnx.backend.test.BackendTest(c2, __name__)

# Current status
# Ran 930 tests in 0.948s OK (skipped=574)

# We should investigate.

# dimension issue
# Error message is: Input X must be 4-dimensional. X: {1,1,3}
# ONNX Runtime expects a 4-dimension vector for operator ConvTranspose.
# file onnxruntime/core/providers/cpu/nn/conv_transpose.cc
backend_test.exclude(r'(convtranspose)')

# Type not supported
backend_test.exclude(r'(FLOAT16)')

# Operator not supported
backend_test.exclude(r'(test_expand)')
backend_test.exclude(r'(maxpool)')
backend_test.exclude(r'(AvgPool)')
backend_test.exclude(r'(max_one_input)|(max_example_cpu)|(max_two_inputs)')
backend_test.exclude(r'(min_one_input)|(min_example_cpu)|(min_two_inputs)')
backend_test.exclude(r'(mean_example)|(mean_one_input)|(mean_two_inputs)')
backend_test.exclude(r'(sum_example)|(sum_one_input)|(sum_two_inputs)')
backend_test.exclude(r'(BatchNorm)')
backend_test.exclude(r'(GLU)|(PReLU)|(PoissonNLLLLoss)|(Softsign)')
backend_test.exclude(r'(Linear_cpu)')
backend_test.exclude(r'(broadcast)')
backend_test.exclude(r'(addconstant)|(addmm)|(basic)|(lstm)')
backend_test.exclude(r'(_mm)|(non_float)|(test_operator_params_cpu)')
backend_test.exclude(r'(pow_cpu)|(rnn_cpu)|(rnn_single)|(_gru_)')
backend_test.exclude("(precomputed_((pads)|(strides)))")

# Exclude deep learning
backend_test.exclude(r'(test_vgg19|test_vgg)') # Too long.
backend_test.exclude(r'(alexnet)') # Too long.
backend_test.exclude(r'(densenet)') # Too long.
backend_test.exclude(r'(inception)') # Too long.
backend_test.exclude(r'(resnet)') # Too long.
backend_test.exclude(r'(shufflenet)') # Too long.
backend_test.exclude(r'(squeezenet)') # Too long.
backend_test.exclude(r'(zfnet)') # Too long.

# Exclude experimental
backend_test.exclude(r'(test_mvn_cpu)')
backend_test.exclude(r'(test_acosh_cpu.*'
'|test_acosh_example_cpu.*'
'|test_asinh_cpu.*'
'|test_asinh_example_cpu.*'
'|test_atanh_cpu.*'
'|test_atanh_example_cpu.*'
'|test_convtranspose_1d_cpu.*'
'|test_convtranspose_3d_cpu.*'
'|test_cosh_cpu.*'
'|test_cosh_example_cpu.*'
'|test_dynamic_slice_cpu.*'
'|test_dynamic_slice_default_axes_cpu.*'
'|test_dynamic_slice_end_out_of_bounds_cpu.*'
'|test_dynamic_slice_neg_cpu.*'
'|test_dynamic_slice_start_out_of_bounds_cpu.*'
'|test_eyelike_populate_off_main_diagonal_cpu.*'
'|test_eyelike_with_dtype_cpu.*'
'|test_eyelike_without_dtype_cpu.*'
'|test_gru_seq_length_cpu.*'
'|test_maxunpool_export_with_output_shape_cpu.*'
'|test_maxunpool_export_without_output_shape_cpu.*'
'|test_onehot_with_axis_cpu.*'
'|test_onehot_without_axis_cpu.*'
'|test_scan_sum_cpu.*'
'|test_scatter_with_axis_cpu.*'
'|test_scatter_without_axis_cpu.*'
'|test_sign_cpu.*'
'|test_sinh_cpu.*'
'|test_sinh_example_cpu.*'
'|test_AvgPool1d_cpu.*'
'|test_AvgPool1d_stride_cpu.*'
'|test_AvgPool2d_cpu.*'
'|test_AvgPool2d_stride_cpu.*'
'|test_AvgPool3d_cpu.*'
'|test_AvgPool3d_stride1_pad0_gpu_input_cpu.*'
'|test_AvgPool3d_stride_cpu.*'
'|test_BatchNorm1d_3d_input_eval_cpu.*'
'|test_BatchNorm2d_eval_cpu.*'
'|test_BatchNorm2d_momentum_eval_cpu.*'
'|test_BatchNorm3d_eval_cpu.*'
'|test_BatchNorm3d_momentum_eval_cpu.*'
'|test_GLU_cpu.*'
'|test_GLU_dim_cpu.*'
'|test_Linear_cpu.*'
'|test_PReLU_1d_cpu.*'
'|test_PReLU_1d_multiparam_cpu.*'
'|test_PReLU_2d_cpu.*'
'|test_PReLU_2d_multiparam_cpu.*'
'|test_PReLU_3d_cpu.*'
'|test_PReLU_3d_multiparam_cpu.*'
'|test_PoissonNLLLLoss_no_reduce_cpu.*'
'|test_Softsign_cpu.*'
'|test_operator_add_broadcast_cpu.*'
'|test_operator_add_size1_broadcast_cpu.*'
'|test_operator_add_size1_right_broadcast_cpu.*'
'|test_operator_add_size1_singleton_broadcast_cpu.*'
'|test_operator_addconstant_cpu.*'
'|test_operator_addmm_cpu.*'
'|test_operator_basic_cpu.*'
'|test_operator_lstm_cpu.*'
'|test_operator_mm_cpu.*'
'|test_operator_non_float_params_cpu.*'
'|test_operator_params_cpu.*'
'|test_operator_pow_cpu.*'
'|test_operator_rnn_cpu.*'
'|test_operator_rnn_single_layer_cpu.*'
'|test_sign_model_cpu.*'
'|test_mvn.*' #No schema registered for 'MeanVarianceNormalization'!
')')

# import all test cases at global scope to make
# them visible to python.unittest.
Expand Down
10 changes: 5 additions & 5 deletions tools/ci_build/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def get_config_build_dir(build_dir, config):
# build directory per configuration
return os.path.join(build_dir, config)

def run_subprocess(args, cwd=None, capture=False, dll_path=None, check=True):
def run_subprocess(args, cwd=None, capture=False, dll_path=None):
log.debug("Running subprocess in '{0}'\n{1}".format(cwd or os.getcwd(), args))
my_env = os.environ.copy()
if dll_path:
Expand All @@ -122,9 +122,9 @@ def run_subprocess(args, cwd=None, capture=False, dll_path=None, check=True):
my_env["LD_LIBRARY_PATH"] = dll_path

if (capture):
result = subprocess.run(args, cwd=cwd, check=check, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=my_env)
result = subprocess.run(args, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=my_env)
else:
result = subprocess.run(args, cwd=cwd, check=check, env=my_env)
result = subprocess.run(args, cwd=cwd, check=True, env=my_env)

return result

Expand Down Expand Up @@ -409,8 +409,8 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enab
run_subprocess([sys.executable, 'onnxruntime_test_python_backend.py'], cwd=cwd, dll_path=dll_path)
run_subprocess([sys.executable, os.path.join(source_dir,'onnxruntime','test','onnx','gen_test_models.py'),'--output_dir','test_models'], cwd=cwd)
run_subprocess([os.path.join(cwd,'onnx_test_runner'), 'test_models'], cwd=cwd)
#The following one may fail
run_subprocess([sys.executable, 'onnx_backend_test_series.py'], cwd=cwd, dll_path=dll_path, check=False)
if config != 'Debug':
run_subprocess([sys.executable, 'onnx_backend_test_series.py'], cwd=cwd, dll_path=dll_path)
try:
import onnxmltools
import keras
Expand Down
4 changes: 2 additions & 2 deletions tools/ci_build/github/linux/docker/scripts/install_deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ else
#Install ONNX
#5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3
#bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0
#eb4b7c2cc2a0d34c0127e26c2c1cb5e712467e1e is v1.3.0 latest
for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "eb4b7c2cc2a0d34c0127e26c2c1cb5e712467e1e"; do
#0a4d5abdf4939ab0842a5eadcc16a3bf0738f901 is v1.3.0 latest
for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "0a4d5abdf4939ab0842a5eadcc16a3bf0738f901"; do
if [ -z ${lastest_onnx_version+x} ]; then
echo "first pass";
else
Expand Down