From ec1d33868f8802365268d3746043e673a150af68 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Tue, 2 Jul 2019 10:51:30 -0700 Subject: [PATCH 1/4] comment out flaky test --- tests/python/unittest/test_profiler.py | 33 ++++++++++++++------------ 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index ab7d29f104ca..e39b1a528a2f 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -434,21 +434,24 @@ def test_custom_operator_profiling_multiple_custom_ops_symbolic(): 'symbolic', \ 'test_custom_operator_profiling_multiple_custom_ops_symbolic.json') -def test_custom_operator_profiling_naive_engine(): - # run the three tests above using Naive Engine - run_in_spawned_process(test_custom_operator_profiling, \ - {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ - 'test_custom_operator_profiling_naive.json') - run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ - {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ - 'imperative', \ - 'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json') - run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ - {'MXNET_ENGINE_TYPE' : "NaiveEngine", \ - 'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ - 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ - 'symbolic', \ - 'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json') +''' +Flaky, commenting out for now, will come back +''' +# def test_custom_operator_profiling_naive_engine(): +# # run the three tests above using Naive Engine +# run_in_spawned_process(test_custom_operator_profiling, \ +# {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ +# 'test_custom_operator_profiling_naive.json') +# run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ +# {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ +# 'imperative', \ +# 'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json') +# run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ +# {'MXNET_ENGINE_TYPE' : "NaiveEngine", \ +# 'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ +# 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ +# 'symbolic', \ +# 'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json') if __name__ == '__main__': import nose From 1eda9a19ac6b3657798e4d675fd8aca7206364ff Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Tue, 2 Jul 2019 11:10:56 -0700 Subject: [PATCH 2/4] Update test_profiler.py --- tests/python/unittest/test_profiler.py | 34 ++++++++++++-------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index e39b1a528a2f..efb9424101b7 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -434,24 +434,22 @@ def test_custom_operator_profiling_multiple_custom_ops_symbolic(): 'symbolic', \ 'test_custom_operator_profiling_multiple_custom_ops_symbolic.json') -''' -Flaky, commenting out for now, will come back -''' -# def test_custom_operator_profiling_naive_engine(): -# # run the three tests above using Naive Engine -# run_in_spawned_process(test_custom_operator_profiling, \ -# {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ -# 'test_custom_operator_profiling_naive.json') -# run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ -# {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ -# 'imperative', \ -# 'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json') -# run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ -# {'MXNET_ENGINE_TYPE' : "NaiveEngine", \ -# 'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ -# 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ -# 'symbolic', \ -# 'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json') +@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/15406") +def test_custom_operator_profiling_naive_engine(): + # run the three tests above using Naive Engine + run_in_spawned_process(test_custom_operator_profiling, \ + {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ + 'test_custom_operator_profiling_naive.json') + run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ + {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ + 'imperative', \ + 'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json') + run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ + {'MXNET_ENGINE_TYPE' : "NaiveEngine", \ + 'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ + 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ + 'symbolic', \ + 'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json') if __name__ == '__main__': import nose From 3e370b086095bac26db314422cf49674d811725b Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Tue, 2 Jul 2019 12:45:37 -0700 Subject: [PATCH 3/4] Update test_profiler.py --- tests/python/unittest/test_profiler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index efb9424101b7..61f7366fee3a 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -427,6 +427,7 @@ def create_operator(self, ctx, shapes, dtypes): and '_plus_scalar' in target_dict['Time']['operator'] profiler.set_state('stop') +@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/15406") def test_custom_operator_profiling_multiple_custom_ops_symbolic(): run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ {'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ From feb704fabf5abf3d2a72f4aeccc0e3a3ad112a52 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Tue, 2 Jul 2019 13:06:09 -0700 Subject: [PATCH 4/4] Update test_profiler.py --- tests/python/unittest/test_profiler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index 61f7366fee3a..b9447f951764 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -24,6 +24,7 @@ import json from collections import OrderedDict from common import run_in_spawned_process +import unittest def enable_profiler(profile_filename, run=True, continuous_dump=False, aggregate_stats=False): profiler.set_config(profile_symbolic=True,