From cc0efb8106f3dca70cd498cebf9089894431cc9b Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Wed, 24 Jul 2019 14:10:40 -0700 Subject: [PATCH 01/22] update precious flaky naive engine test --- tests/python/unittest/test_profiler.py | 66 +++++++++++--------------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index b9447f951764..3f155e23c9c7 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -275,6 +275,8 @@ def test_aggregate_duplication(): file_name = 'test_aggregate_duplication.json' enable_profiler(profile_filename = file_name, run=True, continuous_dump=True, \ aggregate_stats=True) + # clear aggregate stats + profiler.dumps(reset = True) inp = mx.nd.zeros(shape=(100, 100)) y = mx.nd.sqrt(inp) inp = inp + 1 @@ -332,6 +334,8 @@ def create_operator(self, ctx, in_shapes, in_dtypes): file_name = 'test_custom_operator_profiling.json' enable_profiler(profile_filename = file_name, run=True, continuous_dump=True,\ aggregate_stats=True) + # clear aggregate stats + profiler.dumps(reset = True) x = mx.nd.array([0, 1, 2, 3]) x.attach_grad() with mx.autograd.record(): @@ -347,8 +351,15 @@ def create_operator(self, ctx, in_shapes, in_dtypes): and 'MySigmoid::_zeros' in target_dict['Time']['Custom Operator'] profiler.set_state('stop') -def test_custom_operator_profiling_multiple_custom_ops_imperative(seed = None, \ - mode = 'imperative', file_name = None): +def check_custom_operator_profiling_multiple_custom_ops_output(debug_str): + target_dict = json.loads(debug_str) + assert 'Time' in target_dict and 'Custom Operator' in target_dict['Time'] \ + and 'MyAdd1::pure_python' in target_dict['Time']['Custom Operator'] \ + and 'MyAdd2::pure_python' in target_dict['Time']['Custom Operator'] \ + and 'MyAdd1::_plus_scalar' in target_dict['Time']['Custom Operator'] \ + and 'MyAdd2::_plus_scalar' in target_dict['Time']['Custom Operator'] + +def custom_operator_profiling_multiple_custom_ops(seed, mode, file_name): class MyAdd(mx.operator.CustomOp): def forward(self, is_train, req, in_data, out_data, aux): self.assign(out_data[0], req[0], in_data[0] + 1) @@ -392,65 +403,44 @@ def infer_shape(self, in_shape): def create_operator(self, ctx, shapes, dtypes): return MyAdd() - if file_name is None: - file_name = 'test_custom_operator_profiling_multiple_custom_ops_imperative.json' enable_profiler(profile_filename = file_name, run=True, continuous_dump=True,\ aggregate_stats=True) + # clear aggregate stats + profiler.dumps(reset = True) inp = mx.nd.zeros(shape=(100, 100)) if mode == 'imperative': - x = inp + 1 y = mx.nd.Custom(inp, op_type='MyAdd1') z = mx.nd.Custom(inp, op_type='MyAdd2') elif mode == 'symbolic': a = mx.symbol.Variable('a') - b = a + 1 - c = mx.symbol.Custom(data=a, op_type='MyAdd1') - d = mx.symbol.Custom(data=a, op_type='MyAdd2') + b = mx.symbol.Custom(data=a, op_type='MyAdd1') + c = mx.symbol.Custom(data=a, op_type='MyAdd2') b.bind(mx.cpu(), {'a': inp}).forward() c.bind(mx.cpu(), {'a': inp}).forward() - d.bind(mx.cpu(), {'a': inp}).forward() mx.nd.waitall() profiler.dump(False) debug_str = profiler.dumps(format = 'json') - target_dict = json.loads(debug_str) - ''' - We are calling _plus_scalar within MyAdd1 and MyAdd2 and outside both the custom - operators, so in aggregate stats we should have three different kinds of - _plus_scalar under domains "Custom Operator" and "operator" - ''' - assert 'Time' in target_dict and 'Custom Operator' in target_dict['Time'] \ - and 'MyAdd1::pure_python' in target_dict['Time']['Custom Operator'] \ - and 'MyAdd2::pure_python' in target_dict['Time']['Custom Operator'] \ - and 'MyAdd1::_plus_scalar' in target_dict['Time']['Custom Operator'] \ - and 'MyAdd2::_plus_scalar' in target_dict['Time']['Custom Operator'] \ - and '_plus_scalar' not in target_dict['Time']['Custom Operator'] \ - and 'operator' in target_dict['Time'] \ - and '_plus_scalar' in target_dict['Time']['operator'] + check_custom_operator_profiling_multiple_custom_ops_output(debug_str) profiler.set_state('stop') - -@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/15406") + def test_custom_operator_profiling_multiple_custom_ops_symbolic(): - run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ - {'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ - 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ - 'symbolic', \ + custom_operator_profiling_multiple_custom_ops(None, 'symbolic', \ 'test_custom_operator_profiling_multiple_custom_ops_symbolic.json') -@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/15406") +def test_custom_operator_profiling_multiple_custom_ops_imperative(): + custom_operator_profiling_multiple_custom_ops(None, 'imperative', \ + 'test_custom_operator_profiling_multiple_custom_ops_imperative.json') + def test_custom_operator_profiling_naive_engine(): # run the three tests above using Naive Engine run_in_spawned_process(test_custom_operator_profiling, \ {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ 'test_custom_operator_profiling_naive.json') - run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ - {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ - 'imperative', \ + run_in_spawned_process(custom_operator_profiling_multiple_custom_ops, \ + {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, 'imperative', \ 'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json') - run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ - {'MXNET_ENGINE_TYPE' : "NaiveEngine", \ - 'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ - 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ - 'symbolic', \ + run_in_spawned_process(custom_operator_profiling_multiple_custom_ops, \ + {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, 'symbolic', \ 'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json') if __name__ == '__main__': From b52662bb3d62391e9e299b66195decd004842f71 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Wed, 24 Jul 2019 14:13:08 -0700 Subject: [PATCH 02/22] retriever tests From af6931cc2582229a1125a629faebebe58df7c5d0 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Wed, 24 Jul 2019 15:43:15 -0700 Subject: [PATCH 03/22] retrigger tests From ae288eb0640125eff49c85decc2d6e18588f1549 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Wed, 24 Jul 2019 15:43:37 -0700 Subject: [PATCH 04/22] retrigger tests From 2947358710d22e42c22c405205bc90ea60024791 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Thu, 25 Jul 2019 10:43:34 -0700 Subject: [PATCH 05/22] retrigger tests From 62377d08249074a7d35eb233af4024818e23e58b Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Thu, 25 Jul 2019 10:43:58 -0700 Subject: [PATCH 06/22] retrigger tests From ca354afb0bcf8a5c2f51c24be716a12604da30e9 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Thu, 25 Jul 2019 13:21:13 -0700 Subject: [PATCH 07/22] retrigger tests From 59bc51841438436c4c864da671bbbd2364cc8cf8 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Fri, 26 Jul 2019 10:48:47 -0700 Subject: [PATCH 08/22] Update test_profiler.py --- tests/python/unittest/test_profiler.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index 3f155e23c9c7..40d940586087 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -267,23 +267,23 @@ def check_sorting(debug_str, sort_by, ascending): test_profile_event(False) for sb in sort_by_options: for asc in ascending_options: - debug_str = profiler.dumps(format = 'json', sort_by = sb, ascending = asc) + debug_str = profiler.dumps(format='json', sort_by=sb, ascending=asc) check_sorting(debug_str, sb, asc) profiler.set_state('stop') def test_aggregate_duplication(): file_name = 'test_aggregate_duplication.json' - enable_profiler(profile_filename = file_name, run=True, continuous_dump=True, \ + enable_profiler(profile_filename=file_name, run=True, continuous_dump=True, \ aggregate_stats=True) # clear aggregate stats - profiler.dumps(reset = True) + profiler.dumps(reset=True) inp = mx.nd.zeros(shape=(100, 100)) y = mx.nd.sqrt(inp) inp = inp + 1 inp = inp + 1 mx.nd.waitall() profiler.dump(False) - debug_str = profiler.dumps(format = 'json') + debug_str = profiler.dumps(format='json') target_dict = json.loads(debug_str) assert 'Time' in target_dict and 'operator' in target_dict['Time'] \ and 'sqrt' in target_dict['Time']['operator'] \ @@ -295,7 +295,7 @@ def test_aggregate_duplication(): assert target_dict['Time']['operator']['_plus_scalar']['Count'] == 2 profiler.set_state('stop') -def test_custom_operator_profiling(seed = None, file_name = None): +def test_custom_operator_profiling(seed=None, file_name=None): class Sigmoid(mx.operator.CustomOp): def forward(self, is_train, req, in_data, out_data, aux): x = in_data[0].asnumpy() @@ -332,10 +332,10 @@ def create_operator(self, ctx, in_shapes, in_dtypes): if file_name is None: file_name = 'test_custom_operator_profiling.json' - enable_profiler(profile_filename = file_name, run=True, continuous_dump=True,\ + enable_profiler(profile_filename=file_name, run=True, continuous_dump=True,\ aggregate_stats=True) # clear aggregate stats - profiler.dumps(reset = True) + profiler.dumps(reset=True) x = mx.nd.array([0, 1, 2, 3]) x.attach_grad() with mx.autograd.record(): @@ -343,7 +343,7 @@ def create_operator(self, ctx, in_shapes, in_dtypes): y.backward() mx.nd.waitall() profiler.dump(False) - debug_str = profiler.dumps(format = 'json') + debug_str = profiler.dumps(format='json') target_dict = json.loads(debug_str) assert 'Time' in target_dict and 'Custom Operator' in target_dict['Time'] \ and 'MySigmoid::pure_python' in target_dict['Time']['Custom Operator'] \ @@ -403,10 +403,10 @@ def infer_shape(self, in_shape): def create_operator(self, ctx, shapes, dtypes): return MyAdd() - enable_profiler(profile_filename = file_name, run=True, continuous_dump=True,\ + enable_profiler(profile_filename=file_name, run=True, continuous_dump=True,\ aggregate_stats=True) # clear aggregate stats - profiler.dumps(reset = True) + profiler.dumps(reset=True) inp = mx.nd.zeros(shape=(100, 100)) if mode == 'imperative': y = mx.nd.Custom(inp, op_type='MyAdd1') @@ -419,7 +419,7 @@ def create_operator(self, ctx, shapes, dtypes): c.bind(mx.cpu(), {'a': inp}).forward() mx.nd.waitall() profiler.dump(False) - debug_str = profiler.dumps(format = 'json') + debug_str = profiler.dumps(format='json') check_custom_operator_profiling_multiple_custom_ops_output(debug_str) profiler.set_state('stop') From 42c54b7751c7e54fc6ba51b046bd098db331b55f Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Fri, 26 Jul 2019 13:49:15 -0700 Subject: [PATCH 09/22] retrigger tests From 7301151437e4a1231179ae934912b6d0128645fb Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Fri, 26 Jul 2019 13:49:28 -0700 Subject: [PATCH 10/22] retrigger tests From 583f2efe72f02b9bfcc3b1788d95a21dd45c6bf0 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Sat, 27 Jul 2019 21:37:49 -0700 Subject: [PATCH 11/22] retrigger tests From e13825f92fb3a8794cf458226e7dccc3a047722e Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Sat, 27 Jul 2019 23:08:43 -0700 Subject: [PATCH 12/22] retrigger tests From 39ae04f4eb13d2a2a75414b85c47b669a9b61b30 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Sun, 28 Jul 2019 13:18:01 -0700 Subject: [PATCH 13/22] retrigger tests From c6930bc23f1005d204fa07d81120a614cd9eb7a3 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Mon, 29 Jul 2019 17:02:03 -0700 Subject: [PATCH 14/22] retrigger tests From 4c9b3d6fa25119d262899875d9f5187d71efafcc Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Mon, 29 Jul 2019 17:03:42 -0700 Subject: [PATCH 15/22] Update test_profiler.py --- tests/python/unittest/test_profiler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index 40d940586087..3977548ecd5f 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -419,6 +419,7 @@ def create_operator(self, ctx, shapes, dtypes): c.bind(mx.cpu(), {'a': inp}).forward() mx.nd.waitall() profiler.dump(False) + print(profiler.dumps()) debug_str = profiler.dumps(format='json') check_custom_operator_profiling_multiple_custom_ops_output(debug_str) profiler.set_state('stop') From beb4169b6d96d7b3971e1415442f51ca99a7632e Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Mon, 29 Jul 2019 20:31:36 -0700 Subject: [PATCH 16/22] retrigger tests From 49b25bb08770f5957a289c1aed476ba66a28d7da Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Mon, 29 Jul 2019 20:31:53 -0700 Subject: [PATCH 17/22] retrigger tests From e8a25a9615fb814352f8ca1e00a218ba49588483 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Mon, 29 Jul 2019 20:32:09 -0700 Subject: [PATCH 18/22] retrigger tests From 0ca0dc26839ac6d28ad41746747877ee00795ecd Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Mon, 29 Jul 2019 22:25:06 -0700 Subject: [PATCH 19/22] retrigger tests From 6065642234c76f668a40dd6ae6e2d0f9fa8d7eee Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Mon, 29 Jul 2019 22:25:28 -0700 Subject: [PATCH 20/22] Update test_profiler.py --- tests/python/unittest/test_profiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index 3977548ecd5f..40d940586087 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -419,7 +419,6 @@ def create_operator(self, ctx, shapes, dtypes): c.bind(mx.cpu(), {'a': inp}).forward() mx.nd.waitall() profiler.dump(False) - print(profiler.dumps()) debug_str = profiler.dumps(format='json') check_custom_operator_profiling_multiple_custom_ops_output(debug_str) profiler.set_state('stop') From f88b7b0382586710f61b732de61527ec7f24ee7c Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Tue, 30 Jul 2019 12:58:46 -0700 Subject: [PATCH 21/22] retrigger tests From 96c7342851a02fcf6b36eaad44b6d1fd9c580fd0 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Tue, 30 Jul 2019 13:50:15 -0700 Subject: [PATCH 22/22] Update test_profiler.py --- tests/python/unittest/test_profiler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index 40d940586087..33f59845557b 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -415,8 +415,10 @@ def create_operator(self, ctx, shapes, dtypes): a = mx.symbol.Variable('a') b = mx.symbol.Custom(data=a, op_type='MyAdd1') c = mx.symbol.Custom(data=a, op_type='MyAdd2') - b.bind(mx.cpu(), {'a': inp}).forward() - c.bind(mx.cpu(), {'a': inp}).forward() + y = b.bind(mx.cpu(), {'a': inp}) + z = c.bind(mx.cpu(), {'a': inp}) + yy = y.forward() + zz = z.forward() mx.nd.waitall() profiler.dump(False) debug_str = profiler.dumps(format='json')