From e3ea6ad734dae61398bcb682057f5aae2e7051c9 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Mon, 11 Mar 2019 11:28:11 -0700 Subject: [PATCH 1/6] Reenable test_operator_gpu.py:test_bulking. --- tests/python/gpu/test_operator_gpu.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index fdc8a5b30828..c12c94b41f5c 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -2103,7 +2103,6 @@ def test_bilinear_sampler_versions(): @with_seed() -@unittest.skip("test fails on windows gpu. temporarily disabled till it gets fixed. tracked at https://github.com/apache/incubator-mxnet/issues/14368") def test_bulking(): # Return the execution time of a model with the specified limits to the bulked op segments def test_bulking_helper(data_shape, num_ops, num_iterations, From a8d95a20ebd61e75af4203bb585fdf1cc80411f6 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Mon, 11 Mar 2019 14:16:24 -0700 Subject: [PATCH 2/6] Add temporary debug output. --- ci/windows/test_py2_gpu.ps1 | 2 +- ci/windows/test_py3_gpu.ps1 | 2 +- src/executor/graph_executor.cc | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 index 8a6c8e9b44f9..88919f0808bc 100644 --- a/ci/windows/test_py2_gpu.ps1 +++ b/ci/windows/test_py2_gpu.ps1 @@ -25,7 +25,7 @@ $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python27\Scripts\pip install -r tests\requirements.txt C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py if (! $?) { Throw ("Error running tests") } diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 index 5fbc9f2f8036..dad698fd4f64 100644 --- a/ci/windows/test_py3_gpu.ps1 +++ b/ci/windows/test_py3_gpu.ps1 @@ -25,7 +25,7 @@ $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python37\Scripts\pip install -r tests\requirements.txt C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest if (! $?) { Throw ("Error running unittest") } -C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py if (! $?) { Throw ("Error running tests") } diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 3d74bfb9a663..09458983f5fa 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -1201,6 +1201,8 @@ void GraphExecutor::InitOpSegs() { bool is_training = num_forward_nodes_ != total_num_nodes; + LOG(INFO) << "Fwd=" << Imperative::BulkExecMaxNodeTrainFwd() << "\tBwd=" << Imperative::BulkExecMaxNodeTrainBwd() << "\tBulk=" << Imperative::PreferBulkExecTrain(); + if (prefer_bulk_exec_train && is_training) { // Bulk the forward portion of the graph per the bulk segment max size for forward training this->BulkOpSegs(0, num_forward_nodes_, Imperative::BulkExecMaxNodeTrainFwd()); From d55b85155977a0723f618cdc8b3dd0cbf91f7b8f Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Mon, 11 Mar 2019 16:17:01 -0700 Subject: [PATCH 3/6] Add test_gluon_gpu.py:test_bulking to test spawn approach. --- ci/windows/test_py2_gpu.ps1 | 14 ++++++++------ ci/windows/test_py3_gpu.ps1 | 14 ++++++++------ src/executor/graph_executor.cc | 4 +++- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 index 88919f0808bc..f3c316177983 100644 --- a/ci/windows/test_py2_gpu.ps1 +++ b/ci/windows/test_py2_gpu.ps1 @@ -23,11 +23,13 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python27\Scripts\pip install -r tests\requirements.txt -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest -if (! $?) { Throw ("Error running unittest") } -C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +#C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +#if (! $?) { Throw ("Error running unittest") } +C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking if (! $?) { Throw ("Error running tests") } -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py -if (! $?) { Throw ("Error running tests") } -C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train +C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_gluon.xml tests\python\gpu\test_gluon_gpu.py:test_bulking if (! $?) { Throw ("Error running tests") } +#C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +#if (! $?) { Throw ("Error running tests") } +#C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train +#if (! $?) { Throw ("Error running tests") } diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 index dad698fd4f64..698946046d3d 100644 --- a/ci/windows/test_py3_gpu.ps1 +++ b/ci/windows/test_py3_gpu.ps1 @@ -23,11 +23,13 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python37\Scripts\pip install -r tests\requirements.txt -C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest -if (! $?) { Throw ("Error running unittest") } -C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +#C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +#if (! $?) { Throw ("Error running unittest") } +C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking if (! $?) { Throw ("Error running tests") } -C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py -if (! $?) { Throw ("Error running tests") } -C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train +C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_gluon.xml tests\python\gpu\test_gluon_gpu.py:test_bulking if (! $?) { Throw ("Error running tests") } +#C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +#if (! $?) { Throw ("Error running tests") } +#C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train +#if (! $?) { Throw ("Error running tests") } diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 09458983f5fa..6fa8d7b65745 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -1201,7 +1201,9 @@ void GraphExecutor::InitOpSegs() { bool is_training = num_forward_nodes_ != total_num_nodes; - LOG(INFO) << "Fwd=" << Imperative::BulkExecMaxNodeTrainFwd() << "\tBwd=" << Imperative::BulkExecMaxNodeTrainBwd() << "\tBulk=" << Imperative::PreferBulkExecTrain(); + LOG(INFO) << "Fwd=" << Imperative::BulkExecMaxNodeTrainFwd() << + " Bwd=" << Imperative::BulkExecMaxNodeTrainBwd() << + " Bulk=" << Imperative::PreferBulkExecTrain(); if (prefer_bulk_exec_train && is_training) { // Bulk the forward portion of the graph per the bulk segment max size for forward training From e53fa8627737397dfc55e0f352bf96ac5e63a6a7 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Mon, 11 Mar 2019 16:42:03 -0700 Subject: [PATCH 4/6] Reorder testing to see test_gluon_gpu.py:test_bulking result. --- ci/windows/test_py2_gpu.ps1 | 4 ++-- ci/windows/test_py3_gpu.ps1 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 index f3c316177983..2115a9f0e8e1 100644 --- a/ci/windows/test_py2_gpu.ps1 +++ b/ci/windows/test_py2_gpu.ps1 @@ -25,10 +25,10 @@ $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python27\Scripts\pip install -r tests\requirements.txt #C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest #if (! $?) { Throw ("Error running unittest") } -C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking -if (! $?) { Throw ("Error running tests") } C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_gluon.xml tests\python\gpu\test_gluon_gpu.py:test_bulking if (! $?) { Throw ("Error running tests") } +C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking +if (! $?) { Throw ("Error running tests") } #C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py #if (! $?) { Throw ("Error running tests") } #C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 index 698946046d3d..192825def8dc 100644 --- a/ci/windows/test_py3_gpu.ps1 +++ b/ci/windows/test_py3_gpu.ps1 @@ -25,10 +25,10 @@ $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python37\Scripts\pip install -r tests\requirements.txt #C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest #if (! $?) { Throw ("Error running unittest") } -C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking -if (! $?) { Throw ("Error running tests") } C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_gluon.xml tests\python\gpu\test_gluon_gpu.py:test_bulking if (! $?) { Throw ("Error running tests") } +C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking +if (! $?) { Throw ("Error running tests") } #C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py #if (! $?) { Throw ("Error running tests") } #C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train From 61e67e658af2fd02ec028be3546b9566def188b0 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Mon, 11 Mar 2019 17:39:16 -0700 Subject: [PATCH 5/6] Change test_operator_gpu.py:test_bulking to use spawn approach. --- tests/python/gpu/test_operator_gpu.py | 91 ++++++++++++--------------- 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index c12c94b41f5c..38809921f8c6 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -2102,62 +2102,51 @@ def test_bilinear_sampler_versions(): assert_almost_equal(exe.grad_dict['grid'].asnumpy(), exe_list[ref_idx].grad_dict['grid'].asnumpy(), rtol=1e-3, atol=1e-5) -@with_seed() -def test_bulking(): - # Return the execution time of a model with the specified limits to the bulked op segments - def test_bulking_helper(data_shape, num_ops, num_iterations, - max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training): - orig_environ = os.environ.copy() - try: - # Explore different ways of setting the env vars. - # The framework does not cache the bulked seg size env var lookups during symbolic. - os.environ['MXNET_EXEC_BULK_EXEC_TRAIN'] = str(enable_bulking_in_training) - if max_fwd_segment_size == max_bwd_segment_size: - os.environ['MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN'] = str(max_fwd_segment_size) - os.environ.pop('MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD', None) - os.environ.pop('MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD', None) - else: - os.environ.pop('MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN', None) - os.environ['MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD'] = str(max_fwd_segment_size) - os.environ['MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD'] = str(max_bwd_segment_size) - - ctx = default_context() - # build symbol - X = mx.sym.Variable('X') - sym = mx.sym.flip(X, axis=0) - for _ in range(num_ops-1): - sym = mx.sym.flip(sym, axis=0) - x = mx.ndarray.zeros(data_shape) - dx = mx.ndarray.zeros(data_shape) - dy = mx.ndarray.ones(data_shape) - exe = sym.bind(ctx=ctx, args=[x], args_grad = {'X':dx}) - - # time a number of forward() and backward() executions after some warm-up iterations - warmups = 1 - for i in range(num_iterations+warmups): - if i == warmups: - start = time.time() - exe.forward(is_train=True) - exe.backward(dy) - dx.wait_to_read() - time_per_iteration = (time.time() - start) / num_iterations - finally: - os.environ.clear() - os.environ.update(orig_environ) - return time_per_iteration - +# isolated execution bulking test function to be invoked with different env var settings +def _test_bulking_in_process(seed, time_per_iteration): data_shape = (10,) num_ops = 1000 num_iterations = 20 + ctx = default_context() + # build symbol + X = mx.sym.Variable('X') + sym = mx.sym.flip(X, axis=0) + for _ in range(num_ops-1): + sym = mx.sym.flip(sym, axis=0) + x = mx.ndarray.zeros(data_shape) + dx = mx.ndarray.zeros(data_shape) + dy = mx.ndarray.ones(data_shape) + exe = sym.bind(ctx=ctx, args=[x], args_grad = {'X':dx}) + + # time a number of forward() and backward() executions after some warm-up iterations + warmups = 1 + for i in range(num_iterations+warmups): + if i == warmups: + start = time.time() + exe.forward(is_train=True) + exe.backward(dy) + dx.wait_to_read() + time_per_iteration.value = (time.time() - start) / num_iterations + +@with_seed() +def test_bulking(): # test case format: (max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training) test_cases = [(0,0,True), (1,1,True), (15,15,False), (15,0,True), (0,15,True), (15,15,True)] times = {} times_str = '' for seg_sizes in test_cases: - times[seg_sizes] = test_bulking_helper(data_shape, num_ops, num_iterations, - seg_sizes[0], seg_sizes[1], seg_sizes[2]) - times_str +=\ + # Create shared variable to return measured time from test process + time_per_iteration = mp.Manager().Value('d', 0.0) + if not run_in_spawned_process(_test_bulking_in_process, + {'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD' : seg_sizes[0], + 'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD' : seg_sizes[1], + 'MXNET_EXEC_BULK_EXEC_TRAIN' : seg_sizes[2]}, + time_per_iteration): + # skip test since the python version can't run it properly. Warning msg was logged. + return + times[seg_sizes] = time_per_iteration.value + times_str += \ '\n runtime of (fwd,bwd,enable) op seg setting ({},{},{}) =\t{:.1f} msec'.format( seg_sizes[0], seg_sizes[1], seg_sizes[2], 1000.0 * times[seg_sizes]) @@ -2169,12 +2158,12 @@ def test_bulking_helper(data_shape, num_ops, num_iterations, print(times_str) # Non-bulked times[0,0,True], times[1,1,True] and times[15,15,False] should be about the same, # slower than both half-bulked times[0,15,True] and times[15,0,True] - assert slowest_half_bulked_time < fastest_non_bulked_time,\ - 'A half-bulked exec time is slower than the non-bulked time by {} secs! {}'\ + assert slowest_half_bulked_time < fastest_non_bulked_time, \ + 'A half-bulked exec time is slower than the non-bulked time by {} secs! {}' \ .format(slowest_half_bulked_time - fastest_non_bulked_time, times_str) # The fully bulked times[15,15,True] should be faster than both half-bulked runs - assert fully_bulked_time < fastest_half_bulked_time,\ - 'The fully-bulked exec time is slower than a half-bulked time by {} secs! {}'\ + assert fully_bulked_time < fastest_half_bulked_time, \ + 'The fully-bulked exec time is slower than a half-bulked time by {} secs! {}' \ .format(fully_bulked_time - fastest_half_bulked_time, times_str) From 40a6615f7efa46991590f55f0237acff97313ae2 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Tue, 12 Mar 2019 11:11:54 -0700 Subject: [PATCH 6/6] Remove diagnostic output and revert windows ci changes. --- ci/windows/test_py2_gpu.ps1 | 14 ++++++-------- ci/windows/test_py3_gpu.ps1 | 14 ++++++-------- src/executor/graph_executor.cc | 4 ---- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 index 2115a9f0e8e1..8a6c8e9b44f9 100644 --- a/ci/windows/test_py2_gpu.ps1 +++ b/ci/windows/test_py2_gpu.ps1 @@ -23,13 +23,11 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python27\Scripts\pip install -r tests\requirements.txt -#C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest -#if (! $?) { Throw ("Error running unittest") } -C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_gluon.xml tests\python\gpu\test_gluon_gpu.py:test_bulking +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } -C:\Python27\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +if (! $?) { Throw ("Error running tests") } +C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train if (! $?) { Throw ("Error running tests") } -#C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py -#if (! $?) { Throw ("Error running tests") } -#C:\Python27\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error tests\python\train -#if (! $?) { Throw ("Error running tests") } diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 index 192825def8dc..5fbc9f2f8036 100644 --- a/ci/windows/test_py3_gpu.ps1 +++ b/ci/windows/test_py3_gpu.ps1 @@ -23,13 +23,11 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home') C:\Python37\Scripts\pip install -r tests\requirements.txt -#C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest -#if (! $?) { Throw ("Error running unittest") } -C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_gluon.xml tests\python\gpu\test_gluon_gpu.py:test_bulking +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py if (! $?) { Throw ("Error running tests") } -C:\Python37\python.exe -m nose -v -s --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_bulking +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +if (! $?) { Throw ("Error running tests") } +C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train if (! $?) { Throw ("Error running tests") } -#C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py -#if (! $?) { Throw ("Error running tests") } -#C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train -#if (! $?) { Throw ("Error running tests") } diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 6fa8d7b65745..3d74bfb9a663 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -1201,10 +1201,6 @@ void GraphExecutor::InitOpSegs() { bool is_training = num_forward_nodes_ != total_num_nodes; - LOG(INFO) << "Fwd=" << Imperative::BulkExecMaxNodeTrainFwd() << - " Bwd=" << Imperative::BulkExecMaxNodeTrainBwd() << - " Bulk=" << Imperative::PreferBulkExecTrain(); - if (prefer_bulk_exec_train && is_training) { // Bulk the forward portion of the graph per the bulk segment max size for forward training this->BulkOpSegs(0, num_forward_nodes_, Imperative::BulkExecMaxNodeTrainFwd());