Remove duplicate @with_seed decorators (#19336)

* Remove duplicate @with_seed decorators

function_scope_seed pytest autuse fixture defined in conftest.py already manages
the seeds of every test function.

* Fix

* Remove @with_seed from newly introduced tests

conftest.py

@@ -204,7 +204,7 @@ def test_not_ok_with_random_data():
     except:
         logging.warning('Unable to import numpy/mxnet. Skip setting function-level seed.')
 
-    seed_message = 'np/mx/python random seeds are set to {}, use MXNET_TEST_SEED={} to reproduce.'
+    seed_message = 'Setting np/mx/python random seeds to {}. Use MXNET_TEST_SEED={} to reproduce.'
     seed_message = seed_message.format(seed, seed)
 
     # Always log seed on DEBUG log level. This makes sure we can find out the
@@ -215,12 +215,13 @@ def test_not_ok_with_random_data():
     yield  # run the test
 
     if request.node.rep_setup.failed:
-        logging.info("Setting up a test failed: {}", request.node.nodeid)
+        logging.error("Setting up a test failed: {}", request.node.nodeid)
     elif request.node.rep_call.outcome == 'failed':
-        # Either request.node.rep_setup.failed or request.node.rep_setup.passed
-        # should be True
+        # Either request.node.rep_setup.failed or request.node.rep_setup.passed should be True
         assert request.node.rep_setup.passed
-        # On failure also log seed on INFO log level
-        logging.info(seed_message)
+        # On failure also log seed on WARNING log level
+        error_message = 'Error seen with seeded test, use MXNET_TEST_SEED={} to reproduce'
+        error_message = error_message.format(seed)
+        logging.warning(error_message)
 
     random.setstate(old_state)

tests/nightly/common.py

@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import functools
+import logging
+import os
+import random
+
+import mxnet as mx
+import numpy as np
+
+
+def with_seed(seed=None):
+    """
+    A decorator for test functions that manages rng seeds.
+
+    Parameters
+    ----------
+
+    seed : the seed to pass to np.random and mx.random
+
+
+    This tests decorator sets the np, mx and python random seeds identically
+    prior to each test, then outputs those seeds if the test fails or
+    if the test requires a fixed seed (as a reminder to make the test
+    more robust against random data).
+
+    @with_seed()
+    def test_ok_with_random_data():
+        ...
+
+    @with_seed(1234)
+    def test_not_ok_with_random_data():
+        ...
+
+    Use of the @with_seed() decorator for all tests creates
+    tests isolation and reproducability of failures.  When a
+    test fails, the decorator outputs the seed used.  The user
+    can then set the environment variable MXNET_TEST_SEED to
+    the value reported, then rerun the test with:
+
+        pytest --verbose --capture=no <test_module_name.py>::<failing_test>
+
+    To run a test repeatedly, set MXNET_TEST_COUNT=<NNN> in the environment.
+    To see the seeds of even the passing tests, add '--log-level=DEBUG' to pytest.
+    """
+    def test_helper(orig_test):
+        @functools.wraps(orig_test)
+        def test_new(*args, **kwargs):
+            test_count = int(os.getenv('MXNET_TEST_COUNT', '1'))
+            env_seed_str = os.getenv('MXNET_TEST_SEED')
+            for i in range(test_count):
+                if seed is not None:
+                    this_test_seed = seed
+                    log_level = logging.INFO
+                elif env_seed_str is not None:
+                    this_test_seed = int(env_seed_str)
+                    log_level = logging.INFO
+                else:
+                    this_test_seed = np.random.randint(0, np.iinfo(np.int32).max)
+                    log_level = logging.DEBUG
+                post_test_state = np.random.get_state()
+                np.random.seed(this_test_seed)
+                mx.random.seed(this_test_seed)
+                random.seed(this_test_seed)
+                # 'pytest --logging-level=DEBUG' shows this msg even with an ensuing core dump.
+                test_count_msg = '{} of {}: '.format(i+1,test_count) if test_count > 1 else ''
+                pre_test_msg = ('{}Setting test np/mx/python random seeds, use MXNET_TEST_SEED={}'
+                                ' to reproduce.').format(test_count_msg, this_test_seed)
+                on_err_test_msg = ('{}Error seen with seeded test, use MXNET_TEST_SEED={}'
+                                ' to reproduce.').format(test_count_msg, this_test_seed)
+                logging.log(log_level, pre_test_msg)
+                try:
+                    orig_test(*args, **kwargs)
+                except:
+                    # With exceptions, repeat test_msg at WARNING level to be sure it's seen.
+                    if log_level < logging.WARNING:
+                        logging.warning(on_err_test_msg)
+                    raise
+                finally:
+                    # Provide test-isolation for any test having this decorator
+                    mx.nd.waitall()
+                    np.random.set_state(post_test_state)
+        return test_new
+    return test_helper
+

tests/nightly/estimator/test_estimator_cnn.py

@@ -27,9 +27,6 @@
 from mxnet.gluon.contrib.estimator import estimator
 from mxnet.gluon.model_zoo import vision
 
-# use with_seed decorator in python/unittest/common.py
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'python', 'unittest'))
-from common import with_seed
 import pytest
 
 
@@ -93,7 +90,6 @@ def get_net(model_name, context):
     return net, input_shape, label_shape, loss_axis
 
 
-@with_seed()
 def test_estimator_cpu():
     '''
     Test estimator by doing one pass over each model with synthetic data
@@ -125,8 +121,7 @@ def test_estimator_cpu():
                 epochs=1)
 
 
-# using fixed seed to reduce flakiness in accuracy assertion
-@with_seed(7)
+@pytest.mark.seed(7)  # using fixed seed to reduce flakiness in accuracy assertion
 @pytest.mark.skipif(mx.context.num_gpus() < 1, reason="skip if no GPU")
 def test_estimator_gpu():
     '''

tests/nightly/estimator/test_sentiment_rnn.py

@@ -32,9 +32,6 @@
 from mxnet.gluon import nn, rnn
 from mxnet.gluon.contrib.estimator import estimator
 
-# use with_seed decorator in python/unittest/common.py
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'python', 'unittest'))
-from common import with_seed
 import pytest
 
 
@@ -205,7 +202,6 @@ def run(net, train_dataloader, test_dataloader, num_epochs, ctx, lr):
     return acc
 
 
-@with_seed()
 def test_estimator_cpu():
     '''
     Test estimator by doing one pass over each model with synthetic data
@@ -241,8 +237,7 @@ def test_estimator_cpu():
         run(net, train_dataloader, val_dataloader, num_epochs=num_epochs, ctx=ctx, lr=lr)
 
 
-# using fixed seed to reduce flakiness in accuracy assertion
-@with_seed(7)
+@pytest.mark.seed(7)  # using fixed seed to reduce flakiness in accuracy assertion
 @pytest.mark.skipif(mx.context.num_gpus() < 1, reason="skip if no GPU")
 def test_estimator_gpu():
     '''

tests/nightly/test_large_array.py

@@ -22,9 +22,6 @@
 import numpy as np
 import mxnet as mx
 
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-sys.path.append(os.path.join(curr_path, '../python/unittest/'))
-
 from mxnet.test_utils import rand_ndarray, assert_almost_equal, rand_coord_2d, default_context, check_symbolic_forward, create_2d_tensor
 from mxnet import gluon, nd
 from common import with_seed

tests/nightly/test_large_vector.py

@@ -22,9 +22,6 @@
 import numpy as np
 import mxnet as mx
 
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-sys.path.append(os.path.join(curr_path, '../python/unittest/'))
-
 from mxnet.test_utils import rand_ndarray, assert_almost_equal, rand_coord_2d, create_vector
 from mxnet import gluon, nd
 from common import with_seed

tests/nightly/test_np_large_array.py

@@ -27,7 +27,6 @@
 
 from mxnet.test_utils import rand_ndarray, assert_almost_equal, rand_coord_2d, default_context, check_symbolic_forward, create_2d_np_tensor, use_np
 from mxnet import gluon, np, npx
-from common import with_seed
 import pytest
 from tests.python.unittest.common import assertRaises
 from mxnet.base import MXNetError

tests/nightly/test_np_random.py

@@ -16,8 +16,6 @@
 # under the License.
 
 # pylint: skip-file
-from __future__ import absolute_import
-from __future__ import division
 import itertools
 import os
 import sys
@@ -32,15 +30,14 @@
 from mxnet import np, npx, autograd
 from mxnet.gluon import HybridBlock
 from mxnet.test_utils import same, assert_almost_equal, rand_shape_nd, rand_ndarray, use_np
-from common import with_seed, retry
+from common import retry
 from mxnet.test_utils import verify_generator, gen_buckets_probs_with_ppf, assert_exception, is_op_runnable, collapse_sum_like
 from mxnet.ndarray.ndarray import py_slice
 from mxnet.base import integer_types
 import scipy.stats as ss
 
 
 @retry(5)
-@with_seed()
 @use_np
 def test_np_exponential():
     samples = 1000000
@@ -56,7 +53,6 @@ def test_np_exponential():
 
 
 @retry(5)
-@with_seed()
 @use_np
 def test_np_uniform():
     types = [None, "float32", "float64"]
@@ -76,7 +72,6 @@ def test_np_uniform():
 
 
 @retry(5)
-@with_seed()
 @use_np
 def test_np_logistic():
     samples = 1000000
@@ -93,7 +88,6 @@ def test_np_logistic():
 
 
 @retry(5)
-@with_seed()
 @use_np
 def test_np_gumbel():
     samples = 1000000
@@ -109,7 +103,6 @@ def test_np_gumbel():
 
 
 @retry(5)
-@with_seed()
 @use_np
 def test_np_normal():
     types = [None, "float32", "float64"]
@@ -129,7 +122,6 @@ def test_np_normal():
 
 
 @retry(5)
-@with_seed()
 @use_np
 def test_np_gamma():
     types = [None, "float32", "float64"]
@@ -156,7 +148,6 @@ def generator_mx(x): return np.random.gamma(
 
 
 @retry(5)
-@with_seed()
 @use_np
 def test_np_laplace():
     types = [None, "float32", "float64"]

tests/python/gpu/test_amp.py

@@ -31,7 +31,7 @@
 from mxnet.operator import get_all_registered_operators_grouped
 curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
 sys.path.insert(0, os.path.join(curr_path, '../unittest'))
-from common import with_seed, assert_raises_cudnn_not_satisfied
+from common import assert_raises_cudnn_not_satisfied
 sys.path.insert(0, os.path.join(curr_path, '../train'))
 set_default_context(mx.gpu(0))
 
@@ -96,7 +96,6 @@ def test_amp_coverage(amp_tests):
         f"python/mxnet/amp/lists/symbol_fp16.py) - please add them. " \
         f"\n{guidelines}"
 
-@with_seed()
 @pytest.mark.skip(reason='Error during waitall(). Tracked in #18099')
 @assert_raises_cudnn_not_satisfied(min_version='5.1.10')
 def test_amp_conversion_rnn(amp_tests):
@@ -112,7 +111,6 @@ def test_amp_conversion_rnn(amp_tests):
         mx.test_utils.assert_almost_equal(out.asnumpy(), out2.asnumpy(), atol=1e-2, rtol=1e-2)
 
 
-@with_seed()
 def test_fp16_casting(amp_tests):
     data = mx.sym.var("data")
     out1 = mx.sym.amp_cast(data, dtype="float16")

tests/python/gpu/test_fusion.py

@@ -26,7 +26,6 @@
 
 curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
 sys.path.insert(0, os.path.join(curr_path, '../unittest'))
-from common import with_seed
 
 def check_fused_symbol(sym, **kwargs):
     inputs = sym.list_inputs()
@@ -250,14 +249,12 @@ def check_leakyrelu_ops():
     check_fused_symbol(mx.sym.LeakyReLU(a+b, act_type='gelu'), a=arr1, b=arr2)
 
 
-@with_seed()
 def test_fusion():
     check_unary_ops()
     check_binary_ops()
     check_other_ops()
     check_leakyrelu_ops()
 
-@with_seed()
 def test_fusion_compiler_cache():
     # Stresses the internal cache of CUfunctions by creating the same kernel multiple times and
     # on multiple GPUs if available.
@@ -276,7 +273,6 @@ def test_fusion_compiler_cache():
     if num_gpus > 1:
         check_fused_symbol(a+b, ctx=mx.gpu(1), a=arr1, b=arr2)
 
-@with_seed()
 @use_np
 def test_fusion_boolean_inputs():
     from mxnet.gluon import HybridBlock
@@ -296,7 +292,6 @@ def hybrid_forward(self, F, valid_length):
     out = foo(mx.np.ones((10,), ctx=mx.gpu(), dtype=np.bool))
     mx.npx.waitall()
 
-@with_seed()
 def test_fusion_different_dimensions():
     from mxnet.gluon import HybridBlock
 
@@ -320,7 +315,6 @@ def hybrid_forward(self, F, x):
     assert np.all(out.asnumpy() == np.ones((10,10)))
     assert out.shape == (10,10,1)
 
-@with_seed()
 def test_input_reorder():
     class Block(gluon.HybridBlock):
         def __init__(self, **kwargs):
@@ -354,7 +348,6 @@ def hybrid_forward(self, F, x, y, z):
         for key in ['result'] + list(range(len(arg_data))):
             assert_allclose(arrays['0'][key].asnumpy(), arrays['1'][key].asnumpy())
 
-@with_seed()
 def test_fusion_cycle():
     class Test(gluon.nn.HybridBlock):
         def __init__(self, **kwargs):