From ca5553b7a8e52db151f46c2deead02e36014f040 Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Thu, 5 Jul 2018 22:54:10 +0000 Subject: [PATCH 1/7] add flakiness checker --- tools/flakiness_checker.py | 87 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tools/flakiness_checker.py diff --git a/tools/flakiness_checker.py b/tools/flakiness_checker.py new file mode 100644 index 000000000000..74ac6f899f11 --- /dev/null +++ b/tools/flakiness_checker.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" Checks a given test for flakiness +Takes the file name and function name of a test, as well as, optionally, +the number of trials to run and the random seed to use +""" + +import subprocess +import sys +import os +import random +import argparse + + +DEFAULT_NUM_TRIALS = 500 + +def run_test_trials(test_file, test_name, num_trials, seed): + test_path = test_file + ":" + test_name + + new_env = os.environ + new_env["MXNET_TEST_COUNT"] = str(num_trials) + if seed is None: + print("Using random seed") + else: + new_env["MXNET_TEST_SEED"] = seed + + code = subprocess.call(["nosetests", "-s","--verbose",test_path], env = new_env) + print("nosetests completed with return code " + str(code)) + +def find_test_path(test_file): + test_file += ".py" + test_path = test_file + top = str(subprocess.check_output(["git", "rev-parse", "--show-toplevel"]), + errors= "strict").strip() + + for (path, names, files) in os.walk(top): + if test_file in files: + test_path = path + "/" + test_path + return test_path + raise FileNotFoundError("Could not find " + test_file) + + +class NameAction(argparse.Action): + """Parses command line argument to get test file and test name""" + def __call__(self, parser, namespace, values, option_string=None): + name = values.split(".") + setattr(namespace, "test_path", find_test_path(name[0])) + setattr(namespace, "test_name", name[1]) + +def parse_args(): + parser = argparse.ArgumentParser(description="Check test for flakiness") + + parser.add_argument("test", action=NameAction, + help="file name and and function name of test, " + "provided in the format: file_name.test_name") + + parser.add_argument("-n", "--num-trials", + default=DEFAULT_NUM_TRIALS, type=int, + help="number of test trials, passed as " + "MXNET_TEST_COUNT, defaults to 500") + + parser.add_argument("-s", "--seed", + help="random seed, passed as MXNET_TEST_SEED") + + args = parser.parse_args() + return args + + +if __name__ == "__main__": + args = parse_args() + + run_test_trials(args.test_path, args.test_name, args.num_trials, args.seed) From 68ea32ec89a3c3c8d5d174d83181a092bf9eba7c Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Fri, 6 Jul 2018 17:49:42 +0000 Subject: [PATCH 2/7] fixed style and argument parsing --- tools/flakiness_checker.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/tools/flakiness_checker.py b/tools/flakiness_checker.py index 74ac6f899f11..d875b62e769a 100644 --- a/tools/flakiness_checker.py +++ b/tools/flakiness_checker.py @@ -25,6 +25,7 @@ import os import random import argparse +import re DEFAULT_NUM_TRIALS = 500 @@ -32,21 +33,23 @@ def run_test_trials(test_file, test_name, num_trials, seed): test_path = test_file + ":" + test_name - new_env = os.environ + new_env = os.environ.copy() new_env["MXNET_TEST_COUNT"] = str(num_trials) if seed is None: print("Using random seed") else: new_env["MXNET_TEST_SEED"] = seed - code = subprocess.call(["nosetests", "-s","--verbose",test_path], env = new_env) + code = subprocess.call(["nosetests", "-s","--verbose",test_path], + env = new_env) + print("nosetests completed with return code " + str(code)) def find_test_path(test_file): test_file += ".py" - test_path = test_file - top = str(subprocess.check_output(["git", "rev-parse", "--show-toplevel"]), - errors= "strict").strip() + test_path = os.path.basename(test_file) + top = str(subprocess.check_output(["git", "rev-parse", + "--show-toplevel"]), errors= "strict").strip() for (path, names, files) in os.walk(top): if test_file in files: @@ -54,12 +57,12 @@ def find_test_path(test_file): return test_path raise FileNotFoundError("Could not find " + test_file) - class NameAction(argparse.Action): """Parses command line argument to get test file and test name""" def __call__(self, parser, namespace, values, option_string=None): - name = values.split(".") - setattr(namespace, "test_path", find_test_path(name[0])) + name = re.split("\.py:|\.", values) + setattr(namespace, "test_path", + find_test_path(os.path.basename(name[0]))) setattr(namespace, "test_name", name[1]) def parse_args(): @@ -67,15 +70,17 @@ def parse_args(): parser.add_argument("test", action=NameAction, help="file name and and function name of test, " - "provided in the format: file_name.test_name") + "provided in the format: : " + "or .") - parser.add_argument("-n", "--num-trials", + parser.add_argument("-n", "--num-trials", metavar="N", default=DEFAULT_NUM_TRIALS, type=int, help="number of test trials, passed as " "MXNET_TEST_COUNT, defaults to 500") parser.add_argument("-s", "--seed", - help="random seed, passed as MXNET_TEST_SEED") + help="test seed, passed as MXNET_TEST_SEED, " + "defaults to random seed") args = parser.parse_args() return args @@ -83,5 +88,7 @@ def parse_args(): if __name__ == "__main__": args = parse_args() + print(args.test_path) + print(args.test_name) run_test_trials(args.test_path, args.test_name, args.num_trials, args.seed) From 14df1267028e5eb247d2785dd157462f6b0fcfef Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Fri, 6 Jul 2018 19:25:32 +0000 Subject: [PATCH 3/7] added verbosity option, further documentation, etc. --- tools/flakiness_checker.py | 64 ++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/tools/flakiness_checker.py b/tools/flakiness_checker.py index d875b62e769a..5fa809fb3322 100644 --- a/tools/flakiness_checker.py +++ b/tools/flakiness_checker.py @@ -28,41 +28,50 @@ import re -DEFAULT_NUM_TRIALS = 500 +DEFAULT_NUM_TRIALS = 10000 +DEFAULT_VERBOSITY = 2 + +def run_test_trials(args): + test_path = args.test_path + ":" + args.test_name + print("testing: " + test_path) -def run_test_trials(test_file, test_name, num_trials, seed): - test_path = test_file + ":" + test_name - new_env = os.environ.copy() - new_env["MXNET_TEST_COUNT"] = str(num_trials) - if seed is None: + new_env["MXNET_TEST_COUNT"] = str(args.num_trials) + + if args.seed is None: print("Using random seed") else: - new_env["MXNET_TEST_SEED"] = seed + new_env["MXNET_TEST_SEED"] = str(args.seed) + + verbosity = "--verbosity=" + str(args.verbosity) - code = subprocess.call(["nosetests", "-s","--verbose",test_path], + code = subprocess.call(["nosetests", verbosity, test_path], env = new_env) print("nosetests completed with return code " + str(code)) def find_test_path(test_file): + """Searches for the test file and returns the path if found + As a default, the currend working directory is the top of the search. + If a directory was provided as part of the argument, the directory will be + joined with cwd unless it was an absolute path, in which case, the + absolute path will be used instead. + """ test_file += ".py" - test_path = os.path.basename(test_file) - top = str(subprocess.check_output(["git", "rev-parse", - "--show-toplevel"]), errors= "strict").strip() + test_path = os.path.split(test_file) + top = os.path.join(os.getcwd(), test_path[0]) - for (path, names, files) in os.walk(top): - if test_file in files: - test_path = path + "/" + test_path - return test_path - raise FileNotFoundError("Could not find " + test_file) + for (path, dirs, files) in os.walk(top): + if test_path[1] in files: + return os.path.join(path, test_path[1]) + raise FileNotFoundError("Could not find " + test_path[1] + + "in directory: " + top) class NameAction(argparse.Action): """Parses command line argument to get test file and test name""" def __call__(self, parser, namespace, values, option_string=None): name = re.split("\.py:|\.", values) - setattr(namespace, "test_path", - find_test_path(os.path.basename(name[0]))) + setattr(namespace, "test_path", find_test_path(name[0])) setattr(namespace, "test_name", name[1]) def parse_args(): @@ -70,25 +79,28 @@ def parse_args(): parser.add_argument("test", action=NameAction, help="file name and and function name of test, " - "provided in the format: : " - "or .") + "provided in the format: : " + "or /:") parser.add_argument("-n", "--num-trials", metavar="N", default=DEFAULT_NUM_TRIALS, type=int, help="number of test trials, passed as " "MXNET_TEST_COUNT, defaults to 500") - parser.add_argument("-s", "--seed", + parser.add_argument("-s", "--seed", type=int, help="test seed, passed as MXNET_TEST_SEED, " - "defaults to random seed") - + "defaults to random seed") + + parser.add_argument("-v", "--verbosity", + default=DEFAULT_VERBOSITY, type=int, + help="logging level, passed to nosetests") + + args = parser.parse_args() return args if __name__ == "__main__": args = parse_args() - print(args.test_path) - print(args.test_name) - run_test_trials(args.test_path, args.test_name, args.num_trials, args.seed) + run_test_trials(args) From e03e96461bbfc4243a77332acf9d44738db83581 Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Fri, 6 Jul 2018 21:28:05 +0000 Subject: [PATCH 4/7] added logging --- tools/flakiness_checker.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/flakiness_checker.py b/tools/flakiness_checker.py index 5fa809fb3322..68b9c94a7a65 100644 --- a/tools/flakiness_checker.py +++ b/tools/flakiness_checker.py @@ -26,20 +26,22 @@ import random import argparse import re +import logging +logging.basicConfig(level=logging.INFO) DEFAULT_NUM_TRIALS = 10000 DEFAULT_VERBOSITY = 2 def run_test_trials(args): test_path = args.test_path + ":" + args.test_name - print("testing: " + test_path) + logging.info("Testing: %s", test_path) new_env = os.environ.copy() new_env["MXNET_TEST_COUNT"] = str(args.num_trials) if args.seed is None: - print("Using random seed") + logging.info("No test seed provided, using random seed") else: new_env["MXNET_TEST_SEED"] = str(args.seed) @@ -48,7 +50,7 @@ def run_test_trials(args): code = subprocess.call(["nosetests", verbosity, test_path], env = new_env) - print("nosetests completed with return code " + str(code)) + logging.info("Nosetests terminated with exit code %d", code) def find_test_path(test_file): """Searches for the test file and returns the path if found From 16bf6b7f1a51a5ce9cbea3a2efcd45c2e7ad1e2d Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Fri, 6 Jul 2018 21:53:19 +0000 Subject: [PATCH 5/7] Added check for invalid argument --- tools/flakiness_checker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/flakiness_checker.py b/tools/flakiness_checker.py index 68b9c94a7a65..78cdde85f277 100644 --- a/tools/flakiness_checker.py +++ b/tools/flakiness_checker.py @@ -73,6 +73,8 @@ class NameAction(argparse.Action): """Parses command line argument to get test file and test name""" def __call__(self, parser, namespace, values, option_string=None): name = re.split("\.py:|\.", values) + if len(name) != 2: + raise ValueError("Invalid argument format") setattr(namespace, "test_path", find_test_path(name[0])) setattr(namespace, "test_name", name[1]) From 8a06042edc9a8cad034c3dda1c6098408dafb700 Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Fri, 6 Jul 2018 22:19:19 +0000 Subject: [PATCH 6/7] updated error message for specificity --- tools/flakiness_checker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/flakiness_checker.py b/tools/flakiness_checker.py index 78cdde85f277..6106506ba442 100644 --- a/tools/flakiness_checker.py +++ b/tools/flakiness_checker.py @@ -74,7 +74,9 @@ class NameAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): name = re.split("\.py:|\.", values) if len(name) != 2: - raise ValueError("Invalid argument format") + raise ValueError("Invalid argument format for test. Format: " + ". or" + " /:") setattr(namespace, "test_path", find_test_path(name[0])) setattr(namespace, "test_name", name[1]) From 137c44e06fd4f04e1ac78e06e7898e5bae1b1b0f Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Tue, 10 Jul 2018 18:04:38 -0700 Subject: [PATCH 7/7] fixed help message --- tools/flakiness_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/flakiness_checker.py b/tools/flakiness_checker.py index 6106506ba442..79fa3b1854f0 100644 --- a/tools/flakiness_checker.py +++ b/tools/flakiness_checker.py @@ -85,7 +85,7 @@ def parse_args(): parser.add_argument("test", action=NameAction, help="file name and and function name of test, " - "provided in the format: : " + "provided in the format: . " "or /:") parser.add_argument("-n", "--num-trials", metavar="N",