From 46b3c0fb8118265319f2508c92f62e061f978887 Mon Sep 17 00:00:00 2001 From: Carl Tsai Date: Wed, 8 Aug 2018 15:32:35 -0700 Subject: [PATCH] changed print statments to use logging changed Jenkinsfile to use redesigned system included config file for dependency analyzer minor fixes --- tools/flaky_tests/Jenkinsfile | 43 ++++++++-------------- tools/flaky_tests/check_branch.py | 16 ++++---- tools/flaky_tests/dependency_analyzer.py | 17 +++++---- tools/flaky_tests/diff_collator.py | 13 +++---- tools/flaky_tests/flakiness_checker.py | 4 +- tools/flaky_tests/test_dependencies.config | 22 +++++++++++ 6 files changed, 63 insertions(+), 52 deletions(-) create mode 100644 tools/flaky_tests/test_dependencies.config diff --git a/tools/flaky_tests/Jenkinsfile b/tools/flaky_tests/Jenkinsfile index a8a5eee34ed6..1d40e0e405cf 100644 --- a/tools/flaky_tests/Jenkinsfile +++ b/tools/flaky_tests/Jenkinsfile @@ -19,40 +19,27 @@ err = null -def init_git() { - deleteDir() - retry(5) { - try{ - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source code with $(exc)" - sleep 2 - } - } +node('mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node)labels(linux_gpu: 'mxnetlinux-gpu') -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %%SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" - command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker': '') - command = command.replaceAll('%PLATFORM%', platform) - command = command.replaceAll('%FUNCTION_NAME%', function_name) - command = command.replaceAll('%SHARED_MEM%', shared_mem) - sh command -} - -try{ +utils.main_wrapper( +core_logic: { stage('Flakiness Check'){ - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/flakiness_check'){ - init_git() - docker_run('ubuntu_gpu', 'run_flakiness_checker', true) + utils.init_git() + utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) + utils.docker_run('ubuntu_gpu', 'run_flakiness_checker', false) } } } } +, +failure_handler: { +} +) diff --git a/tools/flaky_tests/check_branch.py b/tools/flaky_tests/check_branch.py index 48892c71e385..d86efa860e20 100644 --- a/tools/flaky_tests/check_branch.py +++ b/tools/flaky_tests/check_branch.py @@ -25,6 +25,7 @@ import os import subprocess import json +import sys import flakiness_checker import diff_collator @@ -88,7 +89,7 @@ def time_test(test): try: n = int(TIME_BUDGET / total_time) except ZeroDivisionError: - logger.Error("Total time for tests was 0") + logger.error("Total time for tests was 0") return [] logger.debug("total_time: %f | num_trials: %d", total_time, n) @@ -112,17 +113,17 @@ def check_tests(tests): def output_results(flaky, nonflaky): - print("Following tests failed flakiness checker:") + logger.info("Following tests failed flakiness checker:") if not flaky: - print("None") + logger.info("None") for test in flaky: - print("%s:%s".format(test[0], test[1])) + logger.info("%s:%s", test[0], test[1]) - print("Following tests passed flakiness checker:") + logger.info("Following tests passed flakiness checker:") if not nonflaky: - print("None") + logger.info("None") for test in nonflaky: - print("{}:{}".format(test[0], test[1])) + logger.info("%s:%s", test[0], test[1]) logger.info("[Results]\tTotal: %d\tFlaky: %d\tNon-flaky: %d", len(flaky) + len(nonflaky), len(flaky), len(nonflaky)) @@ -138,6 +139,7 @@ def output_results(flaky, nonflaky): diff_output = diff_collator.get_diff_output(args) changes = diff_collator.parser(diff_output) + diff_collator.output_changes(changes) changes = {k:set(v.keys()) for k, v in changes.items()} tests = select_tests(changes) diff --git a/tools/flaky_tests/dependency_analyzer.py b/tools/flaky_tests/dependency_analyzer.py index 9d6c067fec0f..eedc915b0c28 100644 --- a/tools/flaky_tests/dependency_analyzer.py +++ b/tools/flaky_tests/dependency_analyzer.py @@ -37,7 +37,7 @@ import json DEFAULT_CONFIG_FILE = os.path.join( - os.path.dirname(__file__), "config.json") + os.path.dirname(__file__), "test_dependencies.config") logger = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def find_dependents(dependencies, top): try: file_deps = read_config(DEFAULT_CONFIG_FILE) - except FileNotFoundError: + except IOError: file_deps = {} logger.WARNING("No config file found, " "continuing with no file dependencies") @@ -115,21 +115,22 @@ def visit_Attribute(self, node): try: dependents |= find_dependents_file(dependents - dependencies, filename) - except RecursionError as re: - logger.error("Encountered recursion error when seaching {}: {}", + except RuntimeError as re: + logger.error("Encountered recursion error when seaching %s: %s", filename, re.args[0]) return dependents def output_results(dependents): + logger.info("Dependencies:") for filename in dependents.keys(): - print(filename) + logger.info(filename) if not dependents[filename]: - print("None") + logger.info("None") continue for func in dependents[filename]: - print("\t{}".format(func)) + logger.info("\t%s", func) def parse_args(): @@ -152,7 +153,7 @@ def __call__(self, parser, namespace, values, option_string=None): help="list of dependent functions, " "in the format: :") - arg_parse.add_argument( + arg_parser.add_argument( "--logging-level", "-l", dest="level", default="INFO", help="logging level, defaults to INFO") diff --git a/tools/flaky_tests/diff_collator.py b/tools/flaky_tests/diff_collator.py index 33e294894e89..76965c90d611 100644 --- a/tools/flaky_tests/diff_collator.py +++ b/tools/flaky_tests/diff_collator.py @@ -126,11 +126,12 @@ def parse_patch(patch): if line.startswith("+def "): func_name = line.split()[1].split("(")[0] changes[func_name] = [] + logger.debug("\tFound new top-level function: %s", func_name) return file_name, changes -def output_changes(changes, verbosity): +def output_changes(changes, verbosity=2): """ Output changes in an easy to understand format Three verbosity levels: @@ -147,23 +148,21 @@ def output_changes(changes, verbosity): 5:5 func_c """ - if not verbosity: - verbosity = 2 logger.debug("verbosity: %d", verbosity) if not changes: logger.info("No changes found") else: for file_name, chunks in changes.items(): - print(file_name) + logger.info(file_name) if verbosity < 2: continue for func_name, ranges in chunks.items(): - print("\t{}".format(func_name)) + logger.info("\t%s", func_name) if verbosity < 3: continue for (start, end) in ranges: - print("\t\t{}:{}".format(start, end)) + logger.info("\t\t%s:%s", start, end) @@ -212,7 +211,7 @@ def parse_args(): changes = parser(diff_output) for file_name, chunks in changes.items(): - if not re.fullmatch(args.expr, file_name): + if not re.match(args.expr, file_name): del changes[file_name] output_changes(changes, args.verbosity) diff --git a/tools/flaky_tests/flakiness_checker.py b/tools/flaky_tests/flakiness_checker.py index c2713f3f69bd..606558802adb 100755 --- a/tools/flaky_tests/flakiness_checker.py +++ b/tools/flaky_tests/flakiness_checker.py @@ -69,7 +69,7 @@ def find_test_path(self, test_file): test_path = os.path.split(test_file) top = os.path.join(os.getcwd(), test_path[0]) - for (path, dirs, files) in os.walk(top): + for (path, _ , files) in os.walk(top): if test_path[1] in files: return os.path.join(path, test_path[1]) @@ -122,7 +122,7 @@ def parse_args(): except AttributeError: logging.basicConfig(level=logging.INFO) logging.warning("Invalid logging level: %s", args.level) - logger.debug("args: $s", args) + logger.debug("args: %s", args) code = run_test_trials(args.test_path, args.test_name, args.num_trials, args.seed, args.args) diff --git a/tools/flaky_tests/test_dependencies.config b/tools/flaky_tests/test_dependencies.config new file mode 100644 index 000000000000..20575a0823f7 --- /dev/null +++ b/tools/flaky_tests/test_dependencies.config @@ -0,0 +1,22 @@ +{ + "tests/python/gpu/test_operator_gpu.py": [ + "test_operator.py", + "test_optimizer.py", + "test_random.py", + "test_exc_handling.py", + "test_sparse_ndarray.py", + "test_sparse_operator.py", + "test_ndarray.py" + ], + "tests/python/gpu/test_gluon_gpu.py": [ + "test_gluon.py", + "test_loss.py", + "test_gluon_rnn.py" + ], + "tests/python/mkl/test_quantization_mkldnn.py": [ + "test_quantization.py" + ], + "tests/python/quantization_gpu/test_quantization_gpu.py": [ + "test_quantization.py" + ] +} \ No newline at end of file