Skip to content

Commit

Permalink
test: remove flaky test functionality
Browse files Browse the repository at this point in the history
Reverts nodejs/node-v0.x-archive#8689

PR-URL: #812
Reviewed-By: Jeremiah Senkpiel <[email protected]>
Reviewed-By: Johan Bergström <[email protected]>
Reviewed-By: Ben Noordhuis <[email protected]>
Reviewed-By: Colin Ihrig <[email protected]>
  • Loading branch information
rvagg committed Feb 16, 2015
1 parent fc6507d commit 20f8e7f
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 41 deletions.
1 change: 0 additions & 1 deletion test/internet/internet.status

This file was deleted.

4 changes: 0 additions & 4 deletions test/parallel/simple.status

This file was deleted.

1 change: 0 additions & 1 deletion test/pummel/pummel.status

This file was deleted.

48 changes: 13 additions & 35 deletions tools/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,8 @@

class ProgressIndicator(object):

def __init__(self, cases, flaky_tests_mode):
def __init__(self, cases):
self.cases = cases
self.flaky_tests_mode = flaky_tests_mode
self.parallel_queue = Queue(len(cases))
self.sequential_queue = Queue(len(cases))
for case in cases:
Expand Down Expand Up @@ -248,19 +247,13 @@ def HasRun(self, output):
self._done += 1
command = basename(output.command[-1])
if output.UnexpectedOutput():
status_line = 'not ok %i - %s' % (self._done, command)
if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare":
status_line = status_line + " # TODO : Fix flaky test"
print status_line
print 'not ok %i - %s' % (self._done, command)
for l in output.output.stderr.splitlines():
print '#' + l
for l in output.output.stdout.splitlines():
print '#' + l
else:
status_line = 'ok %i - %s' % (self._done, command)
if FLAKY in output.test.outcomes:
status_line = status_line + " # TODO : Fix flaky test"
print status_line
print 'ok %i - %s' % (self._done, command)

duration = output.test.duration

Expand All @@ -278,8 +271,8 @@ def Done(self):

class CompactProgressIndicator(ProgressIndicator):

def __init__(self, cases, flaky_tests_mode, templates):
super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
def __init__(self, cases, templates):
super(CompactProgressIndicator, self).__init__(cases)
self.templates = templates
self.last_status_length = 0
self.start_time = time.time()
Expand Down Expand Up @@ -334,29 +327,29 @@ def PrintProgress(self, name):

class ColorProgressIndicator(CompactProgressIndicator):

def __init__(self, cases, flaky_tests_mode):
def __init__(self, cases):
templates = {
'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
'stdout': "\033[1m%s\033[0m",
'stderr': "\033[31m%s\033[0m",
}
super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
super(ColorProgressIndicator, self).__init__(cases, templates)

def ClearLine(self, last_line_length):
print "\033[1K\r",


class MonochromeProgressIndicator(CompactProgressIndicator):

def __init__(self, cases, flaky_tests_mode):
def __init__(self, cases):
templates = {
'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
'stdout': '%s',
'stderr': '%s',
'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
'max_length': 78
}
super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
super(MonochromeProgressIndicator, self).__init__(cases, templates)

def ClearLine(self, last_line_length):
print ("\r" + (" " * last_line_length) + "\r"),
Expand Down Expand Up @@ -776,8 +769,8 @@ def GetVmFlags(self, testcase, mode):
def GetTimeout(self, mode):
return self.timeout * TIMEOUT_SCALEFACTOR[mode]

def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
def RunTestCases(cases_to_run, progress, tasks):
progress = PROGRESS_INDICATORS[progress](cases_to_run)
return progress.Run(tasks)


Expand All @@ -801,7 +794,6 @@ def BuildRequirements(context, requirements, mode, scons_flags):
TIMEOUT = 'timeout'
CRASH = 'crash'
SLOW = 'slow'
FLAKY = 'flaky'


class Expression(object):
Expand Down Expand Up @@ -1248,9 +1240,6 @@ def BuildOptions():
default=False, action="store_true")
result.add_option("--cat", help="Print the source of the tests",
default=False, action="store_true")
result.add_option("--flaky-tests",
help="Regard tests marked as flaky (run|skip|dontcare)",
default="run")
result.add_option("--warn-unused", help="Report unused rules",
default=False, action="store_true")
result.add_option("-j", help="The number of parallel tasks to run",
Expand Down Expand Up @@ -1280,35 +1269,24 @@ def ProcessOptions(options):
options.mode = options.mode.split(',')
if options.J:
options.j = multiprocessing.cpu_count()
def CheckTestMode(name, option):
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
return False
return True
if not CheckTestMode("--flaky-tests", options.flaky_tests):
return False
return True


REPORT_TEMPLATE = """\
Total: %(total)i tests
* %(skipped)4d tests will be skipped
* %(nocrash)4d tests are expected to be flaky but not crash
* %(pass)4d tests are expected to pass
* %(fail_ok)4d tests are expected to fail that we won't fix
* %(fail)4d tests are expected to fail that we should fix\
"""

def PrintReport(cases):
def IsFlaky(o):
return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o)
def IsFailOk(o):
return (len(o) == 2) and (FAIL in o) and (OKAY in o)
unskipped = [c for c in cases if not SKIP in c.outcomes]
print REPORT_TEMPLATE % {
'total': len(cases),
'skipped': len(cases) - len(unskipped),
'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]),
'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]),
'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]),
'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]])
Expand Down Expand Up @@ -1486,15 +1464,15 @@ def Main():

result = None
def DoSkip(case):
return SKIP in case.outcomes or SLOW in case.outcomes or (FLAKY in case.outcomes and options.flaky_tests == "skip")
return SKIP in case.outcomes or SLOW in case.outcomes
cases_to_run = [ c for c in all_cases if not DoSkip(c) ]
if len(cases_to_run) == 0:
print "No tests to run."
return 1
else:
try:
start = time.time()
if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
if RunTestCases(cases_to_run, options.progress, options.j):
result = 0
else:
result = 1
Expand Down

0 comments on commit 20f8e7f

Please sign in to comment.