Skip to content

Commit

Permalink
test: add Actions annotation output
Browse files Browse the repository at this point in the history
It's possible to annotate failures in Actions by printing
"::error file={},line={},col={}::{message}". This methos is preferrable
over using a problem matcher because problem matchers only allow
single-line messages, whereas ::error allows multi-line messages.

PR-URL: #34590
Reviewed-By: Rich Trott <[email protected]>
Reviewed-By: Richard Lau <[email protected]>
  • Loading branch information
mmarchini committed Aug 7, 2020
1 parent a11aa9b commit 46bef7b
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 16 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test-asan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@ jobs:
- name: Build
run: make build-ci -j2 V=1
- name: Test
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
4 changes: 2 additions & 2 deletions .github/workflows/test-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
- name: Build
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
- name: Test
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"

test-linux-with-quic:
runs-on: ubuntu-latest
Expand All @@ -42,4 +42,4 @@ jobs:
- name: Build
run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --experimental-quic"
- name: Test
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
2 changes: 1 addition & 1 deletion .github/workflows/test-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,4 @@ jobs:
- name: Build
run: make build-ci -j8 V=1 CONFIG_FLAGS="--error-on-warn --experimental-quic"
- name: Test
run: make run-ci -j8 V=1 TEST_CI_ARGS="-p dots"
run: make run-ci -j8 V=1 TEST_CI_ARGS="-p actions"
49 changes: 37 additions & 12 deletions tools/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,25 @@ def __init__(self, cases, flaky_tests_mode):
self.lock = threading.Lock()
self.shutdown_event = threading.Event()

def GetFailureOutput(self, failure):
output = []
if failure.output.stderr:
output += ["--- stderr ---" ]
output += [failure.output.stderr.strip()]
if failure.output.stdout:
output += ["--- stdout ---"]
output += [failure.output.stdout.strip()]
output += ["Command: %s" % EscapeCommand(failure.command)]
if failure.HasCrashed():
output += ["--- %s ---" % PrintCrashed(failure.output.exit_code)]
if failure.HasTimedOut():
output += ["--- TIMEOUT ---"]
output = "\n".join(output)
return output

def PrintFailureOutput(self, failure):
print(self.GetFailureOutput(failure))

def PrintFailureHeader(self, test):
if test.IsNegative():
negative_marker = '[negative] '
Expand Down Expand Up @@ -224,17 +243,7 @@ def Done(self):
print()
for failed in self.failed:
self.PrintFailureHeader(failed.test)
if failed.output.stderr:
print("--- stderr ---")
print(failed.output.stderr.strip())
if failed.output.stdout:
print("--- stdout ---")
print(failed.output.stdout.strip())
print("Command: %s" % EscapeCommand(failed.command))
if failed.HasCrashed():
print("--- %s ---" % PrintCrashed(failed.output.exit_code))
if failed.HasTimedOut():
print("--- TIMEOUT ---")
self.PrintFailureOutput(failed)
if len(self.failed) == 0:
print("===")
print("=== All tests succeeded")
Expand Down Expand Up @@ -288,6 +297,21 @@ def HasRun(self, output):
sys.stdout.write('.')
sys.stdout.flush()

class ActionsAnnotationProgressIndicator(DotsProgressIndicator):
def GetAnnotationInfo(self, test, output):
traceback = output.stdout + output.stderr
find_full_path = re.search(r' +at .*\(.*%s:([0-9]+):([0-9]+)' % test.file, traceback)
col = line = 0
if find_full_path:
line, col = map(int, find_full_path.groups())
root_path = abspath(join(dirname(__file__), '../')) + os.sep
filename = test.file.replace(root_path, "")
return filename, line, col

def PrintFailureOutput(self, failure):
output = self.GetFailureOutput(failure)
filename, line, column = self.GetAnnotationInfo(failure.test, failure.output)
print("::error file=%s,line=%d,col=%d::%s" % (filename, line, column, output.replace('\n', '%0A')))

class TapProgressIndicator(SimpleProgressIndicator):

Expand Down Expand Up @@ -496,6 +520,7 @@ def ClearLine(self, last_line_length):
PROGRESS_INDICATORS = {
'verbose': VerboseProgressIndicator,
'dots': DotsProgressIndicator,
'actions': ActionsAnnotationProgressIndicator,
'color': ColorProgressIndicator,
'tap': TapProgressIndicator,
'mono': MonochromeProgressIndicator,
Expand Down Expand Up @@ -1299,7 +1324,7 @@ def BuildOptions():
result.add_option('--logfile', dest='logfile',
help='write test output to file. NOTE: this only applies the tap progress indicator')
result.add_option("-p", "--progress",
help="The style of progress indicator (verbose, dots, color, mono, tap)",
help="The style of progress indicator (%s)" % ", ".join(PROGRESS_INDICATORS.keys()),
choices=list(PROGRESS_INDICATORS.keys()), default="mono")
result.add_option("--report", help="Print a summary of the tests to be run",
default=False, action="store_true")
Expand Down

0 comments on commit 46bef7b

Please sign in to comment.