From 26e50f116233e64d56aef40d491679e76e8d38a1 Mon Sep 17 00:00:00 2001 From: Katerina Koukiou Date: Fri, 3 Feb 2017 10:30:28 +0100 Subject: [PATCH] junitxml: adjust junitxml output file to comply with JUnit xsd Change XML file structure in the manner that failures in call and errors in teardown in one test will appear under separate testcase elements in the XML report. --- CHANGELOG.rst | 9 ++++++++- _pytest/junitxml.py | 38 ++++++++++++++++++++++++++++++++++++-- testing/test_junitxml.py | 23 +++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ee3eae09700..44ac7baf375 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -87,7 +87,10 @@ Bug Fixes 3.0.7 (unreleased) ======================= -* +* Change junitxml.py to produce reports that comply with Junitxml schema. + If the same test fails with failure in call and then errors in teardown + we split testcase element into two, one containing the error and the other + the failure. (`#2228`_) Thanks to `@kkoukiou`_ for the PR. * @@ -95,6 +98,10 @@ Bug Fixes * +.. _@kkoukiou: https://github.com/KKoukiou + +.. _#2228: https://github.com/pytest-dev/pytest/issues/2228 + 3.0.6 (2017-01-22) ================== diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py index f486ea10cf4..4f7792aecf5 100644 --- a/_pytest/junitxml.py +++ b/_pytest/junitxml.py @@ -273,6 +273,9 @@ def __init__(self, logfile, prefix): self.node_reporters = {} # nodeid -> _NodeReporter self.node_reporters_ordered = [] self.global_properties = [] + # List of reports that failed on call but teardown is pending. + self.open_reports = [] + self.cnt_double_fail_tests = 0 def finalize(self, report): nodeid = getattr(report, 'nodeid', report) @@ -332,14 +335,33 @@ def pytest_runtest_logreport(self, report): -> teardown node2 -> teardown node1 """ + close_report = None if report.passed: if report.when == "call": # ignore setup/teardown reporter = self._opentestcase(report) reporter.append_pass(report) elif report.failed: + if report.when == "teardown": + # The following vars are needed when xdist plugin is used + report_wid = getattr(report, "worker_id", None) + report_ii = getattr(report, "item_index", None) + close_report = next( + (rep for rep in self.open_reports + if (rep.nodeid == report.nodeid and + getattr(rep, "item_index", None) == report_ii and + getattr(rep, "worker_id", None) == report_wid + ) + ), None) + if close_report: + # We need to open new testcase in case we have failure in + # call and error in teardown in order to follow junit + # schema + self.finalize(close_report) + self.cnt_double_fail_tests += 1 reporter = self._opentestcase(report) if report.when == "call": reporter.append_failure(report) + self.open_reports.append(report) else: reporter.append_error(report) elif report.skipped: @@ -348,6 +370,17 @@ def pytest_runtest_logreport(self, report): self.update_testcase_duration(report) if report.when == "teardown": self.finalize(report) + report_wid = getattr(report, "worker_id", None) + report_ii = getattr(report, "item_index", None) + close_report = next( + (rep for rep in self.open_reports + if (rep.nodeid == report.nodeid and + getattr(rep, "item_index", None) == report_ii and + getattr(rep, "worker_id", None) == report_wid + ) + ), None) + if close_report: + self.open_reports.remove(close_report) def update_testcase_duration(self, report): """accumulates total duration for nodeid from given report and updates @@ -380,8 +413,9 @@ def pytest_sessionfinish(self): suite_stop_time = time.time() suite_time_delta = suite_stop_time - self.suite_start_time - numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + self.stats['error'] - + numtests = (self.stats['passed'] + self.stats['failure'] + + self.stats['skipped'] + self.stats['error'] - + self.cnt_double_fail_tests) logfile.write('') logfile.write(Junit.testsuite( diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 70c02332c15..b4e4c5b1491 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -189,6 +189,29 @@ def test_function(arg): fnode.assert_attr(message="test teardown failure") assert "ValueError" in fnode.toxml() + def test_call_failure_teardown_error(self, testdir): + testdir.makepyfile(""" + import pytest + + @pytest.fixture + def arg(): + yield + raise Exception("Teardown Exception") + def test_function(arg): + raise Exception("Call Exception") + """) + result, dom = runandparse(testdir) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(errors=1, failures=1, tests=1) + first, second = dom.find_by_tag("testcase") + if not first or not second or first == second: + assert 0 + fnode = first.find_first_by_tag("failure") + fnode.assert_attr(message="Exception: Call Exception") + snode = second.find_first_by_tag("error") + snode.assert_attr(message="test teardown failure") + def test_skip_contains_name_reason(self, testdir): testdir.makepyfile(""" import pytest