diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ee3eae09700..44ac7baf375 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -87,7 +87,10 @@ Bug Fixes 3.0.7 (unreleased) ======================= -* +* Change junitxml.py to produce reports that comply with Junitxml schema. + If the same test fails with failure in call and then errors in teardown + we split testcase element into two, one containing the error and the other + the failure. (`#2228`_) Thanks to `@kkoukiou`_ for the PR. * @@ -95,6 +98,10 @@ Bug Fixes * +.. _@kkoukiou: https://github.com/KKoukiou + +.. _#2228: https://github.com/pytest-dev/pytest/issues/2228 + 3.0.6 (2017-01-22) ================== diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py index f486ea10cf4..4f7792aecf5 100644 --- a/_pytest/junitxml.py +++ b/_pytest/junitxml.py @@ -273,6 +273,9 @@ def __init__(self, logfile, prefix): self.node_reporters = {} # nodeid -> _NodeReporter self.node_reporters_ordered = [] self.global_properties = [] + # List of reports that failed on call but teardown is pending. + self.open_reports = [] + self.cnt_double_fail_tests = 0 def finalize(self, report): nodeid = getattr(report, 'nodeid', report) @@ -332,14 +335,33 @@ def pytest_runtest_logreport(self, report): -> teardown node2 -> teardown node1 """ + close_report = None if report.passed: if report.when == "call": # ignore setup/teardown reporter = self._opentestcase(report) reporter.append_pass(report) elif report.failed: + if report.when == "teardown": + # The following vars are needed when xdist plugin is used + report_wid = getattr(report, "worker_id", None) + report_ii = getattr(report, "item_index", None) + close_report = next( + (rep for rep in self.open_reports + if (rep.nodeid == report.nodeid and + getattr(rep, "item_index", None) == report_ii and + getattr(rep, "worker_id", None) == report_wid + ) + ), None) + if close_report: + # We need to open new testcase in case we have failure in + # call and error in teardown in order to follow junit + # schema + self.finalize(close_report) + self.cnt_double_fail_tests += 1 reporter = self._opentestcase(report) if report.when == "call": reporter.append_failure(report) + self.open_reports.append(report) else: reporter.append_error(report) elif report.skipped: @@ -348,6 +370,17 @@ def pytest_runtest_logreport(self, report): self.update_testcase_duration(report) if report.when == "teardown": self.finalize(report) + report_wid = getattr(report, "worker_id", None) + report_ii = getattr(report, "item_index", None) + close_report = next( + (rep for rep in self.open_reports + if (rep.nodeid == report.nodeid and + getattr(rep, "item_index", None) == report_ii and + getattr(rep, "worker_id", None) == report_wid + ) + ), None) + if close_report: + self.open_reports.remove(close_report) def update_testcase_duration(self, report): """accumulates total duration for nodeid from given report and updates @@ -380,8 +413,9 @@ def pytest_sessionfinish(self): suite_stop_time = time.time() suite_time_delta = suite_stop_time - self.suite_start_time - numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + self.stats['error'] - + numtests = (self.stats['passed'] + self.stats['failure'] + + self.stats['skipped'] + self.stats['error'] - + self.cnt_double_fail_tests) logfile.write('') logfile.write(Junit.testsuite( diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 70c02332c15..b4e4c5b1491 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -189,6 +189,29 @@ def test_function(arg): fnode.assert_attr(message="test teardown failure") assert "ValueError" in fnode.toxml() + def test_call_failure_teardown_error(self, testdir): + testdir.makepyfile(""" + import pytest + + @pytest.fixture + def arg(): + yield + raise Exception("Teardown Exception") + def test_function(arg): + raise Exception("Call Exception") + """) + result, dom = runandparse(testdir) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(errors=1, failures=1, tests=1) + first, second = dom.find_by_tag("testcase") + if not first or not second or first == second: + assert 0 + fnode = first.find_first_by_tag("failure") + fnode.assert_attr(message="Exception: Call Exception") + snode = second.find_first_by_tag("error") + snode.assert_attr(message="test teardown failure") + def test_skip_contains_name_reason(self, testdir): testdir.makepyfile(""" import pytest