WIP Change outcome to 'passed' for xfail unless it's strict

CHANGELOG.rst

@@ -79,6 +79,20 @@
 
 *
 
+**Changes**
+
+* Change ``report.outcome`` for ``xpassed`` tests to ``"passed"`` in non-strict
+  mode and ``"failed"`` in strict mode. Thanks to `@hackebrot`_ for the PR
+  (`#1795`_) and `@gprasad84`_ for report (`#1546`_).
+
+* Tests marked with ``xfail(strict=False)`` (the default) now appear in
+  JUnitXML reports as passing tests instead of skipped.
+  Thanks to `@hackebrot`_ for the PR (`#1795`_).
+
+.. _#1795: https://github.com/pytest-dev/pytest/pull/1795
+.. _#1546: https://github.com/pytest-dev/pytest/issues/1546
+.. _@gprasad84: https://github.com/gprasad84
+
 .. _#1210: https://github.com/pytest-dev/pytest/issues/1210
 .. _#1435: https://github.com/pytest-dev/pytest/issues/1435
 .. _#1471: https://github.com/pytest-dev/pytest/issues/1471

_pytest/skipping.py

@@ -220,6 +220,18 @@ def check_strict_xfail(pyfuncitem):
             pytest.fail('[XPASS(strict)] ' + explanation, pytrace=False)
 
 
+def _is_unittest_unexpected_success_a_failure():
+    """Return if the test suite should fail if a @expectedFailure unittest test PASSES.
+
+    From https://docs.python.org/3/library/unittest.html?highlight=unittest#unittest.TestResult.wasSuccessful:
+        Changed in version 3.4: Returns False if there were any
+        unexpectedSuccesses from tests marked with the expectedFailure() decorator.
+
+    TODO: this should be moved to the "compat" module.
+    """
+    return sys.version_info >= (3, 4)
+
+
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
@@ -228,9 +240,15 @@ def pytest_runtest_makereport(item, call):
     evalskip = getattr(item, '_evalskip', None)
     # unitttest special case, see setting of _unexpectedsuccess
     if hasattr(item, '_unexpectedsuccess') and rep.when == "call":
-        # we need to translate into how pytest encodes xpass
-        rep.wasxfail = "reason: " + repr(item._unexpectedsuccess)
-        rep.outcome = "failed"
+        if item._unexpectedsuccess:
+            rep.longrepr = "Unexpected success: {0}".format(item._unexpectedsuccess)
+        else:
+            rep.longrepr = "Unexpected success"
+        if _is_unittest_unexpected_success_a_failure():
+            rep.outcome = "failed"
+        else:
+            rep.outcome = "passed"
+            rep.wasxfail = rep.longrepr
     elif item.config.option.runxfail:
         pass   # don't interefere
     elif call.excinfo and call.excinfo.errisinstance(pytest.xfail.Exception):
@@ -245,8 +263,15 @@ def pytest_runtest_makereport(item, call):
                 rep.outcome = "skipped"
                 rep.wasxfail = evalxfail.getexplanation()
         elif call.when == "call":
-            rep.outcome = "failed"  # xpass outcome
-            rep.wasxfail = evalxfail.getexplanation()
+            strict_default = item.config.getini('xfail_strict')
+            is_strict_xfail = evalxfail.get('strict', strict_default)
+            explanation = evalxfail.getexplanation()
+            if is_strict_xfail:
+                rep.outcome = "failed"
+                rep.longrepr = "[XPASS(strict)] {0}".format(explanation)
+            else:
+                rep.outcome = "passed"
+                rep.wasxfail = explanation
     elif evalskip is not None and rep.skipped and type(rep.longrepr) is tuple:
         # skipped by mark.skipif; change the location of the failure
         # to point to the item definition, otherwise it will display
@@ -260,7 +285,7 @@ def pytest_report_teststatus(report):
     if hasattr(report, "wasxfail"):
         if report.skipped:
             return "xfailed", "x", "xfail"
-        elif report.failed:
+        elif report.passed:
             return "xpassed", "X", ("XPASS", {'yellow': True})
 
 # called by the terminalreporter instance/plugin

testing/python/metafunc.py

@@ -1080,22 +1080,23 @@ def test_increment(n, expected):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=2, skipped=1)
 
-    def test_xfail_passing_is_xpass(self, testdir):
+    @pytest.mark.parametrize('strict', [True, False])
+    def test_xfail_passing_is_xpass(self, testdir, strict):
         s = """
             import pytest
 
             @pytest.mark.parametrize(("n", "expected"), [
                 (1, 2),
-                pytest.mark.xfail("sys.version > 0", reason="some bug")((2, 3)),
+                pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})((2, 3)),
                 (3, 4),
             ])
             def test_increment(n, expected):
                 assert n + 1 == expected
-        """
+        """.format(strict=strict)
         testdir.makepyfile(s)
         reprec = testdir.inline_run()
-        # xpass is fail, obviously :)
-        reprec.assertoutcome(passed=2, failed=1)
+        passed, failed = (2, 1) if strict else (3, 0)
+        reprec.assertoutcome(passed=passed, failed=failed)
 
     def test_parametrize_called_in_generate_tests(self, testdir):
         s = """

testing/test_config.py

@@ -592,6 +592,7 @@ def test_setuppy_fallback(self, tmpdir):
         assert inicfg == {}
 
     def test_nothing(self, tmpdir):
+        tmpdir.chdir()
         rootdir, inifile, inicfg = determine_setup(None, [tmpdir])
         assert rootdir == tmpdir
         assert inifile is None
@@ -603,6 +604,7 @@ def test_with_specific_inifile(self, tmpdir):
         assert rootdir == tmpdir
 
     def test_with_arg_outside_cwd_without_inifile(self, tmpdir):
+        tmpdir.chdir()
         a = tmpdir.mkdir("a")
         b = tmpdir.mkdir("b")
         rootdir, inifile, inicfg = determine_setup(None, [a, b])

testing/test_junitxml.py

@@ -100,7 +100,7 @@ def test_xpass():
         result, dom = runandparse(testdir)
         assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(name="pytest", errors=0, failures=1, skips=3, tests=5)
+        node.assert_attr(name="pytest", errors=0, failures=1, skips=2, tests=5)
 
     def test_summing_simple_with_errors(self, testdir):
         testdir.makepyfile("""
@@ -115,13 +115,16 @@ def test_fail():
             def test_error(fixture):
                 pass
             @pytest.mark.xfail
+            def test_xfail():
+                assert False
+            @pytest.mark.xfail(strict=True)
             def test_xpass():
-                assert 1
+                assert True
         """)
         result, dom = runandparse(testdir)
         assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(name="pytest", errors=1, failures=1, skips=1, tests=4)
+        node.assert_attr(name="pytest", errors=1, failures=2, skips=1, tests=5)
 
     def test_timing_function(self, testdir):
         testdir.makepyfile("""
@@ -346,16 +349,33 @@ def test_xpass():
         result, dom = runandparse(testdir)
         # assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(skips=1, tests=1)
+        node.assert_attr(skips=0, tests=1)
         tnode = node.find_first_by_tag("testcase")
         tnode.assert_attr(
             file="test_xfailure_xpass.py",
             line="1",
             classname="test_xfailure_xpass",
             name="test_xpass")
-        fnode = tnode.find_first_by_tag("skipped")
-        fnode.assert_attr(message="xfail-marked test passes unexpectedly")
-        # assert "ValueError" in fnode.toxml()
+
+    def test_xfailure_xpass_strict(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+            @pytest.mark.xfail(strict=True, reason="This needs to fail!")
+            def test_xpass():
+                pass
+        """)
+        result, dom = runandparse(testdir)
+        # assert result.ret
+        node = dom.find_first_by_tag("testsuite")
+        node.assert_attr(skips=0, tests=1)
+        tnode = node.find_first_by_tag("testcase")
+        tnode.assert_attr(
+            file="test_xfailure_xpass_strict.py",
+            line="1",
+            classname="test_xfailure_xpass_strict",
+            name="test_xpass")
+        fnode = tnode.find_first_by_tag("failure")
+        fnode.assert_attr(message="[XPASS(strict)] This needs to fail!")
 
     def test_collect_error(self, testdir):
         testdir.makepyfile("syntax error")

testing/test_skipping.py

@@ -145,15 +145,29 @@ def test_func():
     def test_xfail_xpassed(self, testdir):
         item = testdir.getitem("""
             import pytest
-            @pytest.mark.xfail
+            @pytest.mark.xfail(reason="this is an xfail")
+            def test_func():
+                assert 1
+        """)
+        reports = runtestprotocol(item, log=False)
+        assert len(reports) == 3
+        callreport = reports[1]
+        assert callreport.passed
+        assert callreport.wasxfail == "this is an xfail"
+
+    def test_xfail_xpassed_strict(self, testdir):
+        item = testdir.getitem("""
+            import pytest
+            @pytest.mark.xfail(strict=True, reason="nope")
             def test_func():
                 assert 1
         """)
         reports = runtestprotocol(item, log=False)
         assert len(reports) == 3
         callreport = reports[1]
         assert callreport.failed
-        assert callreport.wasxfail == ""
+        assert callreport.longrepr == "[XPASS(strict)] nope"
+        assert not hasattr(callreport, "wasxfail")
 
     def test_xfail_run_anyway(self, testdir):
         testdir.makepyfile("""

testing/test_unittest.py

@@ -419,8 +419,9 @@ def setup_class(cls):
                 def test_method(self):
                     pass
         """)
+        from _pytest.skipping import _is_unittest_unexpected_success_a_failure
+        should_fail = _is_unittest_unexpected_success_a_failure()
         result = testdir.runpytest("-rxs")
-        assert result.ret == 0
         result.stdout.fnmatch_lines_random([
             "*XFAIL*test_trial_todo*",
             "*trialselfskip*",
@@ -429,8 +430,9 @@ def test_method(self):
             "*i2wanto*",
             "*sys.version_info*",
             "*skip_in_method*",
-            "*4 skipped*3 xfail*1 xpass*",
+            "*1 failed*4 skipped*3 xfailed*" if should_fail else "*4 skipped*3 xfail*1 xpass*",
         ])
+        assert result.ret == (1 if should_fail else 0)
 
     def test_trial_error(self, testdir):
         testdir.makepyfile("""
@@ -587,24 +589,62 @@ def test_hello(self, arg1):
     assert "TypeError" in result.stdout.str()
     assert result.ret == 1
 
+
 @pytest.mark.skipif("sys.version_info < (2,7)")
-def test_unittest_unexpected_failure(testdir):
-    testdir.makepyfile("""
+@pytest.mark.parametrize('runner', ['pytest', 'unittest'])
+def test_unittest_expected_failure_for_failing_test_is_xfail(testdir, runner):
+    script = testdir.makepyfile("""
         import unittest
         class MyTestCase(unittest.TestCase):
             @unittest.expectedFailure
-            def test_func1(self):
-                assert 0
+            def test_failing_test_is_xfail(self):
+                assert False
+        if __name__ == '__main__':
+            unittest.main()
+    """)
+    if runner == 'pytest':
+        result = testdir.runpytest("-rxX")
+        result.stdout.fnmatch_lines([
+            "*XFAIL*MyTestCase*test_failing_test_is_xfail*",
+            "*1 xfailed*",
+        ])
+    else:
+        result = testdir.runpython(script)
+        result.stderr.fnmatch_lines([
+            "*1 test in*",
+            "*OK*(expected failures=1)*",
+        ])
+    assert result.ret == 0
+
+
+@pytest.mark.skipif("sys.version_info < (2,7)")
+@pytest.mark.parametrize('runner', ['pytest', 'unittest'])
+def test_unittest_expected_failure_for_passing_test_is_fail(testdir, runner):
+    script = testdir.makepyfile("""
+        import unittest
+        class MyTestCase(unittest.TestCase):
             @unittest.expectedFailure
-            def test_func2(self):
-                assert 1
+            def test_passing_test_is_fail(self):
+                assert True
+        if __name__ == '__main__':
+            unittest.main()
     """)
-    result = testdir.runpytest("-rxX")
-    result.stdout.fnmatch_lines([
-        "*XFAIL*MyTestCase*test_func1*",
-        "*XPASS*MyTestCase*test_func2*",
-        "*1 xfailed*1 xpass*",
-    ])
+    from _pytest.skipping import _is_unittest_unexpected_success_a_failure
+    should_fail = _is_unittest_unexpected_success_a_failure()
+    if runner == 'pytest':
+        result = testdir.runpytest("-rxX")
+        result.stdout.fnmatch_lines([
+            "*MyTestCase*test_passing_test_is_fail*",
+            "*1 failed*" if should_fail else "*1 xpassed*",
+        ])
+    else:
+        result = testdir.runpython(script)
+        result.stderr.fnmatch_lines([
+            "*1 test in*",
+            "*(unexpected successes=1)*",
+        ])
+
+    assert result.ret == (1 if should_fail else 0)
 
 
 @pytest.mark.parametrize('fix_type, stmt', [