Make cache plugin cumulative

_pytest/cacheprovider.py

@@ -105,27 +105,22 @@ def __init__(self, config):
         self.config = config
         active_keys = 'lf', 'failedfirst'
         self.active = any(config.getvalue(key) for key in active_keys)
-        if self.active:
-            self.lastfailed = config.cache.get("cache/lastfailed", {})
-        else:
-            self.lastfailed = {}
+        self.lastfailed = config.cache.get("cache/lastfailed", {})
 
     def pytest_report_header(self):
         if self.active:
             if not self.lastfailed:
                 mode = "run all (no recorded failures)"
             else:
-                mode = "rerun last %d failures%s" % (
-                    len(self.lastfailed),
+                mode = "rerun previous failures%s" % (
                     " first" if self.config.getvalue("failedfirst") else "")
             return "run-last-failure: %s" % mode
 
     def pytest_runtest_logreport(self, report):
-        if report.failed and "xfail" not in report.keywords:
+        if (report.when == 'call' and report.passed) or report.skipped:
+            self.lastfailed.pop(report.nodeid, None)
+        elif report.failed:
             self.lastfailed[report.nodeid] = True
-        elif not report.failed:
-            if report.when == "call":
-                self.lastfailed.pop(report.nodeid, None)
 
     def pytest_collectreport(self, report):
         passed = report.outcome in ('passed', 'skipped')
@@ -147,11 +142,11 @@ def pytest_collection_modifyitems(self, session, config, items):
                     previously_failed.append(item)
                 else:
                     previously_passed.append(item)
-            if not previously_failed and previously_passed:
+            if not previously_failed:
                 # running a subset of all tests with recorded failures outside
                 # of the set of tests currently executing
-                pass
-            elif self.config.getvalue("lf"):
+                return
+            if self.config.getvalue("lf"):
                 items[:] = previously_failed
                 config.hook.pytest_deselected(items=previously_passed)
             else:
@@ -161,8 +156,9 @@ def pytest_sessionfinish(self, session):
         config = self.config
         if config.getvalue("cacheshow") or hasattr(config, "slaveinput"):
             return
-        prev_failed = config.cache.get("cache/lastfailed", None) is not None
-        if (session.testscollected and prev_failed) or self.lastfailed:
+
+        saved_lastfailed = config.cache.get("cache/lastfailed", {})
+        if saved_lastfailed != self.lastfailed:
             config.cache.set("cache/lastfailed", self.lastfailed)
 
 

changelog/2621.feature

@@ -0,0 +1,2 @@
+``--last-failed`` now remembers forever when a test has failed and only forgets it if it passes again. This makes it
+easy to fix a test suite by selectively running files and fixing tests incrementally.

testing/test_cache.py

@@ -437,3 +437,102 @@ def test_lastfailed_creates_cache_when_needed(self, testdir):
         testdir.makepyfile(test_errored='def test_error():\n    assert False')
         testdir.runpytest('-q', '--lf')
         assert os.path.exists('.cache')
+
+    def test_xfail_not_considered_failure(self, testdir):
+        testdir.makepyfile('''
+            import pytest
+            @pytest.mark.xfail
+            def test():
+                assert 0
+        ''')
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines('*1 xfailed*')
+        assert self.get_cached_last_failed(testdir) == []
+
+    def test_xfail_strict_considered_failure(self, testdir):
+        testdir.makepyfile('''
+            import pytest
+            @pytest.mark.xfail(strict=True)
+            def test():
+                pass
+        ''')
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines('*1 failed*')
+        assert self.get_cached_last_failed(testdir) == ['test_xfail_strict_considered_failure.py::test']
+
+    @pytest.mark.parametrize('mark', ['mark.xfail', 'mark.skip'])
+    def test_failed_changed_to_xfail_or_skip(self, testdir, mark):
+        testdir.makepyfile('''
+            import pytest
+            def test():
+                assert 0
+        ''')
+        result = testdir.runpytest()
+        assert self.get_cached_last_failed(testdir) == ['test_failed_changed_to_xfail_or_skip.py::test']
+        assert result.ret == 1
+
+        testdir.makepyfile('''
+            import pytest
+            @pytest.{mark}
+            def test():
+                assert 0
+        '''.format(mark=mark))
+        result = testdir.runpytest()
+        assert result.ret == 0
+        assert self.get_cached_last_failed(testdir) == []
+        assert result.ret == 0
+
+    def get_cached_last_failed(self, testdir):
+        config = testdir.parseconfigure()
+        return sorted(config.cache.get("cache/lastfailed", {}))
+
+    def test_cache_cumulative(self, testdir):
+        """
+        Test workflow where user fixes errors gradually file by file using --lf.
+        """
+        # 1. initial run
+        test_bar = testdir.makepyfile(test_bar="""
+            def test_bar_1():
+                pass
+            def test_bar_2():
+                assert 0
+        """)
+        test_foo = testdir.makepyfile(test_foo="""
+            def test_foo_3():
+                pass
+            def test_foo_4():
+                assert 0
+        """)
+        testdir.runpytest()
+        assert self.get_cached_last_failed(testdir) == ['test_bar.py::test_bar_2', 'test_foo.py::test_foo_4']
+
+        # 2. fix test_bar_2, run only test_bar.py
+        testdir.makepyfile(test_bar="""
+            def test_bar_1():
+                pass
+            def test_bar_2():
+                pass
+        """)
+        result = testdir.runpytest(test_bar)
+        result.stdout.fnmatch_lines('*2 passed*')
+        # ensure cache does not forget that test_foo_4 failed once before
+        assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4']
+
+        result = testdir.runpytest('--last-failed')
+        result.stdout.fnmatch_lines('*1 failed, 3 deselected*')
+        assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4']
+
+        # 3. fix test_foo_4, run only test_foo.py
+        test_foo = testdir.makepyfile(test_foo="""
+            def test_foo_3():
+                pass
+            def test_foo_4():
+                pass
+        """)
+        result = testdir.runpytest(test_foo, '--last-failed')
+        result.stdout.fnmatch_lines('*1 passed, 1 deselected*')
+        assert self.get_cached_last_failed(testdir) == []
+
+        result = testdir.runpytest('--last-failed')
+        result.stdout.fnmatch_lines('*4 passed*')
+        assert self.get_cached_last_failed(testdir) == []