Make cache plugin always remember failed tests

_pytest/cacheprovider.py

@@ -105,27 +105,22 @@ def __init__(self, config):
         self.config = config
         active_keys = 'lf', 'failedfirst'
         self.active = any(config.getvalue(key) for key in active_keys)
-        if self.active:
-            self.lastfailed = config.cache.get("cache/lastfailed", {})
-        else:
-            self.lastfailed = {}
+        self.lastfailed = config.cache.get("cache/lastfailed", {})
 
     def pytest_report_header(self):
         if self.active:
             if not self.lastfailed:
                 mode = "run all (no recorded failures)"
             else:
-                mode = "rerun last %d failures%s" % (
-                    len(self.lastfailed),
+                mode = "rerun previous failures%s" % (
                     " first" if self.config.getvalue("failedfirst") else "")
             return "run-last-failure: %s" % mode
 
     def pytest_runtest_logreport(self, report):
-        if report.failed and "xfail" not in report.keywords:
+        if report.passed and report.when == 'call':
+            self.lastfailed.pop(report.nodeid, None)
+        elif report.failed:
             self.lastfailed[report.nodeid] = True
-        elif not report.failed:
-            if report.when == "call":
-                self.lastfailed.pop(report.nodeid, None)
 
     def pytest_collectreport(self, report):
         passed = report.outcome in ('passed', 'skipped')

testing/test_cache.py

@@ -437,3 +437,58 @@ def test_lastfailed_creates_cache_when_needed(self, testdir):
         testdir.makepyfile(test_errored='def test_error():\n    assert False')
         testdir.runpytest('-q', '--lf')
         assert os.path.exists('.cache')
+
+    def get_cached_last_failed(self, testdir):
+        config = testdir.parseconfigure()
+        return sorted(config.cache.get("cache/lastfailed", {}))
+
+    def test_cache_cumulative(self, testdir):
+        """
+        Test workflow where user fixes errors gradually file by file using --lf.
+        """
+        # 1. initial run
+        test_bar = testdir.makepyfile(test_bar="""
+            def test_bar_1():
+                pass
+            def test_bar_2():
+                assert 0
+        """)
+        test_foo = testdir.makepyfile(test_foo="""
+            def test_foo_3():
+                pass
+            def test_foo_4():
+                assert 0
+        """)
+        testdir.runpytest()
+        assert self.get_cached_last_failed(testdir) == ['test_bar.py::test_bar_2', 'test_foo.py::test_foo_4']
+
+        # 2. fix test_bar_2, run only test_bar.py
+        testdir.makepyfile(test_bar="""
+            def test_bar_1():
+                pass
+            def test_bar_2():
+                pass
+        """)
+        result = testdir.runpytest(test_bar)
+        result.stdout.fnmatch_lines('*2 passed*')
+        # ensure cache does not forget that test_foo_4 failed once before
+        assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4']
+
+        result = testdir.runpytest('--last-failed')
+        result.stdout.fnmatch_lines('*1 failed, 3 deselected*')
+        assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4']
+
+        # 3. fix test_foo_4, run only test_foo.py
+        test_foo = testdir.makepyfile(test_foo="""
+            def test_foo_3():
+                pass
+            def test_foo_4():
+                pass
+        """)
+        result = testdir.runpytest(test_foo, '--last-failed')
+        result.stdout.fnmatch_lines('*1 passed, 1 deselected*')
+        assert self.get_cached_last_failed(testdir) == []
+
+        result = testdir.runpytest('--last-failed')
+        result.stdout.fnmatch_lines('*4 passed*')
+        assert self.get_cached_last_failed(testdir) == []