From 43064f5b61815496afa52a802ee6809fee0c06e2 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 10 Jan 2025 12:34:52 -0300 Subject: [PATCH 1/7] Improve stepwise to not forget failed tests Now `--stepwise` will remember the last failed test, even if the previous pytest invocations did not pass `--stepwise`. Previously it would always clear the cache if not active, which hinders certain interactive workflows, which is the prime use cases for the flag. --- changelog/13122.improvement.rst | 11 ++++++++++ src/_pytest/stepwise.py | 2 -- testing/test_cacheprovider.py | 4 ++-- testing/test_stepwise.py | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 changelog/13122.improvement.rst diff --git a/changelog/13122.improvement.rst b/changelog/13122.improvement.rst new file mode 100644 index 00000000000..d7372380e81 --- /dev/null +++ b/changelog/13122.improvement.rst @@ -0,0 +1,11 @@ +Improve the ``--stepwise`` flag to not forget the last failed test in case pytest is executed later without the flag. + +This enables the following workflow: + +1. Execute pytest with ``--stepwise``, pytest then stops at the first failing test; +2. User iteratively updates the code and runs the test in isolation, without the ``--stepwise`` flag + (for example in an IDE), until it is fixed. +3. At this point, the user can execute pytest with ``--stepwise`` again and pytest will continue from the previously + failed test, and if it passes, continue with the next tests. + +Previously, at step 3, pytest would start from the beginning, forgetting the failed tests. diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index c7860808c35..82d1fe3aee5 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -47,8 +47,6 @@ def pytest_sessionfinish(session: Session) -> None: # Do not update cache if this process is a xdist worker to prevent # race conditions (#10641). return - # Clear the list of failing tests if the plugin is not active. - session.config.cache.set(STEPWISE_CACHE_DIR, []) class StepwisePlugin: diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 1ba4659d84e..ca417e86ee5 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -104,7 +104,7 @@ def test_cache_failure_warns( pytester.makepyfile("def test_error(): raise Exception") result = pytester.runpytest() assert result.ret == 1 - # warnings from nodeids, lastfailed, and stepwise + # warnings from nodeids and lastfailed result.stdout.fnmatch_lines( [ # Validate location/stacklevel of warning from cacheprovider. @@ -113,7 +113,7 @@ def test_cache_failure_warns( " */cacheprovider.py:*: PytestCacheWarning: could not create cache path " f"{unwritable_cache_dir}/v/cache/nodeids: *", ' config.cache.set("cache/nodeids", sorted(self.cached_nodeids))', - "*1 failed, 3 warnings in*", + "*1 failed, 2 warnings in*", ] ) diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index affdb73375e..02239e8b5f1 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -358,3 +358,41 @@ def test_one(): with stepwise_cache_file.open(encoding="utf-8") as file_handle: observed_value = file_handle.readlines() assert [expected_value] == observed_value + + +def test_do_not_clear_cache_if_disabled(pytester: Pytester) -> None: + """ + If pytest is run without --step-wise, do not clear the stepwise cache. + + Keeping the cache around is important for this workflow: + + 1. Run tests with --stepwise + 2. Stop at the failing test, and iterate over it changing the code and running it in isolation + (in the IDE for example). + 3. Run tests with --stepwise again - at this point we expect to start from the failing test, which should now pass, + and continue with the next tests. + """ + pytester.makepyfile( + """ + def test_1(): + pass + def test_2(): + assert False + def test_3(): + pass + """ + ) + result = pytester.runpytest("--stepwise") + result.stdout.fnmatch_lines( + ["*::test_2 - assert False*", "*failed, continuing from this test next run*"] + ) + + # Run a specific test without passing `--stepwise`. + result = pytester.runpytest("-k", "test_1") + result.stdout.fnmatch_lines(["*1 passed*"]) + + # Running with `--stepwise` should continue from the last failing test. + result = pytester.runpytest("--stepwise") + result.stdout.fnmatch_lines( + ["*::test_2 - assert False*", "*failed, continuing from this test next run*"] + ) From 40b38f97c977e51f165aa32ccfe95ec3a0e5f8c8 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 10 Jan 2025 13:48:29 -0300 Subject: [PATCH 2/7] Add --stepwise-reset to restart the stepwise workflow --- changelog/13122.improvement.rst | 4 +++- src/_pytest/stepwise.py | 16 +++++++++++++++- testing/test_stepwise.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/changelog/13122.improvement.rst b/changelog/13122.improvement.rst index d7372380e81..808e19dcc00 100644 --- a/changelog/13122.improvement.rst +++ b/changelog/13122.improvement.rst @@ -1,4 +1,4 @@ -Improve the ``--stepwise`` flag to not forget the last failed test in case pytest is executed later without the flag. +Improve the ``--stepwise``/``--sw`` flag to not forget the last failed test in case pytest is executed later without the flag. This enables the following workflow: @@ -9,3 +9,5 @@ This enables the following workflow: failed test, and if it passes, continue with the next tests. Previously, at step 3, pytest would start from the beginning, forgetting the failed tests. + +Also added the new ``--stepwise-reset``/``--sw-reset``, allowing the user to explicitly reset the stepwise state and restart the workflow from the beginning. diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index 82d1fe3aee5..28db3bfc6e2 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -30,11 +30,23 @@ def pytest_addoption(parser: Parser) -> None: help="Ignore the first failing test but stop on the next failing test. " "Implicitly enables --stepwise.", ) + group.addoption( + "--sw-reset", + "--stepwise-reset", + action="store_true", + default=False, + dest="stepwise_reset", + help="Resets stepwise state, restarting the stepwise workflow. " + "Implicitly enables --stepwise.", + ) def pytest_configure(config: Config) -> None: + # --stepwise-skip implies stepwise. if config.option.stepwise_skip: - # allow --stepwise-skip to work on its own merits. + config.option.stepwise = True + # --stepwise-clear implies stepwise. + if config.option.stepwise_reset: config.option.stepwise = True if config.getoption("stepwise"): config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin") @@ -58,6 +70,8 @@ def __init__(self, config: Config) -> None: self.cache: Cache = config.cache self.lastfailed: str | None = self.cache.get(STEPWISE_CACHE_DIR, None) self.skip: bool = config.getoption("stepwise_skip") + if config.getoption("stepwise_reset"): + self.lastfailed = None def pytest_sessionstart(self, session: Session) -> None: self.session = session diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index 02239e8b5f1..4d3d1e91b7f 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -396,3 +396,34 @@ def test_3(): result.stdout.fnmatch_lines( ["*::test_2 - assert False*", "*failed, continuing from this test next run*"] ) + + +def test_stepwise_reset(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_1(): + pass + def test_2(): + assert False + def test_3(): + pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + ] + ) + + # Running with --stepwise-reset restarts the stepwise workflow. + result = pytester.runpytest("-v", "--stepwise-reset") + result.stdout.fnmatch_lines( + [ + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + ] + ) From 18378585628247d92e668c05b2a1254a544d7d7c Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 10 Jan 2025 19:37:06 -0300 Subject: [PATCH 3/7] Tweak changelog --- changelog/13122.improvement.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/changelog/13122.improvement.rst b/changelog/13122.improvement.rst index 808e19dcc00..4079696d676 100644 --- a/changelog/13122.improvement.rst +++ b/changelog/13122.improvement.rst @@ -3,11 +3,9 @@ Improve the ``--stepwise``/``--sw`` flag to not forget the last failed test in c This enables the following workflow: 1. Execute pytest with ``--stepwise``, pytest then stops at the first failing test; -2. User iteratively updates the code and runs the test in isolation, without the ``--stepwise`` flag - (for example in an IDE), until it is fixed. -3. At this point, the user can execute pytest with ``--stepwise`` again and pytest will continue from the previously - failed test, and if it passes, continue with the next tests. +2. Iteratively update the code and run the test in isolation, without the ``--stepwise`` flag (for example in an IDE), until it is fixed. +3. Execute pytest with ``--stepwise`` again and pytest will continue from the previously failed test, and if it passes, continue on to the next tests. -Previously, at step 3, pytest would start from the beginning, forgetting the failed tests. +Previously, at step 3, pytest would start from the beginning, forgetting the previously failed test. Also added the new ``--stepwise-reset``/``--sw-reset``, allowing the user to explicitly reset the stepwise state and restart the workflow from the beginning. From b6f1dadeb201677bd08601283649c9dcc5491895 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 10 Jan 2025 19:44:26 -0300 Subject: [PATCH 4/7] Tighten test --- testing/test_stepwise.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index 4d3d1e91b7f..dbf49f73535 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -384,7 +384,11 @@ def test_3(): ) result = pytester.runpytest("--stepwise") result.stdout.fnmatch_lines( - ["*::test_2 - assert False*", "*failed, continuing from this test next run*"] + [ + "*::test_2 - assert False*", + "*failed, continuing from this test next run*", + "=* 1 failed, 1 passed in *", + ] ) # Run a specific test without passing `--stepwise`. @@ -394,7 +398,12 @@ def test_3(): # Running with `--stepwise` should continue from the last failing test. result = pytester.runpytest("--stepwise") result.stdout.fnmatch_lines( - ["*::test_2 - assert False*", "*failed, continuing from this test next run*"] + [ + "stepwise: skipping 1 already passed items.", + "*::test_2 - assert False*", + "*failed, continuing from this test next run*", + "=* 1 failed, 1 deselected in *", + ] ) From 0c30a580606f6984897bd84f6b940e1a025f29bd Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sun, 12 Jan 2025 10:12:16 -0300 Subject: [PATCH 5/7] Show cache information and reset cache if test count changes --- changelog/13122.improvement.rst | 18 ++-- src/_pytest/stepwise.py | 113 ++++++++++++++++++++----- testing/test_stepwise.py | 145 +++++++++++++++++++++++++++----- 3 files changed, 231 insertions(+), 45 deletions(-) diff --git a/changelog/13122.improvement.rst b/changelog/13122.improvement.rst index 4079696d676..c302713b320 100644 --- a/changelog/13122.improvement.rst +++ b/changelog/13122.improvement.rst @@ -1,11 +1,15 @@ -Improve the ``--stepwise``/``--sw`` flag to not forget the last failed test in case pytest is executed later without the flag. +The ``--stepwise`` mode received a number of improvements: -This enables the following workflow: +* It no longer forgets the last failed test in case pytest is executed later without the flag. -1. Execute pytest with ``--stepwise``, pytest then stops at the first failing test; -2. Iteratively update the code and run the test in isolation, without the ``--stepwise`` flag (for example in an IDE), until it is fixed. -3. Execute pytest with ``--stepwise`` again and pytest will continue from the previously failed test, and if it passes, continue on to the next tests. + This enables the following workflow: -Previously, at step 3, pytest would start from the beginning, forgetting the previously failed test. + 1. Execute pytest with ``--stepwise``, pytest then stops at the first failing test; + 2. Iteratively update the code and run the test in isolation, without the ``--stepwise`` flag (for example in an IDE), until it is fixed. + 3. Execute pytest with ``--stepwise`` again and pytest will continue from the previously failed test, and if it passes, continue on to the next tests. -Also added the new ``--stepwise-reset``/``--sw-reset``, allowing the user to explicitly reset the stepwise state and restart the workflow from the beginning. + Previously, at step 3, pytest would start from the beginning, forgetting the previously failed test. + + This change however might cause issues if the ``--stepwise`` mode is used far apart in time, as the state might get stale, so the internal state will be reset automatically in case the test suite changes (for now only the number of tests are considered for this, we might change/improve this on the future). + +* New ``--stepwise-reset``/``--sw-reset`` flag, allowing the user to explicitly reset the stepwise state and restart the workflow from the beginning. diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index 28db3bfc6e2..7ee869bcef8 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -1,5 +1,10 @@ from __future__ import annotations +import dataclasses +from datetime import datetime +from typing import Any +from typing import TYPE_CHECKING + from _pytest import nodes from _pytest.cacheprovider import Cache from _pytest.config import Config @@ -8,7 +13,12 @@ from _pytest.reports import TestReport -STEPWISE_CACHE_DIR = "cache/stepwise" +if TYPE_CHECKING: + from typing import ClassVar + + from typing_extensions import Self + +STEPWISE_CACHE_DIR = "cache/stepwise2" def pytest_addoption(parser: Parser) -> None: @@ -61,17 +71,63 @@ def pytest_sessionfinish(session: Session) -> None: return +@dataclasses.dataclass +class StepwiseCacheInfo: + # The nodeid of the last failed test. + last_failed: str | None + + # The number of tests in the last time --stepwise was run. + # We use this information as a simple way to invalidate the cache information, avoiding + # confusing behavior in case the cache is stale. + last_test_count: int | None + + # The date when the cache was last updated, for information purposes only. + last_cache_date_str: str + + _DATE_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S" + + @property + def last_cache_date(self) -> datetime: + return datetime.strptime(self.last_cache_date_str, self._DATE_FORMAT) + + @classmethod + def empty(cls) -> Self: + return cls( + last_failed=None, + last_test_count=None, + last_cache_date_str=datetime.now().strftime(cls._DATE_FORMAT), + ) + + def update_date_to_now(self) -> None: + self.last_cache_date_str = datetime.now().strftime(self._DATE_FORMAT) + + class StepwisePlugin: def __init__(self, config: Config) -> None: self.config = config self.session: Session | None = None - self.report_status = "" + self.report_status: list[str] = [] assert config.cache is not None self.cache: Cache = config.cache - self.lastfailed: str | None = self.cache.get(STEPWISE_CACHE_DIR, None) self.skip: bool = config.getoption("stepwise_skip") - if config.getoption("stepwise_reset"): - self.lastfailed = None + self.reset: bool = config.getoption("stepwise_reset") + self.cached_info = self._load_cached_info() + + def _load_cached_info(self) -> StepwiseCacheInfo: + cached_dict: dict[str, Any] | None = self.cache.get(STEPWISE_CACHE_DIR, None) + if cached_dict: + try: + return StepwiseCacheInfo( + cached_dict["last_failed"], + cached_dict["last_test_count"], + cached_dict["last_cache_date_str"], + ) + except Exception as e: + error = f"{type(e).__name__}: {e}" + self.report_status.append(f"error reading cache, discarding ({error})") + + # Cache not found or error during load, return a new cache. + return StepwiseCacheInfo.empty() def pytest_sessionstart(self, session: Session) -> None: self.session = session @@ -79,23 +135,41 @@ def pytest_sessionstart(self, session: Session) -> None: def pytest_collection_modifyitems( self, config: Config, items: list[nodes.Item] ) -> None: - if not self.lastfailed: - self.report_status = "no previously failed tests, not skipping." + last_test_count = self.cached_info.last_test_count + self.cached_info.last_test_count = len(items) + + if self.reset: + self.report_status.append("resetting state, not skipping.") + self.cached_info.last_failed = None + return + + if not self.cached_info.last_failed: + self.report_status.append("no previously failed tests, not skipping.") + return + + if last_test_count is not None and last_test_count != len(items): + self.report_status.append( + f"test count changed, not skipping (now {len(items)} tests, previously {last_test_count})." + ) + self.cached_info.last_failed = None return - # check all item nodes until we find a match on last failed + # Check all item nodes until we find a match on last failed. failed_index = None for index, item in enumerate(items): - if item.nodeid == self.lastfailed: + if item.nodeid == self.cached_info.last_failed: failed_index = index break # If the previously failed test was not found among the test items, # do not skip any tests. if failed_index is None: - self.report_status = "previously failed test not found, not skipping." + self.report_status.append("previously failed test not found, not skipping.") else: - self.report_status = f"skipping {failed_index} already passed items." + self.report_status.append( + f"skipping {failed_index} already passed items (cache from {self.cached_info.last_cache_date}," + f" use --sw-reset to discard)." + ) deselected = items[:failed_index] del items[:failed_index] config.hook.pytest_deselected(items=deselected) @@ -105,13 +179,13 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: if self.skip: # Remove test from the failed ones (if it exists) and unset the skip option # to make sure the following tests will not be skipped. - if report.nodeid == self.lastfailed: - self.lastfailed = None + if report.nodeid == self.cached_info.last_failed: + self.cached_info.last_failed = None self.skip = False else: # Mark test as the last failing and interrupt the test session. - self.lastfailed = report.nodeid + self.cached_info.last_failed = report.nodeid assert self.session is not None self.session.shouldstop = ( "Test failed, continuing from this test next run." @@ -121,12 +195,12 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: # If the test was actually run and did pass. if report.when == "call": # Remove test from the failed ones, if exists. - if report.nodeid == self.lastfailed: - self.lastfailed = None + if report.nodeid == self.cached_info.last_failed: + self.cached_info.last_failed = None - def pytest_report_collectionfinish(self) -> str | None: + def pytest_report_collectionfinish(self) -> list[str] | None: if self.config.get_verbosity() >= 0 and self.report_status: - return f"stepwise: {self.report_status}" + return [f"stepwise: {x}" for x in self.report_status] return None def pytest_sessionfinish(self) -> None: @@ -134,4 +208,5 @@ def pytest_sessionfinish(self) -> None: # Do not update cache if this process is a xdist worker to prevent # race conditions (#10641). return - self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed) + self.cached_info.update_date_to_now() + self.cache.set(STEPWISE_CACHE_DIR, dataclasses.asdict(self.cached_info)) diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index dbf49f73535..8e040cca28e 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -1,6 +1,8 @@ -# mypy: allow-untyped-defs +# mypy: disallow-untyped-defs from __future__ import annotations +from collections.abc import Sequence +import json from pathlib import Path from _pytest.cacheprovider import Cache @@ -84,7 +86,7 @@ def broken_pytester(pytester: Pytester) -> Pytester: return pytester -def _strip_resource_warnings(lines): +def _strip_resource_warnings(lines: Sequence[str]) -> Sequence[str]: # Strip unreliable ResourceWarnings, so no-output assertions on stderr can work. # (https://github.com/pytest-dev/pytest/issues/5088) return [ @@ -114,7 +116,10 @@ def test_data(expected): result.stdout.fnmatch_lines(["stepwise: no previously failed tests, not skipping."]) result = pytester.runpytest("-v", "--stepwise") result.stdout.fnmatch_lines( - ["stepwise: skipping 4 already passed items.", "*1 failed, 4 deselected*"] + [ + "stepwise: skipping 4 already passed items (cache from *, use --sw-reset to discard).", + "*1 failed, 4 deselected*", + ] ) @@ -360,9 +365,9 @@ def test_one(): assert [expected_value] == observed_value -def test_do_not_clear_cache_if_disabled(pytester: Pytester) -> None: +def test_do_not_reset_cache_if_disabled(pytester: Pytester) -> None: """ - If pytest is run without --step-wise, do not clear the stepwise cache. + If pytest is run without --stepwise, do not clear the stepwise cache. Keeping the cache around is important for this workflow: @@ -374,12 +379,9 @@ def test_do_not_clear_cache_if_disabled(pytester: Pytester) -> None: """ pytester.makepyfile( """ - def test_1(): - pass - def test_2(): - assert False - def test_3(): - pass + def test_1(): pass + def test_2(): assert False + def test_3(): pass """ ) result = pytester.runpytest("--stepwise") @@ -399,7 +401,7 @@ def test_3(): result = pytester.runpytest("--stepwise") result.stdout.fnmatch_lines( [ - "stepwise: skipping 1 already passed items.", + "stepwise: skipping 1 already passed items (cache from *, use --sw-reset to discard).", "*::test_2 - assert False*", "*failed, continuing from this test next run*", "=* 1 failed, 1 deselected in *", @@ -407,23 +409,32 @@ def test_3(): ) -def test_stepwise_reset(pytester: Pytester) -> None: +def test_reset(pytester: Pytester) -> None: pytester.makepyfile( """ - def test_1(): - pass - def test_2(): - assert False - def test_3(): - pass + def test_1(): pass + def test_2(): assert False + def test_3(): pass """ ) result = pytester.runpytest("--stepwise", "-v") result.stdout.fnmatch_lines( [ + "stepwise: no previously failed tests, not skipping.", "*::test_1 *PASSED*", "*::test_2 *FAILED*", "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: skipping 1 already passed items (cache from *, use --sw-reset to discard).", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 deselected in *", ] ) @@ -431,8 +442,104 @@ def test_3(): result = pytester.runpytest("-v", "--stepwise-reset") result.stdout.fnmatch_lines( [ + "stepwise: resetting state, not skipping.", "*::test_1 *PASSED*", "*::test_2 *FAILED*", "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + +def test_change_test_count(pytester: Pytester) -> None: + # Run initially with 3 tests. + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): assert False + def test_3(): pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: no previously failed tests, not skipping.", + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + # Change the number of tests, which invalidates the test cache. + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): assert False + def test_3(): pass + def test_4(): pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: test count changed, not skipping (now 4 tests, previously 3).", + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + # Fix the failing test and run again. + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): pass + def test_3(): pass + def test_4(): pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: skipping 1 already passed items (cache from *, use --sw-reset to discard).", + "*::test_2 *PASSED*", + "*::test_3 *PASSED*", + "*::test_4 *PASSED*", + "* 3 passed, 1 deselected in *", + ] + ) + + +def test_cache_error(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_1(): pass + """ + ) + # Run stepwise normally to generate the cache information. + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: no previously failed tests, not skipping.", + "*::test_1 *PASSED*", + "* 1 passed in *", + ] + ) + + # Corrupt the cache. + cache_file = pytester.path / f".pytest_cache/v/{STEPWISE_CACHE_DIR}" + assert cache_file.is_file() + cache_file.write_text(json.dumps({"invalid": True}), encoding="UTF-8") + + # Check we run as if the cache did not exist, but also show an error message. + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: error reading cache, discarding (KeyError: *", + "stepwise: no previously failed tests, not skipping.", + "*::test_1 *PASSED*", + "* 1 passed in *", ] ) From 1f7a1517dbcf381980e2ca1839209e0ca21a595d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 13 Jan 2025 12:59:43 -0300 Subject: [PATCH 6/7] Code review requests --- src/_pytest/stepwise.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index 7ee869bcef8..c25f260a1ca 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -14,11 +14,9 @@ if TYPE_CHECKING: - from typing import ClassVar - from typing_extensions import Self -STEPWISE_CACHE_DIR = "cache/stepwise2" +STEPWISE_CACHE_DIR = "cache/stepwise" def pytest_addoption(parser: Parser) -> None: @@ -52,11 +50,8 @@ def pytest_addoption(parser: Parser) -> None: def pytest_configure(config: Config) -> None: - # --stepwise-skip implies stepwise. - if config.option.stepwise_skip: - config.option.stepwise = True - # --stepwise-clear implies stepwise. - if config.option.stepwise_reset: + # --stepwise-skip/--stepwise-reset implies stepwise. + if config.option.stepwise_skip or config.option.stepwise_reset: config.option.stepwise = True if config.getoption("stepwise"): config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin") @@ -84,22 +79,20 @@ class StepwiseCacheInfo: # The date when the cache was last updated, for information purposes only. last_cache_date_str: str - _DATE_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S" - @property def last_cache_date(self) -> datetime: - return datetime.strptime(self.last_cache_date_str, self._DATE_FORMAT) + return datetime.fromisoformat(self.last_cache_date_str) @classmethod def empty(cls) -> Self: return cls( last_failed=None, last_test_count=None, - last_cache_date_str=datetime.now().strftime(cls._DATE_FORMAT), + last_cache_date_str=datetime.now().isoformat(), ) def update_date_to_now(self) -> None: - self.last_cache_date_str = datetime.now().strftime(self._DATE_FORMAT) + self.last_cache_date_str = datetime.now().isoformat() class StepwisePlugin: @@ -122,7 +115,7 @@ def _load_cached_info(self) -> StepwiseCacheInfo: cached_dict["last_test_count"], cached_dict["last_cache_date_str"], ) - except Exception as e: + except (KeyError, TypeError) as e: error = f"{type(e).__name__}: {e}" self.report_status.append(f"error reading cache, discarding ({error})") From 2fcaa79358ae781ae68cc3a335f93aff7c80d521 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 15 Jan 2025 14:48:06 -0300 Subject: [PATCH 7/7] Show cache age instead of update date --- src/_pytest/stepwise.py | 6 +++++- testing/test_stepwise.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index c25f260a1ca..8901540eb59 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -2,6 +2,7 @@ import dataclasses from datetime import datetime +from datetime import timedelta from typing import Any from typing import TYPE_CHECKING @@ -159,8 +160,11 @@ def pytest_collection_modifyitems( if failed_index is None: self.report_status.append("previously failed test not found, not skipping.") else: + cache_age = datetime.now() - self.cached_info.last_cache_date + # Round up to avoid showing microseconds. + cache_age = timedelta(seconds=int(cache_age.total_seconds())) self.report_status.append( - f"skipping {failed_index} already passed items (cache from {self.cached_info.last_cache_date}," + f"skipping {failed_index} already passed items (cache from {cache_age} ago," f" use --sw-reset to discard)." ) deselected = items[:failed_index] diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index 8e040cca28e..d2ad3bae500 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -117,7 +117,7 @@ def test_data(expected): result = pytester.runpytest("-v", "--stepwise") result.stdout.fnmatch_lines( [ - "stepwise: skipping 4 already passed items (cache from *, use --sw-reset to discard).", + "stepwise: skipping 4 already passed items (cache from * ago, use --sw-reset to discard).", "*1 failed, 4 deselected*", ] )