diff --git a/CHANGES.rst b/CHANGES.rst index 68783aa..8c13f67 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,10 +4,16 @@ Changelog 14.1 (unreleased) ----------------- +Features +++++++++ + - Fix compatibility with pytest 8.2. (`#267 `_) +- Add ``--fail-on-flaky`` option to fail the test run with custom exit code + when test passed on rerun. + 14.0 (2024-03-13) ----------------- diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index d11e1bc..6197444 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -85,6 +85,13 @@ def pytest_addoption(parser): "regex provided. Pass this flag multiple times to accumulate a list " "of regexes to match", ) + group._addoption( + "--fail-on-flaky", + action="store_true", + dest="fail_on_flaky", + help="Fail the test run with exit code 7 if a flaky test passes on a rerun.", + ) + arg_type = "string" parser.addini("reruns", RERUNS_DESC, type=arg_type) parser.addini("reruns_delay", RERUNS_DELAY_DESC, type=arg_type) @@ -614,3 +621,13 @@ def show_rerun(terminalreporter, lines): for rep in rerun: pos = rep.nodeid lines.append(f"RERUN {pos}") + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session, exitstatus): + if exitstatus != 0: + return + + if session.config.option.fail_on_flaky: + if session.config.getvalue("reruns") > 0: + session.exitstatus = 7 diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 4e721ad..ccd7141 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -177,6 +177,28 @@ def test_pass(): assert_outcomes(result, passed=1, rerun=1) +def test_run_with_fail_on_flaky_fails_with_custom_error_code_after_pass_on_rerun( + testdir, +): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()}""" + ) + result = testdir.runpytest("--reruns", "1", "--fail-on-flaky") + assert_outcomes(result, passed=1, rerun=1) + assert result.ret == 7 + + +def test_run_fails_with_code_1_after_consistent_test_failure_even_with_fail_on_flaky( + testdir, +): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--reruns", "1", "--fail-on-flaky") + assert_outcomes(result, passed=0, failed=1, rerun=1) + assert result.ret == 1 + + @pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") def test_rerun_passes_after_temporary_test_crash(testdir): # note: we need two tests because there is a bug where xdist