Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add --session-timeout #165

Merged
merged 15 commits into from
Mar 7, 2024
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ docs/_build/

# Virtual Envs
.env*
venv

# IDE
.idea
Expand Down
41 changes: 39 additions & 2 deletions pytest_timeout.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@
import signal
import sys
import threading
import time
import traceback
from collections import namedtuple

import pytest


__all__ = ("is_debugging", "Settings")
SESSION_TIMEOUT_KEY = pytest.StashKey[float]()


HAVE_SIGALRM = hasattr(signal, "SIGALRM")
Expand All @@ -43,6 +45,11 @@
When specified, disables debugger detection. breakpoint(), pdb.set_trace(), etc.
will be interrupted by the timeout.
""".strip()
SESSION_TIMEOUT_DESC = """
Timeout in seconds for entire session. Default is None which
means no timeout. Timeout is checked between tests, and will not interrupt a test
in progress.
""".strip()

# bdb covers pdb, ipdb, and possibly others
# pydevd covers PyCharm, VSCode, and possibly others
Expand Down Expand Up @@ -79,6 +86,15 @@ def pytest_addoption(parser):
action="store_true",
help=DISABLE_DEBUGGER_DETECTION_DESC,
)
group.addoption(
"--session-timeout",
action="store",
dest="session_timeout",
default=None,
type=float,
metavar="SECONDS",
help=SESSION_TIMEOUT_DESC,
)
parser.addini("timeout", TIMEOUT_DESC)
parser.addini("timeout_method", METHOD_DESC)
parser.addini("timeout_func_only", FUNC_ONLY_DESC, type="bool", default=False)
Expand Down Expand Up @@ -143,6 +159,13 @@ def pytest_configure(config):
config._env_timeout_func_only = settings.func_only
config._env_timeout_disable_debugger_detection = settings.disable_debugger_detection

timeout = config.getoption("--session-timeout")
if timeout is not None:
expire_time = time.time() + timeout
else:
expire_time = 0
config.stash[SESSION_TIMEOUT_KEY] = expire_time


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_protocol(item):
Expand All @@ -161,6 +184,12 @@ def pytest_runtest_protocol(item):
if is_timeout and settings.func_only is False:
hooks.pytest_timeout_cancel_timer(item=item)

# check session timeout
expire_time = item.session.config.stash[SESSION_TIMEOUT_KEY]
if expire_time and (expire_time < time.time()):
timeout = item.session.config.getoption("--session-timeout")
item.session.shouldfail = f"session-timeout: {timeout} sec exceeded"


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_call(item):
Expand All @@ -182,15 +211,23 @@ def pytest_runtest_call(item):
@pytest.hookimpl(tryfirst=True)
def pytest_report_header(config):
"""Add timeout config to pytest header."""
timeout_header = []

if config._env_timeout:
return [
timeout_header.append(
"timeout: %ss\ntimeout method: %s\ntimeout func_only: %s"
% (
config._env_timeout,
config._env_timeout_method,
config._env_timeout_func_only,
)
]
)

session_timeout = config.getoption("--session-timeout")
if session_timeout:
timeout_header.append("session timeout: %ss" % session_timeout)
if timeout_header:
return timeout_header


@pytest.hookimpl(tryfirst=True)
Expand Down
38 changes: 36 additions & 2 deletions test_pytest_timeout.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,14 @@ def test_header(pytester):
def test_x(): pass
"""
)
result = pytester.runpytest_subprocess("--timeout=1")
result = pytester.runpytest_subprocess("--timeout=1", "--session-timeout=2")
result.stdout.fnmatch_lines(
["timeout: 1.0s", "timeout method:*", "timeout func_only:*"]
[
"timeout: 1.0s",
"timeout method:*",
"timeout func_only:*",
"session timeout: 2.0s",
]
)


Expand Down Expand Up @@ -603,3 +608,32 @@ def test_foo():
"pytest_timeout_cancel_timer",
]
)


def test_session_timeout(pytester):
# 2 tests, each with 0.5 sec timeouts
# each using a fixture with 0.5 sec setup and 0.5 sec teardown
# So about 1.5 seconds per test, ~3 sec total,
# Therefore:
# A timeout of 1.25 sec should happen during the teardown of the first test
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

0.25 from an edge is too brittle. experience tells me you can't really work with sub-seconds times in this test suite, it gets flaky otherwise.

Make the total test duration 2s and the session timeout 1s?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All the numbers in this description are now wrong. maybe just describe that this is designed to timeout during the first test and ensure the first test still runs to completion but the 2nd test is not started?

# and the second test should NOT be run
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i don't think this is asserted? not immediately sure how you would assert it, maybe with more verbose output and more output lines to match. or maybe there's something easier.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i don't think this is addressed yet?

pytester.makepyfile(
"""
import time, pytest

@pytest.fixture()
def slow_setup_and_teardown():
time.sleep(0.5)
yield
time.sleep(0.5)

def test_one(slow_setup_and_teardown):
time.sleep(0.5)

def test_two(slow_setup_and_teardown):
time.sleep(0.5)
"""
)
result = pytester.runpytest_subprocess("--session-timeout", "1.25")
result.stdout.fnmatch_lines(["*!! session-timeout: 1.25 sec exceeded !!!*"])
result.assert_outcomes(passed=1)
Loading