Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions tests/tools/test_tirith_security.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,20 @@
@pytest.fixture(autouse=True)
def _reset_resolved_path():
"""Pre-set cached path to skip auto-install in scan tests.

Tests that specifically test ensure_installed / resolve behavior
reset this to None themselves.
"""
_tirith_mod._resolved_path = "tirith"
_tirith_mod._install_thread = None
_tirith_mod._install_failure_reason = ""
_tirith_mod._crash_count = 0
_tirith_mod._circuit_open = False
yield
_tirith_mod._resolved_path = None
_tirith_mod._install_thread = None
_tirith_mod._install_failure_reason = ""
_tirith_mod._crash_count = 0
_tirith_mod._circuit_open = False


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1216,12 +1219,17 @@ def test_repeated_spawn_failure_logs_once(self, mock_cfg, mock_run, caplog):
_tirith_mod._reset_spawn_warning_state()

with caplog.at_level("WARNING", logger="tools.tirith_security"):
for _ in range(15):
for i in range(15):
result = check_command_security("echo hi")
# Behavior must remain the same on every call —
# fail-open allow, with the exception captured in summary.
assert result["action"] == "allow"
assert "unavailable" in result["summary"]
if i < _tirith_mod._CRASH_LIMIT:
# Before circuit breaker opens, summary has the exception
assert "unavailable" in result["summary"]
else:
# After circuit breaker opens, summary is generic
assert "circuit breaker" in result["summary"]

spawn_warnings = [
rec for rec in caplog.records
Expand All @@ -1237,7 +1245,11 @@ def test_repeated_spawn_failure_logs_once(self, mock_cfg, mock_run, caplog):
def test_distinct_exception_types_each_log_once(self, mock_cfg, mock_run, caplog):
"""``FileNotFoundError`` and ``PermissionError`` are distinct
failure modes and each deserves its own first-occurrence log
line; the dedupe key includes the exception class."""
line; the dedupe key includes the exception class.

After _CRASH_LIMIT consecutive failures the circuit breaker opens
and subsequent calls short-circuit without spawning, so we only
see the warnings from the first batch."""
mock_cfg.return_value = {
"tirith_enabled": True, "tirith_path": "tirith",
"tirith_timeout": 5, "tirith_fail_open": True,
Expand All @@ -1248,6 +1260,9 @@ def test_distinct_exception_types_each_log_once(self, mock_cfg, mock_run, caplog
mock_run.side_effect = FileNotFoundError("[WinError 2]")
for _ in range(3):
check_command_security("a")
# Circuit breaker is now open — switching to PermissionError
# won't generate a new warning because the function returns
# before reaching subprocess.run.
mock_run.side_effect = PermissionError("denied")
for _ in range(3):
check_command_security("b")
Expand All @@ -1256,8 +1271,8 @@ def test_distinct_exception_types_each_log_once(self, mock_cfg, mock_run, caplog
rec for rec in caplog.records
if "tirith spawn failed" in rec.message
]
assert len(spawn_warnings) == 2, (
f"expected 2 distinct first-occurrence warnings, "
assert len(spawn_warnings) == 1, (
f"expected 1 warning before circuit breaker opens, "
f"got {len(spawn_warnings)}"
)

Expand Down
46 changes: 45 additions & 1 deletion tools/tirith_security.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ def _load_security_config() -> dict:
_INSTALL_FAILED = False # sentinel: distinct from "not yet tried"
_install_failure_reason: str = "" # reason tag when _resolved_path is _INSTALL_FAILED

# Circuit breaker: after _CRASH_LIMIT consecutive spawn/execution failures,
# disable tirith for the rest of the process to prevent agent hangs.
# Reset on successful execution (see check_command_security).
_CRASH_LIMIT = 3
_crash_count: int = 0
_circuit_open: bool = False

# Background install thread coordination
_install_lock = threading.Lock()
_install_thread: threading.Thread | None = None
Expand Down Expand Up @@ -703,11 +710,21 @@ def check_command_security(command: str) -> dict:
Returns:
{"action": "allow"|"warn"|"block", "findings": [...], "summary": str}
"""
global _crash_count, _circuit_open

cfg = _load_security_config()

if not cfg["tirith_enabled"]:
return {"action": "allow", "findings": [], "summary": ""}

# Circuit breaker: if tirith has crashed _CRASH_LIMIT times in a row,
# stop trying for the rest of the process. Without this, a corrupted
# or missing binary causes every tool call to hit the same spawn failure
# → fail-open → agent retry loop, hanging the user for 20+ minutes
# (issue #41400).
if _circuit_open:
return {"action": "allow", "findings": [], "summary": "tirith disabled (circuit breaker)"}

# Unsupported platform (Windows etc.) — tirith has no binary here and
# never will. Skip the resolver entirely so we don't even try to spawn.
# Pattern-matching guards still run via the rest of approval.py.
Expand Down Expand Up @@ -744,6 +761,14 @@ def check_command_security(command: str) -> dict:
# install marked failed for the day).
spawn_key = f"tirith_spawn_failed:{type(exc).__name__}:{getattr(exc, 'errno', '')}"
_warn_once(spawn_key, "tirith spawn failed: %s", exc)
_crash_count += 1
if _crash_count >= _CRASH_LIMIT:
_circuit_open = True
logger.warning(
"tirith circuit breaker opened after %d consecutive failures; "
"disabling for the rest of the process",
_crash_count,
)
if fail_open:
return {"action": "allow", "findings": [], "summary": f"tirith unavailable: {exc}"}
return {"action": "block", "findings": [], "summary": f"tirith spawn failed (fail-closed): {exc}"}
Expand All @@ -753,6 +778,14 @@ def check_command_security(command: str) -> dict:
"tirith timed out after %ds",
timeout,
)
_crash_count += 1
if _crash_count >= _CRASH_LIMIT:
_circuit_open = True
logger.warning(
"tirith circuit breaker opened after %d consecutive failures; "
"disabling for the rest of the process",
_crash_count,
)
if fail_open:
return {"action": "allow", "findings": [], "summary": f"tirith timed out ({timeout}s)"}
return {"action": "block", "findings": [], "summary": "tirith timed out (fail-closed)"}
Expand All @@ -761,13 +794,24 @@ def check_command_security(command: str) -> dict:
exit_code = result.returncode
if exit_code == 0:
action = "allow"
# Successful execution — reset circuit breaker
_crash_count = 0
elif exit_code == 1:
action = "block"
elif exit_code == 2:
action = "warn"
else:
# Unknown exit code — respect fail_open
# Unknown exit code (includes signal-killed processes like -11/SIGSEGV)
# — respect fail_open
logger.warning("tirith returned unexpected exit code %d", exit_code)
_crash_count += 1
if _crash_count >= _CRASH_LIMIT:
_circuit_open = True
logger.warning(
"tirith circuit breaker opened after %d consecutive failures; "
"disabling for the rest of the process",
_crash_count,
)
if fail_open:
return {"action": "allow", "findings": [], "summary": f"tirith exit code {exit_code} (fail-open)"}
return {"action": "block", "findings": [], "summary": f"tirith exit code {exit_code} (fail-closed)"}
Expand Down
Loading