diff --git a/strix/config/settings.py b/strix/config/settings.py index 42a2c97ea..9c5c223dd 100644 --- a/strix/config/settings.py +++ b/strix/config/settings.py @@ -112,6 +112,12 @@ class RuntimeSettings(BaseSettings): backend: str = Field(default="docker", alias="STRIX_RUNTIME_BACKEND") # Max screenshot/image tool outputs kept live per agent context (0 = none). max_context_images: int = Field(default=3, ge=0, alias="STRIX_MAX_CONTEXT_IMAGES") + # Wall-clock budget for the Caido readiness probe (session_manager -> + # bootstrap_caido -> _login_as_guest). The sandbox entrypoint chowns a + # large toolchain tree before starting caido-cli, which can take minutes + # on a loaded host or CI runner; a fixed attempt count gives up too early + # there. Raise this on slow/shared hosts instead of patching the retry loop. + caido_boot_wait_s: float = Field(default=180.0, gt=0, alias="STRIX_CAIDO_BOOT_WAIT_S") class TelemetrySettings(BaseSettings): diff --git a/strix/runtime/caido_bootstrap.py b/strix/runtime/caido_bootstrap.py index 0b9ad5b17..f77066623 100644 --- a/strix/runtime/caido_bootstrap.py +++ b/strix/runtime/caido_bootstrap.py @@ -13,6 +13,7 @@ import contextlib import json import logging +import time from typing import TYPE_CHECKING from caido_sdk_client import Client, TokenAuthOptions @@ -35,16 +36,26 @@ async def _login_as_guest( session: BaseSandboxSession, *, container_url: str, - attempts: int = 10, + max_wait_s: float = 180.0, ) -> str: """``session.exec`` curl to fetch a guest token; retry until ready. Caido's GraphQL listener may not be up the instant the container - starts. The retry loop also doubles as the Caido readiness probe — - no separate TCP healthcheck needed. + starts — the sandbox entrypoint chowns a large toolchain tree first, + which can take minutes on a loaded host or CI runner. The retry loop + also doubles as the Caido readiness probe — no separate TCP + healthcheck needed — so it is bounded by wall-clock time rather than + a fixed attempt count, to accommodate slow boots without making fast + hosts wait needlessly. """ + deadline = time.monotonic() + max_wait_s last_err: str | None = None - for i in range(1, attempts + 1): + attempt = 0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + attempt += 1 result = await session.exec( "curl", "-fsS", @@ -55,7 +66,7 @@ async def _login_as_guest( "-d", _LOGIN_AS_GUEST_BODY, f"{container_url}/graphql", - timeout=15, + timeout=min(15.0, remaining), ) if result.ok(): try: @@ -74,10 +85,16 @@ async def _login_as_guest( else: stderr = result.stderr.decode("utf-8", errors="replace")[:200] last_err = f"curl exit {result.exit_code}: {stderr}" - logger.debug("loginAsGuest attempt %d/%d failed: %s", i, attempts, last_err) - await asyncio.sleep(min(2.0 * i, 8.0)) - raise RuntimeError(f"loginAsGuest failed after {attempts} attempts: {last_err}") + remaining = deadline - time.monotonic() + if remaining <= 0: + break + logger.debug("loginAsGuest attempt %d failed: %s", attempt, last_err) + await asyncio.sleep(min(2.0 * attempt, 8.0, remaining)) + + raise RuntimeError( + f"loginAsGuest failed after {attempt} attempts over {max_wait_s:.0f}s: {last_err}" + ) async def bootstrap_caido( @@ -85,11 +102,14 @@ async def bootstrap_caido( *, host_url: str, container_url: str, + boot_wait_s: float = 180.0, ) -> Client: """Connect to the in-container Caido sidecar and select a fresh project.""" logger.info("Bootstrapping Caido client (host=%s, container=%s)", host_url, container_url) - access_token = await _login_as_guest(session, container_url=container_url) + access_token = await _login_as_guest( + session, container_url=container_url, max_wait_s=boot_wait_s + ) client = Client(host_url, auth=TokenAuthOptions(token=access_token)) await client.connect() diff --git a/strix/runtime/session_manager.py b/strix/runtime/session_manager.py index 4b61d7351..7240dfb43 100644 --- a/strix/runtime/session_manager.py +++ b/strix/runtime/session_manager.py @@ -183,6 +183,7 @@ def report(phase: str) -> None: session, host_url=host_caido_url, container_url=container_caido_url, + boot_wait_s=load_settings().runtime.caido_boot_wait_s, ) bundle = { diff --git a/tests/test_caido_bootstrap.py b/tests/test_caido_bootstrap.py new file mode 100644 index 000000000..6194847dc --- /dev/null +++ b/tests/test_caido_bootstrap.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from strix.runtime.caido_bootstrap import _login_as_guest + + +@dataclass +class _FakeResult: + exit_code: int + stdout: str = "" + stderr: bytes = b"" + + def ok(self) -> bool: + return self.exit_code == 0 + + +@dataclass +class _FakeSession: + """Stands in for BaseSandboxSession; returns queued exec() results.""" + + results: list[_FakeResult] + sleeps: list[float] = field(default_factory=list) + timeouts: list[float] = field(default_factory=list) + calls: int = 0 + + async def exec(self, *_args: Any, **kwargs: Any) -> _FakeResult: + result = self.results[min(self.calls, len(self.results) - 1)] + self.calls += 1 + self.timeouts.append(kwargs["timeout"]) + return result + + +_FAKE_TOKEN = "guest-token" # noqa: S105 - test fixture, not a real credential + + +def _token_result() -> _FakeResult: + body = {"data": {"loginAsGuest": {"token": {"accessToken": _FAKE_TOKEN}}}} + return _FakeResult(exit_code=0, stdout=json.dumps(body)) + + +def _refused_result() -> _FakeResult: + return _FakeResult(exit_code=7, stderr=b"curl: (7) Failed to connect") + + +async def test_login_as_guest_succeeds_once_caido_is_up(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("strix.runtime.caido_bootstrap.asyncio.sleep", _no_sleep) + session = _FakeSession(results=[_refused_result(), _refused_result(), _token_result()]) + + token = await _login_as_guest(session, container_url="http://127.0.0.1:48080", max_wait_s=30) + + assert token == _FAKE_TOKEN + assert session.calls == 3 + + +async def test_login_as_guest_respects_configurable_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A slow-booting sandbox (connection refused throughout) should be given + the full configured budget rather than giving up after a fixed attempt + count, and the raised error should report elapsed time, not attempts. + """ + monkeypatch.setattr("strix.runtime.caido_bootstrap.asyncio.sleep", _no_sleep) + + # Fake clock: each read advances by 25s, so a 180s budget survives + # ~7 attempts (the old fixed-10-attempt loop only had ~68s total). + fake_now = [0.0] + + def _fake_monotonic() -> float: + fake_now[0] += 25.0 + return fake_now[0] + + monkeypatch.setattr("strix.runtime.caido_bootstrap.time.monotonic", _fake_monotonic) + session = _FakeSession(results=[_refused_result()]) + + with pytest.raises(RuntimeError) as exc_info: + await _login_as_guest(session, container_url="http://127.0.0.1:48080", max_wait_s=180) + + assert "curl exit 7" in str(exc_info.value) + assert "180s" in str(exc_info.value) + assert session.calls >= 2 + + +async def test_login_as_guest_caps_final_attempt_timeout_to_remaining_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The per-attempt curl timeout must never let a single attempt run past + the overall deadline — otherwise a request started just before the + deadline can block session creation for up to another 15s beyond the + configured STRIX_CAIDO_BOOT_WAIT_S budget. + """ + monkeypatch.setattr("strix.runtime.caido_bootstrap.asyncio.sleep", _no_sleep) + + # First read establishes the deadline; second leaves 3s remaining for the + # attempt; every read after that is past the deadline so the loop exits. + fake_now = iter([0.0, 7.0]) + + def _fake_monotonic() -> float: + return next(fake_now, 11.0) + + monkeypatch.setattr("strix.runtime.caido_bootstrap.time.monotonic", _fake_monotonic) + session = _FakeSession(results=[_refused_result()]) + + with pytest.raises(RuntimeError): + await _login_as_guest(session, container_url="http://127.0.0.1:48080", max_wait_s=10) + + assert session.timeouts == [3.0] + + +async def _no_sleep(_seconds: float) -> None: + """asyncio.sleep stub so deadline-bound tests run instantly."""