From 1ce72b9575ec82bdcc3ab14d4553fde2c2dbd580 Mon Sep 17 00:00:00 2001 From: briandevans <252620095+briandevans@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:41:22 -0700 Subject: [PATCH 1/6] fix(dashboard-auth): run bearer token verification off the dashboard event loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `gated_auth_middleware` is an `async def` running on the single uvicorn event loop, but `_verify_bearer` is synchronous and walks every registered session provider calling `verify_session`. The shipped providers verify a JWT against a `PyJWKClient` built with `lifespan=300` and no `timeout=` (`plugins/dashboard_auth/nous/__init__.py:419`, `plugins/dashboard_auth/self_hosted/__init__.py:600`), so it inherits PyJWKClient's 30s default and `fetch_data` does a blocking `urllib.request.urlopen`. Every 300 seconds the next bearer request to traverse the gate performs that fetch inline on the loop, and a slow or blackholed IDP stalls the entire dashboard — every other request, the live feed and the terminal stream — for up to 30 seconds. Offload at the helper rather than at the inner `verify_session` call so the whole provider loop, including its per-provider retries, moves to a worker in one `await` instead of ping-ponging the loop once per provider. The helper body is unchanged. `asyncio.to_thread` copies the caller's context and re-raises the worker's exception in the awaiting frame, so the `ProviderError` arm and the 503 it returns are untouched; this is a pure scheduling change. `hermes_cli/web_server.py`, the module that installs this middleware, already applies the same rule 51 times (rationale at `web_server.py:2373`); `hermes_cli/dashboard_auth/` had zero uses. --- hermes_cli/dashboard_auth/middleware.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/hermes_cli/dashboard_auth/middleware.py b/hermes_cli/dashboard_auth/middleware.py index 5b11e98cf2b16..fb8fc7d862558 100644 --- a/hermes_cli/dashboard_auth/middleware.py +++ b/hermes_cli/dashboard_auth/middleware.py @@ -16,6 +16,7 @@ """ from __future__ import annotations +import asyncio import logging from typing import Awaitable, Callable @@ -297,6 +298,10 @@ def _verify_bearer(request: Request, *, access_token: str): 401, so the desktop retries instead of dropping the user to full re-login. Unlike the cookie path there is no server-side refresh — the desktop owns its refresh token and rotates via ``/auth/native/refresh``. + + Blocking: performs provider IDP I/O (``verify_session`` may re-fetch the + IDP's JWKS). Callers on the event loop must invoke it through + ``asyncio.to_thread``. """ unreachable_provider: str | None = None for provider in list_session_providers(): @@ -356,7 +361,9 @@ async def gated_auth_middleware( bearer = _extract_bearer(request) if bearer: try: - bearer_session = _verify_bearer(request, access_token=bearer) + bearer_session = await asyncio.to_thread( + _verify_bearer, request, access_token=bearer + ) except ProviderError as e: # At least one provider's IDP/JWKS was unreachable and none # verified the token — transient outage, not bad credentials. From 7cb64d8ef764e4c5e3c6caac2e4bf650500cb89c Mon Sep 17 00:00:00 2001 From: briandevans <252620095+briandevans@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:41:45 -0700 Subject: [PATCH 2/6] fix(dashboard-auth): run the cookie-path session verify off the event loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cookie branch of `gated_auth_middleware` runs its own verify loop inline, calling the synchronous `provider.verify_session` for each registered session provider directly on the event loop. This is the hot path: it executes on every gated request that presents an access-token cookie. In steady state the JWKS cache is warm and the verify is a local signature check, so most requests are cheap. But the cache expires every 300 seconds (`_JWKS_CACHE_SECONDS`), and the request unlucky enough to land after expiry pays a blocking `urlopen` to the IDP's JWKS endpoint, bounded only by PyJWKClient's 30s default. Stacked providers multiply this: each provider that cannot be reached raises only after its own timeout, and the loop tries the next one, so the gate can hold the event loop for a multiple of that ceiling while deciding a single request. Offload the individual `verify_session` call rather than the whole loop: the loop body between calls audits and logs against the request, and keeping that on the loop leaves the surrounding control flow — the `ProviderError` arm, the `unreachable_provider` bookkeeping and the 503 — byte-for-byte identical. --- hermes_cli/dashboard_auth/middleware.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hermes_cli/dashboard_auth/middleware.py b/hermes_cli/dashboard_auth/middleware.py index fb8fc7d862558..1b0c8c0e6fb4b 100644 --- a/hermes_cli/dashboard_auth/middleware.py +++ b/hermes_cli/dashboard_auth/middleware.py @@ -428,7 +428,9 @@ async def gated_auth_middleware( unreachable_provider: str | None = None for provider in _ordered_session_providers(provider_hint): try: - session = provider.verify_session(access_token=at) + session = await asyncio.to_thread( + provider.verify_session, access_token=at + ) except ProviderError as e: _log.warning( "dashboard-auth: provider %r unreachable during verify: %s", From 54a7ba7cb80b81c78460b628e814b0f7f64fd728 Mon Sep 17 00:00:00 2001 From: briandevans <252620095+briandevans@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:42:14 -0700 Subject: [PATCH 3/6] fix(dashboard-auth): run the middleware refresh chain off the event loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_attempt_refresh` is the last synchronous provider call left in `gated_auth_middleware`, and it is the worst of the three. Unlike verify, refresh is never served from a cache: every call is a token-endpoint round trip at `_TOKEN_ENDPOINT_TIMEOUT_SEC = 10.0` (`plugins/dashboard_auth/nous/__init__.py:135`, `plugins/dashboard_auth/self_hosted/__init__.py:118`), and the helper walks every registered session provider until one accepts the refresh token. The reachability is routine rather than exceptional. The access-token cookie's Max-Age tracks the token's own lifetime, so the browser evicts it the moment the token lapses while the refresh-token cookie lives for 30 days. A dashboard with several tabs open therefore fires a burst of parallel requests that each carry only the RT — and on the event loop those refreshes serialise, so the freeze is the burst size times the round trip, not one round trip. Offload at the helper, not at the inner `refresh_session`: both the helper and its single caller sit in `gated_auth_middleware`, so one `await` moves the entire provider walk to a worker instead of returning to the loop between providers. The helper body is unchanged, and the audit trail is unaffected — `audit_log` is synchronous and appends under its own lock, and `_client_ip` only reads already-parsed headers, so neither needs the loop. --- hermes_cli/dashboard_auth/middleware.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hermes_cli/dashboard_auth/middleware.py b/hermes_cli/dashboard_auth/middleware.py index 1b0c8c0e6fb4b..bb50c792ddbe7 100644 --- a/hermes_cli/dashboard_auth/middleware.py +++ b/hermes_cli/dashboard_auth/middleware.py @@ -463,7 +463,8 @@ async def gated_auth_middleware( # serve the request transparently; only after every provider rejects # the RT do we fall through to clear-and-relogin. try: - refreshed = _attempt_refresh( + refreshed = await asyncio.to_thread( + _attempt_refresh, request, refresh_token=_rt, provider_hint=provider_hint, @@ -564,6 +565,10 @@ def _attempt_refresh(request: Request, *, refresh_token, provider_hint: str | No no RT or every reachable provider rejects it. If no provider succeeds and at least one raised ``ProviderError``, re-raises with that provider's name so the caller can return 503 without clearing potentially valid cookies. + + Blocking: performs provider IDP I/O (``refresh_session`` is a token-endpoint + round trip). Callers on the event loop must invoke it through + ``asyncio.to_thread``. """ if not refresh_token: return None From cd9b595d1617cb6c175605f1f053723d7b44bf2e Mon Sep 17 00:00:00 2001 From: briandevans <252620095+briandevans@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:42:53 -0700 Subject: [PATCH 4/6] fix(dashboard-auth): run the login and callback provider calls off the event loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `auth_login`, `auth_native_authorize` and `auth_callback` are all `async def` handlers that call synchronous provider methods directly. Unlike the gate's verify path, none of these has a cache in front of it — `start_login` may perform OIDC discovery and `complete_login` is unconditionally a token-endpoint round trip at `_TOKEN_ENDPOINT_TIMEOUT_SEC = 10.0`, with the self-hosted provider also fetching the discovery document (`plugins/dashboard_auth/self_hosted` `__init__.py:524`). Every one of these is a guaranteed network call on the event loop whenever the route runs. The blast radius is the whole dashboard, not the person signing in: while a login round trip is in flight the loop cannot serve any other request, so one user's slow IDP hop stalls every already-authenticated session on the instance. `_redirect_uri(request)` stays on the loop and its result is passed in as an argument — it is a pure header read, so there is nothing to gain from moving it, and evaluating it eagerly keeps the argument list identical to what the provider saw before. --- hermes_cli/dashboard_auth/routes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hermes_cli/dashboard_auth/routes.py b/hermes_cli/dashboard_auth/routes.py index 15af8077fd23c..9fdb6becdc41b 100644 --- a/hermes_cli/dashboard_auth/routes.py +++ b/hermes_cli/dashboard_auth/routes.py @@ -15,6 +15,7 @@ """ from __future__ import annotations +import asyncio import logging import threading import time @@ -202,7 +203,9 @@ async def auth_login(request: Request, provider: str, next: str = ""): return RedirectResponse(url=login_url, status_code=302) try: - ls = p.start_login(redirect_uri=_redirect_uri(request)) + ls = await asyncio.to_thread( + p.start_login, redirect_uri=_redirect_uri(request) + ) except ProviderError as e: audit_log( AuditEvent.LOGIN_FAILURE, @@ -367,7 +370,9 @@ async def auth_native_authorize( raise HTTPException(status_code=503, detail=str(e)) try: - ls = p.start_login(redirect_uri=_redirect_uri(request)) + ls = await asyncio.to_thread( + p.start_login, redirect_uri=_redirect_uri(request) + ) except ProviderError as e: raise HTTPException(status_code=503, detail=f"Provider unreachable: {e}") @@ -467,7 +472,8 @@ async def auth_callback( ) try: - session = p.complete_login( + session = await asyncio.to_thread( + p.complete_login, code=code, state=state, code_verifier=verifier, From c8ef6be0d96ffcf7ea10a8fc29f485745ea03df0 Mon Sep 17 00:00:00 2001 From: briandevans <252620095+briandevans@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:43:29 -0700 Subject: [PATCH 5/6] fix(dashboard-auth): run password login, logout revoke, and native refresh off the event loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last three synchronous provider calls in the router, all reached from `async def` handlers. `complete_password_login` verifies credentials against the provider's backing store; the route already rate-limits it per IP, which is only meaningful if the handler can keep serving requests while one credential check is outstanding. On the event loop the rate limiter and the freeze work against each other — a burst of login attempts blocks the dashboard for the duration of the attempts it is trying to throttle. `auth_logout` is the clearest case: the revoke is explicitly documented as best-effort, and failures are logged and never raised. Logging out is not supposed to be able to hurt anyone, but on the event loop an unreachable IDP makes an ignorable revoke stall the whole instance for the full timeout, once per registered provider, before the user is redirected. `auth_native_refresh` is the desktop's rotation endpoint and mirrors the middleware's `_attempt_refresh` provider stacking, with the same unconditional token-endpoint round trip per provider. Each `to_thread` wraps only the provider call itself; the surrounding `try`/`except` arms, audit calls and control flow are unchanged, and the worker's exception is re-raised in the awaiting frame so `InvalidCredentialsError`, `NotImplementedError`, `RefreshExpiredError` and `ProviderError` still land where they did. --- hermes_cli/dashboard_auth/routes.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hermes_cli/dashboard_auth/routes.py b/hermes_cli/dashboard_auth/routes.py index 9fdb6becdc41b..ac4c05f43f385 100644 --- a/hermes_cli/dashboard_auth/routes.py +++ b/hermes_cli/dashboard_auth/routes.py @@ -713,8 +713,10 @@ async def auth_password_login(request: Request, body: _PasswordLoginBody): raise HTTPException(status_code=404, detail="Unknown provider") try: - session = p.complete_password_login( - username=body.username, password=body.password + session = await asyncio.to_thread( + p.complete_password_login, + username=body.username, + password=body.password, ) except InvalidCredentialsError: audit_log( @@ -771,7 +773,7 @@ async def auth_logout(request: Request): # logged but never raised. for provider in list_providers(): try: - provider.revoke_session(refresh_token=rt) + await asyncio.to_thread(provider.revoke_session, refresh_token=rt) except Exception as e: # noqa: BLE001 — best-effort _log.warning( "dashboard-auth: revoke on %r failed: %s", @@ -942,7 +944,9 @@ async def auth_native_refresh(request: Request, body: _NativeRefreshBody): unreachable: str | None = None for provider in providers: try: - session = provider.refresh_session(refresh_token=body.refresh_token) + session = await asyncio.to_thread( + provider.refresh_session, refresh_token=body.refresh_token + ) except RefreshExpiredError: continue except ProviderError as e: From 1668f46efce68f7f60caa928f76d60849b6879ab Mon Sep 17 00:00:00 2001 From: briandevans <252620095+briandevans@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:50:09 -0700 Subject: [PATCH 6/6] test(dashboard-auth): pin every dashboard-auth provider call to a worker thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine production call sites invoke the synchronous DashboardAuthProvider protocol from `async def` handlers. This locks all nine to a worker thread so a future edit cannot quietly put one back on the event loop. The assertion is structural rather than timing-based: a recording provider calls `asyncio.get_running_loop()` at entry. A RuntimeError means it is on a worker thread (correct); a live loop means the dashboard's loop is being blocked. Each test reverts red with only its own production hunk undone. One timing test is included for the symptom itself — a provider that blocks for two seconds must not delay a concurrent request to a public, provider-free route. It is written around the trap that makes this class of test worthless: `TestClient(app)` outside a `with` block builds a fresh anyio portal, and therefore a fresh event loop, for every request, so two requests can never contend and the test passes even against unpatched code. Every client here is entered as a context manager, pinning one loop for the whole test exactly as uvicorn serves the dashboard. `test_provider_errors_still_surface_unchanged` is a behaviour-preservation guard, not regression coverage: it passes both before and after, and exists so a later refactor cannot swap the offload for something that drops the provider's exception instead of re-raising it in the awaiting frame. New file rather than an append to an existing `test_dashboard_auth_*` module, because it registers its own recording providers and needs its own gate fixture. --- .../test_dashboard_auth_off_loop.py | 471 ++++++++++++++++++ 1 file changed, 471 insertions(+) create mode 100644 tests/hermes_cli/test_dashboard_auth_off_loop.py diff --git a/tests/hermes_cli/test_dashboard_auth_off_loop.py b/tests/hermes_cli/test_dashboard_auth_off_loop.py new file mode 100644 index 0000000000000..3af631a72eea9 --- /dev/null +++ b/tests/hermes_cli/test_dashboard_auth_off_loop.py @@ -0,0 +1,471 @@ +"""Every dashboard-auth provider call must run off the event loop. + +``hermes_cli/dashboard_auth`` gates the dashboard, and the +:class:`~hermes_cli.dashboard_auth.base.DashboardAuthProvider` protocol is +entirely synchronous: ``start_login`` / ``complete_login`` / +``complete_password_login`` / ``verify_session`` / ``refresh_session`` / +``revoke_session`` are all plain ``def``. The shipped providers implement them +with blocking network I/O — ``httpx`` token-endpoint round trips at +``_TOKEN_ENDPOINT_TIMEOUT_SEC = 10.0``, and a ``PyJWKClient`` whose +``fetch_data`` is a blocking ``urllib.request.urlopen`` with a 30s default +ceiling and a 300s cache lifespan. + +Called straight from an ``async def`` handler, each of those blocks the single +uvicorn event loop, freezing the entire dashboard — every other request, the +live feed, the terminal stream — for the duration. + +These tests pin the invariant directly rather than by timing: a recording +provider asks ``asyncio.get_running_loop()`` at call time. A ``RuntimeError`` +means it is on a worker thread (correct); a loop means it was invoked on the +event loop (the bug). One assertion per production call site. + +Two traps this file is written around: + +* ``TestClient(app)`` **without** the context-manager form spins up a fresh + event loop per request, so a concurrency test written that way can never + observe contention and passes even with the fix reverted. Every client here + is entered as a context manager, which pins one anyio portal — and therefore + one event loop — for the whole test. +* The recording provider must not be the stub the other dashboard-auth suites + register, so this file registers its own and restores provider state itself. +""" +from __future__ import annotations + +import asyncio +import contextlib +import threading +import time +from urllib.parse import parse_qs, urlparse + +import pytest +from fastapi.testclient import TestClient + +from hermes_cli import web_server +from hermes_cli.dashboard_auth import clear_providers, register_provider +from hermes_cli.dashboard_auth.base import ( + InvalidCredentialsError, + ProviderError, + Session, +) +from hermes_cli.dashboard_auth.cookies import SESSION_AT_COOKIE, SESSION_RT_COOKIE +from hermes_cli.dashboard_auth.routes import _reset_password_rate_limit +from tests.hermes_cli.conftest_dashboard_auth import StubAuthProvider, _sign + +_BASE_URL = "https://fly-app.fly.dev" +# Any gated (non-allowlisted) route works; /api/auth/me is the cheapest. +_GATED_PATH = "/api/auth/me" +# Public, provider-free, and in the shared allowlist — the probe used to show +# that a slow provider no longer holds the loop. +_PUBLIC_PATH = "/api/health" + + +# --------------------------------------------------------------------------- +# Recording providers +# --------------------------------------------------------------------------- + + +class LoopRecordingProvider(StubAuthProvider): + """``StubAuthProvider`` that records where each provider call executed. + + ``ran_on_loop[method]`` is ``True`` when the method was entered with a + running asyncio event loop — i.e. it was invoked directly from the + dashboard's ``async def`` handler and is blocking it. It is ``False`` when + the method ran on a worker thread, which is what the production code must + arrange. + """ + + name = "stub" + display_name = "Loop-recording stub IdP (test only)" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ran_on_loop: dict[str, bool] = {} + # method name -> seconds to block inside the call. + self.delays: dict[str, float] = {} + # Set as soon as a delayed call has started blocking. + self.entered = threading.Event() + # method name -> exception instance to raise instead of running. + self.raises: dict[str, BaseException] = {} + + def _record(self, method: str) -> None: + try: + asyncio.get_running_loop() + except RuntimeError: + self.ran_on_loop[method] = False + else: + self.ran_on_loop[method] = True + delay = self.delays.get(method, 0.0) + if delay: + self.entered.set() + # Deliberately ``time.sleep`` and not ``asyncio.sleep``: this + # stands in for the blocking socket read a real provider performs. + time.sleep(delay) + exc = self.raises.get(method) + if exc is not None: + raise exc + + def start_login(self, **kwargs): + self._record("start_login") + return super().start_login(**kwargs) + + def complete_login(self, **kwargs): + self._record("complete_login") + return super().complete_login(**kwargs) + + def verify_session(self, **kwargs): + self._record("verify_session") + return super().verify_session(**kwargs) + + def refresh_session(self, **kwargs): + self._record("refresh_session") + return super().refresh_session(**kwargs) + + def revoke_session(self, **kwargs): + self._record("revoke_session") + return super().revoke_session(**kwargs) + + +class LoopRecordingPasswordProvider(LoopRecordingProvider): + """Password variant, for the ``/auth/password-login`` site. + + Kept separate because ``supports_password`` diverts ``/auth/login`` to the + credential form (so ``start_login`` is never reached) and makes + ``/auth/native/authorize`` reject the provider outright. A single provider + cannot cover both the OAuth and the password call sites. + """ + + name = "stubpw" + display_name = "Loop-recording password IdP (test only)" + supports_password = True + + def complete_password_login(self, *, username: str, password: str) -> Session: + self._record("complete_password_login") + if (username, password) != ("admin", "hunter2"): + raise InvalidCredentialsError("bad credentials") + now = int(time.time()) + exp = now + 3600 + return Session( + user_id="stub-user-1", + email="stub@example.test", + display_name="Stub User", + org_id="stub-org-1", + provider=self.name, + expires_at=exp, + access_token=_sign({ + "sub": "stub-user-1", + "email": "stub@example.test", + "name": "Stub User", + "org_id": "stub-org-1", + "exp": exp, + }), + refresh_token=_sign({ + "sub": "stub-user-1", + "kind": "refresh", + "exp": now + 30 * 86400, + }), + ) + + +# --------------------------------------------------------------------------- +# Token helpers — mint tokens the stub's own verify/refresh accept. +# --------------------------------------------------------------------------- + + +def _access_token(ttl: int = 3600) -> str: + return _sign({ + "sub": "stub-user-1", + "email": "stub@example.test", + "name": "Stub User", + "org_id": "stub-org-1", + "exp": int(time.time()) + ttl, + }) + + +def _refresh_token(ttl: int = 30 * 86400) -> str: + return _sign({ + "sub": "stub-user-1", + "kind": "refresh", + "exp": int(time.time()) + ttl, + }) + + +# --------------------------------------------------------------------------- +# Fixture +# --------------------------------------------------------------------------- + + +@pytest.fixture +def gated(): + """Yield ``make(provider) -> TestClient`` with the auth gate engaged. + + The client is entered as a context manager on purpose. ``TestClient`` + outside a ``with`` block builds a throwaway anyio portal per request, which + means a fresh event loop per request and no possible contention between + them; the concurrency test below would then pass against unpatched code. + Entering it pins one portal, and therefore one loop, exactly as uvicorn + serves the real dashboard. + """ + prev_host = getattr(web_server.app.state, "bound_host", None) + prev_port = getattr(web_server.app.state, "bound_port", None) + prev_required = getattr(web_server.app.state, "auth_required", None) + stack = contextlib.ExitStack() + + def _make(provider) -> TestClient: + clear_providers() + register_provider(provider) + web_server.app.state.bound_host = "fly-app.fly.dev" + web_server.app.state.bound_port = 443 + web_server.app.state.auth_required = True + return stack.enter_context( + TestClient( + web_server.app, + base_url=_BASE_URL, + follow_redirects=False, + ) + ) + + try: + yield _make + finally: + stack.close() + clear_providers() + web_server.app.state.bound_host = prev_host + web_server.app.state.bound_port = prev_port + web_server.app.state.auth_required = prev_required + _reset_password_rate_limit() + + +def _assert_off_loop(provider: LoopRecordingProvider, method: str) -> None: + assert method in provider.ran_on_loop, ( + f"{method} was never called — the test did not reach the production " + f"site it is meant to cover (recorded: {sorted(provider.ran_on_loop)})" + ) + assert provider.ran_on_loop[method] is False, ( + f"{method} ran ON the dashboard event loop. Provider methods perform " + f"blocking network I/O and must be dispatched via asyncio.to_thread." + ) + + +# --------------------------------------------------------------------------- +# middleware.py — the auth gate +# --------------------------------------------------------------------------- + + +def test_bearer_verify_runs_off_the_loop(gated): + """``middleware.py`` bearer path: ``_verify_bearer`` -> ``verify_session``.""" + provider = LoopRecordingProvider() + client = gated(provider) + + r = client.get( + _GATED_PATH, + headers={"Authorization": f"Bearer {_access_token()}"}, + ) + + assert r.status_code == 200, r.text + _assert_off_loop(provider, "verify_session") + + +def test_cookie_verify_runs_off_the_loop(gated): + """``middleware.py`` cookie path: the inline ``verify_session`` loop.""" + provider = LoopRecordingProvider() + client = gated(provider) + client.cookies.set(SESSION_AT_COOKIE, _access_token()) + + r = client.get(_GATED_PATH) + + assert r.status_code == 200, r.text + _assert_off_loop(provider, "verify_session") + + +def test_middleware_refresh_runs_off_the_loop(gated): + """``middleware.py`` refresh chain: ``_attempt_refresh`` -> ``refresh_session``. + + Sends only the refresh-token cookie. That is the ordinary expiry shape, not + an edge case: the access-token cookie's Max-Age tracks the token lifetime, + so the browser evicts it on expiry while the RT cookie lives for 30 days. + """ + provider = LoopRecordingProvider() + client = gated(provider) + client.cookies.set(SESSION_RT_COOKIE, _refresh_token()) + + r = client.get(_GATED_PATH) + + assert r.status_code == 200, r.text + _assert_off_loop(provider, "refresh_session") + + +# --------------------------------------------------------------------------- +# routes.py — the OAuth round trip +# --------------------------------------------------------------------------- + + +def test_auth_login_start_runs_off_the_loop(gated): + """``routes.py`` ``auth_login`` -> ``start_login``.""" + provider = LoopRecordingProvider() + client = gated(provider) + + r = client.get("/auth/login", params={"provider": "stub"}) + + assert r.status_code == 302, r.text + _assert_off_loop(provider, "start_login") + + +def test_native_authorize_start_runs_off_the_loop(gated): + """``routes.py`` ``auth_native_authorize`` -> ``start_login``.""" + provider = LoopRecordingProvider() + client = gated(provider) + + r = client.get( + "/auth/native/authorize", + params={ + "provider": "stub", + "code_challenge": "x" * 43, + "code_challenge_method": "S256", + "redirect_uri": "http://127.0.0.1:8765/callback", + "state": "cli-state", + }, + ) + + assert r.status_code == 302, r.text + _assert_off_loop(provider, "start_login") + + +def test_auth_callback_complete_runs_off_the_loop(gated): + """``routes.py`` ``auth_callback`` -> ``complete_login``. + + Walks the real round trip so the PKCE cookie and the state/verifier pair + are the ones the route actually issued. + """ + provider = LoopRecordingProvider() + client = gated(provider) + + start = client.get("/auth/login", params={"provider": "stub"}) + assert start.status_code == 302, start.text + cb = parse_qs(urlparse(start.headers["location"]).query) + + r = client.get( + "/auth/callback", + params={"code": cb["code"][0], "state": cb["state"][0]}, + ) + + assert r.status_code == 302, r.text + _assert_off_loop(provider, "complete_login") + + +def test_password_login_runs_off_the_loop(gated): + """``routes.py`` ``auth_password_login`` -> ``complete_password_login``.""" + provider = LoopRecordingPasswordProvider() + client = gated(provider) + + r = client.post( + "/auth/password-login", + json={"provider": "stubpw", "username": "admin", "password": "hunter2"}, + ) + + assert r.status_code == 200, r.text + _assert_off_loop(provider, "complete_password_login") + + +def test_logout_revoke_runs_off_the_loop(gated): + """``routes.py`` ``auth_logout`` -> ``revoke_session``. + + The revoke is documented as best-effort and its failures are swallowed, so + on the event loop an unreachable IDP turns an ignorable call into a + dashboard-wide stall for the full provider timeout. + """ + provider = LoopRecordingProvider() + client = gated(provider) + client.cookies.set(SESSION_RT_COOKIE, _refresh_token()) + + r = client.post("/auth/logout") + + assert r.status_code == 302, r.text + _assert_off_loop(provider, "revoke_session") + + +def test_native_refresh_runs_off_the_loop(gated): + """``routes.py`` ``auth_native_refresh`` -> ``refresh_session``.""" + provider = LoopRecordingProvider() + client = gated(provider) + + r = client.post( + "/auth/native/refresh", + json={"refresh_token": _refresh_token(), "provider": "stub"}, + ) + + assert r.status_code == 200, r.text + _assert_off_loop(provider, "refresh_session") + + +# --------------------------------------------------------------------------- +# The symptom itself +# --------------------------------------------------------------------------- + + +def test_a_slow_provider_does_not_stall_a_concurrent_request(gated): + """A slow IDP must not freeze the rest of the dashboard. + + One request enters the gate against a provider that blocks for + ``_SLOW`` seconds; a second, public, provider-free request is issued from + another thread while it is in flight. Both share one event loop (the + fixture pins a single portal), so if the provider call runs on the loop the + probe cannot be served until it finishes. + """ + _SLOW = 2.0 + _BUDGET = 1.0 + + provider = LoopRecordingProvider() + provider.delays["verify_session"] = _SLOW + client = gated(provider) + + slow_done = threading.Event() + + def _slow_request() -> None: + try: + client.get( + _GATED_PATH, + headers={"Authorization": f"Bearer {_access_token()}"}, + ) + finally: + slow_done.set() + + worker = threading.Thread(target=_slow_request, daemon=True) + worker.start() + try: + assert provider.entered.wait(timeout=10.0), ( + "the slow provider call never started" + ) + started = time.monotonic() + probe = client.get(_PUBLIC_PATH) + elapsed = time.monotonic() - started + finally: + worker.join(timeout=15.0) + + assert probe.status_code == 200, probe.text + assert slow_done.is_set(), "the slow request never completed" + assert elapsed < _BUDGET, ( + f"a public request waited {elapsed:.2f}s behind a {_SLOW:.1f}s provider " + f"call — the provider call is blocking the event loop" + ) + + +def test_provider_errors_still_surface_unchanged(gated): + """Offloading must not move an exception out of its ``except`` arm. + + ``asyncio.to_thread`` re-raises the worker's exception in the awaiting + frame, so ``ProviderError`` still reaches the handler that turns it into a + 503. This is a behaviour-preservation guard: it passes both before and + after the production change, and is here so a future refactor that swaps + the offload for a fire-and-forget cannot land quietly. + """ + provider = LoopRecordingProvider() + provider.raises["verify_session"] = ProviderError("stub") + client = gated(provider) + + r = client.get( + _GATED_PATH, + headers={"Authorization": f"Bearer {_access_token()}"}, + ) + + assert r.status_code == 503, r.text + assert "unreachable" in r.json()["detail"]