perf: composite indexes + cursor pagination + clock seam + SSE rate-limit + Ollama sanitization + retry-after web client + WS reconnect jitter

docs/reference/conventions.md

@@ -51,9 +51,11 @@ Controllers return one of three shapes:
   "success": true}`.
 * `PaginatedResponse[T]`: list endpoints that return a page of items
   plus pagination metadata. Wraps `ApiResponse[T]` and adds a
-  `pagination` envelope ({`limit`, `offset`, `total`, `next_cursor`,
-  `has_more`}). Required for any controller method whose return type
-  is a collection. Opaque cursor pagination is the project default.
+  `pagination` envelope ({`limit`, `next_cursor`, `has_more`}).
+  Required for any controller method whose return type is a
+  collection. Opaque HMAC-signed cursor pagination is the project
+  default; clients walk pages via the `next_cursor` token rather than
+  offset arithmetic. There is no `total` count on the wire.
 * `Response[ApiResponse[T]]`: only when status code or response
   headers must be customized (e.g. setting a `Location` header on a
   201, attaching a `Retry-After` header on a 429).

scripts/loop_bound_init_baseline.txt

@@ -29,7 +29,7 @@ src/synthorg/integrations/oauth/token_manager.py:70:OAuthTokenManager:_lifecycle
 src/synthorg/integrations/rate_limiting/shared_state.py:71:SharedRateLimitCoordinator:_window_lock:Lock
 src/synthorg/integrations/rate_limiting/shared_state.py:80:SharedRateLimitCoordinator:_lifecycle_lock:Lock
 src/synthorg/integrations/tunnel/ngrok_adapter.py:76:NgrokAdapter:_lifecycle_lock:Lock
-src/synthorg/memory/embedding/fine_tune_orchestrator.py:97:FineTuneOrchestrator:_op_lock:Lock
+src/synthorg/memory/embedding/fine_tune_orchestrator.py:99:FineTuneOrchestrator:_op_lock:Lock
 src/synthorg/meta/chief_of_staff/monitor.py:82:OrgInflectionMonitor:_stop_event:Event
 src/synthorg/meta/chief_of_staff/monitor.py:83:OrgInflectionMonitor:_lifecycle_lock:Lock
 src/synthorg/notifications/adapters/ntfy.py:116:NtfyNotificationSink:_lifecycle_lock:Lock

scripts/mock_spec_baseline.txt

@@ -353,13 +353,13 @@ tests/unit/api/controllers/test_ontology.py:53:24
 tests/unit/api/controllers/test_ontology.py:54:22
 tests/unit/api/controllers/test_ontology.py:55:20
 tests/unit/api/controllers/test_provider_health.py:349:20
-tests/unit/api/controllers/test_providers.py:192:44
-tests/unit/api/controllers/test_providers.py:215:44
-tests/unit/api/controllers/test_providers.py:262:17
-tests/unit/api/controllers/test_providers.py:263:40
-tests/unit/api/controllers/test_providers.py:265:40
-tests/unit/api/controllers/test_providers.py:270:63
-tests/unit/api/controllers/test_providers.py:295:63
+tests/unit/api/controllers/test_providers.py:198:44
+tests/unit/api/controllers/test_providers.py:221:44
+tests/unit/api/controllers/test_providers.py:268:17
+tests/unit/api/controllers/test_providers.py:269:40
+tests/unit/api/controllers/test_providers.py:271:40
+tests/unit/api/controllers/test_providers.py:276:63
+tests/unit/api/controllers/test_providers.py:301:63
 tests/unit/api/controllers/test_setup.py:479:28
 tests/unit/api/controllers/test_setup.py:591:12
 tests/unit/api/controllers/test_setup.py:646:17
@@ -2480,15 +2480,15 @@ tests/unit/providers/drivers/test_litellm_driver.py:470:15
 tests/unit/providers/drivers/test_litellm_driver.py:676:19
 tests/unit/providers/drivers/test_mappers.py:225:15
 tests/unit/providers/drivers/test_mappers.py:229:13
-tests/unit/providers/management/test_local_models.py:97:17
-tests/unit/providers/management/test_local_models.py:104:26
-tests/unit/providers/management/test_local_models.py:128:17
-tests/unit/providers/management/test_local_models.py:135:26
-tests/unit/providers/management/test_local_models.py:155:26
-tests/unit/providers/management/test_local_models.py:173:24
-tests/unit/providers/management/test_local_models.py:176:24
-tests/unit/providers/management/test_local_models.py:225:25
-tests/unit/providers/management/test_local_models.py:247:25
+tests/unit/providers/management/test_local_models.py:99:17
+tests/unit/providers/management/test_local_models.py:106:26
+tests/unit/providers/management/test_local_models.py:130:17
+tests/unit/providers/management/test_local_models.py:137:26
+tests/unit/providers/management/test_local_models.py:157:26
+tests/unit/providers/management/test_local_models.py:175:24
+tests/unit/providers/management/test_local_models.py:178:24
+tests/unit/providers/management/test_local_models.py:227:25
+tests/unit/providers/management/test_local_models.py:249:25
 tests/unit/providers/management/test_service.py:174:24
 tests/unit/providers/management/test_service.py:175:33
 tests/unit/providers/management/test_service.py:178:30

scripts/no_magic_numbers_baseline.txt

@@ -62,8 +62,8 @@ src/synthorg/api/controllers/departments.py:229:49
 src/synthorg/api/controllers/departments.py:392:29
 src/synthorg/api/controllers/escalations.py:133:29
 src/synthorg/api/controllers/evaluation_config_versions.py:46:29
-src/synthorg/api/controllers/events.py:54:34
-src/synthorg/api/controllers/events.py:101:36
+src/synthorg/api/controllers/events.py:57:34
+src/synthorg/api/controllers/events.py:104:36
 src/synthorg/api/controllers/integration_health.py:77:29
 src/synthorg/api/controllers/meetings.py:35:34
 src/synthorg/api/controllers/meetings.py:36:28
@@ -94,12 +94,12 @@ src/synthorg/api/controllers/ontology.py:359:29
 src/synthorg/api/controllers/ontology.py:450:29
 src/synthorg/api/controllers/personalities.py:76:29
 src/synthorg/api/controllers/projects.py:71:29
-src/synthorg/api/controllers/providers.py:1389:29
+src/synthorg/api/controllers/providers.py:1414:29
 src/synthorg/api/controllers/reports.py:144:29
 src/synthorg/api/controllers/requests.py:89:29
 src/synthorg/api/controllers/role_versions.py:44:29
-src/synthorg/api/controllers/scaling.py:192:29
-src/synthorg/api/controllers/settings.py:298:29
+src/synthorg/api/controllers/scaling.py:214:29
+src/synthorg/api/controllers/settings.py:320:29
 src/synthorg/api/controllers/simulations.py:291:29
 src/synthorg/api/controllers/subworkflows.py:128:29
 src/synthorg/api/controllers/tasks.py:172:29
@@ -790,11 +790,11 @@ src/synthorg/providers/health_prober.py:48:35
 src/synthorg/providers/management/_capability_helpers.py:29:26
 src/synthorg/providers/management/_helpers.py:29:35
 src/synthorg/providers/management/audit_service.py:74:21
-src/synthorg/providers/management/local_models.py:29:31
-src/synthorg/providers/management/local_models.py:30:33
-src/synthorg/providers/management/local_models.py:31:16
-src/synthorg/providers/management/local_models.py:32:23
-src/synthorg/providers/management/local_models.py:33:26
+src/synthorg/providers/management/local_models.py:30:31
+src/synthorg/providers/management/local_models.py:31:33
+src/synthorg/providers/management/local_models.py:32:16
+src/synthorg/providers/management/local_models.py:33:23
+src/synthorg/providers/management/local_models.py:34:26
 src/synthorg/providers/probing.py:29:39
 src/synthorg/security/audit.py:30:45
 src/synthorg/security/audit.py:99:21

scripts/run_affected_tests.py

@@ -200,6 +200,32 @@ def _affected_test_dirs(changed: list[str]) -> tuple[list[str], bool]:
     r"worker '(?P<worker>\w+)' crashed while running '(?P<test>[^']+)'",
 )
 
+# A different xdist signature for the same underlying problem: when a
+# worker terminates abnormally between tests (rather than during one),
+# xdist prints ``[gwN] node down: Not properly terminated`` instead of
+# the canonical ``worker 'gwN' crashed while running '...'`` form. The
+# Python 3.14 + Windows ProactorEventLoop IOCP teardown race produces
+# both signatures depending on exactly when the IOCP cleanup blew up,
+# and both should be treated as native-level crash advisory rather
+# than a real test failure. Captures the worker id only -- there is
+# no associated test id in this signature.
+_NODE_DOWN_RE = re.compile(
+    r"\[(?P<worker>gw\d+)\] node down: Not properly terminated",
+)
+
+# pytest-xdist's scheduler raises Python-level exceptions when it
+# tries to assign work to a worker that already disappeared (KeyError
+# in ``loadscope.py``) or asserts on a residual ``crashitem`` while a
+# worker reports finished. Both are downstream consequences of the
+# native-level worker death captured by ``_NODE_DOWN_RE`` /
+# ``_WORKER_CRASH_RE`` above, not independent regressions, so the
+# classifier folds them into the crash-advisory branch when paired
+# with at least one observed crash signature.
+_XDIST_INTERNAL_ERROR_RE = re.compile(
+    r"^INTERNALERROR>",
+    re.MULTILINE,
+)
+
 # pytest in ``-q`` mode prints ``FAILED <test_id> - <reason>`` (or just
 # ``FAILED <test_id>``) at the start of a line for every failure in the
 # session summary.  ``\S+`` captures up to the first whitespace; valid
@@ -563,6 +589,11 @@ def _parse_worker_crashes(stdout: str) -> tuple[tuple[str, str], ...]:
     )
 
 
+def _parse_node_down(stdout: str) -> tuple[str, ...]:
+    """Return the worker ids of every ``[gwN] node down`` announcement."""
+    return tuple(m.group("worker") for m in _NODE_DOWN_RE.finditer(stdout))
+
+
 def _parse_test_failures(stdout: str) -> tuple[str, ...]:
     """Extract test ids from ``FAILED`` summary lines."""
     return tuple(m.group("test") for m in _FAILED_RE.finditer(stdout))
@@ -588,13 +619,21 @@ def _classify_isolation_outcome(
     * Crashes only, no repeats -> crash advisory (treat as pass; the
       gate exits 0 and prints a hint about Windows ProactorEventLoop /
       cross-worktree contention).
+    * Worker(s) went ``node down`` AND the only summary signal is an
+      ``INTERNALERROR>`` traceback (xdist scheduler crashed because
+      its workers vanished) -> crash advisory.  Without the parser
+      branch the dead-worker chain reads as "non-zero returncode +
+      no parsable test signal" and falls through to fail-closed,
+      blocking the push on documented native-level flakiness.
     * No crashes, no failures, returncode 0 -> pass.
     * No parsable signal but returncode non-zero -> fail closed
       (regression) so degraded output never silently passes.
     """
     crashes = _parse_worker_crashes(stdout)
     crashed_tests = tuple(test for _, test in crashes)
     crashed_set = set(crashed_tests)
+    node_down_workers = _parse_node_down(stdout)
+    has_internal_error = bool(_XDIST_INTERNAL_ERROR_RE.search(stdout))
     failed_tests_raw = _parse_test_failures(stdout)
     real_failures = tuple(t for t in failed_tests_raw if t not in crashed_set)
 
@@ -623,6 +662,17 @@ def _classify_isolation_outcome(
             exit_code=0,
             crashed_tests=crashed_tests,
         )
+    # ``node down`` without a paired ``crashed while running`` line means
+    # the worker died between tests, so the test names are not
+    # recoverable -- surface the worker ids in their place so the
+    # advisory banner still has something to print and the
+    # ``crash_advisory`` invariant (``crashed_tests`` non-empty) holds.
+    if node_down_workers and has_internal_error and returncode != 0:
+        return IsolationOutcome(
+            kind="crash_advisory",
+            exit_code=0,
+            crashed_tests=tuple(f"<worker {w}>" for w in node_down_workers),
+        )
     if returncode == 0:
         return IsolationOutcome(kind="pass", exit_code=0)
     return IsolationOutcome(kind="regression", exit_code=returncode)

src/synthorg/api/controllers/events.py

@@ -19,7 +19,10 @@
 from synthorg.api.dto import ApiResponse
 from synthorg.api.guards import _READ_ROLES, require_approval_roles, require_read_access
 from synthorg.api.path_params import QUERY_MAX_LENGTH, PathId
-from synthorg.api.rate_limits import per_op_rate_limit_from_policy
+from synthorg.api.rate_limits import (
+    per_op_concurrency_from_policy,
+    per_op_rate_limit_from_policy,
+)
 from synthorg.api.state import AppState  # noqa: TC001
 from synthorg.communication.event_stream.interrupt import (
     Interrupt,
@@ -524,7 +527,14 @@ class EventStreamController(Controller):
     @get(
         "/stream",
         media_type="text/event-stream",
-        guards=[require_read_access],
+        guards=[
+            require_read_access,
+            per_op_rate_limit_from_policy("events.stream", key="user_or_ip"),
+        ],
+        opt=per_op_concurrency_from_policy(
+            "events.stream",
+            key="user",
+        ),
     )
     async def stream(
         self,