From a9cf722fc90feda96a40615256c1dd6f5ecbc163 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 12:12:17 +0200 Subject: [PATCH 01/13] fix: audit cleanup C - persistence, concurrency & data integrity (#1708) Closes #1708. Lifecycle (canonical pattern per docs/reference/lifecycle-sync.md): - providers/health_prober.py: add _lifecycle_lock + _stop_failed + drain timeout - meta/chief_of_staff/monitor.py: add full canonical pattern + cooperative loop - backup/scheduler.py: convert sync start to async + canonical pattern - hr/pruning/service.py: convert sync start to async + canonical pattern - integrations/tunnel/ngrok_adapter.py: add _lifecycle_lock for uniformity - client/continuous.py: rename _lock to _lifecycle_lock, document in-place runner Race conditions (threading.Lock for sync methods callable from threadpool): - api/auth/ticket_store.py: lock count-and-insert in create() - tools/mcp/cache.py: lock get/put/invalidate - integrations/webhooks/replay_protection.py: lock nonce dedup in check() Currency aggregation invariant: - budget/trends.py: _assert_single_currency in bucket_cost_records and project_daily_spend (raises MixedCurrencyAggregationError on mixed input) Idempotency: - api/controllers/simulations.py: reject duplicate simulation_id with 409 - communication/event_stream/stream.py: per-session dedup window keyed by event.id (60s TTL, bounded per session) - api/controllers/backup.py: Idempotency-Key header now mandatory (HTTP 400) Escalation factory: - communication/conflict_resolution/escalation/factory.py: refactor if/elif dispatch into a registry map shape (queue store + decision processor) Documentation: - docs/research/lgpl-postgres-driver-decision.md: ADR for accept-with-ADR on LGPL psycopg + audit-finding resolutions (#61, #127 false-positives) - docs/licensing.md: third-party LGPL note for the postgres extra Tests added: thread-safety, lifecycle (concurrent start, restart, drain timeout marks the service unrestartable), currency rejection, idempotency dedup, registry dispatch. 26134 unit tests pass. --- docs/licensing.md | 13 ++ .../research/lgpl-postgres-driver-decision.md | 79 ++++++++++ scripts/mock_spec_baseline.txt | 65 ++++++--- src/synthorg/api/auth/ticket_store.py | 67 +++++---- src/synthorg/api/controllers/backup.py | 52 ++++--- src/synthorg/api/controllers/simulations.py | 16 ++ src/synthorg/backup/scheduler.py | 138 ++++++++++++++---- src/synthorg/backup/service.py | 2 +- src/synthorg/budget/trends.py | 15 ++ src/synthorg/client/continuous.py | 45 +++++- .../conflict_resolution/escalation/factory.py | 122 ++++++++++++---- .../communication/event_stream/stream.py | 82 ++++++++++- src/synthorg/hr/pruning/service.py | 123 +++++++++++++--- .../integrations/tunnel/ngrok_adapter.py | 93 +++++++----- .../webhooks/replay_protection.py | 89 ++++++----- src/synthorg/meta/chief_of_staff/monitor.py | 134 +++++++++++++---- .../observability/events/event_stream.py | 1 + src/synthorg/providers/health_prober.py | 109 +++++++++++--- .../settings/subscribers/backup_subscriber.py | 2 +- src/synthorg/tools/mcp/cache.py | 86 ++++++----- .../auth/test_ticket_store_threadsafety.py | 105 +++++++++++++ tests/unit/api/controllers/test_backup.py | 35 ++++- .../test_backup_required_idempotency.py | 96 ++++++++++++ .../test_simulations_idempotency.py | 118 +++++++++++++++ tests/unit/backup/test_scheduler.py | 8 +- tests/unit/backup/test_scheduler_lifecycle.py | 70 +++++++++ tests/unit/budget/test_trends_currency.py | 108 ++++++++++++++ .../unit/client/test_continuous_lifecycle.py | 121 +++++++++++++++ .../escalation/test_factory_registry.py | 104 +++++++++++++ .../event_stream/test_stream_dedup.py | 116 +++++++++++++++ tests/unit/hr/pruning/test_service.py | 8 +- .../unit/hr/pruning/test_service_lifecycle.py | 89 +++++++++++ .../test_ngrok_adapter_lifecycle.py | 86 +++++++++++ .../test_replay_protection_threadsafety.py | 74 ++++++++++ .../chief_of_staff/test_monitor_lifecycle.py | 76 ++++++++++ .../providers/test_health_prober_lifecycle.py | 102 +++++++++++++ tests/unit/settings/test_backup_subscriber.py | 2 +- .../unit/tools/mcp/test_cache_threadsafety.py | 70 +++++++++ 38 files changed, 2390 insertions(+), 331 deletions(-) create mode 100644 docs/research/lgpl-postgres-driver-decision.md create mode 100644 tests/unit/api/auth/test_ticket_store_threadsafety.py create mode 100644 tests/unit/api/controllers/test_backup_required_idempotency.py create mode 100644 tests/unit/api/controllers/test_simulations_idempotency.py create mode 100644 tests/unit/backup/test_scheduler_lifecycle.py create mode 100644 tests/unit/budget/test_trends_currency.py create mode 100644 tests/unit/client/test_continuous_lifecycle.py create mode 100644 tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py create mode 100644 tests/unit/communication/event_stream/test_stream_dedup.py create mode 100644 tests/unit/hr/pruning/test_service_lifecycle.py create mode 100644 tests/unit/integrations/test_ngrok_adapter_lifecycle.py create mode 100644 tests/unit/integrations/test_replay_protection_threadsafety.py create mode 100644 tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py create mode 100644 tests/unit/providers/test_health_prober_lifecycle.py create mode 100644 tests/unit/tools/mcp/test_cache_threadsafety.py diff --git a/docs/licensing.md b/docs/licensing.md index a7222a65a4..0003edbf5d 100644 --- a/docs/licensing.md +++ b/docs/licensing.md @@ -109,6 +109,19 @@ This means: --- +## Third-Party Dependency Licenses + +SynthOrg's default install (SQLite-only) carries only permissive licenses (MIT, Apache-2.0, BSD, ISC). The optional `[postgres]` extra adds two LGPL-3.0-or-later components: + +- `psycopg[binary]`: PostgreSQL adapter +- `psycopg-pool`: connection pool for psycopg + +These are linked dynamically (separate `pip`-installable packages) and the LGPL anti-circumvention clause is satisfied by the standard `pip` replacement workflow. Operators who redistribute combined binaries that include the `postgres` extra must publish a NOTICE listing the LGPL components and preserve replacement-version flexibility. Operators using SQLite (the default) carry no LGPL obligations. + +See [`docs/research/lgpl-postgres-driver-decision.md`](research/lgpl-postgres-driver-decision.md) for the full rationale. + +--- + ## Contributor License Agreement (CLA) We require a [Contributor License Agreement](https://github.com/Aureliolo/synthorg/blob/main/.github/CLA.md) before merging external contributions. The CLA: diff --git a/docs/research/lgpl-postgres-driver-decision.md b/docs/research/lgpl-postgres-driver-decision.md new file mode 100644 index 0000000000..d4954a20fe --- /dev/null +++ b/docs/research/lgpl-postgres-driver-decision.md @@ -0,0 +1,79 @@ +--- +title: "LGPL Postgres Driver Decision" +issue: 1708 +audit_findings: + - "_audit/runs/2026-05-01-225703/findings/119-license-compat.md" + - "_audit/runs/2026-05-01-225703/findings/61-migration-parity.md" + - "_audit/runs/2026-05-01-225703/findings/127-lifecycle-lock-pattern.md" +date: 2026-05-02 +--- + +# LGPL Postgres Driver Decision + +**Issue**: #1708 (audit cleanup C: persistence, concurrency & data integrity) +**Status**: Decided 2026-05-02 + +## Bottom line + +SynthOrg keeps `psycopg[binary]==3.3.3` and `psycopg-pool==3.3.0` (both LGPL-3.0-or-later) inside the optional `[postgres]` extra. The drivers are linked dynamically, the extra is opt-in, and the BUSL-1.1 narrowed Additional Use Grant does not require redistribution under terms incompatible with LGPL. SQLite remains the default backend for new operators. + +## Context + +The 2026-05-01 codebase audit (agent 119, license compatibility) flagged the optional `postgres` extra as carrying two LGPL-3.0-or-later dependencies and recommended one of: vendor, swap, or accept-with-ADR. Three options were considered. + +| Option | What it costs | What it preserves | +|---|---|---| +| **Accept-with-ADR** (chosen) | One ADR + a one-line note in `docs/licensing.md` | Existing 50+ Postgres repository implementations, LISTEN/NOTIFY cross-instance notify channel, JSONB query layer, dual-backend conformance suite (#1505 + #1559), psycopg-pool's connection pooling and async semantics | +| Swap to `asyncpg` (BSD-3-Clause) | 2-3 days of work; rewrite of every file under `src/synthorg/persistence/postgres/`; LISTEN/NOTIFY rewire; revalidation of the conformance suite; new async-cursor / type-codec idioms | Permissive-license footprint | +| Vendor / fork psycopg | Indefinite maintenance burden; security patches lag upstream; no realistic path because psycopg is single-licensed LGPL upstream | Same as accept-with-ADR with worse long-term ergonomics | + +## Why LGPL is acceptable here + +1. **Dynamic linking, not static.** Python imports `psycopg` at runtime via the standard ABI. The LGPL-3.0-or-later anti-circumvention clauses (sections 4-6) cover redistribution of "Combined Works"; they require that operators who redistribute a combined binary must allow the LGPL portion to be replaced. Since psycopg is a separate `pip`-installable package, that condition is satisfied by default: operators can pin a different psycopg version, swap the binary wheel, or replace it entirely without touching SynthOrg's code. + +2. **Optional extra.** Operators install the postgres extra explicitly (`pip install synthorg[postgres]` or `uv sync --extra postgres`). The default install path (SQLite-only) carries no LGPL dependencies. Operators who object to LGPL distribution simply do not install the extra. + +3. **BUSL-1.1 narrowed Additional Use Grant does not conflict.** Our Additional Use Grant restricts production use by competing-use cases and 500+ employee organizations; it does not impose redistribution terms that contradict LGPL. The two licenses operate on orthogonal axes (licensing-the-source vs. distribution-of-binaries-with-replacement-rights). A SynthOrg redistributor must satisfy both: BUSL for SynthOrg's own source, LGPL for the psycopg portion of any combined binary they ship. + +4. **Industry precedent.** psycopg2 (older sibling, also LGPL) ships in major commercial-license SaaS frameworks (e.g. Sentry, GitLab CE/EE) without ever triggering compliance complications. The dynamic-linkage interpretation is settled in the Python ecosystem. + +## Consequences + +- **For operators using the `postgres` extra**: LGPL-3.0-or-later distribution terms apply to the psycopg portion of any combined binary you redistribute. Practically, this means publishing a NOTICE that lists `psycopg` and `psycopg-pool` as LGPL components and offering replacement-version flexibility (the `pip install` workflow already provides this). +- **For operators using SQLite** (the default): No LGPL components in the dependency graph. No additional obligations. +- **For SynthOrg upstream**: No code changes; no rewrite of the persistence layer; the dual-backend conformance suite remains the source of truth for SQLite ↔ Postgres parity. + +## Audit-finding resolutions + +This ADR also closes two stale findings from the same audit run: + +### #61: SQLite migration `idx_wfe_definition_revision` + +The audit reported SQLite was missing the `20260424185325_add_idx_wfe_definition_revision.sql` migration that exists in `src/synthorg/persistence/postgres/revisions/`. + +**Verified false positive**: SQLite's baseline migration (`src/synthorg/persistence/sqlite/revisions/00000000000000_baseline.sql`) already contains the index at lines 498-499: + +```sql +CREATE INDEX `idx_wfe_definition_revision` + ON `workflow_executions` (`definition_id`, `definition_revision`); +``` + +The same index also lives at `src/synthorg/persistence/sqlite/schema.sql:543`. SQLite's revision history was squashed at some point (per `docs/guides/persistence-migrations.md` §"Squash") and absorbed all prior incremental migrations into the baseline; Postgres was not squashed, so its history retains the original 2026-04-24 file. The two backends are at schema parity. `atlas migrate validate --env sqlite` and `atlas schema diff --env sqlite` both confirm parity. **No new SQL needed.** + +### #127: Lifecycle lock false positives + +The audit listed two services as missing the canonical lifecycle pattern: + +- **`src/synthorg/communication/conflict_resolution/escalation/sweeper.py`**: already compliant. `_lifecycle_lock` at line 80, `_stop_failed` at line 87, drain timeout at line 88, full canonical pattern in `start()` (lines 90-123) and `stop()` (lines 125-215). +- **`src/synthorg/hr/training/service.py`**: has no `start()` / `stop()` methods. `TrainingService` is a stateless pipeline orchestrator (with idempotency state); the canonical lifecycle pattern does not apply. The audit was misclassifying the service. + +The other six services flagged by agent 127 (health_prober, monitor, scheduler, pruning service, ngrok_adapter, continuous mode) **are** non-compliant and are addressed in this PR. + +## References + +- [LGPL-3.0-or-later text](https://www.gnu.org/licenses/lgpl-3.0.html) §4 (Combined Works), §5 (Combined Libraries) +- [GNU LGPL FAQ](https://www.gnu.org/licenses/gpl-faq.html#LGPLDistributionsAndLargerWorks) on dynamic linkage interpretation +- [BUSL-1.1 text](https://github.com/Aureliolo/synthorg/blob/main/LICENSE) and Additional Use Grant +- [`docs/licensing.md`](../licensing.md): operator-facing licensing summary +- [`docs/guides/persistence-migrations.md`](../guides/persistence-migrations.md): squash workflow context for #61 +- [`docs/reference/lifecycle-sync.md`](../reference/lifecycle-sync.md): canonical lifecycle pattern referenced for #127 diff --git a/scripts/mock_spec_baseline.txt b/scripts/mock_spec_baseline.txt index e7b4263ca6..e0197005fd 100644 --- a/scripts/mock_spec_baseline.txt +++ b/scripts/mock_spec_baseline.txt @@ -303,26 +303,34 @@ tests/unit/api/controllers/test_approvals_helpers.py:511:22 tests/unit/api/controllers/test_approvals_helpers.py:512:38 tests/unit/api/controllers/test_backup.py:73:14 tests/unit/api/controllers/test_backup.py:74:16 -tests/unit/api/controllers/test_backup.py:81:12 -tests/unit/api/controllers/test_backup.py:98:32 -tests/unit/api/controllers/test_backup.py:109:32 -tests/unit/api/controllers/test_backup.py:126:31 -tests/unit/api/controllers/test_backup.py:145:29 -tests/unit/api/controllers/test_backup.py:160:29 -tests/unit/api/controllers/test_backup.py:179:32 -tests/unit/api/controllers/test_backup.py:193:32 -tests/unit/api/controllers/test_backup.py:213:38 -tests/unit/api/controllers/test_backup.py:236:38 -tests/unit/api/controllers/test_backup.py:267:38 -tests/unit/api/controllers/test_backup.py:281:38 -tests/unit/api/controllers/test_backup.py:297:38 -tests/unit/api/controllers/test_backup.py:313:38 -tests/unit/api/controllers/test_backup.py:367:19 -tests/unit/api/controllers/test_backup.py:370:25 -tests/unit/api/controllers/test_backup.py:371:24 -tests/unit/api/controllers/test_backup.py:372:32 -tests/unit/api/controllers/test_backup.py:373:29 -tests/unit/api/controllers/test_backup.py:374:34 +tests/unit/api/controllers/test_backup.py:79:26 +tests/unit/api/controllers/test_backup.py:89:18 +tests/unit/api/controllers/test_backup.py:102:12 +tests/unit/api/controllers/test_backup.py:119:32 +tests/unit/api/controllers/test_backup.py:134:32 +tests/unit/api/controllers/test_backup.py:155:31 +tests/unit/api/controllers/test_backup.py:174:29 +tests/unit/api/controllers/test_backup.py:189:29 +tests/unit/api/controllers/test_backup.py:208:32 +tests/unit/api/controllers/test_backup.py:222:32 +tests/unit/api/controllers/test_backup.py:242:38 +tests/unit/api/controllers/test_backup.py:265:38 +tests/unit/api/controllers/test_backup.py:296:38 +tests/unit/api/controllers/test_backup.py:310:38 +tests/unit/api/controllers/test_backup.py:326:38 +tests/unit/api/controllers/test_backup.py:342:38 +tests/unit/api/controllers/test_backup.py:396:19 +tests/unit/api/controllers/test_backup.py:399:25 +tests/unit/api/controllers/test_backup.py:400:24 +tests/unit/api/controllers/test_backup.py:401:32 +tests/unit/api/controllers/test_backup.py:402:29 +tests/unit/api/controllers/test_backup.py:403:34 +tests/unit/api/controllers/test_backup_required_idempotency.py:45:14 +tests/unit/api/controllers/test_backup_required_idempotency.py:46:28 +tests/unit/api/controllers/test_backup_required_idempotency.py:47:16 +tests/unit/api/controllers/test_backup_required_idempotency.py:49:26 +tests/unit/api/controllers/test_backup_required_idempotency.py:55:12 +tests/unit/api/controllers/test_backup_required_idempotency.py:82:22 tests/unit/api/controllers/test_collaboration.py:357:23 tests/unit/api/controllers/test_company.py:108:31 tests/unit/api/controllers/test_coordination.py:77:18 @@ -412,6 +420,17 @@ tests/unit/api/controllers/test_setup_has_gpu.py:76:27 tests/unit/api/controllers/test_setup_locales.py:246:33 tests/unit/api/controllers/test_setup_locales.py:269:33 tests/unit/api/controllers/test_setup_locales.py:316:33 +tests/unit/api/controllers/test_simulations_idempotency.py:36:16 +tests/unit/api/controllers/test_simulations_idempotency.py:52:38 +tests/unit/api/controllers/test_simulations_idempotency.py:54:30 +tests/unit/api/controllers/test_simulations_idempotency.py:55:21 +tests/unit/api/controllers/test_simulations_idempotency.py:56:34 +tests/unit/api/controllers/test_simulations_idempotency.py:57:31 +tests/unit/api/controllers/test_simulations_idempotency.py:58:38 +tests/unit/api/controllers/test_simulations_idempotency.py:59:16 +tests/unit/api/controllers/test_simulations_idempotency.py:61:32 +tests/unit/api/controllers/test_simulations_idempotency.py:62:12 +tests/unit/api/controllers/test_simulations_idempotency.py:68:11 tests/unit/api/controllers/test_sse_keepalive_setting.py:28:31 tests/unit/api/controllers/test_sse_keepalive_setting.py:29:41 tests/unit/api/controllers/test_sse_revalidate.py:44:21 @@ -564,6 +583,8 @@ tests/unit/api/test_state.py:343:19 tests/unit/api/test_state.py:350:19 tests/unit/backup/test_scheduler.py:14:14 tests/unit/backup/test_scheduler.py:15:28 +tests/unit/backup/test_scheduler_lifecycle.py:21:14 +tests/unit/backup/test_scheduler_lifecycle.py:22:28 tests/unit/backup/test_service.py:21:14 tests/unit/backup/test_service.py:23:21 tests/unit/backup/test_service.py:24:22 @@ -646,6 +667,8 @@ tests/unit/communication/bus/test_nats_consumer_config.py:31:9 tests/unit/communication/bus/test_nats_consumer_config.py:33:24 tests/unit/communication/bus/test_nats_consumer_config.py:33:47 tests/unit/communication/bus/test_nats_consumer_config.py:44:12 +tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py:37:32 +tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py:37:55 tests/unit/communication/loop_prevention/test_circuit_breaker.py:293:15 tests/unit/communication/loop_prevention/test_circuit_breaker.py:294:20 tests/unit/communication/loop_prevention/test_circuit_breaker.py:316:15 @@ -2168,6 +2191,8 @@ tests/unit/meta/chief_of_staff/test_monitor.py:118:18 tests/unit/meta/chief_of_staff/test_monitor.py:137:18 tests/unit/meta/chief_of_staff/test_monitor.py:164:18 tests/unit/meta/chief_of_staff/test_monitor.py:178:23 +tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py:21:14 +tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py:22:20 tests/unit/meta/mcp/test_all_handlers_wired.py:75:11 tests/unit/meta/mcp/test_all_handlers_wired.py:76:22 tests/unit/meta/mcp/test_all_handlers_wired.py:92:14 diff --git a/src/synthorg/api/auth/ticket_store.py b/src/synthorg/api/auth/ticket_store.py index fa333bc2c5..fdcce484ac 100644 --- a/src/synthorg/api/auth/ticket_store.py +++ b/src/synthorg/api/auth/ticket_store.py @@ -5,6 +5,14 @@ uses ``time.monotonic()`` for expiry so it is immune to wall-clock adjustments. +The mutating methods (``create``, ``validate_and_consume``, +``cleanup_expired``) hold a ``threading.Lock`` so the store is safe +under both single-threaded asyncio handlers and Litestar's threadpool +dispatch. The lock spans count-and-insert, pop-and-validate, and +bulk eviction blocks where a thread switch between the read and the +mutating step would otherwise let a racing caller exceed the per-user +cap or observe a half-mutated dict. + .. note:: The store is per-process -- if the ASGI server runs multiple @@ -15,6 +23,7 @@ import math import secrets +import threading from pydantic import BaseModel, ConfigDict @@ -105,6 +114,7 @@ def __init__( self._max_pending = max_pending_per_user self._clock: Clock = clock if clock is not None else SystemClock() self._tickets: dict[str, _TicketEntry] = {} + self._lock = threading.Lock() @property def ttl_seconds(self) -> float: @@ -136,22 +146,23 @@ def create(self, user: AuthenticatedUser) -> str: Returns: URL-safe random token string. """ - now = self._clock.monotonic() - user_pending = sum( - 1 - for e in self._tickets.values() - if e.user.user_id == user.user_id and now <= e.expires_at - ) - if user_pending >= self._max_pending: - msg = f"Ticket limit exceeded for user {user.user_id}" - raise TicketLimitExceededError(msg) - - ticket = secrets.token_urlsafe(get_auth_token_bytes()) - entry = _TicketEntry( - user=user, - expires_at=self._clock.monotonic() + self._ttl, - ) - self._tickets[ticket] = entry + with self._lock: + now = self._clock.monotonic() + user_pending = sum( + 1 + for e in self._tickets.values() + if e.user.user_id == user.user_id and now <= e.expires_at + ) + if user_pending >= self._max_pending: + msg = f"Ticket limit exceeded for user {user.user_id}" + raise TicketLimitExceededError(msg) + + ticket = secrets.token_urlsafe(get_auth_token_bytes()) + entry = _TicketEntry( + user=user, + expires_at=self._clock.monotonic() + self._ttl, + ) + self._tickets[ticket] = entry logger.info( API_WS_TICKET_ISSUED, user_id=user.user_id, @@ -164,9 +175,8 @@ def validate_and_consume(self, ticket: str) -> AuthenticatedUser | None: """Validate and consume a ticket (single-use). Atomically removes the ticket via ``dict.pop`` before - checking expiry. In the single-threaded asyncio event loop, - ``dict.pop`` cannot be interleaved with another coroutine, - so concurrent calls on the same ticket are safely serialised. + checking expiry, holding ``self._lock`` so concurrent threads + racing on the same ticket cannot both succeed. Args: ticket: Raw ticket string from the client. @@ -174,7 +184,8 @@ def validate_and_consume(self, ticket: str) -> AuthenticatedUser | None: Returns: The bound ``AuthenticatedUser``, or ``None``. """ - entry = self._tickets.pop(ticket, None) + with self._lock: + entry = self._tickets.pop(ticket, None) if entry is None: logger.warning(API_WS_TICKET_INVALID, reason="not_found") return None @@ -200,19 +211,23 @@ def cleanup_expired(self) -> int: Called periodically by a background task to prevent unbounded memory growth from tickets that are requested - but never consumed. + but never consumed. Holds ``self._lock`` across the + snapshot-and-delete pass so a concurrent ``create()`` cannot + change the dict size mid-iteration. Returns: Number of entries removed. """ - now = self._clock.monotonic() - expired = [k for k, v in self._tickets.items() if now > v.expires_at] - for k in expired: - self._tickets.pop(k, None) + with self._lock: + now = self._clock.monotonic() + expired = [k for k, v in self._tickets.items() if now > v.expires_at] + for k in expired: + self._tickets.pop(k, None) + remaining = len(self._tickets) if expired: logger.info( API_WS_TICKET_CLEANUP, removed=len(expired), - remaining=len(self._tickets), + remaining=remaining, ) return len(expired) diff --git a/src/synthorg/api/controllers/backup.py b/src/synthorg/api/controllers/backup.py index d02e1f4b83..e930220784 100644 --- a/src/synthorg/api/controllers/backup.py +++ b/src/synthorg/api/controllers/backup.py @@ -86,24 +86,26 @@ async def create_backup( self, state: State, idempotency_key: Annotated[ - str | None, + str, Parameter( header="Idempotency-Key", description=( - "RFC-style retry-safe key. Same key within 24h " - "returns the cached manifest instead of starting " - "a second backup." + "RFC-style retry-safe key. Required: identical keys " + "within 24h return the cached manifest instead of " + "starting a second backup. Without a key a 5xx-driven " + "client retry could launch concurrent backups, " + "violating the at-most-one-running invariant." ), - required=False, + required=True, min_length=1, ), - ] = None, + ], ) -> ApiResponse[BackupManifest]: """Trigger a manual backup. Args: state: Application state. - idempotency_key: Optional caller-supplied retry token. + idempotency_key: Required caller-supplied retry token. Returns: Manifest of the created backup. @@ -133,27 +135,23 @@ async def _do_backup() -> BackupManifest: msg = "Backup operation failed" raise InternalServerException(msg) from exc - if idempotency_key: - outcome = await app_state.idempotency_service.run_idempotent( - scope=NotBlankStr("backup"), - key=NotBlankStr(idempotency_key), - callback=lambda: _do_backup_as_dict(_do_backup), + outcome = await app_state.idempotency_service.run_idempotent( + scope=NotBlankStr("backup"), + key=NotBlankStr(idempotency_key), + callback=lambda: _do_backup_as_dict(_do_backup), + ) + if outcome.timed_out: + # Discriminated 409 path: distinct from a callback that + # legitimately returned ``None``. + logger.warning( + IDEMPOTENCY_CLAIM_IN_FLIGHT, + scope="backup", + idempotency_key=idempotency_key, + endpoint="backup.create", ) - if outcome.timed_out: - # Discriminated 409 path: distinct from a callback - # that legitimately returned ``None``. - logger.warning( - IDEMPOTENCY_CLAIM_IN_FLIGHT, - scope="backup", - idempotency_key=idempotency_key, - endpoint="backup.create", - ) - msg = "Concurrent in-flight backup with this idempotency key" - raise ConflictError(msg) - return ApiResponse(data=BackupManifest.model_validate(outcome.result)) - - manifest = await _do_backup() - return ApiResponse(data=manifest) + msg = "Concurrent in-flight backup with this idempotency key" + raise ConflictError(msg) + return ApiResponse(data=BackupManifest.model_validate(outcome.result)) @get() async def list_backups( diff --git a/src/synthorg/api/controllers/simulations.py b/src/synthorg/api/controllers/simulations.py index f2115f8a24..3c7209a77d 100644 --- a/src/synthorg/api/controllers/simulations.py +++ b/src/synthorg/api/controllers/simulations.py @@ -274,6 +274,22 @@ async def start_simulation( """ app_state: AppState = state.app_state sim_state = app_state.client_simulation_state + # Idempotency guard (audit #133): a JetStream redelivery or + # HTTP 5xx retry of /simulations/start with the same + # ``simulation_id`` would otherwise spawn a second runner that + # races the first on ``simulation_store.update_status``, + # corrupting metrics with last-write-wins. Reject the second + # request with HTTP 409 Conflict so the caller can fall back + # to ``GET /simulations/{id}`` to observe the in-flight run. + with contextlib.suppress(KeyError): + existing = await sim_state.simulation_store.get(data.config.simulation_id) + msg = ( + f"Simulation {data.config.simulation_id!r} already exists " + f"(status={existing.status!r}); cannot start a second runner " + "for the same id" + ) + raise ConflictError(msg) + record = SimulationRecord( simulation_id=data.config.simulation_id, config=data.config, diff --git a/src/synthorg/backup/scheduler.py b/src/synthorg/backup/scheduler.py index 5ae41c45cb..9eb3d7ce19 100644 --- a/src/synthorg/backup/scheduler.py +++ b/src/synthorg/backup/scheduler.py @@ -1,11 +1,10 @@ """Backup scheduler -- periodic background backup task.""" import asyncio -import contextlib from typing import TYPE_CHECKING from synthorg.backup.models import BackupTrigger -from synthorg.observability import get_logger +from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.backup import ( BACKUP_FAILED, BACKUP_SCHEDULER_RESCHEDULED, @@ -33,39 +32,104 @@ def __init__(self, service: BackupService, interval_hours: int) -> None: self._interval_seconds = interval_hours * 3600 self._task: asyncio.Task[None] | None = None self._wake_event = asyncio.Event() + # Per ``docs/reference/lifecycle-sync.md``: dedicated + # lifecycle primitives, drain timeout, unrestartable flag. + self._stop_event: asyncio.Event = asyncio.Event() + self._lifecycle_lock: asyncio.Lock = asyncio.Lock() + self._stop_failed: bool = False + self._stop_drain_timeout_seconds: float = 30.0 @property def is_running(self) -> bool: """Whether the scheduler loop is currently active.""" return self._task is not None and not self._task.done() - def start(self) -> None: + async def start(self) -> None: """Start the background scheduler loop. - Creates an ``asyncio.Task`` running ``_run_loop``. - No-op if already running. + Creates an ``asyncio.Task`` running ``_run_loop``. Idempotent + + concurrent-safe per the canonical lifecycle pattern. + Refuses to start if a previous ``stop()`` exceeded the drain + deadline (the orphan task may still own the backup lock). """ - if self.is_running: - return - self._wake_event.clear() - self._task = asyncio.create_task( - self._run_loop(), - name="backup-scheduler", - ) - logger.info( - BACKUP_SCHEDULER_STARTED, - interval_hours=self._interval_seconds // 3600, - ) + async with self._lifecycle_lock: + if self._stop_failed: + msg = ( + "BackupScheduler is unrestartable after a " + "timed-out stop; construct a fresh scheduler instead" + ) + logger.warning( + BACKUP_FAILED, + error=msg, + note="unrestartable", + ) + raise RuntimeError(msg) + if self.is_running: + return + self._wake_event.clear() + self._stop_event.clear() + self._task = asyncio.create_task( + self._run_loop(), + name="backup-scheduler", + ) + logger.info( + BACKUP_SCHEDULER_STARTED, + interval_hours=self._interval_seconds // 3600, + ) async def stop(self) -> None: - """Cancel the background scheduler and wait for it to finish.""" - if self._task is None: - return - self._task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._task - self._task = None - logger.info(BACKUP_SCHEDULER_STOPPED) + """Cancel the background scheduler and wait for it to finish. + + Drain is shielded with a hard deadline; on timeout the + scheduler is marked unrestartable so a subsequent ``start()`` + cannot stack a second loop on top of an orphan task. + """ + async with self._lifecycle_lock: + self._stop_event.set() + self._wake_event.set() + task = self._task + if task is None: + return + task.cancel() + + async def _drain() -> None: + try: + await task + except asyncio.CancelledError: + pass + except MemoryError, RecursionError: + raise + except Exception as exc: + logger.warning( + BACKUP_FAILED, + error_type=type(exc).__name__, + error=safe_error_description(exc), + note="shutdown", + ) + + drain_task: asyncio.Task[None] = asyncio.create_task(_drain()) + try: + await asyncio.wait_for( + asyncio.shield(drain_task), + timeout=self._stop_drain_timeout_seconds, + ) + except TimeoutError: + self._stop_failed = True + logger.error( # noqa: TRY400 + BACKUP_FAILED, + error=( + "stop exceeded hard deadline; scheduler marked unrestartable" + ), + timeout_seconds=self._stop_drain_timeout_seconds, + ) + raise + self._task = None + logger.info(BACKUP_SCHEDULER_STOPPED) + # Recreate primitives outside the (released) lock so a + # subsequent ``start()`` on a different event loop can rebind. + self._lifecycle_lock = asyncio.Lock() + self._stop_event = asyncio.Event() + self._wake_event = asyncio.Event() def reschedule(self, interval_hours: int) -> None: """Update the interval and interrupt the current sleep. @@ -92,24 +156,34 @@ def reschedule(self, interval_hours: int) -> None: async def _run_loop(self) -> None: """Sleep-and-backup loop. - Logs and suppresses errors except ``MemoryError`` and - ``RecursionError``. + Honors ``self._stop_event``: when set, the loop exits cleanly + without firing another backup. ``self._wake_event`` still + interrupts the sleep for ``reschedule()``. """ - while True: + while not self._stop_event.is_set(): self._wake_event.clear() - with contextlib.suppress(TimeoutError): + try: await asyncio.wait_for( self._wake_event.wait(), timeout=self._interval_seconds, ) + except TimeoutError: + pass + except asyncio.CancelledError: + raise + if self._stop_event.is_set(): + return logger.debug(BACKUP_SCHEDULER_TICK) try: await self._service.create_backup(BackupTrigger.SCHEDULED) except MemoryError, RecursionError: raise - except Exception: - logger.error( + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning( BACKUP_FAILED, - error="Scheduled backup failed", - exc_info=True, + error_type=type(exc).__name__, + error=safe_error_description(exc), + note="scheduled_run", ) diff --git a/src/synthorg/backup/service.py b/src/synthorg/backup/service.py index 4e06e9feb3..4de70ebbfc 100644 --- a/src/synthorg/backup/service.py +++ b/src/synthorg/backup/service.py @@ -94,7 +94,7 @@ def on_shutdown(self) -> bool: async def start(self) -> None: """Start the backup scheduler if backups are enabled.""" if self._config.enabled: - self._scheduler.start() + await self._scheduler.start() async def stop(self) -> None: """Stop the backup scheduler.""" diff --git a/src/synthorg/budget/trends.py b/src/synthorg/budget/trends.py index d9a75f1cee..5b0f48e012 100644 --- a/src/synthorg/budget/trends.py +++ b/src/synthorg/budget/trends.py @@ -14,6 +14,7 @@ from pydantic import AwareDatetime, BaseModel, ConfigDict, Field +from synthorg.budget._tracker_helpers import _assert_single_currency from synthorg.constants import BUDGET_ROUNDING_PRECISION if TYPE_CHECKING: @@ -231,7 +232,13 @@ def bucket_cost_records( Returns: Sorted tuple of data points, one per bucket. + + Raises: + MixedCurrencyAggregationError: If *records* span multiple + currencies. Summing across currencies would produce a + meaningless monetary total. """ + _assert_single_currency(records) bucket_starts = generate_bucket_starts(start, end, bucket_size) sums: dict[datetime, list[float]] = defaultdict(list) @@ -431,6 +438,13 @@ def project_daily_spend( Returns: Budget forecast with daily projections. + + Raises: + MixedCurrencyAggregationError: If *records* span multiple + currencies. The forecast would conflate currencies on the + avg-daily-spend computation and return a meaningless + projection; raising at the boundary surfaces the bug at + the call site. """ today = (now or datetime.now(UTC)).date() @@ -443,6 +457,7 @@ def project_daily_spend( avg_daily_spend=0.0, ) + _assert_single_currency(records) avg_daily, confidence, _ = _compute_daily_spend(records) projections = _build_projections(avg_daily, horizon_days, today) projected_total = round( diff --git a/src/synthorg/client/continuous.py b/src/synthorg/client/continuous.py index 9009a4746a..9a68e8821d 100644 --- a/src/synthorg/client/continuous.py +++ b/src/synthorg/client/continuous.py @@ -1,4 +1,18 @@ -"""Continuous (always-on) simulation mode.""" +"""Continuous (always-on) simulation mode. + +``ContinuousMode`` is an **in-place runner**: ``start()`` executes the +simulation loop on the calling coroutine and only returns once +``stop()`` has been signalled. This shape differs from the canonical +service lifecycle pattern (``docs/reference/lifecycle-sync.md``) +where ``start()`` spawns a background task and returns immediately. +The canonical pattern's drain timeout / unrestartable flag therefore +does not apply here -- there is no orphan task to drain post-stop. + +What carries over from the canonical pattern is the +``self._lifecycle_lock``: it serialises the running-flag check and +spans the full body of ``start()`` and ``stop()`` so concurrent +callers cannot both observe ``_running=False`` and proceed. +""" import asyncio from collections import deque @@ -43,7 +57,12 @@ def __init__( self._config = config self._runner = runner self._stop_event = asyncio.Event() - self._lock = asyncio.Lock() + # Per ``docs/reference/lifecycle-sync.md`` the lifecycle lock + # is named distinctly from any hot-path lock so a hot-path + # contention cannot block lifecycle transitions. ContinuousMode + # has no hot-path lock today, but the rename keeps the + # codebase uniform across services. + self._lifecycle_lock = asyncio.Lock() self._runs_completed = 0 self._running = False @@ -70,7 +89,16 @@ async def start( if not self._config.enabled: logger.debug(CONTINUOUS_MODE_DISABLED) return [] - async with self._lock: + # Acquire the lifecycle lock briefly to gate the ``_running`` + # transition. Unlike a service that spawns a background + # task, ``start()`` runs the loop on the calling coroutine, + # so the lock does not need to span the loop body -- it only + # needs to make the "is the runner already busy?" check + # atomic against concurrent callers. Holding the lock for + # the full loop would deadlock a second caller: it would + # queue on the lock until the first finished, then enter and + # find ``_running=False``, never observing the conflict. + async with self._lifecycle_lock: if self._running: msg = "ContinuousMode is already running" raise RuntimeError(msg) @@ -97,10 +125,17 @@ async def start( except TimeoutError: continue finally: - async with self._lock: + async with self._lifecycle_lock: self._running = False return list(results) def stop(self) -> None: - """Signal continuous mode to stop after the current run.""" + """Signal continuous mode to stop after the current run. + + Synchronous on purpose: only sets the stop event so a caller + outside the loop can signal teardown without contending with + the running ``start()`` coroutine. The lifecycle lock is not + acquired here because the lock guards only the ``_running`` + flag transition, not the long-lived loop body. + """ self._stop_event.set() diff --git a/src/synthorg/communication/conflict_resolution/escalation/factory.py b/src/synthorg/communication/conflict_resolution/escalation/factory.py index ddae98f95d..e991540789 100644 --- a/src/synthorg/communication/conflict_resolution/escalation/factory.py +++ b/src/synthorg/communication/conflict_resolution/escalation/factory.py @@ -1,9 +1,18 @@ -"""Factories for the escalation queue backend and decision processor.""" +"""Factories for the escalation queue backend and decision processor. +Both factories dispatch via small registry maps (per audit #69) so +adding a new backend or decision strategy is a single registry entry +rather than a new branch in an if/elif chain. The shape mirrors +``synthorg.persistence.registry.PersistenceBackendRegistry`` and the +``match/case`` dispatch in ``synthorg.communication.bus``. +""" + +from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING from synthorg.communication.conflict_resolution.escalation.config import ( - EscalationQueueConfig, # noqa: TC001 + EscalationQueueConfig, ) from synthorg.communication.conflict_resolution.escalation.in_memory_store import ( InMemoryEscalationStore, @@ -17,8 +26,8 @@ WinnerSelectProcessor, ) from synthorg.communication.conflict_resolution.escalation.protocol import ( - DecisionProcessor, # noqa: TC001 - EscalationQueueStore, # noqa: TC001 + DecisionProcessor, + EscalationQueueStore, ) from synthorg.observability import get_logger from synthorg.observability.events.api import API_APP_STARTUP @@ -33,6 +42,13 @@ logger = get_logger(__name__) +type _QueueStoreFactory = Callable[ + [EscalationQueueConfig, "PersistenceBackend | None"], + EscalationQueueStore, +] +type _DecisionProcessorFactory = Callable[[], DecisionProcessor] + + def _require_persistence( config_backend: str, persistence: PersistenceBackend | None, @@ -64,6 +80,46 @@ def _require_persistence( return persistence +def _build_memory_store( + config: EscalationQueueConfig, + persistence: PersistenceBackend | None, +) -> EscalationQueueStore: + del config, persistence + return InMemoryEscalationStore() + + +def _build_sqlite_store( + config: EscalationQueueConfig, + persistence: PersistenceBackend | None, +) -> EscalationQueueStore: + del config + backend = _require_persistence("sqlite", persistence) + return backend.build_escalations() + + +def _build_postgres_store( + config: EscalationQueueConfig, + persistence: PersistenceBackend | None, +) -> EscalationQueueStore: + backend = _require_persistence("postgres", persistence) + # Pass the notify channel only when cross-instance notify is + # enabled so the repo's NOTIFY publishing is a true no-op for + # single-worker deployments. + notify_channel: str | None = None + if config.cross_instance_notify in {"auto", "on"}: + notify_channel = config.notify_channel + return backend.build_escalations(notify_channel=notify_channel) + + +_QUEUE_STORE_FACTORIES: Mapping[str, _QueueStoreFactory] = MappingProxyType( + { + "memory": _build_memory_store, + "sqlite": _build_sqlite_store, + "postgres": _build_postgres_store, + }, +) + + def build_escalation_queue_store( config: EscalationQueueConfig, persistence: PersistenceBackend | None = None, @@ -80,25 +136,18 @@ def build_escalation_queue_store( Raises: ValueError: ``backend`` is ``sqlite`` or ``postgres`` but the - persistence backend is missing or of a mismatched type. + persistence backend is missing or of a mismatched type, or + ``backend`` is not a registered key. """ - if config.backend == "memory": - return InMemoryEscalationStore() - if config.backend == "sqlite": - store_backend = _require_persistence("sqlite", persistence) - return store_backend.build_escalations() - if config.backend == "postgres": - store_backend = _require_persistence("postgres", persistence) - # Pass the notify channel only when cross-instance notify is - # enabled so the repo's NOTIFY publishing is a true no-op for - # single-worker deployments. - notify_channel: str | None = None - if config.cross_instance_notify in {"auto", "on"}: - notify_channel = config.notify_channel - return store_backend.build_escalations(notify_channel=notify_channel) - # Defensive: the Literal union is exhaustive today. - msg = f"Unknown escalation queue backend: {config.backend!r}" # type: ignore[unreachable] - raise ValueError(msg) + factory = _QUEUE_STORE_FACTORIES.get(config.backend) + if factory is None: + available = sorted(_QUEUE_STORE_FACTORIES) or ["(none)"] + msg = ( + f"Unknown escalation queue backend: {config.backend!r}. " + f"Registered backends: {', '.join(available)}" + ) + raise ValueError(msg) + return factory(config, persistence) def build_escalation_notify_subscriber( @@ -171,6 +220,16 @@ def build_escalation_notify_subscriber( ) +_DECISION_PROCESSOR_FACTORIES: Mapping[str, _DecisionProcessorFactory] = ( + MappingProxyType( + { + "winner": WinnerSelectProcessor, + "hybrid": HybridDecisionProcessor, + }, + ) +) + + def build_decision_processor( config: EscalationQueueConfig, ) -> DecisionProcessor: @@ -182,11 +241,16 @@ def build_decision_processor( Returns: The concrete decision processor selected by ``config.decision_strategy``. + + Raises: + ValueError: ``decision_strategy`` is not a registered key. """ - if config.decision_strategy == "winner": - return WinnerSelectProcessor() - if config.decision_strategy == "hybrid": - return HybridDecisionProcessor() - # Defensive: the Literal union is exhaustive today. - msg = f"Unknown decision_strategy: {config.decision_strategy!r}" # type: ignore[unreachable] - raise ValueError(msg) + factory = _DECISION_PROCESSOR_FACTORIES.get(config.decision_strategy) + if factory is None: + available = sorted(_DECISION_PROCESSOR_FACTORIES) or ["(none)"] + msg = ( + f"Unknown decision_strategy: {config.decision_strategy!r}. " + f"Registered strategies: {', '.join(available)}" + ) + raise ValueError(msg) + return factory() diff --git a/src/synthorg/communication/event_stream/stream.py b/src/synthorg/communication/event_stream/stream.py index db1018e7fb..b2a4b65694 100644 --- a/src/synthorg/communication/event_stream/stream.py +++ b/src/synthorg/communication/event_stream/stream.py @@ -8,6 +8,7 @@ import asyncio import contextlib +from collections import OrderedDict from datetime import UTC, datetime from uuid import uuid4 @@ -15,14 +16,18 @@ AgUiEventType, StreamEvent, ) +from synthorg.core.clock import Clock, SystemClock from synthorg.observability import get_logger from synthorg.observability.events.event_stream import ( + EVENT_STREAM_HUB_PUBLISH_DEDUPED, EVENT_STREAM_HUB_PUBLISH_FAILED, ) logger = get_logger(__name__) _DEFAULT_MAX_QUEUE_SIZE = 256 +_DEFAULT_DEDUP_TTL_SECONDS = 60.0 +_DEFAULT_DEDUP_MAX_ENTRIES_PER_SESSION = 1024 class EventStreamHub: @@ -38,14 +43,36 @@ class EventStreamHub: publisher). """ - __slots__ = ("_lock", "_max_queue_size", "_subscribers") + __slots__ = ( + "_clock", + "_dedup_max_entries_per_session", + "_dedup_ttl_seconds", + "_lock", + "_max_queue_size", + "_seen_event_ids", + "_subscribers", + ) def __init__( self, max_queue_size: int = _DEFAULT_MAX_QUEUE_SIZE, + *, + dedup_ttl_seconds: float = _DEFAULT_DEDUP_TTL_SECONDS, + dedup_max_entries_per_session: int = _DEFAULT_DEDUP_MAX_ENTRIES_PER_SESSION, + clock: Clock | None = None, ) -> None: self._max_queue_size = max_queue_size + self._dedup_ttl_seconds = dedup_ttl_seconds + self._dedup_max_entries_per_session = dedup_max_entries_per_session + self._clock: Clock = clock if clock is not None else SystemClock() self._subscribers: dict[str, list[asyncio.Queue[StreamEvent]]] = {} + # Per-session insertion-ordered map of ``event.id`` -> + # ``monotonic_seen_at``. Bounded per session and TTL-evicted on + # publish so a long-lived session cannot grow the dedup window + # without bound. Audit #133: retried publishes (e.g. webhook + # handler that catches a transient publish failure and retries) + # would otherwise emit the same event twice to all subscribers. + self._seen_event_ids: dict[str, OrderedDict[str, float]] = {} self._lock = asyncio.Lock() async def subscribe( @@ -93,6 +120,12 @@ async def publish(self, event: StreamEvent) -> None: Best-effort: if a subscriber queue is full, the event is dropped for that subscriber (never blocks the publisher). + Deduplicates by ``event.id`` within a per-session sliding + window so an upstream retry (e.g. webhook handler that + catches a transient publish failure and retries) cannot + double-deliver. The first publish wins; subsequent publishes + with the same id within the TTL are skipped and logged. + The subscriber list is snapshotted under the lock and ``put_nowait`` is invoked outside the lock so a slow consumer's ``QueueFull`` warning cannot serialize other publishers behind @@ -101,7 +134,17 @@ async def publish(self, event: StreamEvent) -> None: Args: event: The stream event to publish. """ + now = self._clock.monotonic() async with self._lock: + if self._is_duplicate_locked(event, now): + logger.warning( + EVENT_STREAM_HUB_PUBLISH_DEDUPED, + session_id=event.session_id, + event_id=event.id, + ttl_seconds=self._dedup_ttl_seconds, + ) + return + self._record_published_locked(event, now) queues_snapshot = list(self._subscribers.get(event.session_id, ())) if not queues_snapshot: return @@ -116,6 +159,43 @@ async def publish(self, event: StreamEvent) -> None: note="Subscriber queue full, event dropped", ) + def _is_duplicate_locked(self, event: StreamEvent, now: float) -> bool: + """Return ``True`` if *event* was already published within the TTL. + + Caller must hold ``self._lock``. Evicts expired entries from + the per-session window before testing membership so a stale + entry does not falsely register as a duplicate. + """ + seen = self._seen_event_ids.get(event.session_id) + if seen is None: + return False + cutoff = now - self._dedup_ttl_seconds + while seen: + oldest_id, oldest_ts = next(iter(seen.items())) + if oldest_ts >= cutoff: + break + del seen[oldest_id] + if not seen: + del self._seen_event_ids[event.session_id] + return False + return event.id in seen + + def _record_published_locked( + self, + event: StreamEvent, + now: float, + ) -> None: + """Record *event* as published in the per-session dedup window. + + Caller must hold ``self._lock``. Bounds each session's window + by ``self._dedup_max_entries_per_session`` so a single noisy + session cannot exhaust memory. + """ + seen = self._seen_event_ids.setdefault(event.session_id, OrderedDict()) + seen[event.id] = now + while len(seen) > self._dedup_max_entries_per_session: + seen.popitem(last=False) + async def publish_raw( self, *, diff --git a/src/synthorg/hr/pruning/service.py b/src/synthorg/hr/pruning/service.py index 02bfcf0c01..5a45d7a80e 100644 --- a/src/synthorg/hr/pruning/service.py +++ b/src/synthorg/hr/pruning/service.py @@ -13,7 +13,6 @@ """ import asyncio -import contextlib from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING from uuid import uuid4 @@ -95,6 +94,14 @@ def __init__( # noqa: PLR0913 self._on_notification = on_notification self._task: asyncio.Task[None] | None = None self._wake_event = asyncio.Event() + self._stop_event: asyncio.Event = asyncio.Event() + # Per ``docs/reference/lifecycle-sync.md``: dedicated lifecycle + # primitives, kept distinct from the hot-path + # ``_processing_lock`` so a concurrent pruning cycle cannot + # block lifecycle transitions. + self._lifecycle_lock: asyncio.Lock = asyncio.Lock() + self._stop_failed: bool = False + self._stop_drain_timeout_seconds: float = 30.0 self._pending_requests: dict[str, PruningRequest] = {} self._completed: list[PruningRecord] = [] self._processed_approval_ids: set[str] = set() @@ -125,26 +132,85 @@ def is_running(self) -> bool: """Whether the scheduler loop is currently active.""" return self._task is not None and not self._task.done() - def start(self) -> None: - """Start the background pruning scheduler.""" - if self.is_running: - return - self._wake_event.clear() - self._task = asyncio.create_task( - self._run_loop(), - name="pruning-scheduler", - ) - logger.info(HR_PRUNING_SCHEDULER_STARTED) + async def start(self) -> None: + """Start the background pruning scheduler. + + Idempotent + concurrent-safe per ``docs/reference/lifecycle-sync.md``: + serialises on ``self._lifecycle_lock`` so concurrent callers + cannot double-spawn the run loop. + """ + async with self._lifecycle_lock: + if self._stop_failed: + msg = ( + "PruningService is unrestartable after a " + "timed-out stop; construct a fresh service instead" + ) + logger.warning( + HR_PRUNING_POLICY_ERROR, + error=msg, + note="unrestartable", + ) + raise RuntimeError(msg) + if self.is_running: + return + self._wake_event.clear() + self._stop_event.clear() + self._task = asyncio.create_task( + self._run_loop(), + name="pruning-scheduler", + ) + logger.info(HR_PRUNING_SCHEDULER_STARTED) async def stop(self) -> None: - """Stop the background scheduler gracefully.""" - if self._task is None: - return - self._task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._task - self._task = None - logger.info(HR_PRUNING_SCHEDULER_STOPPED) + """Stop the background scheduler gracefully. + + Drain is shielded with a hard deadline; on timeout the service + is marked unrestartable. + """ + async with self._lifecycle_lock: + self._stop_event.set() + self._wake_event.set() + task = self._task + if task is None: + return + task.cancel() + + async def _drain() -> None: + try: + await task + except asyncio.CancelledError: + pass + except MemoryError, RecursionError: + raise + except Exception as exc: + logger.warning( + HR_PRUNING_POLICY_ERROR, + error_type=type(exc).__name__, + error=safe_error_description(exc), + note="shutdown", + ) + + drain_task: asyncio.Task[None] = asyncio.create_task(_drain()) + try: + await asyncio.wait_for( + asyncio.shield(drain_task), + timeout=self._stop_drain_timeout_seconds, + ) + except TimeoutError: + self._stop_failed = True + logger.error( # noqa: TRY400 + HR_PRUNING_POLICY_ERROR, + error=("stop exceeded hard deadline; service marked unrestartable"), + timeout_seconds=self._stop_drain_timeout_seconds, + ) + raise + self._task = None + logger.info(HR_PRUNING_SCHEDULER_STOPPED) + # Recreate primitives outside the (released) lock so a + # subsequent ``start()`` on a different event loop can rebind. + self._lifecycle_lock = asyncio.Lock() + self._stop_event = asyncio.Event() + self._wake_event = asyncio.Event() def wake(self) -> None: """Trigger an early pruning cycle.""" @@ -676,18 +742,31 @@ def _handle_rejected(self, item: ApprovalItem) -> None: # ── Scheduler Loop ──────────────────────────────────────── async def _run_loop(self) -> None: - """Sleep-and-check scheduler loop.""" - while True: - with contextlib.suppress(TimeoutError): + """Sleep-and-check scheduler loop. + + Honors ``self._stop_event`` so the canonical ``stop()`` drain + wakes the loop cooperatively. ``self._wake_event`` continues + to interrupt the sleep for ad-hoc ``wake()`` triggers. + """ + while not self._stop_event.is_set(): + try: await asyncio.wait_for( self._wake_event.wait(), timeout=self._config.evaluation_interval_seconds, ) + except TimeoutError: + pass + except asyncio.CancelledError: + raise self._wake_event.clear() + if self._stop_event.is_set(): + return try: await self.run_pruning_cycle() except MemoryError, RecursionError: raise + except asyncio.CancelledError: + raise except Exception as exc: # Drop ``logger.exception`` -- the scheduler-loop # traceback can carry FiringRequest fields and diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index 42f6293be8..1429a82a33 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -54,6 +54,14 @@ def __init__( self._port = port self._public_url: str | None = None self._tunnel: object | None = None + # Per ``docs/reference/lifecycle-sync.md``: a dedicated + # lifecycle lock serialises ``start`` / ``stop``. No drain + # timeout / unrestartable flag here because the adapter does + # not own a background task; it forwards to pyngrok in a + # worker thread and the lock is sufficient to prevent two + # ``start()`` calls from racing on the single-tunnel + # invariant. + self._lifecycle_lock: asyncio.Lock = asyncio.Lock() async def start(self) -> str: """Start the ngrok tunnel. @@ -66,33 +74,39 @@ async def start(self) -> str: ngrok service down, etc.). ``pyngrok`` itself is a required runtime dependency so an ImportError here is a build / install bug rather than a runtime concern. + RuntimeError: If a tunnel is already active on this + adapter instance. """ - auth_token = os.environ.get(self._auth_token_env, "").strip() - if auth_token: - conf.get_default().auth_token = auth_token - - try: - tunnel = await asyncio.to_thread(ngrok.connect, self._port, "http") - self._tunnel = tunnel - self._public_url = str(tunnel.public_url) - except Exception as exc: - # ngrok auth token env var may be echoed in exception - # messages; scrub + drop traceback. - logger.warning( - TUNNEL_ERROR, - error_type=type(exc).__name__, - error=safe_error_description(exc), + async with self._lifecycle_lock: + if self._tunnel is not None: + msg = "ngrok tunnel already active on this adapter" + raise RuntimeError(msg) + auth_token = os.environ.get(self._auth_token_env, "").strip() + if auth_token: + conf.get_default().auth_token = auth_token + + try: + tunnel = await asyncio.to_thread(ngrok.connect, self._port, "http") + self._tunnel = tunnel + self._public_url = str(tunnel.public_url) + except Exception as exc: + # ngrok auth token env var may be echoed in exception + # messages; scrub + drop traceback. + logger.warning( + TUNNEL_ERROR, + error_type=type(exc).__name__, + error=safe_error_description(exc), + ) + msg = f"Failed to start ngrok tunnel: {type(exc).__name__}" + raise TunnelError(msg) from exc + + logger.info( + TUNNEL_STARTED, + public_url=self._public_url, + port=self._port, + note="tunnel exposes localhost publicly", ) - msg = f"Failed to start ngrok tunnel: {type(exc).__name__}" - raise TunnelError(msg) from exc - - logger.info( - TUNNEL_STARTED, - public_url=self._public_url, - port=self._port, - note="tunnel exposes localhost publicly", - ) - return self._public_url + return self._public_url async def stop(self) -> None: """Stop the ngrok tunnel (best-effort cleanup). @@ -105,20 +119,21 @@ async def stop(self) -> None: anyway, and retaining the handle would block subsequent ``start()`` calls on this adapter instance. """ - if self._tunnel is None: - return - try: - await asyncio.to_thread(ngrok.disconnect, self._public_url) - except Exception as exc: - logger.warning( - TUNNEL_ERROR, - phase="disconnect", - error_type=type(exc).__name__, - error=safe_error_description(exc), - ) - self._tunnel = None - self._public_url = None - logger.info(TUNNEL_STOPPED) + async with self._lifecycle_lock: + if self._tunnel is None: + return + try: + await asyncio.to_thread(ngrok.disconnect, self._public_url) + except Exception as exc: + logger.warning( + TUNNEL_ERROR, + phase="disconnect", + error_type=type(exc).__name__, + error=safe_error_description(exc), + ) + self._tunnel = None + self._public_url = None + logger.info(TUNNEL_STOPPED) async def get_url(self) -> str | None: """Return the current public URL, or ``None`` if stopped.""" diff --git a/src/synthorg/integrations/webhooks/replay_protection.py b/src/synthorg/integrations/webhooks/replay_protection.py index e7c2a2b96e..42b53566a8 100644 --- a/src/synthorg/integrations/webhooks/replay_protection.py +++ b/src/synthorg/integrations/webhooks/replay_protection.py @@ -2,10 +2,18 @@ Prevents replay attacks by tracking nonces and validating timestamps within a configurable window. + +The mutating ``check`` method holds a ``threading.Lock`` so concurrent +threadpool-dispatched webhook handlers cannot both pass the nonce +duplicate test and insert the same nonce. Without the lock, two +identical webhook deliveries arriving simultaneously could each see +the nonce as fresh (line 192) and both proceed (line 199), losing the +replay-protection guarantee. """ import hashlib import math +import threading from collections import OrderedDict from synthorg.core.clock import Clock, SystemClock @@ -81,6 +89,7 @@ def __init__( self._max_entries = max_entries self._seen: OrderedDict[str, float] = OrderedDict() self._clock: Clock = clock if clock is not None else SystemClock() + self._lock = threading.Lock() def check_freshness(self, timestamp: float | None) -> bool: """Validate timestamp staleness only (no nonce dedup). @@ -119,7 +128,7 @@ def check_freshness(self, timestamp: float | None) -> bool: return False return True - def check( + def check( # noqa: PLR0911 self, *, nonce: str | None, @@ -168,43 +177,55 @@ def check( ) return False - self._evict(now) - - if nonce is not None: - # Reject oversized nonces before touching the cache. - # An attacker who could send arbitrarily long nonces - # would otherwise be able to make each hash computation - # increasingly expensive even though the cache entry - # itself is fixed-size. - if len(nonce) > MAX_NONCE_CHARS: - logger.warning( - WEBHOOK_REPLAY_DETECTED, - reason="nonce exceeds max size", - nonce_length=len(nonce), - max_nonce_chars=MAX_NONCE_CHARS, - ) - return False - # Store a fixed-size SHA-256 digest instead of the raw - # attacker-controlled string. Bounds per-entry memory - # independent of nonce length and removes any concern - # about echoing the nonce back in log output below. - key = _fingerprint_nonce(nonce) + if nonce is None: + with self._lock: + self._evict_locked(now) + return True + + # Reject oversized nonces before touching the cache. + # An attacker who could send arbitrarily long nonces + # would otherwise be able to make each hash computation + # increasingly expensive even though the cache entry + # itself is fixed-size. + if len(nonce) > MAX_NONCE_CHARS: + logger.warning( + WEBHOOK_REPLAY_DETECTED, + reason="nonce exceeds max size", + nonce_length=len(nonce), + max_nonce_chars=MAX_NONCE_CHARS, + ) + return False + + # Store a fixed-size SHA-256 digest instead of the raw + # attacker-controlled string. Bounds per-entry memory + # independent of nonce length and removes any concern + # about echoing the nonce back in log output below. + key = _fingerprint_nonce(nonce) + with self._lock: + self._evict_locked(now) if key in self._seen: - logger.warning( - WEBHOOK_REPLAY_DETECTED, - reason="duplicate nonce", - nonce_fingerprint=key[:16], - ) - return False - self._seen[key] = now - # Bound the store: evict oldest insertion(s) if over limit. - while len(self._seen) > self._max_entries: - self._seen.popitem(last=False) + duplicate = True + else: + duplicate = False + self._seen[key] = now + # Bound the store: evict oldest insertion(s) if over limit. + while len(self._seen) > self._max_entries: + self._seen.popitem(last=False) + if duplicate: + logger.warning( + WEBHOOK_REPLAY_DETECTED, + reason="duplicate nonce", + nonce_fingerprint=key[:16], + ) + return False return True - def _evict(self, now: float) -> None: - """Remove nonces older than the window.""" + def _evict_locked(self, now: float) -> None: + """Remove nonces older than the window. + + Caller must hold ``self._lock``. + """ cutoff = now - self._window # OrderedDict preserves insertion order; stop at the first # non-expired entry since later insertions are always newer. diff --git a/src/synthorg/meta/chief_of_staff/monitor.py b/src/synthorg/meta/chief_of_staff/monitor.py index da4ec1e1fc..4ac29649da 100644 --- a/src/synthorg/meta/chief_of_staff/monitor.py +++ b/src/synthorg/meta/chief_of_staff/monitor.py @@ -7,11 +7,10 @@ """ import asyncio -import contextlib from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING -from synthorg.observability import get_logger +from synthorg.observability import get_logger, safe_error_description from synthorg.observability.background_tasks import log_task_exceptions from synthorg.observability.events.chief_of_staff import ( COS_INFLECTION_CHECK_FAILED, @@ -62,37 +61,108 @@ def __init__( self._interval_s = check_interval_minutes * 60 self._last_snapshot: OrgSignalSnapshot | None = None self._task: asyncio.Task[None] | None = None + # Per ``docs/reference/lifecycle-sync.md`` the lifecycle + # primitives are constructed eagerly so a racing ``stop()`` + # cannot observe a half-published lock attribute. + self._stop_event: asyncio.Event = asyncio.Event() + self._lifecycle_lock: asyncio.Lock = asyncio.Lock() + self._stop_failed: bool = False + self._stop_drain_timeout_seconds: float = 30.0 async def start(self) -> None: - """Start the background monitoring loop.""" - if self._task is not None: - return - self._task = asyncio.create_task( - self._loop(), - name="cos-monitor-loop", - ) - self._task.add_done_callback( - log_task_exceptions(logger, COS_MONITOR_LOOP_DIED), - ) - logger.info( - COS_MONITOR_STARTED, - interval_minutes=self._interval_s // 60, - ) + """Start the background monitoring loop. + + Idempotent + concurrent-safe per the canonical lifecycle + pattern: serialises on ``self._lifecycle_lock`` so concurrent + callers cannot both observe ``_task is None`` and double-spawn. + """ + async with self._lifecycle_lock: + if self._stop_failed: + msg = ( + "OrgInflectionMonitor is unrestartable after a " + "timed-out stop; construct a fresh monitor instead" + ) + logger.warning( + COS_INFLECTION_CHECK_FAILED, + error=msg, + note="unrestartable", + ) + raise RuntimeError(msg) + if self._task is not None and not self._task.done(): + return + self._stop_event.clear() + self._task = asyncio.create_task( + self._loop(), + name="cos-monitor-loop", + ) + self._task.add_done_callback( + log_task_exceptions(logger, COS_MONITOR_LOOP_DIED), + ) + logger.info( + COS_MONITOR_STARTED, + interval_minutes=self._interval_s // 60, + ) async def stop(self) -> None: - """Stop the monitoring loop gracefully.""" - if self._task is None: - return - self._task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._task - self._task = None - self._last_snapshot = None - logger.info(COS_MONITOR_STOPPED) + """Stop the monitoring loop gracefully. + + Holds ``self._lifecycle_lock`` so a concurrent ``start()`` + cannot recreate the task mid-stop. Drain is shielded with a + hard deadline; on timeout the monitor is marked unrestartable. + """ + async with self._lifecycle_lock: + self._stop_event.set() + task = self._task + if task is None: + return + task.cancel() + + async def _drain() -> None: + try: + await task + except asyncio.CancelledError: + pass + except MemoryError, RecursionError: + raise + except Exception as exc: + logger.warning( + COS_INFLECTION_CHECK_FAILED, + error_type=type(exc).__name__, + error=safe_error_description(exc), + note="shutdown", + ) + + drain_task: asyncio.Task[None] = asyncio.create_task(_drain()) + try: + await asyncio.wait_for( + asyncio.shield(drain_task), + timeout=self._stop_drain_timeout_seconds, + ) + except TimeoutError: + self._stop_failed = True + logger.error( # noqa: TRY400 + COS_INFLECTION_CHECK_FAILED, + error=("stop exceeded hard deadline; monitor marked unrestartable"), + timeout_seconds=self._stop_drain_timeout_seconds, + ) + raise + self._task = None + self._last_snapshot = None + logger.info(COS_MONITOR_STOPPED) + # Recreate primitives outside the (released) lock so a fresh + # event loop binding works for subsequent ``start()`` calls. + self._lifecycle_lock = asyncio.Lock() + self._stop_event = asyncio.Event() async def _loop(self) -> None: - """Periodic snapshot collection and inflection check.""" - while True: + """Periodic snapshot collection and inflection check. + + Uses ``wait_for(_stop_event.wait(), timeout=interval)`` instead + of plain ``asyncio.sleep`` so cancellation cooperatively wakes + the loop and the canonical drain timeout has a chance to + complete the shutdown promptly. + """ + while not self._stop_event.is_set(): try: await self._tick() except asyncio.CancelledError: @@ -101,7 +171,15 @@ async def _loop(self) -> None: raise except Exception: logger.exception(COS_INFLECTION_CHECK_FAILED) - await asyncio.sleep(self._interval_s) + try: + await asyncio.wait_for( + self._stop_event.wait(), + timeout=self._interval_s, + ) + except TimeoutError: + continue + except asyncio.CancelledError: + raise async def _tick(self) -> None: """Single monitoring tick.""" diff --git a/src/synthorg/observability/events/event_stream.py b/src/synthorg/observability/events/event_stream.py index 7aed34fd03..88a9090b6e 100644 --- a/src/synthorg/observability/events/event_stream.py +++ b/src/synthorg/observability/events/event_stream.py @@ -17,3 +17,4 @@ EVENT_STREAM_HUB_STARTED: Final[str] = "event_stream.hub.started" EVENT_STREAM_HUB_STOPPED: Final[str] = "event_stream.hub.stopped" EVENT_STREAM_HUB_PUBLISH_FAILED: Final[str] = "event_stream.hub.publish_failed" +EVENT_STREAM_HUB_PUBLISH_DEDUPED: Final[str] = "event_stream.hub.publish_deduped" diff --git a/src/synthorg/providers/health_prober.py b/src/synthorg/providers/health_prober.py index 1c7d3d60bc..da162b66bb 100644 --- a/src/synthorg/providers/health_prober.py +++ b/src/synthorg/providers/health_prober.py @@ -8,7 +8,6 @@ """ import asyncio -import contextlib import time from datetime import UTC, datetime from typing import TYPE_CHECKING, Final @@ -16,7 +15,7 @@ import httpx -from synthorg.observability import get_logger +from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.provider import ( PROVIDER_HEALTH_PROBE_FAILED, PROVIDER_HEALTH_PROBE_SKIPPED, @@ -148,7 +147,10 @@ class ProviderHealthProber: "_discovery_policy_loader", "_health_tracker", "_interval", + "_lifecycle_lock", + "_stop_drain_timeout_seconds", "_stop_event", + "_stop_failed", "_task", ) @@ -171,27 +173,98 @@ def __init__( self._interval = interval_seconds self._stop_event = asyncio.Event() self._task: asyncio.Task[None] | None = None + # Per ``docs/reference/lifecycle-sync.md`` the lifecycle lock, + # stop event, drain timeout, and unrestartable flag are + # constructed eagerly so a racing ``stop()`` cannot observe a + # half-published lock attribute. + self._lifecycle_lock: asyncio.Lock = asyncio.Lock() + self._stop_failed: bool = False + self._stop_drain_timeout_seconds: float = 30.0 async def start(self) -> None: - """Start the background probe loop.""" - if self._task is not None: - return - self._stop_event.clear() - self._task = asyncio.create_task(self._run_loop()) - logger.info( - PROVIDER_HEALTH_PROBER_STARTED, - interval_seconds=self._interval, - ) + """Start the background probe loop. + + Idempotent + concurrent-safe: concurrent ``start()`` calls + serialise on ``self._lifecycle_lock`` so at most one task is + spawned even when multiple callers race on the ``_task is + None`` check. After a timed-out stop the prober is marked + unrestartable; constructing a fresh instance is required. + """ + async with self._lifecycle_lock: + if self._stop_failed: + msg = ( + "ProviderHealthProber is unrestartable after a " + "timed-out stop; construct a fresh prober instead" + ) + logger.warning( + PROVIDER_HEALTH_PROBER_CYCLE_FAILED, + error=msg, + note="unrestartable", + ) + raise RuntimeError(msg) + if self._task is not None and not self._task.done(): + return + self._stop_event.clear() + self._task = asyncio.create_task( + self._run_loop(), + name="provider-health-prober", + ) + logger.info( + PROVIDER_HEALTH_PROBER_STARTED, + interval_seconds=self._interval, + ) async def stop(self) -> None: - """Stop the background probe loop gracefully.""" - self._stop_event.set() - if self._task is not None: - self._task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._task + """Stop the background probe loop gracefully. + + Holds ``self._lifecycle_lock`` so a concurrent ``start()`` + cannot recreate the task mid-stop. The drain is shielded from + the outer ``wait_for`` so a hung downstream cannot indefinitely + hold the lifecycle lock; on timeout the prober is marked + unrestartable. + """ + async with self._lifecycle_lock: + self._stop_event.set() + task = self._task + if task is None: + return + task.cancel() + + async def _drain() -> None: + try: + await task + except asyncio.CancelledError: + pass + except MemoryError, RecursionError: + raise + except Exception as exc: + logger.warning( + PROVIDER_HEALTH_PROBER_CYCLE_FAILED, + error_type=type(exc).__name__, + error=safe_error_description(exc), + note="shutdown", + ) + + drain_task: asyncio.Task[None] = asyncio.create_task(_drain()) + try: + await asyncio.wait_for( + asyncio.shield(drain_task), + timeout=self._stop_drain_timeout_seconds, + ) + except TimeoutError: + self._stop_failed = True + logger.error( # noqa: TRY400 + PROVIDER_HEALTH_PROBER_CYCLE_FAILED, + error=("stop exceeded hard deadline; prober marked unrestartable"), + timeout_seconds=self._stop_drain_timeout_seconds, + ) + raise self._task = None - logger.info(PROVIDER_HEALTH_PROBER_STOPPED) + logger.info(PROVIDER_HEALTH_PROBER_STOPPED) + # Recreate primitives outside the (released) lock so a + # subsequent ``start()`` on a different event loop can rebind. + self._lifecycle_lock = asyncio.Lock() + self._stop_event = asyncio.Event() async def _run_loop(self) -> None: """Main loop: probe all, then sleep until next cycle or stop.""" diff --git a/src/synthorg/settings/subscribers/backup_subscriber.py b/src/synthorg/settings/subscribers/backup_subscriber.py index 7197e11a4c..4a2c24f7ea 100644 --- a/src/synthorg/settings/subscribers/backup_subscriber.py +++ b/src/synthorg/settings/subscribers/backup_subscriber.py @@ -110,7 +110,7 @@ async def _toggle_scheduler(self) -> None: enabled = str(result.value).lower() == "true" if enabled and not scheduler.is_running: - scheduler.start() + await scheduler.start() logger.info( SETTINGS_SUBSCRIBER_NOTIFIED, subscriber=self.subscriber_name, diff --git a/src/synthorg/tools/mcp/cache.py b/src/synthorg/tools/mcp/cache.py index 7b961d2fd9..0475e4a276 100644 --- a/src/synthorg/tools/mcp/cache.py +++ b/src/synthorg/tools/mcp/cache.py @@ -5,6 +5,7 @@ """ import copy +import threading from collections import OrderedDict from typing import Any @@ -26,8 +27,9 @@ class MCPResultCache: """TTL + LRU-bounded cache for MCP tool results. - Safe for use within a single asyncio event loop, where coroutine - interleaving cannot cause concurrent mutations to the cache dict. + Thread-safe via an internal ``threading.Lock`` so concurrent + threadpool-dispatched tool invocations cannot interleave the + read-decision-write blocks in :meth:`get` and :meth:`put`. Keys are derived from tool name and arguments. Args: @@ -48,6 +50,7 @@ def __init__( self._cache: OrderedDict[tuple[str, Any], tuple[float, ToolExecutionResult]] = ( OrderedDict() ) + self._lock = threading.Lock() def get( self, @@ -67,27 +70,34 @@ def get( Cached ``ToolExecutionResult`` or ``None``. """ key = self._make_key(tool_name, arguments) - entry = self._cache.get(key) - if entry is None: - logger.debug(MCP_CACHE_MISS, tool_name=tool_name) + hit: ToolExecutionResult | None = None + miss_reason: str | None = None + with self._lock: + entry = self._cache.get(key) + if entry is not None: + timestamp, result = entry + if self._clock.monotonic() - timestamp > self._ttl_seconds: + del self._cache[key] + miss_reason = "expired" + else: + self._cache.move_to_end(key) + hit = result + + if hit is None: + if miss_reason is None: + logger.debug(MCP_CACHE_MISS, tool_name=tool_name) + else: + logger.debug( + MCP_CACHE_MISS, + tool_name=tool_name, + reason=miss_reason, + ) record_cache_operation(cache_name=_CACHE_NAME, outcome="miss") return None - timestamp, result = entry - if self._clock.monotonic() - timestamp > self._ttl_seconds: - del self._cache[key] - logger.debug( - MCP_CACHE_MISS, - tool_name=tool_name, - reason="expired", - ) - record_cache_operation(cache_name=_CACHE_NAME, outcome="miss") - return None - - self._cache.move_to_end(key) logger.debug(MCP_CACHE_HIT, tool_name=tool_name) record_cache_operation(cache_name=_CACHE_NAME, outcome="hit") - return copy.deepcopy(result) + return copy.deepcopy(hit) def put( self, @@ -106,23 +116,26 @@ def put( result: The ``ToolExecutionResult`` to cache. """ key = self._make_key(tool_name, arguments) - - # Remove existing entry to refresh position - if key in self._cache: - del self._cache[key] - - # Evict oldest if at capacity - while len(self._cache) >= self._max_size > 0: - evicted_key, _ = self._cache.popitem(last=False) + evicted: list[tuple[str, Any]] = [] + with self._lock: + # Remove existing entry to refresh position + if key in self._cache: + del self._cache[key] + + # Evict oldest if at capacity + while len(self._cache) >= self._max_size > 0: + evicted_key, _ = self._cache.popitem(last=False) + evicted.append(evicted_key) + + if self._max_size > 0: + self._cache[key] = (self._clock.monotonic(), copy.deepcopy(result)) + for evicted_key in evicted: logger.debug( MCP_CACHE_EVICT, evicted_tool=evicted_key[0], ) record_cache_operation(cache_name=_CACHE_NAME, outcome="evict") - if self._max_size > 0: - self._cache[key] = (self._clock.monotonic(), copy.deepcopy(result)) - def invalidate( self, tool_name: str | None = None, @@ -133,13 +146,14 @@ def invalidate( tool_name: If provided, only invalidate entries for this tool. If ``None``, clear all entries. """ - if tool_name is None: - self._cache.clear() - return - - keys_to_remove = [k for k in self._cache if k[0] == tool_name] - for key in keys_to_remove: - del self._cache[key] + with self._lock: + if tool_name is None: + self._cache.clear() + return + + keys_to_remove = [k for k in self._cache if k[0] == tool_name] + for key in keys_to_remove: + del self._cache[key] @staticmethod def _make_key( diff --git a/tests/unit/api/auth/test_ticket_store_threadsafety.py b/tests/unit/api/auth/test_ticket_store_threadsafety.py new file mode 100644 index 0000000000..8161048a06 --- /dev/null +++ b/tests/unit/api/auth/test_ticket_store_threadsafety.py @@ -0,0 +1,105 @@ +"""Thread-safety tests for WsTicketStore. + +The store's mutating methods (``create``, ``validate_and_consume``, +``cleanup_expired``) are synchronous because they were written for +single-threaded asyncio. Litestar routes async handlers on the loop, +but operators may also dispatch sync handlers via the threadpool. +A ``threading.Lock`` guards count-and-insert, pop-and-validate, and +bulk eviction so concurrent thread access cannot exceed the per-user +cap or double-consume a ticket. +""" + +import contextlib +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from synthorg.api.auth.models import AuthenticatedUser, AuthMethod +from synthorg.api.auth.ticket_store import TicketLimitExceededError, WsTicketStore +from synthorg.api.guards import HumanRole + + +def _make_user(user_id: str = "user-1") -> AuthenticatedUser: + return AuthenticatedUser( + user_id=user_id, + username="testadmin", + role=HumanRole.CEO, + auth_method=AuthMethod.WS_TICKET, + ) + + +@pytest.mark.unit +class TestWsTicketStoreThreadSafety: + """Concurrent access from a thread pool must honor invariants.""" + + def test_concurrent_create_honors_per_user_cap(self) -> None: + """100 threads racing on create() for one user yield exactly cap accepts.""" + store = WsTicketStore(max_pending_per_user=5) + user = _make_user() + + def attempt() -> str | None: + try: + return store.create(user) + except TicketLimitExceededError: + return None + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(attempt) for _ in range(100)] + results = [f.result() for f in futures] + + successes = [r for r in results if r is not None] + assert len(successes) == 5 + assert len(set(successes)) == 5 + + def test_concurrent_create_distinct_users_independent(self) -> None: + """Different users do not share the cap under concurrency.""" + store = WsTicketStore(max_pending_per_user=3) + + def attempt(user_id: str) -> str | None: + try: + return store.create(_make_user(user_id=user_id)) + except TicketLimitExceededError: + return None + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(attempt, f"user-{i % 4}") for i in range(40)] + results = [f.result() for f in futures] + + successes = [r for r in results if r is not None] + assert len(successes) == 12 + assert len(set(successes)) == 12 + + def test_concurrent_validate_and_consume_single_winner(self) -> None: + """A ticket can be consumed by exactly one thread under concurrency.""" + store = WsTicketStore() + user = _make_user() + ticket = store.create(user) + + def attempt() -> AuthenticatedUser | None: + return store.validate_and_consume(ticket) + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(attempt) for _ in range(32)] + results = [f.result() for f in futures] + + accepted = [r for r in results if r is not None] + assert len(accepted) == 1 + assert accepted[0].user_id == user.user_id + + def test_concurrent_create_and_cleanup_no_corruption(self) -> None: + """Mixed create / cleanup_expired calls do not raise or corrupt state.""" + store = WsTicketStore(ttl_seconds=30.0, max_pending_per_user=5) + + def task(i: int) -> None: + if i % 3 == 0: + store.cleanup_expired() + return + with contextlib.suppress(TicketLimitExceededError): + store.create(_make_user(user_id=f"user-{i % 8}")) + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(task, i) for i in range(80)] + for f in futures: + f.result() + # If we reach here without RuntimeError ("dictionary changed size + # during iteration") or KeyError, the lock did its job. diff --git a/tests/unit/api/controllers/test_backup.py b/tests/unit/api/controllers/test_backup.py index 12324e717f..36cf659940 100644 --- a/tests/unit/api/controllers/test_backup.py +++ b/tests/unit/api/controllers/test_backup.py @@ -73,6 +73,27 @@ def _make_state_and_service() -> tuple[MagicMock, AsyncMock]: service = AsyncMock() app_state = MagicMock() app_state.backup_service = service + # The controller now wraps every backup in idempotency_service. + # Mock the service so run_idempotent invokes the callback inline + # and returns a fresh outcome with the manifest dict. + idempotency_service = MagicMock() + + async def _run_idempotent( + *, + scope: object, + key: object, + callback: Any, + ) -> Any: + del scope, key + result = await callback() + outcome = MagicMock() + outcome.timed_out = False + outcome.result = result + outcome.fresh = True + return outcome + + idempotency_service.run_idempotent = _run_idempotent + app_state.idempotency_service = idempotency_service # Pagination requires a real cursor secret; MagicMock's default # attribute resolution would hand back a Mock to ``paginate_cursor`` # which ultimately fails the HMAC pipeline. @@ -98,11 +119,15 @@ async def test_create_backup_calls_service_with_manual_trigger(self) -> None: service.create_backup = AsyncMock(return_value=manifest) ctrl = _controller() - result = await ctrl.create_backup.fn(ctrl, state=state) + result = await ctrl.create_backup.fn( + ctrl, + state=state, + idempotency_key="test-key-001", + ) service.create_backup.assert_awaited_once_with(BackupTrigger.MANUAL) assert isinstance(result, ApiResponse) - assert result.data is manifest + assert result.data == manifest async def test_create_backup_returns_409_on_in_progress(self) -> None: state, service = _make_state_and_service() @@ -112,7 +137,11 @@ async def test_create_backup_returns_409_on_in_progress(self) -> None: ctrl = _controller() with pytest.raises(ConflictError) as exc_info: - await ctrl.create_backup.fn(ctrl, state=state) + await ctrl.create_backup.fn( + ctrl, + state=state, + idempotency_key="test-key-002", + ) assert exc_info.value.status_code == 409 diff --git a/tests/unit/api/controllers/test_backup_required_idempotency.py b/tests/unit/api/controllers/test_backup_required_idempotency.py new file mode 100644 index 0000000000..c74903c217 --- /dev/null +++ b/tests/unit/api/controllers/test_backup_required_idempotency.py @@ -0,0 +1,96 @@ +"""Idempotency-Key is mandatory for POST /admin/backups. + +Per audit #133 (idempotency / retry safety): without a key, a +network-flake-driven 5xx retry could launch concurrent backups and +violate the at-most-one-running invariant. The header is now +required by Litestar's parameter validation; missing or empty values +yield HTTP 400. + +The shape of these tests intentionally avoids spinning up the full +Litestar app: we inspect the route handler's parameter signature and +verify the controller correctly invokes the idempotency service when +the key is supplied. +""" + +import inspect +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from synthorg.api.controllers.backup import BackupController +from synthorg.api.cursor import CursorSecret +from synthorg.backup.models import ( + BackupComponent, + BackupManifest, + BackupTrigger, +) + +pytestmark = pytest.mark.unit + + +def _make_manifest() -> BackupManifest: + return BackupManifest( + synthorg_version="0.3.2", + timestamp="2026-03-18T12:00:00+00:00", + trigger=BackupTrigger.MANUAL, + components=(BackupComponent.PERSISTENCE,), + size_bytes=4096, + checksum="sha256:" + "a" * 64, + backup_id="abc123def456", + ) + + +def _make_state(*, run_idempotent: Any) -> MagicMock: + service = AsyncMock() + service.create_backup = AsyncMock(return_value=_make_manifest()) + app_state = MagicMock() + app_state.backup_service = service + idempotency_service = MagicMock() + idempotency_service.run_idempotent = run_idempotent + app_state.idempotency_service = idempotency_service + app_state.cursor_secret = CursorSecret.from_key( + "test-key-32-bytes-padding0000000", + ) + state = MagicMock() + state.app_state = app_state + return state + + +class TestRequiredIdempotencyKey: + """The header is declared mandatory and the handler delegates to the service.""" + + def test_signature_marks_idempotency_key_required(self) -> None: + sig = inspect.signature(BackupController.create_backup.fn) + param = sig.parameters["idempotency_key"] + # A required parameter has no default. Annotated[str, Parameter(...)] + # without a default value reflects the required header. + assert param.default is inspect.Parameter.empty + + async def test_handler_invokes_idempotency_service(self) -> None: + captured: dict[str, object] = {} + + async def fake_run_idempotent( + *, + scope: object, + key: object, + callback: Any, + ) -> Any: + captured["scope"] = scope + captured["key"] = key + await callback() + outcome = MagicMock() + outcome.timed_out = False + outcome.result = _make_manifest().model_dump(mode="json") + outcome.fresh = True + return outcome + + ctrl = BackupController(owner=BackupController) # type: ignore[arg-type] + state = _make_state(run_idempotent=fake_run_idempotent) + await ctrl.create_backup.fn( + ctrl, + state=state, + idempotency_key="key-abc-123", + ) + assert str(captured["scope"]) == "backup" + assert str(captured["key"]) == "key-abc-123" diff --git a/tests/unit/api/controllers/test_simulations_idempotency.py b/tests/unit/api/controllers/test_simulations_idempotency.py new file mode 100644 index 0000000000..f544a17cb0 --- /dev/null +++ b/tests/unit/api/controllers/test_simulations_idempotency.py @@ -0,0 +1,118 @@ +"""Idempotency guard tests for ``POST /simulations/``. + +Per audit #133: a redelivered ``start_simulation`` request with the +same ``simulation_id`` must not spawn a second runner that races the +first on the in-memory store. The controller now rejects the second +request with HTTP 409 Conflict. +""" + +import contextlib +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from synthorg.api.controllers.simulations import ( + SimulationController, + StartSimulationPayload, +) +from synthorg.client.models import SimulationConfig +from synthorg.client.store import SimulationRecord +from synthorg.core.domain_errors import ConflictError + +pytestmark = pytest.mark.unit + + +def _make_config(simulation_id: str = "sim-001") -> SimulationConfig: + return SimulationConfig( + simulation_id=simulation_id, + project_id="proj-1", + clients_per_round=1, + requirements_per_client=1, + ) + + +def _make_state_with_existing(record: SimulationRecord | None) -> MagicMock: + """Build a mocked Litestar state whose simulation_store returns *record*.""" + sim_state = MagicMock() + if record is None: + + async def _raise(_id: str) -> SimulationRecord: + del _id + msg = "Simulation not found" + raise KeyError(msg) + + sim_state.simulation_store.get = _raise + else: + + async def _return(_id: str) -> SimulationRecord: + del _id + return record + + sim_state.simulation_store.get = _return + sim_state.simulation_store.save = AsyncMock() + sim_state.background_tasks = set() + sim_state.intake_engine = MagicMock() + sim_state.pool = MagicMock() + sim_state.pool.list_clients = AsyncMock(return_value=()) + sim_state.feedback_store = MagicMock() + sim_state.feedback_store.record = MagicMock() + app_state = MagicMock() + app_state.client_simulation_state = sim_state + app_state.config_resolver = MagicMock() + state = MagicMock() + state.app_state = app_state + return state + + +def _make_request() -> MagicMock: + return MagicMock() + + +class TestSimulationsIdempotency: + """Duplicate ``simulation_id`` is rejected with HTTP 409.""" + + async def test_duplicate_id_rejected_with_conflict(self) -> None: + existing = SimulationRecord( + simulation_id="sim-001", + config=_make_config(), + status="running", + ) + state = _make_state_with_existing(existing) + ctrl = SimulationController(owner=SimulationController) # type: ignore[arg-type] + payload = StartSimulationPayload(config=_make_config()) + + with pytest.raises(ConflictError) as exc: + await ctrl.start_simulation.fn( + ctrl, + request=_make_request(), + state=state, + data=payload, + ) + assert exc.value.status_code == 409 + assert "already exists" in str(exc.value) + + async def test_first_request_passes_idempotency_check(self) -> None: + """A fresh ``simulation_id`` survives the idempotency check. + + We cannot easily exercise the full happy path here without a + full app fixture (the runner requires intake_engine etc.). + The check verifies the controller progresses past the + idempotency guard and reaches ``simulation_store.save``. + """ + state = _make_state_with_existing(None) + ctrl = SimulationController(owner=SimulationController) # type: ignore[arg-type] + payload = StartSimulationPayload(config=_make_config(simulation_id="sim-002")) + + # The handler will reach .save() then attempt to spawn the + # runner. We tolerate any post-save error since this test + # only verifies idempotency-guard behaviour, not the runner + # plumbing exercised in the integration suite. + with contextlib.suppress(Exception): + await ctrl.start_simulation.fn( + ctrl, + request=_make_request(), + state=state, + data=payload, + ) + sim_store = state.app_state.client_simulation_state.simulation_store + sim_store.save.assert_awaited_once() diff --git a/tests/unit/backup/test_scheduler.py b/tests/unit/backup/test_scheduler.py index 37c07ca761..27d6a5bfda 100644 --- a/tests/unit/backup/test_scheduler.py +++ b/tests/unit/backup/test_scheduler.py @@ -47,7 +47,7 @@ async def test_start_creates_background_task(self) -> None: "synthorg.backup.scheduler.asyncio.create_task", side_effect=_close_coro_side_effect(mock_task), ) as mock_ct: - scheduler.start() + await scheduler.start() assert scheduler.is_running mock_ct.assert_called_once() # type: ignore[unreachable] @@ -64,8 +64,8 @@ async def test_start_is_noop_when_already_running(self) -> None: "synthorg.backup.scheduler.asyncio.create_task", side_effect=_close_coro_side_effect(mock_task), ) as mock_ct: - scheduler.start() - scheduler.start() # second call should be no-op + await scheduler.start() + await scheduler.start() # second call should be no-op assert mock_ct.call_count == 1 @@ -123,7 +123,7 @@ async def test_is_running_true_after_start(self) -> None: "synthorg.backup.scheduler.asyncio.create_task", side_effect=_close_coro_side_effect(mock_task), ): - scheduler.start() + await scheduler.start() assert scheduler.is_running diff --git a/tests/unit/backup/test_scheduler_lifecycle.py b/tests/unit/backup/test_scheduler_lifecycle.py new file mode 100644 index 0000000000..7e93296836 --- /dev/null +++ b/tests/unit/backup/test_scheduler_lifecycle.py @@ -0,0 +1,70 @@ +"""Canonical lifecycle pattern tests for ``BackupScheduler``. + +The unit ``test_scheduler.py`` covers happy-path start / stop / loop +behaviour. This module verifies the lock-driven concurrency safety +added per ``docs/reference/lifecycle-sync.md``: concurrent start, +restart after clean stop, and unrestartable flag after a drain +timeout. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from synthorg.backup.scheduler import BackupScheduler + +pytestmark = pytest.mark.unit + + +def _make_scheduler() -> BackupScheduler: + service = MagicMock() + service.create_backup = AsyncMock() + return BackupScheduler(service, interval_hours=1) + + +class TestBackupSchedulerLifecycleLock: + """Canonical pattern compliance.""" + + async def test_concurrent_starts_spawn_one_task(self) -> None: + scheduler = _make_scheduler() + try: + await asyncio.gather( + scheduler.start(), + scheduler.start(), + scheduler.start(), + ) + assert scheduler._task is not None + finally: + await scheduler.stop() + + async def test_restart_after_clean_stop(self) -> None: + scheduler = _make_scheduler() + await scheduler.start() + await scheduler.stop() + assert scheduler._task is None + await scheduler.start() + assert scheduler.is_running + await scheduler.stop() + + async def test_unrestartable_after_drain_timeout(self) -> None: + scheduler = _make_scheduler() + scheduler._stop_drain_timeout_seconds = 0.05 + + async def hung_loop(self: BackupScheduler) -> None: + del self + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + await asyncio.sleep(1.0) + + with patch.object(BackupScheduler, "_run_loop", hung_loop): + await scheduler.start() + await asyncio.sleep(0) + + with pytest.raises(TimeoutError): + await scheduler.stop() + assert scheduler._stop_failed is True + + with pytest.raises(RuntimeError, match="unrestartable"): + await scheduler.start() diff --git a/tests/unit/budget/test_trends_currency.py b/tests/unit/budget/test_trends_currency.py new file mode 100644 index 0000000000..07194c89b3 --- /dev/null +++ b/tests/unit/budget/test_trends_currency.py @@ -0,0 +1,108 @@ +"""Currency-invariant tests for budget trends. + +Both ``bucket_cost_records`` and ``project_daily_spend`` aggregate +``record.cost`` across cost records. Mixing currencies in the input +silently produces a meaningless monetary aggregate, so both call +``_assert_single_currency`` at the boundary and raise +``MixedCurrencyAggregationError`` (HTTP 409) instead. +""" + +from datetime import UTC, datetime, timedelta + +import pytest + +from synthorg.budget.cost_record import CostRecord +from synthorg.budget.errors import MixedCurrencyAggregationError +from synthorg.budget.trends import ( + BucketSize, + bucket_cost_records, + project_daily_spend, +) + +pytestmark = pytest.mark.unit + +_NOW = datetime(2026, 5, 1, 12, 0, 0, tzinfo=UTC) + + +def _record( + currency: str, + cost: float = 0.10, + ts: datetime | None = None, +) -> CostRecord: + return CostRecord( + agent_id="agent-a", + task_id="task-001", + provider="test-provider", + model="test-small-001", + input_tokens=100, + output_tokens=50, + cost=cost, + currency=currency, + timestamp=ts or _NOW, + ) + + +class TestBucketCostRecordsCurrency: + """`bucket_cost_records` rejects mixed-currency input.""" + + def test_single_currency_aggregates_cleanly(self) -> None: + records = ( + _record("EUR", 0.10), + _record("EUR", 0.20), + ) + result = bucket_cost_records( + records, + _NOW, + _NOW + timedelta(hours=1), + BucketSize.HOUR, + ) + assert result[0].value == pytest.approx(0.30) + + def test_mixed_currency_raises(self) -> None: + records = ( + _record("EUR", 0.10), + _record("USD", 0.20), + ) + with pytest.raises(MixedCurrencyAggregationError) as exc: + bucket_cost_records( + records, + _NOW, + _NOW + timedelta(hours=1), + BucketSize.HOUR, + ) + assert exc.value.currencies == frozenset({"EUR", "USD"}) + + def test_empty_records_no_error(self) -> None: + result = bucket_cost_records( + (), + _NOW, + _NOW + timedelta(hours=1), + BucketSize.HOUR, + ) + assert all(point.value == 0.0 for point in result) + + +class TestProjectDailySpendCurrency: + """`project_daily_spend` rejects mixed-currency input.""" + + def test_single_currency_projects_cleanly(self) -> None: + records = ( + _record("EUR", 1.00, _NOW - timedelta(days=2)), + _record("EUR", 2.00, _NOW - timedelta(days=1)), + ) + forecast = project_daily_spend(records, horizon_days=7, now=_NOW) + assert forecast.avg_daily_spend > 0 + + def test_mixed_currency_raises(self) -> None: + records = ( + _record("EUR", 1.00, _NOW - timedelta(days=2)), + _record("USD", 2.00, _NOW - timedelta(days=1)), + ) + with pytest.raises(MixedCurrencyAggregationError) as exc: + project_daily_spend(records, horizon_days=7, now=_NOW) + assert exc.value.currencies == frozenset({"EUR", "USD"}) + + def test_empty_records_no_error(self) -> None: + forecast = project_daily_spend((), horizon_days=7, now=_NOW) + assert forecast.avg_daily_spend == 0.0 + assert forecast.confidence == 0.0 diff --git a/tests/unit/client/test_continuous_lifecycle.py b/tests/unit/client/test_continuous_lifecycle.py new file mode 100644 index 0000000000..c843e8af69 --- /dev/null +++ b/tests/unit/client/test_continuous_lifecycle.py @@ -0,0 +1,121 @@ +"""Lifecycle tests for ``ContinuousMode``. + +ContinuousMode is an in-place runner (``start()`` executes the loop +synchronously on the caller until ``stop()`` is signalled). The +``_lifecycle_lock`` serialises concurrent ``start()`` calls so the +"already running" RuntimeError is raised reliably, and the lock +spans the full body so a racing caller cannot enter mid-loop. +""" + +import asyncio +from typing import Any + +import pytest + +from synthorg.client.config import ContinuousModeConfig +from synthorg.client.continuous import ContinuousMode +from synthorg.client.models import SimulationConfig, SimulationMetrics + +pytestmark = pytest.mark.unit + + +class _FakeRunner: + """Records run() invocations and returns canned metrics.""" + + def __init__(self) -> None: + self.calls = 0 + + async def run( + self, + *, + sim_config: SimulationConfig, + clients: tuple[Any, ...], + ) -> tuple[SimulationMetrics, list[Any]]: + del sim_config, clients + self.calls += 1 + await asyncio.sleep(0) + return ( + SimulationMetrics( + total_requirements=1, + total_tasks_created=1, + tasks_accepted=1, + ), + [], + ) + + +def _sim_config() -> SimulationConfig: + return SimulationConfig( + project_id="proj-1", + clients_per_round=1, + requirements_per_client=1, + ) + + +class TestContinuousModeLifecycleLock: + """The lifecycle lock prevents concurrent start() from running twice.""" + + async def test_double_start_raises_when_already_running(self) -> None: + runner = _FakeRunner() + mode = ContinuousMode( + config=ContinuousModeConfig( + enabled=True, + request_interval_sec=10.0, + max_concurrent_requests=1, + ), + runner=runner, # type: ignore[arg-type] + ) + + first = asyncio.create_task( + mode.start(sim_config=_sim_config(), clients=()), + ) + # Yield so the first task acquires the lock and starts the + # loop; without the yield, the second start() may run first. + await asyncio.sleep(0) + await asyncio.sleep(0) + + with pytest.raises(RuntimeError, match="already running"): + await mode.start(sim_config=_sim_config(), clients=()) + + mode.stop() + await first + + async def test_disabled_short_circuits_without_acquiring_lock(self) -> None: + runner = _FakeRunner() + mode = ContinuousMode( + config=ContinuousModeConfig(enabled=False), + runner=runner, # type: ignore[arg-type] + ) + results = await mode.start(sim_config=_sim_config(), clients=()) + assert results == [] + assert runner.calls == 0 + + async def test_stop_releases_runner_loop(self) -> None: + runner = _FakeRunner() + mode = ContinuousMode( + config=ContinuousModeConfig( + enabled=True, + request_interval_sec=0.001, + max_concurrent_requests=1, + ), + runner=runner, # type: ignore[arg-type] + ) + + task = asyncio.create_task( + mode.start(sim_config=_sim_config(), clients=()), + ) + # Let the loop run once. + await asyncio.sleep(0.01) + mode.stop() + results = await task + assert len(results) >= 1 + assert mode.runs_completed >= 1 + + async def test_lifecycle_lock_attribute_present(self) -> None: + """Smoke test: the canonical lock name is in place.""" + runner = _FakeRunner() + mode = ContinuousMode( + config=ContinuousModeConfig(enabled=True), + runner=runner, # type: ignore[arg-type] + ) + assert isinstance(mode._lifecycle_lock, asyncio.Lock) diff --git a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py new file mode 100644 index 0000000000..af856183c1 --- /dev/null +++ b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py @@ -0,0 +1,104 @@ +"""Tests for the registry-based escalation factory dispatch. + +Per audit #69: the factory used to dispatch on a hardcoded if/elif +chain. It now consults a frozen registry map keyed by the config +discriminator. These tests verify each registered branch builds the +expected store and that an unregistered key raises ValueError with a +helpful message listing the available options. +""" + +from typing import cast +from unittest.mock import MagicMock + +import pytest + +from synthorg.communication.conflict_resolution.escalation.config import ( + EscalationQueueConfig, +) +from synthorg.communication.conflict_resolution.escalation.factory import ( + build_decision_processor, + build_escalation_queue_store, +) +from synthorg.communication.conflict_resolution.escalation.in_memory_store import ( + InMemoryEscalationStore, +) +from synthorg.communication.conflict_resolution.escalation.processors import ( + HybridDecisionProcessor, + WinnerSelectProcessor, +) +from synthorg.persistence.protocol import PersistenceBackend + +pytestmark = pytest.mark.unit + + +def _fake_persistence(backend_name: str) -> PersistenceBackend: + backend = MagicMock(spec=PersistenceBackend) + backend.backend_name = backend_name + backend.build_escalations = MagicMock(return_value=MagicMock()) + return cast(PersistenceBackend, backend) + + +class TestQueueStoreRegistry: + """``build_escalation_queue_store`` dispatches via the registry map.""" + + def test_memory_backend_returns_in_memory_store(self) -> None: + config = EscalationQueueConfig(backend="memory") + store = build_escalation_queue_store(config) + assert isinstance(store, InMemoryEscalationStore) + + def test_sqlite_backend_calls_build_escalations(self) -> None: + config = EscalationQueueConfig(backend="sqlite") + backend = _fake_persistence("sqlite") + build_escalation_queue_store(config, backend) + backend.build_escalations.assert_called_once_with() # type: ignore[attr-defined] + + def test_postgres_backend_passes_notify_channel_when_enabled(self) -> None: + config = EscalationQueueConfig( + backend="postgres", + cross_instance_notify="on", + notify_channel="escalations", + ) + backend = _fake_persistence("postgres") + build_escalation_queue_store(config, backend) + backend.build_escalations.assert_called_once_with( # type: ignore[attr-defined] + notify_channel="escalations", + ) + + def test_postgres_backend_off_passes_none_channel(self) -> None: + config = EscalationQueueConfig( + backend="postgres", + cross_instance_notify="off", + ) + backend = _fake_persistence("postgres") + build_escalation_queue_store(config, backend) + backend.build_escalations.assert_called_once_with( # type: ignore[attr-defined] + notify_channel=None, + ) + + def test_sqlite_without_persistence_raises(self) -> None: + config = EscalationQueueConfig(backend="sqlite") + with pytest.raises(ValueError, match="connected persistence backend"): + build_escalation_queue_store(config, persistence=None) + + def test_postgres_with_sqlite_persistence_raises(self) -> None: + config = EscalationQueueConfig(backend="postgres") + backend = _fake_persistence("sqlite") + with pytest.raises( + ValueError, + match=r"config\.backend='postgres'", + ): + build_escalation_queue_store(config, backend) + + +class TestDecisionProcessorRegistry: + """``build_decision_processor`` dispatches via the registry map.""" + + def test_winner_strategy_returns_winner_select(self) -> None: + config = EscalationQueueConfig(decision_strategy="winner") + processor = build_decision_processor(config) + assert isinstance(processor, WinnerSelectProcessor) + + def test_hybrid_strategy_returns_hybrid(self) -> None: + config = EscalationQueueConfig(decision_strategy="hybrid") + processor = build_decision_processor(config) + assert isinstance(processor, HybridDecisionProcessor) diff --git a/tests/unit/communication/event_stream/test_stream_dedup.py b/tests/unit/communication/event_stream/test_stream_dedup.py new file mode 100644 index 0000000000..11c23af3b6 --- /dev/null +++ b/tests/unit/communication/event_stream/test_stream_dedup.py @@ -0,0 +1,116 @@ +"""Dedup tests for ``EventStreamHub.publish``. + +Per audit #133: a retried publish (e.g. a webhook handler that +catches a transient publish failure) must not deliver the same event +twice to subscribers. The hub keeps a per-session sliding-window of +seen ``event.id`` values; identical ids within the TTL are skipped +and logged. +""" + +import asyncio +from datetime import UTC, datetime + +import pytest + +from synthorg.communication.event_stream.stream import EventStreamHub +from synthorg.communication.event_stream.types import ( + AgUiEventType, + StreamEvent, +) +from tests._shared.fake_clock import FakeClock + +pytestmark = pytest.mark.unit + + +def _event(*, event_id: str, session_id: str = "session-1") -> StreamEvent: + return StreamEvent( + id=event_id, + type=AgUiEventType.TEXT_MESSAGE_CONTENT, + timestamp=datetime(2026, 5, 1, 12, 0, 0, tzinfo=UTC), + session_id=session_id, + correlation_id=None, + agent_id=None, + payload={}, + ) + + +class TestEventStreamHubDedup: + """Per-session dedup window.""" + + async def test_duplicate_event_id_within_ttl_is_skipped(self) -> None: + clock = FakeClock() + hub = EventStreamHub(dedup_ttl_seconds=60.0, clock=clock) + queue = await hub.subscribe("session-1") + + event = _event(event_id="evt-001") + await hub.publish(event) + await hub.publish(event) # duplicate + + delivered: list[StreamEvent] = [] + try: + while True: + delivered.append(queue.get_nowait()) + except asyncio.QueueEmpty: + pass + assert len(delivered) == 1 + + async def test_duplicate_after_ttl_is_redelivered(self) -> None: + clock = FakeClock() + hub = EventStreamHub(dedup_ttl_seconds=60.0, clock=clock) + queue = await hub.subscribe("session-1") + + event = _event(event_id="evt-001") + await hub.publish(event) + clock.advance(61.0) + await hub.publish(event) + + delivered: list[StreamEvent] = [] + try: + while True: + delivered.append(queue.get_nowait()) + except asyncio.QueueEmpty: + pass + assert len(delivered) == 2 + + async def test_distinct_ids_all_delivered(self) -> None: + clock = FakeClock() + hub = EventStreamHub(dedup_ttl_seconds=60.0, clock=clock) + queue = await hub.subscribe("session-1") + + for i in range(5): + await hub.publish(_event(event_id=f"evt-{i}")) + + delivered: list[StreamEvent] = [] + try: + while True: + delivered.append(queue.get_nowait()) + except asyncio.QueueEmpty: + pass + assert len(delivered) == 5 + + async def test_different_sessions_independent_dedup(self) -> None: + clock = FakeClock() + hub = EventStreamHub(dedup_ttl_seconds=60.0, clock=clock) + q1 = await hub.subscribe("session-1") + q2 = await hub.subscribe("session-2") + + # Same event id, different sessions -- must each receive one. + await hub.publish(_event(event_id="shared-id", session_id="session-1")) + await hub.publish(_event(event_id="shared-id", session_id="session-2")) + + assert q1.qsize() == 1 + assert q2.qsize() == 1 + + async def test_dedup_window_bounded_per_session(self) -> None: + """The per-session map is bounded so a noisy session cannot leak memory.""" + clock = FakeClock() + hub = EventStreamHub( + dedup_ttl_seconds=300.0, + dedup_max_entries_per_session=8, + clock=clock, + ) + await hub.subscribe("session-1") + for i in range(100): + await hub.publish(_event(event_id=f"evt-{i}")) + seen = hub._seen_event_ids["session-1"] + assert len(seen) <= 8 diff --git a/tests/unit/hr/pruning/test_service.py b/tests/unit/hr/pruning/test_service.py index d92672bcaf..5f302fd74e 100644 --- a/tests/unit/hr/pruning/test_service.py +++ b/tests/unit/hr/pruning/test_service.py @@ -769,7 +769,7 @@ async def test_start_begins_background_task(self) -> None: policies=(NeverEligiblePolicy(),), config=PruningServiceConfig(evaluation_interval_seconds=60.0), ) - service.start() + await service.start() assert service.is_running await service.stop() assert not service.is_running @@ -779,8 +779,8 @@ async def test_double_start_is_idempotent(self) -> None: policies=(NeverEligiblePolicy(),), config=PruningServiceConfig(evaluation_interval_seconds=60.0), ) - service.start() - service.start() # Should not raise or create a second task. + await service.start() + await service.start() # Should not raise or create a second task. assert service.is_running await service.stop() @@ -831,7 +831,7 @@ async def patched_cycle(**kwargs: object) -> object: service.run_pruning_cycle = patched_cycle # type: ignore[assignment] with patch("asyncio.wait_for", side_effect=patched_wait_for): - service.start() + await service.start() await loop_waiting.wait() service.wake() diff --git a/tests/unit/hr/pruning/test_service_lifecycle.py b/tests/unit/hr/pruning/test_service_lifecycle.py new file mode 100644 index 0000000000..a3bfb9d827 --- /dev/null +++ b/tests/unit/hr/pruning/test_service_lifecycle.py @@ -0,0 +1,89 @@ +"""Canonical lifecycle pattern tests for ``PruningService``. + +The unit ``test_service.py`` covers happy-path start / stop. This +module verifies the lock-driven concurrency safety added per +``docs/reference/lifecycle-sync.md``: concurrent start, restart after +clean stop, and unrestartable flag after a drain timeout. +""" + +import asyncio +from unittest.mock import patch + +import pytest + +from synthorg.api.approval_store import ApprovalStore +from synthorg.hr.pruning.models import PruningServiceConfig +from synthorg.hr.pruning.service import PruningService +from synthorg.hr.registry import AgentRegistryService + +pytestmark = pytest.mark.unit + + +class _FakeOffboarding: + async def offboard(self, request: object) -> None: + del request + + +class _FakeTracker: + pass + + +def _make_service() -> PruningService: + return PruningService( + policies=(), + registry=AgentRegistryService(), + tracker=_FakeTracker(), # type: ignore[arg-type] + approval_store=ApprovalStore(), + offboarding_service=_FakeOffboarding(), # type: ignore[arg-type] + config=PruningServiceConfig(evaluation_interval_seconds=3600.0), + ) + + +class TestPruningServiceLifecycleLock: + """Canonical pattern compliance.""" + + async def test_concurrent_starts_spawn_one_task(self) -> None: + service = _make_service() + try: + await asyncio.gather( + service.start(), + service.start(), + service.start(), + ) + assert service.is_running + finally: + await service.stop() + + async def test_restart_after_clean_stop(self) -> None: + service = _make_service() + await service.start() + await service.stop() + # After a clean stop, the service must accept a restart on a + # fresh ``_task``. Cannot assert ``not is_running`` here: + # mypy narrows ``is_running`` to ``False`` after the property + # access and would then flag every subsequent ``await + # service.start()`` / ``stop()`` as unreachable. + await service.start() + await service.stop() + + async def test_unrestartable_after_drain_timeout(self) -> None: + service = _make_service() + service._stop_drain_timeout_seconds = 0.05 + + async def hung_loop(self: PruningService) -> None: + del self + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + await asyncio.sleep(1.0) + + with patch.object(PruningService, "_run_loop", hung_loop): + await service.start() + await asyncio.sleep(0) + + with pytest.raises(TimeoutError): + await service.stop() + assert service._stop_failed is True + + with pytest.raises(RuntimeError, match="unrestartable"): + await service.start() diff --git a/tests/unit/integrations/test_ngrok_adapter_lifecycle.py b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py new file mode 100644 index 0000000000..c650cadfeb --- /dev/null +++ b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py @@ -0,0 +1,86 @@ +"""Lifecycle tests for ``NgrokAdapter``. + +Verifies the adapter holds its lifecycle lock across both ``start`` +and ``stop`` so concurrent invocations cannot create or tear down two +tunnels under the single-tunnel invariant. +""" + +import asyncio +from typing import Any +from unittest.mock import patch + +import pytest + +from synthorg.integrations.tunnel.ngrok_adapter import NgrokAdapter + +pytestmark = pytest.mark.unit + + +class _FakeTunnel: + def __init__(self, public_url: str = "https://fake.ngrok.io") -> None: + self.public_url = public_url + + +def _fake_connect(_port: int, _proto: str) -> _FakeTunnel: + return _FakeTunnel() + + +def _fake_disconnect(_url: Any) -> None: + return None + + +class TestNgrokAdapterLifecycle: + """Adapter must serialise concurrent start / stop calls.""" + + async def test_double_start_raises(self) -> None: + """A second start() while a tunnel is active raises RuntimeError.""" + adapter = NgrokAdapter() + with ( + patch( + "synthorg.integrations.tunnel.ngrok_adapter.ngrok.connect", + _fake_connect, + ), + patch( + "synthorg.integrations.tunnel.ngrok_adapter.ngrok.disconnect", + _fake_disconnect, + ), + ): + url = await adapter.start() + assert url == "https://fake.ngrok.io" + with pytest.raises(RuntimeError, match="already active"): + await adapter.start() + await adapter.stop() + + async def test_concurrent_starts_yield_one_tunnel(self) -> None: + """Two simultaneous start() calls: exactly one wins.""" + adapter = NgrokAdapter() + with ( + patch( + "synthorg.integrations.tunnel.ngrok_adapter.ngrok.connect", + _fake_connect, + ), + patch( + "synthorg.integrations.tunnel.ngrok_adapter.ngrok.disconnect", + _fake_disconnect, + ), + ): + results = await asyncio.gather( + adapter.start(), + adapter.start(), + return_exceptions=True, + ) + successes = [r for r in results if isinstance(r, str)] + errors = [r for r in results if isinstance(r, RuntimeError)] + assert len(successes) == 1 + assert len(errors) == 1 + await adapter.stop() + + async def test_stop_without_start_is_noop(self) -> None: + """stop() before any start() returns cleanly without disconnecting.""" + adapter = NgrokAdapter() + with patch( + "synthorg.integrations.tunnel.ngrok_adapter.ngrok.disconnect", + _fake_disconnect, + ): + await adapter.stop() # Must not raise. + assert adapter._tunnel is None diff --git a/tests/unit/integrations/test_replay_protection_threadsafety.py b/tests/unit/integrations/test_replay_protection_threadsafety.py new file mode 100644 index 0000000000..5bf9fb60cf --- /dev/null +++ b/tests/unit/integrations/test_replay_protection_threadsafety.py @@ -0,0 +1,74 @@ +"""Thread-safety tests for ReplayProtector. + +Two identical webhook payloads arriving simultaneously must not both +pass the nonce duplicate check. The ``threading.Lock`` around the +check-and-insert block guarantees exactly one accept per nonce. +""" + +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from synthorg.integrations.webhooks.replay_protection import ReplayProtector +from tests._shared.fake_clock import FakeClock + +pytestmark = pytest.mark.unit + + +def _epoch(clock: FakeClock) -> float: + return clock.now().timestamp() + + +class TestReplayProtectorThreadSafety: + """Concurrent thread access must remain safe.""" + + def test_concurrent_identical_nonces_yield_single_accept(self) -> None: + clock = FakeClock() + protector = ReplayProtector(window_seconds=300, clock=clock) + ts = _epoch(clock) + + def attempt() -> bool: + return protector.check(nonce="duplicate-nonce", timestamp=ts) + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(attempt) for _ in range(64)] + results = [f.result() for f in futures] + + accepts = [r for r in results if r] + rejects = [r for r in results if not r] + assert len(accepts) == 1 + assert len(rejects) == 63 + + def test_concurrent_distinct_nonces_all_accepted(self) -> None: + clock = FakeClock() + protector = ReplayProtector(window_seconds=300, clock=clock) + ts = _epoch(clock) + + def attempt(i: int) -> bool: + return protector.check(nonce=f"nonce-{i}", timestamp=ts) + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(attempt, i) for i in range(64)] + results = [f.result() for f in futures] + + assert all(results) + + def test_concurrent_eviction_does_not_corrupt(self) -> None: + clock = FakeClock() + protector = ReplayProtector( + window_seconds=300, + max_entries=8, + clock=clock, + ) + ts = _epoch(clock) + + def attempt(i: int) -> None: + protector.check(nonce=f"nonce-{i}", timestamp=ts) + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [pool.submit(attempt, i) for i in range(128)] + for f in futures: + f.result() + # The bounded store must stay within max_entries even under + # concurrent insert pressure. + assert len(protector._seen) <= 8 diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py new file mode 100644 index 0000000000..679ff298d4 --- /dev/null +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -0,0 +1,76 @@ +"""Canonical lifecycle pattern tests for ``OrgInflectionMonitor``. + +The unit ``test_monitor.py`` covers happy-path start / stop / tick. +This module verifies the lock-driven concurrency safety added per +``docs/reference/lifecycle-sync.md``: concurrent start, restart after +clean stop, and unrestartable flag after a drain timeout. +""" + +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest + +from synthorg.meta.chief_of_staff.inflection import OrgInflectionDetector +from synthorg.meta.chief_of_staff.monitor import OrgInflectionMonitor + +pytestmark = pytest.mark.unit + + +def _make_monitor() -> OrgInflectionMonitor: + builder = AsyncMock() + builder.build = AsyncMock(return_value=None) + return OrgInflectionMonitor( + detector=OrgInflectionDetector(), + snapshot_builder=builder, + sinks=(), + check_interval_minutes=60, + ) + + +class TestOrgInflectionMonitorLifecycleLock: + """Canonical pattern compliance.""" + + async def test_concurrent_starts_spawn_one_task(self) -> None: + monitor = _make_monitor() + try: + await asyncio.gather( + monitor.start(), + monitor.start(), + monitor.start(), + ) + assert monitor._task is not None + finally: + await monitor.stop() + + async def test_restart_after_clean_stop(self) -> None: + monitor = _make_monitor() + await monitor.start() + await monitor.stop() + # Cannot assert ``_task is None`` here -- mypy narrows the + # type and flags the subsequent ``start()`` as unreachable. + await monitor.start() + await monitor.stop() + + async def test_unrestartable_after_drain_timeout(self) -> None: + monitor = _make_monitor() + monitor._stop_drain_timeout_seconds = 0.05 + + async def hung_loop(self: OrgInflectionMonitor) -> None: + del self + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + # Suppress cancellation -- simulates a stuck drain. + await asyncio.sleep(1.0) + + with patch.object(OrgInflectionMonitor, "_loop", hung_loop): + await monitor.start() + await asyncio.sleep(0) + + with pytest.raises(TimeoutError): + await monitor.stop() + assert monitor._stop_failed is True + + with pytest.raises(RuntimeError, match="unrestartable"): + await monitor.start() diff --git a/tests/unit/providers/test_health_prober_lifecycle.py b/tests/unit/providers/test_health_prober_lifecycle.py new file mode 100644 index 0000000000..3cd7e39033 --- /dev/null +++ b/tests/unit/providers/test_health_prober_lifecycle.py @@ -0,0 +1,102 @@ +"""Lifecycle tests for ``ProviderHealthProber``. + +Verifies the canonical pattern (per ``docs/reference/lifecycle-sync.md``): + +* concurrent ``start()`` calls spawn at most one background task, +* a re-``start()`` after ``stop()`` works, +* a ``stop()`` whose drain exceeds the hard deadline marks the + prober unrestartable so the next ``start()`` refuses. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from synthorg.config.schema import ProviderConfig +from synthorg.providers.health import ProviderHealthTracker +from synthorg.providers.health_prober import ProviderHealthProber +from synthorg.settings.resolver import ConfigResolver + +pytestmark = pytest.mark.unit + + +def _make_prober() -> ProviderHealthProber: + config = MagicMock(spec=ProviderConfig) + config.base_url = "http://localhost:11434" + config.litellm_provider = "ollama" + config.auth_type = "none" + config.api_key = None + resolver = MagicMock(spec=ConfigResolver) + resolver.get_provider_configs = AsyncMock( + spec=ConfigResolver.get_provider_configs, + return_value={"test-local": config}, + ) + resolver.get_int = AsyncMock(spec=ConfigResolver.get_int, return_value=11434) + return ProviderHealthProber( + ProviderHealthTracker(), + resolver, + discovery_policy_loader=None, + interval_seconds=3600, + ) + + +class TestProviderHealthProberLifecycle: + """Canonical lifecycle pattern.""" + + async def test_concurrent_starts_spawn_one_task(self) -> None: + prober = _make_prober() + try: + await asyncio.gather( + prober.start(), + prober.start(), + prober.start(), + ) + assert prober._task is not None + task = prober._task + await asyncio.gather( + prober.start(), + prober.start(), + ) + assert prober._task is task + finally: + await prober.stop() + + async def test_restart_after_clean_stop(self) -> None: + prober = _make_prober() + await prober.start() + await prober.stop() + # After a clean stop the prober must restart. + await prober.start() + await prober.stop() + + async def test_unrestartable_after_drain_timeout(self) -> None: + """A drain that exceeds the deadline marks the service unrestartable.""" + prober = _make_prober() + prober._stop_drain_timeout_seconds = 0.05 + + # Replace _run_loop with a coroutine that swallows cancellation + # so the drain hangs and triggers the timeout path. + cancel_started = asyncio.Event() + + async def hung_loop(self: ProviderHealthProber) -> None: + del self + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cancel_started.set() + # Suppress cancellation; this simulates a stuck drain. + await asyncio.sleep(1.0) + + with patch.object(ProviderHealthProber, "_run_loop", hung_loop): + await prober.start() + # Let the hung loop start executing. + await asyncio.sleep(0) + + with pytest.raises(TimeoutError): + await prober.stop() + assert prober._stop_failed is True + + # Subsequent start must refuse. + with pytest.raises(RuntimeError, match="unrestartable"): + await prober.start() diff --git a/tests/unit/settings/test_backup_subscriber.py b/tests/unit/settings/test_backup_subscriber.py index 3544575c76..3bcb753a6f 100644 --- a/tests/unit/settings/test_backup_subscriber.py +++ b/tests/unit/settings/test_backup_subscriber.py @@ -29,7 +29,7 @@ def _make_subscriber( """ scheduler = MagicMock() type(scheduler).is_running = PropertyMock(return_value=scheduler_running) - scheduler.start = MagicMock() + scheduler.start = AsyncMock() scheduler.stop = AsyncMock() scheduler.reschedule = MagicMock() diff --git a/tests/unit/tools/mcp/test_cache_threadsafety.py b/tests/unit/tools/mcp/test_cache_threadsafety.py new file mode 100644 index 0000000000..fe0b4c89a1 --- /dev/null +++ b/tests/unit/tools/mcp/test_cache_threadsafety.py @@ -0,0 +1,70 @@ +"""Thread-safety tests for MCPResultCache. + +Concurrent ``get`` / ``put`` / ``invalidate`` calls from a thread +pool must not raise (``RuntimeError: dictionary changed size``, +``KeyError`` on the post-delete deepcopy path) and must not lose the +LRU eviction guarantee. +""" + +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from synthorg.tools.base import ToolExecutionResult +from synthorg.tools.mcp.cache import MCPResultCache + +pytestmark = pytest.mark.unit + + +class TestMCPResultCacheThreadSafety: + """Concurrent thread access must remain safe.""" + + def test_concurrent_get_put_no_corruption(self) -> None: + cache = MCPResultCache(max_size=64, ttl_seconds=120.0) + + def writer(i: int) -> None: + cache.put(f"tool-{i % 8}", {"i": i}, ToolExecutionResult(content=str(i))) + + def reader(i: int) -> None: + cache.get(f"tool-{i % 8}", {"i": i}) + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [] + for i in range(200): + fn = writer if i % 2 == 0 else reader + futures.append(pool.submit(fn, i)) + for f in futures: + f.result() + + def test_concurrent_invalidate_does_not_raise(self) -> None: + cache = MCPResultCache(max_size=64, ttl_seconds=120.0) + for i in range(32): + cache.put(f"tool-{i % 4}", {"i": i}, ToolExecutionResult(content=str(i))) + + def writer(i: int) -> None: + cache.put(f"tool-{i % 4}", {"i": i}, ToolExecutionResult(content=str(i))) + + def invalidator(i: int) -> None: + del i + cache.invalidate(tool_name="tool-0") + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [ + pool.submit(invalidator if i % 5 == 0 else writer, i) + for i in range(120) + ] + for f in futures: + f.result() + + def test_eviction_under_concurrency_keeps_max_size(self) -> None: + cache = MCPResultCache(max_size=8, ttl_seconds=120.0) + + def writer(i: int) -> None: + cache.put(f"tool-{i}", {}, ToolExecutionResult(content=str(i))) + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [pool.submit(writer, i) for i in range(64)] + for f in futures: + f.result() + # Internal cache must stay bounded. + assert len(cache._cache) <= 8 From e93e7b0384b972f6d2e08d8e8d6987b7891da6e3 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 13:08:43 +0200 Subject: [PATCH 02/13] fix: address pre-PR review findings for #1708 Closes #1708. CRITICAL fixes: - simulations.py: replace racy get-then-save with atomic SimulationStore.register_if_absent so two concurrent requests with the same simulation_id cannot both spawn runners (security TOCTOU) - web/src/api/endpoints/backup.ts: send required Idempotency-Key header (auto-generated UUID per call) so the backend mandatory-key change does not break the dashboard - web/src/api/endpoints/clients.ts: detect HTTP 409 on startSimulation and fall back to getSimulation, so retries observe the in-flight runner instead of throwing MAJOR fixes: - Strip 'audit #N' references from 6 source / test docstrings (per CLAUDE.md no-issue-back-references rule) - event_stream/stream.py: drop _seen_event_ids[session_id] when the last subscriber unsubscribes; prevents per-session dedup-map leak on long-lived hubs with churn (+ 2 tests) - docs/reference/lifecycle-sync.md: add the 6 newly-compliant services to canonical-examples list, document in-place runner variant for ContinuousMode - docs/design/observability.md: enumerate EVENT_STREAM_HUB_PUBLISH_DEDUPED in event-stream events table - docs/openapi/openapi.json: regenerated via scripts/export_openapi.py MEDIUM fixes: - ngrok_adapter.py: include safe_error_description(exc) in TunnelError message body (operator triage without log access) - replay_protection.py: simplify lock-then-test ternary - backup.py: add max_length=255 to Idempotency-Key Parameter - docs/design/backup.md, docs/design/client-simulation.md: document the new mandatory-key and 409 contracts - docs/licensing.md: tighten ADR cross-reference wording - tests/integration: new test_start_simulation_duplicate_id_returns_409 LOW fixes: - cache.py docstring: spell out the deepcopy invariant - escalation/factory.py: expand _require_persistence rationale Pre-PR review pipeline: 16 agents launched in parallel; 14 reported. Pre-existing / out-of-scope items skipped per triage; 3 invalid findings (PEP 758 false positives, asyncio race under GIL) ignored. Triage: _audit/pre-pr-review/triage.md. --- docs/design/backup.md | 2 +- docs/design/client-simulation.md | 11 ++++ docs/design/observability.md | 7 +++ docs/licensing.md | 2 +- docs/reference/lifecycle-sync.md | 9 +++ scripts/mock_spec_baseline.txt | 35 +++++------ src/synthorg/api/controllers/backup.py | 5 ++ src/synthorg/api/controllers/simulations.py | 32 +++++----- src/synthorg/client/store.py | 17 ++++++ .../conflict_resolution/escalation/factory.py | 16 +++-- .../communication/event_stream/stream.py | 15 ++++- .../integrations/tunnel/ngrok_adapter.py | 5 +- .../webhooks/replay_protection.py | 6 +- src/synthorg/tools/mcp/cache.py | 5 ++ .../api/controllers/test_client_simulation.py | 30 ++++++++++ .../test_backup_required_idempotency.py | 18 +++--- .../test_simulations_idempotency.py | 60 ++++++++----------- .../escalation/test_factory_registry.py | 10 ++-- .../event_stream/test_stream_dedup.py | 33 ++++++++-- web/src/api/endpoints/backup.ts | 16 ++++- web/src/api/endpoints/clients.ts | 25 ++++++-- 21 files changed, 248 insertions(+), 111 deletions(-) diff --git a/docs/design/backup.md b/docs/design/backup.md index fe765eb9bb..8a175ac55f 100644 --- a/docs/design/backup.md +++ b/docs/design/backup.md @@ -26,7 +26,7 @@ The backup system protects persistent data (persistence DB, agent memory, and co | Scheduled | Configurable interval (default: 6h) | Background, non-blocking | | Pre-shutdown | `Company.shutdown()` / SIGTERM | Synchronous, skips compression | | Post-startup | After config load, before accepting tasks | Snapshot as recovery point | -| Manual | `POST /api/v1/admin/backups` | On-demand, returns manifest | +| Manual | `POST /api/v1/admin/backups` | On-demand, returns manifest. **Requires the `Idempotency-Key` header** (RFC-style retry-safe key, max 255 chars); identical keys within 24h return the cached manifest instead of starting a second backup so a 5xx-driven client retry cannot launch concurrent backups and violate the at-most-one-running invariant. Missing or empty header yields HTTP 400. | | Pre-migration | Before restore operations | Safety net, automatic | ## Restore Flow diff --git a/docs/design/client-simulation.md b/docs/design/client-simulation.md index c078c0197a..94d874bb5a 100644 --- a/docs/design/client-simulation.md +++ b/docs/design/client-simulation.md @@ -224,6 +224,17 @@ task lifecycle state machine. `ContinuousMode` provides event-driven always-on simulation with scheduled requirement generation and review triggers. +### Idempotency + +`POST /api/v1/simulations/` registers the run via +`SimulationStore.register_if_absent`, an atomic check-and-insert under the +store's lock. A redelivered request (JetStream redelivery, HTTP 5xx-driven +retry, etc.) carrying the same `simulation_id` returns HTTP 409 Conflict +instead of spawning a second runner that races the first on +`update_status` and corrupts metrics. Clients that supply their own +`simulation_id` get retry safety for free; clients that omit it receive a +fresh UUID per call and never collide. + --- ## Configuration diff --git a/docs/design/observability.md b/docs/design/observability.md index 72bf9fa0ac..3f868c4de3 100644 --- a/docs/design/observability.md +++ b/docs/design/observability.md @@ -197,6 +197,13 @@ names. Format: `".."` (e.g., `"api.request.started"`). All MCP handler log calls go through `logger.warning(EVENT, error_type=type(exc).__name__, error=safe_error_description(exc))` on credential-sensitive paths (never `logger.exception(..., error=str(exc))`) to avoid leaking secrets through traceback frame-locals (SEC-1). +**Event stream events (`observability/events/event_stream.py`):** + +| Constant | Level | When fired | +|----------|-------|------------| +| `EVENT_STREAM_HUB_PUBLISH_FAILED` | WARNING | A subscriber queue rejected the event (full); the publisher continues (best-effort fan-out). | +| `EVENT_STREAM_HUB_PUBLISH_DEDUPED` | WARNING | An event was rejected as a duplicate within the per-session sliding-window TTL (default 60s). The hub keys dedup on `event.id`; identical ids within the window are dropped so an upstream retry (e.g. webhook handler that catches a transient publish failure and retries) cannot deliver the same event twice. The window is bounded per session (default 1024 entries, evicted on insert) so a noisy session cannot exhaust memory. | + **API entry-point boundary events (`observability/events/api.py`):** | Constant | Level | When fired | diff --git a/docs/licensing.md b/docs/licensing.md index 0003edbf5d..bdaf240511 100644 --- a/docs/licensing.md +++ b/docs/licensing.md @@ -118,7 +118,7 @@ SynthOrg's default install (SQLite-only) carries only permissive licenses (MIT, These are linked dynamically (separate `pip`-installable packages) and the LGPL anti-circumvention clause is satisfied by the standard `pip` replacement workflow. Operators who redistribute combined binaries that include the `postgres` extra must publish a NOTICE listing the LGPL components and preserve replacement-version flexibility. Operators using SQLite (the default) carry no LGPL obligations. -See [`docs/research/lgpl-postgres-driver-decision.md`](research/lgpl-postgres-driver-decision.md) for the full rationale. +The retention of LGPL drivers in the optional extra was evaluated explicitly: the [LGPL Postgres Driver Decision](research/lgpl-postgres-driver-decision.md) compares the trade-offs (asyncpg swap, vendor / fork, accept-with-ADR) and documents why dynamic linkage in an opt-in extra is acceptable under BUSL-1.1's narrowed Additional Use Grant. --- diff --git a/docs/reference/lifecycle-sync.md b/docs/reference/lifecycle-sync.md index a24e2d491d..ac3a136565 100644 --- a/docs/reference/lifecycle-sync.md +++ b/docs/reference/lifecycle-sync.md @@ -25,3 +25,12 @@ For services whose `stop()` drains across `await` boundaries, wrap the drain in - `IntegrationsHealthProber` - `EscalationNotifySubscriber` - `EscalationSweeper` +- `ProviderHealthProber` (`providers/health_prober.py`) +- `OrgInflectionMonitor` (`meta/chief_of_staff/monitor.py`) +- `BackupScheduler` (`backup/scheduler.py`) +- `PruningService` (`hr/pruning/service.py`) +- `NgrokAdapter` (`integrations/tunnel/ngrok_adapter.py`): lifecycle lock only; no spawned background task, so the drain timeout / unrestartable flag do not apply. + +### In-place runner variant + +`ContinuousMode` (`client/continuous.py`) is **not** a background-task service: `start()` runs the simulation loop on the calling coroutine and only returns when `stop()` signals the stop event. The lifecycle lock therefore guards only the `_running` flag transition (acquire briefly at the top of `start()` to check-and-set, release before the loop body, re-acquire in the `finally` to clear the flag). Holding the lock across the full body would deadlock a second concurrent caller: it would queue on the lock until the first finished and then enter an empty state. Document this distinction when adding new in-place runners. diff --git a/scripts/mock_spec_baseline.txt b/scripts/mock_spec_baseline.txt index e0197005fd..f11e283628 100644 --- a/scripts/mock_spec_baseline.txt +++ b/scripts/mock_spec_baseline.txt @@ -325,12 +325,12 @@ tests/unit/api/controllers/test_backup.py:400:24 tests/unit/api/controllers/test_backup.py:401:32 tests/unit/api/controllers/test_backup.py:402:29 tests/unit/api/controllers/test_backup.py:403:34 -tests/unit/api/controllers/test_backup_required_idempotency.py:45:14 -tests/unit/api/controllers/test_backup_required_idempotency.py:46:28 -tests/unit/api/controllers/test_backup_required_idempotency.py:47:16 -tests/unit/api/controllers/test_backup_required_idempotency.py:49:26 -tests/unit/api/controllers/test_backup_required_idempotency.py:55:12 -tests/unit/api/controllers/test_backup_required_idempotency.py:82:22 +tests/unit/api/controllers/test_backup_required_idempotency.py:43:14 +tests/unit/api/controllers/test_backup_required_idempotency.py:44:28 +tests/unit/api/controllers/test_backup_required_idempotency.py:45:16 +tests/unit/api/controllers/test_backup_required_idempotency.py:47:26 +tests/unit/api/controllers/test_backup_required_idempotency.py:53:12 +tests/unit/api/controllers/test_backup_required_idempotency.py:80:22 tests/unit/api/controllers/test_collaboration.py:357:23 tests/unit/api/controllers/test_company.py:108:31 tests/unit/api/controllers/test_coordination.py:77:18 @@ -420,17 +420,18 @@ tests/unit/api/controllers/test_setup_has_gpu.py:76:27 tests/unit/api/controllers/test_setup_locales.py:246:33 tests/unit/api/controllers/test_setup_locales.py:269:33 tests/unit/api/controllers/test_setup_locales.py:316:33 -tests/unit/api/controllers/test_simulations_idempotency.py:36:16 -tests/unit/api/controllers/test_simulations_idempotency.py:52:38 -tests/unit/api/controllers/test_simulations_idempotency.py:54:30 -tests/unit/api/controllers/test_simulations_idempotency.py:55:21 -tests/unit/api/controllers/test_simulations_idempotency.py:56:34 -tests/unit/api/controllers/test_simulations_idempotency.py:57:31 -tests/unit/api/controllers/test_simulations_idempotency.py:58:38 -tests/unit/api/controllers/test_simulations_idempotency.py:59:16 -tests/unit/api/controllers/test_simulations_idempotency.py:61:32 -tests/unit/api/controllers/test_simulations_idempotency.py:62:12 -tests/unit/api/controllers/test_simulations_idempotency.py:68:11 +tests/unit/api/controllers/test_simulations_idempotency.py:40:16 +tests/unit/api/controllers/test_simulations_idempotency.py:41:52 +tests/unit/api/controllers/test_simulations_idempotency.py:44:38 +tests/unit/api/controllers/test_simulations_idempotency.py:46:30 +tests/unit/api/controllers/test_simulations_idempotency.py:47:21 +tests/unit/api/controllers/test_simulations_idempotency.py:48:34 +tests/unit/api/controllers/test_simulations_idempotency.py:49:31 +tests/unit/api/controllers/test_simulations_idempotency.py:50:38 +tests/unit/api/controllers/test_simulations_idempotency.py:51:16 +tests/unit/api/controllers/test_simulations_idempotency.py:53:32 +tests/unit/api/controllers/test_simulations_idempotency.py:54:12 +tests/unit/api/controllers/test_simulations_idempotency.py:60:11 tests/unit/api/controllers/test_sse_keepalive_setting.py:28:31 tests/unit/api/controllers/test_sse_keepalive_setting.py:29:41 tests/unit/api/controllers/test_sse_revalidate.py:44:21 diff --git a/src/synthorg/api/controllers/backup.py b/src/synthorg/api/controllers/backup.py index e930220784..d4ddac64a7 100644 --- a/src/synthorg/api/controllers/backup.py +++ b/src/synthorg/api/controllers/backup.py @@ -98,6 +98,11 @@ async def create_backup( ), required=True, min_length=1, + # Bound the key length so a malicious client cannot + # exhaust the durable idempotency store with arbitrarily + # large keys; 255 chars is plenty for UUIDs / SHAs and + # matches common header-value column widths. + max_length=255, ), ], ) -> ApiResponse[BackupManifest]: diff --git a/src/synthorg/api/controllers/simulations.py b/src/synthorg/api/controllers/simulations.py index 3c7209a77d..7700e57971 100644 --- a/src/synthorg/api/controllers/simulations.py +++ b/src/synthorg/api/controllers/simulations.py @@ -274,29 +274,27 @@ async def start_simulation( """ app_state: AppState = state.app_state sim_state = app_state.client_simulation_state - # Idempotency guard (audit #133): a JetStream redelivery or - # HTTP 5xx retry of /simulations/start with the same - # ``simulation_id`` would otherwise spawn a second runner that - # races the first on ``simulation_store.update_status``, - # corrupting metrics with last-write-wins. Reject the second - # request with HTTP 409 Conflict so the caller can fall back - # to ``GET /simulations/{id}`` to observe the in-flight run. - with contextlib.suppress(KeyError): - existing = await sim_state.simulation_store.get(data.config.simulation_id) - msg = ( - f"Simulation {data.config.simulation_id!r} already exists " - f"(status={existing.status!r}); cannot start a second runner " - "for the same id" - ) - raise ConflictError(msg) - record = SimulationRecord( simulation_id=data.config.simulation_id, config=data.config, status="running", started_at=datetime.now(UTC), ) - await sim_state.simulation_store.save(record) + # A JetStream redelivery or HTTP 5xx retry of /simulations/start + # with the same ``simulation_id`` would otherwise spawn a second + # runner that races the first on + # ``simulation_store.update_status``, corrupting metrics with + # last-write-wins. ``register_if_absent`` performs the check + # and insert atomically under the store's lock, so two + # concurrent callers cannot both observe absence and proceed. + # The losing caller gets HTTP 409 and can fall back to + # ``GET /simulations/{id}`` to observe the in-flight run. + if not await sim_state.simulation_store.register_if_absent(record): + msg = ( + f"Simulation {data.config.simulation_id!r} already exists; " + "cannot start a second runner for the same id" + ) + raise ConflictError(msg) _publish_event(request, WsEventType.SIMULATION_STARTED, record) async def runner_task() -> None: diff --git a/src/synthorg/client/store.py b/src/synthorg/client/store.py index 261e95208f..549e07ddbd 100644 --- a/src/synthorg/client/store.py +++ b/src/synthorg/client/store.py @@ -174,6 +174,23 @@ async def save(self, record: SimulationRecord) -> None: async with self._lock: self._runs[record.simulation_id] = record + async def register_if_absent(self, record: SimulationRecord) -> bool: + """Atomically insert *record* if no entry exists for its id. + + Returns ``True`` when *record* was inserted (the caller is the + "winner" and should spawn the runner), ``False`` when a record + for ``record.simulation_id`` already exists (the caller should + return HTTP 409 and let the existing runner finish). + + The check-and-insert happens under ``self._lock`` so two + concurrent callers cannot both observe absence and proceed. + """ + async with self._lock: + if record.simulation_id in self._runs: + return False + self._runs[record.simulation_id] = record + return True + async def get(self, simulation_id: str) -> SimulationRecord: """Return the record by id or raise ``KeyError``.""" async with self._lock: diff --git a/src/synthorg/communication/conflict_resolution/escalation/factory.py b/src/synthorg/communication/conflict_resolution/escalation/factory.py index e991540789..c37e645dd3 100644 --- a/src/synthorg/communication/conflict_resolution/escalation/factory.py +++ b/src/synthorg/communication/conflict_resolution/escalation/factory.py @@ -1,8 +1,8 @@ """Factories for the escalation queue backend and decision processor. -Both factories dispatch via small registry maps (per audit #69) so -adding a new backend or decision strategy is a single registry entry -rather than a new branch in an if/elif chain. The shape mirrors +Both factories dispatch via small registry maps so adding a new backend +or decision strategy is a single registry entry rather than a new +branch in an if/elif chain. The shape mirrors ``synthorg.persistence.registry.PersistenceBackendRegistry`` and the ``match/case`` dispatch in ``synthorg.communication.bus``. """ @@ -53,7 +53,15 @@ def _require_persistence( config_backend: str, persistence: PersistenceBackend | None, ) -> PersistenceBackend: - """Reject a missing or mismatched persistence backend, logging before raise.""" + """Reject a missing or mismatched persistence backend, logging before raise. + + The escalation queue store's backend (``memory`` / ``sqlite`` / + ``postgres``) MUST line up with the operator's persistence choice. + A mismatch -- typically because someone hand-injected a backend + instance that does not match the configuration -- is surfaced at + construction time rather than allowed to silently fall back to an + in-memory state or to interact with the wrong driver. + """ if persistence is None: msg = f"{config_backend} backend requires a connected persistence backend" logger.warning( diff --git a/src/synthorg/communication/event_stream/stream.py b/src/synthorg/communication/event_stream/stream.py index b2a4b65694..0aee5f4239 100644 --- a/src/synthorg/communication/event_stream/stream.py +++ b/src/synthorg/communication/event_stream/stream.py @@ -69,9 +69,9 @@ def __init__( # Per-session insertion-ordered map of ``event.id`` -> # ``monotonic_seen_at``. Bounded per session and TTL-evicted on # publish so a long-lived session cannot grow the dedup window - # without bound. Audit #133: retried publishes (e.g. webhook - # handler that catches a transient publish failure and retries) - # would otherwise emit the same event twice to all subscribers. + # without bound. Without this map, retried publishes (e.g. a + # webhook handler that catches a transient publish failure and + # retries) would emit the same event twice to all subscribers. self._seen_event_ids: dict[str, OrderedDict[str, float]] = {} self._lock = asyncio.Lock() @@ -113,6 +113,15 @@ async def unsubscribe( queues.remove(queue) if not queues: del self._subscribers[session_id] + # Drop the per-session dedup window once the last + # subscriber leaves so a long-lived hub with churn + # cannot leak per-session state for sessions that + # have no subscribers. The TTL eviction in + # ``publish()`` only fires on publishes; without this + # cleanup, the dedup map for a finished session would + # only shed entries on the rare case of a stray + # publish to that session. + self._seen_event_ids.pop(session_id, None) async def publish(self, event: StreamEvent) -> None: """Fan out an event to all subscribers for its session. diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index 1429a82a33..a9814cd153 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -92,12 +92,13 @@ async def start(self) -> str: except Exception as exc: # ngrok auth token env var may be echoed in exception # messages; scrub + drop traceback. + safe_desc = safe_error_description(exc) logger.warning( TUNNEL_ERROR, error_type=type(exc).__name__, - error=safe_error_description(exc), + error=safe_desc, ) - msg = f"Failed to start ngrok tunnel: {type(exc).__name__}" + msg = f"Failed to start ngrok tunnel: {safe_desc}" raise TunnelError(msg) from exc logger.info( diff --git a/src/synthorg/integrations/webhooks/replay_protection.py b/src/synthorg/integrations/webhooks/replay_protection.py index 42b53566a8..a1180a74ca 100644 --- a/src/synthorg/integrations/webhooks/replay_protection.py +++ b/src/synthorg/integrations/webhooks/replay_protection.py @@ -203,10 +203,8 @@ def check( # noqa: PLR0911 key = _fingerprint_nonce(nonce) with self._lock: self._evict_locked(now) - if key in self._seen: - duplicate = True - else: - duplicate = False + duplicate = key in self._seen + if not duplicate: self._seen[key] = now # Bound the store: evict oldest insertion(s) if over limit. while len(self._seen) > self._max_entries: diff --git a/src/synthorg/tools/mcp/cache.py b/src/synthorg/tools/mcp/cache.py index 0475e4a276..dd7fddcb99 100644 --- a/src/synthorg/tools/mcp/cache.py +++ b/src/synthorg/tools/mcp/cache.py @@ -30,6 +30,11 @@ class MCPResultCache: Thread-safe via an internal ``threading.Lock`` so concurrent threadpool-dispatched tool invocations cannot interleave the read-decision-write blocks in :meth:`get` and :meth:`put`. + Both methods ``copy.deepcopy`` the cached value on the way in and + out so a caller mutating the returned ``ToolExecutionResult`` -- + e.g. appending to a list field, replacing a dict entry -- cannot + poison the next cache hit. The deepcopy is intentional, not an + inefficiency to optimise away. Keys are derived from tool name and arguments. Args: diff --git a/tests/integration/api/controllers/test_client_simulation.py b/tests/integration/api/controllers/test_client_simulation.py index 21c856f285..8f4d85a401 100644 --- a/tests/integration/api/controllers/test_client_simulation.py +++ b/tests/integration/api/controllers/test_client_simulation.py @@ -315,6 +315,36 @@ async def test_get_missing_simulation_returns_404( resp = client.get("/api/v1/simulations/missing") assert resp.status_code == 404 + async def test_start_simulation_duplicate_id_returns_409( + self, + fake_persistence: FakePersistenceBackend, + fake_message_bus: FakeMessageBus, + ) -> None: + """A second start with the same simulation_id is rejected.""" + with _build_client(fake_persistence, fake_message_bus) as client: + client.headers.update(make_auth_headers("ceo")) + client.post( + "/api/v1/clients/", + json={ + "client_id": "sim", + "name": "Sim", + "persona": "Persona", + }, + ) + payload = { + "config": { + "simulation_id": "fixed-sim-001", + "project_id": "proj-1", + "rounds": 1, + "clients_per_round": 1, + "requirements_per_client": 1, + }, + } + first = client.post("/api/v1/simulations/", json=payload) + assert first.status_code == 201 + second = client.post("/api/v1/simulations/", json=payload) + assert second.status_code == 409 + class TestReviewController: async def test_missing_task_returns_404( diff --git a/tests/unit/api/controllers/test_backup_required_idempotency.py b/tests/unit/api/controllers/test_backup_required_idempotency.py index c74903c217..fe547af673 100644 --- a/tests/unit/api/controllers/test_backup_required_idempotency.py +++ b/tests/unit/api/controllers/test_backup_required_idempotency.py @@ -1,15 +1,13 @@ """Idempotency-Key is mandatory for POST /admin/backups. -Per audit #133 (idempotency / retry safety): without a key, a -network-flake-driven 5xx retry could launch concurrent backups and -violate the at-most-one-running invariant. The header is now -required by Litestar's parameter validation; missing or empty values -yield HTTP 400. - -The shape of these tests intentionally avoids spinning up the full -Litestar app: we inspect the route handler's parameter signature and -verify the controller correctly invokes the idempotency service when -the key is supplied. +Without a key, a network-flake-driven 5xx retry could launch +concurrent backups and violate the at-most-one-running invariant. +The header is required by Litestar's parameter validation; missing +or empty values yield HTTP 400. + +These tests avoid spinning up the full Litestar app: we inspect the +route handler's parameter signature and verify the controller +correctly invokes the idempotency service when the key is supplied. """ import inspect diff --git a/tests/unit/api/controllers/test_simulations_idempotency.py b/tests/unit/api/controllers/test_simulations_idempotency.py index f544a17cb0..f24e4afa09 100644 --- a/tests/unit/api/controllers/test_simulations_idempotency.py +++ b/tests/unit/api/controllers/test_simulations_idempotency.py @@ -1,9 +1,9 @@ """Idempotency guard tests for ``POST /simulations/``. -Per audit #133: a redelivered ``start_simulation`` request with the -same ``simulation_id`` must not spawn a second runner that races the -first on the in-memory store. The controller now rejects the second -request with HTTP 409 Conflict. +A redelivered ``start_simulation`` request with the same +``simulation_id`` must not spawn a second runner that races the first +on the in-memory store. The controller rejects the second request +with HTTP 409 Conflict. """ import contextlib @@ -16,7 +16,6 @@ StartSimulationPayload, ) from synthorg.client.models import SimulationConfig -from synthorg.client.store import SimulationRecord from synthorg.core.domain_errors import ConflictError pytestmark = pytest.mark.unit @@ -31,24 +30,17 @@ def _make_config(simulation_id: str = "sim-001") -> SimulationConfig: ) -def _make_state_with_existing(record: SimulationRecord | None) -> MagicMock: - """Build a mocked Litestar state whose simulation_store returns *record*.""" - sim_state = MagicMock() - if record is None: - - async def _raise(_id: str) -> SimulationRecord: - del _id - msg = "Simulation not found" - raise KeyError(msg) - - sim_state.simulation_store.get = _raise - else: - - async def _return(_id: str) -> SimulationRecord: - del _id - return record +def _make_state(*, claim_succeeds: bool) -> MagicMock: + """Build a mocked Litestar state with a controllable register_if_absent. - sim_state.simulation_store.get = _return + When *claim_succeeds* is ``True``, ``register_if_absent`` returns + True (fresh id), simulating the first request. When ``False`` it + returns False (id already registered), simulating a duplicate. + """ + sim_state = MagicMock() + sim_state.simulation_store.register_if_absent = AsyncMock( + return_value=claim_succeeds, + ) sim_state.simulation_store.save = AsyncMock() sim_state.background_tasks = set() sim_state.intake_engine = MagicMock() @@ -72,12 +64,8 @@ class TestSimulationsIdempotency: """Duplicate ``simulation_id`` is rejected with HTTP 409.""" async def test_duplicate_id_rejected_with_conflict(self) -> None: - existing = SimulationRecord( - simulation_id="sim-001", - config=_make_config(), - status="running", - ) - state = _make_state_with_existing(existing) + # Store reports the id was already present (claim fails). + state = _make_state(claim_succeeds=False) ctrl = SimulationController(owner=SimulationController) # type: ignore[arg-type] payload = StartSimulationPayload(config=_make_config()) @@ -90,6 +78,8 @@ async def test_duplicate_id_rejected_with_conflict(self) -> None: ) assert exc.value.status_code == 409 assert "already exists" in str(exc.value) + sim_store = state.app_state.client_simulation_state.simulation_store + sim_store.register_if_absent.assert_awaited_once() async def test_first_request_passes_idempotency_check(self) -> None: """A fresh ``simulation_id`` survives the idempotency check. @@ -97,16 +87,16 @@ async def test_first_request_passes_idempotency_check(self) -> None: We cannot easily exercise the full happy path here without a full app fixture (the runner requires intake_engine etc.). The check verifies the controller progresses past the - idempotency guard and reaches ``simulation_store.save``. + idempotency guard and calls ``register_if_absent``. """ - state = _make_state_with_existing(None) + state = _make_state(claim_succeeds=True) ctrl = SimulationController(owner=SimulationController) # type: ignore[arg-type] payload = StartSimulationPayload(config=_make_config(simulation_id="sim-002")) - # The handler will reach .save() then attempt to spawn the - # runner. We tolerate any post-save error since this test - # only verifies idempotency-guard behaviour, not the runner - # plumbing exercised in the integration suite. + # The handler will reach the register call then attempt to + # spawn the runner. We tolerate any post-claim error since + # this test only verifies idempotency-guard behaviour, not + # the runner plumbing exercised in the integration suite. with contextlib.suppress(Exception): await ctrl.start_simulation.fn( ctrl, @@ -115,4 +105,4 @@ async def test_first_request_passes_idempotency_check(self) -> None: data=payload, ) sim_store = state.app_state.client_simulation_state.simulation_store - sim_store.save.assert_awaited_once() + sim_store.register_if_absent.assert_awaited_once() diff --git a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py index af856183c1..adb4a2572e 100644 --- a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py +++ b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py @@ -1,10 +1,10 @@ """Tests for the registry-based escalation factory dispatch. -Per audit #69: the factory used to dispatch on a hardcoded if/elif -chain. It now consults a frozen registry map keyed by the config -discriminator. These tests verify each registered branch builds the -expected store and that an unregistered key raises ValueError with a -helpful message listing the available options. +The factory consults a frozen registry map keyed by the config +discriminator rather than a hardcoded if/elif chain. These tests +verify each registered branch builds the expected store and that an +unregistered key raises ValueError with a helpful message listing the +available options. """ from typing import cast diff --git a/tests/unit/communication/event_stream/test_stream_dedup.py b/tests/unit/communication/event_stream/test_stream_dedup.py index 11c23af3b6..ff545cc707 100644 --- a/tests/unit/communication/event_stream/test_stream_dedup.py +++ b/tests/unit/communication/event_stream/test_stream_dedup.py @@ -1,10 +1,9 @@ """Dedup tests for ``EventStreamHub.publish``. -Per audit #133: a retried publish (e.g. a webhook handler that -catches a transient publish failure) must not deliver the same event -twice to subscribers. The hub keeps a per-session sliding-window of -seen ``event.id`` values; identical ids within the TTL are skipped -and logged. +A retried publish (e.g. a webhook handler that catches a transient +publish failure) must not deliver the same event twice to subscribers. +The hub keeps a per-session sliding-window of seen ``event.id`` +values; identical ids within the TTL are skipped and logged. """ import asyncio @@ -114,3 +113,27 @@ async def test_dedup_window_bounded_per_session(self) -> None: await hub.publish(_event(event_id=f"evt-{i}")) seen = hub._seen_event_ids["session-1"] assert len(seen) <= 8 + + async def test_unsubscribe_clears_dedup_window_for_session(self) -> None: + """Last unsubscribe drops the per-session dedup window.""" + clock = FakeClock() + hub = EventStreamHub(dedup_ttl_seconds=60.0, clock=clock) + queue = await hub.subscribe("session-1") + + await hub.publish(_event(event_id="evt-001")) + assert "session-1" in hub._seen_event_ids + + await hub.unsubscribe("session-1", queue) + assert "session-1" not in hub._seen_event_ids + + async def test_unsubscribe_with_remaining_subscribers_keeps_dedup(self) -> None: + """Dedup window stays while any subscriber remains for the session.""" + clock = FakeClock() + hub = EventStreamHub(dedup_ttl_seconds=60.0, clock=clock) + q1 = await hub.subscribe("session-1") + await hub.subscribe("session-1") + + await hub.publish(_event(event_id="evt-001")) + await hub.unsubscribe("session-1", q1) + # Other subscriber still present, dedup map should persist. + assert "session-1" in hub._seen_event_ids diff --git a/web/src/api/endpoints/backup.ts b/web/src/api/endpoints/backup.ts index a428e08d3c..1960a6e98c 100644 --- a/web/src/api/endpoints/backup.ts +++ b/web/src/api/endpoints/backup.ts @@ -2,8 +2,20 @@ import { apiClient, unwrap, unwrapVoid } from '../client' import type { BackupInfo, BackupManifest, RestoreRequest, RestoreResponse } from '../types/backup' import type { ApiResponse } from '../types/http' -export async function createBackup(): Promise { - const response = await apiClient.post>('/admin/backups') +export async function createBackup(idempotencyKey?: string): Promise { + // The backend requires the Idempotency-Key header on POST + // /admin/backups so a 5xx-driven retry cannot launch concurrent + // backups and violate the at-most-one-running invariant. Callers + // may supply their own key (recommended for retry semantics); + // otherwise we mint a fresh UUID per call so first-time submissions + // still satisfy the contract without forcing every caller to think + // about it. + const key = idempotencyKey ?? crypto.randomUUID() + const response = await apiClient.post>( + '/admin/backups', + null, + { headers: { 'Idempotency-Key': key } }, + ) return unwrap(response) } diff --git a/web/src/api/endpoints/clients.ts b/web/src/api/endpoints/clients.ts index 27a9ed15a1..8b4ae245e3 100644 --- a/web/src/api/endpoints/clients.ts +++ b/web/src/api/endpoints/clients.ts @@ -1,3 +1,5 @@ +import axios from 'axios' + import { apiClient, unwrap, unwrapPaginated, type PaginatedResult } from '../client' import type { ApiResponse, PaginatedResponse, PaginationParams } from '../types/http' @@ -262,11 +264,24 @@ export async function getSimulation( export async function startSimulation( config: SimulationConfig, ): Promise { - const response = await apiClient.post>( - '/simulations/', - { config }, - ) - return unwrap(response) + try { + const response = await apiClient.post>( + '/simulations/', + { config }, + ) + return unwrap(response) + } catch (err) { + // The backend returns HTTP 409 when a simulation with + // ``config.simulation_id`` is already registered (a redelivery + // or 5xx-driven retry of the same request). Fall back to + // fetching the existing run so the caller's retry path is + // idempotent: retries observe the in-flight runner instead of + // raising and forcing the user to refresh. + if (axios.isAxiosError(err) && err.response?.status === 409) { + return getSimulation(config.simulation_id) + } + throw err + } } export async function cancelSimulation( From 7458d894675bdd77eaa5f4812a4f8520aba85204 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 13:51:57 +0200 Subject: [PATCH 03/13] fix: babysit round 1, 19 findings (17 coderabbit inline + 2 outside-diff) Findings (per the rolling CodeRabbit review on PR #1717): Concurrency: - 5 services: stop no longer reassigns _lifecycle_lock; only the loop-bound stop_event / wake_event is recreated. Replacing the lock let a caller queued on the old lock and a fresh caller acquiring the new lock both proceed concurrently. Affects scheduler.py, monitor.py, health_prober.py, hr/pruning/service.py, and the canonical sweeper.py (same pattern, same bug). - stream.py: validate dedup_ttl_seconds and dedup_max_entries_per_session at construction so the trim loop cannot popitem an empty OrderedDict at publish time. Logging: - ticket_store.py: log API_WS_TICKET_LIMIT_EXCEEDED before raising TicketLimitExceededError so cap rejections appear in audit logs (new event constant added to events/api.py). - ngrok_adapter.py: log TUNNEL_ERROR before raising on duplicate start. Refactors: - replay_protection.py: split check() into check() + _check_nonce() reusing check_freshness() so the public method stays under 50 lines. Docstring: - continuous.py: module docstring now matches implementation (lifecycle lock guards the running-flag transition, not the loop body; stop() does not acquire the lock). Tests: - 6 test files: add spec=ConcreteClass to bare Mock/AsyncMock call sites (test_scheduler_lifecycle, test_monitor_lifecycle, test_backup_subscriber, test_backup_required_idempotency, test_backup, test_simulations_idempotency). assert_awaited_once replaces assert_called_once where applicable. - test_ticket_store_threadsafety.py: threading.Event start gate so worker bodies hit store.create together rather than as the pool fills (tighter contention on the lock). - test_cache_threadsafety.py: seed a shared key so the reader path exercises the locked hit branch (move_to_end + deepcopy). - test_factory_registry.py: cover the unknown-key fallback (both registries) via model_construct bypassing Pydantic literal validation. - test_stream_dedup.py: 2 new tests for unsubscribe dedup-window cleanup (already in the prior PR commit). Frontend: - backup.ts: trim whitespace-only Idempotency-Key inputs and fall through to a fresh UUID; ?? alone would forward an empty string and the server rejects it. 26138 unit tests + web type-check + web lint clean; mypy + ruff clean. --- scripts/mock_spec_baseline.txt | 84 +++++++------------ src/synthorg/api/auth/ticket_store.py | 8 ++ src/synthorg/backup/scheduler.py | 9 +- src/synthorg/client/continuous.py | 12 ++- .../conflict_resolution/escalation/sweeper.py | 23 ++--- .../communication/event_stream/stream.py | 18 ++++ src/synthorg/hr/pruning/service.py | 9 +- .../integrations/tunnel/ngrok_adapter.py | 6 ++ .../webhooks/replay_protection.py | 51 +++++------ src/synthorg/meta/chief_of_staff/monitor.py | 9 +- src/synthorg/observability/events/api.py | 1 + src/synthorg/providers/health_prober.py | 10 ++- .../auth/test_ticket_store_threadsafety.py | 18 ++++ tests/unit/api/controllers/test_backup.py | 10 ++- .../test_backup_required_idempotency.py | 17 ++-- .../test_simulations_idempotency.py | 23 +++-- tests/unit/backup/test_scheduler_lifecycle.py | 5 +- .../escalation/test_factory_registry.py | 24 ++++++ .../chief_of_staff/test_monitor_lifecycle.py | 7 +- tests/unit/settings/test_backup_subscriber.py | 23 +++-- .../unit/tools/mcp/test_cache_threadsafety.py | 14 +++- web/src/api/endpoints/backup.ts | 7 +- 22 files changed, 250 insertions(+), 138 deletions(-) diff --git a/scripts/mock_spec_baseline.txt b/scripts/mock_spec_baseline.txt index f11e283628..ec8ce8f074 100644 --- a/scripts/mock_spec_baseline.txt +++ b/scripts/mock_spec_baseline.txt @@ -301,36 +301,29 @@ tests/unit/api/controllers/test_approvals_helpers.py:493:22 tests/unit/api/controllers/test_approvals_helpers.py:494:38 tests/unit/api/controllers/test_approvals_helpers.py:511:22 tests/unit/api/controllers/test_approvals_helpers.py:512:38 -tests/unit/api/controllers/test_backup.py:73:14 -tests/unit/api/controllers/test_backup.py:74:16 -tests/unit/api/controllers/test_backup.py:79:26 -tests/unit/api/controllers/test_backup.py:89:18 -tests/unit/api/controllers/test_backup.py:102:12 -tests/unit/api/controllers/test_backup.py:119:32 -tests/unit/api/controllers/test_backup.py:134:32 -tests/unit/api/controllers/test_backup.py:155:31 -tests/unit/api/controllers/test_backup.py:174:29 -tests/unit/api/controllers/test_backup.py:189:29 -tests/unit/api/controllers/test_backup.py:208:32 -tests/unit/api/controllers/test_backup.py:222:32 -tests/unit/api/controllers/test_backup.py:242:38 -tests/unit/api/controllers/test_backup.py:265:38 -tests/unit/api/controllers/test_backup.py:296:38 -tests/unit/api/controllers/test_backup.py:310:38 -tests/unit/api/controllers/test_backup.py:326:38 -tests/unit/api/controllers/test_backup.py:342:38 -tests/unit/api/controllers/test_backup.py:396:19 -tests/unit/api/controllers/test_backup.py:399:25 -tests/unit/api/controllers/test_backup.py:400:24 -tests/unit/api/controllers/test_backup.py:401:32 -tests/unit/api/controllers/test_backup.py:402:29 -tests/unit/api/controllers/test_backup.py:403:34 -tests/unit/api/controllers/test_backup_required_idempotency.py:43:14 -tests/unit/api/controllers/test_backup_required_idempotency.py:44:28 -tests/unit/api/controllers/test_backup_required_idempotency.py:45:16 -tests/unit/api/controllers/test_backup_required_idempotency.py:47:26 -tests/unit/api/controllers/test_backup_required_idempotency.py:53:12 -tests/unit/api/controllers/test_backup_required_idempotency.py:80:22 +tests/unit/api/controllers/test_backup.py:76:16 +tests/unit/api/controllers/test_backup.py:93:18 +tests/unit/api/controllers/test_backup.py:106:12 +tests/unit/api/controllers/test_backup.py:123:32 +tests/unit/api/controllers/test_backup.py:138:32 +tests/unit/api/controllers/test_backup.py:159:31 +tests/unit/api/controllers/test_backup.py:178:29 +tests/unit/api/controllers/test_backup.py:193:29 +tests/unit/api/controllers/test_backup.py:212:32 +tests/unit/api/controllers/test_backup.py:226:32 +tests/unit/api/controllers/test_backup.py:246:38 +tests/unit/api/controllers/test_backup.py:269:38 +tests/unit/api/controllers/test_backup.py:300:38 +tests/unit/api/controllers/test_backup.py:314:38 +tests/unit/api/controllers/test_backup.py:330:38 +tests/unit/api/controllers/test_backup.py:346:38 +tests/unit/api/controllers/test_backup.py:400:19 +tests/unit/api/controllers/test_backup.py:403:25 +tests/unit/api/controllers/test_backup.py:404:24 +tests/unit/api/controllers/test_backup.py:405:32 +tests/unit/api/controllers/test_backup.py:406:29 +tests/unit/api/controllers/test_backup.py:407:34 +tests/unit/api/controllers/test_backup_required_idempotency.py:87:22 tests/unit/api/controllers/test_collaboration.py:357:23 tests/unit/api/controllers/test_company.py:108:31 tests/unit/api/controllers/test_coordination.py:77:18 @@ -420,18 +413,11 @@ tests/unit/api/controllers/test_setup_has_gpu.py:76:27 tests/unit/api/controllers/test_setup_locales.py:246:33 tests/unit/api/controllers/test_setup_locales.py:269:33 tests/unit/api/controllers/test_setup_locales.py:316:33 -tests/unit/api/controllers/test_simulations_idempotency.py:40:16 -tests/unit/api/controllers/test_simulations_idempotency.py:41:52 -tests/unit/api/controllers/test_simulations_idempotency.py:44:38 -tests/unit/api/controllers/test_simulations_idempotency.py:46:30 -tests/unit/api/controllers/test_simulations_idempotency.py:47:21 -tests/unit/api/controllers/test_simulations_idempotency.py:48:34 -tests/unit/api/controllers/test_simulations_idempotency.py:49:31 -tests/unit/api/controllers/test_simulations_idempotency.py:50:38 -tests/unit/api/controllers/test_simulations_idempotency.py:51:16 -tests/unit/api/controllers/test_simulations_idempotency.py:53:32 -tests/unit/api/controllers/test_simulations_idempotency.py:54:12 -tests/unit/api/controllers/test_simulations_idempotency.py:60:11 +tests/unit/api/controllers/test_simulations_idempotency.py:55:30 +tests/unit/api/controllers/test_simulations_idempotency.py:56:21 +tests/unit/api/controllers/test_simulations_idempotency.py:57:34 +tests/unit/api/controllers/test_simulations_idempotency.py:58:31 +tests/unit/api/controllers/test_simulations_idempotency.py:59:38 tests/unit/api/controllers/test_sse_keepalive_setting.py:28:31 tests/unit/api/controllers/test_sse_keepalive_setting.py:29:41 tests/unit/api/controllers/test_sse_revalidate.py:44:21 @@ -584,8 +570,6 @@ tests/unit/api/test_state.py:343:19 tests/unit/api/test_state.py:350:19 tests/unit/backup/test_scheduler.py:14:14 tests/unit/backup/test_scheduler.py:15:28 -tests/unit/backup/test_scheduler_lifecycle.py:21:14 -tests/unit/backup/test_scheduler_lifecycle.py:22:28 tests/unit/backup/test_service.py:21:14 tests/unit/backup/test_service.py:23:21 tests/unit/backup/test_service.py:24:22 @@ -2192,8 +2176,6 @@ tests/unit/meta/chief_of_staff/test_monitor.py:118:18 tests/unit/meta/chief_of_staff/test_monitor.py:137:18 tests/unit/meta/chief_of_staff/test_monitor.py:164:18 tests/unit/meta/chief_of_staff/test_monitor.py:178:23 -tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py:21:14 -tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py:22:20 tests/unit/meta/mcp/test_all_handlers_wired.py:75:11 tests/unit/meta/mcp/test_all_handlers_wired.py:76:22 tests/unit/meta/mcp/test_all_handlers_wired.py:92:14 @@ -2864,14 +2846,8 @@ tests/unit/security/timeout/test_scheduler.py:474:32 tests/unit/security/timeout/test_scheduler.py:497:19 tests/unit/security/timeout/test_timeout_checker.py:38:18 tests/unit/security/timeout/test_timeout_checker.py:147:22 -tests/unit/settings/test_backup_subscriber.py:30:16 -tests/unit/settings/test_backup_subscriber.py:32:22 -tests/unit/settings/test_backup_subscriber.py:33:21 -tests/unit/settings/test_backup_subscriber.py:34:27 -tests/unit/settings/test_backup_subscriber.py:36:14 -tests/unit/settings/test_backup_subscriber.py:39:23 -tests/unit/settings/test_backup_subscriber.py:42:17 -tests/unit/settings/test_backup_subscriber.py:51:27 +tests/unit/settings/test_backup_subscriber.py:45:17 +tests/unit/settings/test_backup_subscriber.py:54:27 tests/unit/settings/test_bridge_config_wiring.py:184:40 tests/unit/settings/test_bridge_config_wiring.py:185:19 tests/unit/settings/test_bridge_config_wiring.py:191:40 diff --git a/src/synthorg/api/auth/ticket_store.py b/src/synthorg/api/auth/ticket_store.py index fdcce484ac..b38bde7057 100644 --- a/src/synthorg/api/auth/ticket_store.py +++ b/src/synthorg/api/auth/ticket_store.py @@ -37,6 +37,7 @@ API_WS_TICKET_EXPIRED, API_WS_TICKET_INVALID, API_WS_TICKET_ISSUED, + API_WS_TICKET_LIMIT_EXCEEDED, ) logger = get_logger(__name__) @@ -154,6 +155,13 @@ def create(self, user: AuthenticatedUser) -> str: if e.user.user_id == user.user_id and now <= e.expires_at ) if user_pending >= self._max_pending: + logger.warning( + API_WS_TICKET_LIMIT_EXCEEDED, + user_id=user.user_id, + username=user.username, + pending=user_pending, + cap=self._max_pending, + ) msg = f"Ticket limit exceeded for user {user.user_id}" raise TicketLimitExceededError(msg) diff --git a/src/synthorg/backup/scheduler.py b/src/synthorg/backup/scheduler.py index 9eb3d7ce19..cfab032633 100644 --- a/src/synthorg/backup/scheduler.py +++ b/src/synthorg/backup/scheduler.py @@ -125,9 +125,12 @@ async def _drain() -> None: raise self._task = None logger.info(BACKUP_SCHEDULER_STOPPED) - # Recreate primitives outside the (released) lock so a - # subsequent ``start()`` on a different event loop can rebind. - self._lifecycle_lock = asyncio.Lock() + # Recreate the loop-bound events outside the (released) lock + # so a subsequent ``start()`` on a different event loop can + # rebind them. ``self._lifecycle_lock`` MUST stay the same + # instance for the service's lifetime: replacing it would let + # a caller queued on the old lock and a fresh caller acquiring + # the new lock both proceed concurrently. self._stop_event = asyncio.Event() self._wake_event = asyncio.Event() diff --git a/src/synthorg/client/continuous.py b/src/synthorg/client/continuous.py index 9a68e8821d..bb72185b3d 100644 --- a/src/synthorg/client/continuous.py +++ b/src/synthorg/client/continuous.py @@ -9,9 +9,15 @@ does not apply here -- there is no orphan task to drain post-stop. What carries over from the canonical pattern is the -``self._lifecycle_lock``: it serialises the running-flag check and -spans the full body of ``start()`` and ``stop()`` so concurrent -callers cannot both observe ``_running=False`` and proceed. +``self._lifecycle_lock``: it serialises the ``_running`` flag check +and is held only briefly at the top of ``start()`` (acquire, check, +set, release) and again in the ``finally`` to clear the flag. The +lock does NOT span the loop body -- holding it across the run loop +would deadlock a concurrent ``start()`` caller (it would queue on +the lock until the first finished, then enter an empty state). +``stop()`` is synchronous and does not acquire the lock; it merely +sets ``self._stop_event`` so the running ``start()`` coroutine +observes the signal on its next iteration. """ import asyncio diff --git a/src/synthorg/communication/conflict_resolution/escalation/sweeper.py b/src/synthorg/communication/conflict_resolution/escalation/sweeper.py index 9458489371..ffc5023739 100644 --- a/src/synthorg/communication/conflict_resolution/escalation/sweeper.py +++ b/src/synthorg/communication/conflict_resolution/escalation/sweeper.py @@ -200,18 +200,19 @@ async def _drain() -> None: raise self._task = None logger.info(CONFLICT_ESCALATION_SWEEPER_STOPPED) - # Re-create the lifecycle primitives outside the (now + # Re-create the loop-bound stop event outside the (now # released) lock so a subsequent ``start()`` on a different - # event loop can re-bind them. ``asyncio.Lock`` and - # ``asyncio.Event`` bind to the running loop on first - # ``acquire`` / ``set``; the loop they were last bound to - # may be closed (test pattern: fresh-per-test event loops), - # so reusing the instances would raise ``RuntimeError: ... - # is bound to a different event loop``. The recreate runs - # AFTER the ``async with`` exits to avoid swapping the lock - # while we still hold it. Production single-loop wiring - # constructs the sweeper once and never hits this path. - self._lifecycle_lock = asyncio.Lock() + # event loop can re-bind it. ``asyncio.Event`` binds to the + # running loop on first ``set()``; the loop it was last bound + # to may be closed (test pattern: fresh-per-test event loops), + # so reusing the instance would raise ``RuntimeError: ... is + # bound to a different event loop``. ``self._lifecycle_lock`` + # MUST stay the same instance for the service's lifetime: + # replacing it would let a caller queued on the old lock and a + # fresh caller acquiring the new lock both proceed + # concurrently, breaking the start/stop serialisation. Tests + # that span multiple event loops construct a fresh sweeper + # instance per loop instead of reusing one across loops. self._stop_event = asyncio.Event() async def _run(self) -> None: diff --git a/src/synthorg/communication/event_stream/stream.py b/src/synthorg/communication/event_stream/stream.py index 0aee5f4239..ce3e279874 100644 --- a/src/synthorg/communication/event_stream/stream.py +++ b/src/synthorg/communication/event_stream/stream.py @@ -61,6 +61,24 @@ def __init__( dedup_max_entries_per_session: int = _DEFAULT_DEDUP_MAX_ENTRIES_PER_SESSION, clock: Clock | None = None, ) -> None: + # Fail-fast on bad inputs; otherwise the trim loop in + # ``_record_published_locked`` would call ``popitem`` on an + # empty OrderedDict at publish time when + # ``dedup_max_entries_per_session`` is non-positive, and a + # negative TTL would short-circuit the eviction sweep into + # always-stale (every entry instantly "expired"). + if max_queue_size < 1: + msg = f"max_queue_size must be >= 1, got {max_queue_size}" + raise ValueError(msg) + if dedup_ttl_seconds < 0: + msg = f"dedup_ttl_seconds must be >= 0, got {dedup_ttl_seconds}" + raise ValueError(msg) + if dedup_max_entries_per_session < 1: + msg = ( + "dedup_max_entries_per_session must be >= 1, got " + f"{dedup_max_entries_per_session}" + ) + raise ValueError(msg) self._max_queue_size = max_queue_size self._dedup_ttl_seconds = dedup_ttl_seconds self._dedup_max_entries_per_session = dedup_max_entries_per_session diff --git a/src/synthorg/hr/pruning/service.py b/src/synthorg/hr/pruning/service.py index 5a45d7a80e..e35d1bf8bf 100644 --- a/src/synthorg/hr/pruning/service.py +++ b/src/synthorg/hr/pruning/service.py @@ -206,9 +206,12 @@ async def _drain() -> None: raise self._task = None logger.info(HR_PRUNING_SCHEDULER_STOPPED) - # Recreate primitives outside the (released) lock so a - # subsequent ``start()`` on a different event loop can rebind. - self._lifecycle_lock = asyncio.Lock() + # Recreate the loop-bound events outside the (released) lock + # so a subsequent ``start()`` on a different event loop can + # rebind them. ``self._lifecycle_lock`` MUST stay the same + # instance for the service's lifetime: replacing it would let + # a caller queued on the old lock and a fresh caller acquiring + # the new lock both proceed concurrently. self._stop_event = asyncio.Event() self._wake_event = asyncio.Event() diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index a9814cd153..a6da80ad80 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -79,6 +79,12 @@ async def start(self) -> str: """ async with self._lifecycle_lock: if self._tunnel is not None: + logger.warning( + TUNNEL_ERROR, + phase="start", + reason="already_active", + port=self._port, + ) msg = "ngrok tunnel already active on this adapter" raise RuntimeError(msg) auth_token = os.environ.get(self._auth_token_env, "").strip() diff --git a/src/synthorg/integrations/webhooks/replay_protection.py b/src/synthorg/integrations/webhooks/replay_protection.py index a1180a74ca..5fd40cb53d 100644 --- a/src/synthorg/integrations/webhooks/replay_protection.py +++ b/src/synthorg/integrations/webhooks/replay_protection.py @@ -128,7 +128,7 @@ def check_freshness(self, timestamp: float | None) -> bool: return False return True - def check( # noqa: PLR0911 + def check( self, *, nonce: str | None, @@ -136,6 +136,11 @@ def check( # noqa: PLR0911 ) -> bool: """Check whether a request is a replay. + Delegates timestamp freshness to :meth:`check_freshness` and + nonce dedup to :meth:`_check_nonce` so each concern stays + isolated and the function body fits comfortably under the + 50-line limit. + Args: nonce: Request nonce (optional). timestamp: Request timestamp as Unix epoch seconds. @@ -144,8 +149,6 @@ def check( # noqa: PLR0911 ``True`` if the request is safe (not a replay). ``False`` if the request should be rejected. """ - now = self._clock.now().timestamp() - # Fail closed: when neither a nonce nor a timestamp is supplied # the protector has nothing to check against, so accepting the # request would silently downgrade replay protection to a @@ -157,36 +160,29 @@ def check( # noqa: PLR0911 reason="no freshness signal (nonce and timestamp both missing)", ) return False - - # ``float("nan")`` would bypass the window check because - # ``abs(now - nan) > window`` evaluates to ``False``. Reject - # any non-finite timestamp up-front so a malformed header - # cannot silently pass freshness validation. - if timestamp is not None and not math.isfinite(timestamp): - logger.warning( - WEBHOOK_REPLAY_DETECTED, - reason="non-finite timestamp", - ) + if not self.check_freshness(timestamp): return False + now = self._clock.now().timestamp() + return self._check_nonce(nonce=nonce, now=now) - if timestamp is not None and abs(now - timestamp) > self._window: - logger.warning( - WEBHOOK_REPLAY_DETECTED, - reason="timestamp outside window", - skew=abs(now - timestamp), - ) - return False + def _check_nonce(self, *, nonce: str | None, now: float) -> bool: + """Validate the nonce dedup window. + Caller is responsible for freshness checks; this method only + handles the nonce side of replay protection. ``now`` is taken + from the same clock read the caller used so eviction and + dedup observe the same instant. + """ if nonce is None: with self._lock: self._evict_locked(now) return True - # Reject oversized nonces before touching the cache. - # An attacker who could send arbitrarily long nonces - # would otherwise be able to make each hash computation - # increasingly expensive even though the cache entry - # itself is fixed-size. + # Reject oversized nonces before touching the cache. An + # attacker who could send arbitrarily long nonces would + # otherwise be able to make each hash computation + # increasingly expensive even though the cache entry itself + # is fixed-size. if len(nonce) > MAX_NONCE_CHARS: logger.warning( WEBHOOK_REPLAY_DETECTED, @@ -198,8 +194,8 @@ def check( # noqa: PLR0911 # Store a fixed-size SHA-256 digest instead of the raw # attacker-controlled string. Bounds per-entry memory - # independent of nonce length and removes any concern - # about echoing the nonce back in log output below. + # independent of nonce length and removes any concern about + # echoing the nonce back in log output below. key = _fingerprint_nonce(nonce) with self._lock: self._evict_locked(now) @@ -216,7 +212,6 @@ def check( # noqa: PLR0911 nonce_fingerprint=key[:16], ) return False - return True def _evict_locked(self, now: float) -> None: diff --git a/src/synthorg/meta/chief_of_staff/monitor.py b/src/synthorg/meta/chief_of_staff/monitor.py index 4ac29649da..fa62c582f0 100644 --- a/src/synthorg/meta/chief_of_staff/monitor.py +++ b/src/synthorg/meta/chief_of_staff/monitor.py @@ -149,9 +149,12 @@ async def _drain() -> None: self._task = None self._last_snapshot = None logger.info(COS_MONITOR_STOPPED) - # Recreate primitives outside the (released) lock so a fresh - # event loop binding works for subsequent ``start()`` calls. - self._lifecycle_lock = asyncio.Lock() + # Recreate the loop-bound stop event outside the (released) + # lock so a fresh event loop binding works for subsequent + # ``start()`` calls. ``self._lifecycle_lock`` MUST stay the + # same instance for the service lifetime: replacing it would + # let a caller queued on the old lock and a fresh caller + # acquiring the new lock both proceed concurrently. self._stop_event = asyncio.Event() async def _loop(self) -> None: diff --git a/src/synthorg/observability/events/api.py b/src/synthorg/observability/events/api.py index db0a45f03b..cd897e73fd 100644 --- a/src/synthorg/observability/events/api.py +++ b/src/synthorg/observability/events/api.py @@ -77,6 +77,7 @@ API_WS_TICKET_EXPIRED: Final[str] = "api.ws.ticket_expired" API_WS_TICKET_INVALID: Final[str] = "api.ws.ticket_invalid" API_WS_TICKET_CLEANUP: Final[str] = "api.ws.ticket_cleanup" +API_WS_TICKET_LIMIT_EXCEEDED: Final[str] = "api.ws.ticket_limit_exceeded" API_AUDIT_RETENTION: Final[str] = "api.audit.retention" API_WS_AUTH_STAGE: Final[str] = "api.ws.auth_stage" API_WS_AUTH_OK: Final[str] = "api.ws.auth_ok" diff --git a/src/synthorg/providers/health_prober.py b/src/synthorg/providers/health_prober.py index da162b66bb..32ccb9f20e 100644 --- a/src/synthorg/providers/health_prober.py +++ b/src/synthorg/providers/health_prober.py @@ -261,9 +261,13 @@ async def _drain() -> None: raise self._task = None logger.info(PROVIDER_HEALTH_PROBER_STOPPED) - # Recreate primitives outside the (released) lock so a - # subsequent ``start()`` on a different event loop can rebind. - self._lifecycle_lock = asyncio.Lock() + # Recreate the loop-bound stop event outside the (released) + # lock so a subsequent ``start()`` on a different event loop + # can rebind it. ``self._lifecycle_lock`` MUST stay the same + # instance for the service's lifetime: replacing it would let a + # caller queued on the old lock and a fresh caller acquiring + # the new lock both proceed concurrently, breaking the + # serialisation the canonical pattern guarantees. self._stop_event = asyncio.Event() async def _run_loop(self) -> None: diff --git a/tests/unit/api/auth/test_ticket_store_threadsafety.py b/tests/unit/api/auth/test_ticket_store_threadsafety.py index 8161048a06..2f1dbea80a 100644 --- a/tests/unit/api/auth/test_ticket_store_threadsafety.py +++ b/tests/unit/api/auth/test_ticket_store_threadsafety.py @@ -10,6 +10,7 @@ """ import contextlib +import threading from concurrent.futures import ThreadPoolExecutor import pytest @@ -36,8 +37,15 @@ def test_concurrent_create_honors_per_user_cap(self) -> None: """100 threads racing on create() for one user yield exactly cap accepts.""" store = WsTicketStore(max_pending_per_user=5) user = _make_user() + # Hold all worker threads until every future is submitted, then + # release together. Without the gate, futures execute as the + # pool fills, which under-stresses the lock by spreading the + # contention across submission time. With the gate, all 100 + # threads hit ``store.create`` in the same instant. + start_gate = threading.Event() def attempt() -> str | None: + start_gate.wait() try: return store.create(user) except TicketLimitExceededError: @@ -45,6 +53,7 @@ def attempt() -> str | None: with ThreadPoolExecutor(max_workers=16) as pool: futures = [pool.submit(attempt) for _ in range(100)] + start_gate.set() results = [f.result() for f in futures] successes = [r for r in results if r is not None] @@ -54,8 +63,10 @@ def attempt() -> str | None: def test_concurrent_create_distinct_users_independent(self) -> None: """Different users do not share the cap under concurrency.""" store = WsTicketStore(max_pending_per_user=3) + start_gate = threading.Event() def attempt(user_id: str) -> str | None: + start_gate.wait() try: return store.create(_make_user(user_id=user_id)) except TicketLimitExceededError: @@ -63,6 +74,7 @@ def attempt(user_id: str) -> str | None: with ThreadPoolExecutor(max_workers=16) as pool: futures = [pool.submit(attempt, f"user-{i % 4}") for i in range(40)] + start_gate.set() results = [f.result() for f in futures] successes = [r for r in results if r is not None] @@ -74,12 +86,15 @@ def test_concurrent_validate_and_consume_single_winner(self) -> None: store = WsTicketStore() user = _make_user() ticket = store.create(user) + start_gate = threading.Event() def attempt() -> AuthenticatedUser | None: + start_gate.wait() return store.validate_and_consume(ticket) with ThreadPoolExecutor(max_workers=16) as pool: futures = [pool.submit(attempt) for _ in range(32)] + start_gate.set() results = [f.result() for f in futures] accepted = [r for r in results if r is not None] @@ -89,8 +104,10 @@ def attempt() -> AuthenticatedUser | None: def test_concurrent_create_and_cleanup_no_corruption(self) -> None: """Mixed create / cleanup_expired calls do not raise or corrupt state.""" store = WsTicketStore(ttl_seconds=30.0, max_pending_per_user=5) + start_gate = threading.Event() def task(i: int) -> None: + start_gate.wait() if i % 3 == 0: store.cleanup_expired() return @@ -99,6 +116,7 @@ def task(i: int) -> None: with ThreadPoolExecutor(max_workers=16) as pool: futures = [pool.submit(task, i) for i in range(80)] + start_gate.set() for f in futures: f.result() # If we reach here without RuntimeError ("dictionary changed size diff --git a/tests/unit/api/controllers/test_backup.py b/tests/unit/api/controllers/test_backup.py index 36cf659940..27527daafc 100644 --- a/tests/unit/api/controllers/test_backup.py +++ b/tests/unit/api/controllers/test_backup.py @@ -16,6 +16,7 @@ from synthorg.api.controllers.backup import BackupController from synthorg.api.cursor import CursorSecret from synthorg.api.dto import ApiResponse, PaginatedResponse +from synthorg.api.services.idempotency_service import IdempotencyService from synthorg.backup.errors import ( BackupInProgressError, BackupNotFoundError, @@ -29,6 +30,7 @@ RestoreRequest, RestoreResponse, ) +from synthorg.backup.service import BackupService from synthorg.core.domain_errors import ConflictError, NotFoundError, ValidationError from tests.unit.api.conftest import make_auth_headers @@ -70,13 +72,15 @@ def _make_state_and_service() -> tuple[MagicMock, AsyncMock]: Returns: Tuple of (mock_state, mock_backup_service). """ - service = AsyncMock() + service = AsyncMock(spec=BackupService) app_state = MagicMock() app_state.backup_service = service # The controller now wraps every backup in idempotency_service. # Mock the service so run_idempotent invokes the callback inline - # and returns a fresh outcome with the manifest dict. - idempotency_service = MagicMock() + # and returns a fresh outcome with the manifest dict. ``spec=`` on + # the wrapper enforces the interface; ``run_idempotent`` is set to + # an inline async helper that exercises the awaitable contract. + idempotency_service = MagicMock(spec=IdempotencyService) async def _run_idempotent( *, diff --git a/tests/unit/api/controllers/test_backup_required_idempotency.py b/tests/unit/api/controllers/test_backup_required_idempotency.py index fe547af673..414875b51c 100644 --- a/tests/unit/api/controllers/test_backup_required_idempotency.py +++ b/tests/unit/api/controllers/test_backup_required_idempotency.py @@ -15,14 +15,18 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from litestar.datastructures import State from synthorg.api.controllers.backup import BackupController from synthorg.api.cursor import CursorSecret +from synthorg.api.services.idempotency_service import IdempotencyService +from synthorg.api.state import AppState from synthorg.backup.models import ( BackupComponent, BackupManifest, BackupTrigger, ) +from synthorg.backup.service import BackupService pytestmark = pytest.mark.unit @@ -40,17 +44,20 @@ def _make_manifest() -> BackupManifest: def _make_state(*, run_idempotent: Any) -> MagicMock: - service = AsyncMock() - service.create_backup = AsyncMock(return_value=_make_manifest()) - app_state = MagicMock() + service = MagicMock(spec=BackupService) + service.create_backup = AsyncMock( + spec=BackupService.create_backup, + return_value=_make_manifest(), + ) + app_state = MagicMock(spec=AppState) app_state.backup_service = service - idempotency_service = MagicMock() + idempotency_service = MagicMock(spec=IdempotencyService) idempotency_service.run_idempotent = run_idempotent app_state.idempotency_service = idempotency_service app_state.cursor_secret = CursorSecret.from_key( "test-key-32-bytes-padding0000000", ) - state = MagicMock() + state = MagicMock(spec=State) state.app_state = app_state return state diff --git a/tests/unit/api/controllers/test_simulations_idempotency.py b/tests/unit/api/controllers/test_simulations_idempotency.py index f24e4afa09..0ef515888b 100644 --- a/tests/unit/api/controllers/test_simulations_idempotency.py +++ b/tests/unit/api/controllers/test_simulations_idempotency.py @@ -10,13 +10,19 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from litestar.connection import Request +from litestar.datastructures import State from synthorg.api.controllers.simulations import ( SimulationController, StartSimulationPayload, ) +from synthorg.api.state import AppState from synthorg.client.models import SimulationConfig +from synthorg.client.simulation_state import ClientSimulationState +from synthorg.client.store import SimulationStore from synthorg.core.domain_errors import ConflictError +from synthorg.settings.resolver import ConfigResolver pytestmark = pytest.mark.unit @@ -37,27 +43,30 @@ def _make_state(*, claim_succeeds: bool) -> MagicMock: True (fresh id), simulating the first request. When ``False`` it returns False (id already registered), simulating a duplicate. """ - sim_state = MagicMock() - sim_state.simulation_store.register_if_absent = AsyncMock( + sim_store = MagicMock(spec=SimulationStore) + sim_store.register_if_absent = AsyncMock( + spec=SimulationStore.register_if_absent, return_value=claim_succeeds, ) - sim_state.simulation_store.save = AsyncMock() + sim_store.save = AsyncMock(spec=SimulationStore.save) + sim_state = MagicMock(spec=ClientSimulationState) + sim_state.simulation_store = sim_store sim_state.background_tasks = set() sim_state.intake_engine = MagicMock() sim_state.pool = MagicMock() sim_state.pool.list_clients = AsyncMock(return_value=()) sim_state.feedback_store = MagicMock() sim_state.feedback_store.record = MagicMock() - app_state = MagicMock() + app_state = MagicMock(spec=AppState) app_state.client_simulation_state = sim_state - app_state.config_resolver = MagicMock() - state = MagicMock() + app_state.config_resolver = MagicMock(spec=ConfigResolver) + state = MagicMock(spec=State) state.app_state = app_state return state def _make_request() -> MagicMock: - return MagicMock() + return MagicMock(spec=Request) class TestSimulationsIdempotency: diff --git a/tests/unit/backup/test_scheduler_lifecycle.py b/tests/unit/backup/test_scheduler_lifecycle.py index 7e93296836..b06be19175 100644 --- a/tests/unit/backup/test_scheduler_lifecycle.py +++ b/tests/unit/backup/test_scheduler_lifecycle.py @@ -13,13 +13,14 @@ import pytest from synthorg.backup.scheduler import BackupScheduler +from synthorg.backup.service import BackupService pytestmark = pytest.mark.unit def _make_scheduler() -> BackupScheduler: - service = MagicMock() - service.create_backup = AsyncMock() + service = MagicMock(spec=BackupService) + service.create_backup = AsyncMock(spec=BackupService.create_backup) return BackupScheduler(service, interval_hours=1) diff --git a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py index adb4a2572e..a475fe2e1c 100644 --- a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py +++ b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py @@ -102,3 +102,27 @@ def test_hybrid_strategy_returns_hybrid(self) -> None: config = EscalationQueueConfig(decision_strategy="hybrid") processor = build_decision_processor(config) assert isinstance(processor, HybridDecisionProcessor) + + +class TestRegistryFallback: + """The defensive ValueError fallback fires for unregistered keys. + + Pydantic rejects unknown literals at config-construction time, so + these tests bypass validation via ``model_construct`` to drive the + factory directly and confirm that an unknown key surfaces a + helpful error message rather than silently returning ``None`` or + crashing inside the factory closure. + """ + + def test_unknown_queue_backend_raises_value_error(self) -> None: + # Bypass Pydantic literal validation via ``model_construct`` so + # the factory's defensive ValueError fires for the registered + # unknown-key path. + config = EscalationQueueConfig.model_construct(backend="unknown") # type: ignore[arg-type] + with pytest.raises(ValueError, match=r"Unknown escalation queue backend"): + build_escalation_queue_store(config) + + def test_unknown_decision_strategy_raises_value_error(self) -> None: + config = EscalationQueueConfig.model_construct(decision_strategy="unknown") # type: ignore[arg-type] + with pytest.raises(ValueError, match=r"Unknown decision_strategy"): + build_decision_processor(config) diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py index 679ff298d4..21207a9536 100644 --- a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -13,13 +13,16 @@ from synthorg.meta.chief_of_staff.inflection import OrgInflectionDetector from synthorg.meta.chief_of_staff.monitor import OrgInflectionMonitor +from synthorg.meta.signals.snapshot import SnapshotBuilder pytestmark = pytest.mark.unit def _make_monitor() -> OrgInflectionMonitor: - builder = AsyncMock() - builder.build = AsyncMock(return_value=None) + # ``spec=SnapshotBuilder`` auto-mocks ``build`` as an AsyncMock; + # set ``return_value`` instead of reassigning ``builder.build``. + builder = AsyncMock(spec=SnapshotBuilder) + builder.build.return_value = None return OrgInflectionMonitor( detector=OrgInflectionDetector(), snapshot_builder=builder, diff --git a/tests/unit/settings/test_backup_subscriber.py b/tests/unit/settings/test_backup_subscriber.py index 3bcb753a6f..3deafbc9ac 100644 --- a/tests/unit/settings/test_backup_subscriber.py +++ b/tests/unit/settings/test_backup_subscriber.py @@ -4,6 +4,9 @@ import pytest +from synthorg.backup.scheduler import BackupScheduler +from synthorg.backup.service import BackupService +from synthorg.settings.service import SettingsService from synthorg.settings.subscriber import SettingsSubscriber from synthorg.settings.subscribers.backup_subscriber import ( BackupSettingsSubscriber, @@ -27,16 +30,16 @@ def _make_subscriber( Returns: Tuple of (subscriber, mock_backup_service). """ - scheduler = MagicMock() + scheduler = MagicMock(spec=BackupScheduler) type(scheduler).is_running = PropertyMock(return_value=scheduler_running) - scheduler.start = AsyncMock() - scheduler.stop = AsyncMock() - scheduler.reschedule = MagicMock() + scheduler.start = AsyncMock(spec=BackupScheduler.start) + scheduler.stop = AsyncMock(spec=BackupScheduler.stop) + scheduler.reschedule = MagicMock(spec=BackupScheduler.reschedule) - service = MagicMock() + service = MagicMock(spec=BackupService) type(service).scheduler = PropertyMock(return_value=scheduler) - settings_service = MagicMock() + settings_service = MagicMock(spec=SettingsService) async def _mock_get(namespace: str, key: str) -> MagicMock: result = MagicMock() @@ -95,7 +98,11 @@ async def test_enabled_starts_scheduler_when_stopped(self) -> None: await sub.on_settings_changed("backup", "enabled") - service.scheduler.start.assert_called_once() + # ``assert_awaited_once`` (vs ``assert_called_once``) catches a + # regression where the start coroutine is created but never + # awaited -- the call would still be recorded but the + # scheduler would never actually launch. + service.scheduler.start.assert_awaited_once() service.scheduler.stop.assert_not_awaited() async def test_enabled_stops_scheduler_when_running(self) -> None: @@ -118,7 +125,7 @@ async def test_enabled_toggle_is_idempotent(self) -> None: await sub.on_settings_changed("backup", "enabled") await sub.on_settings_changed("backup", "enabled") # Two start() calls -- no crash, idempotent - assert service.scheduler.start.call_count == 2 + assert service.scheduler.start.await_count == 2 @pytest.mark.unit diff --git a/tests/unit/tools/mcp/test_cache_threadsafety.py b/tests/unit/tools/mcp/test_cache_threadsafety.py index fe0b4c89a1..2474a7f6b5 100644 --- a/tests/unit/tools/mcp/test_cache_threadsafety.py +++ b/tests/unit/tools/mcp/test_cache_threadsafety.py @@ -21,12 +21,22 @@ class TestMCPResultCacheThreadSafety: def test_concurrent_get_put_no_corruption(self) -> None: cache = MCPResultCache(max_size=64, ttl_seconds=120.0) + # Seed a shared key so the reader path actually exercises the + # locked hit branch (``move_to_end`` + deepcopy) rather than + # only the miss branch. Without this seed the reader and + # writer payloads never collide, so the test only stresses + # concurrent misses + writes. + shared_args = {"i": -1} + cache.put("shared-tool", shared_args, ToolExecutionResult(content="seed")) def writer(i: int) -> None: - cache.put(f"tool-{i % 8}", {"i": i}, ToolExecutionResult(content=str(i))) + cache.put("shared-tool", shared_args, ToolExecutionResult(content=str(i))) def reader(i: int) -> None: - cache.get(f"tool-{i % 8}", {"i": i}) + del i + # Reader hits the seeded entry on every iteration, exercising + # the get -> move_to_end -> deepcopy path under contention. + cache.get("shared-tool", shared_args) with ThreadPoolExecutor(max_workers=16) as pool: futures = [] diff --git a/web/src/api/endpoints/backup.ts b/web/src/api/endpoints/backup.ts index 1960a6e98c..f4410bdef7 100644 --- a/web/src/api/endpoints/backup.ts +++ b/web/src/api/endpoints/backup.ts @@ -10,7 +10,12 @@ export async function createBackup(idempotencyKey?: string): Promise 0 ? trimmed : crypto.randomUUID() const response = await apiClient.post>( '/admin/backups', null, From 62a768710f6a7df7347f2872e8f4678f07962b1c Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 14:44:12 +0200 Subject: [PATCH 04/13] fix: address reviewer feedback for audit cleanup C round 2 - event-stream: skip dedup recording when no subscribers; document init params - ngrok adapter: introduce TunnelAlreadyActiveError; per-call PyngrokConfig - replay_protection: drop hardcoded line numbers from module docstring - lifecycle services (sweeper, scheduler, monitor, health_prober, pruning): recreate _stop_event INSIDE _lifecycle_lock for canonical pattern - tests: convert state carriers from MagicMock() to SimpleNamespace - tests: replace AsyncMock(spec=...) attribute replacement with MagicMock(spec=...).return_value to preserve auto-mock signatures - tests: tighten test_simulations_idempotency to patch _publish_event + asyncio.create_task instead of swallowing exceptions - mock_spec_baseline: drop 30 frozen entries from test_backup.py + test_backup_subscriber.py converted to spec-bound --- scripts/mock_spec_baseline.txt | 35 +------- src/synthorg/backup/scheduler.py | 18 ++-- .../conflict_resolution/escalation/sweeper.py | 27 +++--- .../communication/event_stream/stream.py | 25 +++++- src/synthorg/hr/pruning/service.py | 18 ++-- src/synthorg/integrations/errors.py | 14 ++++ .../integrations/tunnel/ngrok_adapter.py | 29 +++++-- .../webhooks/replay_protection.py | 4 +- src/synthorg/meta/chief_of_staff/monitor.py | 16 ++-- src/synthorg/providers/health_prober.py | 23 +++-- tests/unit/api/controllers/test_backup.py | 83 +++++++++---------- .../test_backup_required_idempotency.py | 57 +++++++++---- .../test_simulations_idempotency.py | 70 +++++++++++----- .../escalation/test_factory_registry.py | 9 +- .../test_ngrok_adapter_lifecycle.py | 11 ++- tests/unit/settings/test_backup_subscriber.py | 6 +- 16 files changed, 269 insertions(+), 176 deletions(-) diff --git a/scripts/mock_spec_baseline.txt b/scripts/mock_spec_baseline.txt index ec8ce8f074..04bf57ec03 100644 --- a/scripts/mock_spec_baseline.txt +++ b/scripts/mock_spec_baseline.txt @@ -301,29 +301,8 @@ tests/unit/api/controllers/test_approvals_helpers.py:493:22 tests/unit/api/controllers/test_approvals_helpers.py:494:38 tests/unit/api/controllers/test_approvals_helpers.py:511:22 tests/unit/api/controllers/test_approvals_helpers.py:512:38 -tests/unit/api/controllers/test_backup.py:76:16 -tests/unit/api/controllers/test_backup.py:93:18 -tests/unit/api/controllers/test_backup.py:106:12 -tests/unit/api/controllers/test_backup.py:123:32 -tests/unit/api/controllers/test_backup.py:138:32 -tests/unit/api/controllers/test_backup.py:159:31 -tests/unit/api/controllers/test_backup.py:178:29 -tests/unit/api/controllers/test_backup.py:193:29 -tests/unit/api/controllers/test_backup.py:212:32 -tests/unit/api/controllers/test_backup.py:226:32 -tests/unit/api/controllers/test_backup.py:246:38 -tests/unit/api/controllers/test_backup.py:269:38 -tests/unit/api/controllers/test_backup.py:300:38 -tests/unit/api/controllers/test_backup.py:314:38 -tests/unit/api/controllers/test_backup.py:330:38 -tests/unit/api/controllers/test_backup.py:346:38 -tests/unit/api/controllers/test_backup.py:400:19 -tests/unit/api/controllers/test_backup.py:403:25 -tests/unit/api/controllers/test_backup.py:404:24 -tests/unit/api/controllers/test_backup.py:405:32 -tests/unit/api/controllers/test_backup.py:406:29 -tests/unit/api/controllers/test_backup.py:407:34 -tests/unit/api/controllers/test_backup_required_idempotency.py:87:22 +tests/unit/api/controllers/test_backup.py:112:12 +tests/unit/api/controllers/test_backup_required_idempotency.py:72:12 tests/unit/api/controllers/test_collaboration.py:357:23 tests/unit/api/controllers/test_company.py:108:31 tests/unit/api/controllers/test_coordination.py:77:18 @@ -413,11 +392,7 @@ tests/unit/api/controllers/test_setup_has_gpu.py:76:27 tests/unit/api/controllers/test_setup_locales.py:246:33 tests/unit/api/controllers/test_setup_locales.py:269:33 tests/unit/api/controllers/test_setup_locales.py:316:33 -tests/unit/api/controllers/test_simulations_idempotency.py:55:30 -tests/unit/api/controllers/test_simulations_idempotency.py:56:21 -tests/unit/api/controllers/test_simulations_idempotency.py:57:34 -tests/unit/api/controllers/test_simulations_idempotency.py:58:31 -tests/unit/api/controllers/test_simulations_idempotency.py:59:38 +tests/unit/api/controllers/test_simulations_idempotency.py:69:12 tests/unit/api/controllers/test_sse_keepalive_setting.py:28:31 tests/unit/api/controllers/test_sse_keepalive_setting.py:29:41 tests/unit/api/controllers/test_sse_revalidate.py:44:21 @@ -652,8 +627,6 @@ tests/unit/communication/bus/test_nats_consumer_config.py:31:9 tests/unit/communication/bus/test_nats_consumer_config.py:33:24 tests/unit/communication/bus/test_nats_consumer_config.py:33:47 tests/unit/communication/bus/test_nats_consumer_config.py:44:12 -tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py:37:32 -tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py:37:55 tests/unit/communication/loop_prevention/test_circuit_breaker.py:293:15 tests/unit/communication/loop_prevention/test_circuit_breaker.py:294:20 tests/unit/communication/loop_prevention/test_circuit_breaker.py:316:15 @@ -2846,8 +2819,6 @@ tests/unit/security/timeout/test_scheduler.py:474:32 tests/unit/security/timeout/test_scheduler.py:497:19 tests/unit/security/timeout/test_timeout_checker.py:38:18 tests/unit/security/timeout/test_timeout_checker.py:147:22 -tests/unit/settings/test_backup_subscriber.py:45:17 -tests/unit/settings/test_backup_subscriber.py:54:27 tests/unit/settings/test_bridge_config_wiring.py:184:40 tests/unit/settings/test_bridge_config_wiring.py:185:19 tests/unit/settings/test_bridge_config_wiring.py:191:40 diff --git a/src/synthorg/backup/scheduler.py b/src/synthorg/backup/scheduler.py index cfab032633..6f682d5b2c 100644 --- a/src/synthorg/backup/scheduler.py +++ b/src/synthorg/backup/scheduler.py @@ -124,15 +124,17 @@ async def _drain() -> None: ) raise self._task = None + # Recreate the loop-bound events WHILE holding the + # lifecycle lock. Outside the lock, a racing ``start()`` + # could spawn the scheduler loop bound to the OLD events + # before these assignments land, leaving a later stop() + # signalling different events than the running task is + # waiting on. ``self._lifecycle_lock`` itself MUST stay + # the same instance for the service's lifetime; only the + # events are swapped. + self._stop_event = asyncio.Event() + self._wake_event = asyncio.Event() logger.info(BACKUP_SCHEDULER_STOPPED) - # Recreate the loop-bound events outside the (released) lock - # so a subsequent ``start()`` on a different event loop can - # rebind them. ``self._lifecycle_lock`` MUST stay the same - # instance for the service's lifetime: replacing it would let - # a caller queued on the old lock and a fresh caller acquiring - # the new lock both proceed concurrently. - self._stop_event = asyncio.Event() - self._wake_event = asyncio.Event() def reschedule(self, interval_hours: int) -> None: """Update the interval and interrupt the current sleep. diff --git a/src/synthorg/communication/conflict_resolution/escalation/sweeper.py b/src/synthorg/communication/conflict_resolution/escalation/sweeper.py index ffc5023739..d27b76dd5f 100644 --- a/src/synthorg/communication/conflict_resolution/escalation/sweeper.py +++ b/src/synthorg/communication/conflict_resolution/escalation/sweeper.py @@ -199,21 +199,20 @@ async def _drain() -> None: ) raise self._task = None + # Re-create the loop-bound stop event WHILE holding the + # lifecycle lock. Doing it outside the lock would leave a + # window where a racing ``start()`` could spawn ``_run()`` + # bound to the OLD event before this assignment lands; a + # later stop() would then signal a different event than + # the running task is waiting on, stalling shutdown until + # the interval timeout. ``asyncio.Event`` binds to the + # running loop on first ``set()``, so a fresh instance is + # always required across loops; ``self._lifecycle_lock`` + # itself MUST stay the same instance for the service's + # lifetime. Tests that span multiple event loops construct + # a fresh sweeper per loop instead of reusing one. + self._stop_event = asyncio.Event() logger.info(CONFLICT_ESCALATION_SWEEPER_STOPPED) - # Re-create the loop-bound stop event outside the (now - # released) lock so a subsequent ``start()`` on a different - # event loop can re-bind it. ``asyncio.Event`` binds to the - # running loop on first ``set()``; the loop it was last bound - # to may be closed (test pattern: fresh-per-test event loops), - # so reusing the instance would raise ``RuntimeError: ... is - # bound to a different event loop``. ``self._lifecycle_lock`` - # MUST stay the same instance for the service's lifetime: - # replacing it would let a caller queued on the old lock and a - # fresh caller acquiring the new lock both proceed - # concurrently, breaking the start/stop serialisation. Tests - # that span multiple event loops construct a fresh sweeper - # instance per loop instead of reusing one across loops. - self._stop_event = asyncio.Event() async def _run(self) -> None: """Main loop body.""" diff --git a/src/synthorg/communication/event_stream/stream.py b/src/synthorg/communication/event_stream/stream.py index ce3e279874..1fb17a4d4d 100644 --- a/src/synthorg/communication/event_stream/stream.py +++ b/src/synthorg/communication/event_stream/stream.py @@ -41,6 +41,19 @@ class EventStreamHub: max_queue_size: Maximum events buffered per subscriber queue. When full, new events are dropped (never blocks the publisher). + dedup_ttl_seconds: TTL for the per-session dedup window in + seconds. Identical ``event.id`` values published within + this window are skipped. ``0`` disables the time-based + eviction (entries only fall out via the per-session size + bound). Default 60. + dedup_max_entries_per_session: Maximum dedup entries kept per + session. When the bound is hit, the oldest entry is + evicted FIFO. Bounds memory growth even for noisy + sessions that never get TTL-evicted. Default 1024. + clock: Time source used for the dedup TTL. Inject a + ``FakeClock`` from ``tests._shared.fake_clock`` to drive + virtual time in tests; production wiring leaves this + ``None`` so the hub uses ``SystemClock``. """ __slots__ = ( @@ -163,6 +176,17 @@ async def publish(self, event: StreamEvent) -> None: """ now = self._clock.monotonic() async with self._lock: + queues_snapshot = list(self._subscribers.get(event.session_id, ())) + # If no subscribers, the event would be dropped anyway. + # Don't record it in the dedup window: a later retry that + # arrives after the client reconnects within the TTL must + # be delivered, not silently suppressed because the first + # attempt fell on an empty session. Also drop any orphan + # dedup-window state for that session so it cannot grow + # without bound across publish-without-subscribers cycles. + if not queues_snapshot: + self._seen_event_ids.pop(event.session_id, None) + return if self._is_duplicate_locked(event, now): logger.warning( EVENT_STREAM_HUB_PUBLISH_DEDUPED, @@ -172,7 +196,6 @@ async def publish(self, event: StreamEvent) -> None: ) return self._record_published_locked(event, now) - queues_snapshot = list(self._subscribers.get(event.session_id, ())) if not queues_snapshot: return for queue in queues_snapshot: diff --git a/src/synthorg/hr/pruning/service.py b/src/synthorg/hr/pruning/service.py index e35d1bf8bf..0f659575b9 100644 --- a/src/synthorg/hr/pruning/service.py +++ b/src/synthorg/hr/pruning/service.py @@ -205,15 +205,17 @@ async def _drain() -> None: ) raise self._task = None + # Recreate the loop-bound events WHILE holding the + # lifecycle lock. Outside the lock, a racing ``start()`` + # could spawn the run loop bound to the OLD events + # before these assignments land, leaving a later stop() + # signalling different events than the running task is + # waiting on. ``self._lifecycle_lock`` itself MUST stay + # the same instance for the service's lifetime; only the + # events are swapped. + self._stop_event = asyncio.Event() + self._wake_event = asyncio.Event() logger.info(HR_PRUNING_SCHEDULER_STOPPED) - # Recreate the loop-bound events outside the (released) lock - # so a subsequent ``start()`` on a different event loop can - # rebind them. ``self._lifecycle_lock`` MUST stay the same - # instance for the service's lifetime: replacing it would let - # a caller queued on the old lock and a fresh caller acquiring - # the new lock both proceed concurrently. - self._stop_event = asyncio.Event() - self._wake_event = asyncio.Event() def wake(self) -> None: """Trigger an early pruning cycle.""" diff --git a/src/synthorg/integrations/errors.py b/src/synthorg/integrations/errors.py index 5cf0c9c393..da2d70b73b 100644 --- a/src/synthorg/integrations/errors.py +++ b/src/synthorg/integrations/errors.py @@ -192,6 +192,20 @@ class TunnelError(IntegrationError): retryable: ClassVar[bool] = True +class TunnelAlreadyActiveError(IntegrationError): + """A tunnel is already active on the adapter; refuse re-start. + + Lifecycle conflict, not a transient I/O error: the operator must + ``stop()`` the running tunnel before issuing a fresh ``start()``. + Marked non-retryable so the resilience layer does not loop on a + permanent state error and so the API surface returns a 409-style + domain error instead of a generic 500. + """ + + is_retryable = False + retryable: ClassVar[bool] = False + + # -- MCP catalog errors -------------------------------------------------- diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index a6da80ad80..8fdf3b97f0 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -15,7 +15,10 @@ from pyngrok import conf, ngrok # type: ignore[import-untyped] -from synthorg.integrations.errors import TunnelError +from synthorg.integrations.errors import ( + TunnelAlreadyActiveError, + TunnelError, +) from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.integrations import ( TUNNEL_ERROR, @@ -86,13 +89,29 @@ async def start(self) -> str: port=self._port, ) msg = "ngrok tunnel already active on this adapter" - raise RuntimeError(msg) + raise TunnelAlreadyActiveError(msg) + # Build a per-call ``PyngrokConfig`` instead of mutating + # ``conf.get_default().auth_token``. The default config is + # process-global; mutating it from one adapter would + # silently overwrite the auth token any other adapter or + # caller had previously set, and leaving a blank token in + # place would cause subsequent unauthenticated calls to + # silently reuse stale credentials. Per-call config keeps + # the token instance-local. auth_token = os.environ.get(self._auth_token_env, "").strip() - if auth_token: - conf.get_default().auth_token = auth_token + pyngrok_config = ( + conf.PyngrokConfig(auth_token=auth_token) + if auth_token + else conf.PyngrokConfig() + ) try: - tunnel = await asyncio.to_thread(ngrok.connect, self._port, "http") + tunnel = await asyncio.to_thread( + ngrok.connect, + self._port, + "http", + pyngrok_config=pyngrok_config, + ) self._tunnel = tunnel self._public_url = str(tunnel.public_url) except Exception as exc: diff --git a/src/synthorg/integrations/webhooks/replay_protection.py b/src/synthorg/integrations/webhooks/replay_protection.py index 5fd40cb53d..d00980caa0 100644 --- a/src/synthorg/integrations/webhooks/replay_protection.py +++ b/src/synthorg/integrations/webhooks/replay_protection.py @@ -7,8 +7,8 @@ threadpool-dispatched webhook handlers cannot both pass the nonce duplicate test and insert the same nonce. Without the lock, two identical webhook deliveries arriving simultaneously could each see -the nonce as fresh (line 192) and both proceed (line 199), losing the -replay-protection guarantee. +the nonce as fresh and both proceed, losing the replay-protection +guarantee. """ import hashlib diff --git a/src/synthorg/meta/chief_of_staff/monitor.py b/src/synthorg/meta/chief_of_staff/monitor.py index fa62c582f0..19545badb6 100644 --- a/src/synthorg/meta/chief_of_staff/monitor.py +++ b/src/synthorg/meta/chief_of_staff/monitor.py @@ -148,14 +148,16 @@ async def _drain() -> None: raise self._task = None self._last_snapshot = None + # Recreate the loop-bound stop event WHILE holding the + # lifecycle lock. Outside the lock, a racing ``start()`` + # could spawn a monitor task bound to the OLD event + # before this assignment lands, leaving a later stop() + # signalling a different event than the task is waiting + # on. ``self._lifecycle_lock`` itself MUST stay the same + # instance for the service lifetime; only the event is + # swapped. + self._stop_event = asyncio.Event() logger.info(COS_MONITOR_STOPPED) - # Recreate the loop-bound stop event outside the (released) - # lock so a fresh event loop binding works for subsequent - # ``start()`` calls. ``self._lifecycle_lock`` MUST stay the - # same instance for the service lifetime: replacing it would - # let a caller queued on the old lock and a fresh caller - # acquiring the new lock both proceed concurrently. - self._stop_event = asyncio.Event() async def _loop(self) -> None: """Periodic snapshot collection and inflection check. diff --git a/src/synthorg/providers/health_prober.py b/src/synthorg/providers/health_prober.py index 32ccb9f20e..5c157f97d6 100644 --- a/src/synthorg/providers/health_prober.py +++ b/src/synthorg/providers/health_prober.py @@ -260,15 +260,22 @@ async def _drain() -> None: ) raise self._task = None + # Recreate the loop-bound stop event WHILE holding the + # lifecycle lock. Doing it outside the lock leaves a + # window where a racing ``start()`` could acquire the + # lock, spawn a probe task that captures + # ``self._stop_event`` (still the OLD event), and then + # this stop()'s ``self._stop_event = asyncio.Event()`` + # outside the lock would swap in a NEW event. A later + # stop() would signal the new event, but the running + # task is still waiting on the old one, so shutdown + # stalls until the interval timeout. Holding the lock + # across the swap eliminates that race. + # ``self._lifecycle_lock`` itself MUST stay the same + # instance for the service lifetime; only the event is + # swapped. + self._stop_event = asyncio.Event() logger.info(PROVIDER_HEALTH_PROBER_STOPPED) - # Recreate the loop-bound stop event outside the (released) - # lock so a subsequent ``start()`` on a different event loop - # can rebind it. ``self._lifecycle_lock`` MUST stay the same - # instance for the service's lifetime: replacing it would let a - # caller queued on the old lock and a fresh caller acquiring - # the new lock both proceed concurrently, breaking the - # serialisation the canonical pattern guarantees. - self._stop_event = asyncio.Event() async def _run_loop(self) -> None: """Main loop: probe all, then sleep until next cycle or stop.""" diff --git a/tests/unit/api/controllers/test_backup.py b/tests/unit/api/controllers/test_backup.py index 27527daafc..4724a8c8a2 100644 --- a/tests/unit/api/controllers/test_backup.py +++ b/tests/unit/api/controllers/test_backup.py @@ -6,6 +6,7 @@ ``handler.fn(self, ...)``. """ +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -16,7 +17,11 @@ from synthorg.api.controllers.backup import BackupController from synthorg.api.cursor import CursorSecret from synthorg.api.dto import ApiResponse, PaginatedResponse -from synthorg.api.services.idempotency_service import IdempotencyService +from synthorg.api.services.idempotency_service import ( + IdempotencyResult, + IdempotencyService, +) +from synthorg.api.state import AppState from synthorg.backup.errors import ( BackupInProgressError, BackupNotFoundError, @@ -66,14 +71,14 @@ def _make_restore_response( ) -def _make_state_and_service() -> tuple[MagicMock, AsyncMock]: +def _make_state_and_service() -> tuple[SimpleNamespace, AsyncMock]: """Create a mock Litestar State with a mock BackupService in app_state. Returns: Tuple of (mock_state, mock_backup_service). """ service = AsyncMock(spec=BackupService) - app_state = MagicMock() + app_state = MagicMock(spec=AppState) app_state.backup_service = service # The controller now wraps every backup in idempotency_service. # Mock the service so run_idempotent invokes the callback inline @@ -87,14 +92,10 @@ async def _run_idempotent( scope: object, key: object, callback: Any, - ) -> Any: + ) -> IdempotencyResult: del scope, key result = await callback() - outcome = MagicMock() - outcome.timed_out = False - outcome.result = result - outcome.fresh = True - return outcome + return IdempotencyResult(result=result, fresh=True, timed_out=False) idempotency_service.run_idempotent = _run_idempotent app_state.idempotency_service = idempotency_service @@ -103,9 +104,14 @@ async def _run_idempotent( # which ultimately fails the HMAC pipeline. app_state.cursor_secret = CursorSecret.from_key("test-key-32-bytes-padding0000000") - state = MagicMock() - state.app_state = app_state - return state, service + # ``SimpleNamespace`` is the right sentinel for the ``state`` + # carrier here: it has no auto-mocking magic, so ``state.app_state`` + # always returns the assigned object. ``MagicMock(spec=State)`` + # would intercept attribute access via Litestar's + # ``State.__getattr__`` and might hand back a fresh auto-mock + # instead of the spec-bound ``AppState`` we just built; a plain + # ``MagicMock()`` would trip the no-bare-mock gate. + return SimpleNamespace(app_state=app_state), service def _controller() -> BackupController: @@ -120,7 +126,7 @@ class TestCreateBackup: async def test_create_backup_calls_service_with_manual_trigger(self) -> None: state, service = _make_state_and_service() manifest = _make_manifest() - service.create_backup = AsyncMock(return_value=manifest) + service.create_backup.return_value = manifest ctrl = _controller() result = await ctrl.create_backup.fn( @@ -135,9 +141,7 @@ async def test_create_backup_calls_service_with_manual_trigger(self) -> None: async def test_create_backup_returns_409_on_in_progress(self) -> None: state, service = _make_state_and_service() - service.create_backup = AsyncMock( - side_effect=BackupInProgressError("busy"), - ) + service.create_backup.side_effect = BackupInProgressError("busy") ctrl = _controller() with pytest.raises(ConflictError) as exc_info: @@ -156,7 +160,7 @@ class TestListBackups: async def test_list_backups_calls_service(self) -> None: state, service = _make_state_and_service() - service.list_backups = AsyncMock(return_value=()) + service.list_backups.return_value = () ctrl = _controller() result = await ctrl.list_backups.fn(ctrl, state=state) @@ -175,7 +179,7 @@ class TestGetBackup: async def test_get_backup_calls_service_with_id(self) -> None: state, service = _make_state_and_service() manifest = _make_manifest() - service.get_backup = AsyncMock(return_value=manifest) + service.get_backup.return_value = manifest ctrl = _controller() result = await ctrl.get_backup.fn( @@ -190,9 +194,7 @@ async def test_get_backup_calls_service_with_id(self) -> None: async def test_get_backup_raises_404_on_not_found(self) -> None: state, service = _make_state_and_service() - service.get_backup = AsyncMock( - side_effect=BackupNotFoundError("gone"), - ) + service.get_backup.side_effect = BackupNotFoundError("gone") ctrl = _controller() with pytest.raises(NotFoundError): @@ -209,7 +211,7 @@ class TestDeleteBackup: async def test_delete_backup_calls_service_with_id(self) -> None: state, service = _make_state_and_service() - service.delete_backup = AsyncMock(return_value=None) + service.delete_backup.return_value = None ctrl = _controller() result = await ctrl.delete_backup.fn( @@ -223,9 +225,7 @@ async def test_delete_backup_calls_service_with_id(self) -> None: async def test_delete_backup_raises_404_on_not_found(self) -> None: state, service = _make_state_and_service() - service.delete_backup = AsyncMock( - side_effect=BackupNotFoundError("gone"), - ) + service.delete_backup.side_effect = BackupNotFoundError("gone") ctrl = _controller() with pytest.raises(NotFoundError): @@ -243,7 +243,7 @@ class TestRestoreBackup: async def test_restore_calls_service_with_confirm_true(self) -> None: state, service = _make_state_and_service() response = _make_restore_response() - service.restore_from_backup = AsyncMock(return_value=response) + service.restore_from_backup.return_value = response request = RestoreRequest( backup_id="abc123def456", @@ -266,7 +266,7 @@ async def test_restore_calls_service_with_confirm_true(self) -> None: async def test_restore_passes_components_to_service(self) -> None: state, service = _make_state_and_service() response = _make_restore_response() - service.restore_from_backup = AsyncMock(return_value=response) + service.restore_from_backup.return_value = response components = (BackupComponent.PERSISTENCE, BackupComponent.CONFIG) request = RestoreRequest( @@ -297,9 +297,7 @@ async def test_restore_raises_422_without_confirm(self) -> None: async def test_restore_raises_404_on_not_found(self) -> None: state, service = _make_state_and_service() - service.restore_from_backup = AsyncMock( - side_effect=BackupNotFoundError("gone"), - ) + service.restore_from_backup.side_effect = BackupNotFoundError("gone") request = RestoreRequest( backup_id="000000000099", @@ -311,9 +309,7 @@ async def test_restore_raises_404_on_not_found(self) -> None: async def test_restore_raises_409_on_in_progress(self) -> None: state, service = _make_state_and_service() - service.restore_from_backup = AsyncMock( - side_effect=BackupInProgressError("busy"), - ) + service.restore_from_backup.side_effect = BackupInProgressError("busy") request = RestoreRequest( backup_id="abc123def456", @@ -327,9 +323,7 @@ async def test_restore_raises_409_on_in_progress(self) -> None: async def test_restore_raises_422_on_manifest_error(self) -> None: state, service = _make_state_and_service() - service.restore_from_backup = AsyncMock( - side_effect=ManifestError("corrupt manifest"), - ) + service.restore_from_backup.side_effect = ManifestError("corrupt manifest") request = RestoreRequest( backup_id="abc123def456", @@ -343,9 +337,7 @@ async def test_restore_raises_422_on_manifest_error(self) -> None: async def test_restore_raises_500_on_restore_error(self) -> None: state, service = _make_state_and_service() - service.restore_from_backup = AsyncMock( - side_effect=RestoreError("disk failure"), - ) + service.restore_from_backup.side_effect = RestoreError("disk failure") request = RestoreRequest( backup_id="abc123def456", @@ -397,14 +389,15 @@ def _mock_backup_service( session-scoped apps where the factory patch cannot affect the already-created app. """ - mock_svc = MagicMock() + from synthorg.backup.scheduler import BackupScheduler + + mock_svc = MagicMock(spec=BackupService) mock_svc.on_startup = False mock_svc.on_shutdown = False - mock_svc.start = AsyncMock() - mock_svc.stop = AsyncMock() - mock_svc.list_backups = AsyncMock(return_value=[]) - mock_svc.scheduler = MagicMock() - mock_svc.scheduler.stop = AsyncMock() + mock_svc.list_backups.return_value = [] + scheduler = MagicMock(spec=BackupScheduler) + scheduler.stop = AsyncMock(spec=BackupScheduler.stop) + mock_svc.scheduler = scheduler monkeypatch.setattr( "synthorg.api.app.build_backup_service", lambda *_a, **_kw: mock_svc, diff --git a/tests/unit/api/controllers/test_backup_required_idempotency.py b/tests/unit/api/controllers/test_backup_required_idempotency.py index 414875b51c..784f3dda72 100644 --- a/tests/unit/api/controllers/test_backup_required_idempotency.py +++ b/tests/unit/api/controllers/test_backup_required_idempotency.py @@ -11,15 +11,18 @@ """ import inspect +from types import SimpleNamespace from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest -from litestar.datastructures import State from synthorg.api.controllers.backup import BackupController from synthorg.api.cursor import CursorSecret -from synthorg.api.services.idempotency_service import IdempotencyService +from synthorg.api.services.idempotency_service import ( + IdempotencyResult, + IdempotencyService, +) from synthorg.api.state import AppState from synthorg.backup.models import ( BackupComponent, @@ -43,12 +46,17 @@ def _make_manifest() -> BackupManifest: ) -def _make_state(*, run_idempotent: Any) -> MagicMock: +def _make_state( + *, + run_idempotent: Any, +) -> tuple[SimpleNamespace, MagicMock]: + # ``MagicMock(spec=BackupService)`` auto-mocks ``create_backup`` + # as an AsyncMock. Set ``return_value`` on the auto-mock directly + # so the spec-bound interface is preserved (replacing the + # auto-mock with a fresh AsyncMock would discard the bound + # signature). service = MagicMock(spec=BackupService) - service.create_backup = AsyncMock( - spec=BackupService.create_backup, - return_value=_make_manifest(), - ) + service.create_backup.return_value = _make_manifest() app_state = MagicMock(spec=AppState) app_state.backup_service = service idempotency_service = MagicMock(spec=IdempotencyService) @@ -57,9 +65,14 @@ def _make_state(*, run_idempotent: Any) -> MagicMock: app_state.cursor_secret = CursorSecret.from_key( "test-key-32-bytes-padding0000000", ) - state = MagicMock(spec=State) - state.app_state = app_state - return state + # ``SimpleNamespace`` is the right sentinel for the ``state`` + # carrier here: it has no auto-mocking magic, so ``state.app_state`` + # always returns the assigned object. ``MagicMock(spec=State)`` + # would intercept via Litestar's ``State.__getattr__`` and might + # hand back a fresh auto-mock instead of the spec-bound + # ``AppState`` we just built; a plain ``MagicMock()`` would trip + # the no-bare-mock gate. + return SimpleNamespace(app_state=app_state), service class TestRequiredIdempotencyKey: @@ -80,18 +93,18 @@ async def fake_run_idempotent( scope: object, key: object, callback: Any, - ) -> Any: + ) -> IdempotencyResult: captured["scope"] = scope captured["key"] = key await callback() - outcome = MagicMock() - outcome.timed_out = False - outcome.result = _make_manifest().model_dump(mode="json") - outcome.fresh = True - return outcome + return IdempotencyResult( + result=_make_manifest().model_dump(mode="json"), + fresh=True, + timed_out=False, + ) ctrl = BackupController(owner=BackupController) # type: ignore[arg-type] - state = _make_state(run_idempotent=fake_run_idempotent) + state, service = _make_state(run_idempotent=fake_run_idempotent) await ctrl.create_backup.fn( ctrl, state=state, @@ -99,3 +112,11 @@ async def fake_run_idempotent( ) assert str(captured["scope"]) == "backup" assert str(captured["key"]) == "key-abc-123" + # The fake_run_idempotent helper above ``await callback()``-s + # the controller's wrapper, which must delegate to + # ``BackupService.create_backup``. Without this assertion the + # test would pass even if the controller stopped invoking the + # service entirely (e.g. a refactor that returned a stale + # cached manifest from the idempotency layer without ever + # producing a fresh one). + service.create_backup.assert_awaited_once_with(BackupTrigger.MANUAL) diff --git a/tests/unit/api/controllers/test_simulations_idempotency.py b/tests/unit/api/controllers/test_simulations_idempotency.py index 0ef515888b..51fffdf84c 100644 --- a/tests/unit/api/controllers/test_simulations_idempotency.py +++ b/tests/unit/api/controllers/test_simulations_idempotency.py @@ -6,12 +6,13 @@ with HTTP 409 Conflict. """ -import contextlib -from unittest.mock import AsyncMock, MagicMock +import asyncio +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch import pytest from litestar.connection import Request -from litestar.datastructures import State from synthorg.api.controllers.simulations import ( SimulationController, @@ -36,7 +37,7 @@ def _make_config(simulation_id: str = "sim-001") -> SimulationConfig: ) -def _make_state(*, claim_succeeds: bool) -> MagicMock: +def _make_state(*, claim_succeeds: bool) -> SimpleNamespace: """Build a mocked Litestar state with a controllable register_if_absent. When *claim_succeeds* is ``True``, ``register_if_absent`` returns @@ -51,18 +52,21 @@ def _make_state(*, claim_succeeds: bool) -> MagicMock: sim_store.save = AsyncMock(spec=SimulationStore.save) sim_state = MagicMock(spec=ClientSimulationState) sim_state.simulation_store = sim_store + # ``ClientSimulationState`` ``spec=`` already auto-mocks + # ``intake_engine`` / ``pool`` / ``feedback_store`` to mocks that + # mirror the real attribute types; we don't need to override + # them. ``_publish_event`` and the runner spawn are patched in + # the tests so the bodies of these attributes never get touched. sim_state.background_tasks = set() - sim_state.intake_engine = MagicMock() - sim_state.pool = MagicMock() - sim_state.pool.list_clients = AsyncMock(return_value=()) - sim_state.feedback_store = MagicMock() - sim_state.feedback_store.record = MagicMock() app_state = MagicMock(spec=AppState) app_state.client_simulation_state = sim_state app_state.config_resolver = MagicMock(spec=ConfigResolver) - state = MagicMock(spec=State) - state.app_state = app_state - return state + # ``SimpleNamespace`` is the right sentinel for the ``state`` + # carrier here: it has no auto-mocking magic, so ``state.app_state`` + # always returns the assigned object. ``MagicMock(spec=State)`` + # would intercept via Litestar's ``State.__getattr__``; a plain + # ``MagicMock()`` would trip the no-bare-mock gate. + return SimpleNamespace(app_state=app_state) def _make_request() -> MagicMock: @@ -93,20 +97,44 @@ async def test_duplicate_id_rejected_with_conflict(self) -> None: async def test_first_request_passes_idempotency_check(self) -> None: """A fresh ``simulation_id`` survives the idempotency check. - We cannot easily exercise the full happy path here without a - full app fixture (the runner requires intake_engine etc.). - The check verifies the controller progresses past the - idempotency guard and calls ``register_if_absent``. + We patch the WS publish helper and the runner-spawning hooks + so the controller can complete the happy path without + bootstrapping the full app -- this lets the test assert + ``register_if_absent`` was awaited without swallowing + unrelated downstream errors. """ state = _make_state(claim_succeeds=True) ctrl = SimulationController(owner=SimulationController) # type: ignore[arg-type] payload = StartSimulationPayload(config=_make_config(simulation_id="sim-002")) - # The handler will reach the register call then attempt to - # spawn the runner. We tolerate any post-claim error since - # this test only verifies idempotency-guard behaviour, not - # the runner plumbing exercised in the integration suite. - with contextlib.suppress(Exception): + # Patch the boundary collaborators so the handler reaches the + # end of its body without raising. We're not exercising + # publish-ws-event or the background runner here; the + # integration suite covers that. Narrowed patches replace the + # earlier ``contextlib.suppress(Exception)`` which would + # mask any future regression that happened to raise here. + # ``_publish_event`` is patched to a no-op so we don't need + # the WS backbone wired. ``asyncio.create_task`` is patched + # to return a real (immediately-cancelled) Task object so the + # subsequent ``task.add_done_callback`` calls in the + # controller's spawn block work without needing the runner + # body to actually run. + def _spawn_dummy_task(coro: Any, *_a: Any, **_kw: Any) -> asyncio.Task[Any]: + coro.close() + fut: asyncio.Future[None] = asyncio.get_event_loop().create_future() + fut.set_result(None) + return fut # type: ignore[return-value] + + with ( + patch( + "synthorg.api.controllers.simulations._publish_event", + lambda *_a, **_kw: None, + ), + patch( + "synthorg.api.controllers.simulations.asyncio.create_task", + _spawn_dummy_task, + ), + ): await ctrl.start_simulation.fn( ctrl, request=_make_request(), diff --git a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py index a475fe2e1c..5700a76f9e 100644 --- a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py +++ b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py @@ -26,6 +26,9 @@ HybridDecisionProcessor, WinnerSelectProcessor, ) +from synthorg.communication.conflict_resolution.escalation.protocol import ( + EscalationQueueStore, +) from synthorg.persistence.protocol import PersistenceBackend pytestmark = pytest.mark.unit @@ -34,7 +37,11 @@ def _fake_persistence(backend_name: str) -> PersistenceBackend: backend = MagicMock(spec=PersistenceBackend) backend.backend_name = backend_name - backend.build_escalations = MagicMock(return_value=MagicMock()) + # ``backend.build_escalations`` is auto-mocked by ``spec=`` to mirror + # ``PersistenceBackend.build_escalations``; setting ``return_value`` + # avoids replacing the auto-mock with a bare MagicMock and keeps the + # interface contract enforced. + backend.build_escalations.return_value = MagicMock(spec=EscalationQueueStore) return cast(PersistenceBackend, backend) diff --git a/tests/unit/integrations/test_ngrok_adapter_lifecycle.py b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py index c650cadfeb..22374b2708 100644 --- a/tests/unit/integrations/test_ngrok_adapter_lifecycle.py +++ b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py @@ -11,6 +11,7 @@ import pytest +from synthorg.integrations.errors import TunnelAlreadyActiveError from synthorg.integrations.tunnel.ngrok_adapter import NgrokAdapter pytestmark = pytest.mark.unit @@ -21,7 +22,9 @@ def __init__(self, public_url: str = "https://fake.ngrok.io") -> None: self.public_url = public_url -def _fake_connect(_port: int, _proto: str) -> _FakeTunnel: +def _fake_connect(_port: int, _proto: str, **_kwargs: Any) -> _FakeTunnel: + # The real ngrok.connect now receives ``pyngrok_config=`` as a + # keyword arg from the adapter; accept and ignore it in the fake. return _FakeTunnel() @@ -33,7 +36,7 @@ class TestNgrokAdapterLifecycle: """Adapter must serialise concurrent start / stop calls.""" async def test_double_start_raises(self) -> None: - """A second start() while a tunnel is active raises RuntimeError.""" + """A second start() while a tunnel is active raises a domain error.""" adapter = NgrokAdapter() with ( patch( @@ -47,7 +50,7 @@ async def test_double_start_raises(self) -> None: ): url = await adapter.start() assert url == "https://fake.ngrok.io" - with pytest.raises(RuntimeError, match="already active"): + with pytest.raises(TunnelAlreadyActiveError, match="already active"): await adapter.start() await adapter.stop() @@ -70,7 +73,7 @@ async def test_concurrent_starts_yield_one_tunnel(self) -> None: return_exceptions=True, ) successes = [r for r in results if isinstance(r, str)] - errors = [r for r in results if isinstance(r, RuntimeError)] + errors = [r for r in results if isinstance(r, TunnelAlreadyActiveError)] assert len(successes) == 1 assert len(errors) == 1 await adapter.stop() diff --git a/tests/unit/settings/test_backup_subscriber.py b/tests/unit/settings/test_backup_subscriber.py index 3deafbc9ac..dc2699fafe 100644 --- a/tests/unit/settings/test_backup_subscriber.py +++ b/tests/unit/settings/test_backup_subscriber.py @@ -6,6 +6,7 @@ from synthorg.backup.scheduler import BackupScheduler from synthorg.backup.service import BackupService +from synthorg.settings.models import SettingValue from synthorg.settings.service import SettingsService from synthorg.settings.subscriber import SettingsSubscriber from synthorg.settings.subscribers.backup_subscriber import ( @@ -42,7 +43,8 @@ def _make_subscriber( settings_service = MagicMock(spec=SettingsService) async def _mock_get(namespace: str, key: str) -> MagicMock: - result = MagicMock() + del namespace + result = MagicMock(spec=SettingValue) if key == "enabled": result.value = str(enabled) elif key == "schedule_hours": @@ -51,7 +53,7 @@ async def _mock_get(namespace: str, key: str) -> MagicMock: result.value = "" return result - settings_service.get = AsyncMock(side_effect=_mock_get) + settings_service.get = AsyncMock(spec=SettingsService.get, side_effect=_mock_get) sub = BackupSettingsSubscriber( backup_service=service, From 26e0542b649c6635ca9442fff1c30c0eccc61d54 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 15:14:53 +0200 Subject: [PATCH 05/13] fix: round 3 reviewer feedback for audit cleanup C - event_stream: skip TTL eviction loop when dedup_ttl_seconds == 0 (matches docstring contract) - ngrok adapter: idempotent start() returns existing URL instead of raising (facade reconnect semantics) - integrations.errors: drop unused TunnelAlreadyActiveError class - escalation factory tests: add cross_instance_notify=auto branch coverage; assert error messages enumerate available registry keys - test comments: drop forensic mock-gate phrasing per CLAUDE.md (comments explain WHY only, no internal-taxonomy shorthand) - test_stream_dedup: lock in TTL=0 disable behavior - mock_spec_baseline: remove 3 stale entries for sites converted to SimpleNamespace --- scripts/mock_spec_baseline.txt | 3 -- .../communication/event_stream/stream.py | 20 +++++--- src/synthorg/integrations/errors.py | 14 ------ .../integrations/tunnel/ngrok_adapter.py | 26 ++++++---- tests/unit/api/controllers/test_backup.py | 13 +++-- .../test_backup_required_idempotency.py | 13 +++-- .../test_simulations_idempotency.py | 11 +++-- .../escalation/test_factory_registry.py | 29 +++++++++++- .../event_stream/test_stream_dedup.py | 25 ++++++++++ .../test_ngrok_adapter_lifecycle.py | 47 ++++++++++++------- 10 files changed, 129 insertions(+), 72 deletions(-) diff --git a/scripts/mock_spec_baseline.txt b/scripts/mock_spec_baseline.txt index 04bf57ec03..373b4e7447 100644 --- a/scripts/mock_spec_baseline.txt +++ b/scripts/mock_spec_baseline.txt @@ -301,8 +301,6 @@ tests/unit/api/controllers/test_approvals_helpers.py:493:22 tests/unit/api/controllers/test_approvals_helpers.py:494:38 tests/unit/api/controllers/test_approvals_helpers.py:511:22 tests/unit/api/controllers/test_approvals_helpers.py:512:38 -tests/unit/api/controllers/test_backup.py:112:12 -tests/unit/api/controllers/test_backup_required_idempotency.py:72:12 tests/unit/api/controllers/test_collaboration.py:357:23 tests/unit/api/controllers/test_company.py:108:31 tests/unit/api/controllers/test_coordination.py:77:18 @@ -392,7 +390,6 @@ tests/unit/api/controllers/test_setup_has_gpu.py:76:27 tests/unit/api/controllers/test_setup_locales.py:246:33 tests/unit/api/controllers/test_setup_locales.py:269:33 tests/unit/api/controllers/test_setup_locales.py:316:33 -tests/unit/api/controllers/test_simulations_idempotency.py:69:12 tests/unit/api/controllers/test_sse_keepalive_setting.py:28:31 tests/unit/api/controllers/test_sse_keepalive_setting.py:29:41 tests/unit/api/controllers/test_sse_revalidate.py:44:21 diff --git a/src/synthorg/communication/event_stream/stream.py b/src/synthorg/communication/event_stream/stream.py index 1fb17a4d4d..b24fe19e32 100644 --- a/src/synthorg/communication/event_stream/stream.py +++ b/src/synthorg/communication/event_stream/stream.py @@ -219,12 +219,20 @@ def _is_duplicate_locked(self, event: StreamEvent, now: float) -> bool: seen = self._seen_event_ids.get(event.session_id) if seen is None: return False - cutoff = now - self._dedup_ttl_seconds - while seen: - oldest_id, oldest_ts = next(iter(seen.items())) - if oldest_ts >= cutoff: - break - del seen[oldest_id] + # ``dedup_ttl_seconds == 0`` is the documented "disable + # time-based eviction" knob: entries fall out only via the + # per-session size bound. Without this guard the eviction + # cutoff would equal ``now`` and every previously recorded + # entry would test as expired (its ``oldest_ts`` was captured + # at an earlier monotonic reading), draining the window on + # every publish and silently disabling deduplication too. + if self._dedup_ttl_seconds > 0: + cutoff = now - self._dedup_ttl_seconds + while seen: + oldest_id, oldest_ts = next(iter(seen.items())) + if oldest_ts >= cutoff: + break + del seen[oldest_id] if not seen: del self._seen_event_ids[event.session_id] return False diff --git a/src/synthorg/integrations/errors.py b/src/synthorg/integrations/errors.py index da2d70b73b..5cf0c9c393 100644 --- a/src/synthorg/integrations/errors.py +++ b/src/synthorg/integrations/errors.py @@ -192,20 +192,6 @@ class TunnelError(IntegrationError): retryable: ClassVar[bool] = True -class TunnelAlreadyActiveError(IntegrationError): - """A tunnel is already active on the adapter; refuse re-start. - - Lifecycle conflict, not a transient I/O error: the operator must - ``stop()`` the running tunnel before issuing a fresh ``start()``. - Marked non-retryable so the resilience layer does not loop on a - permanent state error and so the API surface returns a 409-style - domain error instead of a generic 500. - """ - - is_retryable = False - retryable: ClassVar[bool] = False - - # -- MCP catalog errors -------------------------------------------------- diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index 8fdf3b97f0..045c928b48 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -15,10 +15,7 @@ from pyngrok import conf, ngrok # type: ignore[import-untyped] -from synthorg.integrations.errors import ( - TunnelAlreadyActiveError, - TunnelError, -) +from synthorg.integrations.errors import TunnelError from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.integrations import ( TUNNEL_ERROR, @@ -69,27 +66,36 @@ def __init__( async def start(self) -> str: """Start the ngrok tunnel. + Idempotent: if a tunnel is already active on this adapter the + existing public URL is returned and the call is logged as a + no-op. Callers (``mcp_service.connect`` and the tunnel facade) + treat ``start()`` as a reconnect-safe entry point, so raising + here would force every caller to wrap the call in a + try/except just to ignore the already-active case. + Returns: - The public URL. + The public URL of the active tunnel. Raises: TunnelError: If the tunnel fails to start (auth rejected, ngrok service down, etc.). ``pyngrok`` itself is a required runtime dependency so an ImportError here is a build / install bug rather than a runtime concern. - RuntimeError: If a tunnel is already active on this - adapter instance. """ async with self._lifecycle_lock: - if self._tunnel is not None: + # ``_public_url`` is the active-tunnel sentinel; it is set + # in lock-step with ``_tunnel`` below and cleared together + # in ``stop()``, so a non-None URL is the canonical + # "tunnel is up" check and avoids a second ``cast``/assert + # to satisfy the type narrowing. + if self._public_url is not None: logger.warning( TUNNEL_ERROR, phase="start", reason="already_active", port=self._port, ) - msg = "ngrok tunnel already active on this adapter" - raise TunnelAlreadyActiveError(msg) + return self._public_url # Build a per-call ``PyngrokConfig`` instead of mutating # ``conf.get_default().auth_token``. The default config is # process-global; mutating it from one adapter would diff --git a/tests/unit/api/controllers/test_backup.py b/tests/unit/api/controllers/test_backup.py index 4724a8c8a2..638371d712 100644 --- a/tests/unit/api/controllers/test_backup.py +++ b/tests/unit/api/controllers/test_backup.py @@ -104,13 +104,12 @@ async def _run_idempotent( # which ultimately fails the HMAC pipeline. app_state.cursor_secret = CursorSecret.from_key("test-key-32-bytes-padding0000000") - # ``SimpleNamespace`` is the right sentinel for the ``state`` - # carrier here: it has no auto-mocking magic, so ``state.app_state`` - # always returns the assigned object. ``MagicMock(spec=State)`` - # would intercept attribute access via Litestar's - # ``State.__getattr__`` and might hand back a fresh auto-mock - # instead of the spec-bound ``AppState`` we just built; a plain - # ``MagicMock()`` would trip the no-bare-mock gate. + # ``state.app_state`` must return the bound ``AppState`` exactly + # as assigned. ``SimpleNamespace`` is a plain attribute container + # with no auto-mocking, so the read is a direct attribute lookup. + # ``MagicMock(spec=State)`` would route the read through + # Litestar's ``State.__getattr__`` and could return a fresh + # auto-mock instead of the bound object. return SimpleNamespace(app_state=app_state), service diff --git a/tests/unit/api/controllers/test_backup_required_idempotency.py b/tests/unit/api/controllers/test_backup_required_idempotency.py index 784f3dda72..e2d137a834 100644 --- a/tests/unit/api/controllers/test_backup_required_idempotency.py +++ b/tests/unit/api/controllers/test_backup_required_idempotency.py @@ -65,13 +65,12 @@ def _make_state( app_state.cursor_secret = CursorSecret.from_key( "test-key-32-bytes-padding0000000", ) - # ``SimpleNamespace`` is the right sentinel for the ``state`` - # carrier here: it has no auto-mocking magic, so ``state.app_state`` - # always returns the assigned object. ``MagicMock(spec=State)`` - # would intercept via Litestar's ``State.__getattr__`` and might - # hand back a fresh auto-mock instead of the spec-bound - # ``AppState`` we just built; a plain ``MagicMock()`` would trip - # the no-bare-mock gate. + # ``state.app_state`` must return the bound ``AppState`` exactly + # as assigned. ``SimpleNamespace`` is a plain attribute container + # with no auto-mocking, so the read is a direct attribute lookup. + # ``MagicMock(spec=State)`` would route the read through + # Litestar's ``State.__getattr__`` and could return a fresh + # auto-mock instead of the bound object. return SimpleNamespace(app_state=app_state), service diff --git a/tests/unit/api/controllers/test_simulations_idempotency.py b/tests/unit/api/controllers/test_simulations_idempotency.py index 51fffdf84c..921ba75f38 100644 --- a/tests/unit/api/controllers/test_simulations_idempotency.py +++ b/tests/unit/api/controllers/test_simulations_idempotency.py @@ -61,11 +61,12 @@ def _make_state(*, claim_succeeds: bool) -> SimpleNamespace: app_state = MagicMock(spec=AppState) app_state.client_simulation_state = sim_state app_state.config_resolver = MagicMock(spec=ConfigResolver) - # ``SimpleNamespace`` is the right sentinel for the ``state`` - # carrier here: it has no auto-mocking magic, so ``state.app_state`` - # always returns the assigned object. ``MagicMock(spec=State)`` - # would intercept via Litestar's ``State.__getattr__``; a plain - # ``MagicMock()`` would trip the no-bare-mock gate. + # ``state.app_state`` must return the bound ``AppState`` exactly + # as assigned. ``SimpleNamespace`` is a plain attribute container + # with no auto-mocking, so the read is a direct attribute lookup. + # ``MagicMock(spec=State)`` would route the read through + # Litestar's ``State.__getattr__`` and could return a fresh + # auto-mock instead of the bound object. return SimpleNamespace(app_state=app_state) diff --git a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py index 5700a76f9e..2140e7e9c7 100644 --- a/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py +++ b/tests/unit/communication/conflict_resolution/escalation/test_factory_registry.py @@ -82,6 +82,21 @@ def test_postgres_backend_off_passes_none_channel(self) -> None: notify_channel=None, ) + def test_postgres_backend_auto_passes_notify_channel(self) -> None: + # ``auto`` and ``on`` share the same factory branch; covering + # ``auto`` explicitly catches a regression where the equality + # check is narrowed back to ``== "on"``. + config = EscalationQueueConfig( + backend="postgres", + cross_instance_notify="auto", + notify_channel="escalations", + ) + backend = _fake_persistence("postgres") + build_escalation_queue_store(config, backend) + backend.build_escalations.assert_called_once_with( # type: ignore[attr-defined] + notify_channel="escalations", + ) + def test_sqlite_without_persistence_raises(self) -> None: config = EscalationQueueConfig(backend="sqlite") with pytest.raises(ValueError, match="connected persistence backend"): @@ -126,10 +141,20 @@ def test_unknown_queue_backend_raises_value_error(self) -> None: # the factory's defensive ValueError fires for the registered # unknown-key path. config = EscalationQueueConfig.model_construct(backend="unknown") # type: ignore[arg-type] - with pytest.raises(ValueError, match=r"Unknown escalation queue backend"): + match = r"Unknown escalation queue backend" + with pytest.raises(ValueError, match=match) as exc: build_escalation_queue_store(config) + # Error message must enumerate the registered backends so a + # caller hitting a typo learns the valid options without + # spelunking the factory module. + message = str(exc.value) + for expected in ("memory", "sqlite", "postgres"): + assert expected in message def test_unknown_decision_strategy_raises_value_error(self) -> None: config = EscalationQueueConfig.model_construct(decision_strategy="unknown") # type: ignore[arg-type] - with pytest.raises(ValueError, match=r"Unknown decision_strategy"): + with pytest.raises(ValueError, match=r"Unknown decision_strategy") as exc: build_decision_processor(config) + message = str(exc.value) + for expected in ("winner", "hybrid"): + assert expected in message diff --git a/tests/unit/communication/event_stream/test_stream_dedup.py b/tests/unit/communication/event_stream/test_stream_dedup.py index ff545cc707..d87595c31e 100644 --- a/tests/unit/communication/event_stream/test_stream_dedup.py +++ b/tests/unit/communication/event_stream/test_stream_dedup.py @@ -137,3 +137,28 @@ async def test_unsubscribe_with_remaining_subscribers_keeps_dedup(self) -> None: await hub.unsubscribe("session-1", q1) # Other subscriber still present, dedup map should persist. assert "session-1" in hub._seen_event_ids + + async def test_zero_ttl_disables_time_based_eviction(self) -> None: + """``dedup_ttl_seconds=0`` keeps entries until the size bound.""" + clock = FakeClock() + hub = EventStreamHub( + dedup_ttl_seconds=0.0, + dedup_max_entries_per_session=8, + clock=clock, + ) + queue = await hub.subscribe("session-1") + await hub.publish(_event(event_id="evt-001")) + # Advancing the clock far past any reasonable TTL must NOT + # evict the entry: ``ttl=0`` disables time-based eviction by + # contract, so the dedup window survives indefinitely until + # the per-session size bound trims the oldest entry. + clock.advance(3600.0) + await hub.publish(_event(event_id="evt-001")) # duplicate + + delivered: list[StreamEvent] = [] + try: + while True: + delivered.append(queue.get_nowait()) + except asyncio.QueueEmpty: + pass + assert len(delivered) == 1 diff --git a/tests/unit/integrations/test_ngrok_adapter_lifecycle.py b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py index 22374b2708..97d62e1926 100644 --- a/tests/unit/integrations/test_ngrok_adapter_lifecycle.py +++ b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py @@ -11,7 +11,6 @@ import pytest -from synthorg.integrations.errors import TunnelAlreadyActiveError from synthorg.integrations.tunnel.ngrok_adapter import NgrokAdapter pytestmark = pytest.mark.unit @@ -35,47 +34,59 @@ def _fake_disconnect(_url: Any) -> None: class TestNgrokAdapterLifecycle: """Adapter must serialise concurrent start / stop calls.""" - async def test_double_start_raises(self) -> None: - """A second start() while a tunnel is active raises a domain error.""" + async def test_double_start_is_idempotent(self) -> None: + """A second start() while active returns the existing URL.""" adapter = NgrokAdapter() + connect_calls: list[int] = [] + + def _counting_connect(*args: Any, **kwargs: Any) -> _FakeTunnel: + connect_calls.append(1) + return _fake_connect(*args, **kwargs) + with ( patch( "synthorg.integrations.tunnel.ngrok_adapter.ngrok.connect", - _fake_connect, + _counting_connect, ), patch( "synthorg.integrations.tunnel.ngrok_adapter.ngrok.disconnect", _fake_disconnect, ), ): - url = await adapter.start() - assert url == "https://fake.ngrok.io" - with pytest.raises(TunnelAlreadyActiveError, match="already active"): - await adapter.start() + first = await adapter.start() + second = await adapter.start() + assert first == "https://fake.ngrok.io" + assert second == first + # The second call must NOT invoke ngrok.connect a second + # time -- idempotency means the existing tunnel is reused + # rather than a fresh one being negotiated upstream. + assert len(connect_calls) == 1 await adapter.stop() async def test_concurrent_starts_yield_one_tunnel(self) -> None: - """Two simultaneous start() calls: exactly one wins.""" + """Two simultaneous start() calls connect once and return the same URL.""" adapter = NgrokAdapter() + connect_calls: list[int] = [] + + def _counting_connect(*args: Any, **kwargs: Any) -> _FakeTunnel: + connect_calls.append(1) + return _fake_connect(*args, **kwargs) + with ( patch( "synthorg.integrations.tunnel.ngrok_adapter.ngrok.connect", - _fake_connect, + _counting_connect, ), patch( "synthorg.integrations.tunnel.ngrok_adapter.ngrok.disconnect", _fake_disconnect, ), ): - results = await asyncio.gather( - adapter.start(), - adapter.start(), - return_exceptions=True, + results = list( + await asyncio.gather(adapter.start(), adapter.start()), ) - successes = [r for r in results if isinstance(r, str)] - errors = [r for r in results if isinstance(r, TunnelAlreadyActiveError)] - assert len(successes) == 1 - assert len(errors) == 1 + assert results == ["https://fake.ngrok.io", "https://fake.ngrok.io"] + assert len(connect_calls) == 1 await adapter.stop() async def test_stop_without_start_is_noop(self) -> None: From 4d700791e52d6e31b13e7456e5c9ff9f088340fa Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 15:58:59 +0200 Subject: [PATCH 06/13] fix: babysit round 4, 13 findings (9 inline + 4 outside-diff coderabbit) Source code: - simulations.py: roll back register_if_absent claim if post-claim setup raises (publish/spawn/callback registration); add SimulationStore.unregister - backup/scheduler.py: add log_task_exceptions done_callback so silent _run_loop deaths are surfaced via BACKUP_FAILED - escalation/factory.py: log API_APP_STARTUP warning with config value + registered keys before raising ValueError on unknown backend / strategy - ngrok_adapter.py: read NGROK_AUTHTOKEN at __init__ time per CLAUDE.md bootstrap-secret exception (no os.environ at runtime) - budget/trends.py: filter records to [start, end) before _assert_single_currency so out-of-window mixed-currency rows do not reject valid partial-range queries Tests: - test_backup.py: == not is for ApiResponse.data; assert_not_called for confirm=false gate - test_continuous_lifecycle: Event-driven runner, drop wall-clock asyncio.sleep gates - test_ngrok_adapter_lifecycle: replace swallowing fake disconnect with strict MagicMock(spec=ngrok.disconnect) + assert_not_called on stop-without-start path - test_replay_protection_threadsafety: threading.Barrier across all 3 ThreadPoolExecutor blocks so workers all hit the critical section simultaneously - test_monitor_lifecycle / test_pruning_service_lifecycle: asyncio.Event gate around hung loops; release+await task after timeout assertion (no leaked tasks) - test_backup_subscriber: assert_not_called instead of assert_not_awaited on scheduler.stop (catches unawaited coroutine creation) --- src/synthorg/api/controllers/simulations.py | 52 ++++++++++++------- src/synthorg/backup/scheduler.py | 8 +++ src/synthorg/budget/trends.py | 19 ++++--- src/synthorg/client/store.py | 13 +++++ .../conflict_resolution/escalation/factory.py | 18 +++++++ .../integrations/tunnel/ngrok_adapter.py | 17 ++++-- tests/unit/api/controllers/test_backup.py | 12 +++-- .../unit/client/test_continuous_lifecycle.py | 26 +++++++--- .../unit/hr/pruning/test_service_lifecycle.py | 16 +++++- .../test_ngrok_adapter_lifecycle.py | 11 +++- .../test_replay_protection_threadsafety.py | 20 +++++-- .../chief_of_staff/test_monitor_lifecycle.py | 20 +++++-- tests/unit/settings/test_backup_subscriber.py | 9 ++-- 13 files changed, 186 insertions(+), 55 deletions(-) diff --git a/src/synthorg/api/controllers/simulations.py b/src/synthorg/api/controllers/simulations.py index 7700e57971..14f5af2da2 100644 --- a/src/synthorg/api/controllers/simulations.py +++ b/src/synthorg/api/controllers/simulations.py @@ -295,7 +295,6 @@ async def start_simulation( "cannot start a second runner for the same id" ) raise ConflictError(msg) - _publish_event(request, WsEventType.SIMULATION_STARTED, record) async def runner_task() -> None: try: @@ -346,25 +345,38 @@ async def runner_task() -> None: if event is not None: _publish_event(request, event, final) - task = asyncio.create_task( - runner_task(), - name=f"simulation-runner[{record.simulation_id}]", - ) - # Register the exception logger FIRST so a task that finishes - # between ``create_task`` and ``background_tasks.add`` still has - # its failure surfaced -- asyncio invokes done-callbacks in the - # order they were registered. Adding the task to the set - # before attaching the logger would let a fast-completing - # failure fire ``discard`` first and silently drop the error. - task.add_done_callback( - log_task_exceptions( - logger, - SIMULATION_RUN_FAILED, - simulation_id=record.simulation_id, - ), - ) - task.add_done_callback(sim_state.background_tasks.discard) - sim_state.background_tasks.add(task) + # Roll back the ``register_if_absent`` claim if any post-claim + # step (publish, runner spawn, callback registration) raises. + # Without rollback the ``simulation_id`` would stay claimed + # forever and block every retry, defeating the very 409-on- + # duplicate guard the claim provides. + try: + _publish_event(request, WsEventType.SIMULATION_STARTED, record) + task = asyncio.create_task( + runner_task(), + name=f"simulation-runner[{record.simulation_id}]", + ) + # Register the exception logger FIRST so a task that + # finishes between ``create_task`` and + # ``background_tasks.add`` still has its failure + # surfaced -- asyncio invokes done-callbacks in the order + # they were registered. Adding the task to the set before + # attaching the logger would let a fast-completing failure + # fire ``discard`` first and silently drop the error. + task.add_done_callback( + log_task_exceptions( + logger, + SIMULATION_RUN_FAILED, + simulation_id=record.simulation_id, + ), + ) + task.add_done_callback(sim_state.background_tasks.discard) + sim_state.background_tasks.add(task) + except MemoryError, RecursionError: + raise + except BaseException: + await sim_state.simulation_store.unregister(record.simulation_id) + raise return ApiResponse(data=_to_response(record)) @post( diff --git a/src/synthorg/backup/scheduler.py b/src/synthorg/backup/scheduler.py index 6f682d5b2c..4152e76a72 100644 --- a/src/synthorg/backup/scheduler.py +++ b/src/synthorg/backup/scheduler.py @@ -5,6 +5,7 @@ from synthorg.backup.models import BackupTrigger from synthorg.observability import get_logger, safe_error_description +from synthorg.observability.background_tasks import log_task_exceptions from synthorg.observability.events.backup import ( BACKUP_FAILED, BACKUP_SCHEDULER_RESCHEDULED, @@ -72,6 +73,13 @@ async def start(self) -> None: self._run_loop(), name="backup-scheduler", ) + # Surface unexpected loop deaths -- without this callback + # an exception inside ``_run_loop`` would set the task to + # ``done`` silently and ``is_running`` would flip to False + # without anyone noticing the scheduled backups stopped. + self._task.add_done_callback( + log_task_exceptions(logger, BACKUP_FAILED, note="scheduler_loop_died"), + ) logger.info( BACKUP_SCHEDULER_STARTED, interval_hours=self._interval_seconds // 3600, diff --git a/src/synthorg/budget/trends.py b/src/synthorg/budget/trends.py index 5b0f48e012..88cbeebe40 100644 --- a/src/synthorg/budget/trends.py +++ b/src/synthorg/budget/trends.py @@ -238,15 +238,22 @@ def bucket_cost_records( currencies. Summing across currencies would produce a meaningless monetary total. """ - _assert_single_currency(records) bucket_starts = generate_bucket_starts(start, end, bucket_size) + # Filter to the requested ``[start, end)`` window before + # validating currency uniformity. Validating the raw input would + # reject otherwise-valid partial-range queries when the caller + # passes a multi-currency dataset and asks for a single-currency + # slice -- the rows outside the window do not contribute to the + # aggregation, so they are not part of the "is this bucket + # meaningful?" question. + in_window_records = tuple( + record for record in records if start <= record.timestamp < end + ) + _assert_single_currency(in_window_records) sums: dict[datetime, list[float]] = defaultdict(list) - for record in records: - ts = record.timestamp - if ts < start or ts >= end: - continue - key = _bucket_key(ts, bucket_size) + for record in in_window_records: + key = _bucket_key(record.timestamp, bucket_size) sums[key].append(record.cost) return tuple( diff --git a/src/synthorg/client/store.py b/src/synthorg/client/store.py index 549e07ddbd..821ac2e3d5 100644 --- a/src/synthorg/client/store.py +++ b/src/synthorg/client/store.py @@ -191,6 +191,19 @@ async def register_if_absent(self, record: SimulationRecord) -> bool: self._runs[record.simulation_id] = record return True + async def unregister(self, simulation_id: str) -> bool: + """Remove a registration if it has not produced state yet. + + Returns ``True`` when the entry was removed, ``False`` when no + entry existed. Used by ``start_simulation`` to roll back a + successful ``register_if_absent`` if the post-claim setup + (event publish, runner spawn) raises -- without rollback the + ``simulation_id`` would stay claimed forever and block every + retry. + """ + async with self._lock: + return self._runs.pop(simulation_id, None) is not None + async def get(self, simulation_id: str) -> SimulationRecord: """Return the record by id or raise ``KeyError``.""" async with self._lock: diff --git a/src/synthorg/communication/conflict_resolution/escalation/factory.py b/src/synthorg/communication/conflict_resolution/escalation/factory.py index c37e645dd3..8ecafb8bb9 100644 --- a/src/synthorg/communication/conflict_resolution/escalation/factory.py +++ b/src/synthorg/communication/conflict_resolution/escalation/factory.py @@ -154,6 +154,17 @@ def build_escalation_queue_store( f"Unknown escalation queue backend: {config.backend!r}. " f"Registered backends: {', '.join(available)}" ) + # Log the misconfiguration before raising so the operator's + # log inventory carries the full context (config value + + # registered keys) even if the caller swallows the exception + # higher up. + logger.warning( + API_APP_STARTUP, + component="escalation_factory", + error=msg, + config_backend=config.backend, + registered=available, + ) raise ValueError(msg) return factory(config, persistence) @@ -260,5 +271,12 @@ def build_decision_processor( f"Unknown decision_strategy: {config.decision_strategy!r}. " f"Registered strategies: {', '.join(available)}" ) + logger.warning( + API_APP_STARTUP, + component="escalation_factory", + error=msg, + decision_strategy=config.decision_strategy, + registered=available, + ) raise ValueError(msg) return factory() diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index 045c928b48..230454cc57 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -50,8 +50,18 @@ def __init__( auth_token_env: str = "NGROK_AUTHTOKEN", # noqa: S107 port: int = 8000, ) -> None: - self._auth_token_env = auth_token_env self._port = port + # The ngrok auth token is a bootstrap secret read from the + # process environment at construction time (the sanctioned + # init-time exception in the configuration-precedence policy: + # bootstrap secrets are env-only with no settings registry + # entry, since they have to be available before + # ``SettingsService`` itself can come up). Reading at + # ``__init__`` keeps the runtime ``start()`` path off the + # ``os.environ`` API and means rotating the token requires a + # fresh adapter instance, which matches how every other + # bootstrap-credential surface in this codebase behaves. + self._auth_token: str = os.environ.get(auth_token_env, "").strip() self._public_url: str | None = None self._tunnel: object | None = None # Per ``docs/reference/lifecycle-sync.md``: a dedicated @@ -104,10 +114,9 @@ async def start(self) -> str: # place would cause subsequent unauthenticated calls to # silently reuse stale credentials. Per-call config keeps # the token instance-local. - auth_token = os.environ.get(self._auth_token_env, "").strip() pyngrok_config = ( - conf.PyngrokConfig(auth_token=auth_token) - if auth_token + conf.PyngrokConfig(auth_token=self._auth_token) + if self._auth_token else conf.PyngrokConfig() ) diff --git a/tests/unit/api/controllers/test_backup.py b/tests/unit/api/controllers/test_backup.py index 638371d712..1f6bc3c696 100644 --- a/tests/unit/api/controllers/test_backup.py +++ b/tests/unit/api/controllers/test_backup.py @@ -189,7 +189,7 @@ async def test_get_backup_calls_service_with_id(self) -> None: service.get_backup.assert_awaited_once_with("abc123def456") assert isinstance(result, ApiResponse) - assert result.data is manifest + assert result.data == manifest async def test_get_backup_raises_404_on_not_found(self) -> None: state, service = _make_state_and_service() @@ -260,7 +260,7 @@ async def test_restore_calls_service_with_confirm_true(self) -> None: components=None, ) assert isinstance(result, ApiResponse) - assert result.data is response + assert result.data == response async def test_restore_passes_components_to_service(self) -> None: state, service = _make_state_and_service() @@ -364,8 +364,12 @@ async def test_service_not_called_when_confirm_false( with pytest.raises(ValidationError): await ctrl.restore_backup.fn(ctrl, state=state, data=request) - # Service must never be called when confirm is false - service.restore_from_backup.assert_not_awaited() + # Service must never be called when confirm is false. + # ``assert_not_called()`` is stricter than ``assert_not_awaited()``: + # the former trips even on an unawaited coroutine, catching a + # regression where the controller forgets the ``await`` but + # still creates the call. + service.restore_from_backup.assert_not_called() @pytest.mark.unit diff --git a/tests/unit/client/test_continuous_lifecycle.py b/tests/unit/client/test_continuous_lifecycle.py index c843e8af69..1dab2dff52 100644 --- a/tests/unit/client/test_continuous_lifecycle.py +++ b/tests/unit/client/test_continuous_lifecycle.py @@ -20,10 +20,18 @@ class _FakeRunner: - """Records run() invocations and returns canned metrics.""" + """Records ``run()`` invocations and signals readiness via ``Event``. + + The ``ready`` event lets a test wait for the runner's first entry + deterministically instead of relying on ``asyncio.sleep(0)`` cycles + to let the inner loop schedule itself. Tests that need to drive + the second ``start()`` only after the first has actually entered + the runner await ``runner.ready.wait()``. + """ def __init__(self) -> None: self.calls = 0 + self.ready = asyncio.Event() async def run( self, @@ -33,7 +41,7 @@ async def run( ) -> tuple[SimulationMetrics, list[Any]]: del sim_config, clients self.calls += 1 - await asyncio.sleep(0) + self.ready.set() return ( SimulationMetrics( total_requirements=1, @@ -69,10 +77,10 @@ async def test_double_start_raises_when_already_running(self) -> None: first = asyncio.create_task( mode.start(sim_config=_sim_config(), clients=()), ) - # Yield so the first task acquires the lock and starts the - # loop; without the yield, the second start() may run first. - await asyncio.sleep(0) - await asyncio.sleep(0) + # Wait until the runner reports it has entered ``run()``; this + # guarantees the first ``start()`` already holds the lifecycle + # lock before the second call below races for it. + await runner.ready.wait() with pytest.raises(RuntimeError, match="already running"): await mode.start(sim_config=_sim_config(), clients=()) @@ -104,8 +112,10 @@ async def test_stop_releases_runner_loop(self) -> None: task = asyncio.create_task( mode.start(sim_config=_sim_config(), clients=()), ) - # Let the loop run once. - await asyncio.sleep(0.01) + # Wait for the first run to enter (the runner sets ``ready`` + # on every entry); racing the wall-clock with a fixed sleep + # is what makes lifecycle tests flaky on busy CI runners. + await runner.ready.wait() mode.stop() results = await task assert len(results) >= 1 diff --git a/tests/unit/hr/pruning/test_service_lifecycle.py b/tests/unit/hr/pruning/test_service_lifecycle.py index a3bfb9d827..b632e0dac1 100644 --- a/tests/unit/hr/pruning/test_service_lifecycle.py +++ b/tests/unit/hr/pruning/test_service_lifecycle.py @@ -69,21 +69,33 @@ async def test_restart_after_clean_stop(self) -> None: async def test_unrestartable_after_drain_timeout(self) -> None: service = _make_service() service._stop_drain_timeout_seconds = 0.05 + # ``release`` lets the test wake the patched loop after the + # timeout assertion. Without it, the suppressed-cancel branch + # would block on a wall-clock sleep and leak a pending task + # past the patch scope; later tests could then observe a + # ``_task`` that does not belong to them. + entered = asyncio.Event() + release = asyncio.Event() async def hung_loop(self: PruningService) -> None: del self + entered.set() try: await asyncio.Event().wait() except asyncio.CancelledError: - await asyncio.sleep(1.0) + await release.wait() with patch.object(PruningService, "_run_loop", hung_loop): await service.start() - await asyncio.sleep(0) + await entered.wait() with pytest.raises(TimeoutError): await service.stop() assert service._stop_failed is True + task = service._task + assert task is not None + release.set() + await task with pytest.raises(RuntimeError, match="unrestartable"): await service.start() diff --git a/tests/unit/integrations/test_ngrok_adapter_lifecycle.py b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py index 97d62e1926..5a9ebd71a4 100644 --- a/tests/unit/integrations/test_ngrok_adapter_lifecycle.py +++ b/tests/unit/integrations/test_ngrok_adapter_lifecycle.py @@ -7,9 +7,10 @@ import asyncio from typing import Any -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest +from pyngrok import ngrok # type: ignore[import-untyped] from synthorg.integrations.tunnel.ngrok_adapter import NgrokAdapter @@ -92,9 +93,15 @@ def _counting_connect(*args: Any, **kwargs: Any) -> _FakeTunnel: async def test_stop_without_start_is_noop(self) -> None: """stop() before any start() returns cleanly without disconnecting.""" adapter = NgrokAdapter() + # Use a strict ``MagicMock`` so we can prove ngrok.disconnect + # was never invoked. The previous swallowing fake silently + # absorbed any accidental call, hiding a regression where + # ``stop()`` would tear down state it never owned. + disconnect_mock = MagicMock(spec=ngrok.disconnect, return_value=None) with patch( "synthorg.integrations.tunnel.ngrok_adapter.ngrok.disconnect", - _fake_disconnect, + disconnect_mock, ): await adapter.stop() # Must not raise. + disconnect_mock.assert_not_called() assert adapter._tunnel is None diff --git a/tests/unit/integrations/test_replay_protection_threadsafety.py b/tests/unit/integrations/test_replay_protection_threadsafety.py index 5bf9fb60cf..8f4b5d2b1b 100644 --- a/tests/unit/integrations/test_replay_protection_threadsafety.py +++ b/tests/unit/integrations/test_replay_protection_threadsafety.py @@ -5,6 +5,7 @@ check-and-insert block guarantees exactly one accept per nonce. """ +import threading from concurrent.futures import ThreadPoolExecutor import pytest @@ -26,11 +27,20 @@ def test_concurrent_identical_nonces_yield_single_accept(self) -> None: clock = FakeClock() protector = ReplayProtector(window_seconds=300, clock=clock) ts = _epoch(clock) + # ``Barrier`` ensures every worker has been scheduled and is + # parked at the same instruction before any of them reaches + # ``protector.check``. Without it ``ThreadPoolExecutor.submit`` + # spawns workers staggered, so the first attempt frequently + # finishes the check-and-insert before the rest even hit the + # lock -- which means the test passes even when the lock is + # ineffective. + barrier = threading.Barrier(64) def attempt() -> bool: + barrier.wait() return protector.check(nonce="duplicate-nonce", timestamp=ts) - with ThreadPoolExecutor(max_workers=16) as pool: + with ThreadPoolExecutor(max_workers=64) as pool: futures = [pool.submit(attempt) for _ in range(64)] results = [f.result() for f in futures] @@ -43,11 +53,13 @@ def test_concurrent_distinct_nonces_all_accepted(self) -> None: clock = FakeClock() protector = ReplayProtector(window_seconds=300, clock=clock) ts = _epoch(clock) + barrier = threading.Barrier(64) def attempt(i: int) -> bool: + barrier.wait() return protector.check(nonce=f"nonce-{i}", timestamp=ts) - with ThreadPoolExecutor(max_workers=16) as pool: + with ThreadPoolExecutor(max_workers=64) as pool: futures = [pool.submit(attempt, i) for i in range(64)] results = [f.result() for f in futures] @@ -61,11 +73,13 @@ def test_concurrent_eviction_does_not_corrupt(self) -> None: clock=clock, ) ts = _epoch(clock) + barrier = threading.Barrier(128) def attempt(i: int) -> None: + barrier.wait() protector.check(nonce=f"nonce-{i}", timestamp=ts) - with ThreadPoolExecutor(max_workers=8) as pool: + with ThreadPoolExecutor(max_workers=128) as pool: futures = [pool.submit(attempt, i) for i in range(128)] for f in futures: f.result() diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py index 21207a9536..ef73145351 100644 --- a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -58,22 +58,36 @@ async def test_restart_after_clean_stop(self) -> None: async def test_unrestartable_after_drain_timeout(self) -> None: monitor = _make_monitor() monitor._stop_drain_timeout_seconds = 0.05 + # ``release`` lets the test wake the patched loop after the + # timeout assertion, so the test never leaves a wall-clock- + # sensitive ``asyncio.sleep(1.0)`` task hanging in the + # background. Without this, the next test can race the + # leftover task and observe `_task` from this one. + entered = asyncio.Event() + release = asyncio.Event() async def hung_loop(self: OrgInflectionMonitor) -> None: del self + entered.set() try: await asyncio.Event().wait() except asyncio.CancelledError: - # Suppress cancellation -- simulates a stuck drain. - await asyncio.sleep(1.0) + # Suppress cancellation -- simulates a stuck drain -- + # but block on a controllable event instead of sleep + # so the test can release the loop deterministically. + await release.wait() with patch.object(OrgInflectionMonitor, "_loop", hung_loop): await monitor.start() - await asyncio.sleep(0) + await entered.wait() with pytest.raises(TimeoutError): await monitor.stop() assert monitor._stop_failed is True + task = monitor._task + assert task is not None + release.set() + await task with pytest.raises(RuntimeError, match="unrestartable"): await monitor.start() diff --git a/tests/unit/settings/test_backup_subscriber.py b/tests/unit/settings/test_backup_subscriber.py index dc2699fafe..e0a646569e 100644 --- a/tests/unit/settings/test_backup_subscriber.py +++ b/tests/unit/settings/test_backup_subscriber.py @@ -105,7 +105,10 @@ async def test_enabled_starts_scheduler_when_stopped(self) -> None: # awaited -- the call would still be recorded but the # scheduler would never actually launch. service.scheduler.start.assert_awaited_once() - service.scheduler.stop.assert_not_awaited() + # ``assert_not_called`` catches an unawaited coroutine + # (call recorded but never awaited) which ``assert_not_awaited`` + # would silently pass through. + service.scheduler.stop.assert_not_called() async def test_enabled_stops_scheduler_when_running(self) -> None: sub, service = _make_subscriber( @@ -151,7 +154,7 @@ async def test_advisory_key_does_not_start_scheduler( await sub.on_settings_changed("backup", key) service.scheduler.start.assert_not_called() - service.scheduler.stop.assert_not_awaited() + service.scheduler.stop.assert_not_called() async def test_schedule_hours_reschedules_without_toggle(self) -> None: """schedule_hours calls reschedule but does not stop/start scheduler.""" @@ -160,7 +163,7 @@ async def test_schedule_hours_reschedules_without_toggle(self) -> None: await sub.on_settings_changed("backup", "schedule_hours") service.scheduler.start.assert_not_called() - service.scheduler.stop.assert_not_awaited() + service.scheduler.stop.assert_not_called() service.scheduler.reschedule.assert_called_once() @pytest.mark.parametrize( From 630ff7e668e6802e826be2c821405afd13c42f62 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 16:40:57 +0200 Subject: [PATCH 07/13] fix: babysit round 5, 6 findings (6 inline coderabbit) Source code: - simulations.py: cancel + drain spawned runner_task before unregister on rollback path; extract _attach_runner_callbacks and _rollback_register_if_absent helpers - budget/trends.py: docstring matches in_window_records check (currency error fires on the post-filter set, not raw input) Tests: - test_continuous_lifecycle: module docstring reflects actual lock semantics (released before run loop body) - test_pruning_service_lifecycle: patch asyncio.create_task and assert exactly one spawn under concurrent start() callers - test_replay_protection: threading.Barrier.wait(timeout=5) on all three pools so a stalled worker fails fast instead of hanging the suite - test_monitor_lifecycle: concurrent-starts now uses a blocking builder.build (asyncio.Event) and counts asyncio.create_task spawns to assert exactly one task --- src/synthorg/api/controllers/simulations.py | 89 +++++++++++++++---- src/synthorg/budget/trends.py | 8 +- .../unit/client/test_continuous_lifecycle.py | 12 ++- .../unit/hr/pruning/test_service_lifecycle.py | 33 +++++-- .../test_replay_protection_threadsafety.py | 12 ++- .../chief_of_staff/test_monitor_lifecycle.py | 50 +++++++++-- 6 files changed, 166 insertions(+), 38 deletions(-) diff --git a/src/synthorg/api/controllers/simulations.py b/src/synthorg/api/controllers/simulations.py index 14f5af2da2..0635120107 100644 --- a/src/synthorg/api/controllers/simulations.py +++ b/src/synthorg/api/controllers/simulations.py @@ -112,6 +112,67 @@ async def _mark_failed( ) +def _attach_runner_callbacks( + task: asyncio.Task[None], + *, + sim_state: Any, + simulation_id: str, +) -> None: + """Wire the failure logger + background-task discard to a runner. + + The exception logger is registered FIRST so a task that finishes + between ``create_task`` and the ``add`` below still has its + failure surfaced -- asyncio invokes done-callbacks in the order + they were registered. Adding the task to the set before attaching + the logger would let a fast-completing failure fire ``discard`` + first and silently drop the error. + """ + task.add_done_callback( + log_task_exceptions( + logger, + SIMULATION_RUN_FAILED, + simulation_id=simulation_id, + ), + ) + task.add_done_callback(sim_state.background_tasks.discard) + sim_state.background_tasks.add(task) + + +async def _rollback_register_if_absent( + spawned_task: asyncio.Task[None] | None, + *, + sim_state: Any, + simulation_id: str, +) -> None: + """Tear down a partially-constructed simulation start. + + If the runner task was spawned before the post-claim setup raised, + cancel and drain it before unregistering -- otherwise the orphan + runner would race the unregister and either re-claim the + ``simulation_id`` via ``update_status`` or silently corrupt the + store. ``shield`` is unnecessary here because the caller is the + request handler, not a coroutine guarding against external + cancellation. + """ + if spawned_task is not None: + spawned_task.cancel() + try: + await spawned_task + except asyncio.CancelledError: + pass + except MemoryError, RecursionError: + raise + except Exception as drain_exc: + logger.warning( + SIMULATION_RUN_FAILED, + simulation_id=simulation_id, + stage="rollback_drain", + error_type=type(drain_exc).__name__, + error=safe_error_description(drain_exc), + ) + await sim_state.simulation_store.unregister(simulation_id) + + async def _run_in_background( *, app_state: AppState, @@ -350,32 +411,26 @@ async def runner_task() -> None: # Without rollback the ``simulation_id`` would stay claimed # forever and block every retry, defeating the very 409-on- # duplicate guard the claim provides. + spawned_task: asyncio.Task[None] | None = None try: _publish_event(request, WsEventType.SIMULATION_STARTED, record) - task = asyncio.create_task( + spawned_task = asyncio.create_task( runner_task(), name=f"simulation-runner[{record.simulation_id}]", ) - # Register the exception logger FIRST so a task that - # finishes between ``create_task`` and - # ``background_tasks.add`` still has its failure - # surfaced -- asyncio invokes done-callbacks in the order - # they were registered. Adding the task to the set before - # attaching the logger would let a fast-completing failure - # fire ``discard`` first and silently drop the error. - task.add_done_callback( - log_task_exceptions( - logger, - SIMULATION_RUN_FAILED, - simulation_id=record.simulation_id, - ), + _attach_runner_callbacks( + spawned_task, + sim_state=sim_state, + simulation_id=record.simulation_id, ) - task.add_done_callback(sim_state.background_tasks.discard) - sim_state.background_tasks.add(task) except MemoryError, RecursionError: raise except BaseException: - await sim_state.simulation_store.unregister(record.simulation_id) + await _rollback_register_if_absent( + spawned_task, + sim_state=sim_state, + simulation_id=record.simulation_id, + ) raise return ApiResponse(data=_to_response(record)) diff --git a/src/synthorg/budget/trends.py b/src/synthorg/budget/trends.py index 88cbeebe40..c8fd5ac9e0 100644 --- a/src/synthorg/budget/trends.py +++ b/src/synthorg/budget/trends.py @@ -234,9 +234,11 @@ def bucket_cost_records( Sorted tuple of data points, one per bucket. Raises: - MixedCurrencyAggregationError: If *records* span multiple - currencies. Summing across currencies would produce a - meaningless monetary total. + MixedCurrencyAggregationError: If the records remaining after + filtering to ``[start, end)`` span multiple currencies. + Summing across currencies would produce a meaningless + monetary total. Records outside the window are not part + of the aggregation and are not validated. """ bucket_starts = generate_bucket_starts(start, end, bucket_size) # Filter to the requested ``[start, end)`` window before diff --git a/tests/unit/client/test_continuous_lifecycle.py b/tests/unit/client/test_continuous_lifecycle.py index 1dab2dff52..836a2966b7 100644 --- a/tests/unit/client/test_continuous_lifecycle.py +++ b/tests/unit/client/test_continuous_lifecycle.py @@ -1,10 +1,14 @@ """Lifecycle tests for ``ContinuousMode``. ContinuousMode is an in-place runner (``start()`` executes the loop -synchronously on the caller until ``stop()`` is signalled). The -``_lifecycle_lock`` serialises concurrent ``start()`` calls so the -"already running" RuntimeError is raised reliably, and the lock -spans the full body so a racing caller cannot enter mid-loop. +on the calling coroutine until ``stop()`` is signalled). The +``_lifecycle_lock`` serialises only the ``_running`` flag check at +the top of ``start()`` (acquire / check / set / release) and again +in the ``finally`` to clear the flag; it is NOT held across the run +loop body, so a second ``start()`` raises ``RuntimeError`` rather +than queuing behind the first. ``stop()`` is synchronous and does +not acquire the lock; it sets ``self._stop_event`` so the running +``start()`` coroutine observes the signal on its next iteration. """ import asyncio diff --git a/tests/unit/hr/pruning/test_service_lifecycle.py b/tests/unit/hr/pruning/test_service_lifecycle.py index b632e0dac1..ff432c9bd0 100644 --- a/tests/unit/hr/pruning/test_service_lifecycle.py +++ b/tests/unit/hr/pruning/test_service_lifecycle.py @@ -44,13 +44,36 @@ class TestPruningServiceLifecycleLock: async def test_concurrent_starts_spawn_one_task(self) -> None: service = _make_service() + # Patch ``asyncio.create_task`` (as resolved through the + # service module so the patch reaches the call site) to count + # spawn invocations: ``service.is_running`` alone would also + # be true if the lock leaked and three loop tasks raced. The + # canonical lifecycle contract is "exactly one spawn under + # concurrent ``start()`` callers", which we now assert + # directly. + original_create_task = asyncio.create_task + spawned: list[asyncio.Task[object]] = [] + + def _counting_create_task( + coro: object, + **kwargs: object, + ) -> asyncio.Task[object]: + task: asyncio.Task[object] = original_create_task(coro, **kwargs) # type: ignore[arg-type] + spawned.append(task) + return task + try: - await asyncio.gather( - service.start(), - service.start(), - service.start(), - ) + with patch( + "synthorg.hr.pruning.service.asyncio.create_task", + _counting_create_task, + ): + await asyncio.gather( + service.start(), + service.start(), + service.start(), + ) assert service.is_running + assert len(spawned) == 1 finally: await service.stop() diff --git a/tests/unit/integrations/test_replay_protection_threadsafety.py b/tests/unit/integrations/test_replay_protection_threadsafety.py index 8f4b5d2b1b..ddc8167819 100644 --- a/tests/unit/integrations/test_replay_protection_threadsafety.py +++ b/tests/unit/integrations/test_replay_protection_threadsafety.py @@ -37,7 +37,13 @@ def test_concurrent_identical_nonces_yield_single_accept(self) -> None: barrier = threading.Barrier(64) def attempt() -> bool: - barrier.wait() + # ``timeout`` keeps the barrier from holding the test + # process indefinitely if a worker never arrives (e.g. + # interpreter crash, GIL deadlock). A broken barrier + # bubbles up via ``BrokenBarrierError`` and the future's + # ``result()`` re-raises it -- a fast, diagnosable + # failure in place of a CI hang. + barrier.wait(timeout=5) return protector.check(nonce="duplicate-nonce", timestamp=ts) with ThreadPoolExecutor(max_workers=64) as pool: @@ -56,7 +62,7 @@ def test_concurrent_distinct_nonces_all_accepted(self) -> None: barrier = threading.Barrier(64) def attempt(i: int) -> bool: - barrier.wait() + barrier.wait(timeout=5) return protector.check(nonce=f"nonce-{i}", timestamp=ts) with ThreadPoolExecutor(max_workers=64) as pool: @@ -76,7 +82,7 @@ def test_concurrent_eviction_does_not_corrupt(self) -> None: barrier = threading.Barrier(128) def attempt(i: int) -> None: - barrier.wait() + barrier.wait(timeout=5) protector.check(nonce=f"nonce-{i}", timestamp=ts) with ThreadPoolExecutor(max_workers=128) as pool: diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py index ef73145351..270c9b4121 100644 --- a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -35,15 +35,53 @@ class TestOrgInflectionMonitorLifecycleLock: """Canonical pattern compliance.""" async def test_concurrent_starts_spawn_one_task(self) -> None: - monitor = _make_monitor() + # The first spawned monitor task must still be alive when + # the peer ``start()`` calls run -- otherwise a fast-finishing + # builder lets the first task complete before the others + # land, the lifecycle lock releases, and the test cannot + # distinguish "lock works" from "lock leaked but task already + # finished". Block ``builder.build`` on a controllable Event + # so the spawned task is guaranteed to be running through + # the gather. + block = asyncio.Event() + + async def blocking_build(*_args: object, **_kwargs: object) -> None: + await block.wait() + + builder = AsyncMock(spec=SnapshotBuilder) + builder.build.side_effect = blocking_build + monitor = OrgInflectionMonitor( + detector=OrgInflectionDetector(), + snapshot_builder=builder, + sinks=(), + check_interval_minutes=60, + ) + + original_create_task = asyncio.create_task + spawned: list[asyncio.Task[object]] = [] + + def _counting_create_task( + coro: object, + **kwargs: object, + ) -> asyncio.Task[object]: + task: asyncio.Task[object] = original_create_task(coro, **kwargs) # type: ignore[arg-type] + spawned.append(task) + return task + try: - await asyncio.gather( - monitor.start(), - monitor.start(), - monitor.start(), - ) + with patch( + "synthorg.meta.chief_of_staff.monitor.asyncio.create_task", + _counting_create_task, + ): + await asyncio.gather( + monitor.start(), + monitor.start(), + monitor.start(), + ) assert monitor._task is not None + assert len(spawned) == 1 finally: + block.set() await monitor.stop() async def test_restart_after_clean_stop(self) -> None: From 51c84674eda2ed42d073e9349490d837d60b1164 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 17:27:41 +0200 Subject: [PATCH 08/13] fix: babysit round 6, 4 findings (3 inline + 1 duplicate-but-valid) Source code: - simulations.py: capture and log original exception in except BaseException before _rollback_register_if_absent + re-raise; without this the rollback drain log was the only trace of a failed start Tests: - test_continuous_lifecycle: reword sequencing comment to drop misleading 'lock still held' claim (lock released before run loop body; ready.wait() only proves run() has entered) - test_pruning_service_lifecycle.test_restart_after_clean_stop: assert is_running is True after the second start() so a silent restart no-op cannot pass - test_pruning_service_lifecycle.test_unrestartable_after_drain_timeout / test_monitor_lifecycle.test_unrestartable_after_drain_timeout: wrap timeout/assertion blocks in try/finally so release.set() + saved_task await always run, even when an assertion above raises (no orphan task leaks into next test) --- src/synthorg/api/controllers/simulations.py | 15 +++++++- .../unit/client/test_continuous_lifecycle.py | 10 ++++-- .../unit/hr/pruning/test_service_lifecycle.py | 34 ++++++++++++------- .../chief_of_staff/test_monitor_lifecycle.py | 23 ++++++++----- 4 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/synthorg/api/controllers/simulations.py b/src/synthorg/api/controllers/simulations.py index 0635120107..885d37b0d5 100644 --- a/src/synthorg/api/controllers/simulations.py +++ b/src/synthorg/api/controllers/simulations.py @@ -425,7 +425,20 @@ async def runner_task() -> None: ) except MemoryError, RecursionError: raise - except BaseException: + except BaseException as exc: + # Log the rollback trigger before tearing down -- without + # this entry, a failure between ``register_if_absent`` and + # the callback wiring would leave only the rollback drain + # log, with no record of the original cause for the start + # that the operator would have to chase across components. + logger.warning( + SIMULATION_RUN_FAILED, + simulation_id=record.simulation_id, + stage="post_claim_setup", + spawned_task=spawned_task is not None, + error_type=type(exc).__name__, + error=safe_error_description(exc), + ) await _rollback_register_if_absent( spawned_task, sim_state=sim_state, diff --git a/tests/unit/client/test_continuous_lifecycle.py b/tests/unit/client/test_continuous_lifecycle.py index 836a2966b7..7bbbf326f4 100644 --- a/tests/unit/client/test_continuous_lifecycle.py +++ b/tests/unit/client/test_continuous_lifecycle.py @@ -81,9 +81,13 @@ async def test_double_start_raises_when_already_running(self) -> None: first = asyncio.create_task( mode.start(sim_config=_sim_config(), clients=()), ) - # Wait until the runner reports it has entered ``run()``; this - # guarantees the first ``start()`` already holds the lifecycle - # lock before the second call below races for it. + # ``runner.ready.wait()`` only proves ``run()`` has entered; + # it does NOT prove the lifecycle lock is still held (the + # current ``ContinuousMode`` releases the lock before the + # run loop body). What it gives us is sequencing: the first + # ``start()`` has already passed the ``_running`` check and + # set the flag, so the second call below sees ``_running == + # True`` and is forced down the "already running" branch. await runner.ready.wait() with pytest.raises(RuntimeError, match="already running"): diff --git a/tests/unit/hr/pruning/test_service_lifecycle.py b/tests/unit/hr/pruning/test_service_lifecycle.py index ff432c9bd0..40c7b6303d 100644 --- a/tests/unit/hr/pruning/test_service_lifecycle.py +++ b/tests/unit/hr/pruning/test_service_lifecycle.py @@ -82,11 +82,16 @@ async def test_restart_after_clean_stop(self) -> None: await service.start() await service.stop() # After a clean stop, the service must accept a restart on a - # fresh ``_task``. Cannot assert ``not is_running`` here: - # mypy narrows ``is_running`` to ``False`` after the property - # access and would then flag every subsequent ``await - # service.start()`` / ``stop()`` as unreachable. + # fresh ``_task``. Cannot assert ``not is_running`` between + # the calls because mypy narrows ``is_running`` to ``False`` + # after the property access and would then flag every + # subsequent ``await service.start()`` / ``stop()`` as + # unreachable. await service.start() + # Positive assertion proves the second ``start()`` actually + # took effect: without this, a regression where ``start()`` + # silently no-ops after a stop would still pass the test. + assert service.is_running await service.stop() async def test_unrestartable_after_drain_timeout(self) -> None: @@ -111,14 +116,19 @@ async def hung_loop(self: PruningService) -> None: with patch.object(PruningService, "_run_loop", hung_loop): await service.start() await entered.wait() - - with pytest.raises(TimeoutError): - await service.stop() - assert service._stop_failed is True - task = service._task - assert task is not None - release.set() - await task + saved_task = service._task + try: + with pytest.raises(TimeoutError): + await service.stop() + assert service._stop_failed is True + assert saved_task is not None + finally: + # ``finally`` so a failed assertion above still + # releases the hung loop and drains the orphan task, + # rather than leaking it into the next test run. + release.set() + if saved_task is not None: + await saved_task with pytest.raises(RuntimeError, match="unrestartable"): await service.start() diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py index 270c9b4121..289214a070 100644 --- a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -118,14 +118,21 @@ async def hung_loop(self: OrgInflectionMonitor) -> None: with patch.object(OrgInflectionMonitor, "_loop", hung_loop): await monitor.start() await entered.wait() - - with pytest.raises(TimeoutError): - await monitor.stop() - assert monitor._stop_failed is True - task = monitor._task - assert task is not None - release.set() - await task + saved_task = monitor._task + try: + with pytest.raises(TimeoutError): + await monitor.stop() + assert monitor._stop_failed is True + assert saved_task is not None + finally: + # ``finally`` so a failed assertion above (e.g. the + # service forgot to mark itself unrestartable) still + # releases the hung loop and drains the orphan task. + # Otherwise the leak would silently propagate into + # the next test run. + release.set() + if saved_task is not None: + await saved_task with pytest.raises(RuntimeError, match="unrestartable"): await monitor.start() From 03887ab8bd4d8574c5d4b84a7220d7da0aaddc55 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 17:53:16 +0200 Subject: [PATCH 09/13] fix: babysit round 7, 1 finding (1 inline coderabbit) - test_monitor_lifecycle.test_restart_after_clean_stop: assert monitor._task is not None after the second start() so a silent restart no-op cannot pass the test (mirrors the PruningService equivalent) --- .../unit/meta/chief_of_staff/test_monitor_lifecycle.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py index 289214a070..bf6c632929 100644 --- a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -88,9 +88,15 @@ async def test_restart_after_clean_stop(self) -> None: monitor = _make_monitor() await monitor.start() await monitor.stop() - # Cannot assert ``_task is None`` here -- mypy narrows the - # type and flags the subsequent ``start()`` as unreachable. + # Cannot assert ``_task is None`` between the calls -- mypy + # narrows the type and flags the subsequent ``start()`` as + # unreachable. await monitor.start() + # Positive assertion: the second ``start()`` actually + # rebuilds the loop task. Without this, a regression where + # ``start()`` silently no-ops after a stop would still pass + # the test. + assert monitor._task is not None await monitor.stop() async def test_unrestartable_after_drain_timeout(self) -> None: From df6b61882214e9c46014052db7c1cda865d8b793 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 18:52:28 +0200 Subject: [PATCH 10/13] fix: babysit round 9, 17 findings (17 coderabbit) Domain errors (CLAUDE.md naming convention): - BackupUnrestartableError (backup/errors.py) replaces RuntimeError in BackupScheduler.start() - PruningUnrestartableError (hr/errors.py) replaces RuntimeError in PruningService.start() - InflectionMonitorLifecycleError (meta/chief_of_staff/monitor.py) replaces RuntimeError in OrgInflectionMonitor.start() - ProviderLifecycleConflictError (providers/errors.py) replaces RuntimeError in ProviderHealthProber.start() Source code: - backup.py controller: idempotency_key str -> NotBlankStr; cached BackupManifest.model_validate wrapped in try/except + BACKUP_FAILED log + InternalServerException - SimulationStore.unregister: compare-and-delete via expected= snapshot so a fresh retry winning the slot is preserved instead of being silently deleted by the loser's rollback - event_stream/stream.py: drop dead post-lock if-not-queues branch (already handled inside lock) - ngrok_adapter: materialise public_url BEFORE assigning _tunnel; cleanup-disconnect on conversion failure - replay_protection: sample clock once per check() and pass the same now to freshness + nonce eviction (closes boundary replay gap) - backup_subscriber: try/except around scheduler.start() with structured log + re-raise Tests: - test_scheduler_lifecycle: spawn-count assertion + Event-driven hung loop with finally cleanup - test_trends_currency: derive currencies from DEFAULT_CURRENCY (validator-allowlist forces a real ISO code for the alternate); add out-of-window mixed-currency regression test - test_health_prober_lifecycle: Event-driven hung loop with finally release+await - All four lifecycle tests: pytest.raises now expects the new domain-error class instead of RuntimeError Frontend: - clients.ts startSimulation 409 path: deep-compare existing config to incoming config (field-by-field, type-safe); only inherit the in-flight run when configs match, else rethrow --- src/synthorg/api/controllers/backup.py | 23 ++++++- src/synthorg/api/controllers/simulations.py | 11 +++- src/synthorg/backup/errors.py | 19 ++++++ src/synthorg/backup/scheduler.py | 3 +- src/synthorg/client/store.py | 35 ++++++++--- .../communication/event_stream/stream.py | 2 - src/synthorg/hr/errors.py | 18 ++++++ src/synthorg/hr/pruning/service.py | 3 +- .../integrations/tunnel/ngrok_adapter.py | 33 +++++++++- .../webhooks/replay_protection.py | 23 +++++-- src/synthorg/meta/chief_of_staff/monitor.py | 19 +++++- src/synthorg/providers/errors.py | 16 +++++ src/synthorg/providers/health_prober.py | 3 +- .../settings/subscribers/backup_subscriber.py | 25 +++++++- tests/unit/backup/test_scheduler_lifecycle.py | 63 +++++++++++++++---- tests/unit/budget/test_trends_currency.py | 62 +++++++++++++++--- .../unit/hr/pruning/test_service_lifecycle.py | 3 +- .../chief_of_staff/test_monitor_lifecycle.py | 7 ++- .../providers/test_health_prober_lifecycle.py | 34 ++++++---- web/src/api/endpoints/clients.ts | 28 +++++++-- 20 files changed, 358 insertions(+), 72 deletions(-) diff --git a/src/synthorg/api/controllers/backup.py b/src/synthorg/api/controllers/backup.py index d4ddac64a7..e5a0c87e6a 100644 --- a/src/synthorg/api/controllers/backup.py +++ b/src/synthorg/api/controllers/backup.py @@ -86,7 +86,7 @@ async def create_backup( self, state: State, idempotency_key: Annotated[ - str, + NotBlankStr, Parameter( header="Idempotency-Key", description=( @@ -156,7 +156,26 @@ async def _do_backup() -> BackupManifest: ) msg = "Concurrent in-flight backup with this idempotency key" raise ConflictError(msg) - return ApiResponse(data=BackupManifest.model_validate(outcome.result)) + try: + manifest = BackupManifest.model_validate(outcome.result) + except (ValueError, TypeError) as exc: + # A corrupt or stale cached payload (e.g. schema added a + # field after the entry was stored) would otherwise leak + # the raw pydantic ValidationError. Surface a 5xx instead + # so the operator gets a stable error and the failure is + # visible in logs. + logger.error( # noqa: TRY400 + BACKUP_FAILED, + scope="backup", + idempotency_key=idempotency_key, + endpoint="backup.create", + stage="cached_manifest_validate", + error_type=type(exc).__name__, + error=safe_error_description(exc), + ) + msg = "Cached backup manifest failed validation; rerun the backup" + raise InternalServerException(msg) from exc + return ApiResponse(data=manifest) @get() async def list_backups( diff --git a/src/synthorg/api/controllers/simulations.py b/src/synthorg/api/controllers/simulations.py index 885d37b0d5..93c6abd519 100644 --- a/src/synthorg/api/controllers/simulations.py +++ b/src/synthorg/api/controllers/simulations.py @@ -142,7 +142,7 @@ async def _rollback_register_if_absent( spawned_task: asyncio.Task[None] | None, *, sim_state: Any, - simulation_id: str, + record: SimulationRecord, ) -> None: """Tear down a partially-constructed simulation start. @@ -153,7 +153,12 @@ async def _rollback_register_if_absent( store. ``shield`` is unnecessary here because the caller is the request handler, not a coroutine guarding against external cancellation. + + Passes the originally-claimed ``record`` to ``unregister`` so the + compare-and-delete semantics protect a fresh retry that might have + won the slot between the failure and this rollback running. """ + simulation_id = record.simulation_id if spawned_task is not None: spawned_task.cancel() try: @@ -170,7 +175,7 @@ async def _rollback_register_if_absent( error_type=type(drain_exc).__name__, error=safe_error_description(drain_exc), ) - await sim_state.simulation_store.unregister(simulation_id) + await sim_state.simulation_store.unregister(simulation_id, expected=record) async def _run_in_background( @@ -442,7 +447,7 @@ async def runner_task() -> None: await _rollback_register_if_absent( spawned_task, sim_state=sim_state, - simulation_id=record.simulation_id, + record=record, ) raise return ApiResponse(data=_to_response(record)) diff --git a/src/synthorg/backup/errors.py b/src/synthorg/backup/errors.py index cfab834cfa..096cf9f11d 100644 --- a/src/synthorg/backup/errors.py +++ b/src/synthorg/backup/errors.py @@ -4,6 +4,10 @@ can catch the entire family with a single except clause. """ +from typing import ClassVar + +from synthorg.core.domain_errors import ConflictError + class BackupError(Exception): """Base exception for all backup operations.""" @@ -31,3 +35,18 @@ class RetentionError(BackupError): class BackupNotFoundError(BackupError): """Raised when a requested backup ID does not exist.""" + + +class BackupUnrestartableError(ConflictError): + """Raised when ``BackupScheduler.start()`` is called after a timed-out stop. + + The scheduler refuses to spawn a fresh loop on top of an orphan + task that may still own the backup lock, so the request is + rejected with HTTP 409. Inherits :class:`ConflictError` so the + centralised ``EXCEPTION_HANDLERS`` routing produces the right + RFC 9457 response. + """ + + default_message: ClassVar[str] = ( + "Backup scheduler is unrestartable after a timed-out stop" + ) diff --git a/src/synthorg/backup/scheduler.py b/src/synthorg/backup/scheduler.py index 4152e76a72..4d1f9cd00b 100644 --- a/src/synthorg/backup/scheduler.py +++ b/src/synthorg/backup/scheduler.py @@ -3,6 +3,7 @@ import asyncio from typing import TYPE_CHECKING +from synthorg.backup.errors import BackupUnrestartableError from synthorg.backup.models import BackupTrigger from synthorg.observability import get_logger, safe_error_description from synthorg.observability.background_tasks import log_task_exceptions @@ -64,7 +65,7 @@ async def start(self) -> None: error=msg, note="unrestartable", ) - raise RuntimeError(msg) + raise BackupUnrestartableError(msg) if self.is_running: return self._wake_event.clear() diff --git a/src/synthorg/client/store.py b/src/synthorg/client/store.py index 821ac2e3d5..003232a241 100644 --- a/src/synthorg/client/store.py +++ b/src/synthorg/client/store.py @@ -191,18 +191,37 @@ async def register_if_absent(self, record: SimulationRecord) -> bool: self._runs[record.simulation_id] = record return True - async def unregister(self, simulation_id: str) -> bool: - """Remove a registration if it has not produced state yet. + async def unregister( + self, + simulation_id: str, + *, + expected: SimulationRecord | None = None, + ) -> bool: + """Compare-and-delete the registration if it matches *expected*. Returns ``True`` when the entry was removed, ``False`` when no - entry existed. Used by ``start_simulation`` to roll back a - successful ``register_if_absent`` if the post-claim setup - (event publish, runner spawn) raises -- without rollback the - ``simulation_id`` would stay claimed forever and block every - retry. + entry existed OR a different record now occupies the slot. Used + by ``start_simulation`` to roll back a successful + ``register_if_absent`` if the post-claim setup (event publish, + runner spawn) raises -- without rollback the ``simulation_id`` + would stay claimed forever and block every retry. + + Passing ``expected`` makes the rollback safe under concurrent + retries: if the original claim has already been replaced by a + new run between the failure and the rollback (a fast retry + succeeded while the loser was still tearing down), the new run + is preserved instead of being silently deleted. ``expected=None`` + keeps the old unconditional-delete semantics for callers that + don't have a snapshot to compare against. """ async with self._lock: - return self._runs.pop(simulation_id, None) is not None + current = self._runs.get(simulation_id) + if current is None: + return False + if expected is not None and current is not expected: + return False + del self._runs[simulation_id] + return True async def get(self, simulation_id: str) -> SimulationRecord: """Return the record by id or raise ``KeyError``.""" diff --git a/src/synthorg/communication/event_stream/stream.py b/src/synthorg/communication/event_stream/stream.py index b24fe19e32..cea03bcd19 100644 --- a/src/synthorg/communication/event_stream/stream.py +++ b/src/synthorg/communication/event_stream/stream.py @@ -196,8 +196,6 @@ async def publish(self, event: StreamEvent) -> None: ) return self._record_published_locked(event, now) - if not queues_snapshot: - return for queue in queues_snapshot: try: queue.put_nowait(event) diff --git a/src/synthorg/hr/errors.py b/src/synthorg/hr/errors.py index afc636b137..400dbfffcb 100644 --- a/src/synthorg/hr/errors.py +++ b/src/synthorg/hr/errors.py @@ -8,6 +8,10 @@ should override ``is_retryable = True`` explicitly. """ +from typing import ClassVar + +from synthorg.core.domain_errors import ConflictError + class HRError(Exception): """Base error for all HR operations. @@ -113,6 +117,20 @@ class PruningError(HRError): """Error during the pruning process.""" +class PruningUnrestartableError(ConflictError): + """Raised when ``PruningService.start()`` is called after a timed-out stop. + + Mirrors :class:`BackupUnrestartableError`: a stuck drain leaves an + orphan loop that may still hold references the new instance would + race; the canonical lifecycle pattern marks the service unrestartable + and forces operators to construct a fresh one. + """ + + default_message: ClassVar[str] = ( + "Pruning service is unrestartable after a timed-out stop" + ) + + # ── Personalities ─────────────────────────────────────────────── diff --git a/src/synthorg/hr/pruning/service.py b/src/synthorg/hr/pruning/service.py index 0f659575b9..05e2461163 100644 --- a/src/synthorg/hr/pruning/service.py +++ b/src/synthorg/hr/pruning/service.py @@ -21,6 +21,7 @@ from synthorg.core.enums import ApprovalRiskLevel, ApprovalStatus from synthorg.core.types import NotBlankStr from synthorg.hr.enums import FiringReason +from synthorg.hr.errors import PruningUnrestartableError from synthorg.hr.models import FiringRequest from synthorg.hr.pruning.models import ( PruningEvaluation, @@ -150,7 +151,7 @@ async def start(self) -> None: error=msg, note="unrestartable", ) - raise RuntimeError(msg) + raise PruningUnrestartableError(msg) if self.is_running: return self._wake_event.clear() diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index 230454cc57..b9d34da350 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -127,8 +127,15 @@ async def start(self) -> str: "http", pyngrok_config=pyngrok_config, ) - self._tunnel = tunnel - self._public_url = str(tunnel.public_url) + # Materialise the public URL BEFORE assigning the + # tunnel handle so a converter / attribute-access + # failure on ``tunnel.public_url`` cannot leave the + # adapter in a half-started state where ``_tunnel`` + # exists but ``_public_url`` is still ``None``. Such a + # half-state would later cause ``stop()`` to call + # ``ngrok.disconnect(None)`` on a tunnel the adapter + # never fully owned. + public_url = str(tunnel.public_url) except Exception as exc: # ngrok auth token env var may be echoed in exception # messages; scrub + drop traceback. @@ -138,9 +145,31 @@ async def start(self) -> str: error_type=type(exc).__name__, error=safe_desc, ) + # If ``tunnel`` was created but the URL conversion + # failed afterwards, best-effort disconnect upstream + # so we don't orphan an open tunnel on the ngrok + # side. Failures here are logged but not raised -- + # the caller already gets ``TunnelError``. + local_tunnel = locals().get("tunnel") + if local_tunnel is not None: + try: + await asyncio.to_thread( + ngrok.disconnect, + getattr(local_tunnel, "public_url", None), + ) + except Exception as cleanup_exc: + logger.warning( + TUNNEL_ERROR, + phase="cleanup", + error_type=type(cleanup_exc).__name__, + error=safe_error_description(cleanup_exc), + ) msg = f"Failed to start ngrok tunnel: {safe_desc}" raise TunnelError(msg) from exc + self._public_url = public_url + self._tunnel = tunnel + logger.info( TUNNEL_STARTED, public_url=self._public_url, diff --git a/src/synthorg/integrations/webhooks/replay_protection.py b/src/synthorg/integrations/webhooks/replay_protection.py index d00980caa0..aa0cc4bb4b 100644 --- a/src/synthorg/integrations/webhooks/replay_protection.py +++ b/src/synthorg/integrations/webhooks/replay_protection.py @@ -107,6 +107,17 @@ def check_freshness(self, timestamp: float | None) -> bool: ``False`` if the timestamp is non-finite or outside the configured window. """ + return self._check_freshness_at(timestamp, self._clock.now().timestamp()) + + def _check_freshness_at(self, timestamp: float | None, now: float) -> bool: + """Validate timestamp staleness against a caller-supplied *now*. + + Allows :meth:`check` to sample the clock exactly once and pass + the same snapshot to both the freshness check and the nonce + eviction so a clock advance between two reads cannot open a + boundary replay window where the freshness check uses one + ``now`` and the nonce eviction uses another. + """ if timestamp is None: return True if not math.isfinite(timestamp): @@ -115,9 +126,6 @@ def check_freshness(self, timestamp: float | None) -> bool: reason="non-finite timestamp", ) return False - # Capture once so the comparison and the logged ``skew`` field - # cannot disagree if the clock advances between calls. - now = self._clock.now().timestamp() skew = abs(now - timestamp) if skew > self._window: logger.warning( @@ -160,9 +168,14 @@ def check( reason="no freshness signal (nonce and timestamp both missing)", ) return False - if not self.check_freshness(timestamp): - return False + # Sample the clock once per ``check()`` call and reuse the + # snapshot for both freshness and nonce-eviction decisions so + # a clock advance mid-call cannot open a boundary replay + # window where the freshness check observes one ``now`` and + # the nonce eviction observes another. now = self._clock.now().timestamp() + if not self._check_freshness_at(timestamp, now): + return False return self._check_nonce(nonce=nonce, now=now) def _check_nonce(self, *, nonce: str | None, now: float) -> bool: diff --git a/src/synthorg/meta/chief_of_staff/monitor.py b/src/synthorg/meta/chief_of_staff/monitor.py index 19545badb6..50426e5279 100644 --- a/src/synthorg/meta/chief_of_staff/monitor.py +++ b/src/synthorg/meta/chief_of_staff/monitor.py @@ -8,8 +8,9 @@ import asyncio from datetime import UTC, datetime, timedelta -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar +from synthorg.core.domain_errors import ConflictError from synthorg.observability import get_logger, safe_error_description from synthorg.observability.background_tasks import log_task_exceptions from synthorg.observability.events.chief_of_staff import ( @@ -30,6 +31,20 @@ logger = get_logger(__name__) +class InflectionMonitorLifecycleError(ConflictError): + """Raised when ``OrgInflectionMonitor.start()`` is called after a timed-out stop. + + Mirrors :class:`BackupUnrestartableError`: a stuck drain leaves an + orphan loop the new instance would race; the canonical lifecycle + pattern marks the monitor unrestartable so operators must construct + a fresh one. + """ + + default_message: ClassVar[str] = ( + "OrgInflectionMonitor is unrestartable after a timed-out stop" + ) + + class OrgInflectionMonitor: """Background loop for org-level inflection detection. @@ -87,7 +102,7 @@ async def start(self) -> None: error=msg, note="unrestartable", ) - raise RuntimeError(msg) + raise InflectionMonitorLifecycleError(msg) if self._task is not None and not self._task.done(): return self._stop_event.clear() diff --git a/src/synthorg/providers/errors.py b/src/synthorg/providers/errors.py index 436754516b..1b4ed4fc17 100644 --- a/src/synthorg/providers/errors.py +++ b/src/synthorg/providers/errors.py @@ -9,8 +9,24 @@ from types import MappingProxyType from typing import Any, ClassVar, Final, Literal +from synthorg.core.domain_errors import ConflictError from synthorg.core.error_taxonomy import ErrorCategory, ErrorCode + +class ProviderLifecycleConflictError(ConflictError): + """Raised when ``ProviderHealthProber.start()`` is called after a timed-out stop. + + Mirrors :class:`BackupUnrestartableError` -- a stuck drain leaves + the prober's loop alive on the original instance, so the canonical + lifecycle pattern marks the prober unrestartable rather than + layering a second loop on top of an orphan task. + """ + + default_message: ClassVar[str] = ( + "ProviderHealthProber is unrestartable after a timed-out stop" + ) + + ProviderErrorLabel = Literal[ "rate_limit", "timeout", diff --git a/src/synthorg/providers/health_prober.py b/src/synthorg/providers/health_prober.py index 5c157f97d6..c7ba08a814 100644 --- a/src/synthorg/providers/health_prober.py +++ b/src/synthorg/providers/health_prober.py @@ -29,6 +29,7 @@ ProviderDiscoveryPolicy, is_url_allowed, ) +from synthorg.providers.errors import ProviderLifecycleConflictError from synthorg.providers.health import ProviderHealthRecord, ProviderHealthTracker if TYPE_CHECKING: @@ -201,7 +202,7 @@ async def start(self) -> None: error=msg, note="unrestartable", ) - raise RuntimeError(msg) + raise ProviderLifecycleConflictError(msg) if self._task is not None and not self._task.done(): return self._stop_event.clear() diff --git a/src/synthorg/settings/subscribers/backup_subscriber.py b/src/synthorg/settings/subscribers/backup_subscriber.py index 4a2c24f7ea..3b77d2af4f 100644 --- a/src/synthorg/settings/subscribers/backup_subscriber.py +++ b/src/synthorg/settings/subscribers/backup_subscriber.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from synthorg.observability import get_logger +from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.settings import SETTINGS_SUBSCRIBER_NOTIFIED if TYPE_CHECKING: @@ -110,7 +110,28 @@ async def _toggle_scheduler(self) -> None: enabled = str(result.value).lower() == "true" if enabled and not scheduler.is_running: - await scheduler.start() + try: + await scheduler.start() + except MemoryError, RecursionError: + raise + except Exception as exc: + # Surface a startup failure here -- without this branch + # the exception would propagate from the subscriber + # callback with no context tying it back to the + # setting that triggered it. The structured log entry + # gives the operator the namespace/key plus the + # scrubbed error before re-raising so the dispatcher + # still records the failure. + logger.error( # noqa: TRY400 + SETTINGS_SUBSCRIBER_NOTIFIED, + subscriber=self.subscriber_name, + namespace="backup", + key="enabled", + note="scheduler.start() failed", + error_type=type(exc).__name__, + error=safe_error_description(exc), + ) + raise logger.info( SETTINGS_SUBSCRIBER_NOTIFIED, subscriber=self.subscriber_name, diff --git a/tests/unit/backup/test_scheduler_lifecycle.py b/tests/unit/backup/test_scheduler_lifecycle.py index b06be19175..597294cdb6 100644 --- a/tests/unit/backup/test_scheduler_lifecycle.py +++ b/tests/unit/backup/test_scheduler_lifecycle.py @@ -12,6 +12,7 @@ import pytest +from synthorg.backup.errors import BackupUnrestartableError from synthorg.backup.scheduler import BackupScheduler from synthorg.backup.service import BackupService @@ -29,13 +30,33 @@ class TestBackupSchedulerLifecycleLock: async def test_concurrent_starts_spawn_one_task(self) -> None: scheduler = _make_scheduler() + original_create_task = asyncio.create_task + spawned: list[asyncio.Task[object]] = [] + + def _counting_create_task( + coro: object, + **kwargs: object, + ) -> asyncio.Task[object]: + task: asyncio.Task[object] = original_create_task(coro, **kwargs) # type: ignore[arg-type] + spawned.append(task) + return task + try: - await asyncio.gather( - scheduler.start(), - scheduler.start(), - scheduler.start(), - ) + with patch( + "synthorg.backup.scheduler.asyncio.create_task", + _counting_create_task, + ): + await asyncio.gather( + scheduler.start(), + scheduler.start(), + scheduler.start(), + ) assert scheduler._task is not None + # Exactly one spawn under three concurrent ``start()`` calls + # is the canonical lifecycle contract; ``_task is not None`` + # alone would also pass if the lock leaked and the same task + # got re-assigned each time. + assert len(spawned) == 1 finally: await scheduler.stop() @@ -51,21 +72,37 @@ async def test_restart_after_clean_stop(self) -> None: async def test_unrestartable_after_drain_timeout(self) -> None: scheduler = _make_scheduler() scheduler._stop_drain_timeout_seconds = 0.05 + # ``release`` lets the test wake the patched loop after the + # timeout assertion. Without it the suppressed-cancel branch + # would block on a wall-clock sleep and leak a pending task + # past the patch scope; later tests could then observe a + # ``_task`` that does not belong to them. + entered = asyncio.Event() + release = asyncio.Event() async def hung_loop(self: BackupScheduler) -> None: del self + entered.set() try: await asyncio.Event().wait() except asyncio.CancelledError: - await asyncio.sleep(1.0) + await release.wait() with patch.object(BackupScheduler, "_run_loop", hung_loop): await scheduler.start() - await asyncio.sleep(0) - - with pytest.raises(TimeoutError): - await scheduler.stop() - assert scheduler._stop_failed is True - - with pytest.raises(RuntimeError, match="unrestartable"): + await entered.wait() + saved_task = scheduler._task + try: + with pytest.raises(TimeoutError): + await scheduler.stop() + assert scheduler._stop_failed is True + assert saved_task is not None + finally: + # ``finally`` so a failed assertion above still + # releases the hung loop and drains the orphan task. + release.set() + if saved_task is not None: + await saved_task + + with pytest.raises(BackupUnrestartableError, match="unrestartable"): await scheduler.start() diff --git a/tests/unit/budget/test_trends_currency.py b/tests/unit/budget/test_trends_currency.py index 07194c89b3..2a68ba17d9 100644 --- a/tests/unit/budget/test_trends_currency.py +++ b/tests/unit/budget/test_trends_currency.py @@ -12,6 +12,7 @@ import pytest from synthorg.budget.cost_record import CostRecord +from synthorg.budget.currency import DEFAULT_CURRENCY from synthorg.budget.errors import MixedCurrencyAggregationError from synthorg.budget.trends import ( BucketSize, @@ -22,6 +23,13 @@ pytestmark = pytest.mark.unit _NOW = datetime(2026, 5, 1, 12, 0, 0, tzinfo=UTC) +# ``CurrencyCode`` validates against the project's known ISO 4217 +# allowlist, so the alternate currency MUST be a real ISO code -- we +# can't substitute a synthetic string here. ``EUR`` is picked only +# because it is distinct from ``DEFAULT_CURRENCY``; the specific +# choice is irrelevant to what these tests prove. +_PRIMARY_CURRENCY = DEFAULT_CURRENCY +_ALTERNATE_CURRENCY = "USD" if DEFAULT_CURRENCY != "USD" else "EUR" def _record( @@ -47,8 +55,8 @@ class TestBucketCostRecordsCurrency: def test_single_currency_aggregates_cleanly(self) -> None: records = ( - _record("EUR", 0.10), - _record("EUR", 0.20), + _record(_PRIMARY_CURRENCY, 0.10), + _record(_PRIMARY_CURRENCY, 0.20), ) result = bucket_cost_records( records, @@ -60,8 +68,8 @@ def test_single_currency_aggregates_cleanly(self) -> None: def test_mixed_currency_raises(self) -> None: records = ( - _record("EUR", 0.10), - _record("USD", 0.20), + _record(_PRIMARY_CURRENCY, 0.10), + _record(_ALTERNATE_CURRENCY, 0.20), ) with pytest.raises(MixedCurrencyAggregationError) as exc: bucket_cost_records( @@ -70,7 +78,9 @@ def test_mixed_currency_raises(self) -> None: _NOW + timedelta(hours=1), BucketSize.HOUR, ) - assert exc.value.currencies == frozenset({"EUR", "USD"}) + assert exc.value.currencies == frozenset( + {_PRIMARY_CURRENCY, _ALTERNATE_CURRENCY}, + ) def test_empty_records_no_error(self) -> None: result = bucket_cost_records( @@ -81,26 +91,58 @@ def test_empty_records_no_error(self) -> None: ) assert all(point.value == 0.0 for point in result) + def test_out_of_window_mixed_currency_no_error(self) -> None: + """Mixed currencies entirely outside ``[start, end)`` must not raise. + + The ``_assert_single_currency`` guard runs on the post-filter + slice, so out-of-range rows do not contribute to the + aggregation and do not need to be currency-uniform. Without + this regression, a partial-range query against a long-lived + multi-currency dataset would raise even though the bucket + itself is consistent. + """ + records = ( + _record( + _PRIMARY_CURRENCY, + 0.10, + ts=_NOW - timedelta(hours=10), + ), + _record( + _ALTERNATE_CURRENCY, + 0.20, + ts=_NOW - timedelta(hours=20), + ), + ) + result = bucket_cost_records( + records, + _NOW, + _NOW + timedelta(hours=1), + BucketSize.HOUR, + ) + assert all(point.value == 0.0 for point in result) + class TestProjectDailySpendCurrency: """`project_daily_spend` rejects mixed-currency input.""" def test_single_currency_projects_cleanly(self) -> None: records = ( - _record("EUR", 1.00, _NOW - timedelta(days=2)), - _record("EUR", 2.00, _NOW - timedelta(days=1)), + _record(_PRIMARY_CURRENCY, 1.00, _NOW - timedelta(days=2)), + _record(_PRIMARY_CURRENCY, 2.00, _NOW - timedelta(days=1)), ) forecast = project_daily_spend(records, horizon_days=7, now=_NOW) assert forecast.avg_daily_spend > 0 def test_mixed_currency_raises(self) -> None: records = ( - _record("EUR", 1.00, _NOW - timedelta(days=2)), - _record("USD", 2.00, _NOW - timedelta(days=1)), + _record(_PRIMARY_CURRENCY, 1.00, _NOW - timedelta(days=2)), + _record(_ALTERNATE_CURRENCY, 2.00, _NOW - timedelta(days=1)), ) with pytest.raises(MixedCurrencyAggregationError) as exc: project_daily_spend(records, horizon_days=7, now=_NOW) - assert exc.value.currencies == frozenset({"EUR", "USD"}) + assert exc.value.currencies == frozenset( + {_PRIMARY_CURRENCY, _ALTERNATE_CURRENCY}, + ) def test_empty_records_no_error(self) -> None: forecast = project_daily_spend((), horizon_days=7, now=_NOW) diff --git a/tests/unit/hr/pruning/test_service_lifecycle.py b/tests/unit/hr/pruning/test_service_lifecycle.py index 40c7b6303d..034d0a8534 100644 --- a/tests/unit/hr/pruning/test_service_lifecycle.py +++ b/tests/unit/hr/pruning/test_service_lifecycle.py @@ -12,6 +12,7 @@ import pytest from synthorg.api.approval_store import ApprovalStore +from synthorg.hr.errors import PruningUnrestartableError from synthorg.hr.pruning.models import PruningServiceConfig from synthorg.hr.pruning.service import PruningService from synthorg.hr.registry import AgentRegistryService @@ -130,5 +131,5 @@ async def hung_loop(self: PruningService) -> None: if saved_task is not None: await saved_task - with pytest.raises(RuntimeError, match="unrestartable"): + with pytest.raises(PruningUnrestartableError, match="unrestartable"): await service.start() diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py index bf6c632929..c9e8b736c5 100644 --- a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -12,7 +12,10 @@ import pytest from synthorg.meta.chief_of_staff.inflection import OrgInflectionDetector -from synthorg.meta.chief_of_staff.monitor import OrgInflectionMonitor +from synthorg.meta.chief_of_staff.monitor import ( + InflectionMonitorLifecycleError, + OrgInflectionMonitor, +) from synthorg.meta.signals.snapshot import SnapshotBuilder pytestmark = pytest.mark.unit @@ -140,5 +143,5 @@ async def hung_loop(self: OrgInflectionMonitor) -> None: if saved_task is not None: await saved_task - with pytest.raises(RuntimeError, match="unrestartable"): + with pytest.raises(InflectionMonitorLifecycleError, match="unrestartable"): await monitor.start() diff --git a/tests/unit/providers/test_health_prober_lifecycle.py b/tests/unit/providers/test_health_prober_lifecycle.py index 3cd7e39033..aa44dbee5e 100644 --- a/tests/unit/providers/test_health_prober_lifecycle.py +++ b/tests/unit/providers/test_health_prober_lifecycle.py @@ -14,6 +14,7 @@ import pytest from synthorg.config.schema import ProviderConfig +from synthorg.providers.errors import ProviderLifecycleConflictError from synthorg.providers.health import ProviderHealthTracker from synthorg.providers.health_prober import ProviderHealthProber from synthorg.settings.resolver import ConfigResolver @@ -74,29 +75,38 @@ async def test_unrestartable_after_drain_timeout(self) -> None: """A drain that exceeds the deadline marks the service unrestartable.""" prober = _make_prober() prober._stop_drain_timeout_seconds = 0.05 - # Replace _run_loop with a coroutine that swallows cancellation - # so the drain hangs and triggers the timeout path. - cancel_started = asyncio.Event() + # so the drain hangs and triggers the timeout path. ``release`` + # lets the test wake the patched loop after the timeout + # assertion so the orphan task drains deterministically + # instead of leaking past the patch scope and racing the next + # test. + entered = asyncio.Event() + release = asyncio.Event() async def hung_loop(self: ProviderHealthProber) -> None: del self + entered.set() try: await asyncio.Event().wait() except asyncio.CancelledError: - cancel_started.set() # Suppress cancellation; this simulates a stuck drain. - await asyncio.sleep(1.0) + await release.wait() with patch.object(ProviderHealthProber, "_run_loop", hung_loop): await prober.start() - # Let the hung loop start executing. - await asyncio.sleep(0) - - with pytest.raises(TimeoutError): - await prober.stop() - assert prober._stop_failed is True + await entered.wait() + saved_task = prober._task + try: + with pytest.raises(TimeoutError): + await prober.stop() + assert prober._stop_failed is True + assert saved_task is not None + finally: + release.set() + if saved_task is not None: + await saved_task # Subsequent start must refuse. - with pytest.raises(RuntimeError, match="unrestartable"): + with pytest.raises(ProviderLifecycleConflictError, match="unrestartable"): await prober.start() diff --git a/web/src/api/endpoints/clients.ts b/web/src/api/endpoints/clients.ts index 8b4ae245e3..c9d97c9b36 100644 --- a/web/src/api/endpoints/clients.ts +++ b/web/src/api/endpoints/clients.ts @@ -261,6 +261,19 @@ export async function getSimulation( return unwrap(response) } +function configsEqual(a: SimulationConfig, b: SimulationConfig): boolean { + // Field-by-field compare matches the actual ``SimulationConfig`` + // shape (five primitive fields) and stays correct under key-order + // shifts that ``JSON.stringify`` would silently misreport. + return ( + a.simulation_id === b.simulation_id && + a.project_id === b.project_id && + a.rounds === b.rounds && + a.clients_per_round === b.clients_per_round && + a.requirements_per_client === b.requirements_per_client + ) +} + export async function startSimulation( config: SimulationConfig, ): Promise { @@ -273,12 +286,17 @@ export async function startSimulation( } catch (err) { // The backend returns HTTP 409 when a simulation with // ``config.simulation_id`` is already registered (a redelivery - // or 5xx-driven retry of the same request). Fall back to - // fetching the existing run so the caller's retry path is - // idempotent: retries observe the in-flight runner instead of - // raising and forcing the user to refresh. + // or 5xx-driven retry of the same request). Make the retry path + // idempotent only when the existing run was started with the + // SAME config -- if the configs differ, the caller passed a + // different request that happened to collide on + // ``simulation_id`` and should see the 409 surface instead of + // silently inheriting an unrelated in-flight runner. if (axios.isAxiosError(err) && err.response?.status === 409) { - return getSimulation(config.simulation_id) + const existing = await getSimulation(config.simulation_id) + if (configsEqual(existing.config, config)) { + return existing + } } throw err } From dafc6f4a3bf15ece751111dc376a9d5a4d3ccd51 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 19:26:47 +0200 Subject: [PATCH 11/13] fix: babysit round 10, 6 findings (5 inline + 1 outside-diff) - backup.py: drop runtime NotBlankStr() wrapping in run_idempotent (Annotated alias is not a constructor; type validation runs through Pydantic at the boundary, not on a no-op call) - pruning/service.py stop(): cooperative drain (await wait_for(shield(task), timeout)) before escalating to task.cancel(); only cancel + mark unrestartable on TimeoutError - ngrok_adapter idempotent-start: log TUNNEL_ALREADY_ACTIVE at INFO instead of TUNNEL_ERROR at WARNING (legitimate reconnects are not failures and were triggering tunnel-failure alerts) - monitor.py stop(): replace shield+create_task drain helper with direct wait_for(shield(task)); add explicit cancel + unrestartable mark on TimeoutError; cooperative cancel handling for the no-deadline path - health_prober: add Clock seam (clock=None constructor param defaulting to SystemClock); convert _execute_probe from staticmethod to instance method to use the injected clock for monotonic timing - replay_protection _evict_locked: walk every entry instead of early-exiting on the first non-expired one (round 9's sample-outside-lock change broke the insertion-order = timestamp-order assumption under contention) - test_monitor_lifecycle.test_unrestartable_after_drain_timeout: suppress CancelledError on the post-cleanup task await (Python 3.11+ retains the cancelled state even when the coroutine catches and returns) --- src/synthorg/api/controllers/backup.py | 13 +++-- src/synthorg/hr/pruning/service.py | 48 +++++++++++-------- .../integrations/tunnel/ngrok_adapter.py | 10 ++-- .../webhooks/replay_protection.py | 20 +++++--- src/synthorg/meta/chief_of_staff/monitor.py | 45 +++++++++-------- .../observability/events/integrations.py | 1 + src/synthorg/providers/health_prober.py | 14 ++++-- .../chief_of_staff/test_monitor_lifecycle.py | 11 ++++- 8 files changed, 104 insertions(+), 58 deletions(-) diff --git a/src/synthorg/api/controllers/backup.py b/src/synthorg/api/controllers/backup.py index e5a0c87e6a..9a03ba5e59 100644 --- a/src/synthorg/api/controllers/backup.py +++ b/src/synthorg/api/controllers/backup.py @@ -45,7 +45,7 @@ NotFoundError, ValidationError, ) -from synthorg.core.types import NotBlankStr +from synthorg.core.types import NotBlankStr # noqa: TC001 from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.backup import ( BACKUP_FAILED, @@ -140,9 +140,16 @@ async def _do_backup() -> BackupManifest: msg = "Backup operation failed" raise InternalServerException(msg) from exc + # ``NotBlankStr`` is an Annotated type alias, not a callable + # constructor; calling it at runtime returns the underlying + # ``str`` without running the AfterValidator (which only fires + # through Pydantic). The literal "backup" and the + # already-validated header value satisfy the parameter contract + # directly, so pass them as plain strings instead of fake- + # wrapping them in a no-op call. outcome = await app_state.idempotency_service.run_idempotent( - scope=NotBlankStr("backup"), - key=NotBlankStr(idempotency_key), + scope="backup", + key=idempotency_key, callback=lambda: _do_backup_as_dict(_do_backup), ) if outcome.timed_out: diff --git a/src/synthorg/hr/pruning/service.py b/src/synthorg/hr/pruning/service.py index 05e2461163..1ac839696c 100644 --- a/src/synthorg/hr/pruning/service.py +++ b/src/synthorg/hr/pruning/service.py @@ -165,8 +165,13 @@ async def start(self) -> None: async def stop(self) -> None: """Stop the background scheduler gracefully. - Drain is shielded with a hard deadline; on timeout the service - is marked unrestartable. + First signals the run loop to exit cleanly via ``_stop_event`` + and waits up to ``_stop_drain_timeout_seconds`` for the + in-flight pruning cycle to finish. Only escalates to + ``task.cancel()`` if the cooperative drain times out -- on + timeout the service is also marked unrestartable so a fresh + ``start()`` does not stack a second loop on top of the orphan + task that may still own pruning state. """ async with self._lifecycle_lock: self._stop_event.set() @@ -174,30 +179,18 @@ async def stop(self) -> None: task = self._task if task is None: return - task.cancel() - - async def _drain() -> None: - try: - await task - except asyncio.CancelledError: - pass - except MemoryError, RecursionError: - raise - except Exception as exc: - logger.warning( - HR_PRUNING_POLICY_ERROR, - error_type=type(exc).__name__, - error=safe_error_description(exc), - note="shutdown", - ) - - drain_task: asyncio.Task[None] = asyncio.create_task(_drain()) try: await asyncio.wait_for( - asyncio.shield(drain_task), + asyncio.shield(task), timeout=self._stop_drain_timeout_seconds, ) except TimeoutError: + # Cooperative drain missed the deadline. Cancel hard, + # mark unrestartable, and re-raise so the caller sees + # the timeout. The running cycle owns repository + # state we cannot safely interrupt twice, so we do + # NOT chase the cancellation with a second wait. + task.cancel() self._stop_failed = True logger.error( # noqa: TRY400 HR_PRUNING_POLICY_ERROR, @@ -205,6 +198,19 @@ async def _drain() -> None: timeout_seconds=self._stop_drain_timeout_seconds, ) raise + except asyncio.CancelledError: + # The running cycle was cancelled before it observed + # ``_stop_event``. Drained successfully. + pass + except MemoryError, RecursionError: + raise + except Exception as exc: + logger.warning( + HR_PRUNING_POLICY_ERROR, + error_type=type(exc).__name__, + error=safe_error_description(exc), + note="shutdown", + ) self._task = None # Recreate the loop-bound events WHILE holding the # lifecycle lock. Outside the lock, a racing ``start()`` diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index b9d34da350..e2a19cc631 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -18,6 +18,7 @@ from synthorg.integrations.errors import TunnelError from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.integrations import ( + TUNNEL_ALREADY_ACTIVE, TUNNEL_ERROR, TUNNEL_STARTED, TUNNEL_STOPPED, @@ -99,10 +100,13 @@ async def start(self) -> str: # "tunnel is up" check and avoids a second ``cast``/assert # to satisfy the type narrowing. if self._public_url is not None: - logger.warning( - TUNNEL_ERROR, + # Idempotent reconnect path -- a legitimate caller + # observing an already-active tunnel is not a failure. + # Logging at WARNING with ``TUNNEL_ERROR`` would + # trigger tunnel-failure alerting on every retry. + logger.info( + TUNNEL_ALREADY_ACTIVE, phase="start", - reason="already_active", port=self._port, ) return self._public_url diff --git a/src/synthorg/integrations/webhooks/replay_protection.py b/src/synthorg/integrations/webhooks/replay_protection.py index aa0cc4bb4b..402d012d04 100644 --- a/src/synthorg/integrations/webhooks/replay_protection.py +++ b/src/synthorg/integrations/webhooks/replay_protection.py @@ -230,13 +230,19 @@ def _check_nonce(self, *, nonce: str | None, now: float) -> bool: def _evict_locked(self, now: float) -> None: """Remove nonces older than the window. - Caller must hold ``self._lock``. + Caller must hold ``self._lock``. Walks every entry instead of + early-exiting on the first non-expired one: the caller in + ``check()`` samples ``now`` BEFORE acquiring the lock, which + means under contention the insertion order in ``self._seen`` + no longer matches timestamp order (a thread that read an + older ``now`` can win the lock after a thread with a newer + ``now`` already inserted, leaving an older timestamp behind a + newer one in the ordered map). Walking all entries keeps the + duplicate-reject window pinned to ``self._window`` even when + that ordering invariant is broken. The walk is O(n) but + bounded by ``self._max_entries``. """ cutoff = now - self._window - # OrderedDict preserves insertion order; stop at the first - # non-expired entry since later insertions are always newer. - while self._seen: - nonce, ts = next(iter(self._seen.items())) - if ts >= cutoff: - break + expired = [nonce for nonce, ts in self._seen.items() if ts < cutoff] + for nonce in expired: del self._seen[nonce] diff --git a/src/synthorg/meta/chief_of_staff/monitor.py b/src/synthorg/meta/chief_of_staff/monitor.py index 50426e5279..94ddcd40a7 100644 --- a/src/synthorg/meta/chief_of_staff/monitor.py +++ b/src/synthorg/meta/chief_of_staff/monitor.py @@ -130,30 +130,24 @@ async def stop(self) -> None: task = self._task if task is None: return - task.cancel() - - async def _drain() -> None: - try: - await task - except asyncio.CancelledError: - pass - except MemoryError, RecursionError: - raise - except Exception as exc: - logger.warning( - COS_INFLECTION_CHECK_FAILED, - error_type=type(exc).__name__, - error=safe_error_description(exc), - note="shutdown", - ) - - drain_task: asyncio.Task[None] = asyncio.create_task(_drain()) try: + # Cooperative drain: ``_stop_event`` is already set, + # so the loop wakes from its ``wait_for`` and exits. + # ``shield`` keeps the drain timeout from cancelling + # an in-flight check that's about to terminate + # naturally. No helper task is created -- a direct + # await means there is no orphan wrapper to leak when + # ``wait_for`` times out. await asyncio.wait_for( - asyncio.shield(drain_task), + asyncio.shield(task), timeout=self._stop_drain_timeout_seconds, ) except TimeoutError: + # Cooperative drain missed the deadline. Cancel hard + # and mark the monitor unrestartable so a later + # ``start()`` cannot stack a second loop on top of an + # orphan task that may still own snapshot state. + task.cancel() self._stop_failed = True logger.error( # noqa: TRY400 COS_INFLECTION_CHECK_FAILED, @@ -161,6 +155,19 @@ async def _drain() -> None: timeout_seconds=self._stop_drain_timeout_seconds, ) raise + except asyncio.CancelledError: + # Loop was already cancelled before observing + # ``_stop_event``; treat as drained successfully. + pass + except MemoryError, RecursionError: + raise + except Exception as exc: + logger.warning( + COS_INFLECTION_CHECK_FAILED, + error_type=type(exc).__name__, + error=safe_error_description(exc), + note="shutdown", + ) self._task = None self._last_snapshot = None # Recreate the loop-bound stop event WHILE holding the diff --git a/src/synthorg/observability/events/integrations.py b/src/synthorg/observability/events/integrations.py index 4b3086c0de..0949e20724 100644 --- a/src/synthorg/observability/events/integrations.py +++ b/src/synthorg/observability/events/integrations.py @@ -110,6 +110,7 @@ TUNNEL_STARTED: Final[str] = "integrations.tunnel.started" TUNNEL_STOPPED: Final[str] = "integrations.tunnel.stopped" TUNNEL_ERROR: Final[str] = "integrations.tunnel.error" +TUNNEL_ALREADY_ACTIVE: Final[str] = "integrations.tunnel.already_active" # -- Webhook bridge ------------------------------------------------------ diff --git a/src/synthorg/providers/health_prober.py b/src/synthorg/providers/health_prober.py index c7ba08a814..b47d5896a7 100644 --- a/src/synthorg/providers/health_prober.py +++ b/src/synthorg/providers/health_prober.py @@ -8,13 +8,13 @@ """ import asyncio -import time from datetime import UTC, datetime from typing import TYPE_CHECKING, Final from urllib.parse import urlparse import httpx +from synthorg.core.clock import Clock, SystemClock from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.provider import ( PROVIDER_HEALTH_PROBE_FAILED, @@ -144,6 +144,7 @@ class ProviderHealthProber: """ __slots__ = ( + "_clock", "_config_resolver", "_discovery_policy_loader", "_health_tracker", @@ -164,6 +165,7 @@ def __init__( Callable[[], Awaitable[ProviderDiscoveryPolicy]] | None ) = None, interval_seconds: int = _DEFAULT_INTERVAL_SECONDS, + clock: Clock | None = None, ) -> None: if interval_seconds < 1: msg = f"interval_seconds must be >= 1, got {interval_seconds}" @@ -172,6 +174,10 @@ def __init__( self._config_resolver = config_resolver self._discovery_policy_loader = discovery_policy_loader self._interval = interval_seconds + # ``Clock`` seam per ``CLAUDE.md`` -- tests inject ``FakeClock`` + # so the lifecycle drain timeout and probe-cycle interval can + # be driven on virtual time instead of wall time. + self._clock: Clock = clock if clock is not None else SystemClock() self._stop_event = asyncio.Event() self._task: asyncio.Task[None] | None = None # Per ``docs/reference/lifecycle-sync.md`` the lifecycle lock, @@ -427,8 +433,8 @@ async def _probe_one( latency_ms=round(elapsed_ms, 1), ) - @staticmethod async def _execute_probe( + self, url: str, headers: dict[str, str], ) -> tuple[float, bool, str | None]: @@ -441,7 +447,7 @@ async def _execute_probe( Returns: Tuple of (elapsed_ms, success, error_message). """ - start = time.monotonic() + start = self._clock.monotonic() success = False error_msg: str | None = None @@ -465,5 +471,5 @@ async def _execute_probe( except Exception as exc: error_msg = _truncate(f"{type(exc).__name__}: {exc}") - elapsed_ms = (time.monotonic() - start) * 1000 + elapsed_ms = (self._clock.monotonic() - start) * 1000 return elapsed_ms, success, error_msg diff --git a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py index c9e8b736c5..a43ea0f91a 100644 --- a/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py +++ b/tests/unit/meta/chief_of_staff/test_monitor_lifecycle.py @@ -7,6 +7,7 @@ """ import asyncio +import contextlib from unittest.mock import AsyncMock, patch import pytest @@ -139,9 +140,17 @@ async def hung_loop(self: OrgInflectionMonitor) -> None: # releases the hung loop and drains the orphan task. # Otherwise the leak would silently propagate into # the next test run. + # + # ``suppress(CancelledError)`` covers the canonical + # post-cancel state: ``stop()`` called ``task.cancel()`` + # and the hung loop caught it, but the task remains + # marked cancelled in Python 3.11+ semantics. Awaiting + # it after ``release.set()`` re-raises the residual + # cancellation; that's expected, not a regression. release.set() if saved_task is not None: - await saved_task + with contextlib.suppress(asyncio.CancelledError): + await saved_task with pytest.raises(InflectionMonitorLifecycleError, match="unrestartable"): await monitor.start() From d00336026c72f546e40ae624b569b08a4eb691c7 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 19:50:15 +0200 Subject: [PATCH 12/13] fix: babysit round 11, 2 findings (2 coderabbit, 1 skipped as factually wrong) - pruning/service.py stop(): distinguish external CancelledError from drain-success. If task is still running when CancelledError is caught, the cancel was aimed at stop() (not the loop); re-raise so the external cancellation propagates instead of pretending the service drained - ngrok_adapter: emit NGROK_TUNNEL_STARTED (provider-scoped) instead of the global TUNNEL_STARTED. The tunnel controller already emits the global event after the adapter returns; emitting both inflates metrics keyed on the global event Skipped (factually wrong, logged): - backup.py:168 'use PEP 758 except ValueError, TypeError as exc' is invalid Python. CLAUDE.md explicitly says: 'as exc requires parens (except (A, B) as exc:)'. The current parenthesized form is correct. --- src/synthorg/hr/pruning/service.py | 12 +++++++++--- src/synthorg/integrations/tunnel/ngrok_adapter.py | 8 ++++++-- src/synthorg/observability/events/integrations.py | 7 +++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/synthorg/hr/pruning/service.py b/src/synthorg/hr/pruning/service.py index 1ac839696c..114189084f 100644 --- a/src/synthorg/hr/pruning/service.py +++ b/src/synthorg/hr/pruning/service.py @@ -199,9 +199,15 @@ async def stop(self) -> None: ) raise except asyncio.CancelledError: - # The running cycle was cancelled before it observed - # ``_stop_event``. Drained successfully. - pass + # Distinguish external cancellation of ``stop()`` from + # the running cycle being cancelled before observing + # ``_stop_event``. If the loop task is still alive, the + # cancel was aimed at us, not at it -- re-raise so the + # external cancellation propagates and we don't pretend + # the service drained when it actually didn't. + if not task.done(): + raise + # Loop already finished; drained successfully. except MemoryError, RecursionError: raise except Exception as exc: diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index e2a19cc631..bbc59fc657 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -18,9 +18,9 @@ from synthorg.integrations.errors import TunnelError from synthorg.observability import get_logger, safe_error_description from synthorg.observability.events.integrations import ( + NGROK_TUNNEL_STARTED, TUNNEL_ALREADY_ACTIVE, TUNNEL_ERROR, - TUNNEL_STARTED, TUNNEL_STOPPED, ) @@ -174,8 +174,12 @@ async def start(self) -> str: self._public_url = public_url self._tunnel = tunnel + # Provider-scoped log only -- the tunnel controller emits + # the global ``TUNNEL_STARTED`` after this method returns. + # Emitting both events here would double-count metrics + # keyed on the global event. logger.info( - TUNNEL_STARTED, + NGROK_TUNNEL_STARTED, public_url=self._public_url, port=self._port, note="tunnel exposes localhost publicly", diff --git a/src/synthorg/observability/events/integrations.py b/src/synthorg/observability/events/integrations.py index 0949e20724..b736c4c5ed 100644 --- a/src/synthorg/observability/events/integrations.py +++ b/src/synthorg/observability/events/integrations.py @@ -111,6 +111,13 @@ TUNNEL_STOPPED: Final[str] = "integrations.tunnel.stopped" TUNNEL_ERROR: Final[str] = "integrations.tunnel.error" TUNNEL_ALREADY_ACTIVE: Final[str] = "integrations.tunnel.already_active" +# Provider-scoped twin of ``TUNNEL_STARTED``. The high-level +# ``TUNNEL_STARTED`` is emitted by the tunnel controller after the +# adapter returns; the adapter emits this provider-scoped variant so +# operators can distinguish "the controller started a tunnel" from +# "the ngrok provider opened the upstream connection" without +# double-counting metrics keyed on the global event. +NGROK_TUNNEL_STARTED: Final[str] = "integrations.tunnel.ngrok.started" # -- Webhook bridge ------------------------------------------------------ From f2145f672c6613432e14561d2438b0b4d7811da2 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Sat, 2 May 2026 20:15:03 +0200 Subject: [PATCH 13/13] fix: babysit round 12, 1 finding (1 coderabbit) - ngrok_adapter.start(): replace fragile locals().get('tunnel') lookup with explicit tunnel: Any = None initialised before the try block. The cleanup path now references an unconditionally-defined symbol; the previous form would have raised NameError if a tooling refactor renamed 'tunnel' without updating the locals() string. Type is Any because pyngrok ships untyped stubs. --- src/synthorg/integrations/tunnel/ngrok_adapter.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/synthorg/integrations/tunnel/ngrok_adapter.py b/src/synthorg/integrations/tunnel/ngrok_adapter.py index bbc59fc657..987682b9b5 100644 --- a/src/synthorg/integrations/tunnel/ngrok_adapter.py +++ b/src/synthorg/integrations/tunnel/ngrok_adapter.py @@ -12,6 +12,7 @@ import asyncio import os +from typing import Any from pyngrok import conf, ngrok # type: ignore[import-untyped] @@ -124,6 +125,13 @@ async def start(self) -> str: else conf.PyngrokConfig() ) + # Initialise ``tunnel`` to ``None`` BEFORE the try block + # so the cleanup branch below can reference it + # unconditionally without ``locals()`` introspection or + # ``UnboundLocalError`` if ``ngrok.connect`` itself raises. + # Typed ``Any`` because ``pyngrok`` ships untyped stubs and + # the returned object exposes ``public_url``. + tunnel: Any = None try: tunnel = await asyncio.to_thread( ngrok.connect, @@ -154,12 +162,11 @@ async def start(self) -> str: # so we don't orphan an open tunnel on the ngrok # side. Failures here are logged but not raised -- # the caller already gets ``TunnelError``. - local_tunnel = locals().get("tunnel") - if local_tunnel is not None: + if tunnel is not None: try: await asyncio.to_thread( ngrok.disconnect, - getattr(local_tunnel, "public_url", None), + getattr(tunnel, "public_url", None), ) except Exception as cleanup_exc: logger.warning(