diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..6e981b6c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -854,6 +854,267 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) otherwise. `_write_sse` relies on `_begin_sse`'s already-set marker rather than touching it itself, since it is only ever called after a prior successful header flush. +- `proxy_capability`'s image-generation endpoint rewrite no longer compares + `agent.provider_name` against the literal `"openrouter"`: provider identity + is a display/admin alias, never a routing condition. `ModelAgent` gained an + explicit, declared `image_generation_endpoint` field (`None` by default, + meaning no rewrite); both the immediate-race and sequential-failover call + sites in `proxy_capability` now branch on that field instead. No shipped + agent config or discovery path set the old hardcoded name today, so this is + not a behavior change for any current deployment -- a future OpenRouter (or + any other provider's) image-capable agent must declare + `image_generation_endpoint` explicitly to get the endpoint rewrite. The + identical rewrite condition was duplicated verbatim at both call sites + inside `proxy_capability` (the immediate-race `call()` closure and the + sequential-failover loop); CodeRabbit's review flagged the duplication and + a new module-level `_capability_provider_endpoint(agent, endpoint)` helper + now backs both, following this file's own established precedent + (`_log_retry_outcome`, extracted for the same reason: two near-identical + branches that must not be allowed to silently diverge) rather than the + `@staticmethod` shape CodeRabbit's own suggested diff used. +- `batch_routing.py` and `cost_router.py` shared a single-word KV config + category (`"routing"`) for feature flags -- both module constants + (`_ROUTING_CATEGORY`, `_EMBEDDING_CONFIG_CATEGORY`) and + `batch_job_registry.py`'s direct literal read now use the two-word + `"routing_config"` category name, matching the naming convention + (`conventions.require_object_name`) that already governs other configurable + identifiers. This is one renamed, still-shared category (batch, embedding, + and job-retention flags all still read/write the same category, as before) + -- not three separate categories. + **Migration contract (added after review on #1017):** on the Postgres-backed + boundary, `pg_llm_batch.PostgresConfigStore` keys `com_config` by the literal + `f"{category}.{key}"` string as its SQL primary key, so the call-site rename + alone *does* orphan any row a prior deployment already persisted under + `routing.` -- readers now ask for `routing_config.`, get an exact + miss, and silently fall back to their hardcoded Python default instead of the + operator's configured value. `kv_config.get_config_store()` now runs an + idempotent, additive-only backfill (`_migrate_legacy_categories`, + `_LEGACY_CATEGORY_MIGRATIONS["routing"] -> "routing_config"`) at every boot, + for the seven keys this codebase has ever written under the old category: + a value already present under `routing_config` always wins and is never + overwritten (so an operator's explicit reconfiguration after an earlier + backfill survives a later restart); a legacy `routing.` value with no + `routing_config` counterpart yet is copied forward. This uses only the + `get`/`set` the minimal `ConfigStore` protocol guarantees, so it applies + identically to the in-memory/seeded path and the real Postgres adapter. The + legacy `routing.` rows themselves are left in place, not deleted (the + protocol exposes no delete operation) -- they become harmless once every + reader has moved to `routing_config`, and removing `_LEGACY_CATEGORY_MIGRATIONS["routing"]` + is left as a bounded follow-up once every deployment has booted at least + once against the new category (tracked as gap G-17's residual item in + `ContextualWisdomLab/.github`'s `docs/product-technical-gap-baseline.md`). + See `tests/test_kv_config_store.py`'s four new tests (backfill-from-legacy, + new-value-precedence, idempotent-across-reconnects, in-memory-seed-path) for + the RED-before-GREEN evidence. + **Second review round on #1017 found three further defects.** Fixed + independently and concurrently by reviewer `seonghobae` (pushed directly + to this PR branch; merged here rather than force-pushed over, per this + repo's concurrent-agent-commit policy): (1) the migration call originally + sat inside the same broad `try/except Exception` that falls back to an + ephemeral `InMemoryConfigStore` when `pg_llm_batch` is unavailable or + Postgres is unreachable at construction -- a transient failure purely in + the migration's own reads on an otherwise successfully connected store + fell into that same except and silently discarded visibility into *all* + of the caller's real durable config, not just the migrated keys. The + migration call (renamed public, `migrate_legacy_categories`) now runs + after that except block, and seed-application was moved out there too + for the same reason, so either failure propagates instead. (3) the + `RoutingPolicy` docstring still advertised the pre-rename `routing` + category. (2) only the `get_config_store()` factory ran the migration, so + a caller injecting an already-constructed `ConfigStore` directly (e.g. a + real Postgres-backed store already carrying legacy `routing.*` rows) + never got it migrated. Reconciled with a RED test + (`tests/test_routing_config_compatibility.py`) expecting the fix at the + precise consumption boundary: `RoutingPolicy.__init__` now calls + `migrate_legacy_categories(config_store)` directly, which -- since + `CostRoutingCoordinator.__init__` constructs `RoutingPolicy(self.config)` + from the same shared object before `build_job_registry(self.config)` + runs -- transitively covers that consumer too for the common + (default-`routing_policy`) path. + **Third review round found the remaining gap in (2):** a caller supplying + its own pre-built `routing_policy` (bypassing `RoutingPolicy.__init__` + entirely) still left `build_job_registry`'s `batch_job_retention_seconds` + read -- and, transitively, any later embedding-category read against that + same shared `self.config` -- unmigrated. `build_job_registry()` itself now + also calls `migrate_legacy_categories(config_store)` before its own read, + since it is unconditionally invoked in `CostRoutingCoordinator.__init__` + regardless of whether `routing_policy` was custom-supplied. Eight more + tests added across all three rounds: injected-store migration (at both + the `RoutingPolicy` and `CostRoutingCoordinator`/custom-`routing_policy` + boundaries), migration-failure propagation, and the persistence/ + validation fixes described next -- every one confirmed genuinely RED + before its fix and GREEN after, verified directly by temporarily + reverting each fix and re-running, not assumed. + **Fourth review round:** the `ConfigStore` protocol has no conditional/ + compare-and-swap write, so a genuinely concurrent operator update landing + between `migrate_legacy_categories`'s read of the replacement key and its + own write could, in principle, be clobbered back to the stale legacy + value. Added a re-check of the replacement key immediately before the + write, narrowing that window from "the whole legacy-value read" to just + the gap between the re-check and the write itself -- this does not + eliminate the race (a real conditional-write primitive would, but neither + this protocol nor `pg_llm_batch.PostgresConfigStore.set()`, an + unconditional upsert, currently exposes one; extending that is the owning + repository's boundary, not something to work around here). New + deterministic test (`_InterleavedWriteConfigBackend`) simulates the exact + interleaving without relying on real threading timing, confirmed + genuinely RED before the re-check and GREEN after. Documented as a known, + narrow residual (realistic exposure: a multi-replica Postgres-backed + deployment restarting at the same moment an operator reconfigures the + exact same key) tracked alongside gap G-17 in `ContextualWisdomLab/ + .github`'s `docs/product-technical-gap-baseline.md`. + **Fifth review round found two more defects, one of them a real + regression from the third round's fix.** (1) `build_job_registry()`'s + unconditional `migrate_legacy_categories(config_store)` call (added in + the third round) crashed with `AttributeError` on the function's own + documented "injectable test path": config-store doubles that expose only + `get_secret` (no full `get`/`set` surface), a shape + `tests/test_batch_job_registry_boundaries.py` already exercised and + passed before that change. Confirmed by direct reproduction -- reverting + just the guard failed exactly 4 pre-existing tests in that file with the + same `AttributeError` -- before restoring it. Both Devin Review and + CodeRabbit flagged this independently in the same round; reviewer + `seonghobae` also independently pushed a fix straight to this PR branch + while this round's fix was in progress here. Reconciled by merging (not + force-pushing) and keeping `seonghobae`'s version, which reuses the + existing `get` probe already computed a few lines below instead of a + second `getattr` lookup: `build_job_registry()` now runs the migration + only when `config_store` exposes callable `get` and `set`, the same + "inapplicable, not an error" contract the rest of the function already + applies to a duck-typed secret-only store. (2) a caller supplying *both* + a pre-built `routing_policy` and a pre-built `job_registry` to + `CostRoutingCoordinator` bypassed the migration entirely: neither + `RoutingPolicy(self.config)` nor `build_job_registry(self.config)` is + constructed when its own keyword argument is already supplied, so + `self.config` -- read directly by `CostRoutingCoordinator._embedding_request_limits()` + -- never got migrated for that specific combination. This exact edge + case had been reasoned through and consciously deferred in the third + round (no current production caller supplies both); Devin Review and + CodeRabbit both re-flagged it independently in this round, which raised + confidence it was worth closing rather than continuing to defer. + `CostRoutingCoordinator.__init__` now also calls + `migrate_legacy_categories(self.config)` unconditionally, immediately + after `self.config` is set and before either optional object is + constructed -- independent of, and in addition to, the + `RoutingPolicy.__init__`-level call. New test + (`test_cost_routing_coordinator_migrates_with_both_custom_policy_and_registry`) + confirmed genuinely RED (asserted the migrated `12345` token limit, got + the unmigrated default `280000`) before the fix. +- `ALLOWED_AGENT_PATCH_KEYS`/`ALLOWED_AGENT_CREATE_KEYS` in `server.py` -- + the HTTP-layer request-validation allowlists, entirely separate from + `patch_agent`/`add_agent`'s own field handling in `orchestrator.py` -- + did not include `image_generation_endpoint`, so despite the persistence + fix below, every real HTTP request setting it was rejected with an + `unknown_fields` error before `patch_agent`/`add_agent` ever saw the + field. Added to both allowlists; new end-to-end HTTP test in + `tests/test_agent_pool_db.py` exercises a live PATCH and POST through the + actual server, not just the in-process `orchestrator.py` methods. +- `ModelAgent.image_generation_endpoint` (added above in this same PR) was + never wired into `_AgentPoolStore`, the durable `--state-db`/`agents_db` + SQLite persistence used across process restarts: the field was absent from + `_AGENT_COLUMNS`, the `agent_pool` schema, its `INSERT`/`UPDATE` + statements, and the restart-time `SELECT` -- an operator setting it via + `patch_agent` would see it silently vanish on the very next restart, and + `patch_agent`'s own field allowlist did not accept the key at all, so there + was no way to set it through the admin API in the first place. Fixed: the + column (nullable `TEXT`, non-empty-or-null `CHECK`) is now created for + fresh installs and added via the same guarded `ALTER TABLE` migration + pattern already used for `reasoning_effort_supported`/`max_output_tokens`/ + `context_window`/`stream_usage_supported`; `_insert_agent`/`save`/`load_all` + all round-trip it; `patch_agent` accepts and clears it; the admin + `_agent_to_admin_payload` view surfaces it; and the `/api/v1/agents/*` + OpenAPI patch schema in `api_contract.py` documents it. Separately, + `ModelAgent.__post_init__` now rejects a non-string or empty + `image_generation_endpoint` at construction time (matching the existing + `auth_scheme` validation pattern) instead of letting a malformed value from + JSON survive construction and fail only on the first image request that + reaches it. New tests in `tests/test_agent_pool_db.py`: a full patch -> + restart -> clear -> restart round trip, and a rejected-blank-value/ + no-mutation test. +- `tests/test_psychometric_routing.py` no longer fails collection for the + entire test suite in an environment without `numpy`/`fast_mlsirm` + installed. Only the one test that exercises the real `fast_mlsirm`-backed + scoring path now uses `pytest.importorskip` (mirroring + `psychometric_routing.py`'s own lazy, fail-closed-on-`ImportError` design); + the other four tests in that module never needed either optional package + and now run unconditionally. +- **Sixth review round found a real regression from the first round's own + fix.** Removing `proxy_capability`'s `agent.provider_name == "openrouter"` + hardcode in favor of the declarative `ModelAgent.image_generation_endpoint` + field (first bullet above) never wired the new field into the discovery + path: `agent_from_discovered` built every auto-discovered agent with + `image_generation_endpoint=None`, so a freshly discovered OpenRouter model + silently lost image-generation routing the moment it was ever proxied an + `images/generations` capability request -- exactly the deployment the + removed hardcode used to cover, contradicting this PR's own earlier claim + that no current discovery path relied on it. Devin Review re-flagged this + independently on this round. Fixed: `DiscoveredModel` gained an + `image_generation_endpoint: str | None = None` field, `_parse_openai_compatible` + sets it to `"images"` unconditionally for every OpenRouter-sourced row + (mirroring the removed hardcode 1:1, not gated on that row's own + capabilities/modalities), and `agent_from_discovered` now threads + `discovered.image_generation_endpoint` into the built `ModelAgent`. New + regression tests in `tests/test_model_discovery.py` cover the + discovery-to-agent thread and confirm non-OpenRouter providers stay + unaffected. The companion concurrent-migration-write finding from the same + round (`kv_config.migrate_legacy_categories`'s check-then-set race) was + already investigated, narrowed, and tracked as gap G-17 in a prior round + (see this file's Devin-review concurrency note above); Devin re-flagging it + again this round does not change that a real fix needs a conditional-write + primitive neither the `ConfigStore` protocol nor + `pg_llm_batch.PostgresConfigStore.set()` currently exposes. +- **Seventh review round found the same regression's other half.** The + sixth round's fix threaded `image_generation_endpoint` through direct, + in-process discovery, but the durable catalog-persistence boundary + (`provider_catalog_store.normalize_discovered_model`/ + `_restore_model_semantics`, reached via `record_success`/ + `serving_models`) still reconstructed `DiscoveredModel` without the + field, so a refresh-then-restart round trip through either catalog + backend silently dropped an OpenRouter model's image-generation routing + again. Devin Review caught this immediately after the sixth-round fix + landed. Fixed without a schema migration: `provider_bootstrap. + serving_tags_for_discovered` now emits an + `image_generation_endpoint:` generic serving tag (the same + mechanism already round-tripping capabilities/modalities through the + existing `model_serving_tag` table on both the in-memory and Postgres + backends), `normalize_discovered_model` passes the field straight + through for the live in-flight path, and `_restore_model_semantics` + parses the tag back out on restore. New regression test in + `tests/test_provider_catalog_store.py` runs an OpenRouter model through + `record_success`/`serving_models` and confirms the restored model's + `image_generation_endpoint` survives the round trip. +- **Same round, one more edge on the fix above.** The naive + `image_generation_endpoint:` serving tag from the previous bullet + would itself be silently case-folded or dropped by the catalog store's own + tag-validation charset (`[a-z][a-z0-9_]*(?::[a-z0-9_]+)?`) for any endpoint + value containing a slash, hyphen, or uppercase letter -- today's only real + value (`"images"`) happens to fit that charset, but nothing enforced that a + future provider's endpoint would. Devin Review flagged this immediately. + Fixed by hex-encoding the UTF-8 bytes (`model_discovery. + encode_image_generation_endpoint_tag`/`decode_image_generation_endpoint_tag`, + used by both `provider_bootstrap.serving_tags_for_discovered` and + `provider_catalog_store._restore_model_semantics`): hex output only ever + contains lowercase `0-9a-f`, so it always matches the tag charset + regardless of the source string's content, and a malformed persisted + payload decodes to `None` (fail closed) rather than raising and taking + down every other model's restore. New tests cover a + slash/hyphen/uppercase endpoint's full catalog-store round trip, the + encode/decode pair directly, and a malformed-payload decode. +- **Same round, a namespace collision in the fix above.** `model.capabilities` + values are stored as raw, unprefixed serving tags alongside the reserved + `image_generation_endpoint:` tag. A provider response is untrusted + evidence; if it ever declared a capability string that happened to start + with `"image_generation_endpoint:"` (adversarial or merely coincidental), + the restore path would read that raw capability as the real tag and + silently route every image request to whatever endpoint it encoded after + the next restart, with no endpoint ever legitimately configured. + CodeRabbit flagged this immediately after the hex-encoding fix landed. + Fixed by dropping any capability colliding with the reserved prefix + before `serving_tags_for_discovered` persists it (the namespaced + `capability:` tag it also emits is unaffected and still + round-trips the capability normally). New regression test proves a + colliding capability cannot spoof the restored + `image_generation_endpoint`. ### Added diff --git a/contextual_orchestrator/api_contract.py b/contextual_orchestrator/api_contract.py index 03f89ba3a..6d8b19df3 100644 --- a/contextual_orchestrator/api_contract.py +++ b/contextual_orchestrator/api_contract.py @@ -523,6 +523,12 @@ ] }, "stream_usage_supported": {"type": "boolean"}, + "image_generation_endpoint": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "null"}, + ] + }, }, }, }, diff --git a/contextual_orchestrator/batch_job_registry.py b/contextual_orchestrator/batch_job_registry.py index 21c7fc8a7..b4ceed35e 100644 --- a/contextual_orchestrator/batch_job_registry.py +++ b/contextual_orchestrator/batch_job_registry.py @@ -42,6 +42,8 @@ from contextlib import contextmanager from typing import Any, Callable, Iterator, Optional +from .kv_config import migrate_legacy_categories + # Registry entries expire after this many seconds so abandoned jobs do # not accumulate forever. Seven days comfortably outlives every batch # backend's own completion window. @@ -424,7 +426,26 @@ def build_job_registry(config_store: Any) -> JobRegistryFactory: ``queue`` extra), registries stay in-process dicts — exactly the pre-Valkey behavior — so nothing changes for deployments that have not opted in. + + Migrates legacy KV categories on ``config_store`` before reading + ``batch_job_retention_seconds`` below: ``CostRoutingCoordinator`` + normally triggers the same migration as a side effect of constructing + its default ``RoutingPolicy`` from the same shared store first, but a + caller supplying its own pre-built ``routing_policy`` (skipping that + construction) or calling this function directly would otherwise never + see a legacy-persisted retention value. Gated on ``config_store`` + actually exposing the ``get``/``set`` pair the migration needs, the + same capability probe already used below for the secret and retention + reads: ``config_store`` is documented as ``Any`` and this factory's own + "injectable test path" accepts stores that implement only + ``get_secret`` (no ``get``/``set``), for which the migration -- which + has no legacy categories to move without a full ``ConfigStore`` -- is + simply inapplicable rather than an error. """ + get = getattr(config_store, "get", None) + set_config = getattr(config_store, "set", None) + if callable(get) and callable(set_config): + migrate_legacy_categories(config_store) from .credentials import get_credential try: @@ -442,9 +463,8 @@ def build_job_registry(config_store: Any) -> JobRegistryFactory: return JobRegistryFactory(None) client = redis.Redis.from_url(str(url)) retention = DEFAULT_RETENTION_SECONDS - get = getattr(config_store, "get", None) if callable(get): - configured = get("routing", "batch_job_retention_seconds", DEFAULT_RETENTION_SECONDS) + configured = get("routing_config", "batch_job_retention_seconds", DEFAULT_RETENTION_SECONDS) if type(configured) is int and configured >= 1: retention = configured return JobRegistryFactory(client, retention_seconds=retention) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 5650e441d..cadc62de2 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -39,10 +39,11 @@ JobRegistryFactory, _claim_renewal_interval_seconds, ) +from .kv_config import migrate_legacy_categories _LOGGER = logging.getLogger(__name__) -_ROUTING_CATEGORY = "routing" +_ROUTING_CATEGORY = "routing_config" _PROVIDER_CUSTOM_ID_MAX_LENGTH = 64 @@ -84,7 +85,7 @@ class RoutingDecision: class RoutingPolicy: """Decides sync vs batch from hints + KV-configured thresholds. - Config (category ``routing``): + Config (category ``routing_config``): * ``batch_enabled`` (bool, default ``True``) — master switch. When off, everything runs sync. @@ -96,6 +97,12 @@ class RoutingPolicy: """ def __init__(self, config_store: Any) -> None: + # get_config_store() runs this at factory-build time, but a caller may + # construct and inject its own store directly here instead -- e.g. a + # real Postgres-backed store already carrying pre-existing legacy + # routing.* rows. Migrate at this actual consumption boundary so + # every RoutingPolicy, however constructed, sees migrated values. + migrate_legacy_categories(config_store) self._config = config_store def _batch_enabled(self) -> bool: @@ -474,8 +481,8 @@ def _assemble_payload(self, requests: List[BatchRequest]) -> str: return f"memory://{uuid.uuid4().hex}" @staticmethod - def _run(coro: Any) -> Any: - return asyncio.run(coro) + def _run_batch_coroutine(batch_coroutine: Any) -> Any: + return asyncio.run(batch_coroutine) def submit(self, requests: List[BatchRequest], metadata: Optional[Dict[str, Any]] = None) -> BatchJob: """Upload JSONL + create a batch job via the pg-llm-batch client.""" @@ -491,7 +498,7 @@ async def _submit() -> Dict[str, Any]: metadata=metadata, ) - job_payload = self._run(_submit()) + job_payload = self._run_batch_coroutine(_submit()) batch_id = job_payload["id"] # Tracked requests are stored as JSON primitives (not dataclass # instances) so the registry can be a JSON-backed Valkey mapping; @@ -514,7 +521,7 @@ def poll(self, job: BatchJob) -> Dict[str, Any]: async def _poll() -> Dict[str, Any]: return await self._client.get_batch_status(job.job_id, self._endpoint_alias) - status = self._run(_poll()) + status = self._run_batch_coroutine(_poll()) return { "job_id": job.job_id, "status": status.get("status"), @@ -532,7 +539,7 @@ def retrieve(self, job: BatchJob) -> List[BatchResultItem]: async def _download() -> Dict[str, Any]: return await self._client.download_results(job.job_id, self._endpoint_alias) - payload = self._run(_download()) + payload = self._run_batch_coroutine(_download()) if not payload.get("success"): reason = payload.get("reason") or payload.get("error") _LOGGER.warning( @@ -1218,8 +1225,8 @@ def _assemble_payload(self, requests: List[EmbeddingBatchRequest]) -> str: return f"memory://{uuid.uuid4().hex}" @staticmethod - def _run(coro: Any) -> Any: - return asyncio.run(coro) + def _run_batch_coroutine(batch_coroutine: Any) -> Any: + return asyncio.run(batch_coroutine) def submit( self, requests: List[EmbeddingBatchRequest], metadata: Optional[Dict[str, Any]] = None @@ -1248,7 +1255,7 @@ async def _submit() -> Dict[str, Any]: metadata=job_metadata, ) - job_payload = self._run(_submit()) + job_payload = self._run_batch_coroutine(_submit()) batch_id = job_payload["id"] self._jobs[batch_id] = { "endpoint_alias": self._endpoint_alias, @@ -1272,7 +1279,7 @@ def poll(self, job: BatchJob) -> Dict[str, Any]: async def _poll() -> Dict[str, Any]: return await self._client.get_batch_status(job.job_id, self._endpoint_alias) - status = self._run(_poll()) + status = self._run_batch_coroutine(_poll()) return { "job_id": job.job_id, "status": status.get("status"), @@ -1290,7 +1297,7 @@ def retrieve(self, job: BatchJob) -> List[EmbeddingBatchResultItem]: async def _download() -> Dict[str, Any]: return await self._client.download_results(job.job_id, self._endpoint_alias) - payload = self._run(_download()) + payload = self._run_batch_coroutine(_download()) if not payload.get("success"): reason = payload.get("reason") or payload.get("error") _LOGGER.warning( diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 751c9e366..2f9bd0fb5 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -42,7 +42,7 @@ ) from .batch_job_registry import ClaimNotAcquired, JobRegistryFactory, build_job_registry from .cost_ledger import CostLedger, PriceBook, PriceEntry -from .kv_config import InMemoryConfigStore +from .kv_config import InMemoryConfigStore, migrate_legacy_categories from .model_discovery import _currency_is_comparable from .token_counting import ( TokenCountUnavailable, @@ -55,7 +55,7 @@ "race_usage_context", default=None ) -_EMBEDDING_CONFIG_CATEGORY = "routing" +_EMBEDDING_CONFIG_CATEGORY = "routing_config" _DEFAULT_EMBEDDING_MAX_TOKENS_PER_REQUEST = 280_000 _DEFAULT_EMBEDDING_MAX_CHARS_PER_PART = 240_000 _DEFAULT_EMBEDDING_MAX_INPUTS_PER_REQUEST = 1 @@ -91,6 +91,13 @@ def __init__( ) -> None: self.orchestrator = orchestrator self.config = config_store or InMemoryConfigStore() + # Unconditional, not left to RoutingPolicy.__init__/build_job_registry: + # this coordinator's own _embedding_request_limits() reads + # self.config directly, and a caller supplying BOTH a custom + # routing_policy AND a custom job_registry would otherwise construct + # neither of those (each only built when its own kwarg is unset), + # leaving self.config never migrated at all for that combination. + migrate_legacy_categories(self.config) self.price_book = price_book or PriceBook(self.config) self.ledger = ledger or CostLedger(self.price_book) self._race_usage_context = _RACE_USAGE_CONTEXT diff --git a/contextual_orchestrator/kv_config.py b/contextual_orchestrator/kv_config.py index fbb7278ae..baa2564b5 100644 --- a/contextual_orchestrator/kv_config.py +++ b/contextual_orchestrator/kv_config.py @@ -14,16 +14,128 @@ The ``get(category, key, default)`` / ``set(category, key, value)`` shape is deliberately identical to ``pg_llm_batch.PostgresConfigStore`` so the two are -drop-in interchangeable. +drop-in interchangeable. Those generic parameter names are therefore retained +only at that external compatibility boundary; organization-owned internals use +bounded-context names. """ from __future__ import annotations from typing import Any, Dict, Iterable, Optional, Protocol, Tuple +# Sentinel distinguishing "no value stored" from any real stored value +# (including ``None``) when probing a category/key pair during migration. +_MISSING_CONFIG_VALUE = object() + +# Renamed KV categories that may already hold persisted rows under their old +# name on a Postgres-backed deployment (``pg_llm_batch.PostgresConfigStore`` +# keys each value at the literal ``f"{category}.{key}"`` string as a SQL +# PRIMARY KEY, so a bare category rename silently orphans any row already +# written under the old identity -- see ContextualWisdomLab/contextual-orchestrator#1017 +# review discussion). Each entry names the finite, explicitly enumerated set +# of keys this codebase has ever written under the old category; the +# ``ConfigStore`` protocol has no category-enumeration method, so unknown +# keys cannot be discovered automatically. Bounded removal condition: delete +# an entry once every deployment has booted at least once against the +# renamed category (tracked as a docs/product-technical-gap-baseline.md G-17 +# follow-up in ContextualWisdomLab/.github). +_LEGACY_CATEGORY_MIGRATIONS: Dict[str, Tuple[str, Tuple[str, ...]]] = { + "routing": ( + "routing_config", + ( + "batch_enabled", + "interactive_forces_sync", + "batch_min_tokens", + "embedding_max_tokens_per_request", + "embedding_max_chars_per_part", + "embedding_max_inputs_per_request", + "batch_job_retention_seconds", + ), + ), +} + + +def migrate_legacy_categories(config_store: "ConfigStore") -> None: + """Backfill renamed KV categories so already-persisted values survive. + + Idempotent and additive only: a value already present under the new + category name always wins and is never overwritten by this migration, + so re-running it on every boot (including every reconnect/reload) is + safe and produces one deterministic outcome regardless of call order. + Only ``get``/``set`` are used, matching the minimal ``ConfigStore`` + protocol every backend (in-memory, the Postgres adapter, and test + doubles) implements. + + Public (not the factory's private implementation detail) because a + ``ConfigStore`` reaches its real consumers (``RoutingPolicy``, + ``CostRoutingCoordinator``, ``build_job_registry``) two ways: built by + :func:`get_config_store`, or injected directly by a caller that + constructed its own store. Only the factory path ran this before; an + injected store carrying legacy ``routing.*`` rows never got migrated. + Call this at every real consumer's construction boundary, not only here. + + Concurrency note (Devin-review finding on #1017): the ``ConfigStore`` + protocol offers no conditional/compare-and-swap write, so a genuinely + concurrent operator update landing between this function's read of the + replacement key and its own ``set`` could, in principle, be clobbered + back to the stale legacy value. Re-checking the replacement key + immediately before the write (below) narrows that window to just the + gap between that re-check and the write itself, rather than the whole + legacy-value read in between -- it does not eliminate the race, since + doing that would need a real conditional-write primitive + (``INSERT ... ON CONFLICT DO NOTHING`` or equivalent) that neither this + protocol nor ``pg_llm_batch.PostgresConfigStore.set()`` (an + unconditional upsert) currently exposes; that is this migration's + owning repository's boundary to extend, not something to work around + here. In practice this migration runs once per process construction + (a boot-time compatibility shim, not a steady-state hot path), so the + realistic exposure is a multi-replica Postgres-backed deployment + restarting at the same moment an operator reconfigures the exact same + key -- narrow, but real; tracked as a residual item alongside gap G-17 + in ``ContextualWisdomLab/.github``'s + ``docs/product-technical-gap-baseline.md``. + """ + for legacy_category, (replacement_category, config_keys) in _LEGACY_CATEGORY_MIGRATIONS.items(): + for config_key in config_keys: + if ( + config_store.get( + replacement_category, + config_key, + _MISSING_CONFIG_VALUE, + ) + is not _MISSING_CONFIG_VALUE + ): + continue + legacy_config_value = config_store.get( + legacy_category, + config_key, + _MISSING_CONFIG_VALUE, + ) + if legacy_config_value is _MISSING_CONFIG_VALUE: + continue + if ( + config_store.get( + replacement_category, + config_key, + _MISSING_CONFIG_VALUE, + ) + is not _MISSING_CONFIG_VALUE + ): + continue + config_store.set( + replacement_category, + config_key, + legacy_config_value, + ) + class ConfigStore(Protocol): - """Minimal KV config contract shared by every backend.""" + """Minimal KV config contract shared by every backend. + + ``category``, ``key``, ``value`` and ``default`` intentionally mirror the + released ``pg_llm_batch.PostgresConfigStore`` protocol and are treated as + adapter-boundary vocabulary rather than organization-owned domain names. + """ def get(self, category: str, key: str, default: Any = None) -> Any: """Return the configured value or ``default`` when unset.""" @@ -43,44 +155,44 @@ class InMemoryConfigStore: """ def __init__(self, seed: Optional[Dict[str, Dict[str, Any]]] = None) -> None: - self._tree: Dict[str, Dict[str, Any]] = {} - self._secrets: Dict[str, str] = {} + self._config_tree: Dict[str, Dict[str, Any]] = {} + self._secret_values: Dict[str, str] = {} if seed: - for category, entries in seed.items(): - for key, value in entries.items(): - self.set(category, key, value) + for config_category, category_entries in seed.items(): + for config_key, config_value in category_entries.items(): + self.set(config_category, config_key, config_value) def get(self, category: str, key: str, default: Any = None) -> Any: """Return the value stored under ``category``/``key`` or ``default``.""" - return self._tree.get(category, {}).get(key, default) + return self._config_tree.get(category, {}).get(key, default) def set(self, category: str, key: str, value: Any) -> None: """Store ``value`` under ``category``/``key``.""" - self._tree.setdefault(category, {})[key] = value + self._config_tree.setdefault(category, {})[key] = value def get_category(self, category: str) -> Dict[str, Any]: """Return a copy of every key/value pair under ``category``.""" - return dict(self._tree.get(category, {})) + return dict(self._config_tree.get(category, {})) def show_config(self) -> Iterable[Tuple[str, str, Any]]: """Yield ``(category, key, value)`` for every configured entry.""" - for category, entries in sorted(self._tree.items()): - for key, value in sorted(entries.items()): - yield category, key, value + for config_category, category_entries in sorted(self._config_tree.items()): + for config_key, config_value in sorted(category_entries.items()): + yield config_category, config_key, config_value def set_secret(self, secret_name: str, secret_value: str) -> None: """Store a secret (kept apart from plain config, never surfaced by show_config).""" - self._secrets[secret_name] = secret_value + self._secret_values[secret_name] = secret_value def get_secret(self, secret_name: str, default: Any = None) -> Any: """Return a stored secret or ``default`` when absent.""" - return self._secrets.get(secret_name, default) + return self._secret_values.get(secret_name, default) def require_secret(self, secret_name: str) -> str: """Return a stored secret, raising when it is not configured.""" - if secret_name not in self._secrets: + if secret_name not in self._secret_values: raise KeyError(f"secret {secret_name!r} is not configured") - return self._secrets[secret_name] + return self._secret_values[secret_name] class PostgresConfigStoreAdapter: @@ -92,31 +204,31 @@ class PostgresConfigStoreAdapter: """ def __init__(self, config_store: Any, secret_store: Any = None) -> None: - self._config = config_store - self._secret = secret_store + self._postgres_config_store = config_store + self._postgres_secret_store = secret_store def get(self, category: str, key: str, default: Any = None) -> Any: """Delegate reads to the underlying ``PostgresConfigStore``.""" - return self._config.get(category, key, default) + return self._postgres_config_store.get(category, key, default) def set(self, category: str, key: str, value: Any) -> None: """Delegate writes to the underlying ``PostgresConfigStore``.""" - self._config.set(category, key, value) + self._postgres_config_store.set(category, key, value) def get_secret(self, secret_name: str, default: Any = None) -> Any: """Return a secret from the backing ``SecretStore`` when configured.""" - if self._secret is None: + if self._postgres_secret_store is None: return default try: - return self._secret.require_secret(secret_name) + return self._postgres_secret_store.require_secret(secret_name) except Exception: return default def require_secret(self, secret_name: str) -> str: """Return a required secret from the backing ``SecretStore``.""" - if self._secret is None: + if self._postgres_secret_store is None: raise KeyError(f"secret {secret_name!r} is not configured") - return self._secret.require_secret(secret_name) + return self._postgres_secret_store.require_secret(secret_name) def get_config_store( @@ -130,20 +242,32 @@ def get_config_store( With no DSN, an :class:`InMemoryConfigStore` is returned (the standalone / test default). With a DSN, the ``pg_llm_batch`` Postgres-backed stores are used when ``pg_llm_batch`` is importable; otherwise the call degrades to the - in-memory store so the orchestrator never hard-depends on Postgres. + in-memory store so the orchestrator never hard-depends on Postgres. Once a + Postgres-backed adapter has been constructed, however, compatibility + migration failures propagate so a transient persistence outage cannot + silently switch a live process to default/in-memory routing policy. """ if not postgres_dsn: - return InMemoryConfigStore(seed=seed) + in_memory_config_store = InMemoryConfigStore(seed=seed) + migrate_legacy_categories(in_memory_config_store) + return in_memory_config_store try: # pragma: no cover - exercised only with pg_llm_batch + Postgres present from pg_llm_batch import PostgresConfigStore, SecretStore # type: ignore - config_store = PostgresConfigStore(postgres_dsn) - secret_store = SecretStore(postgres_dsn, fernet_key=fernet_key) - adapter = PostgresConfigStoreAdapter(config_store, secret_store) - if seed: - for category, entries in seed.items(): - for key, value in entries.items(): - adapter.set(category, key, value) - return adapter - except Exception: # pragma: no cover - fall back when deps/DB unavailable - return InMemoryConfigStore(seed=seed) + postgres_config_store = PostgresConfigStore(postgres_dsn) + postgres_secret_store = SecretStore(postgres_dsn, fernet_key=fernet_key) + config_store_adapter = PostgresConfigStoreAdapter( + postgres_config_store, + postgres_secret_store, + ) + except Exception: # pragma: no cover - fall back when deps/DB cannot initialize + in_memory_config_store = InMemoryConfigStore(seed=seed) + migrate_legacy_categories(in_memory_config_store) + return in_memory_config_store + + if seed: + for config_category, category_entries in seed.items(): + for config_key, config_value in category_entries.items(): + config_store_adapter.set(config_category, config_key, config_value) + migrate_legacy_categories(config_store_adapter) + return config_store_adapter diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index b4ae2f135..670e1cdba 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -515,6 +515,7 @@ class DiscoveredModel: zdr_capable: bool = False evidence_only: bool = False spend_admitted: bool = True + image_generation_endpoint: str | None = None class ProviderDiscoveryError(RuntimeError): @@ -1395,6 +1396,18 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ else None ), privacy_policy_urls=_privacy_policy_urls(source, row), + # OpenRouter's image-generation REST path ("images") differs + # from the generic OpenAI-compatible "images/generations" + # convention TaskOrchestrator.proxy_capability defaults to. + # This mirrors the removed provider_name == "openrouter" + # hardcode 1:1 for every OpenRouter-sourced model (not + # gated on this row's own capabilities/modalities) so a + # discovered OpenRouter agent keeps working the moment it is + # ever proxied an "images/generations" capability request, + # exactly like the hardcode it replaces. + image_generation_endpoint=( + "images" if source.provider_name == "openrouter" else None + ), evidence_only=( source.provider_name == "opencode_go" and model_id not in _OPENCODE_GO_CHAT_MODELS @@ -1805,6 +1818,39 @@ def agent_id_for(discovered: DiscoveredModel) -> str: return f"{discovered.provider_name}_{_slug(discovered.model_id)}" +def encode_image_generation_endpoint_tag(value: str) -> str: + """Return a serving tag carrying ``value`` losslessly through the tag charset. + + The catalog store's generic serving-tag mechanism only persists tags + matching ``[a-z][a-z0-9_]*(?::[a-z0-9_]+)?`` (``provider_catalog_store. + _normalize_tags``): lowercase, underscore-separated words. An + ``image_generation_endpoint`` value is provider-declared free text (a + REST path such as OpenRouter's ``"images"`` today, but not guaranteed to + stay that simple) -- a slash, hyphen, or uppercase letter would otherwise + be silently case-folded or dropped by that filter on restore, corrupting + or losing the endpoint rather than raising. Hex-encoding the UTF-8 bytes + survives any string content while itself only ever producing lowercase + hex digits, so it always matches the tag charset. + """ + return f"image_generation_endpoint:{value.encode('utf-8').hex()}" + + +def decode_image_generation_endpoint_tag(tag: str) -> str | None: + """Return the endpoint value :func:`encode_image_generation_endpoint_tag` encoded. + + ``None`` for a tag whose payload is not valid hex/UTF-8 rather than + raising: a persisted store's tag rows are trusted evidence today, but + treating a malformed value as absent (fail closed) rather than crashing + restoration keeps one corrupted row from taking down every other model's + restore. + """ + encoded = tag.removeprefix("image_generation_endpoint:") + try: + return bytes.fromhex(encoded).decode("utf-8") + except ValueError: + return None + + def privacy_tags_for_discovered(discovered: DiscoveredModel) -> tuple[str, ...]: """Translate only explicit provider privacy evidence into agent tags.""" return ( @@ -1877,6 +1923,7 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> disabled=True, max_output_tokens=discovered.max_output_tokens, context_window=discovered.context_window, + image_generation_endpoint=discovered.image_generation_endpoint, ) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 9d168b3b3..01d910031 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -597,6 +597,17 @@ class ModelAgent: endpoint_equivalence: dict[str, Any] | None = None # Provider-declared support for the Chat Completions terminal usage frame. stream_usage_supported: bool = False + # Declared image-generation endpoint path override for this exact + # deployment, when it differs from the OpenAI-compatible default + # ("images/generations"). ``None`` means no override -- the request is + # sent to the same endpoint path as every other capability. Set this + # explicitly (via agent-pool config or discovery metadata) for a provider + # whose image API lives at a different path (e.g. OpenRouter serves image + # generation at "images"); never infer it from ``provider_name`` at + # request-routing time -- provider identity is a display/admin alias, not + # a routing condition (see docs/planning/adrs -- provider-group hardcoding + # is prohibited in selection/fallback/endpoint-routing decisions). + image_generation_endpoint: str | None = None def __post_init__(self) -> None: require_object_name(self.id, "agent.id") @@ -622,6 +633,10 @@ def __post_init__(self) -> None: raise TypeError("reasoning_effort_supported must be true, false, or null") if type(self.stream_usage_supported) is not bool: raise TypeError("stream_usage_supported must be a boolean") + if self.image_generation_endpoint is not None and ( + type(self.image_generation_endpoint) is not str or not self.image_generation_endpoint + ): + raise TypeError("image_generation_endpoint must be a non-empty string or null") if self.endpoint_equivalence is not None: contract = EndpointEquivalenceContract(**self.endpoint_equivalence) object.__setattr__(self, "endpoint_equivalence", dict(contract.__dict__)) @@ -647,6 +662,7 @@ def to_config(self) -> dict[str, Any]: "reasoning_effort_supported": self.reasoning_effort_supported, "endpoint_equivalence": self.endpoint_equivalence, "stream_usage_supported": self.stream_usage_supported, + "image_generation_endpoint": self.image_generation_endpoint, } @property @@ -682,6 +698,7 @@ def from_dict(cls, value: dict[str, Any]) -> "ModelAgent": # pragma: no cover reasoning_effort_supported=value.get("reasoning_effort_supported"), endpoint_equivalence=value.get("endpoint_equivalence"), stream_usage_supported=value.get("stream_usage_supported", False), + image_generation_endpoint=value.get("image_generation_endpoint"), ) @@ -1475,6 +1492,23 @@ def _log_provider_rejected_permanent(agent: ModelAgent, attempts: int, last_erro ) +def _capability_provider_endpoint(agent: ModelAgent, endpoint: str) -> str: + """Return the declared image-generation endpoint override when applicable. + + ``TaskOrchestrator.proxy_capability``'s immediate-race and sequential- + failover paths both compute this identical rewrite once their retry loop + ends up selecting an agent. It used to be duplicated verbatim in both + branches; extracting it here (mirroring ``_log_retry_outcome`` below, + added for the same reason) makes it impossible for the two call sites to + diverge if this condition is ever revised. + """ + return ( + agent.image_generation_endpoint + if agent.image_generation_endpoint and endpoint == "images/generations" + else endpoint + ) + + def _log_retry_outcome( agent: ModelAgent, attempt: int, @@ -3114,6 +3148,7 @@ class _AgentPoolStore: "context_window", "reasoning_effort_supported", "stream_usage_supported", + "image_generation_endpoint", } ) @@ -3153,7 +3188,10 @@ def _create_normalized_schema(cls, conn: sqlite3.Connection) -> None: context_window INTEGER, reasoning_effort_supported INTEGER, stream_usage_supported INTEGER NOT NULL DEFAULT 0, + image_generation_endpoint TEXT, CONSTRAINT agent_pool_disabled_flag_check CHECK (disabled IN (0, 1)), + CONSTRAINT agent_pool_image_generation_endpoint_check + CHECK (image_generation_endpoint IS NULL OR length(image_generation_endpoint) > 0), CONSTRAINT agent_pool_max_output_tokens_check CHECK ( max_output_tokens IS NULL @@ -3212,8 +3250,9 @@ def _insert_agent(cls, conn: sqlite3.Connection, agent: "ModelAgent") -> None: INSERT INTO agent_pool ( agent_id, model_name, base_url, api_key_env, credential_key, priority, disabled, provider_name, local_credential_key, auth_scheme, - max_output_tokens, context_window, reasoning_effort_supported, stream_usage_supported - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + max_output_tokens, context_window, reasoning_effort_supported, stream_usage_supported, + image_generation_endpoint + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( config["id"], @@ -3230,6 +3269,7 @@ def _insert_agent(cls, conn: sqlite3.Connection, agent: "ModelAgent") -> None: config["context_window"], config["reasoning_effort_supported"], int(config["stream_usage_supported"]), + config["image_generation_endpoint"], ), ) conn.executemany( @@ -3301,6 +3341,12 @@ def _initialize_schema(cls, conn: sqlite3.Connection) -> None: "CHECK (stream_usage_supported IN (0, 1))" ) columns.add("stream_usage_supported") + if "image_generation_endpoint" not in columns: + conn.execute( + "ALTER TABLE agent_pool ADD COLUMN image_generation_endpoint TEXT " + "CHECK (image_generation_endpoint IS NULL OR length(image_generation_endpoint) > 0)" + ) + columns.add("image_generation_endpoint") if not cls._AGENT_COLUMNS.issubset(columns): missing = ", ".join(sorted(cls._AGENT_COLUMNS - columns)) raise RuntimeError(f"unsupported agent_pool schema; missing columns: {missing}") @@ -3396,7 +3442,8 @@ def save(self, agent: "ModelAgent") -> None: priority = ?, disabled = ?, provider_name = ?, local_credential_key = ?, auth_scheme = ?, max_output_tokens = ?, context_window = ?, - reasoning_effort_supported = ?, stream_usage_supported = ? + reasoning_effort_supported = ?, stream_usage_supported = ?, + image_generation_endpoint = ? WHERE agent_id = ? """, ( @@ -3413,6 +3460,7 @@ def save(self, agent: "ModelAgent") -> None: config["context_window"], config["reasoning_effort_supported"], int(config["stream_usage_supported"]), + config["image_generation_endpoint"], agent.id, ), ) @@ -3509,7 +3557,8 @@ def load_all(self) -> list["ModelAgent"]: SELECT agent_id, model_name, base_url, api_key_env, credential_key, priority, disabled, provider_name, local_credential_key, auth_scheme, max_output_tokens, context_window, - reasoning_effort_supported, stream_usage_supported + reasoning_effort_supported, stream_usage_supported, + image_generation_endpoint FROM agent_pool ORDER BY agent_id """ ).fetchall() @@ -3575,6 +3624,7 @@ def load_all(self) -> list["ModelAgent"]: context_window=row[11], reasoning_effort_supported=(None if row[12] is None else bool(row[12])), stream_usage_supported=bool(row[13]), + image_generation_endpoint=row[14], group_name=group_by_agent.get(row[0], ""), endpoint_equivalence=contract_by_agent.get(row[0]), ) @@ -5989,6 +6039,10 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, patched = replace( patched, stream_usage_supported=patch["stream_usage_supported"] ) + if "image_generation_endpoint" in patch: + patched = replace( + patched, image_generation_endpoint=patch["image_generation_endpoint"] + ) updated_candidates = [patched if agent.id == worker_agent_id else agent for agent in self.candidates] updated_agents = [agent for agent in updated_candidates if not agent.disabled] @@ -7566,11 +7620,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: if key not in self._ORCHESTRATION_ONLY_KEYS } payload["model"] = agent.model - provider_endpoint = ( - "images" - if agent.provider_name == "openrouter" and endpoint == "images/generations" - else endpoint - ) + provider_endpoint = _capability_provider_endpoint(agent, endpoint) return ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary else self.client.proxy_send(agent, provider_endpoint, payload) @@ -7622,11 +7672,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: if key not in self._ORCHESTRATION_ONLY_KEYS } payload["model"] = agent.model - provider_endpoint = ( - "images" - if agent.provider_name == "openrouter" and endpoint == "images/generations" - else endpoint - ) + provider_endpoint = _capability_provider_endpoint(agent, endpoint) started_at = time.perf_counter() try: result = ( @@ -8371,6 +8417,7 @@ def _agent_to_admin_payload(self, agent: ModelAgent) -> dict[str, Any]: "max_output_tokens": agent.max_output_tokens, "context_window": agent.context_window, "stream_usage_supported": agent.stream_usage_supported, + "image_generation_endpoint": agent.image_generation_endpoint, "group_name": agent.group_name, "group_routing": self._group_router.member_report(agent.id) if agent.group_name else None, } diff --git a/contextual_orchestrator/provider_bootstrap.py b/contextual_orchestrator/provider_bootstrap.py index 5b9f9af2a..749a1a9ef 100644 --- a/contextual_orchestrator/provider_bootstrap.py +++ b/contextual_orchestrator/provider_bootstrap.py @@ -33,6 +33,7 @@ agent_from_discovered, agent_id_for, discover_all_models, + encode_image_generation_endpoint_tag, privacy_tags_for_discovered, is_routable_discovered_model, refresh_price_book, @@ -180,6 +181,9 @@ def is_chat_serving_candidate(model: DiscoveredModel) -> bool: return is_routable_discovered_model(model) +_IMAGE_GENERATION_ENDPOINT_TAG_PREFIX = "image_generation_endpoint:" + + def serving_tags_for_discovered(model: DiscoveredModel) -> tuple[str, ...]: """Return only provider-declared capabilities, modalities, and cost evidence.""" return tuple( @@ -189,10 +193,26 @@ def serving_tags_for_discovered(model: DiscoveredModel) -> tuple[str, ...]: *(("cost:free",) if model.is_free else ()), *(("spend:blocked",) if not model.spend_admitted else ()), *privacy_tags_for_discovered(model), - *model.capabilities, + # A provider-declared capability string is untrusted: + # dropping any that happens to collide with this module's + # own reserved image_generation_endpoint: prefix keeps a + # provider response from spoofing (or, more likely, + # coincidentally colliding with) the restored image-routing + # endpoint on the next catalog restore (CodeRabbit finding + # on #1017). + *( + value + for value in model.capabilities + if not value.casefold().startswith(_IMAGE_GENERATION_ENDPOINT_TAG_PREFIX) + ), *(f"capability:{value}" for value in model.capabilities), *(f"input:{value}" for value in model.input_modalities), *(f"output:{value}" for value in model.output_modalities), + *( + (encode_image_generation_endpoint_tag(model.image_generation_endpoint),) + if model.image_generation_endpoint + else () + ), ) ) ) diff --git a/contextual_orchestrator/provider_catalog_store.py b/contextual_orchestrator/provider_catalog_store.py index d9074c575..a0a98bef6 100644 --- a/contextual_orchestrator/provider_catalog_store.py +++ b/contextual_orchestrator/provider_catalog_store.py @@ -23,6 +23,7 @@ DiscoveredModel, ModelUnitPrice, ProviderModelSource, + decode_image_generation_endpoint_tag, ) if TYPE_CHECKING: @@ -372,6 +373,7 @@ def normalize_discovered_model( privacy_policy_urls=tuple(model.privacy_policy_urls), zdr_capable=bool(model.zdr_capable), spend_admitted=bool(model.spend_admitted), + image_generation_endpoint=model.image_generation_endpoint, ) @@ -412,6 +414,15 @@ def _restore_model_semantics( ), privacy_policy_urls=tuple(model.privacy_policy_urls), zdr_capable=bool(model.zdr_capable), + image_generation_endpoint=next( + ( + decoded + for tag in normalized + if tag.startswith("image_generation_endpoint:") + and (decoded := decode_image_generation_endpoint_tag(tag)) is not None + ), + None, + ), ) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index eb6a77519..25fff6265 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -283,7 +283,7 @@ def server_close(self) -> None: ALLOWED_AGENT_PATCH_KEYS = { "status", "priority", "tags", "provider_exclusions", "group_name", "endpoint_equivalence", "stream_usage_supported", "max_output_tokens", - "context_window", + "context_window", "image_generation_endpoint", } ALLOWED_AGENT_CREATE_KEYS = { "id", @@ -301,6 +301,7 @@ def server_close(self) -> None: "stream_usage_supported", "max_output_tokens", "context_window", + "image_generation_endpoint", } ALLOWED_MODEL_GROUP_KEYS = {"group_name", "member_agent_ids"} ALLOWED_MODEL_GROUP_PATCH_KEYS = {"member_agent_ids"} diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index 78b556392..19aa28fa6 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -155,6 +155,68 @@ def test_stream_usage_capability_patch_survives_restart() -> None: assert restored._agent(agent.id).stream_usage_supported is True +def test_image_generation_endpoint_override_survives_restart() -> None: + """A declared image-endpoint override, set via admin patch, is not lost on reload. + + Devin-review finding on ContextualWisdomLab/contextual-orchestrator#1017: + ``image_generation_endpoint`` was added to ``ModelAgent`` and ``patch_agent``'s + in-memory replace, but never reached the persisted ``agent_pool`` schema, its + INSERT/UPDATE statements, or the restart-time SELECT -- so a durable + (``agents_db``-backed) deployment would silently drop the override on the + very first restart, exactly the failure mode this class of test already + guards for every other patchable field. + """ + with tempfile.TemporaryDirectory() as directory: + db = os.path.join(directory, "pool.db") + agent = ModelAgent("persisted_agent", "model-x") + first = TaskOrchestrator([agent], agents_db=db) + + updated = first.patch_agent( + "default", "persisted_agent", {"image_generation_endpoint": "images"} + ) + assert updated["image_generation_endpoint"] == "images" + with sqlite3.connect(db) as connection: + assert connection.execute( + "SELECT image_generation_endpoint FROM agent_pool WHERE agent_id = ?", + (agent.id,), + ).fetchone() == ("images",) + + restored = TaskOrchestrator([agent], agents_db=db) + assert restored._agent(agent.id).image_generation_endpoint == "images" + + cleared = restored.patch_agent( + "default", "persisted_agent", {"image_generation_endpoint": None} + ) + assert cleared["image_generation_endpoint"] is None + reloaded_again = TaskOrchestrator([agent], agents_db=db) + assert reloaded_again._agent(agent.id).image_generation_endpoint is None + + +def test_add_agent_rejects_blank_image_generation_endpoint_without_mutation() -> None: + """An empty-string override is rejected upfront, not accepted then failing late. + + Devin-review finding: a non-string or empty ``image_generation_endpoint`` + loaded from JSON previously survived ``ModelAgent`` construction and would + only fail the first time an image request actually tried to use it. + ``__post_init__`` now validates it the same way ``auth_scheme`` is validated. + """ + with tempfile.TemporaryDirectory() as directory: + db = os.path.join(directory, "pool.db") + orchestrator = TaskOrchestrator(_seed(), agents_db=db) + with pytest.raises(TypeError, match="image_generation_endpoint must be"): + orchestrator.add_agent( + "default", + { + "id": "bad_endpoint_agent", + "model": "model-x", + "base_url": "https://api.openai.com/v1", + "credential_key": "OPENAI_API_KEY", + "image_generation_endpoint": "", + }, + ) + assert "bad_endpoint_agent" not in {a.id for a in orchestrator.agents} + + def test_agent_pool_persists_limit_metadata_across_restart() -> None: with tempfile.TemporaryDirectory() as directory: db = os.path.join(directory, "pool.db") @@ -502,12 +564,35 @@ def test_http_create_and_delete_worker_agents() -> None: ) assert status == 404 and wrong_pool_create["error"]["code"] == "agent_not_found" + # Devin-review finding on #1017: ALLOWED_AGENT_PATCH_KEYS/ + # ALLOWED_AGENT_CREATE_KEYS in server.py are a separate HTTP-layer + # validation allowlist from patch_agent/add_agent's own field + # handling in orchestrator.py -- adding the field to the latter + # without the former meant every real HTTP request setting + # image_generation_endpoint was rejected as an unknown field before + # patch_agent/add_agent ever saw it. + status, image_patched = _call( + f"{base}/general_agent", "PATCH", token, {"image_generation_endpoint": "images"} + ) + assert status == 200 and image_patched["image_generation_endpoint"] == "images" + + status, image_created = _call( + base, + "POST", + token, + {**NEW_AGENT, "id": "image_capable_agent", "image_generation_endpoint": "images"}, + ) + assert status == 201 and image_created["image_generation_endpoint"] == "images" + status, unknown = _call(base, "POST", token, {**NEW_AGENT, "id": "extra_agent", "surprise": 1}) assert status == 400 and unknown["error"]["code"] == "unknown_fields" status, removed = _call(f"{base}/coding_agent", "DELETE", token) assert status == 200 and removed["removed"] == "coding_agent" + status, removed_image_agent = _call(f"{base}/image_capable_agent", "DELETE", token) + assert status == 200 and removed_image_agent["removed"] == "image_capable_agent" + status, _ = _call(f"{base}/ghost_agent", "DELETE", token) assert status == 404 finally: diff --git a/tests/test_batch_embeddings.py b/tests/test_batch_embeddings.py index 153581df0..c43c8a761 100644 --- a/tests/test_batch_embeddings.py +++ b/tests/test_batch_embeddings.py @@ -570,8 +570,8 @@ def test_batch_embeddings_split_oversized_inputs_before_backend() -> None: ] orchestrator = TaskOrchestrator(agents) config = InMemoryConfigStore() - config.set("routing", "embedding_max_tokens_per_request", 4) - config.set("routing", "embedding_max_chars_per_part", 200) + config.set("routing_config", "embedding_max_tokens_per_request", 4) + config.set("routing_config", "embedding_max_chars_per_part", 200) price_book = PriceBook(config) price_book.set_price(PriceEntry("acme-provider", "text-embedding-test", 1.0, 0.0)) backend = _RecordingEmbeddingBackend() @@ -626,8 +626,8 @@ def test_batch_embeddings_char_guard_splits_no_whitespace_input() -> None: ] orchestrator = TaskOrchestrator(agents) config = InMemoryConfigStore() - config.set("routing", "embedding_max_tokens_per_request", 100) - config.set("routing", "embedding_max_chars_per_part", 5) + config.set("routing_config", "embedding_max_tokens_per_request", 100) + config.set("routing_config", "embedding_max_chars_per_part", 5) backend = _RecordingEmbeddingBackend() coordinator = CostRoutingCoordinator( orchestrator, diff --git a/tests/test_batch_routing.py b/tests/test_batch_routing.py index 2b84dd018..8436d8cbc 100644 --- a/tests/test_batch_routing.py +++ b/tests/test_batch_routing.py @@ -61,7 +61,7 @@ def test_bulk_priority_routes_batch_but_interactive_forces_sync() -> None: def test_batch_min_tokens_threshold_from_config() -> None: config = InMemoryConfigStore() - config.set("routing", "batch_min_tokens", 500) + config.set("routing_config", "batch_min_tokens", 500) policy = RoutingPolicy(config) assert policy.decide(RoutingHints(), prompt_tokens=200).channel == "sync" assert policy.decide(RoutingHints(), prompt_tokens=800).channel == "batch" @@ -69,7 +69,7 @@ def test_batch_min_tokens_threshold_from_config() -> None: def test_batch_token_threshold_stays_sync_when_prompt_count_unavailable() -> None: config = InMemoryConfigStore() - config.set("routing", "batch_min_tokens", 500) + config.set("routing_config", "batch_min_tokens", 500) decision = RoutingPolicy(config).decide(RoutingHints(), prompt_tokens=None) assert decision.channel == "sync" assert "unavailable" in decision.reason @@ -77,7 +77,7 @@ def test_batch_token_threshold_stays_sync_when_prompt_count_unavailable() -> Non def test_batch_disabled_config_forces_sync() -> None: config = InMemoryConfigStore() - config.set("routing", "batch_enabled", False) + config.set("routing_config", "batch_enabled", False) policy = RoutingPolicy(config) assert policy.decide(RoutingHints(latency_tolerant=True)).channel == "sync" diff --git a/tests/test_batch_routing_semantic_identifiers.py b/tests/test_batch_routing_semantic_identifiers.py new file mode 100644 index 000000000..eae4b520a --- /dev/null +++ b/tests/test_batch_routing_semantic_identifiers.py @@ -0,0 +1,17 @@ +"""Regression coverage for semantic identifiers in batch-routing internals.""" + +import inspect + +from contextual_orchestrator.batch_routing import ( + PgLlmBatchBackend, + PgLlmBatchEmbeddingBackend, +) + + +def test_pg_batch_backends_expose_semantic_coroutine_runner_names() -> None: + """Keep organization-owned async bridges specific to the batch context.""" + for batch_backend_type in (PgLlmBatchBackend, PgLlmBatchEmbeddingBackend): + assert hasattr(batch_backend_type, "_run_batch_coroutine") + assert not hasattr(batch_backend_type, "_run") + coroutine_runner = batch_backend_type._run_batch_coroutine + assert list(inspect.signature(coroutine_runner).parameters) == ["batch_coroutine"] diff --git a/tests/test_kv_config_store.py b/tests/test_kv_config_store.py index 2fb11e568..55f181433 100644 --- a/tests/test_kv_config_store.py +++ b/tests/test_kv_config_store.py @@ -8,10 +8,19 @@ import pytest +from contextual_orchestrator import ( + CostRoutingCoordinator, + ModelAgent, + RoutingPolicy, + TaskOrchestrator, +) +from contextual_orchestrator.batch_job_registry import JobRegistryFactory from contextual_orchestrator.kv_config import ( + _LEGACY_CATEGORY_MIGRATIONS, InMemoryConfigStore, PostgresConfigStoreAdapter, get_config_store, + migrate_legacy_categories, ) @@ -20,15 +29,18 @@ class _ConfigBackend: def __init__(self, postgres_dsn: str) -> None: self.postgres_dsn = postgres_dsn - self.values: dict[tuple[str, str], Any] = {} + self.config_values: dict[tuple[str, str], Any] = {} def get(self, category: str, key: str, default: Any = None) -> Any: - """Return one stored value or the caller default.""" - return self.values.get((category, key), default) + """Return one stored value or the caller default. + + The generic parameter names mirror the external pg_llm_batch protocol. + """ + return self.config_values.get((category, key), default) def set(self, category: str, key: str, value: Any) -> None: - """Record one delegated write.""" - self.values[(category, key)] = value + """Record one delegated write using the external adapter vocabulary.""" + self.config_values[(category, key)] = value class _SecretBackend: @@ -37,89 +49,92 @@ class _SecretBackend: def __init__(self, postgres_dsn: str, *, fernet_key: str | None = None) -> None: self.postgres_dsn = postgres_dsn self.fernet_key = fernet_key - self.values = {"OPENAI_API_KEY": "runtime-secret"} + self.secret_values = {"OPENAI_API_KEY": "runtime-secret"} def require_secret(self, secret_name: str) -> str: """Return one configured secret or raise the backend lookup error.""" - return self.values[secret_name] + return self.secret_values[secret_name] def test_in_memory_config_and_secrets_keep_their_boundaries() -> None: """Round-trip configuration while keeping secrets out of enumeration.""" assert InMemoryConfigStore().get("routing_policy", "missing_key", "fallback") == "fallback" - store = InMemoryConfigStore( + config_store = InMemoryConfigStore( {"routing_policy": {"quality_floor": 0.8}, "agent_pool": {"enabled": True}} ) - store.set("routing_policy", "quality_floor", 0.9) - store.set_secret("OPENAI_API_KEY", "runtime-secret") - - assert store.get("routing_policy", "quality_floor") == 0.9 - assert store.get("routing_policy", "missing_key", "fallback") == "fallback" - assert store.get_category("missing_category") == {} - category = store.get_category("routing_policy") - category["quality_floor"] = 0 - assert store.get("routing_policy", "quality_floor") == 0.9 - assert list(store.show_config()) == [ + config_store.set("routing_policy", "quality_floor", 0.9) + config_store.set_secret("OPENAI_API_KEY", "runtime-secret") + + assert config_store.get("routing_policy", "quality_floor") == 0.9 + assert config_store.get("routing_policy", "missing_key", "fallback") == "fallback" + assert config_store.get_category("missing_category") == {} + routing_config = config_store.get_category("routing_policy") + routing_config["quality_floor"] = 0 + assert config_store.get("routing_policy", "quality_floor") == 0.9 + assert list(config_store.show_config()) == [ ("agent_pool", "enabled", True), ("routing_policy", "quality_floor", 0.9), ] - assert "runtime-secret" not in repr(list(store.show_config())) - assert store.get_secret("OPENAI_API_KEY") == "runtime-secret" - assert store.get_secret("MISSING_API_KEY", "fallback") == "fallback" - assert store.require_secret("OPENAI_API_KEY") == "runtime-secret" + assert "runtime-secret" not in repr(list(config_store.show_config())) + assert config_store.get_secret("OPENAI_API_KEY") == "runtime-secret" + assert config_store.get_secret("MISSING_API_KEY", "fallback") == "fallback" + assert config_store.require_secret("OPENAI_API_KEY") == "runtime-secret" with pytest.raises(KeyError, match="MISSING_API_KEY"): - store.require_secret("MISSING_API_KEY") + config_store.require_secret("MISSING_API_KEY") def test_postgres_adapter_delegates_config_and_fails_closed_without_secrets() -> None: """Keep adapter configuration usable while absent secret storage stays closed.""" - config = _ConfigBackend("postgresql://example/config") - adapter = PostgresConfigStoreAdapter(config) - adapter.set("routing_policy", "quality_floor", 0.9) + config_backend = _ConfigBackend("postgresql://example/config") + config_store_adapter = PostgresConfigStoreAdapter(config_backend) + config_store_adapter.set("routing_policy", "quality_floor", 0.9) - assert adapter.get("routing_policy", "quality_floor") == 0.9 - assert adapter.get("routing_policy", "missing_key", "fallback") == "fallback" - assert adapter.get_secret("OPENAI_API_KEY", "fallback") == "fallback" + assert config_store_adapter.get("routing_policy", "quality_floor") == 0.9 + assert config_store_adapter.get("routing_policy", "missing_key", "fallback") == "fallback" + assert config_store_adapter.get_secret("OPENAI_API_KEY", "fallback") == "fallback" with pytest.raises(KeyError, match="OPENAI_API_KEY"): - adapter.require_secret("OPENAI_API_KEY") + config_store_adapter.require_secret("OPENAI_API_KEY") def test_postgres_adapter_returns_secret_and_hides_lookup_failure() -> None: """Return authorized secrets but expose only the supplied default on lookup failure.""" - secrets = _SecretBackend("postgresql://example/config") - adapter = PostgresConfigStoreAdapter(_ConfigBackend("postgresql://example/config"), secrets) + secret_backend = _SecretBackend("postgresql://example/config") + config_store_adapter = PostgresConfigStoreAdapter( + _ConfigBackend("postgresql://example/config"), + secret_backend, + ) - assert adapter.get_secret("OPENAI_API_KEY") == "runtime-secret" - assert adapter.require_secret("OPENAI_API_KEY") == "runtime-secret" - assert adapter.get_secret("MISSING_API_KEY", "fallback") == "fallback" + assert config_store_adapter.get_secret("OPENAI_API_KEY") == "runtime-secret" + assert config_store_adapter.require_secret("OPENAI_API_KEY") == "runtime-secret" + assert config_store_adapter.get_secret("MISSING_API_KEY", "fallback") == "fallback" def test_config_store_factory_uses_in_memory_backend_without_a_dsn() -> None: """Use the dependency-free backend when no database boundary is requested.""" - store = get_config_store(seed={"routing_policy": {"quality_floor": 0.8}}) + config_store = get_config_store(seed={"routing_policy": {"quality_floor": 0.8}}) - assert isinstance(store, InMemoryConfigStore) - assert store.get("routing_policy", "quality_floor") == 0.8 + assert isinstance(config_store, InMemoryConfigStore) + assert config_store.get("routing_policy", "quality_floor") == 0.8 def test_config_store_factory_builds_and_seeds_the_postgres_adapter(monkeypatch) -> None: """Pass the explicit DSN/key to pg_llm_batch and apply initial configuration.""" - module = types.ModuleType("pg_llm_batch") - module.PostgresConfigStore = _ConfigBackend - module.SecretStore = _SecretBackend - monkeypatch.setitem(sys.modules, "pg_llm_batch", module) + pg_llm_batch_module = types.ModuleType("pg_llm_batch") + pg_llm_batch_module.PostgresConfigStore = _ConfigBackend + pg_llm_batch_module.SecretStore = _SecretBackend + monkeypatch.setitem(sys.modules, "pg_llm_batch", pg_llm_batch_module) - store = get_config_store( + config_store = get_config_store( "postgresql://example/config", fernet_key="test-fernet-key", seed={"routing_policy": {"quality_floor": 0.8}}, ) - assert isinstance(store, PostgresConfigStoreAdapter) - assert store.get("routing_policy", "quality_floor") == 0.8 - assert store.get_secret("OPENAI_API_KEY") == "runtime-secret" - assert store._config.postgres_dsn == "postgresql://example/config" - assert store._secret.fernet_key == "test-fernet-key" + assert isinstance(config_store, PostgresConfigStoreAdapter) + assert config_store.get("routing_policy", "quality_floor") == 0.8 + assert config_store.get_secret("OPENAI_API_KEY") == "runtime-secret" + assert config_store._postgres_config_store.postgres_dsn == "postgresql://example/config" + assert config_store._postgres_secret_store.fernet_key == "test-fernet-key" def test_config_store_factory_falls_back_when_postgres_is_unavailable(monkeypatch) -> None: @@ -128,15 +143,306 @@ class _UnavailableConfigBackend: def __init__(self, _postgres_dsn: str) -> None: raise ConnectionError("database unavailable") - module = types.ModuleType("pg_llm_batch") - module.PostgresConfigStore = _UnavailableConfigBackend - module.SecretStore = _SecretBackend - monkeypatch.setitem(sys.modules, "pg_llm_batch", module) + pg_llm_batch_module = types.ModuleType("pg_llm_batch") + pg_llm_batch_module.PostgresConfigStore = _UnavailableConfigBackend + pg_llm_batch_module.SecretStore = _SecretBackend + monkeypatch.setitem(sys.modules, "pg_llm_batch", pg_llm_batch_module) - store = get_config_store( + config_store = get_config_store( "postgresql://example/config", seed={"routing_policy": {"quality_floor": 0.8}}, ) - assert isinstance(store, InMemoryConfigStore) - assert store.get("routing_policy", "quality_floor") == 0.8 + assert isinstance(config_store, InMemoryConfigStore) + assert config_store.get("routing_policy", "quality_floor") == 0.8 + + +def test_config_store_factory_propagates_live_postgres_migration_failure(monkeypatch) -> None: + """Fail closed when a constructed Postgres store cannot read migration state.""" + class _UnreadableConfigBackend(_ConfigBackend): + def get(self, category: str, key: str, default: Any = None) -> Any: + """Represent a live backend whose reads fail during compatibility migration.""" + raise ConnectionError("database read failed") + + pg_llm_batch_module = types.ModuleType("pg_llm_batch") + pg_llm_batch_module.PostgresConfigStore = _UnreadableConfigBackend + pg_llm_batch_module.SecretStore = _SecretBackend + monkeypatch.setitem(sys.modules, "pg_llm_batch", pg_llm_batch_module) + + with pytest.raises(ConnectionError, match="database read failed"): + get_config_store("postgresql://example/config") + + +def _postgres_backed_config_store(monkeypatch, config_backend: "_ConfigBackend"): + """Return a ``get_config_store`` result backed by the given live-DSN double. + + ``config_backend`` stands in for the persisted + ``pg_llm_batch.PostgresConfigStore`` surface (``get``/``set`` over + ``com_config``, keyed by ``category.key`` exactly as the real Postgres + table's primary key is built), so a value written into it before this call + simulates a row a previous, unmigrated deployment already persisted to + production Postgres. + """ + pg_llm_batch_module = types.ModuleType("pg_llm_batch") + pg_llm_batch_module.PostgresConfigStore = lambda postgres_dsn: config_backend + pg_llm_batch_module.SecretStore = _SecretBackend + monkeypatch.setitem(sys.modules, "pg_llm_batch", pg_llm_batch_module) + return get_config_store("postgresql://example/config") + + +def test_renamed_category_backfills_every_key_already_persisted_under_the_old_name( + monkeypatch, +) -> None: + """A pre-existing Postgres row under a renamed category is not orphaned. + + RED without the ContextualWisdomLab/contextual-orchestrator#1017 review + fix: renaming ``_ROUTING_CATEGORY``/``_EMBEDDING_CONFIG_CATEGORY`` from + ``"routing"`` to ``"routing_config"`` at the call sites alone leaves any + row a prior deployment already wrote under ``routing.`` invisible to + readers that now ask for ``routing_config.`` -- ``PostgresConfigStore`` + keys ``com_config`` by the literal ``f"{category}.{key}"`` string, so the + lookup is an exact miss and every caller silently falls back to its + hardcoded Python default instead of the operator's persisted value. + """ + legacy_category, (replacement_category, config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + config_backend = _ConfigBackend("postgresql://example/config") + for config_index, config_key in enumerate(config_keys): + config_backend.set(legacy_category, config_key, config_index) + + config_store = _postgres_backed_config_store(monkeypatch, config_backend) + + for config_index, config_key in enumerate(config_keys): + assert config_store.get(replacement_category, config_key) == config_index + + +class _InterleavedWriteConfigBackend: + """Deterministically simulates an operator write landing mid-migration. + + Devin-review finding on #1017: ``migrate_legacy_categories`` reads the + replacement key (absent), reads the legacy key, then writes the + replacement key -- with no conditional/compare-and-swap primitive + available, a real concurrent operator write to the replacement key + between the first read and the final write would be silently + overwritten by the stale legacy value. This double makes that + interleaving deterministic instead of timing-dependent: the first read + of the replacement key reports it absent (as a genuinely fresh key + would), and every read after that reports the value an operator is + modeled as having written in the interim. + """ + + def __init__(self, legacy_value: object, interleaved_value: object) -> None: + self._legacy_value = legacy_value + self._interleaved_value = interleaved_value + self._replacement_reads = 0 + + def get(self, category: str, key: str, default: Any = None) -> Any: + """Report the replacement key as absent once, then interleaved-written.""" + legacy_category, (replacement_category, _keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + if category == legacy_category: + return self._legacy_value + if category == replacement_category: + self._replacement_reads += 1 + return default if self._replacement_reads == 1 else self._interleaved_value + return default # pragma: no cover - no other category is read here + + def set(self, category: str, key: str, value: Any) -> None: + """Record every write so the test can assert none clobbers the interleaved value.""" + self.last_set = (category, key, value) + + +def test_migration_re_checks_immediately_before_writing_to_narrow_the_race( +) -> None: + """A write landing between the absence-check and the legacy-value read survives. + + Proves the re-check added directly above the ``set`` call actually does + something: without it, this exact interleaving would silently overwrite + the operator's real value with the stale legacy one. + """ + _legacy_category, (_replacement_category, config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + config_key = config_keys[0] + backend = _InterleavedWriteConfigBackend( + legacy_value="stale_legacy_value", interleaved_value="operator_write_mid_migration" + ) + + migrate_legacy_categories(backend) + + assert not hasattr(backend, "last_set") + + +def test_renamed_category_migration_never_overwrites_an_explicit_new_value( + monkeypatch, +) -> None: + """An operator's explicit new-category value always wins over legacy backfill.""" + legacy_category, (replacement_category, config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + config_key = config_keys[0] + config_backend = _ConfigBackend("postgresql://example/config") + config_backend.set(legacy_category, config_key, "stale_legacy_value") + config_backend.set(replacement_category, config_key, "explicit_new_value") + + config_store = _postgres_backed_config_store(monkeypatch, config_backend) + + assert config_store.get(replacement_category, config_key) == "explicit_new_value" + + +def test_renamed_category_migration_is_idempotent_across_reconnects(monkeypatch) -> None: + """Re-running the migration on every reconnect/restart is stable, not clobbering. + + Simulates two process restarts against the same persisted backend: the + first restart backfills the legacy value, an operator then explicitly + reconfigures the new key, and a second restart must retain that explicit + reconfiguration rather than re-copying the now-stale legacy row forward. + """ + legacy_category, (replacement_category, config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + config_key = config_keys[0] + config_backend = _ConfigBackend("postgresql://example/config") + config_backend.set(legacy_category, config_key, "stale_legacy_value") + + first_boot_store = _postgres_backed_config_store(monkeypatch, config_backend) + assert first_boot_store.get(replacement_category, config_key) == "stale_legacy_value" + + config_backend.set(replacement_category, config_key, "operator_reconfigured_value") + + second_boot_store = _postgres_backed_config_store(monkeypatch, config_backend) + assert ( + second_boot_store.get(replacement_category, config_key) + == "operator_reconfigured_value" + ) + assert config_backend.get(legacy_category, config_key) == "stale_legacy_value" + + +def test_renamed_category_migration_also_backfills_the_in_memory_seed_path() -> None: + """The same migration applies without a DSN, for seeded standalone/test runs.""" + legacy_category, (replacement_category, config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + config_key = config_keys[0] + + config_store = get_config_store( + seed={legacy_category: {config_key: "seeded_legacy_value"}} + ) + + assert isinstance(config_store, InMemoryConfigStore) + assert config_store.get(replacement_category, config_key) == "seeded_legacy_value" + + +def test_cost_routing_coordinator_migrates_a_directly_injected_store() -> None: + """A caller-constructed store bypassing get_config_store() is still migrated. + + Devin-review finding on #1017: only the ``get_config_store()`` factory ran + the legacy-category backfill, so a caller building its own store (e.g. a + real Postgres-backed one already carrying persisted ``routing.`` rows) + and injecting it straight into ``CostRoutingCoordinator`` never got it + migrated -- ``RoutingPolicy`` and ``build_job_registry`` would then silently + fall back to their hardcoded defaults for that legacy data. Complements + ``tests/test_routing_config_compatibility.py``'s focused + ``RoutingPolicy``-level unit test with a full ``CostRoutingCoordinator`` + integration check, since that is the real production entry point an + injected store actually arrives through. + """ + legacy_category, (replacement_category, _config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + injected_config_store = InMemoryConfigStore( + seed={legacy_category: {"batch_enabled": False}} + ) + assert injected_config_store.get(replacement_category, "batch_enabled", "unset") == "unset" + + orchestrator = TaskOrchestrator( + [ModelAgent("mock_worker", "mock-model", base_url="mock://a")] + ) + coordinator = CostRoutingCoordinator(orchestrator, injected_config_store) + + assert coordinator.config.get(replacement_category, "batch_enabled") is False + assert coordinator.policy._batch_enabled() is False + + +def test_cost_routing_coordinator_migrates_for_a_caller_supplied_routing_policy() -> None: + """A pre-built RoutingPolicy bypasses RoutingPolicy.__init__'s own migration. + + Devin-review follow-up finding on #1017: migrate_legacy_categories only + ran inside RoutingPolicy.__init__, so a caller supplying its own + already-constructed routing_policy (built from a store never passed + through that constructor) skipped it entirely -- and, since + CostRoutingCoordinator.__init__ only constructs RoutingPolicy when + routing_policy is None, build_job_registry(self.config) was the sole + remaining touch point actually guaranteed to run against self.config in + that case. Migrating there closes the gap for both the job registry's + own batch_job_retention_seconds read and any later embedding-category + read against the same shared self.config object. + """ + legacy_category, (replacement_category, _config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + injected_config_store = InMemoryConfigStore( + seed={legacy_category: {"batch_job_retention_seconds": 3600}} + ) + assert ( + injected_config_store.get(replacement_category, "batch_job_retention_seconds", "unset") + == "unset" + ) + prebuilt_policy = RoutingPolicy(InMemoryConfigStore()) # built from an unrelated store + + orchestrator = TaskOrchestrator( + [ModelAgent("mock_worker", "mock-model", base_url="mock://a")] + ) + coordinator = CostRoutingCoordinator( + orchestrator, injected_config_store, routing_policy=prebuilt_policy + ) + + assert coordinator.policy is prebuilt_policy + assert ( + coordinator.config.get(replacement_category, "batch_job_retention_seconds") == 3600 + ) + + +def test_cost_routing_coordinator_migrates_with_both_custom_policy_and_registry() -> None: + """Neither RoutingPolicy nor build_job_registry runs when both are custom-supplied. + + CodeRabbit/Devin-review follow-up finding on #1017: RoutingPolicy is only + constructed (and only then migrates) when routing_policy is None; + build_job_registry is only called (and only then migrates) when + job_registry is None. A caller supplying BOTH a custom routing_policy AND + a custom job_registry therefore triggered neither migration path, leaving + self.config unmigrated for CostRoutingCoordinator's own + _embedding_request_limits(), which reads self.config directly rather than + through either of those objects. + """ + legacy_category, (replacement_category, _config_keys) = next( + iter(_LEGACY_CATEGORY_MIGRATIONS.items()) + ) + injected_config_store = InMemoryConfigStore( + seed={legacy_category: {"embedding_max_tokens_per_request": 12345}} + ) + assert ( + injected_config_store.get( + replacement_category, "embedding_max_tokens_per_request", "unset" + ) + == "unset" + ) + prebuilt_policy = RoutingPolicy(InMemoryConfigStore()) # unrelated store + prebuilt_registry = JobRegistryFactory(None) # never touches self.config at all + + orchestrator = TaskOrchestrator( + [ModelAgent("mock_worker", "mock-model", base_url="mock://a")] + ) + coordinator = CostRoutingCoordinator( + orchestrator, + injected_config_store, + routing_policy=prebuilt_policy, + job_registry=prebuilt_registry, + ) + + assert coordinator.policy is prebuilt_policy + assert coordinator.job_registry is prebuilt_registry + max_tokens, _max_chars, _max_inputs = coordinator._embedding_request_limits() + assert max_tokens == 12345 diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 1cd2b5e2d..6a9a7332e 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -49,8 +49,10 @@ agent_from_discovered, agent_id_for, apply_openrouter_spend_admission, + decode_image_generation_endpoint_tag, discover_all_models, discover_provider_models, + encode_image_generation_endpoint_tag, free_discovered_models, general_free_serving_candidates, is_routable_discovered_model, @@ -2535,6 +2537,76 @@ def test_agent_from_discovered_preserves_explicit_capabilities() -> None: ) +def test_agent_from_discovered_threads_image_generation_endpoint() -> None: + """Discovery-carried image endpoint capability reaches the built agent. + + Regression test for a gap Devin Review found on + ContextualWisdomLab/contextual-orchestrator#1017: removing the + ``agent.provider_name == "openrouter"`` hardcode from + ``TaskOrchestrator.proxy_capability`` in favor of the declarative + ``ModelAgent.image_generation_endpoint`` field silently broke every + auto-discovered OpenRouter agent's image-generation routing, because + ``agent_from_discovered`` never populated the new field from discovery + evidence. + """ + discovered = DiscoveredModel( + provider_name="openrouter", + model_id="provider/image-model", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + capabilities=("image",), + image_generation_endpoint="images", + ) + + assert agent_from_discovered(discovered).image_generation_endpoint == "images" + + +def test_openrouter_discovery_sets_images_generation_endpoint() -> None: + """``_parse_openai_compatible`` sets OpenRouter's image endpoint capability. + + Mirrors the removed ``provider_name == "openrouter"`` hardcode 1:1 for + every OpenRouter-sourced row -- unconditional on that row's own + capabilities/modalities, exactly like the hardcode it replaces -- so a + freshly discovered OpenRouter agent keeps working the moment it is ever + proxied an ``images/generations`` capability request. + """ + discovered = _parse_openai_compatible( + {"data": [{"id": "provider/chat-model"}]}, + OPENROUTER_SOURCE, + ) + + assert discovered[0].image_generation_endpoint == "images" + + +def test_non_openrouter_discovery_leaves_image_generation_endpoint_unset() -> None: + discovered = _parse_openai_compatible( + {"data": [{"id": "gpt-test"}]}, + OPENAI_SOURCE, + ) + + assert discovered[0].image_generation_endpoint is None + + +def test_image_generation_endpoint_tag_round_trips_arbitrary_content() -> None: + """Hex encoding survives characters the tag store's own charset rejects. + + Regression test for a Devin Review follow-up on + ContextualWisdomLab/contextual-orchestrator#1017: a naive + ``image_generation_endpoint:`` tag would be silently case-folded + or dropped by the catalog store's ``[a-z][a-z0-9_]*(?::[a-z0-9_]+)?`` + tag-validation regex for any value containing a slash, hyphen, or + uppercase letter. + """ + for value in ("images", "v1/Images-Edits", "Images", "a/b-c_d"): + tag = encode_image_generation_endpoint_tag(value) + assert decode_image_generation_endpoint_tag(tag) == value + + +def test_decode_image_generation_endpoint_tag_rejects_malformed_payload() -> None: + assert decode_image_generation_endpoint_tag("image_generation_endpoint:not_hex!") is None + + def test_agent_from_discovered_preserves_explicit_privacy_evidence() -> None: """Every persistence path receives the same provider-declared privacy tags.""" discovered = DiscoveredModel( diff --git a/tests/test_multimodal_model_group_http.py b/tests/test_multimodal_model_group_http.py index ac7ffb970..8c676f753 100644 --- a/tests/test_multimodal_model_group_http.py +++ b/tests/test_multimodal_model_group_http.py @@ -418,13 +418,17 @@ def test_expired_provider_video_job_stops_polling_with_404() -> None: server.shutdown() -def test_openrouter_image_alias_uses_its_dedicated_images_endpoint() -> None: +def test_declared_image_generation_endpoint_override_is_used() -> None: + """The image endpoint is chosen from a declared capability field, not a + hardcoded provider-name comparison (provider names are display aliases, + never routing conditions).""" agent = ModelAgent( "image_member", "provider/image", tags=("image",), group_name="image_group", provider_name="openrouter", + image_generation_endpoint="images", ) orchestrator = TaskOrchestrator([agent]) observed: list[tuple[str, dict]] = [] @@ -443,6 +447,36 @@ def send(_agent: ModelAgent, endpoint: str, payload: dict) -> dict: assert observed == [("images", {"model": "provider/image", "prompt": "diagram"})] +def test_provider_name_alone_never_rewrites_the_image_endpoint() -> None: + """An agent named "openrouter" with no declared image_generation_endpoint + must be routed to the caller's requested endpoint unchanged: provider + identity alone must never drive endpoint selection.""" + agent = ModelAgent( + "image_member", + "provider/image", + tags=("image",), + group_name="image_group", + provider_name="openrouter", + ) + orchestrator = TaskOrchestrator([agent]) + observed: list[tuple[str, dict]] = [] + + def send(_agent: ModelAgent, endpoint: str, payload: dict) -> dict: + observed.append((endpoint, payload)) + return {"model": payload["model"], "data": []} + + orchestrator.client.proxy_send = send # type: ignore[method-assign] + orchestrator.proxy_capability( + {"model": "image-group", "prompt": "diagram", "routing": {"channel": "sync"}}, + capability="image", + endpoint="images/generations", + ) + + assert observed == [ + ("images/generations", {"model": "provider/image", "prompt": "diagram"}) + ] + + def test_capability_request_size_exhaustion_preserves_413_without_penalty() -> None: """Oversized capability requests do not degrade provider health.""" agents = [ diff --git a/tests/test_provider_catalog_store.py b/tests/test_provider_catalog_store.py index 7cfc24efc..35b985243 100644 --- a/tests/test_provider_catalog_store.py +++ b/tests/test_provider_catalog_store.py @@ -16,6 +16,7 @@ _parse_openai_compatible, ) from contextual_orchestrator.privacy_policy_analysis import PrivacyPolicyAssessment +from contextual_orchestrator.provider_bootstrap import serving_tags_for_discovered from contextual_orchestrator.provider_catalog_store import ( PROVIDER_CATALOG_SCHEMA_SQL, InMemoryProviderCatalogStore, @@ -303,6 +304,118 @@ def test_last_known_good_restores_free_and_modality_evidence() -> None: assert store.serving_models(source) == [model] +def test_last_known_good_restores_image_generation_endpoint() -> None: + """A catalog restart round trip cannot silently drop declared image routing. + + Regression test for a gap Devin Review found on + ContextualWisdomLab/contextual-orchestrator#1017: an in-flight discovery + correctly threads ``DiscoveredModel.image_generation_endpoint`` into the + agent it builds, but the catalog persistence boundary + (``normalize_discovered_model``/``_restore_model_semantics``, reached via + ``record_success``/``serving_models``) reconstructed ``DiscoveredModel`` + without that field, so a durable refresh-then-restart round trip through + the catalog store silently lost an OpenRouter model's image-generation + routing. Fixed by round-tripping the field through the store's existing + generic serving-tag mechanism (``serving_tags_for_discovered`` emits + ``image_generation_endpoint:``; ``_restore_model_semantics`` parses + it back out), the same pattern already used for capabilities/modalities -- + no schema change needed. + """ + source = _source(provider="openrouter", credential="OPENROUTER_API_KEY") + model = replace( + _model(source, "provider/image-model", 0), + capabilities=("image",), + currency_code="USD", + is_free=True, + image_generation_endpoint="images", + ) + store = InMemoryProviderCatalogStore() + store.record_success( + source, + [model], + eligible_model_ids={model.model_id}, + serving_tags={model.model_id: serving_tags_for_discovered(model)}, + ) + + assert store.serving_models(source) == [model] + + +def test_image_generation_endpoint_survives_tag_charset_restrictions() -> None: + """A slash/hyphen/uppercase endpoint round-trips through the tag store intact. + + Regression test for a Devin Review follow-up on + ContextualWisdomLab/contextual-orchestrator#1017: the catalog store's + generic serving tags only persist ``[a-z][a-z0-9_]*(?::[a-z0-9_]+)?`` + (``_normalize_tags``), so a naive ``image_generation_endpoint:`` + tag would have this store's own tag validation silently case-fold or + drop any endpoint containing a slash, hyphen, or uppercase letter on + restore -- corrupting or losing it rather than raising. Hex-encoding the + value (``encode_image_generation_endpoint_tag``/ + ``decode_image_generation_endpoint_tag``) keeps it byte-for-byte intact. + """ + source = _source(provider="openrouter", credential="OPENROUTER_API_KEY") + model = replace( + _model(source, "provider/image-model", 0), + capabilities=("image",), + currency_code="USD", + is_free=True, + image_generation_endpoint="v1/Images-Edits", + ) + store = InMemoryProviderCatalogStore() + store.record_success( + source, + [model], + eligible_model_ids={model.model_id}, + serving_tags={model.model_id: serving_tags_for_discovered(model)}, + ) + + assert store.serving_models(source) == [model] + + +def test_colliding_capability_cannot_spoof_the_image_generation_endpoint() -> None: + """A provider-declared capability cannot masquerade as the reserved endpoint tag. + + Regression test for a CodeRabbit finding on + ContextualWisdomLab/contextual-orchestrator#1017: ``model.capabilities`` + values are stored as raw, unprefixed serving tags alongside the reserved + ``image_generation_endpoint:`` tag this module emits. A provider + response is untrusted evidence; if it ever declared a capability string + that happened to start with ``"image_generation_endpoint:"`` (whether + adversarial or merely coincidental), the restore path would read that + raw capability as the real tag and route every image request to + whatever endpoint it encoded -- silently, after a restart, with no + endpoint ever legitimately configured. ``serving_tags_for_discovered`` + drops any capability colliding with the reserved prefix before it is + persisted, so restoration is untouched by it either way. + """ + source = _source(provider="openrouter", credential="OPENROUTER_API_KEY") + spoofed_capability = "image_generation_endpoint:" + "foo".encode("utf-8").hex() + model = replace( + _model(source, "provider/spoofed-model", 0), + capabilities=("chat", spoofed_capability), + currency_code="USD", + is_free=True, + image_generation_endpoint=None, + ) + + tags = serving_tags_for_discovered(model) + + # The reserved bare tag is never emitted for this capability -- it may + # still appear namespaced as "capability:" (ordinary, harmless + # capability round-tripping), just never as the unprefixed collision + # the restore path treats as the real endpoint. + assert spoofed_capability not in tags + store = InMemoryProviderCatalogStore() + store.record_success( + source, + [model], + eligible_model_ids={model.model_id}, + serving_tags={model.model_id: tags}, + ) + restored = store.serving_models(source)[0] + assert restored.image_generation_endpoint is None + + def test_successful_refresh_retains_known_limits_when_metadata_is_unknown() -> None: source = _source() known = replace( diff --git a/tests/test_provider_embedding_batch_backend.py b/tests/test_provider_embedding_batch_backend.py index 0eb661fbb..998ceee08 100644 --- a/tests/test_provider_embedding_batch_backend.py +++ b/tests/test_provider_embedding_batch_backend.py @@ -101,7 +101,7 @@ def test_unknown_tokenizer_fails_before_provider_when_byte_bound_exceeds_budget( AssertionError("provider must not be called") ) config = InMemoryConfigStore() - config.set("routing", "embedding_max_tokens_per_request", 3) + config.set("routing_config", "embedding_max_tokens_per_request", 3) coordinator = CostRoutingCoordinator( TaskOrchestrator([agent], client=client), config, @@ -445,8 +445,8 @@ def test_provider_embedding_requests_are_sharded_by_the_existing_token_limit() - TaskOrchestrator([agent], client=client), embedding_token_counter=_SyntheticExactCounter(), ) - coordinator.config.set("routing", "embedding_max_tokens_per_request", 3) - coordinator.config.set("routing", "embedding_max_inputs_per_request", 2) + coordinator.config.set("routing_config", "embedding_max_tokens_per_request", 3) + coordinator.config.set("routing_config", "embedding_max_inputs_per_request", 2) document = coordinator.complete_embeddings_batch( ["one two", "three four", "five"], diff --git a/tests/test_psychometric_routing.py b/tests/test_psychometric_routing.py index 4c7bc8991..7bc0c2d6f 100644 --- a/tests/test_psychometric_routing.py +++ b/tests/test_psychometric_routing.py @@ -2,16 +2,26 @@ from __future__ import annotations -import numpy as np from pathlib import Path +import pytest + from contextual_orchestrator import ModelAgent, TaskOrchestrator from contextual_orchestrator.psychometric_routing import PsychometricRoutingEvidence def test_fast_mlsirm_fit_uses_judge_acceptance_item_for_context_score(monkeypatch) -> None: - """Criterion IRT rows inform one MLSRM fit; acceptance remains the route score.""" - import fast_mlsirm + """Criterion IRT rows inform one MLSRM fit; acceptance remains the route score. + + ``numpy`` and ``fast_mlsirm`` are optional, lazily-imported dependencies of + ``psychometric_routing.py`` itself (absence means "no psychometric + evidence", not an error) -- this is the only test in the module that + exercises that real code path, so it is the only one scoped to skip when + either package is unavailable, rather than failing collection for the + whole file. + """ + np = pytest.importorskip("numpy") + fast_mlsirm = pytest.importorskip("fast_mlsirm") captured: dict[str, object] = {} diff --git a/tests/test_routing_config_compatibility.py b/tests/test_routing_config_compatibility.py new file mode 100644 index 000000000..9a3a78af2 --- /dev/null +++ b/tests/test_routing_config_compatibility.py @@ -0,0 +1,21 @@ +"""Compatibility coverage for the semantic routing-config KV category.""" + +from contextual_orchestrator.batch_routing import RoutingHints, RoutingPolicy +from contextual_orchestrator.kv_config import InMemoryConfigStore + + +def test_injected_store_migrates_legacy_routing_values_before_policy_reads() -> None: + """Directly injected stores retain persisted routing values after the rename. + + RED until the consumption boundary prepares arbitrary injected stores: + ``get_config_store()`` cannot help a caller that already constructed a + compatible store and passes it directly to ``RoutingPolicy``. + """ + config_store = InMemoryConfigStore( + {"routing": {"batch_enabled": False}} + ) + + routing_policy = RoutingPolicy(config_store) + + assert routing_policy.decide(RoutingHints(latency_tolerant=True)).channel == "sync" + assert config_store.get("routing_config", "batch_enabled") is False diff --git a/tests/test_spend_analytics.py b/tests/test_spend_analytics.py index 7ef56568d..3d6f900ef 100644 --- a/tests/test_spend_analytics.py +++ b/tests/test_spend_analytics.py @@ -4,6 +4,7 @@ import json import threading +from unittest.mock import patch import urllib.request from contextual_orchestrator import ModelAgent, TaskOrchestrator @@ -29,8 +30,17 @@ def _orchestrator(*, price: float | None = None) -> TaskOrchestrator: def test_exact_output_without_prompt_usage_is_explicitly_unavailable() -> None: - orchestrator = _orchestrator() - orchestrator.run([{"role": "user", "content": "account for this"}]) + """Output-only tokenizer usage leaves prompt/cost totals explicitly unavailable, not estimated.""" + # This test owns the raw-output tokenizer fallback contract, not the + # optional fast-mlsirm judge integration. Resolve that optional capability + # deterministically as unavailable so installing an extra package cannot + # change the evidence-source assertion from tokenizer to reported/mixed. + with patch( + "contextual_orchestrator.orchestrator._resolve_fast_mlsirm_components", + return_value=None, + ): + orchestrator = _orchestrator() + orchestrator.run([{"role": "user", "content": "account for this"}]) report = orchestrator.spend_analytics() row = report["by_model"][0] @@ -38,7 +48,7 @@ def test_exact_output_without_prompt_usage_is_explicitly_unavailable() -> None: assert report["totals"]["output_tokens"] > 0 assert report["totals"]["prompt_tokens"] is None assert report["totals"]["cost_usd"] is None - assert row["usage_source"] == "mixed" + assert row["usage_source"] == "tokenizer" assert row["cost_usd"] is None assert not any("estimated" in key for key in row | report["totals"])