diff --git a/CHANGELOG.md b/CHANGELOG.md index b9439d935..409b15238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,22 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- OpenRouter's `ProviderModelSource.evidence_only` is no longer a blanket + `True` set once at module load for every discovered OpenRouter model, + regardless of that model's own evidence (introduced without an ADR update + in `952996ec`, contradicting ADR 0032's "privacy discovery is + model-specific ... discovery does not turn either ambiguous state into + blanket non-support"). `_apply_discovered_model_evidence` now gates + `evidence_only`/`zdr_capable` for OpenRouter's own rows per model, using + the exact same authoritative `/api/v1/endpoints/zdr` feed match already + used to donate ZDR evidence to other providers' matching rows: a row + becomes routable only when its own model id is present on that feed. + Fail-closed is preserved — a model absent from the feed, or any feed + fetch/parse failure, still leaves the row `evidence_only=True` and + unroutable, exactly as before; only models with genuine per-model ZDR + evidence gain a change in outcome. `PROVIDER_MODEL_SOURCES`'s `openrouter` + entry now leaves `evidence_only` at the `False` default like every other + provider source. - Model discovery now treats every KV credential as an independent account/catalog boundary, removes provider-family collapsing, and offers secret-free `--verbose` progress diagnostics. Logical equivalence and latency-based switching remain explicit `model_group` decisions only. - Model-group evidence now reports peak observed RPM and provider-reported TPM over a real 60-second completion window without generating probe traffic or inferring missing usage. - `discover_provider_models`'s primary model-list fetch is now retried once diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index a4d28c819..3a7ae7a94 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -197,7 +197,10 @@ def configured_gateway_source( list_url="https://openrouter.ai/api/v1/models?output_modalities=all", chat_base_url="https://openrouter.ai/api/v1", capabilities=("chat",), - evidence_only=True, + # Not a blanket True: OpenRouter rows are gated per model, in + # _apply_discovered_model_evidence, against OpenRouter's own + # authoritative ZDR feed (ADR 0032 -- privacy discovery is + # model-specific, never inferred/defaulted at the provider level). ), ProviderModelSource( provider_name="opencode_zen", @@ -1097,22 +1100,29 @@ def _apply_discovered_model_evidence( a different upstream endpoint. Exact canonical ids are the only portable identity; suffix matching would transfer privacy evidence to an unrelated model that merely shares a display name. + + OpenRouter's own rows are gated by this exact same feed match, not a + provider-wide default (ADR 0032): a row becomes ``evidence_only=False`` + and ``zdr_capable=True`` only when its own model id is present in + ``zdr_model_ids``. Any other outcome for an OpenRouter row -- the model + genuinely absent from the feed, an empty feed, or a feed-fetch failure + (``zdr_model_ids`` empty) -- leaves it ``evidence_only=True`` and + ``zdr_capable=False``: fail closed, never a blanket default independent + of that model's own feed coverage. """ - if not zdr_model_ids: - return discovered exact_ids = {model_id.strip().casefold() for model_id in zdr_model_ids if model_id.strip()} def matches(model_id: str) -> bool: normalized = model_id.strip().casefold() - return normalized in exact_ids + return bool(exact_ids) and normalized in exact_ids - return [ - replace( - model, - zdr_capable=not model.evidence_only and matches(model.model_id), - ) - for model in discovered - ] + def apply(model: DiscoveredModel) -> DiscoveredModel: + if model.provider_name == "openrouter": + is_zdr_match = matches(model.model_id) + return replace(model, evidence_only=not is_zdr_match, zdr_capable=is_zdr_match) + return replace(model, zdr_capable=not model.evidence_only and matches(model.model_id)) + + return [apply(model) for model in discovered] def discover_provider_models( @@ -1279,9 +1289,12 @@ def discover_all_models( ) except ProviderDiscoveryError as exc: errors.append(exc) - # The OpenRouter catalog is evidence-only; its public ZDR endpoint supplies - # matching privacy evidence for discovered models from other providers. It - # is never selected as an inference upstream here. + # OpenRouter's public ZDR endpoint supplies matching privacy evidence both + # for its own rows and for discovered models from other providers that + # share a canonical model id. An OpenRouter row becomes a usable + # inference upstream only when this same feed attests to that specific + # model (see _apply_discovered_model_evidence); it is never blanket + # evidence-only or blanket eligible as a provider class. return _apply_discovered_model_evidence( _deduplicate_discovered_models(discovered), _openrouter_zdr_model_ids(timeout=timeout), diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e10aad199..a3ac20a9d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,46 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-08-31 OpenRouter per-model ZDR evidence: blanket `evidence_only=True` reconciled against ADR 0032 + +A 5-agent investigation confirmed `ProviderModelSource.evidence_only` was +hardcoded `True` for OpenRouter's provider source declaration only (every +other one of the six provider sources uses the `False` default), set once at +module load before any HTTP call, applying to every discovered OpenRouter +model unconditionally regardless of that specific model's own evidence. That +override (`952996ec`, a bare one-line commit message, no accompanying ADR +amendment) contradicts ADR 0032's binding design for this exact subsystem: +"Privacy discovery is also model-specific rather than inferred from price +... discovery does not turn either ambiguous state into blanket +non-support." Earlier entries in this file and ADR 0041 both repeated the +blanket claim ("OpenRouter is deliberately `evidence_only=True` ... so it +never serves inference") as a stated fact when it was current; that +statement is superseded by this fix and no longer describes current +behavior, though the historical entries and ADR 0041's own decision (not to +add a `models_dev_provider_id` join for OpenRouter, since OpenRouter already +reports its own real pricing) are otherwise unaffected. + +Per-model ZDR evidence for OpenRouter already existed and was actively +computed elsewhere in the same file (`_merge_openrouter_zdr_metadata` / +`_merge_openrouter_provider_privacy` fetch OpenRouter's authoritative +`/api/v1/endpoints/zdr` feed and per-endpoint provider privacy policy) but +was thrown away for OpenRouter's own rows — used exclusively to donate ZDR +status to other providers' rows sharing the same canonical model id. +`_apply_discovered_model_evidence` now gates `evidence_only`/`zdr_capable` +for OpenRouter's own rows through that identical per-model feed match: a row +becomes routable only when its own model id is present on the feed. A model +absent from the feed, or any feed fetch/parse failure, still leaves the row +`evidence_only=True` and unroutable (fail-closed unchanged). See +`CHANGELOG.md` and the PR for the exact diff and test coverage (including a +dedicated negative case for a genuinely non-attested OpenRouter model and a +feed-fetch-failure case). + +A companion, independent blanket filter in +`ContextualWisdomLab/.github`'s `scripts/ci/contextual_orchestrator_review_launcher.py` +(`_routable_discovered_models()`) strips every OpenRouter row before that +repo's own already-correct per-route ZDR mechanism (`zdr_policy.py`'s +`is_zdr_model()` / `openrouter_endpoints_feed`) ever evaluates them. That is +tracked and fixed separately in `.github`, not in this change. + ## 2026-08-30 provider-catalog-sync: no scheduled run has succeeded in 5 days over one provider; workflow check was too strict `provider-catalog-sync.yml` (run `33312773022`, job `99260685380`) failed with `credential diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 0b059fd39..4f6311a4b 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -43,6 +43,7 @@ discover_provider_models, free_discovered_models, general_free_serving_candidates, + is_routable_discovered_model, openrouter_paid_inference_available, refresh_price_book, select_cheapest_discovered_agent, @@ -1510,7 +1511,11 @@ def test_default_sources_request_openrouter_full_modality_catalog() -> None: assert sources["openai"].capabilities == () assert sources["openrouter"].capabilities == ("chat",) assert sources["openrouter"].list_url.endswith("?output_modalities=all") - assert sources["openrouter"].evidence_only is True + # Not a blanket provider-level default: OpenRouter rows are gated + # per model against OpenRouter's own ZDR feed in + # _apply_discovered_model_evidence (ADR 0032), same as every other + # provider source. + assert sources["openrouter"].evidence_only is False assert sources["opencode_zen"].list_url == "https://opencode.ai/zen/v1/models" assert sources["nvidia_nim"].capabilities == ("chat",) assert sources["nvidia_nim_sub"].capabilities == ("chat",) @@ -1622,10 +1627,15 @@ def urlopen(request, timeout=None): discovered, errors = discover_all_models((OPENROUTER_SOURCE, other_source)) assert errors == [] + # OpenRouter's own row is a genuine ZDR-feed match too: it gets to use + # its own evidence for its own serving eligibility, the same as the + # evidence it donates to nvidia_nim's matching row. assert [(model.provider_name, model.zdr_capable) for model in discovered] == [ - ("openrouter", False), + ("openrouter", True), ("nvidia_nim", True), ] + openrouter_model = next(m for m in discovered if m.provider_name == "openrouter") + assert openrouter_model.evidence_only is False def test_openrouter_zdr_evidence_uses_the_registered_kv_credential() -> None: @@ -1711,8 +1721,11 @@ def urlopen(request, timeout=None): discovered, errors = discover_all_models((OPENROUTER_SOURCE, other_source)) assert errors == [] + # OpenRouter's own "openai/shared-model" row is an exact feed match and + # correctly becomes zdr_capable; nvidia_nim's "shared-model" is only a + # suffix of that id and correctly stays unmatched. assert [(model.provider_name, model.zdr_capable) for model in discovered] == [ - ("openrouter", False), + ("openrouter", True), ("nvidia_nim", False), ] @@ -1751,12 +1764,86 @@ def urlopen(request, timeout=None): discovered, errors = discover_all_models((OPENROUTER_SOURCE, other_source)) assert errors == [] + # OpenRouter's own "openai/shared-model" row is still an exact, + # unambiguous feed match on its own id; nvidia_nim's bare "shared-model" + # remains rejected as an ambiguous suffix of two distinct feed entries. assert [(model.provider_name, model.zdr_capable) for model in discovered] == [ - ("openrouter", False), + ("openrouter", True), ("nvidia_nim", False), ] +def test_discover_all_models_openrouter_model_absent_from_zdr_feed_stays_evidence_only() -> None: + """A real negative: an OpenRouter model the feed does not attest to. + + Companion to test_discover_all_models_applies_model_zdr_evidence_to_other_sources + (the positive case). Fixing the blanket evidence_only=True override must + not turn every OpenRouter row into a serving agent -- only rows the feed + itself attests to. This model shares no id with the feed, so it must stay + evidence_only=True/zdr_capable=False and unroutable, exactly like before + the fix, even though the source itself is no longer blanket evidence_only. + """ + register_credential("OPENROUTER_API_KEY", "sk-openrouter") + + def urlopen(request, timeout=None): + return _Response({"data": [{"id": "openai/not-on-zdr-feed"}]}) + + with ( + patch( + "contextual_orchestrator.model_discovery.urllib.request.urlopen", + side_effect=urlopen, + ), + patch( + "contextual_orchestrator.model_discovery._fetch_json_same_host_https", + return_value={"data": [{"model_id": "openai/some-other-model"}]}, + ), + ): + discovered, errors = discover_all_models((OPENROUTER_SOURCE,)) + + assert errors == [] + assert len(discovered) == 1 + model = discovered[0] + assert model.provider_name == "openrouter" + assert model.evidence_only is True + assert model.zdr_capable is False + assert is_routable_discovered_model(model) is False + with pytest.raises(ValueError, match="evidence-only"): + agent_from_discovered(model) + + +def test_discover_all_models_openrouter_zdr_feed_failure_keeps_every_row_evidence_only() -> None: + """Fail closed: a total ZDR-feed fetch failure must not default rows open. + + If the feed cannot be read at all, no OpenRouter row has any evidence -- + positive or negative -- so every row must stay evidence_only=True and + unroutable, never fall back to the (now-default) False. + """ + register_credential("OPENROUTER_API_KEY", "sk-openrouter") + + def urlopen(request, timeout=None): + return _Response({"data": [{"id": "openai/would-have-matched"}]}) + + with ( + patch( + "contextual_orchestrator.model_discovery.urllib.request.urlopen", + side_effect=urlopen, + ), + patch( + "contextual_orchestrator.model_discovery._fetch_json_same_host_https", + side_effect=urllib.error.URLError("zdr feed unreachable"), + ), + ): + discovered, errors = discover_all_models((OPENROUTER_SOURCE,)) + + assert errors == [] + assert len(discovered) == 1 + model = discovered[0] + assert model.provider_name == "openrouter" + assert model.evidence_only is True + assert model.zdr_capable is False + assert is_routable_discovered_model(model) is False + + def test_malformed_openrouter_zdr_data_is_ignored(monkeypatch) -> None: monkeypatch.setattr( "contextual_orchestrator.model_discovery._fetch_json_same_host_https",