Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)

### Fixed

- OpenRouter discovery no longer marks the entire credential account
evidence-only. Authenticated catalog rows may serve ordinary requests, while
ZDR-only requests still require explicit route-level ZDR evidence.
- Model discovery now treats every KV credential as an independent account/catalog boundary, removes provider-family collapsing, and offers secret-free `--verbose` progress diagnostics. Logical equivalence and latency-based switching remain explicit `model_group` decisions only.
- Model-group evidence now reports peak observed RPM and provider-reported TPM over a real 60-second completion window without generating probe traffic or inferring missing usage.
- `discover_provider_models`'s primary model-list fetch is now retried once
Expand Down
46 changes: 37 additions & 9 deletions contextual_orchestrator/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
discover_all_models,
free_discovered_models,
general_free_serving_candidates,
is_discovered_chat_candidate,
is_routable_discovered_model,
refresh_price_book,
select_bootstrap_discovered_agents,
Expand Down Expand Up @@ -412,16 +413,43 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l
_runtime_discovery_sources(orchestrator),
ca_bundle=orchestrator.client.ca_bundle,
)
chat_models = [model for model in discovered if is_routable_discovered_model(model)]
existing_ids = {agent.id for agent in orchestrator.candidates}
agents = [
replace(
agent_from_discovered(model),
disabled=False,
)
for model in chat_models
if agent_id_for(model) not in existing_ids
chat_models = [
model
for model in discovered
if not model.evidence_only and is_discovered_chat_candidate(model)
]
existing_by_id = {agent.id: agent for agent in orchestrator.candidates}
agents = []
for model in chat_models:
existing = existing_by_id.get(agent_id_for(model))
routable = is_routable_discovered_model(model)
if existing is None:
agents.append(replace(agent_from_discovered(model), disabled=not routable))
elif "discovered" not in existing.tags:
continue
elif not routable:
tags = (*existing.tags, "spend:blocked")
if existing.disabled and "spend:blocked" not in existing.tags:
tags = (*tags, "spend:blocked:preserve-disabled")
agents.append(
replace(
existing,
disabled=True,
tags=tuple(dict.fromkeys(tags)),
)
)
elif "spend:blocked" in existing.tags:
agents.append(
replace(
existing,
disabled="spend:blocked:preserve-disabled" in existing.tags,
tags=tuple(
tag
for tag in existing.tags
if tag not in {"spend:blocked", "spend:blocked:preserve-disabled"}
),
)
)
result = (
orchestrator.sync_discovered_agents(agents)
if agents
Expand Down
47 changes: 39 additions & 8 deletions contextual_orchestrator/model_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,6 @@ def configured_gateway_source(
list_url="https://openrouter.ai/api/v1/models?output_modalities=all",
chat_base_url="https://openrouter.ai/api/v1",
capabilities=("chat",),
evidence_only=True,
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
),
ProviderModelSource(
provider_name="opencode_zen",
Expand Down Expand Up @@ -285,6 +284,7 @@ class DiscoveredModel:
privacy_policy_urls: tuple[str, ...] = ()
zdr_capable: bool = False
evidence_only: bool = False
spend_admitted: bool = True


class ProviderDiscoveryError(RuntimeError):
Expand Down Expand Up @@ -1279,13 +1279,23 @@ def discover_all_models(
)
except ProviderDiscoveryError as exc:
errors.append(exc)
# The OpenRouter catalog is evidence-only; its public ZDR endpoint supplies
# matching privacy evidence for discovered models from other providers. It
# is never selected as an inference upstream here.
return _apply_discovered_model_evidence(
# OpenRouter's authenticated catalog supplies routable account-model rows;
# its public ZDR endpoint adds route-specific privacy evidence without
# turning the whole provider account into either ZDR-only or non-serving.
Comment on lines +1282 to +1284

@devin-ai-integration devin-ai-integration Bot Aug 31, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Info: ZDR requests still fail closed

OpenRouter rows gain privacy:zdr only from explicit model evidence. Failed or empty inventories leave them ineligible for zdr_only requests.

Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

routed = _apply_discovered_model_evidence(
_deduplicate_discovered_models(discovered),
_openrouter_zdr_model_ids(timeout=timeout),
), errors
)
if any(
source.provider_name == "openrouter"
and get_credential(source.credential_name)
for source in sources
):
routed = apply_openrouter_spend_admission(
routed,
openrouter_paid_inference_available(timeout=timeout),
)
return routed, errors


def openrouter_paid_inference_available(
Expand Down Expand Up @@ -1321,6 +1331,24 @@ def openrouter_paid_inference_available(
return None


def apply_openrouter_spend_admission(
discovered: Sequence[DiscoveredModel],
paid_available: bool | None,
) -> list[DiscoveredModel]:
"""Fail closed for paid OpenRouter rows without current credit evidence."""
return [
replace(
model,
spend_admitted=(
model.provider_name != "openrouter"
or model.is_free
or paid_available is True
),
)
for model in discovered
]


_SLUG_RE = re.compile(r"[^a-z0-9]+")


Expand Down Expand Up @@ -1367,8 +1395,10 @@ def is_routable_discovered_model(discovered: DiscoveredModel) -> bool:
expose a media-only model with a generic identifier, so the model-name
heuristic is only a fallback for rows with no capability or modality data.
"""
return not discovered.evidence_only and is_discovered_chat_candidate(
discovered
return (
not discovered.evidence_only
and discovered.spend_admitted
and is_discovered_chat_candidate(discovered)
)


Expand All @@ -1393,6 +1423,7 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) ->
tags=(
"discovered",
*(("cost:free",) if discovered.is_free else ()),
*(("spend:blocked",) if not discovered.spend_admitted else ()),
*privacy_tags_for_discovered(discovered),
*discovered.capabilities,
*(f"capability:{value}" for value in discovered.capabilities),
Expand Down
1 change: 1 addition & 0 deletions contextual_orchestrator/provider_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ def serving_tags_for_discovered(model: DiscoveredModel) -> tuple[str, ...]:
(
*_GENERIC_SERVING_TAGS,
*(("cost:free",) if model.is_free else ()),
*(("spend:blocked",) if not model.spend_admitted else ()),
*privacy_tags_for_discovered(model),
*model.capabilities,
*(f"capability:{value}" for value in model.capabilities),
Expand Down
18 changes: 17 additions & 1 deletion contextual_orchestrator/provider_catalog_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import argparse
import json
import os
from dataclasses import dataclass
from dataclasses import dataclass, replace
import threading
from typing import Callable, Mapping, Sequence

Expand All @@ -28,8 +28,10 @@
DiscoveredModel,
ProviderDiscoveryError,
ProviderModelSource,
apply_openrouter_spend_admission,
agent_id_for,
discover_all_models,
openrouter_paid_inference_available,
refresh_price_book,
)
from .privacy_policy_analysis import (
Expand Down Expand Up @@ -593,6 +595,20 @@ def bootstrap_provider_catalog_runtime(
for name in failed_credentials
}
) if failed_credentials else ()
if any(
source.provider_name == "openrouter"
and source.credential_name in failed_credentials
for source in source_tuple
):
snapshot = replace(
snapshot,
models=tuple(
apply_openrouter_spend_admission(
snapshot.models,
openrouter_paid_inference_available(),
)
),
)

usable_models = tuple(
model
Expand Down
2 changes: 2 additions & 0 deletions contextual_orchestrator/provider_catalog_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,7 @@ def normalize_discovered_model(
supports_no_prompt_retention=model.supports_no_prompt_retention,
privacy_policy_urls=tuple(model.privacy_policy_urls),
zdr_capable=bool(model.zdr_capable),
spend_admitted=bool(model.spend_admitted),
)


Expand All @@ -353,6 +354,7 @@ def _restore_model_semantics(
currency_code=model.currency_code,
unit_prices=model.unit_prices,
is_free="cost:free" in normalized,
spend_admitted="spend:blocked" not in normalized,
supports_zero_data_retention=(
True if "privacy:zdr" in normalized else False if "privacy:no_zdr" in normalized else None
),
Expand Down
7 changes: 7 additions & 0 deletions docs/planning/adrs/0032-model-group-cost-aware-discovery.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,13 @@ publish the same fields through model metadata, but a logical model receives a
value only when every deployment agrees. Thus free status never implies privacy,
and paid status never implies retention.

OpenRouter's authenticated catalog rows remain ordinary serving candidates.
The ZDR inventory qualifies only matching account-model routes when a request
requires ZDR; it does not make the entire OpenRouter account evidence-only and
does not exclude non-ZDR routes from ordinary requests. Missing or failed ZDR
evidence therefore fails closed only for `zdr_only` selection, not for general
inference.
Comment on lines +39 to +44

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔍 Research artifact requirement unresolved

The routing change adds citations but no paper artifact or redistribution note. Confirm the documented citation-and-summary fallback satisfies the repository’s research-grounding rule.

Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.


OpenAI catalog rows also retain OpenAI's official data-controls documentation
as policy evidence. Because approval and enablement are organization/project
settings that the Models API does not disclose, discovery leaves the actual ZDR
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ time, in `_parse_openai_compatible` (`_pricing_is_free`, unless a merged row
already carries an explicit `is_free` boolean), which requires a provider's
own model-list response to carry a real per-token price of exactly zero. Of
this gateway's six configured provider sources, only OpenRouter's own API
ever reports real pricing, and OpenRouter is deliberately `evidence_only=True`
(commit `952996ec`, a ZDR-privacy hardening that stays untouched) and so never
serves inference. Of the remaining five, `openai`, `nvidia_nim`,
ever reports real pricing. Commit `952996ec` incorrectly made its entire
authenticated catalog evidence-only even though ZDR evidence is route-specific;
ADR 0032 now restores those rows as serving candidates. Of the remaining five, `openai`, `nvidia_nim`,
`nvidia_nim_sub`, and `bytez` never report pricing in their own `/v1/models`
responses, so `is_free` was never `True` for any of them. `orchestrator/free`
was therefore structurally empty in practice: the one provider ADR 0032
Expand Down Expand Up @@ -87,7 +87,7 @@ constant and its `provider_name == "opencode_zen"` special case with the same
value, now expressed as data), `nvidia_nim` and `nvidia_nim_sub` both set it
to `"nvidia"`, and `openai` sets it to `"openai"`. `openrouter` and `bytez`
keep the `None` default: OpenRouter already reports its own real per-token
pricing and stays `evidence_only=True` regardless, and there is no
pricing, and there is no
Models.dev signal to join for Bytez.

The invocation site in `discover_provider_models` becomes `if
Expand Down
7 changes: 4 additions & 3 deletions docs/product-technical-gap-baseline.md
Original file line number Diff line number Diff line change
Expand Up @@ -630,9 +630,10 @@ This document serves as the baseline for the Contextual Orchestrator (an enterpr
`orchestrator/free` (ADR 0032) was structurally empty in practice: `is_free`
only ever becomes `True` from a provider's own reported per-token price, and
of this gateway's six provider sources only OpenRouter's API ever reports
real pricing — and OpenRouter is deliberately `evidence_only=True` (ZDR
hardening, commit `952996ec`, untouched by this change) so it never serves
inference. Of the remaining five, `openai`, `nvidia_nim`, `nvidia_nim_sub`,
real pricing. Commit `952996ec` incorrectly made the entire OpenRouter account
evidence-only even though its ZDR evidence is route-specific; ADR 0032 now
keeps authenticated OpenRouter rows routable and applies ZDR only during
`zdr_only` selection. Of the remaining five, `openai`, `nvidia_nim`, `nvidia_nim_sub`,
and `bytez` never report pricing themselves. The one existing mitigation, cross-referencing
`opencode_zen` against Models.dev (`https://models.dev/api.json`), only
covers a source that is `bootstrap_required = False` and not always
Expand Down
Loading
Loading