From 7e3b65e993419fc53d5a75bedcb2b7ea439ba276 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:32:50 -0700 Subject: [PATCH 01/89] test: reproduce embedding deployment selected for chat synthesis --- tests/test_chat_model_capability_isolation.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/test_chat_model_capability_isolation.py diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py new file mode 100644 index 000000000..9047249c0 --- /dev/null +++ b/tests/test_chat_model_capability_isolation.py @@ -0,0 +1,83 @@ +"""Regression coverage for isolating non-chat models from chat agent discovery.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + register_credential, + set_backend, +) +from contextual_orchestrator.model_discovery import ( # noqa: E402 + ProviderModelSource, + discover_provider_models, +) + + +class _Response: + """Small context-managed HTTP response used by the offline regression.""" + + def __init__(self, payload: dict[str, object]) -> None: + self._body = json.dumps(payload).encode("utf-8") + + def __enter__(self) -> "_Response": + return self + + def __exit__(self, *_args: object) -> bool: + return False + + def read(self, _size: int = -1) -> bytes: + return self._body + + +@pytest.fixture(autouse=True) +def _fresh_credential_backend(): + """Keep the provider credential registry isolated between tests.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: + """Exclude the exact Azure embedding deployment seen in synthesis alerts.""" + register_credential("GATEWAY_API_KEY", "gateway-secret") + source = ProviderModelSource( + provider_name="enterprise_gateway", + credential_name="GATEWAY_API_KEY", + list_url="https://gateway.example.test/v1/models", + chat_base_url="https://gateway.example.test/v1", + ) + payload = { + "data": [ + {"id": "azure/text-embedding-3-large"}, + {"id": "text_embedding_3_large"}, + {"id": "BAAI/bge-m3"}, + {"id": "openai/whisper-1"}, + {"id": "gpt-4o-mini-transcribe"}, + {"id": "text-moderation-latest"}, + {"id": "company/reranker-v2"}, + {"id": "gpt-5.2"}, + {"id": "qwen/qwen3-235b-a22b-instruct"}, + ] + } + + with patch( + "contextual_orchestrator.model_discovery.urllib.request.urlopen", + return_value=_Response(payload), + ): + discovered = discover_provider_models(source) + + assert [model.model_id for model in discovered] == [ + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ] From 817eaec7a6acb1ed3060aa7557d96d008784e206 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:36:23 -0700 Subject: [PATCH 02/89] fix: isolate chat discovery from embedding endpoints --- contextual_orchestrator/model_discovery.py | 110 +++++++++++++++++---- 1 file changed, 90 insertions(+), 20 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index bd76100c0..bae52cc54 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -1,4 +1,4 @@ -"""Provider model-list discovery: turns registered KV credentials into agent candidates. +"""Provider model-list discovery for chat-agent candidates. Queries each configured provider's model-list endpoint over its OpenAI-compatible (or provider-specific) discovery API and returns :class:`DiscoveredModel` rows that @@ -9,6 +9,12 @@ (the KV registry), and a provider with nothing registered is silently skipped so registering a subset of the five supported keys still works. Stdlib only (``urllib.request``), matching this repo's dependency-free transport convention. + +This module owns the ordinary chat-agent discovery boundary. Provider catalogs may +mix chat, embedding, reranking, transcription, moderation, image, and realtime +models under one ``/models`` endpoint. Clearly non-chat identifiers are rejected +before they can be converted to workers, selected by cost, or persisted into the +chat agent pool. """ from __future__ import annotations @@ -29,6 +35,39 @@ DISCOVERY_TIMEOUT_SECONDS = 15.0 +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_NON_CHAT_EXACT_TOKENS = frozenset( + { + "audio", + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "guard", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "safety", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_NON_CHAT_TOKEN_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + @dataclass(frozen=True) class ProviderModelSource: @@ -84,7 +123,7 @@ class ProviderModelSource: @dataclass(frozen=True) class DiscoveredModel: - """One model found on a provider, with pricing when the provider reports it.""" + """One chat-compatible model found on a provider, with reported pricing.""" provider_name: str model_id: str @@ -104,6 +143,32 @@ def __init__(self, provider_name: str, detail: str) -> None: super().__init__(f"model discovery failed for provider {provider_name!r}: {detail}") +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether a model identifier is eligible for the chat-agent pool. + + The classifier is intentionally conservative and only rejects identifiers + that clearly advertise a non-chat endpoint family. It normalizes provider + prefixes and common separators so values such as + ``azure/text-embedding-3-large`` and ``text_embedding_3_large`` cannot be + assigned to thinker, worker, verifier, or synthesizer roles. + + Unknown identifiers remain eligible until explicit provider capability + metadata is available; this avoids fabricating positive reasoning or tool + capabilities from a name while still preserving ordinary chat discovery. + """ + if type(model_id) is not str: + return False + tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + if not tokens: + return False + for token in tokens: + if token in _NON_CHAT_EXACT_TOKENS: + return False + if token.startswith(_NON_CHAT_TOKEN_PREFIXES): + return False + return True + + def _fetch_json(url: str, *, api_key: str, auth_scheme: str, timeout: float) -> Any: if not url.startswith("https://"): # Every caller passes one of the hardcoded PROVIDER_SOURCES chat_base_url @@ -132,13 +197,14 @@ def _price_per_1k(value: Any) -> float | None: def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: + """Parse one OpenAI-compatible catalog into chat-compatible candidates.""" rows = payload.get("data") if isinstance(payload, dict) else None discovered: list[DiscoveredModel] = [] for row in rows if isinstance(rows, list) else []: if not isinstance(row, dict): continue model_id = row.get("id") - if type(model_id) is not str or not model_id: + if not is_chat_compatible_model_id(model_id): continue pricing = row.get("pricing") if isinstance(row.get("pricing"), dict) else {} discovered.append( @@ -156,13 +222,14 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: + """Parse one Bytez chat catalog without admitting non-chat identifiers.""" rows = payload.get("output") if isinstance(payload, dict) else None discovered: list[DiscoveredModel] = [] for row in rows if isinstance(rows, list) else []: if not isinstance(row, dict): continue model_id = row.get("modelId") - if type(model_id) is not str or not model_id: + if not is_chat_compatible_model_id(model_id): continue discovered.append( DiscoveredModel( @@ -181,7 +248,7 @@ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredMo def discover_provider_models( source: ProviderModelSource, *, timeout: float = DISCOVERY_TIMEOUT_SECONDS ) -> list[DiscoveredModel]: - """Discover one provider's models, or ``[]`` if its credential is not registered.""" + """Discover chat candidates, or ``[]`` when the credential is not registered.""" api_key = get_credential(source.credential_name) if not api_key: return [] @@ -202,7 +269,7 @@ def discover_all_models( *, timeout: float = DISCOVERY_TIMEOUT_SECONDS, ) -> tuple[list[DiscoveredModel], list[ProviderDiscoveryError]]: - """Discover models across every provider with a registered credential. + """Discover chat candidates across providers with registered credentials. One provider's failure never blocks the others: errors are collected and returned alongside whatever models were successfully discovered. @@ -231,7 +298,9 @@ def agent_id_for(discovered: DiscoveredModel) -> str: def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> ModelAgent: - """Build a disabled-by-default ModelAgent for a discovered model (opt-in serving).""" + """Build a disabled chat ModelAgent or reject a non-chat discovery record.""" + if not is_chat_compatible_model_id(discovered.model_id): + raise ValueError("non-chat model cannot be converted into a chat agent") return ModelAgent( id=agent_id_for(discovered), model=discovered.model_id, @@ -246,7 +315,7 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook") -> int: - """Write every discovered model's known pricing into the price book. + """Write every discovered chat model's known pricing into the price book. Returns the number of price rows written. A model without provider-reported pricing is skipped rather than defaulted to 0 -- an unpriced model already @@ -257,6 +326,8 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook written = 0 for model in discovered: + if not is_chat_compatible_model_id(model.model_id): + continue if model.prompt_price_per_1k is None and model.completion_price_per_1k is None: continue price_book.set_price( @@ -275,7 +346,7 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook def select_cheapest_discovered_agent( discovered: list[DiscoveredModel], price_book: "PriceBook" ) -> DiscoveredModel | None: - """Pick the lowest-cost discovered model per the price book (auto-optimization). + """Pick the lowest-cost chat-compatible model per the price book. Reuses :func:`~contextual_orchestrator.batch_routing.cheapest_upstream`, the existing cost-optimizing upstream selector. Call :func:`refresh_price_book` @@ -286,13 +357,14 @@ def select_cheapest_discovered_agent( "auto-pick something free to try," but callers doing real cost comparison should refresh pricing for every candidate they care about first. """ - if not discovered: + eligible = [model for model in discovered if is_chat_compatible_model_id(model.model_id)] + if not eligible: return None - candidates = [{"provider": model.provider_name, "model": model.model_id} for model in discovered] + candidates = [{"provider": model.provider_name, "model": model.model_id} for model in eligible] winner = cheapest_upstream(candidates, price_book) if winner is None: return None - for model in discovered: + for model in eligible: if model.provider_name == winner["provider"] and model.model_id == winner["model"]: return model return None # pragma: no cover - winner always comes from candidates @@ -301,17 +373,15 @@ def select_cheapest_discovered_agent( def select_top_n_cheapest_discovered_agents( discovered: list[DiscoveredModel], price_book: "PriceBook", limit: int ) -> list[DiscoveredModel]: - """Return the ``limit`` lowest-cost discovered models, cheapest first. - - For bootstrapping a CI sidecar (or any first-boot pool) with more than one - enabled agent for failover, without hand-picking which discovered models to - trust. Same pricing contract as :func:`select_cheapest_discovered_agent`. - """ - if limit <= 0 or not discovered: + """Return the ``limit`` cheapest chat-compatible models in ascending cost.""" + if limit <= 0: + return [] + eligible = [model for model in discovered if is_chat_compatible_model_id(model.model_id)] + if not eligible: return [] def _cost(model: DiscoveredModel) -> float: cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) return cost - return sorted(discovered, key=_cost)[:limit] + return sorted(eligible, key=_cost)[:limit] From c765ce469f1f54e0f2d7f15858bfe8b3d83d42f1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:37:26 -0700 Subject: [PATCH 03/89] test: cover chat capability isolation defenses --- tests/test_chat_model_capability_isolation.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index 9047249c0..46b653e2c 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -16,9 +16,17 @@ register_credential, set_backend, ) +from contextual_orchestrator.cost_ledger import PriceBook, PriceEntry # noqa: E402 +from contextual_orchestrator.kv_config import InMemoryConfigStore # noqa: E402 from contextual_orchestrator.model_discovery import ( # noqa: E402 + DiscoveredModel, ProviderModelSource, + agent_from_discovered, discover_provider_models, + is_chat_compatible_model_id, + refresh_price_book, + select_cheapest_discovered_agent, + select_top_n_cheapest_discovered_agents, ) @@ -48,6 +56,19 @@ def _fresh_credential_backend(): set_backend(None) +def _model(model_id: str, *, priced: bool = False) -> DiscoveredModel: + """Build one synthetic discovered model for capability-boundary tests.""" + return DiscoveredModel( + provider_name="enterprise_gateway", + model_id=model_id, + credential_name="GATEWAY_API_KEY", + chat_base_url="https://gateway.example.test/v1", + auth_scheme="Bearer", + prompt_price_per_1k=1.0 if priced else None, + completion_price_per_1k=1.0 if priced else None, + ) + + def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: """Exclude the exact Azure embedding deployment seen in synthesis alerts.""" register_credential("GATEWAY_API_KEY", "gateway-secret") @@ -81,3 +102,74 @@ def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: "gpt-5.2", "qwen/qwen3-235b-a22b-instruct", ] + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + (None, False), + ("", False), + ("---", False), + ("vendor/embeddingv2", False), + ("vendor/reranking-v2", False), + ("vendor/transcriber-v2", False), + ("gpt-5.2", True), + ("qwen/qwen3-instruct", True), + ], +) +def test_chat_compatibility_normalizes_identifiers( + model_id: object, expected: bool +) -> None: + """Normalize provider prefixes and separators without guessing chat features.""" + assert is_chat_compatible_model_id(model_id) is expected # type: ignore[arg-type] + + +def test_bytez_chat_catalog_still_rejects_non_chat_identifiers() -> None: + """Apply the same boundary even when a provider accepts a chat task filter.""" + register_credential("BYTEZ_API_KEY", "bytez-secret") + source = ProviderModelSource( + provider_name="bytez", + credential_name="BYTEZ_API_KEY", + list_url="https://api.bytez.com/models/v2/list/models", + chat_base_url="https://api.bytez.com/models/v2/openai/v1", + auth_scheme="Key", + style="bytez", + task_filter="chat", + ) + payload = { + "output": [ + {"modelId": "vendor/embeddingv2"}, + {"modelId": "vendor/chat-instruct"}, + ] + } + + with patch( + "contextual_orchestrator.model_discovery.urllib.request.urlopen", + return_value=_Response(payload), + ): + discovered = discover_provider_models(source) + + assert [model.model_id for model in discovered] == ["vendor/chat-instruct"] + + +def test_non_chat_discovery_cannot_be_converted_to_agent() -> None: + """Keep manually constructed discovery rows from bypassing the parser filter.""" + with pytest.raises(ValueError, match="non-chat model"): + agent_from_discovered(_model("azure/text-embedding-3-large")) + + +def test_non_chat_discovery_is_not_priced_or_selected_for_chat() -> None: + """Keep price routing from reintroducing an incompatible endpoint model.""" + price_book = PriceBook(InMemoryConfigStore()) + embedding_model = _model("azure/text-embedding-3-large", priced=True) + chat_model = _model("gpt-5.2", priced=True) + price_book.set_price(PriceEntry("enterprise_gateway", "gpt-5.2", 1.0, 1.0)) + + assert refresh_price_book([embedding_model, chat_model], price_book) == 1 + assert price_book.get_price( + "enterprise_gateway", "azure/text-embedding-3-large" + ) is None + assert select_cheapest_discovered_agent([embedding_model], price_book) is None + assert select_top_n_cheapest_discovered_agents( + [embedding_model], price_book, 1 + ) == [] From 6c2d263064b57704ec237599ca64e03f8187edce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:38:09 -0700 Subject: [PATCH 04/89] docs: record embedding-to-chat incident boundary --- .../embedding-chat-capability-isolation.md | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 docs/doctoring/embedding-chat-capability-isolation.md diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md new file mode 100644 index 000000000..88cc334c7 --- /dev/null +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -0,0 +1,88 @@ +# Embedding-to-chat capability isolation incident + +**Status:** Accepted incident decision +**Date:** 2026-08-20 +**Affected consumer:** LineageWeave buyer-surface stack around PR #260 + +## Incident + +A conducted workflow reached the final contextual-orchestrator synthesizer with +`model_group=text-embedding-3-large` and deployment +`azure/text-embedding-3-large`. The gateway rejected the chat operation as +unsupported. Its configured fallback map contained chat-generation model groups, +but no fallback attached to the embedding group. + +The missing fallback was a symptom, not the causal defect. An embedding deployment +had already crossed the chat-agent capability boundary and become eligible for a +worker role. + +## Causal boundary + +Provider-compatible `/models` registries can contain multiple endpoint families. +The current discovery parser accepted every non-empty model identifier and exposed +it to agent creation, price selection, and durable pool synchronization. A catalog +row naming an embedding deployment could therefore be scored for thinker, worker, +verifier, or synthesizer work even though its serving endpoint accepts embedding +input rather than chat messages. + +OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately +from models supported by chat completions. Microsoft likewise demonstrates it with +`client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes +chat, responses, embeddings, image, audio, rerank, and other endpoint families as +distinct operations. A router fallback can choose another deployment for the same +operation; it cannot make an embedding deployment execute a chat operation. + +## Decision + +The ordinary model-discovery module is a **chat-agent discovery boundary**. + +1. Normalize provider prefixes and common separators in model identifiers. +2. Reject identifiers that clearly advertise embedding, reranking, transcription, + moderation/safety, image, audio/speech, realtime, or known embedding-family + transport semantics. +3. Apply the guard while parsing both OpenAI-compatible and Bytez catalogs. +4. Re-check the invariant before converting a discovery record to `ModelAgent`. +5. Re-check the invariant before writing chat pricing or running price-based chat + selection. +6. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, + or verification capabilities from their names. + +This is deliberately a conservative negative filter. A future capability registry +may replace name-based exclusion with authenticated provider metadata, measured +endpoint probes, and separate endpoint-specific pools. Until that evidence exists, +a clearly non-chat model fails closed at the chat boundary. + +## Rejected response + +Adding `text-embedding-3-large` to a chat fallback map is rejected. It would retain +the invalid primary assignment and merely hide it when a fallback happened to be +available. Repeated provider retries are also rejected because the request is +structurally unsupported, not transiently unavailable. + +## Residual operational action + +This change prevents new incompatible catalog rows from entering the chat pool. +A deployment that already persisted such a row must disable or withdraw that stale +discovered agent. The durable provider-bootstrap slice owns exact-set activation and +stale discovered-agent withdrawal; it must import this shared boundary when rebased. + +## Verification evidence + +`tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID +and provider/separator aliases. It also verifies: + +- OpenAI-compatible and Bytez catalog filtering; +- malformed and prefix-only identifier handling; +- agent-conversion rejection; +- exclusion from the price book and cheapest-agent selection. + +## References + +BerriAI. (2026). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. +https://docs.litellm.ai/ + +Microsoft. (2026). *How to switch between OpenAI and Azure OpenAI endpoints*. +Microsoft Learn. https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints + +OpenAI. (2026). *Data controls in the OpenAI platform: Default usage policies by +endpoint*. https://platform.openai.com/docs/models/default-usage-policies-by-endpoint From 35c7bef09fdc09ab70c5d38ae56869c45be55fbe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:47:19 -0700 Subject: [PATCH 05/89] docs: normalize incident references to APA 7 --- .../embedding-chat-capability-isolation.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 88cc334c7..87bb44e97 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -78,11 +78,13 @@ and provider/separator aliases. It also verifies: ## References -BerriAI. (2026). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. -https://docs.litellm.ai/ +BerriAI. (n.d.). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. +Retrieved August 20, 2026, from https://docs.litellm.ai/ -Microsoft. (2026). *How to switch between OpenAI and Azure OpenAI endpoints*. -Microsoft Learn. https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints +Microsoft. (n.d.). *How to switch between OpenAI and Azure OpenAI endpoints*. +Microsoft Learn. Retrieved August 20, 2026, from +https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints -OpenAI. (2026). *Data controls in the OpenAI platform: Default usage policies by -endpoint*. https://platform.openai.com/docs/models/default-usage-policies-by-endpoint +OpenAI. (n.d.). *Data controls in the OpenAI platform: Default usage policies by +endpoint*. Retrieved August 20, 2026, from +https://platform.openai.com/docs/models/default-usage-policies-by-endpoint From f01da9f7c6b26a399bfbd2e3030fd783124fe07d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:51:36 -0700 Subject: [PATCH 06/89] test: reproduce stale embedding agent runtime selection --- tests/test_chat_model_capability_isolation.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index 46b653e2c..f55bd5725 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -11,6 +11,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, register_credential, @@ -28,6 +29,7 @@ select_cheapest_discovered_agent, select_top_n_cheapest_discovered_agents, ) +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 class _Response: @@ -69,6 +71,23 @@ def _model(model_id: str, *, priced: bool = False) -> DiscoveredModel: ) +def _agent( + agent_id: str, + model_id: str, + *, + priority: int = 0, + tags: tuple[str, ...] = ("writing",), +) -> ModelAgent: + """Build one mock-backed runtime agent for selection-path regressions.""" + return ModelAgent( + id=agent_id, + model=model_id, + base_url="mock://local", + priority=priority, + tags=tags, + ) + + def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: """Exclude the exact Azure embedding deployment seen in synthesis alerts.""" register_credential("GATEWAY_API_KEY", "gateway-secret") @@ -173,3 +192,92 @@ def test_non_chat_discovery_is_not_priced_or_selected_for_chat() -> None: assert select_top_n_cheapest_discovered_agents( [embedding_model], price_book, 1 ) == [] + + +def test_stale_embedding_agent_cannot_win_synthesizer_selection() -> None: + """Exclude an already-persisted embedding row even when it has high priority.""" + embedding_agent = _agent( + "embedding_agent", + "azure/text-embedding-3-large", + priority=10_000, + ) + chat_agent = _agent("chat_agent", "gpt-5.2") + orchestrator = TaskOrchestrator([embedding_agent, chat_agent]) + + assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent + + +def test_all_non_chat_agents_fail_before_synthesis() -> None: + """Fail closed when a stale pool contains no chat-compatible worker.""" + orchestrator = TaskOrchestrator( + [_agent("embedding_agent", "azure/text-embedding-3-large")] + ) + + with pytest.raises(RuntimeError, match="chat-compatible"): + orchestrator._select_agent("Produce the final answer.", "synthesizer") + + +def test_generated_plan_reselects_non_chat_agent_assignment() -> None: + """Do not trust a generated plan that names a stale embedding agent directly.""" + embedding_agent = _agent( + "embedding_agent", + "azure/text-embedding-3-large", + priority=10_000, + ) + chat_agent = _agent("chat_agent", "gpt-5.2") + orchestrator = TaskOrchestrator([embedding_agent, chat_agent]) + raw_plan = json.dumps( + { + "steps": [ + { + "id": 0, + "role": "worker", + "agent_id": "chat_agent", + "subtask": "Execute the task.", + "access": [], + }, + { + "id": 1, + "role": "synthesizer", + "agent_id": "embedding_agent", + "subtask": "Produce the final answer.", + "access": [0], + }, + ] + } + ) + + steps = orchestrator._parse_workflow_plan(raw_plan) + + assert steps[-1].agent_id == "chat_agent" + + +def test_failover_candidates_exclude_stale_embedding_agents() -> None: + """Keep cross-agent retry from falling through to an incompatible endpoint.""" + chat_agent = _agent("chat_agent", "gpt-5.2") + embedding_agent = _agent( + "embedding_agent", + "azure/text-embedding-3-large", + priority=10_000, + ) + orchestrator = TaskOrchestrator([chat_agent, embedding_agent]) + + candidates = orchestrator._failover_candidates( + chat_agent, + "Produce the final answer.", + "synthesizer", + ) + + assert candidates == [chat_agent] + + +def test_model_client_rejects_non_chat_model_before_mock_or_network_call() -> None: + """Keep the provider boundary fail-closed even when selection is bypassed.""" + client = ModelClient() + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + + with pytest.raises(ValueError, match="chat-compatible"): + client.chat( + embedding_agent, + [{"role": "user", "content": "Produce the final answer."}], + ) From 266561c99c69b9caa6dcb273d8e449a8d2280c7c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:02 -0700 Subject: [PATCH 07/89] fix: enforce chat capability at runtime boundaries --- contextual_orchestrator/chat_capability.py | 151 +++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 contextual_orchestrator/chat_capability.py diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py new file mode 100644 index 000000000..567dc8234 --- /dev/null +++ b/contextual_orchestrator/chat_capability.py @@ -0,0 +1,151 @@ +"""Shared chat-capability classification and runtime fail-closed guards. + +Provider model catalogs are heterogeneous: a model identifier may name a chat +model, embedding model, reranker, transcription model, image model, or another +endpoint family. The helpers in this module prevent clearly non-chat model IDs +from crossing a chat-agent boundary even when an incompatible agent was persisted +before discovery filtering was introduced. +""" + +from __future__ import annotations + +from dataclasses import replace +from functools import wraps +import re +from typing import Any + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_NON_CHAT_EXACT_TOKENS = frozenset( + { + "audio", + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "guard", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "safety", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_NON_CHAT_TOKEN_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether a model identifier is eligible for a chat-agent pool. + + The classifier intentionally rejects only identifiers that clearly advertise + a non-chat endpoint family. Unknown identifiers remain eligible until an + authenticated capability registry can prove more specific endpoint support. + """ + if type(model_id) is not str: + return False + tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + if not tokens: + return False + for token in tokens: + if token in _NON_CHAT_EXACT_TOKENS: + return False + if token.startswith(_NON_CHAT_TOKEN_PREFIXES): + return False + return True + + +def install_runtime_chat_capability_guards( + model_client_cls: type[Any], + task_orchestrator_cls: type[Any], +) -> None: + """Install idempotent fail-closed guards on the existing runtime classes. + + The project keeps runtime orchestration in one large compatibility module. + Installing the invariant here lets discovery, persisted configuration, plan + parsing, failover, and direct client calls share one classifier without + introducing a circular import from that compatibility module back into model + discovery. + """ + marker = "_chat_capability_guards_installed" + if getattr(task_orchestrator_cls, marker, False): + return + + original_chat = model_client_cls.chat + + @wraps(original_chat) + def guarded_chat(self: Any, agent: Any, *args: Any, **kwargs: Any) -> Any: + """Reject a non-chat endpoint before mock or network transport executes.""" + if not is_chat_compatible_model_id(agent.model): + raise ValueError("model is not chat-compatible and cannot serve a chat request") + return original_chat(self, agent, *args, **kwargs) + + model_client_cls.chat = guarded_chat + + original_ranked_agents = task_orchestrator_cls._ranked_agents + + @wraps(original_ranked_agents) + def guarded_ranked_agents(self: Any, text: str, role: str) -> list[Any]: + """Remove stale non-chat agents before role scoring or selection.""" + ranked = original_ranked_agents(self, text, role) + return [agent for agent in ranked if is_chat_compatible_model_id(agent.model)] + + task_orchestrator_cls._ranked_agents = guarded_ranked_agents + + original_select_agent = task_orchestrator_cls._select_agent + + @wraps(original_select_agent) + def guarded_select_agent(self: Any, text: str, role: str) -> Any: + """Fail with an actionable error when no chat-compatible agent remains.""" + if not self._ranked_agents(text, role): + raise RuntimeError(f"no chat-compatible agent available for role={role}") + return original_select_agent(self, text, role) + + task_orchestrator_cls._select_agent = guarded_select_agent + + original_failover_candidates = task_orchestrator_cls._failover_candidates + + @wraps(original_failover_candidates) + def guarded_failover_candidates( + self: Any, + primary: Any, + text: str, + role: str, + ) -> list[Any]: + """Prevent cross-agent retry from falling through to another endpoint family.""" + candidates = original_failover_candidates(self, primary, text, role) + return [agent for agent in candidates if is_chat_compatible_model_id(agent.model)] + + task_orchestrator_cls._failover_candidates = guarded_failover_candidates + + original_parse_workflow_plan = task_orchestrator_cls._parse_workflow_plan + + @wraps(original_parse_workflow_plan) + def guarded_parse_workflow_plan(self: Any, raw: str) -> list[Any]: + """Reselect any stale non-chat assignment named by a generated plan.""" + steps = original_parse_workflow_plan(self, raw) + agents_by_id = {agent.id: agent for agent in self.agents} + guarded_steps: list[Any] = [] + for step in steps: + assigned = agents_by_id.get(step.agent_id) + if assigned is not None and not is_chat_compatible_model_id(assigned.model): + replacement = self._select_agent(step.subtask, step.role) + step = replace(step, agent_id=replacement.id) + guarded_steps.append(step) + return guarded_steps + + task_orchestrator_cls._parse_workflow_plan = guarded_parse_workflow_plan + setattr(task_orchestrator_cls, marker, True) From 4b50126650b2df86756698502137d75e66b8e9d1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:36 -0700 Subject: [PATCH 08/89] refactor: share chat capability classifier --- contextual_orchestrator/model_discovery.py | 60 +--------------------- 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index bae52cc54..12f8f354c 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -27,6 +27,7 @@ from typing import TYPE_CHECKING, Any from .batch_routing import cheapest_upstream +from .chat_capability import is_chat_compatible_model_id from .credentials import get_credential from .orchestrator import ModelAgent @@ -35,39 +36,6 @@ DISCOVERY_TIMEOUT_SECONDS = 15.0 -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_NON_CHAT_EXACT_TOKENS = frozenset( - { - "audio", - "bge", - "e5", - "embed", - "embedding", - "embeddings", - "guard", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "safety", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_NON_CHAT_TOKEN_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - @dataclass(frozen=True) class ProviderModelSource: @@ -143,32 +111,6 @@ def __init__(self, provider_name: str, detail: str) -> None: super().__init__(f"model discovery failed for provider {provider_name!r}: {detail}") -def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether a model identifier is eligible for the chat-agent pool. - - The classifier is intentionally conservative and only rejects identifiers - that clearly advertise a non-chat endpoint family. It normalizes provider - prefixes and common separators so values such as - ``azure/text-embedding-3-large`` and ``text_embedding_3_large`` cannot be - assigned to thinker, worker, verifier, or synthesizer roles. - - Unknown identifiers remain eligible until explicit provider capability - metadata is available; this avoids fabricating positive reasoning or tool - capabilities from a name while still preserving ordinary chat discovery. - """ - if type(model_id) is not str: - return False - tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - if not tokens: - return False - for token in tokens: - if token in _NON_CHAT_EXACT_TOKENS: - return False - if token.startswith(_NON_CHAT_TOKEN_PREFIXES): - return False - return True - - def _fetch_json(url: str, *, api_key: str, auth_scheme: str, timeout: float) -> Any: if not url.startswith("https://"): # Every caller passes one of the hardcoded PROVIDER_SOURCES chat_base_url From bc004d099468f36ee7e69f59c7b10f97ce7ea658 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:46 -0700 Subject: [PATCH 09/89] fix: activate runtime chat capability guards --- contextual_orchestrator/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index 70dbd71c6..dbf9e2ad0 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -17,6 +17,7 @@ cheapest_upstream, heuristic_embedding, ) +from .chat_capability import install_runtime_chat_capability_guards from .cost_ledger import ( ATTRIBUTION_DIMENSIONS, AttributionDimensions, @@ -37,9 +38,11 @@ from .cost_router import CostRoutingCoordinator from .credentials import NotConfigured, get_credential, register_credential from .kv_config import InMemoryConfigStore, get_config_store -from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents +from .orchestrator import ModelAgent, ModelClient, TaskOrchestrator, WorkflowStep, load_agents from .token_counting import HeuristicTokenCounter, build_token_counter +install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) + __all__ = [ "ModelAgent", "TaskOrchestrator", From 8bf5db19695acbffeaba041e3563690d5dbc17f0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:56:14 -0700 Subject: [PATCH 10/89] test: tighten runtime guard coverage surface --- contextual_orchestrator/chat_capability.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index 567dc8234..98d55cf09 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -2,7 +2,7 @@ Provider model catalogs are heterogeneous: a model identifier may name a chat model, embedding model, reranker, transcription model, image model, or another -endpoint family. The helpers in this module prevent clearly non-chat model IDs +endpoint family. The helpers in this module prevent clearly non-chat model IDs from crossing a chat-agent boundary even when an incompatible agent was persisted before discovery filtering was introduced. """ @@ -52,7 +52,7 @@ def is_chat_compatible_model_id(model_id: str) -> bool: """Return whether a model identifier is eligible for a chat-agent pool. The classifier intentionally rejects only identifiers that clearly advertise - a non-chat endpoint family. Unknown identifiers remain eligible until an + a non-chat endpoint family. Unknown identifiers remain eligible until an authenticated capability registry can prove more specific endpoint support. """ if type(model_id) is not str: @@ -140,8 +140,8 @@ def guarded_parse_workflow_plan(self: Any, raw: str) -> list[Any]: agents_by_id = {agent.id: agent for agent in self.agents} guarded_steps: list[Any] = [] for step in steps: - assigned = agents_by_id.get(step.agent_id) - if assigned is not None and not is_chat_compatible_model_id(assigned.model): + assigned = agents_by_id[step.agent_id] + if not is_chat_compatible_model_id(assigned.model): replacement = self._select_agent(step.subtask, step.role) step = replace(step, agent_id=replacement.id) guarded_steps.append(step) From d33d1d57a7d8da9df6cca8a007446d04784e6dc9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:56:37 -0700 Subject: [PATCH 11/89] test: cover runtime guard installation idempotence --- tests/test_chat_model_capability_isolation.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index f55bd5725..fc1cd024e 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -12,6 +12,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.chat_capability import ( # noqa: E402 + install_runtime_chat_capability_guards, +) from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, register_credential, @@ -143,6 +146,15 @@ def test_chat_compatibility_normalizes_identifiers( assert is_chat_compatible_model_id(model_id) is expected # type: ignore[arg-type] +def test_runtime_guard_installation_is_idempotent() -> None: + """Re-importing package initialization must not stack method wrappers.""" + before = ModelClient.chat + + install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) + + assert ModelClient.chat is before + + def test_bytez_chat_catalog_still_rejects_non_chat_identifiers() -> None: """Apply the same boundary even when a provider accepts a chat task filter.""" register_credential("BYTEZ_API_KEY", "bytez-secret") From 7ae2d4d375c760b7f4db65178edf7764d2bd04c7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:57:28 -0700 Subject: [PATCH 12/89] docs: record stale-agent runtime containment --- .../embedding-chat-capability-isolation.md | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 87bb44e97..02312f61e 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -19,12 +19,19 @@ worker role. ## Causal boundary Provider-compatible `/models` registries can contain multiple endpoint families. -The current discovery parser accepted every non-empty model identifier and exposed +The original discovery parser accepted every non-empty model identifier and exposed it to agent creation, price selection, and durable pool synchronization. A catalog row naming an embedding deployment could therefore be scored for thinker, worker, verifier, or synthesizer work even though its serving endpoint accepts embedding input rather than chat messages. +The first incident fix closed discovery and price-routing boundaries, but further +root-cause tracing showed an already-persisted incompatible `ModelAgent` could still +survive that filter. The runtime ranking path, generated workflow assignment, +cross-agent failover, and direct `ModelClient.chat()` path previously trusted the +persisted model identifier. That stale-state path is sufficient to reproduce the +same unsupported Azure chat operation after a process restart or durable bootstrap. + OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately from models supported by chat completions. Microsoft likewise demonstrates it with `client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes @@ -34,7 +41,7 @@ operation; it cannot make an embedding deployment execute a chat operation. ## Decision -The ordinary model-discovery module is a **chat-agent discovery boundary**. +Chat capability is a **shared runtime invariant**, not only a discovery filter. 1. Normalize provider prefixes and common separators in model identifiers. 2. Reject identifiers that clearly advertise embedding, reranking, transcription, @@ -44,13 +51,18 @@ The ordinary model-discovery module is a **chat-agent discovery boundary**. 4. Re-check the invariant before converting a discovery record to `ModelAgent`. 5. Re-check the invariant before writing chat pricing or running price-based chat selection. -6. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, - or verification capabilities from their names. +6. Remove stale non-chat agents from thinker, worker, verifier, and synthesizer + ranking even if a durable configuration still contains them. +7. Reselect a generated workflow step that explicitly names a stale non-chat agent. +8. Remove non-chat agents from cross-agent failover candidates. +9. Reject a non-chat model at `ModelClient.chat()` before mock or network transport. +10. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, + or verification capabilities from their names. This is deliberately a conservative negative filter. A future capability registry may replace name-based exclusion with authenticated provider metadata, measured endpoint probes, and separate endpoint-specific pools. Until that evidence exists, -a clearly non-chat model fails closed at the chat boundary. +a clearly non-chat model fails closed at every chat boundary. ## Rejected response @@ -61,20 +73,28 @@ structurally unsupported, not transiently unavailable. ## Residual operational action -This change prevents new incompatible catalog rows from entering the chat pool. -A deployment that already persisted such a row must disable or withdraw that stale -discovered agent. The durable provider-bootstrap slice owns exact-set activation and -stale discovered-agent withdrawal; it must import this shared boundary when rebased. +Runtime containment means an already-persisted embedding agent can no longer win +chat selection or failover while stale data is being cleaned up. Durable state must +still converge to the correct exact set: the provider-bootstrap slice owns stale +discovered-agent withdrawal and must import the same shared classifier when rebased. +Runtime rejection is defense in depth, not a substitute for deleting invalid +persistent configuration. ## Verification evidence `tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID -and provider/separator aliases. It also verifies: +and provider/separator aliases. It verifies: - OpenAI-compatible and Bytez catalog filtering; - malformed and prefix-only identifier handling; - agent-conversion rejection; -- exclusion from the price book and cheapest-agent selection. +- exclusion from the price book and cheapest-agent selection; +- exclusion of a high-priority stale embedding agent from synthesizer selection; +- fail-closed behavior when the persisted pool contains only non-chat agents; +- generated-plan reassignment away from a stale embedding agent; +- exclusion from cross-agent failover; +- direct `ModelClient.chat()` rejection before transport; +- idempotent runtime-guard installation. ## References From db7b2095eba7e9dea7db90604389f9046ac0e752 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 14:12:47 +0900 Subject: [PATCH 13/89] fix: integrate chat capability checks explicitly --- contextual_orchestrator/__init__.py | 3 - contextual_orchestrator/chat_capability.py | 86 ------------------- contextual_orchestrator/orchestrator.py | 33 +++++-- tests/test_chat_model_capability_isolation.py | 12 --- 4 files changed, 26 insertions(+), 108 deletions(-) diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index dbf9e2ad0..99c3a9601 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -17,7 +17,6 @@ cheapest_upstream, heuristic_embedding, ) -from .chat_capability import install_runtime_chat_capability_guards from .cost_ledger import ( ATTRIBUTION_DIMENSIONS, AttributionDimensions, @@ -41,8 +40,6 @@ from .orchestrator import ModelAgent, ModelClient, TaskOrchestrator, WorkflowStep, load_agents from .token_counting import HeuristicTokenCounter, build_token_counter -install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) - __all__ = [ "ModelAgent", "TaskOrchestrator", diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index 98d55cf09..795ad81fb 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -9,10 +9,7 @@ from __future__ import annotations -from dataclasses import replace -from functools import wraps import re -from typing import Any _MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") _NON_CHAT_EXACT_TOKENS = frozenset( @@ -66,86 +63,3 @@ def is_chat_compatible_model_id(model_id: str) -> bool: if token.startswith(_NON_CHAT_TOKEN_PREFIXES): return False return True - - -def install_runtime_chat_capability_guards( - model_client_cls: type[Any], - task_orchestrator_cls: type[Any], -) -> None: - """Install idempotent fail-closed guards on the existing runtime classes. - - The project keeps runtime orchestration in one large compatibility module. - Installing the invariant here lets discovery, persisted configuration, plan - parsing, failover, and direct client calls share one classifier without - introducing a circular import from that compatibility module back into model - discovery. - """ - marker = "_chat_capability_guards_installed" - if getattr(task_orchestrator_cls, marker, False): - return - - original_chat = model_client_cls.chat - - @wraps(original_chat) - def guarded_chat(self: Any, agent: Any, *args: Any, **kwargs: Any) -> Any: - """Reject a non-chat endpoint before mock or network transport executes.""" - if not is_chat_compatible_model_id(agent.model): - raise ValueError("model is not chat-compatible and cannot serve a chat request") - return original_chat(self, agent, *args, **kwargs) - - model_client_cls.chat = guarded_chat - - original_ranked_agents = task_orchestrator_cls._ranked_agents - - @wraps(original_ranked_agents) - def guarded_ranked_agents(self: Any, text: str, role: str) -> list[Any]: - """Remove stale non-chat agents before role scoring or selection.""" - ranked = original_ranked_agents(self, text, role) - return [agent for agent in ranked if is_chat_compatible_model_id(agent.model)] - - task_orchestrator_cls._ranked_agents = guarded_ranked_agents - - original_select_agent = task_orchestrator_cls._select_agent - - @wraps(original_select_agent) - def guarded_select_agent(self: Any, text: str, role: str) -> Any: - """Fail with an actionable error when no chat-compatible agent remains.""" - if not self._ranked_agents(text, role): - raise RuntimeError(f"no chat-compatible agent available for role={role}") - return original_select_agent(self, text, role) - - task_orchestrator_cls._select_agent = guarded_select_agent - - original_failover_candidates = task_orchestrator_cls._failover_candidates - - @wraps(original_failover_candidates) - def guarded_failover_candidates( - self: Any, - primary: Any, - text: str, - role: str, - ) -> list[Any]: - """Prevent cross-agent retry from falling through to another endpoint family.""" - candidates = original_failover_candidates(self, primary, text, role) - return [agent for agent in candidates if is_chat_compatible_model_id(agent.model)] - - task_orchestrator_cls._failover_candidates = guarded_failover_candidates - - original_parse_workflow_plan = task_orchestrator_cls._parse_workflow_plan - - @wraps(original_parse_workflow_plan) - def guarded_parse_workflow_plan(self: Any, raw: str) -> list[Any]: - """Reselect any stale non-chat assignment named by a generated plan.""" - steps = original_parse_workflow_plan(self, raw) - agents_by_id = {agent.id: agent for agent in self.agents} - guarded_steps: list[Any] = [] - for step in steps: - assigned = agents_by_id[step.agent_id] - if not is_chat_compatible_model_id(assigned.model): - replacement = self._select_agent(step.subtask, step.role) - step = replace(step, agent_id=replacement.id) - guarded_steps.append(step) - return guarded_steps - - task_orchestrator_cls._parse_workflow_plan = guarded_parse_workflow_plan - setattr(task_orchestrator_cls, marker, True) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index f353bae5e..45b60612c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -31,6 +31,7 @@ import urllib.error import urllib.request +from .chat_capability import is_chat_compatible_model_id from .conventions import require_object_name from .credentials import NotConfigured, get_credential @@ -776,6 +777,8 @@ def chat( ``default_top_p`` are used so request-scoped Completions sampling can be applied without threading kwargs through every orchestrator hop. """ + if not is_chat_compatible_model_id(agent.model): + raise ValueError("model is not chat-compatible and cannot serve a chat request") self._local.usage = None # Expose the effective sampling knobs for request-path tests / diagnostics. effective_temperature = self.default_temperature if temperature is None else temperature @@ -2459,7 +2462,7 @@ def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: raw_steps = data.get("steps") if not isinstance(raw_steps, list) or not (2 <= len(raw_steps) <= self.policy.max_workflow_steps): raise ValueError(f"plan must have 2..{self.policy.max_workflow_steps} steps") - known_agents = {agent.id for agent in self.agents} + known_agents = {agent.id: agent for agent in self.agents} steps: list[WorkflowStep] = [] for index, item in enumerate(raw_steps): if int(item.get("id", -1)) != index: @@ -2474,8 +2477,9 @@ def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: if any(value < 0 or value >= index for value in access): raise ValueError("access may reference only earlier steps") agent_id = item.get("agent_id") - if agent_id not in known_agents: - # The planner named an unknown agent: reselect honestly instead of failing the plan. + assigned = known_agents.get(agent_id) + if assigned is None or not is_chat_compatible_model_id(assigned.model): + # Unknown or stale non-chat assignments are reselected honestly. agent_id = self._select_agent(subtask, role).id steps.append(WorkflowStep(index, role, agent_id, subtask, access)) if steps[-1].role not in {"synthesizer", "worker"}: @@ -2509,10 +2513,21 @@ def _score_agent(self, agent: ModelAgent, role: str, lowered: str) -> tuple[int, def _ranked_agents(self, text: str, role: str) -> list[ModelAgent]: """Agents sorted best-first for a role; the head is the primary, the tail are failovers.""" lowered = text.lower() - return sorted(self.agents, key=lambda agent: self._score_agent(agent, role, lowered), reverse=True) + return [ + agent + for agent in sorted( + self.agents, + key=lambda agent: self._score_agent(agent, role, lowered), + reverse=True, + ) + if is_chat_compatible_model_id(agent.model) + ] def _select_agent(self, text: str, role: str) -> ModelAgent: - selected = self._ranked_agents(text, role)[0] + ranked = self._ranked_agents(text, role) + if not ranked: + raise RuntimeError(f"no chat-compatible agent available for role={role}") + selected = ranked[0] if selected.disabled: # pragma: no cover raise RuntimeError(f"no enabled agent available for role={role}") if role in selected.provider_exclusions: # pragma: no cover @@ -2545,11 +2560,15 @@ def _invoke( def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]: ranked = self._ranked_agents(text, role) - ordered = [primary] + [agent for agent in ranked if agent.id != primary.id] + ordered = [ + agent + for agent in [primary] + [agent for agent in ranked if agent.id != primary.id] + if is_chat_compatible_model_id(agent.model) + ] eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions] healthy = [agent for agent in eligible if not self._circuit_open(agent.id)] # If every eligible agent is circuit-open, still probe them rather than fail with no attempt. - return healthy or eligible or [primary] + return healthy or eligible def _circuit_open(self, agent_id: str) -> bool: with self._circuit_lock: diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index fc1cd024e..f55bd5725 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -12,9 +12,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.chat_capability import ( # noqa: E402 - install_runtime_chat_capability_guards, -) from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, register_credential, @@ -146,15 +143,6 @@ def test_chat_compatibility_normalizes_identifiers( assert is_chat_compatible_model_id(model_id) is expected # type: ignore[arg-type] -def test_runtime_guard_installation_is_idempotent() -> None: - """Re-importing package initialization must not stack method wrappers.""" - before = ModelClient.chat - - install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) - - assert ModelClient.chat is before - - def test_bytez_chat_catalog_still_rejects_non_chat_identifiers() -> None: """Apply the same boundary even when a provider accepts a chat task filter.""" register_credential("BYTEZ_API_KEY", "bytez-secret") From 7739ca0ee77eb2e31160c4bfe0840f276ddc0add Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 14:14:58 +0900 Subject: [PATCH 14/89] docs: normalize capability incident formatting --- docs/doctoring/embedding-chat-capability-isolation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 02312f61e..182878017 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -1,7 +1,7 @@ # Embedding-to-chat capability isolation incident -**Status:** Accepted incident decision -**Date:** 2026-08-20 +**Status:** Accepted incident decision +**Date:** 2026-08-20 **Affected consumer:** LineageWeave buyer-surface stack around PR #260 ## Incident From 35718690dcbe638f62f3f073bd4ffbdbb3701478 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:17:40 -0700 Subject: [PATCH 15/89] ci: repair chat capability guards with source integration --- ...r-chat-capability-explicit-integration.yml | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 .github/workflows/repair-chat-capability-explicit-integration.yml diff --git a/.github/workflows/repair-chat-capability-explicit-integration.yml b/.github/workflows/repair-chat-capability-explicit-integration.yml new file mode 100644 index 000000000..07adf118b --- /dev/null +++ b/.github/workflows/repair-chat-capability-explicit-integration.yml @@ -0,0 +1,320 @@ +name: Repair chat capability source integration + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + +permissions: + contents: write + +concurrency: + group: repair-chat-capability-explicit-integration + cancel-in-progress: false + +jobs: + repair: + name: Rebase, patch, verify, and self-delete + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - name: Checkout exact branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Rebase work onto current main without publishing an unverified commit + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git fetch origin main + git merge --no-commit --no-ff origin/main + + - name: Replace import-time monkeypatches with explicit runtime guards + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + + orchestrator = replace_once( + orchestrator, + "from .conventions import require_object_name\nfrom .credentials import NotConfigured, get_credential\n", + "from .chat_capability import is_chat_compatible_model_id\nfrom .conventions import require_object_name\nfrom .credentials import NotConfigured, get_credential\n", + "chat capability import", + ) + + orchestrator = replace_once( + orchestrator, + " self._local.usage = None\n # Expose the effective sampling knobs for request-path tests / diagnostics.\n", + " if not is_chat_compatible_model_id(agent.model):\n" + " raise ValueError(\n" + " f\"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}\"\n" + " )\n" + " self._local.usage = None\n" + " # Expose the effective sampling knobs for request-path tests / diagnostics.\n", + "ModelClient.chat guard", + ) + + orchestrator = replace_once( + orchestrator, + " answer in fixed chunks so behavior shape stays testable and unchanged.\n" + " \"\"\"\n" + " if agent.base_url.startswith(\"mock://\"):\n", + " answer in fixed chunks so behavior shape stays testable and unchanged.\n" + " \"\"\"\n" + " if not is_chat_compatible_model_id(agent.model):\n" + " raise ValueError(\n" + " f\"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}\"\n" + " )\n" + " if agent.base_url.startswith(\"mock://\"):\n", + "ModelClient.stream_chat guard", + ) + + orchestrator = replace_once( + orchestrator, + " for agent in self.agents\n )\n system = (\n", + " for agent in self.agents\n" + " if is_chat_compatible_model_id(agent.model)\n" + " )\n" + " system = (\n", + "generated planner pool guard", + ) + + orchestrator = replace_once( + orchestrator, + " known_agents = {agent.id for agent in self.agents}\n", + " known_agents = {agent.id: agent for agent in self.agents}\n", + "workflow known-agent map", + ) + + orchestrator = replace_once( + orchestrator, + " agent_id = item.get(\"agent_id\")\n" + " if agent_id not in known_agents:\n" + " # The planner named an unknown agent: reselect honestly instead of failing the plan.\n" + " agent_id = self._select_agent(subtask, role).id\n", + " agent_id = item.get(\"agent_id\")\n" + " assigned_agent = known_agents.get(agent_id)\n" + " if assigned_agent is None or not is_chat_compatible_model_id(assigned_agent.model):\n" + " # Unknown or endpoint-incompatible assignments are never trusted.\n" + " agent_id = self._select_agent(subtask, role).id\n", + "generated workflow assignment guard", + ) + + orchestrator = replace_once( + orchestrator, + " lowered = text.lower()\n" + " return sorted(self.agents, key=lambda agent: self._score_agent(agent, role, lowered), reverse=True)\n", + " lowered = text.lower()\n" + " eligible = [\n" + " agent\n" + " for agent in self.agents\n" + " if is_chat_compatible_model_id(agent.model)\n" + " ]\n" + " return sorted(\n" + " eligible,\n" + " key=lambda agent: self._score_agent(agent, role, lowered),\n" + " reverse=True,\n" + " )\n", + "runtime ranking guard", + ) + + orchestrator = replace_once( + orchestrator, + " def _select_agent(self, text: str, role: str) -> ModelAgent:\n" + " selected = self._ranked_agents(text, role)[0]\n", + " def _select_agent(self, text: str, role: str) -> ModelAgent:\n" + " ranked = self._ranked_agents(text, role)\n" + " if not ranked:\n" + " raise RuntimeError(f\"no chat-compatible agent available for role={role}\")\n" + " selected = ranked[0]\n", + "runtime selection fail-closed guard", + ) + + orchestrator = replace_once( + orchestrator, + " def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]:\n" + " ranked = self._ranked_agents(text, role)\n" + " ordered = [primary] + [agent for agent in ranked if agent.id != primary.id]\n" + " eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions]\n" + " healthy = [agent for agent in eligible if not self._circuit_open(agent.id)]\n" + " # If every eligible agent is circuit-open, still probe them rather than fail with no attempt.\n" + " return healthy or eligible or [primary]\n", + " def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]:\n" + " ranked = self._ranked_agents(text, role)\n" + " primary_candidates = (\n" + " [primary] if is_chat_compatible_model_id(primary.model) else []\n" + " )\n" + " ordered = primary_candidates + [\n" + " agent for agent in ranked if agent.id != primary.id\n" + " ]\n" + " eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions]\n" + " healthy = [agent for agent in eligible if not self._circuit_open(agent.id)]\n" + " # If every eligible agent is circuit-open, still probe compatible agents.\n" + " return healthy or eligible\n", + "runtime failover guard", + ) + + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + Path("contextual_orchestrator/chat_capability.py").write_text( + '''"""Dependency-neutral chat endpoint capability classification. + +Provider model catalogs are heterogeneous: one registry can contain chat, +embedding, reranking, transcription, moderation, image, audio, and realtime +endpoint families. This module supplies the conservative negative classifier used +by discovery and runtime orchestration without importing either subsystem. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_NON_CHAT_EXACT_TOKENS = frozenset( + { + "audio", + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "guard", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "safety", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_NON_CHAT_TOKEN_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether a model identifier can enter an ordinary chat-agent pool. + + Only identifiers that clearly advertise a non-chat endpoint family are + rejected. Unknown identifiers stay eligible until authenticated provider + metadata or a measured endpoint probe establishes a stronger capability. + """ + if type(model_id) is not str: + return False + tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + if not tokens: + return False + for token in tokens: + if token in _NON_CHAT_EXACT_TOKENS: + return False + if token.startswith(_NON_CHAT_TOKEN_PREFIXES): + return False + return True +''', + encoding="utf-8", + ) + + init_path = Path("contextual_orchestrator/__init__.py") + init_text = init_path.read_text(encoding="utf-8") + init_text = replace_once( + init_text, + "from .chat_capability import install_runtime_chat_capability_guards\n", + "", + "remove runtime installer import", + ) + init_text = replace_once( + init_text, + "\ninstall_runtime_chat_capability_guards(ModelClient, TaskOrchestrator)\n", + "", + "remove import-time runtime mutation", + ) + init_path.write_text(init_text, encoding="utf-8") + + tests_path = Path("tests/test_chat_model_capability_isolation.py") + tests = tests_path.read_text(encoding="utf-8") + tests += ''' + + +def test_non_chat_primary_fails_over_only_to_chat_agents() -> None: + """Drop an incompatible primary while retaining a compatible fallback.""" + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + chat_agent = _agent("chat_agent", "gpt-5.2") + orchestrator = TaskOrchestrator([embedding_agent, chat_agent]) + + candidates = orchestrator._failover_candidates( + embedding_agent, + "Produce the final answer.", + "synthesizer", + ) + + assert candidates == [chat_agent] + + +def test_streaming_client_rejects_non_chat_model_before_transport() -> None: + """Apply the same endpoint boundary to streaming chat requests.""" + client = ModelClient() + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + + with pytest.raises(ValueError, match="chat-compatible"): + next( + client.stream_chat( + embedding_agent, + [{"role": "user", "content": "Produce the final answer."}], + ) + ) +''' + tests_path.write_text(tests, encoding="utf-8") + PY + + rm .github/workflows/repair-chat-capability-explicit-integration.yml + + - name: Install hash-locked test dependencies + run: python -m pip install --require-hashes -r fuzz/requirements-property.txt + + - name: Verify source integration and full regression suite + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_model_capability_isolation.py + python -m pytest -q + + - name: Commit verified repair and current main merge + shell: bash + run: | + set -euo pipefail + git add -A + git diff --cached --check + git commit -m "fix: integrate chat capability guards explicitly" + git push origin HEAD:fix/chat-capability-isolation-embedding-models From 14571a5b2d3f2ca6bff665afd2f6734a8a1e242f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:19:13 -0700 Subject: [PATCH 16/89] ci: remove completed one-shot repair workflow --- ...r-chat-capability-explicit-integration.yml | 320 ------------------ 1 file changed, 320 deletions(-) delete mode 100644 .github/workflows/repair-chat-capability-explicit-integration.yml diff --git a/.github/workflows/repair-chat-capability-explicit-integration.yml b/.github/workflows/repair-chat-capability-explicit-integration.yml deleted file mode 100644 index 07adf118b..000000000 --- a/.github/workflows/repair-chat-capability-explicit-integration.yml +++ /dev/null @@ -1,320 +0,0 @@ -name: Repair chat capability source integration - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - -permissions: - contents: write - -concurrency: - group: repair-chat-capability-explicit-integration - cancel-in-progress: false - -jobs: - repair: - name: Rebase, patch, verify, and self-delete - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - name: Checkout exact branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: true - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Rebase work onto current main without publishing an unverified commit - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git fetch origin main - git merge --no-commit --no-ff origin/main - - - name: Replace import-time monkeypatches with explicit runtime guards - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - - orchestrator = replace_once( - orchestrator, - "from .conventions import require_object_name\nfrom .credentials import NotConfigured, get_credential\n", - "from .chat_capability import is_chat_compatible_model_id\nfrom .conventions import require_object_name\nfrom .credentials import NotConfigured, get_credential\n", - "chat capability import", - ) - - orchestrator = replace_once( - orchestrator, - " self._local.usage = None\n # Expose the effective sampling knobs for request-path tests / diagnostics.\n", - " if not is_chat_compatible_model_id(agent.model):\n" - " raise ValueError(\n" - " f\"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}\"\n" - " )\n" - " self._local.usage = None\n" - " # Expose the effective sampling knobs for request-path tests / diagnostics.\n", - "ModelClient.chat guard", - ) - - orchestrator = replace_once( - orchestrator, - " answer in fixed chunks so behavior shape stays testable and unchanged.\n" - " \"\"\"\n" - " if agent.base_url.startswith(\"mock://\"):\n", - " answer in fixed chunks so behavior shape stays testable and unchanged.\n" - " \"\"\"\n" - " if not is_chat_compatible_model_id(agent.model):\n" - " raise ValueError(\n" - " f\"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}\"\n" - " )\n" - " if agent.base_url.startswith(\"mock://\"):\n", - "ModelClient.stream_chat guard", - ) - - orchestrator = replace_once( - orchestrator, - " for agent in self.agents\n )\n system = (\n", - " for agent in self.agents\n" - " if is_chat_compatible_model_id(agent.model)\n" - " )\n" - " system = (\n", - "generated planner pool guard", - ) - - orchestrator = replace_once( - orchestrator, - " known_agents = {agent.id for agent in self.agents}\n", - " known_agents = {agent.id: agent for agent in self.agents}\n", - "workflow known-agent map", - ) - - orchestrator = replace_once( - orchestrator, - " agent_id = item.get(\"agent_id\")\n" - " if agent_id not in known_agents:\n" - " # The planner named an unknown agent: reselect honestly instead of failing the plan.\n" - " agent_id = self._select_agent(subtask, role).id\n", - " agent_id = item.get(\"agent_id\")\n" - " assigned_agent = known_agents.get(agent_id)\n" - " if assigned_agent is None or not is_chat_compatible_model_id(assigned_agent.model):\n" - " # Unknown or endpoint-incompatible assignments are never trusted.\n" - " agent_id = self._select_agent(subtask, role).id\n", - "generated workflow assignment guard", - ) - - orchestrator = replace_once( - orchestrator, - " lowered = text.lower()\n" - " return sorted(self.agents, key=lambda agent: self._score_agent(agent, role, lowered), reverse=True)\n", - " lowered = text.lower()\n" - " eligible = [\n" - " agent\n" - " for agent in self.agents\n" - " if is_chat_compatible_model_id(agent.model)\n" - " ]\n" - " return sorted(\n" - " eligible,\n" - " key=lambda agent: self._score_agent(agent, role, lowered),\n" - " reverse=True,\n" - " )\n", - "runtime ranking guard", - ) - - orchestrator = replace_once( - orchestrator, - " def _select_agent(self, text: str, role: str) -> ModelAgent:\n" - " selected = self._ranked_agents(text, role)[0]\n", - " def _select_agent(self, text: str, role: str) -> ModelAgent:\n" - " ranked = self._ranked_agents(text, role)\n" - " if not ranked:\n" - " raise RuntimeError(f\"no chat-compatible agent available for role={role}\")\n" - " selected = ranked[0]\n", - "runtime selection fail-closed guard", - ) - - orchestrator = replace_once( - orchestrator, - " def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]:\n" - " ranked = self._ranked_agents(text, role)\n" - " ordered = [primary] + [agent for agent in ranked if agent.id != primary.id]\n" - " eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions]\n" - " healthy = [agent for agent in eligible if not self._circuit_open(agent.id)]\n" - " # If every eligible agent is circuit-open, still probe them rather than fail with no attempt.\n" - " return healthy or eligible or [primary]\n", - " def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]:\n" - " ranked = self._ranked_agents(text, role)\n" - " primary_candidates = (\n" - " [primary] if is_chat_compatible_model_id(primary.model) else []\n" - " )\n" - " ordered = primary_candidates + [\n" - " agent for agent in ranked if agent.id != primary.id\n" - " ]\n" - " eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions]\n" - " healthy = [agent for agent in eligible if not self._circuit_open(agent.id)]\n" - " # If every eligible agent is circuit-open, still probe compatible agents.\n" - " return healthy or eligible\n", - "runtime failover guard", - ) - - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - Path("contextual_orchestrator/chat_capability.py").write_text( - '''"""Dependency-neutral chat endpoint capability classification. - -Provider model catalogs are heterogeneous: one registry can contain chat, -embedding, reranking, transcription, moderation, image, audio, and realtime -endpoint families. This module supplies the conservative negative classifier used -by discovery and runtime orchestration without importing either subsystem. -""" - -from __future__ import annotations - -import re - -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_NON_CHAT_EXACT_TOKENS = frozenset( - { - "audio", - "bge", - "e5", - "embed", - "embedding", - "embeddings", - "guard", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "safety", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_NON_CHAT_TOKEN_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - - -def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether a model identifier can enter an ordinary chat-agent pool. - - Only identifiers that clearly advertise a non-chat endpoint family are - rejected. Unknown identifiers stay eligible until authenticated provider - metadata or a measured endpoint probe establishes a stronger capability. - """ - if type(model_id) is not str: - return False - tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - if not tokens: - return False - for token in tokens: - if token in _NON_CHAT_EXACT_TOKENS: - return False - if token.startswith(_NON_CHAT_TOKEN_PREFIXES): - return False - return True -''', - encoding="utf-8", - ) - - init_path = Path("contextual_orchestrator/__init__.py") - init_text = init_path.read_text(encoding="utf-8") - init_text = replace_once( - init_text, - "from .chat_capability import install_runtime_chat_capability_guards\n", - "", - "remove runtime installer import", - ) - init_text = replace_once( - init_text, - "\ninstall_runtime_chat_capability_guards(ModelClient, TaskOrchestrator)\n", - "", - "remove import-time runtime mutation", - ) - init_path.write_text(init_text, encoding="utf-8") - - tests_path = Path("tests/test_chat_model_capability_isolation.py") - tests = tests_path.read_text(encoding="utf-8") - tests += ''' - - -def test_non_chat_primary_fails_over_only_to_chat_agents() -> None: - """Drop an incompatible primary while retaining a compatible fallback.""" - embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") - chat_agent = _agent("chat_agent", "gpt-5.2") - orchestrator = TaskOrchestrator([embedding_agent, chat_agent]) - - candidates = orchestrator._failover_candidates( - embedding_agent, - "Produce the final answer.", - "synthesizer", - ) - - assert candidates == [chat_agent] - - -def test_streaming_client_rejects_non_chat_model_before_transport() -> None: - """Apply the same endpoint boundary to streaming chat requests.""" - client = ModelClient() - embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") - - with pytest.raises(ValueError, match="chat-compatible"): - next( - client.stream_chat( - embedding_agent, - [{"role": "user", "content": "Produce the final answer."}], - ) - ) -''' - tests_path.write_text(tests, encoding="utf-8") - PY - - rm .github/workflows/repair-chat-capability-explicit-integration.yml - - - name: Install hash-locked test dependencies - run: python -m pip install --require-hashes -r fuzz/requirements-property.txt - - - name: Verify source integration and full regression suite - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_model_capability_isolation.py - python -m pytest -q - - - name: Commit verified repair and current main merge - shell: bash - run: | - set -euo pipefail - git add -A - git diff --cached --check - git commit -m "fix: integrate chat capability guards explicitly" - git push origin HEAD:fix/chat-capability-isolation-embedding-models From 480d6efea618d694199b55fe2cb7fb91b9807632 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:32:50 -0700 Subject: [PATCH 17/89] test: reproduce embedding deployment selected for chat synthesis --- tests/test_chat_model_capability_isolation.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/test_chat_model_capability_isolation.py diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py new file mode 100644 index 000000000..9047249c0 --- /dev/null +++ b/tests/test_chat_model_capability_isolation.py @@ -0,0 +1,83 @@ +"""Regression coverage for isolating non-chat models from chat agent discovery.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + register_credential, + set_backend, +) +from contextual_orchestrator.model_discovery import ( # noqa: E402 + ProviderModelSource, + discover_provider_models, +) + + +class _Response: + """Small context-managed HTTP response used by the offline regression.""" + + def __init__(self, payload: dict[str, object]) -> None: + self._body = json.dumps(payload).encode("utf-8") + + def __enter__(self) -> "_Response": + return self + + def __exit__(self, *_args: object) -> bool: + return False + + def read(self, _size: int = -1) -> bytes: + return self._body + + +@pytest.fixture(autouse=True) +def _fresh_credential_backend(): + """Keep the provider credential registry isolated between tests.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: + """Exclude the exact Azure embedding deployment seen in synthesis alerts.""" + register_credential("GATEWAY_API_KEY", "gateway-secret") + source = ProviderModelSource( + provider_name="enterprise_gateway", + credential_name="GATEWAY_API_KEY", + list_url="https://gateway.example.test/v1/models", + chat_base_url="https://gateway.example.test/v1", + ) + payload = { + "data": [ + {"id": "azure/text-embedding-3-large"}, + {"id": "text_embedding_3_large"}, + {"id": "BAAI/bge-m3"}, + {"id": "openai/whisper-1"}, + {"id": "gpt-4o-mini-transcribe"}, + {"id": "text-moderation-latest"}, + {"id": "company/reranker-v2"}, + {"id": "gpt-5.2"}, + {"id": "qwen/qwen3-235b-a22b-instruct"}, + ] + } + + with patch( + "contextual_orchestrator.model_discovery.urllib.request.urlopen", + return_value=_Response(payload), + ): + discovered = discover_provider_models(source) + + assert [model.model_id for model in discovered] == [ + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ] From 54b4403eb4d24c14fc670090410237716171442c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:36:23 -0700 Subject: [PATCH 18/89] fix: isolate chat discovery from embedding endpoints --- contextual_orchestrator/model_discovery.py | 110 +++++++++++++++++---- 1 file changed, 90 insertions(+), 20 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index bd76100c0..bae52cc54 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -1,4 +1,4 @@ -"""Provider model-list discovery: turns registered KV credentials into agent candidates. +"""Provider model-list discovery for chat-agent candidates. Queries each configured provider's model-list endpoint over its OpenAI-compatible (or provider-specific) discovery API and returns :class:`DiscoveredModel` rows that @@ -9,6 +9,12 @@ (the KV registry), and a provider with nothing registered is silently skipped so registering a subset of the five supported keys still works. Stdlib only (``urllib.request``), matching this repo's dependency-free transport convention. + +This module owns the ordinary chat-agent discovery boundary. Provider catalogs may +mix chat, embedding, reranking, transcription, moderation, image, and realtime +models under one ``/models`` endpoint. Clearly non-chat identifiers are rejected +before they can be converted to workers, selected by cost, or persisted into the +chat agent pool. """ from __future__ import annotations @@ -29,6 +35,39 @@ DISCOVERY_TIMEOUT_SECONDS = 15.0 +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_NON_CHAT_EXACT_TOKENS = frozenset( + { + "audio", + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "guard", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "safety", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_NON_CHAT_TOKEN_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + @dataclass(frozen=True) class ProviderModelSource: @@ -84,7 +123,7 @@ class ProviderModelSource: @dataclass(frozen=True) class DiscoveredModel: - """One model found on a provider, with pricing when the provider reports it.""" + """One chat-compatible model found on a provider, with reported pricing.""" provider_name: str model_id: str @@ -104,6 +143,32 @@ def __init__(self, provider_name: str, detail: str) -> None: super().__init__(f"model discovery failed for provider {provider_name!r}: {detail}") +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether a model identifier is eligible for the chat-agent pool. + + The classifier is intentionally conservative and only rejects identifiers + that clearly advertise a non-chat endpoint family. It normalizes provider + prefixes and common separators so values such as + ``azure/text-embedding-3-large`` and ``text_embedding_3_large`` cannot be + assigned to thinker, worker, verifier, or synthesizer roles. + + Unknown identifiers remain eligible until explicit provider capability + metadata is available; this avoids fabricating positive reasoning or tool + capabilities from a name while still preserving ordinary chat discovery. + """ + if type(model_id) is not str: + return False + tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + if not tokens: + return False + for token in tokens: + if token in _NON_CHAT_EXACT_TOKENS: + return False + if token.startswith(_NON_CHAT_TOKEN_PREFIXES): + return False + return True + + def _fetch_json(url: str, *, api_key: str, auth_scheme: str, timeout: float) -> Any: if not url.startswith("https://"): # Every caller passes one of the hardcoded PROVIDER_SOURCES chat_base_url @@ -132,13 +197,14 @@ def _price_per_1k(value: Any) -> float | None: def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: + """Parse one OpenAI-compatible catalog into chat-compatible candidates.""" rows = payload.get("data") if isinstance(payload, dict) else None discovered: list[DiscoveredModel] = [] for row in rows if isinstance(rows, list) else []: if not isinstance(row, dict): continue model_id = row.get("id") - if type(model_id) is not str or not model_id: + if not is_chat_compatible_model_id(model_id): continue pricing = row.get("pricing") if isinstance(row.get("pricing"), dict) else {} discovered.append( @@ -156,13 +222,14 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: + """Parse one Bytez chat catalog without admitting non-chat identifiers.""" rows = payload.get("output") if isinstance(payload, dict) else None discovered: list[DiscoveredModel] = [] for row in rows if isinstance(rows, list) else []: if not isinstance(row, dict): continue model_id = row.get("modelId") - if type(model_id) is not str or not model_id: + if not is_chat_compatible_model_id(model_id): continue discovered.append( DiscoveredModel( @@ -181,7 +248,7 @@ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredMo def discover_provider_models( source: ProviderModelSource, *, timeout: float = DISCOVERY_TIMEOUT_SECONDS ) -> list[DiscoveredModel]: - """Discover one provider's models, or ``[]`` if its credential is not registered.""" + """Discover chat candidates, or ``[]`` when the credential is not registered.""" api_key = get_credential(source.credential_name) if not api_key: return [] @@ -202,7 +269,7 @@ def discover_all_models( *, timeout: float = DISCOVERY_TIMEOUT_SECONDS, ) -> tuple[list[DiscoveredModel], list[ProviderDiscoveryError]]: - """Discover models across every provider with a registered credential. + """Discover chat candidates across providers with registered credentials. One provider's failure never blocks the others: errors are collected and returned alongside whatever models were successfully discovered. @@ -231,7 +298,9 @@ def agent_id_for(discovered: DiscoveredModel) -> str: def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> ModelAgent: - """Build a disabled-by-default ModelAgent for a discovered model (opt-in serving).""" + """Build a disabled chat ModelAgent or reject a non-chat discovery record.""" + if not is_chat_compatible_model_id(discovered.model_id): + raise ValueError("non-chat model cannot be converted into a chat agent") return ModelAgent( id=agent_id_for(discovered), model=discovered.model_id, @@ -246,7 +315,7 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook") -> int: - """Write every discovered model's known pricing into the price book. + """Write every discovered chat model's known pricing into the price book. Returns the number of price rows written. A model without provider-reported pricing is skipped rather than defaulted to 0 -- an unpriced model already @@ -257,6 +326,8 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook written = 0 for model in discovered: + if not is_chat_compatible_model_id(model.model_id): + continue if model.prompt_price_per_1k is None and model.completion_price_per_1k is None: continue price_book.set_price( @@ -275,7 +346,7 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook def select_cheapest_discovered_agent( discovered: list[DiscoveredModel], price_book: "PriceBook" ) -> DiscoveredModel | None: - """Pick the lowest-cost discovered model per the price book (auto-optimization). + """Pick the lowest-cost chat-compatible model per the price book. Reuses :func:`~contextual_orchestrator.batch_routing.cheapest_upstream`, the existing cost-optimizing upstream selector. Call :func:`refresh_price_book` @@ -286,13 +357,14 @@ def select_cheapest_discovered_agent( "auto-pick something free to try," but callers doing real cost comparison should refresh pricing for every candidate they care about first. """ - if not discovered: + eligible = [model for model in discovered if is_chat_compatible_model_id(model.model_id)] + if not eligible: return None - candidates = [{"provider": model.provider_name, "model": model.model_id} for model in discovered] + candidates = [{"provider": model.provider_name, "model": model.model_id} for model in eligible] winner = cheapest_upstream(candidates, price_book) if winner is None: return None - for model in discovered: + for model in eligible: if model.provider_name == winner["provider"] and model.model_id == winner["model"]: return model return None # pragma: no cover - winner always comes from candidates @@ -301,17 +373,15 @@ def select_cheapest_discovered_agent( def select_top_n_cheapest_discovered_agents( discovered: list[DiscoveredModel], price_book: "PriceBook", limit: int ) -> list[DiscoveredModel]: - """Return the ``limit`` lowest-cost discovered models, cheapest first. - - For bootstrapping a CI sidecar (or any first-boot pool) with more than one - enabled agent for failover, without hand-picking which discovered models to - trust. Same pricing contract as :func:`select_cheapest_discovered_agent`. - """ - if limit <= 0 or not discovered: + """Return the ``limit`` cheapest chat-compatible models in ascending cost.""" + if limit <= 0: + return [] + eligible = [model for model in discovered if is_chat_compatible_model_id(model.model_id)] + if not eligible: return [] def _cost(model: DiscoveredModel) -> float: cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) return cost - return sorted(discovered, key=_cost)[:limit] + return sorted(eligible, key=_cost)[:limit] From e55740043c9e5f1072bf68988bbc9a31c16cfcf6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:37:26 -0700 Subject: [PATCH 19/89] test: cover chat capability isolation defenses --- tests/test_chat_model_capability_isolation.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index 9047249c0..46b653e2c 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -16,9 +16,17 @@ register_credential, set_backend, ) +from contextual_orchestrator.cost_ledger import PriceBook, PriceEntry # noqa: E402 +from contextual_orchestrator.kv_config import InMemoryConfigStore # noqa: E402 from contextual_orchestrator.model_discovery import ( # noqa: E402 + DiscoveredModel, ProviderModelSource, + agent_from_discovered, discover_provider_models, + is_chat_compatible_model_id, + refresh_price_book, + select_cheapest_discovered_agent, + select_top_n_cheapest_discovered_agents, ) @@ -48,6 +56,19 @@ def _fresh_credential_backend(): set_backend(None) +def _model(model_id: str, *, priced: bool = False) -> DiscoveredModel: + """Build one synthetic discovered model for capability-boundary tests.""" + return DiscoveredModel( + provider_name="enterprise_gateway", + model_id=model_id, + credential_name="GATEWAY_API_KEY", + chat_base_url="https://gateway.example.test/v1", + auth_scheme="Bearer", + prompt_price_per_1k=1.0 if priced else None, + completion_price_per_1k=1.0 if priced else None, + ) + + def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: """Exclude the exact Azure embedding deployment seen in synthesis alerts.""" register_credential("GATEWAY_API_KEY", "gateway-secret") @@ -81,3 +102,74 @@ def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: "gpt-5.2", "qwen/qwen3-235b-a22b-instruct", ] + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + (None, False), + ("", False), + ("---", False), + ("vendor/embeddingv2", False), + ("vendor/reranking-v2", False), + ("vendor/transcriber-v2", False), + ("gpt-5.2", True), + ("qwen/qwen3-instruct", True), + ], +) +def test_chat_compatibility_normalizes_identifiers( + model_id: object, expected: bool +) -> None: + """Normalize provider prefixes and separators without guessing chat features.""" + assert is_chat_compatible_model_id(model_id) is expected # type: ignore[arg-type] + + +def test_bytez_chat_catalog_still_rejects_non_chat_identifiers() -> None: + """Apply the same boundary even when a provider accepts a chat task filter.""" + register_credential("BYTEZ_API_KEY", "bytez-secret") + source = ProviderModelSource( + provider_name="bytez", + credential_name="BYTEZ_API_KEY", + list_url="https://api.bytez.com/models/v2/list/models", + chat_base_url="https://api.bytez.com/models/v2/openai/v1", + auth_scheme="Key", + style="bytez", + task_filter="chat", + ) + payload = { + "output": [ + {"modelId": "vendor/embeddingv2"}, + {"modelId": "vendor/chat-instruct"}, + ] + } + + with patch( + "contextual_orchestrator.model_discovery.urllib.request.urlopen", + return_value=_Response(payload), + ): + discovered = discover_provider_models(source) + + assert [model.model_id for model in discovered] == ["vendor/chat-instruct"] + + +def test_non_chat_discovery_cannot_be_converted_to_agent() -> None: + """Keep manually constructed discovery rows from bypassing the parser filter.""" + with pytest.raises(ValueError, match="non-chat model"): + agent_from_discovered(_model("azure/text-embedding-3-large")) + + +def test_non_chat_discovery_is_not_priced_or_selected_for_chat() -> None: + """Keep price routing from reintroducing an incompatible endpoint model.""" + price_book = PriceBook(InMemoryConfigStore()) + embedding_model = _model("azure/text-embedding-3-large", priced=True) + chat_model = _model("gpt-5.2", priced=True) + price_book.set_price(PriceEntry("enterprise_gateway", "gpt-5.2", 1.0, 1.0)) + + assert refresh_price_book([embedding_model, chat_model], price_book) == 1 + assert price_book.get_price( + "enterprise_gateway", "azure/text-embedding-3-large" + ) is None + assert select_cheapest_discovered_agent([embedding_model], price_book) is None + assert select_top_n_cheapest_discovered_agents( + [embedding_model], price_book, 1 + ) == [] From be2f191d6c3c3425c37a4f891719748fece57f55 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:38:09 -0700 Subject: [PATCH 20/89] docs: record embedding-to-chat incident boundary --- .../embedding-chat-capability-isolation.md | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 docs/doctoring/embedding-chat-capability-isolation.md diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md new file mode 100644 index 000000000..88cc334c7 --- /dev/null +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -0,0 +1,88 @@ +# Embedding-to-chat capability isolation incident + +**Status:** Accepted incident decision +**Date:** 2026-08-20 +**Affected consumer:** LineageWeave buyer-surface stack around PR #260 + +## Incident + +A conducted workflow reached the final contextual-orchestrator synthesizer with +`model_group=text-embedding-3-large` and deployment +`azure/text-embedding-3-large`. The gateway rejected the chat operation as +unsupported. Its configured fallback map contained chat-generation model groups, +but no fallback attached to the embedding group. + +The missing fallback was a symptom, not the causal defect. An embedding deployment +had already crossed the chat-agent capability boundary and become eligible for a +worker role. + +## Causal boundary + +Provider-compatible `/models` registries can contain multiple endpoint families. +The current discovery parser accepted every non-empty model identifier and exposed +it to agent creation, price selection, and durable pool synchronization. A catalog +row naming an embedding deployment could therefore be scored for thinker, worker, +verifier, or synthesizer work even though its serving endpoint accepts embedding +input rather than chat messages. + +OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately +from models supported by chat completions. Microsoft likewise demonstrates it with +`client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes +chat, responses, embeddings, image, audio, rerank, and other endpoint families as +distinct operations. A router fallback can choose another deployment for the same +operation; it cannot make an embedding deployment execute a chat operation. + +## Decision + +The ordinary model-discovery module is a **chat-agent discovery boundary**. + +1. Normalize provider prefixes and common separators in model identifiers. +2. Reject identifiers that clearly advertise embedding, reranking, transcription, + moderation/safety, image, audio/speech, realtime, or known embedding-family + transport semantics. +3. Apply the guard while parsing both OpenAI-compatible and Bytez catalogs. +4. Re-check the invariant before converting a discovery record to `ModelAgent`. +5. Re-check the invariant before writing chat pricing or running price-based chat + selection. +6. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, + or verification capabilities from their names. + +This is deliberately a conservative negative filter. A future capability registry +may replace name-based exclusion with authenticated provider metadata, measured +endpoint probes, and separate endpoint-specific pools. Until that evidence exists, +a clearly non-chat model fails closed at the chat boundary. + +## Rejected response + +Adding `text-embedding-3-large` to a chat fallback map is rejected. It would retain +the invalid primary assignment and merely hide it when a fallback happened to be +available. Repeated provider retries are also rejected because the request is +structurally unsupported, not transiently unavailable. + +## Residual operational action + +This change prevents new incompatible catalog rows from entering the chat pool. +A deployment that already persisted such a row must disable or withdraw that stale +discovered agent. The durable provider-bootstrap slice owns exact-set activation and +stale discovered-agent withdrawal; it must import this shared boundary when rebased. + +## Verification evidence + +`tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID +and provider/separator aliases. It also verifies: + +- OpenAI-compatible and Bytez catalog filtering; +- malformed and prefix-only identifier handling; +- agent-conversion rejection; +- exclusion from the price book and cheapest-agent selection. + +## References + +BerriAI. (2026). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. +https://docs.litellm.ai/ + +Microsoft. (2026). *How to switch between OpenAI and Azure OpenAI endpoints*. +Microsoft Learn. https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints + +OpenAI. (2026). *Data controls in the OpenAI platform: Default usage policies by +endpoint*. https://platform.openai.com/docs/models/default-usage-policies-by-endpoint From c1fc1ca0525b1eac2001aee053f9d139c47acb55 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:47:19 -0700 Subject: [PATCH 21/89] docs: normalize incident references to APA 7 --- .../embedding-chat-capability-isolation.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 88cc334c7..87bb44e97 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -78,11 +78,13 @@ and provider/separator aliases. It also verifies: ## References -BerriAI. (2026). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. -https://docs.litellm.ai/ +BerriAI. (n.d.). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. +Retrieved August 20, 2026, from https://docs.litellm.ai/ -Microsoft. (2026). *How to switch between OpenAI and Azure OpenAI endpoints*. -Microsoft Learn. https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints +Microsoft. (n.d.). *How to switch between OpenAI and Azure OpenAI endpoints*. +Microsoft Learn. Retrieved August 20, 2026, from +https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints -OpenAI. (2026). *Data controls in the OpenAI platform: Default usage policies by -endpoint*. https://platform.openai.com/docs/models/default-usage-policies-by-endpoint +OpenAI. (n.d.). *Data controls in the OpenAI platform: Default usage policies by +endpoint*. Retrieved August 20, 2026, from +https://platform.openai.com/docs/models/default-usage-policies-by-endpoint From 392ce6c4ebcec4e35ebcc5dfb1ff894b78d4cd9f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:51:36 -0700 Subject: [PATCH 22/89] test: reproduce stale embedding agent runtime selection --- tests/test_chat_model_capability_isolation.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index 46b653e2c..f55bd5725 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -11,6 +11,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, register_credential, @@ -28,6 +29,7 @@ select_cheapest_discovered_agent, select_top_n_cheapest_discovered_agents, ) +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 class _Response: @@ -69,6 +71,23 @@ def _model(model_id: str, *, priced: bool = False) -> DiscoveredModel: ) +def _agent( + agent_id: str, + model_id: str, + *, + priority: int = 0, + tags: tuple[str, ...] = ("writing",), +) -> ModelAgent: + """Build one mock-backed runtime agent for selection-path regressions.""" + return ModelAgent( + id=agent_id, + model=model_id, + base_url="mock://local", + priority=priority, + tags=tags, + ) + + def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: """Exclude the exact Azure embedding deployment seen in synthesis alerts.""" register_credential("GATEWAY_API_KEY", "gateway-secret") @@ -173,3 +192,92 @@ def test_non_chat_discovery_is_not_priced_or_selected_for_chat() -> None: assert select_top_n_cheapest_discovered_agents( [embedding_model], price_book, 1 ) == [] + + +def test_stale_embedding_agent_cannot_win_synthesizer_selection() -> None: + """Exclude an already-persisted embedding row even when it has high priority.""" + embedding_agent = _agent( + "embedding_agent", + "azure/text-embedding-3-large", + priority=10_000, + ) + chat_agent = _agent("chat_agent", "gpt-5.2") + orchestrator = TaskOrchestrator([embedding_agent, chat_agent]) + + assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent + + +def test_all_non_chat_agents_fail_before_synthesis() -> None: + """Fail closed when a stale pool contains no chat-compatible worker.""" + orchestrator = TaskOrchestrator( + [_agent("embedding_agent", "azure/text-embedding-3-large")] + ) + + with pytest.raises(RuntimeError, match="chat-compatible"): + orchestrator._select_agent("Produce the final answer.", "synthesizer") + + +def test_generated_plan_reselects_non_chat_agent_assignment() -> None: + """Do not trust a generated plan that names a stale embedding agent directly.""" + embedding_agent = _agent( + "embedding_agent", + "azure/text-embedding-3-large", + priority=10_000, + ) + chat_agent = _agent("chat_agent", "gpt-5.2") + orchestrator = TaskOrchestrator([embedding_agent, chat_agent]) + raw_plan = json.dumps( + { + "steps": [ + { + "id": 0, + "role": "worker", + "agent_id": "chat_agent", + "subtask": "Execute the task.", + "access": [], + }, + { + "id": 1, + "role": "synthesizer", + "agent_id": "embedding_agent", + "subtask": "Produce the final answer.", + "access": [0], + }, + ] + } + ) + + steps = orchestrator._parse_workflow_plan(raw_plan) + + assert steps[-1].agent_id == "chat_agent" + + +def test_failover_candidates_exclude_stale_embedding_agents() -> None: + """Keep cross-agent retry from falling through to an incompatible endpoint.""" + chat_agent = _agent("chat_agent", "gpt-5.2") + embedding_agent = _agent( + "embedding_agent", + "azure/text-embedding-3-large", + priority=10_000, + ) + orchestrator = TaskOrchestrator([chat_agent, embedding_agent]) + + candidates = orchestrator._failover_candidates( + chat_agent, + "Produce the final answer.", + "synthesizer", + ) + + assert candidates == [chat_agent] + + +def test_model_client_rejects_non_chat_model_before_mock_or_network_call() -> None: + """Keep the provider boundary fail-closed even when selection is bypassed.""" + client = ModelClient() + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + + with pytest.raises(ValueError, match="chat-compatible"): + client.chat( + embedding_agent, + [{"role": "user", "content": "Produce the final answer."}], + ) From 99f5c412633588740fab393f3ed9242efeaaf170 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:02 -0700 Subject: [PATCH 23/89] fix: enforce chat capability at runtime boundaries --- contextual_orchestrator/chat_capability.py | 151 +++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 contextual_orchestrator/chat_capability.py diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py new file mode 100644 index 000000000..567dc8234 --- /dev/null +++ b/contextual_orchestrator/chat_capability.py @@ -0,0 +1,151 @@ +"""Shared chat-capability classification and runtime fail-closed guards. + +Provider model catalogs are heterogeneous: a model identifier may name a chat +model, embedding model, reranker, transcription model, image model, or another +endpoint family. The helpers in this module prevent clearly non-chat model IDs +from crossing a chat-agent boundary even when an incompatible agent was persisted +before discovery filtering was introduced. +""" + +from __future__ import annotations + +from dataclasses import replace +from functools import wraps +import re +from typing import Any + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_NON_CHAT_EXACT_TOKENS = frozenset( + { + "audio", + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "guard", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "safety", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_NON_CHAT_TOKEN_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether a model identifier is eligible for a chat-agent pool. + + The classifier intentionally rejects only identifiers that clearly advertise + a non-chat endpoint family. Unknown identifiers remain eligible until an + authenticated capability registry can prove more specific endpoint support. + """ + if type(model_id) is not str: + return False + tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + if not tokens: + return False + for token in tokens: + if token in _NON_CHAT_EXACT_TOKENS: + return False + if token.startswith(_NON_CHAT_TOKEN_PREFIXES): + return False + return True + + +def install_runtime_chat_capability_guards( + model_client_cls: type[Any], + task_orchestrator_cls: type[Any], +) -> None: + """Install idempotent fail-closed guards on the existing runtime classes. + + The project keeps runtime orchestration in one large compatibility module. + Installing the invariant here lets discovery, persisted configuration, plan + parsing, failover, and direct client calls share one classifier without + introducing a circular import from that compatibility module back into model + discovery. + """ + marker = "_chat_capability_guards_installed" + if getattr(task_orchestrator_cls, marker, False): + return + + original_chat = model_client_cls.chat + + @wraps(original_chat) + def guarded_chat(self: Any, agent: Any, *args: Any, **kwargs: Any) -> Any: + """Reject a non-chat endpoint before mock or network transport executes.""" + if not is_chat_compatible_model_id(agent.model): + raise ValueError("model is not chat-compatible and cannot serve a chat request") + return original_chat(self, agent, *args, **kwargs) + + model_client_cls.chat = guarded_chat + + original_ranked_agents = task_orchestrator_cls._ranked_agents + + @wraps(original_ranked_agents) + def guarded_ranked_agents(self: Any, text: str, role: str) -> list[Any]: + """Remove stale non-chat agents before role scoring or selection.""" + ranked = original_ranked_agents(self, text, role) + return [agent for agent in ranked if is_chat_compatible_model_id(agent.model)] + + task_orchestrator_cls._ranked_agents = guarded_ranked_agents + + original_select_agent = task_orchestrator_cls._select_agent + + @wraps(original_select_agent) + def guarded_select_agent(self: Any, text: str, role: str) -> Any: + """Fail with an actionable error when no chat-compatible agent remains.""" + if not self._ranked_agents(text, role): + raise RuntimeError(f"no chat-compatible agent available for role={role}") + return original_select_agent(self, text, role) + + task_orchestrator_cls._select_agent = guarded_select_agent + + original_failover_candidates = task_orchestrator_cls._failover_candidates + + @wraps(original_failover_candidates) + def guarded_failover_candidates( + self: Any, + primary: Any, + text: str, + role: str, + ) -> list[Any]: + """Prevent cross-agent retry from falling through to another endpoint family.""" + candidates = original_failover_candidates(self, primary, text, role) + return [agent for agent in candidates if is_chat_compatible_model_id(agent.model)] + + task_orchestrator_cls._failover_candidates = guarded_failover_candidates + + original_parse_workflow_plan = task_orchestrator_cls._parse_workflow_plan + + @wraps(original_parse_workflow_plan) + def guarded_parse_workflow_plan(self: Any, raw: str) -> list[Any]: + """Reselect any stale non-chat assignment named by a generated plan.""" + steps = original_parse_workflow_plan(self, raw) + agents_by_id = {agent.id: agent for agent in self.agents} + guarded_steps: list[Any] = [] + for step in steps: + assigned = agents_by_id.get(step.agent_id) + if assigned is not None and not is_chat_compatible_model_id(assigned.model): + replacement = self._select_agent(step.subtask, step.role) + step = replace(step, agent_id=replacement.id) + guarded_steps.append(step) + return guarded_steps + + task_orchestrator_cls._parse_workflow_plan = guarded_parse_workflow_plan + setattr(task_orchestrator_cls, marker, True) From 136426de08461e6e73047a44627d900695404ab2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:36 -0700 Subject: [PATCH 24/89] refactor: share chat capability classifier --- contextual_orchestrator/model_discovery.py | 60 +--------------------- 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index bae52cc54..12f8f354c 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -27,6 +27,7 @@ from typing import TYPE_CHECKING, Any from .batch_routing import cheapest_upstream +from .chat_capability import is_chat_compatible_model_id from .credentials import get_credential from .orchestrator import ModelAgent @@ -35,39 +36,6 @@ DISCOVERY_TIMEOUT_SECONDS = 15.0 -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_NON_CHAT_EXACT_TOKENS = frozenset( - { - "audio", - "bge", - "e5", - "embed", - "embedding", - "embeddings", - "guard", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "safety", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_NON_CHAT_TOKEN_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - @dataclass(frozen=True) class ProviderModelSource: @@ -143,32 +111,6 @@ def __init__(self, provider_name: str, detail: str) -> None: super().__init__(f"model discovery failed for provider {provider_name!r}: {detail}") -def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether a model identifier is eligible for the chat-agent pool. - - The classifier is intentionally conservative and only rejects identifiers - that clearly advertise a non-chat endpoint family. It normalizes provider - prefixes and common separators so values such as - ``azure/text-embedding-3-large`` and ``text_embedding_3_large`` cannot be - assigned to thinker, worker, verifier, or synthesizer roles. - - Unknown identifiers remain eligible until explicit provider capability - metadata is available; this avoids fabricating positive reasoning or tool - capabilities from a name while still preserving ordinary chat discovery. - """ - if type(model_id) is not str: - return False - tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - if not tokens: - return False - for token in tokens: - if token in _NON_CHAT_EXACT_TOKENS: - return False - if token.startswith(_NON_CHAT_TOKEN_PREFIXES): - return False - return True - - def _fetch_json(url: str, *, api_key: str, auth_scheme: str, timeout: float) -> Any: if not url.startswith("https://"): # Every caller passes one of the hardcoded PROVIDER_SOURCES chat_base_url From 050b514688dfd0553a48cd3ea8990aa9847ce9d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:46 -0700 Subject: [PATCH 25/89] fix: activate runtime chat capability guards --- contextual_orchestrator/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index 70dbd71c6..dbf9e2ad0 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -17,6 +17,7 @@ cheapest_upstream, heuristic_embedding, ) +from .chat_capability import install_runtime_chat_capability_guards from .cost_ledger import ( ATTRIBUTION_DIMENSIONS, AttributionDimensions, @@ -37,9 +38,11 @@ from .cost_router import CostRoutingCoordinator from .credentials import NotConfigured, get_credential, register_credential from .kv_config import InMemoryConfigStore, get_config_store -from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents +from .orchestrator import ModelAgent, ModelClient, TaskOrchestrator, WorkflowStep, load_agents from .token_counting import HeuristicTokenCounter, build_token_counter +install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) + __all__ = [ "ModelAgent", "TaskOrchestrator", From fa980bc91bcf93e8e527598ed22bde389990e78d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:56:14 -0700 Subject: [PATCH 26/89] test: tighten runtime guard coverage surface --- contextual_orchestrator/chat_capability.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index 567dc8234..98d55cf09 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -2,7 +2,7 @@ Provider model catalogs are heterogeneous: a model identifier may name a chat model, embedding model, reranker, transcription model, image model, or another -endpoint family. The helpers in this module prevent clearly non-chat model IDs +endpoint family. The helpers in this module prevent clearly non-chat model IDs from crossing a chat-agent boundary even when an incompatible agent was persisted before discovery filtering was introduced. """ @@ -52,7 +52,7 @@ def is_chat_compatible_model_id(model_id: str) -> bool: """Return whether a model identifier is eligible for a chat-agent pool. The classifier intentionally rejects only identifiers that clearly advertise - a non-chat endpoint family. Unknown identifiers remain eligible until an + a non-chat endpoint family. Unknown identifiers remain eligible until an authenticated capability registry can prove more specific endpoint support. """ if type(model_id) is not str: @@ -140,8 +140,8 @@ def guarded_parse_workflow_plan(self: Any, raw: str) -> list[Any]: agents_by_id = {agent.id: agent for agent in self.agents} guarded_steps: list[Any] = [] for step in steps: - assigned = agents_by_id.get(step.agent_id) - if assigned is not None and not is_chat_compatible_model_id(assigned.model): + assigned = agents_by_id[step.agent_id] + if not is_chat_compatible_model_id(assigned.model): replacement = self._select_agent(step.subtask, step.role) step = replace(step, agent_id=replacement.id) guarded_steps.append(step) From 3998ad1f09d2a1711995ddd12f5b2d470c7cbb81 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:56:37 -0700 Subject: [PATCH 27/89] test: cover runtime guard installation idempotence --- tests/test_chat_model_capability_isolation.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index f55bd5725..fc1cd024e 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -12,6 +12,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.chat_capability import ( # noqa: E402 + install_runtime_chat_capability_guards, +) from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, register_credential, @@ -143,6 +146,15 @@ def test_chat_compatibility_normalizes_identifiers( assert is_chat_compatible_model_id(model_id) is expected # type: ignore[arg-type] +def test_runtime_guard_installation_is_idempotent() -> None: + """Re-importing package initialization must not stack method wrappers.""" + before = ModelClient.chat + + install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) + + assert ModelClient.chat is before + + def test_bytez_chat_catalog_still_rejects_non_chat_identifiers() -> None: """Apply the same boundary even when a provider accepts a chat task filter.""" register_credential("BYTEZ_API_KEY", "bytez-secret") From feeac9e2c25eec15abf168b591328b381de76321 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:57:28 -0700 Subject: [PATCH 28/89] docs: record stale-agent runtime containment --- .../embedding-chat-capability-isolation.md | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 87bb44e97..02312f61e 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -19,12 +19,19 @@ worker role. ## Causal boundary Provider-compatible `/models` registries can contain multiple endpoint families. -The current discovery parser accepted every non-empty model identifier and exposed +The original discovery parser accepted every non-empty model identifier and exposed it to agent creation, price selection, and durable pool synchronization. A catalog row naming an embedding deployment could therefore be scored for thinker, worker, verifier, or synthesizer work even though its serving endpoint accepts embedding input rather than chat messages. +The first incident fix closed discovery and price-routing boundaries, but further +root-cause tracing showed an already-persisted incompatible `ModelAgent` could still +survive that filter. The runtime ranking path, generated workflow assignment, +cross-agent failover, and direct `ModelClient.chat()` path previously trusted the +persisted model identifier. That stale-state path is sufficient to reproduce the +same unsupported Azure chat operation after a process restart or durable bootstrap. + OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately from models supported by chat completions. Microsoft likewise demonstrates it with `client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes @@ -34,7 +41,7 @@ operation; it cannot make an embedding deployment execute a chat operation. ## Decision -The ordinary model-discovery module is a **chat-agent discovery boundary**. +Chat capability is a **shared runtime invariant**, not only a discovery filter. 1. Normalize provider prefixes and common separators in model identifiers. 2. Reject identifiers that clearly advertise embedding, reranking, transcription, @@ -44,13 +51,18 @@ The ordinary model-discovery module is a **chat-agent discovery boundary**. 4. Re-check the invariant before converting a discovery record to `ModelAgent`. 5. Re-check the invariant before writing chat pricing or running price-based chat selection. -6. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, - or verification capabilities from their names. +6. Remove stale non-chat agents from thinker, worker, verifier, and synthesizer + ranking even if a durable configuration still contains them. +7. Reselect a generated workflow step that explicitly names a stale non-chat agent. +8. Remove non-chat agents from cross-agent failover candidates. +9. Reject a non-chat model at `ModelClient.chat()` before mock or network transport. +10. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, + or verification capabilities from their names. This is deliberately a conservative negative filter. A future capability registry may replace name-based exclusion with authenticated provider metadata, measured endpoint probes, and separate endpoint-specific pools. Until that evidence exists, -a clearly non-chat model fails closed at the chat boundary. +a clearly non-chat model fails closed at every chat boundary. ## Rejected response @@ -61,20 +73,28 @@ structurally unsupported, not transiently unavailable. ## Residual operational action -This change prevents new incompatible catalog rows from entering the chat pool. -A deployment that already persisted such a row must disable or withdraw that stale -discovered agent. The durable provider-bootstrap slice owns exact-set activation and -stale discovered-agent withdrawal; it must import this shared boundary when rebased. +Runtime containment means an already-persisted embedding agent can no longer win +chat selection or failover while stale data is being cleaned up. Durable state must +still converge to the correct exact set: the provider-bootstrap slice owns stale +discovered-agent withdrawal and must import the same shared classifier when rebased. +Runtime rejection is defense in depth, not a substitute for deleting invalid +persistent configuration. ## Verification evidence `tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID -and provider/separator aliases. It also verifies: +and provider/separator aliases. It verifies: - OpenAI-compatible and Bytez catalog filtering; - malformed and prefix-only identifier handling; - agent-conversion rejection; -- exclusion from the price book and cheapest-agent selection. +- exclusion from the price book and cheapest-agent selection; +- exclusion of a high-priority stale embedding agent from synthesizer selection; +- fail-closed behavior when the persisted pool contains only non-chat agents; +- generated-plan reassignment away from a stale embedding agent; +- exclusion from cross-agent failover; +- direct `ModelClient.chat()` rejection before transport; +- idempotent runtime-guard installation. ## References From 668cf8660c1ff5454316d02d9a1b9d67dcbb8e5e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 14:12:47 +0900 Subject: [PATCH 29/89] fix: integrate chat capability checks explicitly --- contextual_orchestrator/__init__.py | 3 - contextual_orchestrator/chat_capability.py | 86 ------------------- contextual_orchestrator/orchestrator.py | 33 +++++-- tests/test_chat_model_capability_isolation.py | 12 --- 4 files changed, 26 insertions(+), 108 deletions(-) diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index dbf9e2ad0..99c3a9601 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -17,7 +17,6 @@ cheapest_upstream, heuristic_embedding, ) -from .chat_capability import install_runtime_chat_capability_guards from .cost_ledger import ( ATTRIBUTION_DIMENSIONS, AttributionDimensions, @@ -41,8 +40,6 @@ from .orchestrator import ModelAgent, ModelClient, TaskOrchestrator, WorkflowStep, load_agents from .token_counting import HeuristicTokenCounter, build_token_counter -install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) - __all__ = [ "ModelAgent", "TaskOrchestrator", diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index 98d55cf09..795ad81fb 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -9,10 +9,7 @@ from __future__ import annotations -from dataclasses import replace -from functools import wraps import re -from typing import Any _MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") _NON_CHAT_EXACT_TOKENS = frozenset( @@ -66,86 +63,3 @@ def is_chat_compatible_model_id(model_id: str) -> bool: if token.startswith(_NON_CHAT_TOKEN_PREFIXES): return False return True - - -def install_runtime_chat_capability_guards( - model_client_cls: type[Any], - task_orchestrator_cls: type[Any], -) -> None: - """Install idempotent fail-closed guards on the existing runtime classes. - - The project keeps runtime orchestration in one large compatibility module. - Installing the invariant here lets discovery, persisted configuration, plan - parsing, failover, and direct client calls share one classifier without - introducing a circular import from that compatibility module back into model - discovery. - """ - marker = "_chat_capability_guards_installed" - if getattr(task_orchestrator_cls, marker, False): - return - - original_chat = model_client_cls.chat - - @wraps(original_chat) - def guarded_chat(self: Any, agent: Any, *args: Any, **kwargs: Any) -> Any: - """Reject a non-chat endpoint before mock or network transport executes.""" - if not is_chat_compatible_model_id(agent.model): - raise ValueError("model is not chat-compatible and cannot serve a chat request") - return original_chat(self, agent, *args, **kwargs) - - model_client_cls.chat = guarded_chat - - original_ranked_agents = task_orchestrator_cls._ranked_agents - - @wraps(original_ranked_agents) - def guarded_ranked_agents(self: Any, text: str, role: str) -> list[Any]: - """Remove stale non-chat agents before role scoring or selection.""" - ranked = original_ranked_agents(self, text, role) - return [agent for agent in ranked if is_chat_compatible_model_id(agent.model)] - - task_orchestrator_cls._ranked_agents = guarded_ranked_agents - - original_select_agent = task_orchestrator_cls._select_agent - - @wraps(original_select_agent) - def guarded_select_agent(self: Any, text: str, role: str) -> Any: - """Fail with an actionable error when no chat-compatible agent remains.""" - if not self._ranked_agents(text, role): - raise RuntimeError(f"no chat-compatible agent available for role={role}") - return original_select_agent(self, text, role) - - task_orchestrator_cls._select_agent = guarded_select_agent - - original_failover_candidates = task_orchestrator_cls._failover_candidates - - @wraps(original_failover_candidates) - def guarded_failover_candidates( - self: Any, - primary: Any, - text: str, - role: str, - ) -> list[Any]: - """Prevent cross-agent retry from falling through to another endpoint family.""" - candidates = original_failover_candidates(self, primary, text, role) - return [agent for agent in candidates if is_chat_compatible_model_id(agent.model)] - - task_orchestrator_cls._failover_candidates = guarded_failover_candidates - - original_parse_workflow_plan = task_orchestrator_cls._parse_workflow_plan - - @wraps(original_parse_workflow_plan) - def guarded_parse_workflow_plan(self: Any, raw: str) -> list[Any]: - """Reselect any stale non-chat assignment named by a generated plan.""" - steps = original_parse_workflow_plan(self, raw) - agents_by_id = {agent.id: agent for agent in self.agents} - guarded_steps: list[Any] = [] - for step in steps: - assigned = agents_by_id[step.agent_id] - if not is_chat_compatible_model_id(assigned.model): - replacement = self._select_agent(step.subtask, step.role) - step = replace(step, agent_id=replacement.id) - guarded_steps.append(step) - return guarded_steps - - task_orchestrator_cls._parse_workflow_plan = guarded_parse_workflow_plan - setattr(task_orchestrator_cls, marker, True) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index f353bae5e..45b60612c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -31,6 +31,7 @@ import urllib.error import urllib.request +from .chat_capability import is_chat_compatible_model_id from .conventions import require_object_name from .credentials import NotConfigured, get_credential @@ -776,6 +777,8 @@ def chat( ``default_top_p`` are used so request-scoped Completions sampling can be applied without threading kwargs through every orchestrator hop. """ + if not is_chat_compatible_model_id(agent.model): + raise ValueError("model is not chat-compatible and cannot serve a chat request") self._local.usage = None # Expose the effective sampling knobs for request-path tests / diagnostics. effective_temperature = self.default_temperature if temperature is None else temperature @@ -2459,7 +2462,7 @@ def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: raw_steps = data.get("steps") if not isinstance(raw_steps, list) or not (2 <= len(raw_steps) <= self.policy.max_workflow_steps): raise ValueError(f"plan must have 2..{self.policy.max_workflow_steps} steps") - known_agents = {agent.id for agent in self.agents} + known_agents = {agent.id: agent for agent in self.agents} steps: list[WorkflowStep] = [] for index, item in enumerate(raw_steps): if int(item.get("id", -1)) != index: @@ -2474,8 +2477,9 @@ def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: if any(value < 0 or value >= index for value in access): raise ValueError("access may reference only earlier steps") agent_id = item.get("agent_id") - if agent_id not in known_agents: - # The planner named an unknown agent: reselect honestly instead of failing the plan. + assigned = known_agents.get(agent_id) + if assigned is None or not is_chat_compatible_model_id(assigned.model): + # Unknown or stale non-chat assignments are reselected honestly. agent_id = self._select_agent(subtask, role).id steps.append(WorkflowStep(index, role, agent_id, subtask, access)) if steps[-1].role not in {"synthesizer", "worker"}: @@ -2509,10 +2513,21 @@ def _score_agent(self, agent: ModelAgent, role: str, lowered: str) -> tuple[int, def _ranked_agents(self, text: str, role: str) -> list[ModelAgent]: """Agents sorted best-first for a role; the head is the primary, the tail are failovers.""" lowered = text.lower() - return sorted(self.agents, key=lambda agent: self._score_agent(agent, role, lowered), reverse=True) + return [ + agent + for agent in sorted( + self.agents, + key=lambda agent: self._score_agent(agent, role, lowered), + reverse=True, + ) + if is_chat_compatible_model_id(agent.model) + ] def _select_agent(self, text: str, role: str) -> ModelAgent: - selected = self._ranked_agents(text, role)[0] + ranked = self._ranked_agents(text, role) + if not ranked: + raise RuntimeError(f"no chat-compatible agent available for role={role}") + selected = ranked[0] if selected.disabled: # pragma: no cover raise RuntimeError(f"no enabled agent available for role={role}") if role in selected.provider_exclusions: # pragma: no cover @@ -2545,11 +2560,15 @@ def _invoke( def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]: ranked = self._ranked_agents(text, role) - ordered = [primary] + [agent for agent in ranked if agent.id != primary.id] + ordered = [ + agent + for agent in [primary] + [agent for agent in ranked if agent.id != primary.id] + if is_chat_compatible_model_id(agent.model) + ] eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions] healthy = [agent for agent in eligible if not self._circuit_open(agent.id)] # If every eligible agent is circuit-open, still probe them rather than fail with no attempt. - return healthy or eligible or [primary] + return healthy or eligible def _circuit_open(self, agent_id: str) -> bool: with self._circuit_lock: diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index fc1cd024e..f55bd5725 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -12,9 +12,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.chat_capability import ( # noqa: E402 - install_runtime_chat_capability_guards, -) from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, register_credential, @@ -146,15 +143,6 @@ def test_chat_compatibility_normalizes_identifiers( assert is_chat_compatible_model_id(model_id) is expected # type: ignore[arg-type] -def test_runtime_guard_installation_is_idempotent() -> None: - """Re-importing package initialization must not stack method wrappers.""" - before = ModelClient.chat - - install_runtime_chat_capability_guards(ModelClient, TaskOrchestrator) - - assert ModelClient.chat is before - - def test_bytez_chat_catalog_still_rejects_non_chat_identifiers() -> None: """Apply the same boundary even when a provider accepts a chat task filter.""" register_credential("BYTEZ_API_KEY", "bytez-secret") From 64b4e703ffb21497037e333a82ccce6a1ea3067f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 14:14:58 +0900 Subject: [PATCH 30/89] docs: normalize capability incident formatting --- docs/doctoring/embedding-chat-capability-isolation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 02312f61e..182878017 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -1,7 +1,7 @@ # Embedding-to-chat capability isolation incident -**Status:** Accepted incident decision -**Date:** 2026-08-20 +**Status:** Accepted incident decision +**Date:** 2026-08-20 **Affected consumer:** LineageWeave buyer-surface stack around PR #260 ## Incident From 543cdc782d58fa643bf091e40f1527c6046d353e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:24:21 -0700 Subject: [PATCH 31/89] ci: verify final chat capability guards --- .../repair-chat-capability-final.yml | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 .github/workflows/repair-chat-capability-final.yml diff --git a/.github/workflows/repair-chat-capability-final.yml b/.github/workflows/repair-chat-capability-final.yml new file mode 100644 index 000000000..68d3ae801 --- /dev/null +++ b/.github/workflows/repair-chat-capability-final.yml @@ -0,0 +1,221 @@ +name: Verify final chat capability source guards + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/repair-chat-capability-final.yml + +permissions: + contents: read + +concurrency: + group: repair-chat-capability-final + cancel-in-progress: false + +jobs: + prepare: + name: Patch and test with read-only token + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + base_sha: ${{ steps.patch.outputs.base_sha }} + patch_b64: ${{ steps.patch.outputs.patch_b64 }} + steps: + - name: Checkout exact triggering commit without credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Apply final explicit guards and regression tests + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator = replace_once( + orchestrator, + ''' answer in fixed chunks so behavior shape stays testable and unchanged. + """ + if agent.base_url.startswith("mock://"): + ''', + ''' answer in fixed chunks so behavior shape stays testable and unchanged. + """ + if not is_chat_compatible_model_id(agent.model): + raise ValueError( + f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" + ) + if agent.base_url.startswith("mock://"): + ''', + "streaming chat guard", + ) + orchestrator = replace_once( + orchestrator, + ''' pool = "\\n".join( + f"- {agent.id}: model={agent.model}, tags={', '.join(agent.tags) or 'none'}" + for agent in self.agents + ) + ''', + ''' pool = "\\n".join( + f"- {agent.id}: model={agent.model}, tags={', '.join(agent.tags) or 'none'}" + for agent in self.agents + if is_chat_compatible_model_id(agent.model) + ) + ''', + "generated planner inventory guard", + ) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + tests_path = Path("tests/test_chat_model_capability_isolation.py") + tests = tests_path.read_text(encoding="utf-8") + tests += ''' + + +def test_non_chat_primary_fails_over_only_to_chat_agents() -> None: + """Drop an incompatible primary while retaining a compatible fallback.""" + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + chat_agent = _agent("chat_agent", "gpt-5.2") + orchestrator = TaskOrchestrator([embedding_agent, chat_agent]) + + candidates = orchestrator._failover_candidates( + embedding_agent, + "Produce the final answer.", + "synthesizer", + ) + + assert candidates == [chat_agent] + + +def test_streaming_client_rejects_non_chat_model_before_transport() -> None: + """Apply the same endpoint boundary to streaming chat requests.""" + client = ModelClient() + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + + with pytest.raises(ValueError, match="chat-compatible"): + next( + client.stream_chat( + embedding_agent, + [{"role": "user", "content": "Produce the final answer."}], + ) + ) + + +def test_generated_planner_inventory_excludes_non_chat_agents() -> None: + """Do not advertise stale endpoint-incompatible agents to the planner.""" + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + chat_agent = _agent("chat_agent", "gpt-5.2", tags=("reasoning", "writing")) + + class PlannerClient: + def __init__(self) -> None: + self.system_prompt = "" + + def chat(self, _agent, messages, **_kwargs): + self.system_prompt = messages[0]["content"] + return json.dumps( + { + "steps": [ + { + "id": 0, + "role": "worker", + "agent_id": "chat_agent", + "subtask": "Execute the task.", + "access": [], + }, + { + "id": 1, + "role": "synthesizer", + "agent_id": "chat_agent", + "subtask": "Produce the answer.", + "access": [0], + }, + ] + } + ) + + client = PlannerClient() + orchestrator = TaskOrchestrator( + [embedding_agent, chat_agent], + client=client, + ) + + steps = orchestrator._plan_generated("Produce the final answer.") + + assert steps[-1].agent_id == "chat_agent" + assert "embedding_agent" not in client.system_prompt + assert "azure/text-embedding-3-large" not in client.system_prompt + assert "chat_agent" in client.system_prompt + ''' + tests_path.write_text(tests, encoding="utf-8") + PY + rm .github/workflows/repair-chat-capability-final.yml + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Run focused and full verification + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_model_capability_isolation.py + python -m pytest -q + git diff --check + + - name: Export verified patch + id: patch + shell: bash + run: | + set -euo pipefail + git diff --binary > /tmp/chat-capability-final.patch + test -s /tmp/chat-capability-final.patch + echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "patch_b64=$(base64 -w0 /tmp/chat-capability-final.patch)" >> "$GITHUB_OUTPUT" + + commit: + name: Commit verified patch without executing repository code + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Checkout branch without persisted credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: false + + - name: Verify exact base, apply tested patch, and push + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} + PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" + printf '%s' "${PATCH_B64}" | base64 -d > /tmp/chat-capability-final.patch + git apply --index /tmp/chat-capability-final.patch + git diff --cached --check + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "fix: complete explicit chat capability guards" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From 92efe38ee86f9b2a7cbd6e39ca874f6f748e6bd2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:33:05 -0700 Subject: [PATCH 32/89] refactor: keep ModelClient internal to orchestrator module --- contextual_orchestrator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index 99c3a9601..70dbd71c6 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -37,7 +37,7 @@ from .cost_router import CostRoutingCoordinator from .credentials import NotConfigured, get_credential, register_credential from .kv_config import InMemoryConfigStore, get_config_store -from .orchestrator import ModelAgent, ModelClient, TaskOrchestrator, WorkflowStep, load_agents +from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents from .token_counting import HeuristicTokenCounter, build_token_counter __all__ = [ From df20c19e9e71b8148227d7845a04e8516879987f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:40:44 -0700 Subject: [PATCH 33/89] test: separate chat transport from general synthesis roles --- tests/test_chat_transport_role_separation.py | 76 ++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/test_chat_transport_role_separation.py diff --git a/tests/test_chat_transport_role_separation.py b/tests/test_chat_transport_role_separation.py new file mode 100644 index 000000000..f31d14cdf --- /dev/null +++ b/tests/test_chat_transport_role_separation.py @@ -0,0 +1,76 @@ +"""Regression tests for transport compatibility versus ordinary agent roles.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-4o-mini-audio-preview", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: + """Do not reject models that official/provider contracts serve over chat.""" + assert is_chat_compatible_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: + """Keep chat-served policy classifiers out of ordinary worker roles.""" + assert not is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ], +) +def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: + """Preserve capable chat generation models for ordinary agent selection.""" + assert is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "azure/text-embedding-3-large", + "text_embedding_3_large", + "company/reranker-v2", + "gpt-4o-mini-transcribe", + "omni-moderation-latest", + "gpt-image-1", + "gpt-realtime", + "tts-1", + ], +) +def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: + """Reject clear non-chat endpoint families before transport or role routing.""" + assert not is_chat_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) From 8e86976d7cfa9da2b54243e0bd4b50bae35fccd0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:42:29 -0700 Subject: [PATCH 34/89] ci: verify transport and synthesis capability separation --- .../repair-chat-transport-role-separation.yml | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 .github/workflows/repair-chat-transport-role-separation.yml diff --git a/.github/workflows/repair-chat-transport-role-separation.yml b/.github/workflows/repair-chat-transport-role-separation.yml new file mode 100644 index 000000000..4ce59d231 --- /dev/null +++ b/.github/workflows/repair-chat-transport-role-separation.yml @@ -0,0 +1,237 @@ +name: Repair chat transport and role separation + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/repair-chat-transport-role-separation.yml + +permissions: + contents: write + +concurrency: + group: repair-chat-transport-role-separation + cancel-in-progress: false + +jobs: + repair: + name: Patch, verify, and self-delete + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout exact branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Separate endpoint transport from ordinary agent roles + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + Path("contextual_orchestrator/chat_capability.py").write_text( + '''"""Classify chat transport compatibility and ordinary agent-role eligibility. + +Provider catalogs mix endpoint-only models with models served through an +OpenAI-compatible chat transport. Transport compatibility is not the same as +fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and +policy-classification models can use chat transport, while embedding, reranking, +transcription, moderation-endpoint, image-generation, realtime, and speech-only +models cannot. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_TRANSPORT_INCOMPATIBLE_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use the ordinary chat transport. + + The classifier rejects only identifiers that clearly advertise an endpoint + family incompatible with chat messages. Audio-capable and safety-classifier + models remain transport-compatible because providers serve some of them over + ``/chat/completions``. + """ + tokens = _model_tokens(model_id) + if not tokens: + return False + for token in tokens: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: + return False + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): + return False + return True + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models can use chat transport but are specialized + policy classifiers, not general answer synthesizers. This negative role gate + does not infer reasoning, coding, vision, or verification capabilities. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) +''', + encoding="utf-8", + ) + + discovery_path = Path("contextual_orchestrator/model_discovery.py") + discovery = discovery_path.read_text(encoding="utf-8") + discovery = replace_once( + discovery, + "from .chat_capability import is_chat_compatible_model_id\n", + "from .chat_capability import (\n" + " is_chat_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + "model discovery capability imports", + ) + discovery = discovery.replace( + "is_chat_compatible_model_id(", + "is_general_chat_agent_model_id(", + ) + # Keep the transport helper available from this module for compatibility + # with existing callers and focused tests; the ordinary discovery path + # must use the stricter general-role gate. + if "is_chat_compatible_model_id," not in discovery: + raise SystemExit("transport helper import was lost") + discovery_path.write_text(discovery, encoding="utf-8") + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator = replace_once( + orchestrator, + "from .chat_capability import is_chat_compatible_model_id\n", + "from .chat_capability import (\n" + " is_chat_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + "orchestrator capability imports", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " )\n" + " system = (\n", + " if is_general_chat_agent_model_id(agent.model)\n" + " )\n" + " system = (\n", + "planner inventory role gate", + ) + orchestrator = replace_once( + orchestrator, + " if assigned is None or not is_chat_compatible_model_id(assigned.model):\n", + " if assigned is None or not is_general_chat_agent_model_id(assigned.model):\n", + "generated assignment role gate", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " ]\n\n" + " def _select_agent", + " if is_general_chat_agent_model_id(agent.model)\n" + " ]\n\n" + " def _select_agent", + "ranked agent role gate", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " ]\n" + " eligible = [agent for agent in ordered", + " if is_general_chat_agent_model_id(agent.model)\n" + " ]\n" + " eligible = [agent for agent in ordered", + "failover role gate", + ) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + workflow = Path(".github/workflows/repair-chat-transport-role-separation.yml") + workflow.unlink() + PY + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Verify focused and full suites + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py + python -m pytest -q + + - name: Commit verified repair + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + git commit -m "fix: separate chat transport from synthesis roles" + git push origin HEAD:fix/chat-capability-isolation-embedding-models From 4fb286fdaacb0458aece47ff47dbe1fb4aeee08d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:48:35 -0700 Subject: [PATCH 35/89] chore: remove superseded transport-role repair workflow --- .../repair-chat-transport-role-separation.yml | 237 ------------------ 1 file changed, 237 deletions(-) delete mode 100644 .github/workflows/repair-chat-transport-role-separation.yml diff --git a/.github/workflows/repair-chat-transport-role-separation.yml b/.github/workflows/repair-chat-transport-role-separation.yml deleted file mode 100644 index 4ce59d231..000000000 --- a/.github/workflows/repair-chat-transport-role-separation.yml +++ /dev/null @@ -1,237 +0,0 @@ -name: Repair chat transport and role separation - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/repair-chat-transport-role-separation.yml - -permissions: - contents: write - -concurrency: - group: repair-chat-transport-role-separation - cancel-in-progress: false - -jobs: - repair: - name: Patch, verify, and self-delete - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout exact branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: true - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Separate endpoint transport from ordinary agent roles - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - Path("contextual_orchestrator/chat_capability.py").write_text( - '''"""Classify chat transport compatibility and ordinary agent-role eligibility. - -Provider catalogs mix endpoint-only models with models served through an -OpenAI-compatible chat transport. Transport compatibility is not the same as -fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and -policy-classification models can use chat transport, while embedding, reranking, -transcription, moderation-endpoint, image-generation, realtime, and speech-only -models cannot. -""" - -from __future__ import annotations - -import re - -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( - { - "bge", - "e5", - "embed", - "embedding", - "embeddings", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_TRANSPORT_INCOMPATIBLE_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - - -def _model_tokens(model_id: str) -> tuple[str, ...]: - """Normalize one provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: - return () - return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - - -def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether an identifier can use the ordinary chat transport. - - The classifier rejects only identifiers that clearly advertise an endpoint - family incompatible with chat messages. Audio-capable and safety-classifier - models remain transport-compatible because providers serve some of them over - ``/chat/completions``. - """ - tokens = _model_tokens(model_id) - if not tokens: - return False - for token in tokens: - if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: - return False - if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): - return False - return True - - -def is_general_chat_agent_model_id(model_id: str) -> bool: - """Return whether a chat model may enter ordinary orchestration roles. - - Explicit guard and safety models can use chat transport but are specialized - policy classifiers, not general answer synthesizers. This negative role gate - does not infer reasoning, coding, vision, or verification capabilities. - """ - tokens = _model_tokens(model_id) - if not tokens or not is_chat_compatible_model_id(model_id): - return False - return not any( - token == "safety" - or token == "guard" - or token.startswith("nemoguard") - or token.endswith("guard") - for token in tokens - ) -''', - encoding="utf-8", - ) - - discovery_path = Path("contextual_orchestrator/model_discovery.py") - discovery = discovery_path.read_text(encoding="utf-8") - discovery = replace_once( - discovery, - "from .chat_capability import is_chat_compatible_model_id\n", - "from .chat_capability import (\n" - " is_chat_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - "model discovery capability imports", - ) - discovery = discovery.replace( - "is_chat_compatible_model_id(", - "is_general_chat_agent_model_id(", - ) - # Keep the transport helper available from this module for compatibility - # with existing callers and focused tests; the ordinary discovery path - # must use the stricter general-role gate. - if "is_chat_compatible_model_id," not in discovery: - raise SystemExit("transport helper import was lost") - discovery_path.write_text(discovery, encoding="utf-8") - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - orchestrator = replace_once( - orchestrator, - "from .chat_capability import is_chat_compatible_model_id\n", - "from .chat_capability import (\n" - " is_chat_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - "orchestrator capability imports", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " )\n" - " system = (\n", - " if is_general_chat_agent_model_id(agent.model)\n" - " )\n" - " system = (\n", - "planner inventory role gate", - ) - orchestrator = replace_once( - orchestrator, - " if assigned is None or not is_chat_compatible_model_id(assigned.model):\n", - " if assigned is None or not is_general_chat_agent_model_id(assigned.model):\n", - "generated assignment role gate", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " ]\n\n" - " def _select_agent", - " if is_general_chat_agent_model_id(agent.model)\n" - " ]\n\n" - " def _select_agent", - "ranked agent role gate", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " ]\n" - " eligible = [agent for agent in ordered", - " if is_general_chat_agent_model_id(agent.model)\n" - " ]\n" - " eligible = [agent for agent in ordered", - "failover role gate", - ) - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - workflow = Path(".github/workflows/repair-chat-transport-role-separation.yml") - workflow.unlink() - PY - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Verify focused and full suites - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py - python -m pytest -q - - - name: Commit verified repair - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git diff --cached --check - git commit -m "fix: separate chat transport from synthesis roles" - git push origin HEAD:fix/chat-capability-isolation-embedding-models From ea2ca8d4cb82b5e75b34e6855f24651d35d176c9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:48:52 -0700 Subject: [PATCH 36/89] test: remove out-of-scope transport-role experiment --- tests/test_chat_transport_role_separation.py | 76 -------------------- 1 file changed, 76 deletions(-) delete mode 100644 tests/test_chat_transport_role_separation.py diff --git a/tests/test_chat_transport_role_separation.py b/tests/test_chat_transport_role_separation.py deleted file mode 100644 index f31d14cdf..000000000 --- a/tests/test_chat_transport_role_separation.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Regression tests for transport compatibility versus ordinary agent roles.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from contextual_orchestrator.chat_capability import ( # noqa: E402 - is_chat_compatible_model_id, - is_general_chat_agent_model_id, -) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-4o-mini-audio-preview", - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: - """Do not reject models that official/provider contracts serve over chat.""" - assert is_chat_compatible_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "nvidia/llama-3.1-nemoguard-8b-content-safety", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: - """Keep chat-served policy classifiers out of ordinary worker roles.""" - assert not is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-5.2", - "qwen/qwen3-235b-a22b-instruct", - ], -) -def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: - """Preserve capable chat generation models for ordinary agent selection.""" - assert is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "azure/text-embedding-3-large", - "text_embedding_3_large", - "company/reranker-v2", - "gpt-4o-mini-transcribe", - "omni-moderation-latest", - "gpt-image-1", - "gpt-realtime", - "tts-1", - ], -) -def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: - """Reject clear non-chat endpoint families before transport or role routing.""" - assert not is_chat_compatible_model_id(model_id) - assert not is_general_chat_agent_model_id(model_id) From a30c390d80aeb5478d1618f8d572f5af04882533 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:49:20 -0700 Subject: [PATCH 37/89] test: define honest provider-diverse discovery bootstrap --- tests/test_discovery_bootstrap_selection.py | 104 ++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/test_discovery_bootstrap_selection.py diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py new file mode 100644 index 000000000..93d5e3559 --- /dev/null +++ b/tests/test_discovery_bootstrap_selection.py @@ -0,0 +1,104 @@ +"""Regression coverage for honest, provider-diverse discovery bootstrap.""" + +from __future__ import annotations + +from contextual_orchestrator.cost_ledger import PriceBook, PriceEntry +from contextual_orchestrator.kv_config import InMemoryConfigStore +from contextual_orchestrator import model_discovery +from contextual_orchestrator.model_discovery import ( + DiscoveredModel, + select_cheapest_discovered_agent, + select_top_n_cheapest_discovered_agents, +) + + +def _model(provider_name: str, model_id: str) -> DiscoveredModel: + """Build one deterministic OpenAI-compatible discovery fixture.""" + credential_name = f"{provider_name.upper()}_API_KEY" + return DiscoveredModel( + provider_name=provider_name, + model_id=model_id, + credential_name=credential_name, + chat_base_url=f"https://{provider_name}.example/v1", + auth_scheme="Bearer", + ) + + +def _set_price( + price_book: PriceBook, + model: DiscoveredModel, + price_per_1k: float, +) -> None: + """Record one known symmetric prompt/completion price.""" + price_book.set_price( + PriceEntry( + model.provider_name, + model.model_id, + price_per_1k, + price_per_1k, + ) + ) + + +def test_unpriced_discovered_model_is_unknown_not_free() -> None: + """Missing price evidence must not outrank a model with a known price.""" + price_book = PriceBook(InMemoryConfigStore()) + priced = _model("openrouter", "priced-model") + unpriced = _model("bytez", "unpriced-model") + _set_price(price_book, priced, 0.01) + + assert select_cheapest_discovered_agent([unpriced, priced], price_book) is priced + assert select_top_n_cheapest_discovered_agents( + [unpriced, priced], price_book, 2 + ) == [priced, unpriced] + + +def test_bootstrap_selector_prefers_provider_diversity_before_duplicates() -> None: + """The initial failover pool must span providers before repeating one.""" + selector = getattr( + model_discovery, + "select_bootstrap_discovered_agents", + None, + ) + assert callable(selector), "missing provider-diverse bootstrap selector" + + price_book = PriceBook(InMemoryConfigStore()) + router_cheapest = _model("openrouter", "router-cheapest") + router_second = _model("openrouter", "router-second") + nim_model = _model("nvidia_nim", "nim-model") + openai_model = _model("openai", "openai-model") + _set_price(price_book, router_cheapest, 0.01) + _set_price(price_book, router_second, 0.02) + _set_price(price_book, nim_model, 0.5) + _set_price(price_book, openai_model, 1.0) + + selected = selector( + [router_second, openai_model, nim_model, router_cheapest], + price_book, + 3, + ) + + assert selected == [router_cheapest, nim_model, openai_model] + + +def test_bootstrap_selector_is_deterministic_when_every_model_is_unpriced() -> None: + """All-unpriced discovery remains usable but never order-dependent.""" + selector = getattr( + model_discovery, + "select_bootstrap_discovered_agents", + None, + ) + assert callable(selector), "missing provider-diverse bootstrap selector" + + price_book = PriceBook(InMemoryConfigStore()) + router_z = _model("openrouter", "z-model") + router_a = _model("openrouter", "a-model") + nim_b = _model("nvidia_nim", "b-model") + + selected = selector( + [router_z, nim_b, router_a], + price_book, + 3, + ) + + assert selected == [nim_b, router_a, router_z] From 6578dc5c81a8c71099e8754ee93f2bc69e29cc53 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:51:45 -0700 Subject: [PATCH 38/89] test: bind NIM keys to one provider family --- tests/test_discovery_bootstrap_selection.py | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py index 93d5e3559..7357e3cec 100644 --- a/tests/test_discovery_bootstrap_selection.py +++ b/tests/test_discovery_bootstrap_selection.py @@ -81,6 +81,32 @@ def test_bootstrap_selector_prefers_provider_diversity_before_duplicates() -> No assert selected == [router_cheapest, nim_model, openai_model] +def test_bootstrap_selector_treats_nim_primary_and_sub_as_one_outage_domain() -> None: + """Two NIM keys must not displace an independently hosted provider.""" + selector = getattr( + model_discovery, + "select_bootstrap_discovered_agents", + None, + ) + assert callable(selector), "missing provider-diverse bootstrap selector" + + price_book = PriceBook(InMemoryConfigStore()) + nim_primary = _model("nvidia_nim", "primary-model") + nim_sub = _model("nvidia_nim_sub", "sub-model") + openrouter = _model("openrouter", "router-model") + _set_price(price_book, nim_primary, 0.01) + _set_price(price_book, nim_sub, 0.02) + _set_price(price_book, openrouter, 0.5) + + selected = selector( + [nim_sub, openrouter, nim_primary], + price_book, + 2, + ) + + assert selected == [nim_primary, openrouter] + + def test_bootstrap_selector_is_deterministic_when_every_model_is_unpriced() -> None: """All-unpriced discovery remains usable but never order-dependent.""" selector = getattr( From 3f159124f1c8cd7690a3401a3f5073994fbadfb6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:52:07 -0700 Subject: [PATCH 39/89] test: reproduce chat transport and role boundary conflation --- tests/test_chat_transport_role_separation.py | 75 ++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/test_chat_transport_role_separation.py diff --git a/tests/test_chat_transport_role_separation.py b/tests/test_chat_transport_role_separation.py new file mode 100644 index 000000000..27142ebcd --- /dev/null +++ b/tests/test_chat_transport_role_separation.py @@ -0,0 +1,75 @@ +"""Regression tests for chat transport compatibility versus agent-role eligibility.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-audio-mini", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + ], +) +def test_chat_served_models_remain_transport_compatible(model_id: str) -> None: + """Do not reject models that provider contracts serve through chat completions.""" + assert is_chat_compatible_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "nvidia/llama-3.1-nemoguard-8b-topic-control", + ], +) +def test_policy_classifiers_do_not_enter_general_agent_roles(model_id: str) -> None: + """Keep chat-served policy classifiers out of ordinary synthesis roles.""" + assert not is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-audio-mini", + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ], +) +def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: + """Preserve chat generation models for ordinary agent selection.""" + assert is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "azure/text-embedding-3-large", + "text_embedding_3_large", + "company/reranker-v2", + "gpt-4o-mini-transcribe", + "omni-moderation-latest", + "gpt-image-1", + "gpt-realtime", + "tts-1", + ], +) +def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: + """Reject clear non-chat endpoint families before transport or role routing.""" + assert not is_chat_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) From 1ca40150100730f938b8009b333db022de2a7d51 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:52:59 -0700 Subject: [PATCH 40/89] fix: rank unknown prices honestly and diversify bootstrap providers --- contextual_orchestrator/model_discovery.py | 101 +++++++++++++++------ 1 file changed, 71 insertions(+), 30 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index bd76100c0..a0b955c47 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -20,7 +20,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from .batch_routing import cheapest_upstream from .credentials import get_credential from .orchestrator import ModelAgent @@ -249,9 +248,8 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook """Write every discovered model's known pricing into the price book. Returns the number of price rows written. A model without provider-reported - pricing is skipped rather than defaulted to 0 -- an unpriced model already - costs 0 under ``PriceBook.compute_cost``'s "explicit, not silently expensive" - contract, so writing a fabricated 0 row here would just hide that signal. + pricing is skipped rather than defaulted to zero. Discovery ranking treats + that absence as unknown price evidence, never as proof that the model is free. """ from .cost_ledger import PriceEntry @@ -272,46 +270,89 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook return written +def _discovery_price_key( + model: DiscoveredModel, + price_book: "PriceBook", +) -> tuple[int, float, str, str]: + """Rank known prices first, then deterministically order unknown prices.""" + entry = price_book.get_price(model.provider_name, model.model_id) + if entry is None: + return (1, 0.0, model.provider_name, model.model_id) + cost, _currency = price_book.compute_cost( + model.provider_name, + model.model_id, + 1000, + 1000, + ) + return (0, cost, model.provider_name, model.model_id) + + +def _provider_family(provider_name: str) -> str: + """Collapse credentials that share one upstream provider outage domain.""" + if provider_name in {"nvidia_nim", "nvidia_nim_sub"}: + return "nvidia_nim" + return provider_name + + def select_cheapest_discovered_agent( discovered: list[DiscoveredModel], price_book: "PriceBook" ) -> DiscoveredModel | None: - """Pick the lowest-cost discovered model per the price book (auto-optimization). - - Reuses :func:`~contextual_orchestrator.batch_routing.cheapest_upstream`, the - existing cost-optimizing upstream selector. Call :func:`refresh_price_book` - first so discovered pricing is visible; an unpriced candidate costs ``0`` - under that selector's documented contract and is treated as free, not - unknown -- so a genuinely unpriced provider (e.g. Bytez, priced by - GPU-second rather than per token) will always look cheapest here. Fine for - "auto-pick something free to try," but callers doing real cost comparison - should refresh pricing for every candidate they care about first. + """Pick the cheapest candidate with trustworthy price evidence. + + A candidate without a price row is unknown, not free. Known prices therefore + sort first; when every candidate is unpriced, provider and model identifiers + provide deterministic fallback ordering without inventing a monetary value. """ if not discovered: return None - candidates = [{"provider": model.provider_name, "model": model.model_id} for model in discovered] - winner = cheapest_upstream(candidates, price_book) - if winner is None: - return None - for model in discovered: - if model.provider_name == winner["provider"] and model.model_id == winner["model"]: - return model - return None # pragma: no cover - winner always comes from candidates + return min(discovered, key=lambda model: _discovery_price_key(model, price_book)) def select_top_n_cheapest_discovered_agents( discovered: list[DiscoveredModel], price_book: "PriceBook", limit: int ) -> list[DiscoveredModel]: - """Return the ``limit`` lowest-cost discovered models, cheapest first. + """Return up to ``limit`` candidates with known prices before unknown ones.""" + if limit <= 0 or not discovered: + return [] + return sorted( + discovered, + key=lambda model: _discovery_price_key(model, price_book), + )[:limit] - For bootstrapping a CI sidecar (or any first-boot pool) with more than one - enabled agent for failover, without hand-picking which discovered models to - trust. Same pricing contract as :func:`select_cheapest_discovered_agent`. + +def select_bootstrap_discovered_agents( + discovered: list[DiscoveredModel], + price_book: "PriceBook", + limit: int, +) -> list[DiscoveredModel]: + """Build a deterministic, price-honest, provider-diverse initial pool. + + Candidates retain the known-price-first ordering above, but the first pass + takes at most one model from each independent provider family. Remaining + capacity is filled in the same deterministic cost order. NVIDIA NIM primary + and sub credentials are one outage domain, so they participate in the second + pass only after independently hosted providers have had a chance to enter. """ if limit <= 0 or not discovered: return [] - def _cost(model: DiscoveredModel) -> float: - cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) - return cost + ranked = sorted( + discovered, + key=lambda model: _discovery_price_key(model, price_book), + ) + selected: list[DiscoveredModel] = [] + deferred: list[DiscoveredModel] = [] + provider_families: set[str] = set() + + for model in ranked: + family = _provider_family(model.provider_name) + if family in provider_families: + deferred.append(model) + continue + provider_families.add(family) + selected.append(model) + if len(selected) == limit: + return selected - return sorted(discovered, key=_cost)[:limit] + selected.extend(deferred[: limit - len(selected)]) + return selected From 33185ad30fa64f707aaa4618091d6c47baea44b9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:55:16 -0700 Subject: [PATCH 41/89] ci: verify chat transport and role boundaries safely --- .../verify-chat-transport-role-separation.yml | 496 ++++++++++++++++++ 1 file changed, 496 insertions(+) create mode 100644 .github/workflows/verify-chat-transport-role-separation.yml diff --git a/.github/workflows/verify-chat-transport-role-separation.yml b/.github/workflows/verify-chat-transport-role-separation.yml new file mode 100644 index 000000000..21fc78c41 --- /dev/null +++ b/.github/workflows/verify-chat-transport-role-separation.yml @@ -0,0 +1,496 @@ +name: Verify chat transport and role separation + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/verify-chat-transport-role-separation.yml + +permissions: + contents: read + +concurrency: + group: verify-chat-transport-role-separation + cancel-in-progress: false + +jobs: + prepare: + name: Prove RED, patch, and verify with read-only token + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + base_sha: ${{ steps.export.outputs.base_sha }} + patch_b64: ${{ steps.export.outputs.patch_b64 }} + steps: + - name: Checkout exact triggering commit without credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Confirm the transport-role regression is RED + shell: bash + run: | + set -euo pipefail + set +e + python -m pytest -q tests/test_chat_transport_role_separation.py > /tmp/red.log 2>&1 + status=$? + set -e + cat /tmp/red.log + test "${status}" -ne 0 + grep -Eq "is_general_chat_agent_model_id|cannot import name" /tmp/red.log + + - name: Apply the narrow transport and role boundary fix + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + Path("contextual_orchestrator/chat_capability.py").write_text( + '''"""Classify chat transport compatibility and ordinary agent-role eligibility. + +Provider catalogs mix endpoint-only models with models served through an +OpenAI-compatible chat transport. Transport compatibility is not the same as +fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and +policy-classification models can use chat transport, while embedding, reranking, +transcription, moderation-endpoint, image-generation, realtime, and speech-only +models cannot. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_TRANSPORT_INCOMPATIBLE_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use the ordinary chat transport. + + The classifier rejects only identifiers that clearly advertise an endpoint + family incompatible with chat messages. Audio-capable and safety-classifier + models remain transport-compatible because providers serve some of them over + ``/chat/completions``. + """ + tokens = _model_tokens(model_id) + if not tokens: + return False + for token in tokens: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: + return False + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): + return False + return True + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models can use chat transport but are specialized + policy classifiers, not general answer synthesizers. This negative role gate + does not infer reasoning, coding, vision, or verification capabilities. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) +''', + encoding="utf-8", + ) + + discovery_path = Path("contextual_orchestrator/model_discovery.py") + discovery = discovery_path.read_text(encoding="utf-8") + discovery = replace_once( + discovery, + "from .chat_capability import is_chat_compatible_model_id\n", + "from .chat_capability import is_general_chat_agent_model_id\n", + "model discovery role classifier import", + ) + discovery = discovery.replace( + "is_chat_compatible_model_id(", + "is_general_chat_agent_model_id(", + ) + discovery = discovery.replace( + "Clearly non-chat identifiers are rejected", + "Clearly endpoint-incompatible or specialized identifiers are rejected", + ) + discovery = discovery.replace( + "One chat-compatible model found on a provider, with reported pricing.", + "One general chat-agent candidate found on a provider, with reported pricing.", + ) + discovery = discovery.replace( + "chat-compatible candidates", + "general chat-agent candidates", + ) + discovery = discovery.replace( + "Build a disabled chat ModelAgent or reject a non-chat discovery record.", + "Build a disabled general chat agent or reject an ineligible discovery record.", + ) + discovery = discovery.replace( + "Write every discovered chat model's known pricing into the price book.", + "Write every eligible general chat model's known pricing into the price book.", + ) + discovery = discovery.replace( + "lowest-cost chat-compatible model", + "lowest-cost general chat-agent model", + ) + discovery = discovery.replace( + "cheapest chat-compatible models", + "cheapest general chat-agent models", + ) + discovery_path.write_text(discovery, encoding="utf-8") + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator = replace_once( + orchestrator, + "from .chat_capability import is_chat_compatible_model_id\n", + "from .chat_capability import (\n" + " is_chat_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + "orchestrator capability imports", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " )\n" + " system = (\n", + " if is_general_chat_agent_model_id(agent.model)\n" + " )\n" + " system = (\n", + "planner inventory role gate", + ) + orchestrator = replace_once( + orchestrator, + " if assigned is None or not is_chat_compatible_model_id(assigned.model):\n", + " if assigned is None or not is_general_chat_agent_model_id(assigned.model):\n", + "generated assignment role gate", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " ]\n\n" + " def _select_agent", + " if is_general_chat_agent_model_id(agent.model)\n" + " ]\n\n" + " def _select_agent", + "ranked agent role gate", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " ]\n" + " eligible = [agent for agent in ordered", + " if is_general_chat_agent_model_id(agent.model)\n" + " ]\n" + " eligible = [agent for agent in ordered", + "failover role gate", + ) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + incident_test_path = Path("tests/test_chat_model_capability_isolation.py") + incident_test = incident_test_path.read_text(encoding="utf-8") + incident_test = replace_once( + incident_test, + "from contextual_orchestrator.credentials import ( # noqa: E402\n", + "from contextual_orchestrator.chat_capability import ( # noqa: E402\n" + " is_chat_compatible_model_id,\n" + ")\n" + "from contextual_orchestrator.credentials import ( # noqa: E402\n", + "direct classifier test import", + ) + incident_test = replace_once( + incident_test, + " discover_provider_models,\n" + " is_chat_compatible_model_id,\n" + " refresh_price_book,\n", + " discover_provider_models,\n" + " refresh_price_book,\n", + "remove discovery-module classifier re-export dependency", + ) + incident_test = replace_once( + incident_test, + " {\"id\": \"company/reranker-v2\"},\n" + " {\"id\": \"gpt-5.2\"},\n", + " {\"id\": \"company/reranker-v2\"},\n" + " {\"id\": \"nvidia/llama-3.1-nemotron-safety-guard-8b-v3\"},\n" + " {\"id\": \"gpt-audio\"},\n" + " {\"id\": \"gpt-5.2\"},\n", + "mixed provider catalog cases", + ) + incident_test = replace_once( + incident_test, + " assert [model.model_id for model in discovered] == [\n" + " \"gpt-5.2\",\n", + " assert [model.model_id for model in discovered] == [\n" + " \"gpt-audio\",\n" + " \"gpt-5.2\",\n", + "audio chat model discovery expectation", + ) + incident_test_path.write_text(incident_test, encoding="utf-8") + + Path("docs/doctoring/embedding-chat-capability-isolation.md").write_text( + '''# Embedding-to-chat capability isolation incident + +**Status:** Accepted incident decision +**Date:** 2026-08-20 +**Affected consumer:** LineageWeave buyer-surface stack around PR #260 + +## Incident + +A conducted workflow reached the final contextual-orchestrator synthesizer with +`model_group=text-embedding-3-large` and deployment +`azure/text-embedding-3-large`. The gateway rejected the chat operation as +unsupported. Its configured fallback map contained chat-generation model groups, +but no fallback attached to the embedding group. + +The missing fallback was a symptom, not the causal defect. An embedding deployment +had already crossed the ordinary chat-agent capability boundary and become eligible +for a worker role. + +## Causal boundary + +Provider-compatible `/models` registries can contain multiple endpoint families. +The original discovery parser accepted every non-empty model identifier and exposed +it to agent creation, price selection, and durable pool synchronization. A catalog +row naming an embedding deployment could therefore be scored for thinker, worker, +verifier, or synthesizer work even though its serving endpoint accepts embedding +input rather than chat messages. + +The first incident fix closed discovery and price-routing boundaries, but further +root-cause tracing showed an already-persisted incompatible `ModelAgent` could still +survive that filter. Runtime ranking, generated workflow assignment, cross-agent +failover, readiness probing, streaming, and direct `ModelClient.chat()` previously +trusted the persisted model identifier. That stale-state path is sufficient to +reproduce the same unsupported Azure chat operation after a restart or durable +bootstrap. + +Capability classification must distinguish two questions. **Chat transport +compatibility** asks whether a model can be invoked through a chat-completions +operation. **General agent-role eligibility** asks whether that chat-served model is +appropriate for ordinary thinker, worker, verifier, or synthesizer work. OpenAI's +`gpt-audio` is explicitly served through Chat Completions, while NVIDIA content +safety and topic-control NIMs also expose chat-completions interfaces but are +specialized policy classifiers. Rejecting every identifier containing `audio`, +`guard`, or `safety` at the transport layer would therefore create a new endpoint +routing defect while fixing the embedding incident. + +OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately +from chat-completions models. Microsoft demonstrates it with +`client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes +chat, responses, embeddings, image, audio, rerank, and other endpoint families as +distinct operations. A router fallback can choose another deployment for the same +operation; it cannot make an embedding deployment execute a chat operation. + +## Decision + +Endpoint compatibility and ordinary agent-role eligibility are shared runtime +invariants. + +1. Normalize provider prefixes and common separators in model identifiers. +2. At the transport boundary, reject identifiers that clearly advertise embedding, + reranking, transcription, moderation-endpoint, image/video generation, realtime, + or speech-only semantics. +3. Keep provider-documented audio and policy-classifier models transport-compatible + when they are served through chat completions. +4. At discovery and ordinary orchestration-role boundaries, additionally exclude + explicit guard, safety, and NemoGuard policy classifiers. +5. Apply the general-role guard while parsing both OpenAI-compatible and Bytez + catalogs and before converting, pricing, or cost-selecting a discovery record. +6. Remove stale ineligible agents from thinker, worker, verifier, and synthesizer + ranking, generated-plan inventory and assignments, and cross-agent failover. +7. Apply only the transport guard in `ModelClient.chat()`, `stream_chat()`, and + readiness probing so legitimate chat-served specialized models are not falsely + rejected. +8. Fail closed when no general chat agent remains. +9. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, + or verification capabilities from their names. + +This remains a conservative negative filter. A future capability registry may +replace name-based exclusion with authenticated provider metadata, measured endpoint +probes, and separate endpoint-specific pools. + +## Rejected response + +Adding `text-embedding-3-large` to a chat fallback map is rejected. It would retain +the invalid primary assignment and merely hide it when a fallback happened to be +available. Repeated provider retries are also rejected because the request is +structurally unsupported, not transiently unavailable. + +## Residual operational action + +Runtime containment means an already-persisted embedding agent can no longer win +ordinary chat selection or failover while stale data is being cleaned up. Durable +state must still converge to the correct exact set: the provider-bootstrap slice +owns stale discovered-agent withdrawal and must import the shared general-role +classifier when rebased. Runtime rejection is defense in depth, not a substitute +for deleting invalid persistent configuration. + +## Verification evidence + +`tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID, +provider/separator aliases, persisted-state paths, and transport guards. +`tests/test_chat_transport_role_separation.py` proves that provider-documented +chat-served audio and policy models remain transport-compatible while policy +classifiers stay out of ordinary synthesis roles. Together they verify discovery, +conversion, pricing, selection, generated plans, failover, readiness, direct chat, +and streaming boundaries. + +## References + +BerriAI. (n.d.). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. +Retrieved August 20, 2026, from https://docs.litellm.ai/ + +Microsoft. (n.d.). *How to switch between OpenAI and Azure OpenAI endpoints*. +Microsoft Learn. Retrieved August 20, 2026, from +https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints + +NVIDIA. (n.d.). *Deploy NemoGuard NIMs*. Retrieved August 20, 2026, from +https://docs.nvidia.com/nemo/microservices/26.3.1/guardrails/tutorials/deploy-nemoguard-nims.html + +NVIDIA. (n.d.). *Reference—Llama 3.1 Nemotron Safety Guard 8B NIM*. Retrieved +August 20, 2026, from +https://docs.nvidia.com/nim/llama-3-1-nemotron-safety-guard-8b/latest/reference.html + +OpenAI. (n.d.). *Data controls in the OpenAI platform: Default usage policies by +endpoint*. Retrieved August 20, 2026, from +https://platform.openai.com/docs/models/default-usage-policies-by-endpoint + +OpenAI. (n.d.). *gpt-audio model*. Retrieved August 20, 2026, from +https://developers.openai.com/api/docs/models/gpt-audio +''', + encoding="utf-8", + ) + + Path(".github/workflows/verify-chat-transport-role-separation.yml").unlink() + PY + + - name: Snapshot the intended patch before executing tests + shell: bash + run: | + set -euo pipefail + git diff --binary > /tmp/expected.patch + test -s /tmp/expected.patch + + - name: Run focused and full verification + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py + python -m pytest -q + git diff --check + git diff --binary > /tmp/actual.patch + cmp /tmp/expected.patch /tmp/actual.patch + + - name: Export the verified patch + id: export + shell: bash + run: | + set -euo pipefail + echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "patch_b64=$(base64 -w0 /tmp/expected.patch)" >> "$GITHUB_OUTPUT" + + commit: + name: Commit the verified patch without executing repository code + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Checkout the branch without persisted credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: false + + - name: Verify exact base, apply the tested patch, and push + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} + PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" + printf '%s' "${PATCH_B64}" | base64 -d > /tmp/verified.patch + git apply --index /tmp/verified.patch + git diff --cached --check + actual_paths="$(git diff --cached --name-only | sort)" + expected_paths="$(cat <<'EOF' +.github/workflows/verify-chat-transport-role-separation.yml +contextual_orchestrator/chat_capability.py +contextual_orchestrator/model_discovery.py +contextual_orchestrator/orchestrator.py +docs/doctoring/embedding-chat-capability-isolation.md +tests/test_chat_model_capability_isolation.py +EOF +)" + test "${actual_paths}" = "${expected_paths}" + test ! -e .github/workflows/verify-chat-transport-role-separation.yml + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "fix: separate chat transport from synthesis roles" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From a5a3840d84c9584713e115800b47a21d1b69e9fe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:07:19 -0700 Subject: [PATCH 42/89] chore: remove branch-mutating repair workflow --- .../verify-chat-transport-role-separation.yml | 496 ------------------ 1 file changed, 496 deletions(-) delete mode 100644 .github/workflows/verify-chat-transport-role-separation.yml diff --git a/.github/workflows/verify-chat-transport-role-separation.yml b/.github/workflows/verify-chat-transport-role-separation.yml deleted file mode 100644 index 21fc78c41..000000000 --- a/.github/workflows/verify-chat-transport-role-separation.yml +++ /dev/null @@ -1,496 +0,0 @@ -name: Verify chat transport and role separation - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/verify-chat-transport-role-separation.yml - -permissions: - contents: read - -concurrency: - group: verify-chat-transport-role-separation - cancel-in-progress: false - -jobs: - prepare: - name: Prove RED, patch, and verify with read-only token - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - base_sha: ${{ steps.export.outputs.base_sha }} - patch_b64: ${{ steps.export.outputs.patch_b64 }} - steps: - - name: Checkout exact triggering commit without credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Confirm the transport-role regression is RED - shell: bash - run: | - set -euo pipefail - set +e - python -m pytest -q tests/test_chat_transport_role_separation.py > /tmp/red.log 2>&1 - status=$? - set -e - cat /tmp/red.log - test "${status}" -ne 0 - grep -Eq "is_general_chat_agent_model_id|cannot import name" /tmp/red.log - - - name: Apply the narrow transport and role boundary fix - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - Path("contextual_orchestrator/chat_capability.py").write_text( - '''"""Classify chat transport compatibility and ordinary agent-role eligibility. - -Provider catalogs mix endpoint-only models with models served through an -OpenAI-compatible chat transport. Transport compatibility is not the same as -fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and -policy-classification models can use chat transport, while embedding, reranking, -transcription, moderation-endpoint, image-generation, realtime, and speech-only -models cannot. -""" - -from __future__ import annotations - -import re - -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( - { - "bge", - "e5", - "embed", - "embedding", - "embeddings", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_TRANSPORT_INCOMPATIBLE_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - - -def _model_tokens(model_id: str) -> tuple[str, ...]: - """Normalize one provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: - return () - return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - - -def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether an identifier can use the ordinary chat transport. - - The classifier rejects only identifiers that clearly advertise an endpoint - family incompatible with chat messages. Audio-capable and safety-classifier - models remain transport-compatible because providers serve some of them over - ``/chat/completions``. - """ - tokens = _model_tokens(model_id) - if not tokens: - return False - for token in tokens: - if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: - return False - if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): - return False - return True - - -def is_general_chat_agent_model_id(model_id: str) -> bool: - """Return whether a chat model may enter ordinary orchestration roles. - - Explicit guard and safety models can use chat transport but are specialized - policy classifiers, not general answer synthesizers. This negative role gate - does not infer reasoning, coding, vision, or verification capabilities. - """ - tokens = _model_tokens(model_id) - if not tokens or not is_chat_compatible_model_id(model_id): - return False - return not any( - token == "safety" - or token == "guard" - or token.startswith("nemoguard") - or token.endswith("guard") - for token in tokens - ) -''', - encoding="utf-8", - ) - - discovery_path = Path("contextual_orchestrator/model_discovery.py") - discovery = discovery_path.read_text(encoding="utf-8") - discovery = replace_once( - discovery, - "from .chat_capability import is_chat_compatible_model_id\n", - "from .chat_capability import is_general_chat_agent_model_id\n", - "model discovery role classifier import", - ) - discovery = discovery.replace( - "is_chat_compatible_model_id(", - "is_general_chat_agent_model_id(", - ) - discovery = discovery.replace( - "Clearly non-chat identifiers are rejected", - "Clearly endpoint-incompatible or specialized identifiers are rejected", - ) - discovery = discovery.replace( - "One chat-compatible model found on a provider, with reported pricing.", - "One general chat-agent candidate found on a provider, with reported pricing.", - ) - discovery = discovery.replace( - "chat-compatible candidates", - "general chat-agent candidates", - ) - discovery = discovery.replace( - "Build a disabled chat ModelAgent or reject a non-chat discovery record.", - "Build a disabled general chat agent or reject an ineligible discovery record.", - ) - discovery = discovery.replace( - "Write every discovered chat model's known pricing into the price book.", - "Write every eligible general chat model's known pricing into the price book.", - ) - discovery = discovery.replace( - "lowest-cost chat-compatible model", - "lowest-cost general chat-agent model", - ) - discovery = discovery.replace( - "cheapest chat-compatible models", - "cheapest general chat-agent models", - ) - discovery_path.write_text(discovery, encoding="utf-8") - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - orchestrator = replace_once( - orchestrator, - "from .chat_capability import is_chat_compatible_model_id\n", - "from .chat_capability import (\n" - " is_chat_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - "orchestrator capability imports", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " )\n" - " system = (\n", - " if is_general_chat_agent_model_id(agent.model)\n" - " )\n" - " system = (\n", - "planner inventory role gate", - ) - orchestrator = replace_once( - orchestrator, - " if assigned is None or not is_chat_compatible_model_id(assigned.model):\n", - " if assigned is None or not is_general_chat_agent_model_id(assigned.model):\n", - "generated assignment role gate", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " ]\n\n" - " def _select_agent", - " if is_general_chat_agent_model_id(agent.model)\n" - " ]\n\n" - " def _select_agent", - "ranked agent role gate", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " ]\n" - " eligible = [agent for agent in ordered", - " if is_general_chat_agent_model_id(agent.model)\n" - " ]\n" - " eligible = [agent for agent in ordered", - "failover role gate", - ) - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - incident_test_path = Path("tests/test_chat_model_capability_isolation.py") - incident_test = incident_test_path.read_text(encoding="utf-8") - incident_test = replace_once( - incident_test, - "from contextual_orchestrator.credentials import ( # noqa: E402\n", - "from contextual_orchestrator.chat_capability import ( # noqa: E402\n" - " is_chat_compatible_model_id,\n" - ")\n" - "from contextual_orchestrator.credentials import ( # noqa: E402\n", - "direct classifier test import", - ) - incident_test = replace_once( - incident_test, - " discover_provider_models,\n" - " is_chat_compatible_model_id,\n" - " refresh_price_book,\n", - " discover_provider_models,\n" - " refresh_price_book,\n", - "remove discovery-module classifier re-export dependency", - ) - incident_test = replace_once( - incident_test, - " {\"id\": \"company/reranker-v2\"},\n" - " {\"id\": \"gpt-5.2\"},\n", - " {\"id\": \"company/reranker-v2\"},\n" - " {\"id\": \"nvidia/llama-3.1-nemotron-safety-guard-8b-v3\"},\n" - " {\"id\": \"gpt-audio\"},\n" - " {\"id\": \"gpt-5.2\"},\n", - "mixed provider catalog cases", - ) - incident_test = replace_once( - incident_test, - " assert [model.model_id for model in discovered] == [\n" - " \"gpt-5.2\",\n", - " assert [model.model_id for model in discovered] == [\n" - " \"gpt-audio\",\n" - " \"gpt-5.2\",\n", - "audio chat model discovery expectation", - ) - incident_test_path.write_text(incident_test, encoding="utf-8") - - Path("docs/doctoring/embedding-chat-capability-isolation.md").write_text( - '''# Embedding-to-chat capability isolation incident - -**Status:** Accepted incident decision -**Date:** 2026-08-20 -**Affected consumer:** LineageWeave buyer-surface stack around PR #260 - -## Incident - -A conducted workflow reached the final contextual-orchestrator synthesizer with -`model_group=text-embedding-3-large` and deployment -`azure/text-embedding-3-large`. The gateway rejected the chat operation as -unsupported. Its configured fallback map contained chat-generation model groups, -but no fallback attached to the embedding group. - -The missing fallback was a symptom, not the causal defect. An embedding deployment -had already crossed the ordinary chat-agent capability boundary and become eligible -for a worker role. - -## Causal boundary - -Provider-compatible `/models` registries can contain multiple endpoint families. -The original discovery parser accepted every non-empty model identifier and exposed -it to agent creation, price selection, and durable pool synchronization. A catalog -row naming an embedding deployment could therefore be scored for thinker, worker, -verifier, or synthesizer work even though its serving endpoint accepts embedding -input rather than chat messages. - -The first incident fix closed discovery and price-routing boundaries, but further -root-cause tracing showed an already-persisted incompatible `ModelAgent` could still -survive that filter. Runtime ranking, generated workflow assignment, cross-agent -failover, readiness probing, streaming, and direct `ModelClient.chat()` previously -trusted the persisted model identifier. That stale-state path is sufficient to -reproduce the same unsupported Azure chat operation after a restart or durable -bootstrap. - -Capability classification must distinguish two questions. **Chat transport -compatibility** asks whether a model can be invoked through a chat-completions -operation. **General agent-role eligibility** asks whether that chat-served model is -appropriate for ordinary thinker, worker, verifier, or synthesizer work. OpenAI's -`gpt-audio` is explicitly served through Chat Completions, while NVIDIA content -safety and topic-control NIMs also expose chat-completions interfaces but are -specialized policy classifiers. Rejecting every identifier containing `audio`, -`guard`, or `safety` at the transport layer would therefore create a new endpoint -routing defect while fixing the embedding incident. - -OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately -from chat-completions models. Microsoft demonstrates it with -`client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes -chat, responses, embeddings, image, audio, rerank, and other endpoint families as -distinct operations. A router fallback can choose another deployment for the same -operation; it cannot make an embedding deployment execute a chat operation. - -## Decision - -Endpoint compatibility and ordinary agent-role eligibility are shared runtime -invariants. - -1. Normalize provider prefixes and common separators in model identifiers. -2. At the transport boundary, reject identifiers that clearly advertise embedding, - reranking, transcription, moderation-endpoint, image/video generation, realtime, - or speech-only semantics. -3. Keep provider-documented audio and policy-classifier models transport-compatible - when they are served through chat completions. -4. At discovery and ordinary orchestration-role boundaries, additionally exclude - explicit guard, safety, and NemoGuard policy classifiers. -5. Apply the general-role guard while parsing both OpenAI-compatible and Bytez - catalogs and before converting, pricing, or cost-selecting a discovery record. -6. Remove stale ineligible agents from thinker, worker, verifier, and synthesizer - ranking, generated-plan inventory and assignments, and cross-agent failover. -7. Apply only the transport guard in `ModelClient.chat()`, `stream_chat()`, and - readiness probing so legitimate chat-served specialized models are not falsely - rejected. -8. Fail closed when no general chat agent remains. -9. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, - or verification capabilities from their names. - -This remains a conservative negative filter. A future capability registry may -replace name-based exclusion with authenticated provider metadata, measured endpoint -probes, and separate endpoint-specific pools. - -## Rejected response - -Adding `text-embedding-3-large` to a chat fallback map is rejected. It would retain -the invalid primary assignment and merely hide it when a fallback happened to be -available. Repeated provider retries are also rejected because the request is -structurally unsupported, not transiently unavailable. - -## Residual operational action - -Runtime containment means an already-persisted embedding agent can no longer win -ordinary chat selection or failover while stale data is being cleaned up. Durable -state must still converge to the correct exact set: the provider-bootstrap slice -owns stale discovered-agent withdrawal and must import the shared general-role -classifier when rebased. Runtime rejection is defense in depth, not a substitute -for deleting invalid persistent configuration. - -## Verification evidence - -`tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID, -provider/separator aliases, persisted-state paths, and transport guards. -`tests/test_chat_transport_role_separation.py` proves that provider-documented -chat-served audio and policy models remain transport-compatible while policy -classifiers stay out of ordinary synthesis roles. Together they verify discovery, -conversion, pricing, selection, generated plans, failover, readiness, direct chat, -and streaming boundaries. - -## References - -BerriAI. (n.d.). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. -Retrieved August 20, 2026, from https://docs.litellm.ai/ - -Microsoft. (n.d.). *How to switch between OpenAI and Azure OpenAI endpoints*. -Microsoft Learn. Retrieved August 20, 2026, from -https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints - -NVIDIA. (n.d.). *Deploy NemoGuard NIMs*. Retrieved August 20, 2026, from -https://docs.nvidia.com/nemo/microservices/26.3.1/guardrails/tutorials/deploy-nemoguard-nims.html - -NVIDIA. (n.d.). *Reference—Llama 3.1 Nemotron Safety Guard 8B NIM*. Retrieved -August 20, 2026, from -https://docs.nvidia.com/nim/llama-3-1-nemotron-safety-guard-8b/latest/reference.html - -OpenAI. (n.d.). *Data controls in the OpenAI platform: Default usage policies by -endpoint*. Retrieved August 20, 2026, from -https://platform.openai.com/docs/models/default-usage-policies-by-endpoint - -OpenAI. (n.d.). *gpt-audio model*. Retrieved August 20, 2026, from -https://developers.openai.com/api/docs/models/gpt-audio -''', - encoding="utf-8", - ) - - Path(".github/workflows/verify-chat-transport-role-separation.yml").unlink() - PY - - - name: Snapshot the intended patch before executing tests - shell: bash - run: | - set -euo pipefail - git diff --binary > /tmp/expected.patch - test -s /tmp/expected.patch - - - name: Run focused and full verification - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py - python -m pytest -q - git diff --check - git diff --binary > /tmp/actual.patch - cmp /tmp/expected.patch /tmp/actual.patch - - - name: Export the verified patch - id: export - shell: bash - run: | - set -euo pipefail - echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - echo "patch_b64=$(base64 -w0 /tmp/expected.patch)" >> "$GITHUB_OUTPUT" - - commit: - name: Commit the verified patch without executing repository code - needs: prepare - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - steps: - - name: Checkout the branch without persisted credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: false - - - name: Verify exact base, apply the tested patch, and push - shell: bash - env: - GH_TOKEN: ${{ github.token }} - EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} - PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" - printf '%s' "${PATCH_B64}" | base64 -d > /tmp/verified.patch - git apply --index /tmp/verified.patch - git diff --cached --check - actual_paths="$(git diff --cached --name-only | sort)" - expected_paths="$(cat <<'EOF' -.github/workflows/verify-chat-transport-role-separation.yml -contextual_orchestrator/chat_capability.py -contextual_orchestrator/model_discovery.py -contextual_orchestrator/orchestrator.py -docs/doctoring/embedding-chat-capability-isolation.md -tests/test_chat_model_capability_isolation.py -EOF -)" - test "${actual_paths}" = "${expected_paths}" - test ! -e .github/workflows/verify-chat-transport-role-separation.yml - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -m "fix: separate chat transport from synthesis roles" - git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From cabb1b0f3c68647fec5a09284c6c10739297a535 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:07:34 -0700 Subject: [PATCH 43/89] test: remove discarded transport-role experiment --- tests/test_chat_transport_role_separation.py | 75 -------------------- 1 file changed, 75 deletions(-) delete mode 100644 tests/test_chat_transport_role_separation.py diff --git a/tests/test_chat_transport_role_separation.py b/tests/test_chat_transport_role_separation.py deleted file mode 100644 index 27142ebcd..000000000 --- a/tests/test_chat_transport_role_separation.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Regression tests for chat transport compatibility versus agent-role eligibility.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from contextual_orchestrator.chat_capability import ( # noqa: E402 - is_chat_compatible_model_id, - is_general_chat_agent_model_id, -) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-audio-mini", - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "nvidia/llama-3.1-nemoguard-8b-content-safety", - ], -) -def test_chat_served_models_remain_transport_compatible(model_id: str) -> None: - """Do not reject models that provider contracts serve through chat completions.""" - assert is_chat_compatible_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "nvidia/llama-3.1-nemoguard-8b-content-safety", - "nvidia/llama-3.1-nemoguard-8b-topic-control", - ], -) -def test_policy_classifiers_do_not_enter_general_agent_roles(model_id: str) -> None: - """Keep chat-served policy classifiers out of ordinary synthesis roles.""" - assert not is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-audio-mini", - "gpt-5.2", - "qwen/qwen3-235b-a22b-instruct", - ], -) -def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: - """Preserve chat generation models for ordinary agent selection.""" - assert is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "azure/text-embedding-3-large", - "text_embedding_3_large", - "company/reranker-v2", - "gpt-4o-mini-transcribe", - "omni-moderation-latest", - "gpt-image-1", - "gpt-realtime", - "tts-1", - ], -) -def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: - """Reject clear non-chat endpoint families before transport or role routing.""" - assert not is_chat_compatible_model_id(model_id) - assert not is_general_chat_agent_model_id(model_id) From 3e75b1e81c8e1e97ff47acd0e3a97fac7fb3f801 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:10:34 -0700 Subject: [PATCH 44/89] test: reproduce transport and agent-role capability conflation --- tests/test_chat_transport_role_separation.py | 77 ++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/test_chat_transport_role_separation.py diff --git a/tests/test_chat_transport_role_separation.py b/tests/test_chat_transport_role_separation.py new file mode 100644 index 000000000..57655f1e3 --- /dev/null +++ b/tests/test_chat_transport_role_separation.py @@ -0,0 +1,77 @@ +"""Regression coverage for chat transport versus ordinary agent-role eligibility.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-audio-mini", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "nvidia/llama-3.1-nemoguard-8b-topic-control", + ], +) +def test_chat_served_models_remain_transport_compatible(model_id: str) -> None: + """Do not pre-reject models that provider contracts serve through chat completions.""" + assert is_chat_compatible_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "nvidia/llama-3.1-nemoguard-8b-topic-control", + ], +) +def test_policy_classifiers_do_not_enter_general_agent_roles(model_id: str) -> None: + """Keep chat-served policy classifiers out of ordinary synthesis roles.""" + assert not is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-audio-mini", + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ], +) +def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: + """Preserve chat generation models for ordinary agent selection.""" + assert is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "azure/text-embedding-3-large", + "text_embedding_3_large", + "company/reranker-v2", + "gpt-4o-mini-transcribe", + "omni-moderation-latest", + "gpt-image-1", + "sora-2", + "gpt-realtime", + "tts-1", + ], +) +def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: + """Reject endpoint-only model families before transport or ordinary role routing.""" + assert not is_chat_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) From 104ba9486bd0abf4b77c589599b2f97574673e4e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:19:51 -0700 Subject: [PATCH 45/89] ci: verify transport and ordinary-agent capability boundaries --- .../verify-chat-capability-boundaries.yml | 518 ++++++++++++++++++ 1 file changed, 518 insertions(+) create mode 100644 .github/workflows/verify-chat-capability-boundaries.yml diff --git a/.github/workflows/verify-chat-capability-boundaries.yml b/.github/workflows/verify-chat-capability-boundaries.yml new file mode 100644 index 000000000..c4e3b20b1 --- /dev/null +++ b/.github/workflows/verify-chat-capability-boundaries.yml @@ -0,0 +1,518 @@ +name: Verify chat capability boundaries + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/verify-chat-capability-boundaries.yml + +permissions: + contents: read + +concurrency: + group: verify-chat-capability-boundaries + cancel-in-progress: false + +jobs: + prepare: + name: Prove RED, apply GREEN, and verify read-only + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + base_sha: ${{ steps.export.outputs.base_sha }} + patch_b64: ${{ steps.export.outputs.patch_b64 }} + steps: + - name: Checkout exact triggering commit without credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Confirm the transport-role regression is RED + shell: bash + run: | + set -euo pipefail + set +e + python -m pytest -q tests/test_chat_transport_role_separation.py > /tmp/red.log 2>&1 + status=$? + set -e + cat /tmp/red.log + test "${status}" -ne 0 + grep -Eq "is_general_chat_agent_model_id|cannot import name" /tmp/red.log + + - name: Apply the narrow capability-boundary fix + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + Path("contextual_orchestrator/chat_capability.py").write_text( + '''"""Classify chat transport compatibility and ordinary agent-role eligibility. + +Provider catalogs mix endpoint-only models with models served through an +OpenAI-compatible chat transport. Transport compatibility is not the same as +fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and +policy-classification models can use chat transport, while embedding, reranking, +transcription, moderation-endpoint, image-generation, realtime, and speech-only +models cannot. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_TRANSPORT_INCOMPATIBLE_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize a provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use an ordinary chat operation. + + Only identifiers that clearly advertise a non-chat endpoint family are + rejected. Audio-capable and policy-classification models remain transport + compatible when their providers serve them through chat completions. + """ + tokens = _model_tokens(model_id) + if not tokens: + return False + for token in tokens: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: + return False + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): + return False + return True + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models are specialized policy classifiers rather + than general answer synthesizers. This negative role gate does not infer any + positive reasoning, coding, vision, or verification capability from a name. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) +''', + encoding="utf-8", + ) + + discovery_path = Path("contextual_orchestrator/model_discovery.py") + discovery = discovery_path.read_text(encoding="utf-8") + discovery = replace_once( + discovery, + "from .chat_capability import is_chat_compatible_model_id\n", + "from .chat_capability import is_general_chat_agent_model_id\n", + "model discovery capability import", + ) + discovery = discovery.replace( + "is_chat_compatible_model_id(", + "is_general_chat_agent_model_id(", + ) + discovery = discovery.replace( + "Provider model-list discovery for chat-agent candidates.", + "Provider model-list discovery for general chat-agent candidates.", + ) + discovery = discovery.replace( + "Clearly non-chat identifiers are rejected", + "Clearly endpoint-incompatible or specialized identifiers are rejected", + ) + discovery = discovery.replace( + "One chat-compatible model found on a provider, with reported pricing.", + "One general chat-agent candidate found on a provider, with reported pricing.", + ) + discovery = discovery.replace( + "chat-compatible candidates", + "general chat-agent candidates", + ) + discovery = discovery.replace( + "Build a disabled chat ModelAgent or reject a non-chat discovery record.", + "Build a disabled general chat agent or reject an ineligible discovery record.", + ) + discovery = discovery.replace( + "Write every discovered chat model's known pricing into the price book.", + "Write every eligible general chat model's known pricing into the price book.", + ) + discovery = discovery.replace( + "lowest-cost chat-compatible model", + "lowest-cost general chat-agent model", + ) + discovery = discovery.replace( + "cheapest chat-compatible models", + "cheapest general chat-agent models", + ) + discovery_path.write_text(discovery, encoding="utf-8") + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator = replace_once( + orchestrator, + "from .chat_capability import is_chat_compatible_model_id\n", + "from .chat_capability import (\n" + " is_chat_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + "orchestrator capability imports", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " )\n" + " system = (\n", + " if is_general_chat_agent_model_id(agent.model)\n" + " )\n" + " system = (\n", + "planner inventory role gate", + ) + orchestrator = replace_once( + orchestrator, + " if assigned is None or not is_chat_compatible_model_id(assigned.model):\n", + " if assigned is None or not is_general_chat_agent_model_id(assigned.model):\n", + "generated assignment role gate", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " ]\n\n" + " def _select_agent", + " if is_general_chat_agent_model_id(agent.model)\n" + " ]\n\n" + " def _select_agent", + "ranked agent role gate", + ) + orchestrator = replace_once( + orchestrator, + " if is_chat_compatible_model_id(agent.model)\n" + " ]\n" + " eligible = [agent for agent in ordered", + " if is_general_chat_agent_model_id(agent.model)\n" + " ]\n" + " eligible = [agent for agent in ordered", + "failover role gate", + ) + orchestrator = replace_once( + orchestrator, + "raise RuntimeError(f\"no chat-compatible agent available for role={role}\")", + "raise RuntimeError(f\"no general chat-compatible agent available for role={role}\")", + "general-role empty-pool error", + ) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + incident_path = Path("tests/test_chat_model_capability_isolation.py") + incident = incident_path.read_text(encoding="utf-8") + incident = replace_once( + incident, + "from contextual_orchestrator.credentials import ( # noqa: E402\n", + "from contextual_orchestrator.chat_capability import ( # noqa: E402\n" + " is_chat_compatible_model_id,\n" + ")\n" + "from contextual_orchestrator.credentials import ( # noqa: E402\n", + "direct transport classifier import", + ) + incident = replace_once( + incident, + " discover_provider_models,\n" + " is_chat_compatible_model_id,\n" + " refresh_price_book,\n", + " discover_provider_models,\n" + " refresh_price_book,\n", + "remove discovery-module classifier re-export dependency", + ) + incident = replace_once( + incident, + " {\"id\": \"company/reranker-v2\"},\n" + " {\"id\": \"gpt-5.2\"},\n", + " {\"id\": \"company/reranker-v2\"},\n" + " {\"id\": \"nvidia/llama-3.1-nemotron-safety-guard-8b-v3\"},\n" + " {\"id\": \"gpt-audio\"},\n" + " {\"id\": \"gpt-5.2\"},\n", + "mixed discovery capability cases", + ) + incident = replace_once( + incident, + " assert [model.model_id for model in discovered] == [\n" + " \"gpt-5.2\",\n", + " assert [model.model_id for model in discovered] == [\n" + " \"gpt-audio\",\n" + " \"gpt-5.2\",\n", + "audio discovery expectation", + ) + incident = replace_once( + incident, + " (\"vendor/transcriber-v2\", False),\n" + " (\"gpt-5.2\", True),\n", + " (\"vendor/transcriber-v2\", False),\n" + " (\"gpt-audio\", True),\n" + " (\"nvidia/llama-3.1-nemotron-safety-guard-8b-v3\", True),\n" + " (\"gpt-5.2\", True),\n", + "transport-compatible audio and policy cases", + ) + incident_path.write_text(incident, encoding="utf-8") + + Path("docs/doctoring/embedding-chat-capability-isolation.md").write_text( + '''# Embedding-to-chat capability isolation incident + +**Status:** Accepted incident decision +**Date:** 2026-08-20 +**Affected consumer:** LineageWeave buyer-surface stack around PR #260 + +## Incident + +A conducted workflow reached the final contextual-orchestrator synthesizer with +`model_group=text-embedding-3-large` and deployment +`azure/text-embedding-3-large`. The gateway rejected the chat operation as +unsupported. Its configured fallback map contained chat-generation model groups, +but no fallback attached to the embedding group. + +The missing fallback was a symptom, not the causal defect. An embedding deployment +had already crossed the ordinary chat-agent boundary and become eligible for a +worker role. + +## Causal boundary + +Provider-compatible `/models` registries can contain multiple endpoint families. +The original discovery parser accepted every non-empty model identifier and exposed +it to agent creation, price selection, and durable pool synchronization. A catalog +row naming an embedding deployment could therefore be scored for thinker, worker, +verifier, or synthesizer work even though its serving endpoint accepts embedding +input rather than chat messages. + +The first incident fix closed discovery and price-routing boundaries, but an +already-persisted incompatible `ModelAgent` could still survive that filter. +Runtime ranking, generated workflow assignment, cross-agent failover, readiness +probing, streaming, and direct `ModelClient.chat()` previously trusted the persisted +identifier. That stale-state path reproduces the same unsupported Azure operation +after a restart or durable bootstrap. + +Capability classification must answer two different questions. **Chat transport +compatibility** asks whether a model can be invoked through a chat-completions +operation. **General agent-role eligibility** asks whether that chat-served model is +appropriate for ordinary thinker, worker, verifier, or synthesizer work. OpenAI's +`gpt-audio` and `gpt-audio-mini` are served through Chat Completions. NVIDIA content +safety and topic-control NIMs also expose OpenAI-compatible chat-completions +interfaces, but they are specialized policy classifiers rather than general answer +generators. Rejecting every identifier containing `audio`, `guard`, or `safety` at +the transport layer would therefore create a new endpoint-routing defect while +fixing the embedding incident. + +OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately +from chat-completions models. Microsoft demonstrates it with +`client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes +chat, responses, embeddings, image, audio, rerank, and other endpoint families as +distinct operations. A router fallback can choose another deployment for the same +operation; it cannot make an embedding deployment execute a chat operation. + +## Decision + +Transport compatibility and ordinary agent-role eligibility are shared runtime +invariants. + +1. Normalize provider prefixes and common separators in model identifiers. +2. At the transport boundary, reject identifiers that clearly advertise embedding, + reranking, transcription, moderation-endpoint, image or video generation, + realtime, or speech-only semantics. +3. Keep provider-documented audio and policy-classifier models transport-compatible + when they are served through chat completions. +4. At discovery and ordinary orchestration-role boundaries, additionally exclude + explicit guard, safety, and NemoGuard policy classifiers. +5. Apply the general-role guard while parsing OpenAI-compatible and Bytez catalogs + and before converting, pricing, or cost-selecting a discovery record. +6. Remove stale ineligible agents from thinker, worker, verifier, and synthesizer + ranking, generated-plan inventory and assignments, and cross-agent failover. +7. Apply only the transport guard in `ModelClient.chat()`, `stream_chat()`, and + readiness probing so legitimate chat-served specialized models are not falsely + rejected. +8. Fail closed when no general chat agent remains. +9. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, + or verification capabilities from their names. + +This is a conservative negative filter. A future capability registry may replace +name-based exclusion with authenticated provider metadata, measured endpoint probes, +and separate endpoint-specific pools. + +## Rejected response + +Adding `text-embedding-3-large` to a chat fallback map is rejected. It would retain +the invalid primary assignment and merely hide it when a fallback happened to be +available. Repeated provider retries are also rejected because the request is +structurally unsupported, not transiently unavailable. + +## Residual operational action + +Runtime containment means an already-persisted embedding agent can no longer win +ordinary chat selection or failover while stale data is being cleaned up. Durable +state must still converge to the correct exact set: the provider-bootstrap slice +owns stale discovered-agent withdrawal and must import the shared general-role +classifier when rebased. Runtime rejection is defense in depth, not a substitute +for deleting invalid persistent configuration. + +## Verification evidence + +`tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID, +provider/separator aliases, persisted-state paths, and provider I/O boundaries. +`tests/test_chat_transport_role_separation.py` proves that provider-documented +chat-served audio and policy models remain transport-compatible while policy +classifiers stay out of ordinary synthesis roles. Together they cover discovery, +conversion, pricing, selection, generated plans, failover, readiness, direct chat, +and streaming boundaries. + +## References + +BerriAI. (n.d.). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. +Retrieved August 20, 2026, from https://docs.litellm.ai/ + +Microsoft. (n.d.). *How to switch between OpenAI and Azure OpenAI endpoints*. +Microsoft Learn. Retrieved August 20, 2026, from +https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints + +NVIDIA. (n.d.). *Deploy NemoGuard NIMs*. Retrieved August 20, 2026, from +https://docs.nvidia.com/nemo/microservices/26.3.1/guardrails/tutorials/deploy-nemoguard-nims.html + +NVIDIA. (n.d.). *Reference—Llama 3.1 Nemotron Safety Guard 8B NIM*. Retrieved +August 20, 2026, from +https://docs.nvidia.com/nim/llama-3-1-nemotron-safety-guard-8b/latest/reference.html + +OpenAI. (n.d.). *Data controls in the OpenAI platform: Default usage policies by +endpoint*. Retrieved August 20, 2026, from +https://platform.openai.com/docs/models/default-usage-policies-by-endpoint + +OpenAI. (n.d.). *gpt-audio model*. Retrieved August 20, 2026, from +https://developers.openai.com/api/docs/models/gpt-audio + +OpenAI. (n.d.). *gpt-audio-mini model*. Retrieved August 20, 2026, from +https://developers.openai.com/api/docs/models/gpt-audio-mini +''', + encoding="utf-8", + ) + + Path(".github/workflows/verify-chat-capability-boundaries.yml").unlink() + PY + + - name: Snapshot the intended patch before running tests + shell: bash + run: | + set -euo pipefail + git diff --binary > /tmp/expected.patch + test -s /tmp/expected.patch + + - name: Run focused and full verification + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py + python -m pytest -q + git diff --check + git diff --binary > /tmp/actual.patch + cmp /tmp/expected.patch /tmp/actual.patch + + - name: Export the verified patch + id: export + shell: bash + run: | + set -euo pipefail + echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "patch_b64=$(base64 -w0 /tmp/expected.patch)" >> "$GITHUB_OUTPUT" + + commit: + name: Commit verified patch without executing repository code + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Checkout branch without persisted credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: false + + - name: Verify exact base, apply tested patch, and push + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} + PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" + printf '%s' "${PATCH_B64}" | base64 -d > /tmp/verified.patch + git apply --index /tmp/verified.patch + git diff --cached --check + actual_paths="$(git diff --cached --name-only | sort)" + expected_paths="$(cat <<'EOF' +.github/workflows/verify-chat-capability-boundaries.yml +contextual_orchestrator/chat_capability.py +contextual_orchestrator/model_discovery.py +contextual_orchestrator/orchestrator.py +docs/doctoring/embedding-chat-capability-isolation.md +tests/test_chat_model_capability_isolation.py +EOF +)" + test "${actual_paths}" = "${expected_paths}" + test ! -e .github/workflows/verify-chat-capability-boundaries.yml + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "fix: separate chat transport from ordinary agent roles" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From 9984ad2058559e61944fb8db1273093cfca8698a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 15:24:39 +0900 Subject: [PATCH 46/89] chore: remove branch-mutating capability workflow --- .../verify-chat-capability-boundaries.yml | 518 ------------------ 1 file changed, 518 deletions(-) delete mode 100644 .github/workflows/verify-chat-capability-boundaries.yml diff --git a/.github/workflows/verify-chat-capability-boundaries.yml b/.github/workflows/verify-chat-capability-boundaries.yml deleted file mode 100644 index c4e3b20b1..000000000 --- a/.github/workflows/verify-chat-capability-boundaries.yml +++ /dev/null @@ -1,518 +0,0 @@ -name: Verify chat capability boundaries - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/verify-chat-capability-boundaries.yml - -permissions: - contents: read - -concurrency: - group: verify-chat-capability-boundaries - cancel-in-progress: false - -jobs: - prepare: - name: Prove RED, apply GREEN, and verify read-only - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - base_sha: ${{ steps.export.outputs.base_sha }} - patch_b64: ${{ steps.export.outputs.patch_b64 }} - steps: - - name: Checkout exact triggering commit without credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Confirm the transport-role regression is RED - shell: bash - run: | - set -euo pipefail - set +e - python -m pytest -q tests/test_chat_transport_role_separation.py > /tmp/red.log 2>&1 - status=$? - set -e - cat /tmp/red.log - test "${status}" -ne 0 - grep -Eq "is_general_chat_agent_model_id|cannot import name" /tmp/red.log - - - name: Apply the narrow capability-boundary fix - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - Path("contextual_orchestrator/chat_capability.py").write_text( - '''"""Classify chat transport compatibility and ordinary agent-role eligibility. - -Provider catalogs mix endpoint-only models with models served through an -OpenAI-compatible chat transport. Transport compatibility is not the same as -fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and -policy-classification models can use chat transport, while embedding, reranking, -transcription, moderation-endpoint, image-generation, realtime, and speech-only -models cannot. -""" - -from __future__ import annotations - -import re - -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( - { - "bge", - "e5", - "embed", - "embedding", - "embeddings", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_TRANSPORT_INCOMPATIBLE_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - - -def _model_tokens(model_id: str) -> tuple[str, ...]: - """Normalize a provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: - return () - return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - - -def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether an identifier can use an ordinary chat operation. - - Only identifiers that clearly advertise a non-chat endpoint family are - rejected. Audio-capable and policy-classification models remain transport - compatible when their providers serve them through chat completions. - """ - tokens = _model_tokens(model_id) - if not tokens: - return False - for token in tokens: - if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: - return False - if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): - return False - return True - - -def is_general_chat_agent_model_id(model_id: str) -> bool: - """Return whether a chat model may enter ordinary orchestration roles. - - Explicit guard and safety models are specialized policy classifiers rather - than general answer synthesizers. This negative role gate does not infer any - positive reasoning, coding, vision, or verification capability from a name. - """ - tokens = _model_tokens(model_id) - if not tokens or not is_chat_compatible_model_id(model_id): - return False - return not any( - token == "safety" - or token == "guard" - or token.startswith("nemoguard") - or token.endswith("guard") - for token in tokens - ) -''', - encoding="utf-8", - ) - - discovery_path = Path("contextual_orchestrator/model_discovery.py") - discovery = discovery_path.read_text(encoding="utf-8") - discovery = replace_once( - discovery, - "from .chat_capability import is_chat_compatible_model_id\n", - "from .chat_capability import is_general_chat_agent_model_id\n", - "model discovery capability import", - ) - discovery = discovery.replace( - "is_chat_compatible_model_id(", - "is_general_chat_agent_model_id(", - ) - discovery = discovery.replace( - "Provider model-list discovery for chat-agent candidates.", - "Provider model-list discovery for general chat-agent candidates.", - ) - discovery = discovery.replace( - "Clearly non-chat identifiers are rejected", - "Clearly endpoint-incompatible or specialized identifiers are rejected", - ) - discovery = discovery.replace( - "One chat-compatible model found on a provider, with reported pricing.", - "One general chat-agent candidate found on a provider, with reported pricing.", - ) - discovery = discovery.replace( - "chat-compatible candidates", - "general chat-agent candidates", - ) - discovery = discovery.replace( - "Build a disabled chat ModelAgent or reject a non-chat discovery record.", - "Build a disabled general chat agent or reject an ineligible discovery record.", - ) - discovery = discovery.replace( - "Write every discovered chat model's known pricing into the price book.", - "Write every eligible general chat model's known pricing into the price book.", - ) - discovery = discovery.replace( - "lowest-cost chat-compatible model", - "lowest-cost general chat-agent model", - ) - discovery = discovery.replace( - "cheapest chat-compatible models", - "cheapest general chat-agent models", - ) - discovery_path.write_text(discovery, encoding="utf-8") - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - orchestrator = replace_once( - orchestrator, - "from .chat_capability import is_chat_compatible_model_id\n", - "from .chat_capability import (\n" - " is_chat_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - "orchestrator capability imports", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " )\n" - " system = (\n", - " if is_general_chat_agent_model_id(agent.model)\n" - " )\n" - " system = (\n", - "planner inventory role gate", - ) - orchestrator = replace_once( - orchestrator, - " if assigned is None or not is_chat_compatible_model_id(assigned.model):\n", - " if assigned is None or not is_general_chat_agent_model_id(assigned.model):\n", - "generated assignment role gate", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " ]\n\n" - " def _select_agent", - " if is_general_chat_agent_model_id(agent.model)\n" - " ]\n\n" - " def _select_agent", - "ranked agent role gate", - ) - orchestrator = replace_once( - orchestrator, - " if is_chat_compatible_model_id(agent.model)\n" - " ]\n" - " eligible = [agent for agent in ordered", - " if is_general_chat_agent_model_id(agent.model)\n" - " ]\n" - " eligible = [agent for agent in ordered", - "failover role gate", - ) - orchestrator = replace_once( - orchestrator, - "raise RuntimeError(f\"no chat-compatible agent available for role={role}\")", - "raise RuntimeError(f\"no general chat-compatible agent available for role={role}\")", - "general-role empty-pool error", - ) - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - incident_path = Path("tests/test_chat_model_capability_isolation.py") - incident = incident_path.read_text(encoding="utf-8") - incident = replace_once( - incident, - "from contextual_orchestrator.credentials import ( # noqa: E402\n", - "from contextual_orchestrator.chat_capability import ( # noqa: E402\n" - " is_chat_compatible_model_id,\n" - ")\n" - "from contextual_orchestrator.credentials import ( # noqa: E402\n", - "direct transport classifier import", - ) - incident = replace_once( - incident, - " discover_provider_models,\n" - " is_chat_compatible_model_id,\n" - " refresh_price_book,\n", - " discover_provider_models,\n" - " refresh_price_book,\n", - "remove discovery-module classifier re-export dependency", - ) - incident = replace_once( - incident, - " {\"id\": \"company/reranker-v2\"},\n" - " {\"id\": \"gpt-5.2\"},\n", - " {\"id\": \"company/reranker-v2\"},\n" - " {\"id\": \"nvidia/llama-3.1-nemotron-safety-guard-8b-v3\"},\n" - " {\"id\": \"gpt-audio\"},\n" - " {\"id\": \"gpt-5.2\"},\n", - "mixed discovery capability cases", - ) - incident = replace_once( - incident, - " assert [model.model_id for model in discovered] == [\n" - " \"gpt-5.2\",\n", - " assert [model.model_id for model in discovered] == [\n" - " \"gpt-audio\",\n" - " \"gpt-5.2\",\n", - "audio discovery expectation", - ) - incident = replace_once( - incident, - " (\"vendor/transcriber-v2\", False),\n" - " (\"gpt-5.2\", True),\n", - " (\"vendor/transcriber-v2\", False),\n" - " (\"gpt-audio\", True),\n" - " (\"nvidia/llama-3.1-nemotron-safety-guard-8b-v3\", True),\n" - " (\"gpt-5.2\", True),\n", - "transport-compatible audio and policy cases", - ) - incident_path.write_text(incident, encoding="utf-8") - - Path("docs/doctoring/embedding-chat-capability-isolation.md").write_text( - '''# Embedding-to-chat capability isolation incident - -**Status:** Accepted incident decision -**Date:** 2026-08-20 -**Affected consumer:** LineageWeave buyer-surface stack around PR #260 - -## Incident - -A conducted workflow reached the final contextual-orchestrator synthesizer with -`model_group=text-embedding-3-large` and deployment -`azure/text-embedding-3-large`. The gateway rejected the chat operation as -unsupported. Its configured fallback map contained chat-generation model groups, -but no fallback attached to the embedding group. - -The missing fallback was a symptom, not the causal defect. An embedding deployment -had already crossed the ordinary chat-agent boundary and become eligible for a -worker role. - -## Causal boundary - -Provider-compatible `/models` registries can contain multiple endpoint families. -The original discovery parser accepted every non-empty model identifier and exposed -it to agent creation, price selection, and durable pool synchronization. A catalog -row naming an embedding deployment could therefore be scored for thinker, worker, -verifier, or synthesizer work even though its serving endpoint accepts embedding -input rather than chat messages. - -The first incident fix closed discovery and price-routing boundaries, but an -already-persisted incompatible `ModelAgent` could still survive that filter. -Runtime ranking, generated workflow assignment, cross-agent failover, readiness -probing, streaming, and direct `ModelClient.chat()` previously trusted the persisted -identifier. That stale-state path reproduces the same unsupported Azure operation -after a restart or durable bootstrap. - -Capability classification must answer two different questions. **Chat transport -compatibility** asks whether a model can be invoked through a chat-completions -operation. **General agent-role eligibility** asks whether that chat-served model is -appropriate for ordinary thinker, worker, verifier, or synthesizer work. OpenAI's -`gpt-audio` and `gpt-audio-mini` are served through Chat Completions. NVIDIA content -safety and topic-control NIMs also expose OpenAI-compatible chat-completions -interfaces, but they are specialized policy classifiers rather than general answer -generators. Rejecting every identifier containing `audio`, `guard`, or `safety` at -the transport layer would therefore create a new endpoint-routing defect while -fixing the embedding incident. - -OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately -from chat-completions models. Microsoft demonstrates it with -`client.embeddings.create`, not `client.chat.completions.create`. LiteLLM exposes -chat, responses, embeddings, image, audio, rerank, and other endpoint families as -distinct operations. A router fallback can choose another deployment for the same -operation; it cannot make an embedding deployment execute a chat operation. - -## Decision - -Transport compatibility and ordinary agent-role eligibility are shared runtime -invariants. - -1. Normalize provider prefixes and common separators in model identifiers. -2. At the transport boundary, reject identifiers that clearly advertise embedding, - reranking, transcription, moderation-endpoint, image or video generation, - realtime, or speech-only semantics. -3. Keep provider-documented audio and policy-classifier models transport-compatible - when they are served through chat completions. -4. At discovery and ordinary orchestration-role boundaries, additionally exclude - explicit guard, safety, and NemoGuard policy classifiers. -5. Apply the general-role guard while parsing OpenAI-compatible and Bytez catalogs - and before converting, pricing, or cost-selecting a discovery record. -6. Remove stale ineligible agents from thinker, worker, verifier, and synthesizer - ranking, generated-plan inventory and assignments, and cross-agent failover. -7. Apply only the transport guard in `ModelClient.chat()`, `stream_chat()`, and - readiness probing so legitimate chat-served specialized models are not falsely - rejected. -8. Fail closed when no general chat agent remains. -9. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, - or verification capabilities from their names. - -This is a conservative negative filter. A future capability registry may replace -name-based exclusion with authenticated provider metadata, measured endpoint probes, -and separate endpoint-specific pools. - -## Rejected response - -Adding `text-embedding-3-large` to a chat fallback map is rejected. It would retain -the invalid primary assignment and merely hide it when a fallback happened to be -available. Repeated provider retries are also rejected because the request is -structurally unsupported, not transiently unavailable. - -## Residual operational action - -Runtime containment means an already-persisted embedding agent can no longer win -ordinary chat selection or failover while stale data is being cleaned up. Durable -state must still converge to the correct exact set: the provider-bootstrap slice -owns stale discovered-agent withdrawal and must import the shared general-role -classifier when rebased. Runtime rejection is defense in depth, not a substitute -for deleting invalid persistent configuration. - -## Verification evidence - -`tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID, -provider/separator aliases, persisted-state paths, and provider I/O boundaries. -`tests/test_chat_transport_role_separation.py` proves that provider-documented -chat-served audio and policy models remain transport-compatible while policy -classifiers stay out of ordinary synthesis roles. Together they cover discovery, -conversion, pricing, selection, generated plans, failover, readiness, direct chat, -and streaming boundaries. - -## References - -BerriAI. (n.d.). *LiteLLM: Call 100+ LLMs using the OpenAI input/output format*. -Retrieved August 20, 2026, from https://docs.litellm.ai/ - -Microsoft. (n.d.). *How to switch between OpenAI and Azure OpenAI endpoints*. -Microsoft Learn. Retrieved August 20, 2026, from -https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints - -NVIDIA. (n.d.). *Deploy NemoGuard NIMs*. Retrieved August 20, 2026, from -https://docs.nvidia.com/nemo/microservices/26.3.1/guardrails/tutorials/deploy-nemoguard-nims.html - -NVIDIA. (n.d.). *Reference—Llama 3.1 Nemotron Safety Guard 8B NIM*. Retrieved -August 20, 2026, from -https://docs.nvidia.com/nim/llama-3-1-nemotron-safety-guard-8b/latest/reference.html - -OpenAI. (n.d.). *Data controls in the OpenAI platform: Default usage policies by -endpoint*. Retrieved August 20, 2026, from -https://platform.openai.com/docs/models/default-usage-policies-by-endpoint - -OpenAI. (n.d.). *gpt-audio model*. Retrieved August 20, 2026, from -https://developers.openai.com/api/docs/models/gpt-audio - -OpenAI. (n.d.). *gpt-audio-mini model*. Retrieved August 20, 2026, from -https://developers.openai.com/api/docs/models/gpt-audio-mini -''', - encoding="utf-8", - ) - - Path(".github/workflows/verify-chat-capability-boundaries.yml").unlink() - PY - - - name: Snapshot the intended patch before running tests - shell: bash - run: | - set -euo pipefail - git diff --binary > /tmp/expected.patch - test -s /tmp/expected.patch - - - name: Run focused and full verification - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py - python -m pytest -q - git diff --check - git diff --binary > /tmp/actual.patch - cmp /tmp/expected.patch /tmp/actual.patch - - - name: Export the verified patch - id: export - shell: bash - run: | - set -euo pipefail - echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - echo "patch_b64=$(base64 -w0 /tmp/expected.patch)" >> "$GITHUB_OUTPUT" - - commit: - name: Commit verified patch without executing repository code - needs: prepare - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - steps: - - name: Checkout branch without persisted credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: false - - - name: Verify exact base, apply tested patch, and push - shell: bash - env: - GH_TOKEN: ${{ github.token }} - EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} - PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" - printf '%s' "${PATCH_B64}" | base64 -d > /tmp/verified.patch - git apply --index /tmp/verified.patch - git diff --cached --check - actual_paths="$(git diff --cached --name-only | sort)" - expected_paths="$(cat <<'EOF' -.github/workflows/verify-chat-capability-boundaries.yml -contextual_orchestrator/chat_capability.py -contextual_orchestrator/model_discovery.py -contextual_orchestrator/orchestrator.py -docs/doctoring/embedding-chat-capability-isolation.md -tests/test_chat_model_capability_isolation.py -EOF -)" - test "${actual_paths}" = "${expected_paths}" - test ! -e .github/workflows/verify-chat-capability-boundaries.yml - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -m "fix: separate chat transport from ordinary agent roles" - git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From 0ec353d2f8c7916514fb1875f26df0e6094e002c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 15:23:29 +0900 Subject: [PATCH 47/89] fix: separate chat transport from synthesis roles --- contextual_orchestrator/chat_capability.py | 63 +++++++++++++------ contextual_orchestrator/model_discovery.py | 18 +++--- contextual_orchestrator/orchestrator.py | 15 +++-- .../embedding-chat-capability-isolation.md | 53 ++++++++++------ tests/test_chat_model_capability_isolation.py | 9 ++- 5 files changed, 102 insertions(+), 56 deletions(-) diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index 795ad81fb..fc36cb101 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -1,10 +1,11 @@ -"""Shared chat-capability classification and runtime fail-closed guards. +"""Classify chat transport compatibility and ordinary agent-role eligibility. -Provider model catalogs are heterogeneous: a model identifier may name a chat -model, embedding model, reranker, transcription model, image model, or another -endpoint family. The helpers in this module prevent clearly non-chat model IDs -from crossing a chat-agent boundary even when an incompatible agent was persisted -before discovery filtering was introduced. +Provider catalogs mix endpoint-only models with models served through an +OpenAI-compatible chat transport. Transport compatibility is not the same as +fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio +and policy-classification models can use chat transport, while embedding, +reranking, transcription, moderation-endpoint, image-generation, realtime, and +speech-only models cannot. """ from __future__ import annotations @@ -12,15 +13,13 @@ import re _MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_NON_CHAT_EXACT_TOKENS = frozenset( +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( { - "audio", "bge", "e5", "embed", "embedding", "embeddings", - "guard", "gte", "image", "images", @@ -28,7 +27,6 @@ "realtime", "rerank", "reranker", - "safety", "sora", "speech", "transcribe", @@ -37,7 +35,7 @@ "whisper", } ) -_NON_CHAT_TOKEN_PREFIXES = ( +_TRANSPORT_INCOMPATIBLE_PREFIXES = ( "embed", "moderat", "rerank", @@ -46,20 +44,45 @@ def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether a model identifier is eligible for a chat-agent pool. + """Return whether an identifier can use the ordinary chat transport. - The classifier intentionally rejects only identifiers that clearly advertise - a non-chat endpoint family. Unknown identifiers remain eligible until an - authenticated capability registry can prove more specific endpoint support. + The classifier rejects only identifiers that clearly advertise an endpoint + family incompatible with chat messages. Audio-capable and safety-classifier + models remain transport-compatible because providers serve some of them over + ``/chat/completions``. """ - if type(model_id) is not str: - return False - tokens = tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + tokens = _model_tokens(model_id) if not tokens: return False for token in tokens: - if token in _NON_CHAT_EXACT_TOKENS: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: return False - if token.startswith(_NON_CHAT_TOKEN_PREFIXES): + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): return False return True + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models can use chat transport but are specialized + policy classifiers, not general answer synthesizers. This negative role gate + does not infer reasoning, coding, vision, or verification capabilities. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 12f8f354c..00064f575 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -27,7 +27,7 @@ from typing import TYPE_CHECKING, Any from .batch_routing import cheapest_upstream -from .chat_capability import is_chat_compatible_model_id +from .chat_capability import is_general_chat_agent_model_id from .credentials import get_credential from .orchestrator import ModelAgent @@ -146,7 +146,7 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ if not isinstance(row, dict): continue model_id = row.get("id") - if not is_chat_compatible_model_id(model_id): + if not is_general_chat_agent_model_id(model_id): continue pricing = row.get("pricing") if isinstance(row.get("pricing"), dict) else {} discovered.append( @@ -171,7 +171,7 @@ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredMo if not isinstance(row, dict): continue model_id = row.get("modelId") - if not is_chat_compatible_model_id(model_id): + if not is_general_chat_agent_model_id(model_id): continue discovered.append( DiscoveredModel( @@ -240,9 +240,9 @@ def agent_id_for(discovered: DiscoveredModel) -> str: def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> ModelAgent: - """Build a disabled chat ModelAgent or reject a non-chat discovery record.""" - if not is_chat_compatible_model_id(discovered.model_id): - raise ValueError("non-chat model cannot be converted into a chat agent") + """Build a disabled general chat agent or reject an ineligible record.""" + if not is_general_chat_agent_model_id(discovered.model_id): + raise ValueError("model is not eligible for a general chat agent") return ModelAgent( id=agent_id_for(discovered), model=discovered.model_id, @@ -268,7 +268,7 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook written = 0 for model in discovered: - if not is_chat_compatible_model_id(model.model_id): + if not is_general_chat_agent_model_id(model.model_id): continue if model.prompt_price_per_1k is None and model.completion_price_per_1k is None: continue @@ -299,7 +299,7 @@ def select_cheapest_discovered_agent( "auto-pick something free to try," but callers doing real cost comparison should refresh pricing for every candidate they care about first. """ - eligible = [model for model in discovered if is_chat_compatible_model_id(model.model_id)] + eligible = [model for model in discovered if is_general_chat_agent_model_id(model.model_id)] if not eligible: return None candidates = [{"provider": model.provider_name, "model": model.model_id} for model in eligible] @@ -318,7 +318,7 @@ def select_top_n_cheapest_discovered_agents( """Return the ``limit`` cheapest chat-compatible models in ascending cost.""" if limit <= 0: return [] - eligible = [model for model in discovered if is_chat_compatible_model_id(model.model_id)] + eligible = [model for model in discovered if is_general_chat_agent_model_id(model.model_id)] if not eligible: return [] diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index e55506c1e..6366ae107 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -31,7 +31,10 @@ import urllib.error import urllib.request -from .chat_capability import is_chat_compatible_model_id +from .chat_capability import ( + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) from .conventions import require_object_name from .credentials import NotConfigured, get_credential @@ -2450,7 +2453,7 @@ def _plan_generated(self, task: str) -> list[WorkflowStep]: pool = "\n".join( f"- {agent.id}: model={agent.model}, tags={', '.join(agent.tags) or 'none'}" for agent in self.agents - if is_chat_compatible_model_id(agent.model) + if is_general_chat_agent_model_id(agent.model) ) system = ( "You are the workflow conductor. Decompose the user's task into a short workflow.\n" @@ -2492,8 +2495,8 @@ def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: raise ValueError("access may reference only earlier steps") agent_id = item.get("agent_id") assigned = known_agents.get(agent_id) - if assigned is None or not is_chat_compatible_model_id(assigned.model): - # Unknown or stale non-chat assignments are reselected honestly. + if assigned is None or not is_general_chat_agent_model_id(assigned.model): + # Unknown or stale ineligible assignments are reselected honestly. agent_id = self._select_agent(subtask, role).id steps.append(WorkflowStep(index, role, agent_id, subtask, access)) if steps[-1].role not in {"synthesizer", "worker"}: @@ -2534,7 +2537,7 @@ def _ranked_agents(self, text: str, role: str) -> list[ModelAgent]: key=lambda agent: self._score_agent(agent, role, lowered), reverse=True, ) - if is_chat_compatible_model_id(agent.model) + if is_general_chat_agent_model_id(agent.model) ] def _select_agent(self, text: str, role: str) -> ModelAgent: @@ -2577,7 +2580,7 @@ def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> lis ordered = [ agent for agent in [primary] + [agent for agent in ranked if agent.id != primary.id] - if is_chat_compatible_model_id(agent.model) + if is_general_chat_agent_model_id(agent.model) ] eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions] healthy = [agent for agent in eligible if not self._circuit_open(agent.id)] diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 182878017..93429d30b 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -28,9 +28,10 @@ input rather than chat messages. The first incident fix closed discovery and price-routing boundaries, but further root-cause tracing showed an already-persisted incompatible `ModelAgent` could still survive that filter. The runtime ranking path, generated workflow assignment, -cross-agent failover, and direct `ModelClient.chat()` path previously trusted the -persisted model identifier. That stale-state path is sufficient to reproduce the -same unsupported Azure chat operation after a process restart or durable bootstrap. +cross-agent failover, readiness probe, streaming path, and direct +`ModelClient.chat()` path previously trusted the persisted model identifier. That +stale-state path is sufficient to reproduce the same unsupported Azure chat +operation after a process restart or durable bootstrap. OpenAI documents `text-embedding-3-large` under the embeddings endpoint, separately from models supported by chat completions. Microsoft likewise demonstrates it with @@ -41,28 +42,37 @@ operation; it cannot make an embedding deployment execute a chat operation. ## Decision -Chat capability is a **shared runtime invariant**, not only a discovery filter. +Chat transport compatibility and general agent-role eligibility are separate +shared runtime invariants. A provider may expose an audio-capable model or a +policy classifier through Chat Completions while that model remains unsuitable +for ordinary thinker, worker, verifier, or synthesizer work. 1. Normalize provider prefixes and common separators in model identifiers. -2. Reject identifiers that clearly advertise embedding, reranking, transcription, - moderation/safety, image, audio/speech, realtime, or known embedding-family - transport semantics. -3. Apply the guard while parsing both OpenAI-compatible and Bytez catalogs. -4. Re-check the invariant before converting a discovery record to `ModelAgent`. -5. Re-check the invariant before writing chat pricing or running price-based chat - selection. -6. Remove stale non-chat agents from thinker, worker, verifier, and synthesizer +2. At the transport boundary, reject identifiers that clearly advertise embedding, + reranking, transcription, moderation-endpoint, image-generation, realtime, or + speech-only semantics. +3. Keep provider-documented audio and policy-classifier models transport-compatible + when they are served through Chat Completions. +4. At discovery and ordinary orchestration-role boundaries, additionally exclude + explicit guard, safety, and NemoGuard policy classifiers. +5. Apply the general-role guard while parsing both OpenAI-compatible and Bytez + catalogs and before converting, pricing, or cost-selecting a discovery record. +6. Remove stale ineligible agents from thinker, worker, verifier, and synthesizer ranking even if a durable configuration still contains them. -7. Reselect a generated workflow step that explicitly names a stale non-chat agent. -8. Remove non-chat agents from cross-agent failover candidates. -9. Reject a non-chat model at `ModelClient.chat()` before mock or network transport. -10. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, +7. Reselect a generated workflow step that explicitly names a stale ineligible + agent and omit such agents from planner inventory. +8. Remove ineligible agents from cross-agent failover candidates. +9. Apply the transport guard at `ModelClient.chat()`, `stream_chat()`, and + readiness probing before mock or network transport. +10. Fail closed when no general chat agent remains. +11. Leave unknown identifiers eligible without fabricating reasoning, tool, vision, or verification capabilities from their names. This is deliberately a conservative negative filter. A future capability registry may replace name-based exclusion with authenticated provider metadata, measured endpoint probes, and separate endpoint-specific pools. Until that evidence exists, -a clearly non-chat model fails closed at every chat boundary. +a clearly incompatible model fails closed at transport boundaries and a clearly +specialized policy model fails closed at general-role boundaries. ## Rejected response @@ -93,8 +103,10 @@ and provider/separator aliases. It verifies: - fail-closed behavior when the persisted pool contains only non-chat agents; - generated-plan reassignment away from a stale embedding agent; - exclusion from cross-agent failover; -- direct `ModelClient.chat()` rejection before transport; -- idempotent runtime-guard installation. +- direct and streaming `ModelClient` rejection before transport; +- readiness failure with a stable non-chat code before provider access; +- planner inventory and generated-plan isolation; +- distinction between chat-served audio/policy models and general agent roles. ## References @@ -108,3 +120,6 @@ https://learn.microsoft.com/en-us/azure/developer/ai/how-to/switching-endpoints OpenAI. (n.d.). *Data controls in the OpenAI platform: Default usage policies by endpoint*. Retrieved August 20, 2026, from https://platform.openai.com/docs/models/default-usage-policies-by-endpoint + +OpenAI. (n.d.). *GPT-audio model*. Retrieved August 20, 2026, from +https://developers.openai.com/api/docs/models/gpt-audio diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index 4883a9701..23e17d3da 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -12,6 +12,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, +) from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, register_credential, @@ -24,7 +27,6 @@ ProviderModelSource, agent_from_discovered, discover_provider_models, - is_chat_compatible_model_id, refresh_price_book, select_cheapest_discovered_agent, select_top_n_cheapest_discovered_agents, @@ -106,6 +108,8 @@ def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: {"id": "gpt-4o-mini-transcribe"}, {"id": "text-moderation-latest"}, {"id": "company/reranker-v2"}, + {"id": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3"}, + {"id": "gpt-audio"}, {"id": "gpt-5.2"}, {"id": "qwen/qwen3-235b-a22b-instruct"}, ] @@ -118,6 +122,7 @@ def test_embedding_deployments_never_enter_chat_agent_discovery() -> None: discovered = discover_provider_models(source) assert [model.model_id for model in discovered] == [ + "gpt-audio", "gpt-5.2", "qwen/qwen3-235b-a22b-instruct", ] @@ -173,7 +178,7 @@ def test_bytez_chat_catalog_still_rejects_non_chat_identifiers() -> None: def test_non_chat_discovery_cannot_be_converted_to_agent() -> None: """Keep manually constructed discovery rows from bypassing the parser filter.""" - with pytest.raises(ValueError, match="non-chat model"): + with pytest.raises(ValueError, match="general chat agent"): agent_from_discovered(_model("azure/text-embedding-3-large")) From 67b43414cbc667ef2b287869f30c4e54033a0e3b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:33:56 -0700 Subject: [PATCH 48/89] test: reproduce non-chat passthrough and batch bypasses --- ...t_chat_passthrough_capability_isolation.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/test_chat_passthrough_capability_isolation.py diff --git a/tests/test_chat_passthrough_capability_isolation.py b/tests/test_chat_passthrough_capability_isolation.py new file mode 100644 index 000000000..d200257c5 --- /dev/null +++ b/tests/test_chat_passthrough_capability_isolation.py @@ -0,0 +1,113 @@ +"""Regression tests for chat-capability checks on passthrough and batch paths.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 + + +def _embedding_agent() -> ModelAgent: + """Build the stale embedding agent from the production incident.""" + return ModelAgent( + "embedding_agent", + "azure/text-embedding-3-large", + base_url="mock://local", + ) + + +def _chat_agent() -> ModelAgent: + """Build one compatible fallback for explicit-model passthrough tests.""" + return ModelAgent( + "general_chat_agent", + "gpt-5.2", + base_url="mock://local", + tags=("writing",), + ) + + +@pytest.mark.parametrize("endpoint", ["chat/completions", "/v1/chat/completions", "responses"]) +def test_proxy_send_rejects_embedding_before_mock_or_network_transport(endpoint: str) -> None: + """Keep raw OpenAI passthrough from bypassing the chat transport invariant.""" + client = ModelClient() + + with pytest.raises(ValueError, match="chat-compatible"): + client.proxy_send( + _embedding_agent(), + endpoint, + { + "model": "azure/text-embedding-3-large", + "messages": [{"role": "user", "content": "Return JSON."}], + "input": "Return JSON.", + }, + ) + + +def test_explicit_embedding_model_cannot_bypass_through_structured_passthrough() -> None: + """Reject an explicitly requested stale embedding agent before raw proxy transport.""" + orchestrator = TaskOrchestrator([_embedding_agent(), _chat_agent()]) + + with pytest.raises(ValueError, match="chat-compatible"): + orchestrator.proxy_completion( + { + "model": "azure/text-embedding-3-large", + "messages": [{"role": "user", "content": "Return JSON."}], + "response_format": {"type": "json_object"}, + } + ) + + +def test_explicit_embedding_model_cannot_bypass_through_responses_passthrough() -> None: + """Apply the same transport contract to the Responses passthrough path.""" + orchestrator = TaskOrchestrator([_embedding_agent(), _chat_agent()]) + + with pytest.raises(ValueError, match="chat-compatible"): + orchestrator.proxy_completion( + { + "model": "azure/text-embedding-3-large", + "input": "Return JSON.", + }, + endpoint="responses", + ) + + +def test_batch_chat_rejects_embedding_before_mock_or_network_transport() -> None: + """Prevent direct batch callers from submitting embedding models as chat jobs.""" + client = ModelClient() + + with pytest.raises(ValueError, match="chat-compatible"): + client.batch_chat( + _embedding_agent(), + {"task_0": [{"role": "user", "content": "Return JSON."}]}, + ) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + ], +) +def test_chat_served_specialized_models_remain_valid_passthrough_transports(model_id: str) -> None: + """Do not turn ordinary-role exclusion into a false transport rejection.""" + client = ModelClient() + agent = ModelAgent("specialized_chat_agent", model_id, base_url="mock://local") + + response = client.proxy_send( + agent, + "chat/completions", + { + "model": model_id, + "messages": [{"role": "user", "content": "Classify this."}], + }, + ) + + assert response["object"] == "chat.completion" + assert response["model"] == model_id From 06da0c0c257adbf975e77e9f2b3b43829087dd69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 15:34:11 +0900 Subject: [PATCH 49/89] fix(discovery): wire provider-diverse bootstrap into CLI --- contextual_orchestrator/__main__.py | 6 ++-- tests/test_discover_models_cli.py | 46 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 622d2642c..3424629af 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -16,7 +16,7 @@ agent_id_for, discover_all_models, refresh_price_book, - select_top_n_cheapest_discovered_agents, + select_bootstrap_discovered_agents, ) from .orchestrator import ( CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1, @@ -211,7 +211,7 @@ def _discover_models_command(argv: list[str]) -> None: type=_non_negative_int, default=0, metavar="N", - help="Enable the N cheapest discovered agents in --agents-db (auto-optimization bootstrap; " + help="Enable a price-honest, provider-diverse discovered agent pool in --agents-db (auto-optimization bootstrap; " "requires --agents-db; 0 disables, the default, leaving every discovered agent inert).", ) args = parser.parse_args(argv) @@ -229,7 +229,7 @@ def _discover_models_command(argv: list[str]) -> None: ) bootstrap.sync_discovered_agents([agent_from_discovered(model) for model in discovered]) if args.enable_cheapest: - for model in select_top_n_cheapest_discovered_agents(discovered, price_book, args.enable_cheapest): + for model in select_bootstrap_discovered_agents(discovered, price_book, args.enable_cheapest): agent_id = agent_id_for(model) bootstrap.patch_agent("default", agent_id, {"status": "active"}) enabled_agent_ids.append(agent_id) diff --git a/tests/test_discover_models_cli.py b/tests/test_discover_models_cli.py index d00825ffd..444557f75 100644 --- a/tests/test_discover_models_cli.py +++ b/tests/test_discover_models_cli.py @@ -164,3 +164,49 @@ def urlopen(request, timeout=None): by_id = {agent.id: agent for agent in reloaded.candidates} assert by_id["openrouter_cheap_model"].disabled is False assert by_id["openai_pricey_model"].disabled is True + + +def test_enable_cheapest_bootstraps_independent_provider_families(tmp_path) -> None: + """CLI bootstrap must use the provider-diverse selector, not only the cheapest vendor.""" + from contextual_orchestrator import TaskOrchestrator + from contextual_orchestrator.orchestrator import ModelAgent + + set_backend(InMemoryCredentialBackend()) + register_credential("OPENAI_API_KEY", "sk-openai") + register_credential("OPENROUTER_API_KEY", "sk-router") + register_credential("NVIDIA_NIM_API_KEY", "nv-primary") + db_path = str(tmp_path / "pool.db") + stdout = StringIO() + + def urlopen(request, timeout=None): + host = urllib.parse.urlsplit(request.full_url).hostname + payloads = { + "api.openai.com": {"data": [{"id": "openai-model", "pricing": {"prompt": "0.001", "completion": "0.001"}}]}, + "openrouter.ai": {"data": [{"id": "router-model", "pricing": {"prompt": "0.000001", "completion": "0.000001"}}]}, + "integrate.api.nvidia.com": {"data": [{"id": "nim-model", "pricing": {"prompt": "0.000002", "completion": "0.000002"}}]}, + } + return _Response(payloads.get(host, {"data": []})) + + try: + with ( + patch.object( + sys, + "argv", + ["contextual-orchestrator", "discover-models", "--agents-db", db_path, "--enable-cheapest", "3"], + ), + patch.object(sys, "stdout", stdout), + patch("contextual_orchestrator.model_discovery.urllib.request.urlopen", side_effect=urlopen), + ): + main() + finally: + set_backend(None) + + report = json.loads(stdout.getvalue()) + assert report["enabled_agent_ids"] == [ + "openrouter_router_model", + "nvidia_nim_nim_model", + "openai_openai_model", + ] + reloaded = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) + enabled = {agent.id for agent in reloaded.candidates if not agent.disabled} + assert enabled - {"seed_agent"} == set(report["enabled_agent_ids"]) From 50b879baf5e239fbccd6eb7c545e8e6906def338 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:35:02 -0700 Subject: [PATCH 50/89] ci: verify passthrough and batch chat capability guards --- .../verify-chat-passthrough-capability.yml | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 .github/workflows/verify-chat-passthrough-capability.yml diff --git a/.github/workflows/verify-chat-passthrough-capability.yml b/.github/workflows/verify-chat-passthrough-capability.yml new file mode 100644 index 000000000..dc97b53bb --- /dev/null +++ b/.github/workflows/verify-chat-passthrough-capability.yml @@ -0,0 +1,183 @@ +name: Verify chat passthrough capability + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/verify-chat-passthrough-capability.yml + +permissions: + contents: read + +concurrency: + group: verify-chat-passthrough-capability + cancel-in-progress: false + +jobs: + prepare: + name: Prove RED, apply GREEN, and verify read-only + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + base_sha: ${{ steps.export.outputs.base_sha }} + patch_b64: ${{ steps.export.outputs.patch_b64 }} + steps: + - name: Checkout exact triggering commit without credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Confirm passthrough and batch bypass regressions are RED + shell: bash + run: | + set -euo pipefail + set +e + python -m pytest -q tests/test_chat_passthrough_capability_isolation.py > /tmp/red.log 2>&1 + status=$? + set -e + cat /tmp/red.log + test "${status}" -ne 0 + grep -Eq "DID NOT RAISE|Failed" /tmp/red.log + + - name: Apply transport guards to raw passthrough and batch chat + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + path = Path("contextual_orchestrator/orchestrator.py") + text = path.read_text(encoding="utf-8") + text = replace_once( + text, + ''' def proxy_send( + self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Passthrough a full request to one agent, returning the raw provider JSON.""" + if agent.base_url.startswith("mock://"): + ''', + ''' def proxy_send( + self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Passthrough a full request to one agent, returning the raw provider JSON.""" + normalized_endpoint = endpoint.strip("/") + if normalized_endpoint.startswith("v1/"): + normalized_endpoint = normalized_endpoint[3:] + if ( + normalized_endpoint in {"chat/completions", "completions", "responses"} + and not is_chat_compatible_model_id(agent.model) + ): + raise ValueError( + f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" + ) + if agent.base_url.startswith("mock://"): + ''', + "raw passthrough transport guard", + ) + text = replace_once( + text, + ''' ``requests`` maps a caller custom_id to its messages. Suited to eval/benchmark + workloads (24h completion window, ~half the price); real-time chat should keep + using ``chat``. The mock path answers synchronously so tests and local runs work. + """ + if agent.base_url.startswith("mock://"): + ''', + ''' ``requests`` maps a caller custom_id to its messages. Suited to eval/benchmark + workloads (24h completion window, ~half the price); real-time chat should keep + using ``chat``. The mock path answers synchronously so tests and local runs work. + """ + if not is_chat_compatible_model_id(agent.model): + raise ValueError( + f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" + ) + if agent.base_url.startswith("mock://"): + ''', + "batch chat transport guard", + ) + path.write_text(text, encoding="utf-8") + Path(".github/workflows/verify-chat-passthrough-capability.yml").unlink() + PY + + - name: Snapshot intended patch before executing tests + shell: bash + run: | + set -euo pipefail + git diff --binary > /tmp/expected.patch + test -s /tmp/expected.patch + + - name: Run focused and full verification + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_passthrough_capability_isolation.py tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py tests/test_openai_passthrough.py tests/test_local_mlx.py + python -m pytest -q + git diff --check + git diff --binary > /tmp/actual.patch + cmp /tmp/expected.patch /tmp/actual.patch + + - name: Export verified patch + id: export + shell: bash + run: | + set -euo pipefail + echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "patch_b64=$(base64 -w0 /tmp/expected.patch)" >> "$GITHUB_OUTPUT" + + commit: + name: Commit verified patch without executing repository code + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Checkout branch without persisted credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: false + + - name: Verify exact base, apply tested patch, and push + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} + PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" + printf '%s' "${PATCH_B64}" | base64 -d > /tmp/verified.patch + git apply --index /tmp/verified.patch + git diff --cached --check + actual_paths="$(git diff --cached --name-only | sort)" + expected_paths="$(cat <<'EOF' +.github/workflows/verify-chat-passthrough-capability.yml +contextual_orchestrator/orchestrator.py +EOF +)" + test "${actual_paths}" = "${expected_paths}" + test ! -e .github/workflows/verify-chat-passthrough-capability.yml + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "fix: guard raw and batch chat transport boundaries" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From a806e3914e47e03729e0fc4e0785b2d8a895130d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 15:41:31 +0900 Subject: [PATCH 51/89] fix: replace chat capability repair workflow with reviewed source --- .../verify-chat-passthrough-capability.yml | 183 ------------------ contextual_orchestrator/orchestrator.py | 14 ++ 2 files changed, 14 insertions(+), 183 deletions(-) delete mode 100644 .github/workflows/verify-chat-passthrough-capability.yml diff --git a/.github/workflows/verify-chat-passthrough-capability.yml b/.github/workflows/verify-chat-passthrough-capability.yml deleted file mode 100644 index dc97b53bb..000000000 --- a/.github/workflows/verify-chat-passthrough-capability.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Verify chat passthrough capability - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/verify-chat-passthrough-capability.yml - -permissions: - contents: read - -concurrency: - group: verify-chat-passthrough-capability - cancel-in-progress: false - -jobs: - prepare: - name: Prove RED, apply GREEN, and verify read-only - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - base_sha: ${{ steps.export.outputs.base_sha }} - patch_b64: ${{ steps.export.outputs.patch_b64 }} - steps: - - name: Checkout exact triggering commit without credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Confirm passthrough and batch bypass regressions are RED - shell: bash - run: | - set -euo pipefail - set +e - python -m pytest -q tests/test_chat_passthrough_capability_isolation.py > /tmp/red.log 2>&1 - status=$? - set -e - cat /tmp/red.log - test "${status}" -ne 0 - grep -Eq "DID NOT RAISE|Failed" /tmp/red.log - - - name: Apply transport guards to raw passthrough and batch chat - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - - def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - path = Path("contextual_orchestrator/orchestrator.py") - text = path.read_text(encoding="utf-8") - text = replace_once( - text, - ''' def proxy_send( - self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] - ) -> dict[str, Any]: - """Passthrough a full request to one agent, returning the raw provider JSON.""" - if agent.base_url.startswith("mock://"): - ''', - ''' def proxy_send( - self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] - ) -> dict[str, Any]: - """Passthrough a full request to one agent, returning the raw provider JSON.""" - normalized_endpoint = endpoint.strip("/") - if normalized_endpoint.startswith("v1/"): - normalized_endpoint = normalized_endpoint[3:] - if ( - normalized_endpoint in {"chat/completions", "completions", "responses"} - and not is_chat_compatible_model_id(agent.model) - ): - raise ValueError( - f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" - ) - if agent.base_url.startswith("mock://"): - ''', - "raw passthrough transport guard", - ) - text = replace_once( - text, - ''' ``requests`` maps a caller custom_id to its messages. Suited to eval/benchmark - workloads (24h completion window, ~half the price); real-time chat should keep - using ``chat``. The mock path answers synchronously so tests and local runs work. - """ - if agent.base_url.startswith("mock://"): - ''', - ''' ``requests`` maps a caller custom_id to its messages. Suited to eval/benchmark - workloads (24h completion window, ~half the price); real-time chat should keep - using ``chat``. The mock path answers synchronously so tests and local runs work. - """ - if not is_chat_compatible_model_id(agent.model): - raise ValueError( - f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" - ) - if agent.base_url.startswith("mock://"): - ''', - "batch chat transport guard", - ) - path.write_text(text, encoding="utf-8") - Path(".github/workflows/verify-chat-passthrough-capability.yml").unlink() - PY - - - name: Snapshot intended patch before executing tests - shell: bash - run: | - set -euo pipefail - git diff --binary > /tmp/expected.patch - test -s /tmp/expected.patch - - - name: Run focused and full verification - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_passthrough_capability_isolation.py tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py tests/test_openai_passthrough.py tests/test_local_mlx.py - python -m pytest -q - git diff --check - git diff --binary > /tmp/actual.patch - cmp /tmp/expected.patch /tmp/actual.patch - - - name: Export verified patch - id: export - shell: bash - run: | - set -euo pipefail - echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - echo "patch_b64=$(base64 -w0 /tmp/expected.patch)" >> "$GITHUB_OUTPUT" - - commit: - name: Commit verified patch without executing repository code - needs: prepare - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - steps: - - name: Checkout branch without persisted credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: false - - - name: Verify exact base, apply tested patch, and push - shell: bash - env: - GH_TOKEN: ${{ github.token }} - EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} - PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" - printf '%s' "${PATCH_B64}" | base64 -d > /tmp/verified.patch - git apply --index /tmp/verified.patch - git diff --cached --check - actual_paths="$(git diff --cached --name-only | sort)" - expected_paths="$(cat <<'EOF' -.github/workflows/verify-chat-passthrough-capability.yml -contextual_orchestrator/orchestrator.py -EOF -)" - test "${actual_paths}" = "${expected_paths}" - test ! -e .github/workflows/verify-chat-passthrough-capability.yml - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -m "fix: guard raw and batch chat transport boundaries" - git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 6366ae107..0cc65e404 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1148,6 +1148,16 @@ def proxy_send( self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] ) -> dict[str, Any]: """Passthrough a full request to one agent, returning the raw provider JSON.""" + normalized_endpoint = endpoint.strip("/") + if normalized_endpoint.startswith("v1/"): + normalized_endpoint = normalized_endpoint[3:] + if ( + normalized_endpoint in {"chat/completions", "completions", "responses"} + and not is_chat_compatible_model_id(agent.model) + ): + raise ValueError( + f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" + ) if agent.base_url.startswith("mock://"): return self._mock_raw(agent, endpoint, payload) destination = self._validate_provider(agent) # pragma: no cover @@ -1335,6 +1345,10 @@ def batch_chat( workloads (24h completion window, ~half the price); real-time chat should keep using ``chat``. The mock path answers synchronously so tests and local runs work. """ + if not is_chat_compatible_model_id(agent.model): + raise ValueError( + f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" + ) if agent.base_url.startswith("mock://"): results = { custom_id: {"content": self._mock(agent, messages), "usage": None} From a424c9da783c86e83e0700de425cad30880c1bfc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:49:28 -0700 Subject: [PATCH 52/89] test: preserve unknown model identifiers without capability fabrication --- ...est_chat_capability_unknown_identifiers.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_chat_capability_unknown_identifiers.py diff --git a/tests/test_chat_capability_unknown_identifiers.py b/tests/test_chat_capability_unknown_identifiers.py new file mode 100644 index 000000000..1e317cbc6 --- /dev/null +++ b/tests/test_chat_capability_unknown_identifiers.py @@ -0,0 +1,42 @@ +"""Regressions for conservative treatment of unknown model identifiers.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) + + +@pytest.mark.parametrize( + "model_id", + [ + "vendor/vanguard-7b", + "vendor/vanguard-instruct", + ], +) +def test_unknown_names_that_merely_end_with_guard_remain_eligible(model_id: str) -> None: + """Do not fabricate a policy-classifier capability from an unrelated word suffix.""" + assert is_chat_compatible_model_id(model_id) + assert is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "meta-llama/llama-guard-4-12b", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-topic-control", + ], +) +def test_explicit_policy_classifier_markers_remain_role_ineligible(model_id: str) -> None: + """Keep exact guard, safety, and NemoGuard markers out of general synthesis roles.""" + assert is_chat_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) From d71762dcc72368d1af379378714bd1086c9d1a62 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:49:59 -0700 Subject: [PATCH 53/89] fix: avoid inferring guard roles from unrelated suffixes --- contextual_orchestrator/chat_capability.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index fc36cb101..d149a3427 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -83,6 +83,5 @@ def is_general_chat_agent_model_id(model_id: str) -> bool: token == "safety" or token == "guard" or token.startswith("nemoguard") - or token.endswith("guard") for token in tokens ) From 6bd4e0c4427480e3b58fdf4733c193224049c8cc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 00:23:46 -0700 Subject: [PATCH 54/89] ci: verify final chat capability contract --- .../repair-chat-capability-contract-final.yml | 391 ++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 .github/workflows/repair-chat-capability-contract-final.yml diff --git a/.github/workflows/repair-chat-capability-contract-final.yml b/.github/workflows/repair-chat-capability-contract-final.yml new file mode 100644 index 000000000..e4b3ba2ed --- /dev/null +++ b/.github/workflows/repair-chat-capability-contract-final.yml @@ -0,0 +1,391 @@ +name: Verify final chat capability contract + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/repair-chat-capability-contract-final.yml + +permissions: + contents: read + +concurrency: + group: verify-final-chat-capability-contract + cancel-in-progress: false + +jobs: + prepare: + name: Build and test exact patch read-only + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + base_sha: ${{ steps.export.outputs.base_sha }} + patch_b64: ${{ steps.export.outputs.patch_b64 }} + steps: + - name: Checkout exact trigger without credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Materialize two-boundary capability contract + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from __future__ import annotations + + from pathlib import Path + import re + + chat = '''"""Classify chat transport compatibility and ordinary agent-role eligibility. + + Provider catalogs mix endpoint-only models with models served through an + OpenAI-compatible chat transport. Transport compatibility is not the same as + fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and + policy-classification models can use chat transport, while embedding, reranking, + transcription, moderation-endpoint, image-generation, realtime, and speech-only + models cannot. + """ + + from __future__ import annotations + + import re + + _MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") + _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "e5", + "embed", + "embedding", + "embeddings", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } + ) + _TRANSPORT_INCOMPATIBLE_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", + ) + + + def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + + def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use the ordinary chat transport. + + The classifier rejects only identifiers that clearly advertise an endpoint + family incompatible with chat messages. Audio-capable and safety-classifier + models remain transport-compatible because providers serve some of them over + ``/chat/completions``. + """ + tokens = _model_tokens(model_id) + if not tokens: + return False + for token in tokens: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: + return False + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): + return False + return True + + + def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models can use chat transport but are specialized + policy classifiers, not general answer synthesizers. This negative role gate + does not infer reasoning, coding, vision, or verification capabilities. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) + ''' + # The workflow YAML indentation is intentionally removed from generated source. + chat = "\n".join(line[10:] if line.startswith(" ") else line for line in chat.splitlines()) + "\n" + Path("contextual_orchestrator/chat_capability.py").write_text(chat, encoding="utf-8") + + discovery_path = Path("contextual_orchestrator/model_discovery.py") + discovery = discovery_path.read_text(encoding="utf-8") + discovery = re.sub( + r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", + "from .chat_capability import (\n" + " is_chat_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + discovery, + count=1, + flags=re.S, + ) + discovery = discovery.replace( + "is_chat_compatible_model_id(", + "is_general_chat_agent_model_id(", + ) + discovery = discovery.replace( + "Provider model-list discovery for chat-agent candidates.", + "Provider model-list discovery for general chat-agent candidates.", + ) + discovery = discovery.replace( + "Clearly non-chat identifiers are rejected", + "Endpoint-incompatible and specialized policy-model identifiers are rejected", + ) + discovery = discovery.replace( + '"""One chat-compatible model found on a provider, with reported pricing."""', + '"""One general chat-agent candidate found on a provider, with pricing."""', + ) + discovery = discovery.replace( + '"""Build a disabled chat ModelAgent or reject a non-chat discovery record."""', + '"""Build a disabled general chat agent or reject an ineligible record."""', + ) + discovery = discovery.replace( + 'raise ValueError("non-chat model cannot be converted into a chat agent")', + 'raise ValueError("model is not eligible for the general chat agent pool")', + ) + discovery_path.write_text(discovery, encoding="utf-8") + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator = re.sub( + r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", + "from .chat_capability import (\n" + " is_chat_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + orchestrator, + count=1, + flags=re.S, + ) + + def replace_in_method(source: str, method: str) -> str: + start_marker = f" def {method}(" + start = source.find(start_marker) + if start < 0: + raise SystemExit(f"missing method {method}") + next_def = source.find("\n def ", start + len(start_marker)) + end = len(source) if next_def < 0 else next_def + block = source[start:end] + block = block.replace( + "is_chat_compatible_model_id(", + "is_general_chat_agent_model_id(", + ) + return source[:start] + block + source[end:] + + for method in ( + "_plan_generated", + "_parse_workflow_plan", + "_ranked_agents", + "_failover_candidates", + ): + orchestrator = replace_in_method(orchestrator, method) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + transport_test = '''"""Regression tests for transport compatibility versus ordinary agent roles.""" + + from __future__ import annotations + + import sys + from pathlib import Path + + import pytest + + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, + is_general_chat_agent_model_id, + ) + + + @pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-4o-mini-audio-preview", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "meta-llama/Llama-Guard-4-12B", + ], + ) + def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: + """Do not reject models that provider contracts serve over chat.""" + assert is_chat_compatible_model_id(model_id) + + + @pytest.mark.parametrize( + "model_id", + [ + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "meta-llama/Llama-Guard-4-12B", + ], + ) + def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: + """Keep chat-served policy classifiers out of ordinary worker roles.""" + assert not is_general_chat_agent_model_id(model_id) + + + @pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ], + ) + def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: + """Preserve capable chat generation models for ordinary agent selection.""" + assert is_general_chat_agent_model_id(model_id) + + + @pytest.mark.parametrize( + "model_id", + [ + "azure/text-embedding-3-large", + "text_embedding_3_large", + "company/reranker-v2", + "gpt-4o-mini-transcribe", + "omni-moderation-latest", + "gpt-image-1", + "gpt-realtime", + "tts-1", + ], + ) + def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: + """Reject clear non-chat endpoint families before transport or role routing.""" + assert not is_chat_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) + ''' + transport_test = "\n".join(line[10:] if line.startswith(" ") else line for line in transport_test.splitlines()) + "\n" + Path("tests/test_chat_transport_role_separation.py").write_text( + transport_test, + encoding="utf-8", + ) + + incident_test = Path("tests/test_chat_model_capability_isolation.py") + incident = incident_test.read_text(encoding="utf-8") + incident = incident.replace( + 'with pytest.raises(ValueError, match="non-chat model"):', + 'with pytest.raises(ValueError, match="general chat agent pool"):', + ) + incident_test.write_text(incident, encoding="utf-8") + + doctoring = Path("docs/doctoring/embedding-chat-capability-isolation.md") + body = doctoring.read_text(encoding="utf-8") + if "### Transport-versus-role evidence" not in body: + section = ''' + ### Transport-versus-role evidence + + The runtime does not equate a token in a model name with an endpoint family. + OpenAI publishes audio-capable generation through its audio/chat guidance, and + NVIDIA NIM exposes OpenAI-compatible chat APIs for model families that include + specialized policy classifiers. Therefore `audio`, `guard`, and `safety` are not + transport-level disqualifiers. Explicit guard/safety identifiers are excluded at + the ordinary orchestration-role boundary until a specialized role and + authenticated capability record exist. + + ''' + section = "\n".join(line[10:] if line.startswith(" ") else line for line in section.splitlines()) + "\n" + anchor = "\n## References\n" + body = body.replace(anchor, "\n" + section + anchor, 1) if anchor in body else body.rstrip() + "\n\n" + section + references = ( + "NVIDIA Corporation. (n.d.). *NVIDIA NIM for large language models: OpenAI-compatible API reference*. Retrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html", + "OpenAI. (n.d.). *Audio and speech*. Retrieved August 20, 2026, from https://platform.openai.com/docs/guides/audio", + ) + for reference in references: + if reference not in body: + body = body.rstrip() + "\n\n" + reference + "\n" + doctoring.write_text(body, encoding="utf-8") + + # No temporary workflow may survive in the verified patch. + for pattern in ("repair-chat-*.yml", "repair-chat-*.yaml"): + for workflow in Path(".github/workflows").glob(pattern): + workflow.unlink() + PY + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Run focused and full verification + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py + python -m pytest -q + git diff --check + + - name: Export tested patch + id: export + shell: bash + run: | + set -euo pipefail + git diff --binary > /tmp/final-chat-capability.patch + test -s /tmp/final-chat-capability.patch + echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "patch_b64=$(base64 -w0 /tmp/final-chat-capability.patch)" >> "$GITHUB_OUTPUT" + + commit: + name: Commit tested patch without executing repository code + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Checkout branch without credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: false + + - name: Verify base, apply tested patch, and push + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} + PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + printf '%s' "$PATCH_B64" | base64 -d > /tmp/final-chat-capability.patch + git apply --index /tmp/final-chat-capability.patch + git diff --cached --check + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "fix: finalize chat transport capability boundary" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From 290838c9daf820ba83d9190dc61ba089805a7fdd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:25:02 +0900 Subject: [PATCH 55/89] chore: remove completed capability repair workflow --- .../repair-chat-capability-contract-final.yml | 391 ------------------ 1 file changed, 391 deletions(-) delete mode 100644 .github/workflows/repair-chat-capability-contract-final.yml diff --git a/.github/workflows/repair-chat-capability-contract-final.yml b/.github/workflows/repair-chat-capability-contract-final.yml deleted file mode 100644 index e4b3ba2ed..000000000 --- a/.github/workflows/repair-chat-capability-contract-final.yml +++ /dev/null @@ -1,391 +0,0 @@ -name: Verify final chat capability contract - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/repair-chat-capability-contract-final.yml - -permissions: - contents: read - -concurrency: - group: verify-final-chat-capability-contract - cancel-in-progress: false - -jobs: - prepare: - name: Build and test exact patch read-only - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - base_sha: ${{ steps.export.outputs.base_sha }} - patch_b64: ${{ steps.export.outputs.patch_b64 }} - steps: - - name: Checkout exact trigger without credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Materialize two-boundary capability contract - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from __future__ import annotations - - from pathlib import Path - import re - - chat = '''"""Classify chat transport compatibility and ordinary agent-role eligibility. - - Provider catalogs mix endpoint-only models with models served through an - OpenAI-compatible chat transport. Transport compatibility is not the same as - fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and - policy-classification models can use chat transport, while embedding, reranking, - transcription, moderation-endpoint, image-generation, realtime, and speech-only - models cannot. - """ - - from __future__ import annotations - - import re - - _MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") - _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( - { - "bge", - "e5", - "embed", - "embedding", - "embeddings", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } - ) - _TRANSPORT_INCOMPATIBLE_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", - ) - - - def _model_tokens(model_id: str) -> tuple[str, ...]: - """Normalize one provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: - return () - return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - - - def is_chat_compatible_model_id(model_id: str) -> bool: - """Return whether an identifier can use the ordinary chat transport. - - The classifier rejects only identifiers that clearly advertise an endpoint - family incompatible with chat messages. Audio-capable and safety-classifier - models remain transport-compatible because providers serve some of them over - ``/chat/completions``. - """ - tokens = _model_tokens(model_id) - if not tokens: - return False - for token in tokens: - if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: - return False - if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): - return False - return True - - - def is_general_chat_agent_model_id(model_id: str) -> bool: - """Return whether a chat model may enter ordinary orchestration roles. - - Explicit guard and safety models can use chat transport but are specialized - policy classifiers, not general answer synthesizers. This negative role gate - does not infer reasoning, coding, vision, or verification capabilities. - """ - tokens = _model_tokens(model_id) - if not tokens or not is_chat_compatible_model_id(model_id): - return False - return not any( - token == "safety" - or token == "guard" - or token.startswith("nemoguard") - or token.endswith("guard") - for token in tokens - ) - ''' - # The workflow YAML indentation is intentionally removed from generated source. - chat = "\n".join(line[10:] if line.startswith(" ") else line for line in chat.splitlines()) + "\n" - Path("contextual_orchestrator/chat_capability.py").write_text(chat, encoding="utf-8") - - discovery_path = Path("contextual_orchestrator/model_discovery.py") - discovery = discovery_path.read_text(encoding="utf-8") - discovery = re.sub( - r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", - "from .chat_capability import (\n" - " is_chat_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - discovery, - count=1, - flags=re.S, - ) - discovery = discovery.replace( - "is_chat_compatible_model_id(", - "is_general_chat_agent_model_id(", - ) - discovery = discovery.replace( - "Provider model-list discovery for chat-agent candidates.", - "Provider model-list discovery for general chat-agent candidates.", - ) - discovery = discovery.replace( - "Clearly non-chat identifiers are rejected", - "Endpoint-incompatible and specialized policy-model identifiers are rejected", - ) - discovery = discovery.replace( - '"""One chat-compatible model found on a provider, with reported pricing."""', - '"""One general chat-agent candidate found on a provider, with pricing."""', - ) - discovery = discovery.replace( - '"""Build a disabled chat ModelAgent or reject a non-chat discovery record."""', - '"""Build a disabled general chat agent or reject an ineligible record."""', - ) - discovery = discovery.replace( - 'raise ValueError("non-chat model cannot be converted into a chat agent")', - 'raise ValueError("model is not eligible for the general chat agent pool")', - ) - discovery_path.write_text(discovery, encoding="utf-8") - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - orchestrator = re.sub( - r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", - "from .chat_capability import (\n" - " is_chat_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - orchestrator, - count=1, - flags=re.S, - ) - - def replace_in_method(source: str, method: str) -> str: - start_marker = f" def {method}(" - start = source.find(start_marker) - if start < 0: - raise SystemExit(f"missing method {method}") - next_def = source.find("\n def ", start + len(start_marker)) - end = len(source) if next_def < 0 else next_def - block = source[start:end] - block = block.replace( - "is_chat_compatible_model_id(", - "is_general_chat_agent_model_id(", - ) - return source[:start] + block + source[end:] - - for method in ( - "_plan_generated", - "_parse_workflow_plan", - "_ranked_agents", - "_failover_candidates", - ): - orchestrator = replace_in_method(orchestrator, method) - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - transport_test = '''"""Regression tests for transport compatibility versus ordinary agent roles.""" - - from __future__ import annotations - - import sys - from pathlib import Path - - import pytest - - sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - - from contextual_orchestrator.chat_capability import ( # noqa: E402 - is_chat_compatible_model_id, - is_general_chat_agent_model_id, - ) - - - @pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-4o-mini-audio-preview", - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "meta-llama/Llama-Guard-4-12B", - ], - ) - def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: - """Do not reject models that provider contracts serve over chat.""" - assert is_chat_compatible_model_id(model_id) - - - @pytest.mark.parametrize( - "model_id", - [ - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "nvidia/llama-3.1-nemoguard-8b-content-safety", - "meta-llama/Llama-Guard-4-12B", - ], - ) - def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: - """Keep chat-served policy classifiers out of ordinary worker roles.""" - assert not is_general_chat_agent_model_id(model_id) - - - @pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-5.2", - "qwen/qwen3-235b-a22b-instruct", - ], - ) - def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: - """Preserve capable chat generation models for ordinary agent selection.""" - assert is_general_chat_agent_model_id(model_id) - - - @pytest.mark.parametrize( - "model_id", - [ - "azure/text-embedding-3-large", - "text_embedding_3_large", - "company/reranker-v2", - "gpt-4o-mini-transcribe", - "omni-moderation-latest", - "gpt-image-1", - "gpt-realtime", - "tts-1", - ], - ) - def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: - """Reject clear non-chat endpoint families before transport or role routing.""" - assert not is_chat_compatible_model_id(model_id) - assert not is_general_chat_agent_model_id(model_id) - ''' - transport_test = "\n".join(line[10:] if line.startswith(" ") else line for line in transport_test.splitlines()) + "\n" - Path("tests/test_chat_transport_role_separation.py").write_text( - transport_test, - encoding="utf-8", - ) - - incident_test = Path("tests/test_chat_model_capability_isolation.py") - incident = incident_test.read_text(encoding="utf-8") - incident = incident.replace( - 'with pytest.raises(ValueError, match="non-chat model"):', - 'with pytest.raises(ValueError, match="general chat agent pool"):', - ) - incident_test.write_text(incident, encoding="utf-8") - - doctoring = Path("docs/doctoring/embedding-chat-capability-isolation.md") - body = doctoring.read_text(encoding="utf-8") - if "### Transport-versus-role evidence" not in body: - section = ''' - ### Transport-versus-role evidence - - The runtime does not equate a token in a model name with an endpoint family. - OpenAI publishes audio-capable generation through its audio/chat guidance, and - NVIDIA NIM exposes OpenAI-compatible chat APIs for model families that include - specialized policy classifiers. Therefore `audio`, `guard`, and `safety` are not - transport-level disqualifiers. Explicit guard/safety identifiers are excluded at - the ordinary orchestration-role boundary until a specialized role and - authenticated capability record exist. - - ''' - section = "\n".join(line[10:] if line.startswith(" ") else line for line in section.splitlines()) + "\n" - anchor = "\n## References\n" - body = body.replace(anchor, "\n" + section + anchor, 1) if anchor in body else body.rstrip() + "\n\n" + section - references = ( - "NVIDIA Corporation. (n.d.). *NVIDIA NIM for large language models: OpenAI-compatible API reference*. Retrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html", - "OpenAI. (n.d.). *Audio and speech*. Retrieved August 20, 2026, from https://platform.openai.com/docs/guides/audio", - ) - for reference in references: - if reference not in body: - body = body.rstrip() + "\n\n" + reference + "\n" - doctoring.write_text(body, encoding="utf-8") - - # No temporary workflow may survive in the verified patch. - for pattern in ("repair-chat-*.yml", "repair-chat-*.yaml"): - for workflow in Path(".github/workflows").glob(pattern): - workflow.unlink() - PY - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Run focused and full verification - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py - python -m pytest -q - git diff --check - - - name: Export tested patch - id: export - shell: bash - run: | - set -euo pipefail - git diff --binary > /tmp/final-chat-capability.patch - test -s /tmp/final-chat-capability.patch - echo "base_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - echo "patch_b64=$(base64 -w0 /tmp/final-chat-capability.patch)" >> "$GITHUB_OUTPUT" - - commit: - name: Commit tested patch without executing repository code - needs: prepare - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - steps: - - name: Checkout branch without credentials - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: false - - - name: Verify base, apply tested patch, and push - shell: bash - env: - GH_TOKEN: ${{ github.token }} - EXPECTED_SHA: ${{ needs.prepare.outputs.base_sha }} - PATCH_B64: ${{ needs.prepare.outputs.patch_b64 }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" - printf '%s' "$PATCH_B64" | base64 -d > /tmp/final-chat-capability.patch - git apply --index /tmp/final-chat-capability.patch - git diff --cached --check - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -m "fix: finalize chat transport capability boundary" - git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:fix/chat-capability-isolation-embedding-models From a06311311f9fbf7e38fa65575465baa0e18b98f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 00:29:12 -0700 Subject: [PATCH 56/89] test(discovery): expose ambiguous price and duplicate pool gaps --- tests/test_discovery_bootstrap_selection.py | 151 ++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py index 7357e3cec..cb4abba2c 100644 --- a/tests/test_discovery_bootstrap_selection.py +++ b/tests/test_discovery_bootstrap_selection.py @@ -7,6 +7,7 @@ from contextual_orchestrator import model_discovery from contextual_orchestrator.model_discovery import ( DiscoveredModel, + refresh_price_book, select_cheapest_discovered_agent, select_top_n_cheapest_discovered_agents, ) @@ -24,10 +25,34 @@ def _model(provider_name: str, model_id: str) -> DiscoveredModel: ) +def _priced_model( + provider_name: str, + model_id: str, + *, + prompt_price_per_1k: float | None, + completion_price_per_1k: float | None, + currency_code: str = "USD", +) -> DiscoveredModel: + """Build one discovery row carrying provider-reported price evidence.""" + base = _model(provider_name, model_id) + return DiscoveredModel( + provider_name=base.provider_name, + model_id=base.model_id, + credential_name=base.credential_name, + chat_base_url=base.chat_base_url, + auth_scheme=base.auth_scheme, + prompt_price_per_1k=prompt_price_per_1k, + completion_price_per_1k=completion_price_per_1k, + currency_code=currency_code, + ) + + def _set_price( price_book: PriceBook, model: DiscoveredModel, price_per_1k: float, + *, + currency_code: str = "USD", ) -> None: """Record one known symmetric prompt/completion price.""" price_book.set_price( @@ -36,6 +61,7 @@ def _set_price( model.model_id, price_per_1k, price_per_1k, + currency_code, ) ) @@ -53,6 +79,131 @@ def test_unpriced_discovered_model_is_unknown_not_free() -> None: ) == [priced, unpriced] +def test_partial_provider_price_is_unknown_instead_of_fabricating_a_free_component() -> None: + """A missing prompt or completion price cannot become an invented zero.""" + price_book = PriceBook(InMemoryConfigStore()) + partial = _priced_model( + "partial_vendor", + "partial-model", + prompt_price_per_1k=0.001, + completion_price_per_1k=None, + ) + complete = _priced_model( + "openrouter", + "complete-model", + prompt_price_per_1k=1.0, + completion_price_per_1k=1.0, + ) + + assert refresh_price_book([partial, complete], price_book) == 1 + assert price_book.get_price(partial.provider_name, partial.model_id) is None + assert select_cheapest_discovered_agent([partial, complete], price_book) is complete + + +def test_invalid_catalog_prices_are_unknown_not_trusted_cost_evidence() -> None: + """Reject negative, non-finite, and boolean provider price values.""" + assert model_discovery._price_per_1k("-0.000001") is None + assert model_discovery._price_per_1k("nan") is None + assert model_discovery._price_per_1k("inf") is None + assert model_discovery._price_per_1k(True) is None + assert model_discovery._price_per_1k("0") == 0.0 + + +def test_invalid_or_cross_currency_price_rows_do_not_outrank_comparable_usd_cost() -> None: + """Only finite non-negative prices in the configured currency are comparable.""" + price_book = PriceBook(InMemoryConfigStore(), default_currency="USD") + valid = _model("openrouter", "valid-model") + negative = _model("negative_vendor", "negative-model") + non_finite = _model("nan_vendor", "nan-model") + foreign = _model("eur_vendor", "eur-model") + + _set_price(price_book, valid, 1.0) + _set_price(price_book, negative, -100.0) + _set_price(price_book, non_finite, float("nan")) + _set_price(price_book, foreign, 0.000001, currency_code="EUR") + + assert select_cheapest_discovered_agent( + [negative, non_finite, foreign, valid], + price_book, + ) is valid + + +def test_duplicate_serving_identity_cannot_consume_bootstrap_capacity() -> None: + """A repeated provider/model row must not masquerade as failover diversity.""" + selector = getattr( + model_discovery, + "select_bootstrap_discovered_agents", + None, + ) + assert callable(selector), "missing provider-diverse bootstrap selector" + + price_book = PriceBook(InMemoryConfigStore()) + duplicate_first = _model("openrouter", "same-model") + duplicate_second = _model("openrouter", "same-model") + independent = _model("openai", "independent-model") + _set_price(price_book, duplicate_first, 0.01) + _set_price(price_book, independent, 0.02) + + selected = selector( + [duplicate_second, independent, duplicate_first], + price_book, + 3, + ) + top_n = select_top_n_cheapest_discovered_agents( + [duplicate_second, independent, duplicate_first], + price_book, + 3, + ) + + assert [ + (model.provider_name, model.model_id) + for model in selected + ] == [ + ("openrouter", "same-model"), + ("openai", "independent-model"), + ] + assert [ + (model.provider_name, model.model_id) + for model in top_n + ] == [ + ("openrouter", "same-model"), + ("openai", "independent-model"), + ] + + +def test_conflicting_duplicate_prices_are_withheld_as_ambiguous() -> None: + """Do not let provider row order decide the trusted price for one agent id.""" + price_book = PriceBook(InMemoryConfigStore()) + cheap_claim = _priced_model( + "openrouter", + "duplicate-model", + prompt_price_per_1k=0.000001, + completion_price_per_1k=0.000001, + ) + expensive_claim = _priced_model( + "openrouter", + "duplicate-model", + prompt_price_per_1k=100.0, + completion_price_per_1k=100.0, + ) + complete = _priced_model( + "openai", + "complete-model", + prompt_price_per_1k=1.0, + completion_price_per_1k=1.0, + ) + + assert refresh_price_book( + [cheap_claim, expensive_claim, complete], + price_book, + ) == 1 + assert price_book.get_price("openrouter", "duplicate-model") is None + assert select_cheapest_discovered_agent( + [cheap_claim, expensive_claim, complete], + price_book, + ) is complete + + def test_bootstrap_selector_prefers_provider_diversity_before_duplicates() -> None: """The initial failover pool must span providers before repeating one.""" selector = getattr( From 2539ad5797ee63a17d866851658edf031833826f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:32:36 +0900 Subject: [PATCH 57/89] fix(discovery): reject untrusted duplicate price evidence --- contextual_orchestrator/model_discovery.py | 77 ++++++++++++++++++---- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index a0b955c47..329798559 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -14,6 +14,7 @@ from __future__ import annotations import json +import math import re import urllib.error import urllib.request @@ -122,12 +123,15 @@ def _fetch_json(url: str, *, api_key: str, auth_scheme: str, timeout: float) -> def _price_per_1k(value: Any) -> float | None: """OpenAI-compatible providers report USD price per single token; convert to per-1K.""" - if value is None: + if value is None or isinstance(value, bool): return None try: - return float(value) * 1000 + price = float(value) except (TypeError, ValueError): return None + if not math.isfinite(price) or price < 0: + return None + return price * 1000 def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: @@ -253,30 +257,67 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook """ from .cost_ledger import PriceEntry - written = 0 + claims: dict[tuple[str, str], list[tuple[float | None, float | None, str]]] = {} for model in discovered: - if model.prompt_price_per_1k is None and model.completion_price_per_1k is None: + identity = (model.provider_name, model.model_id) + claims.setdefault(identity, []).append( + ( + model.prompt_price_per_1k, + model.completion_price_per_1k, + model.currency_code, + ) + ) + + written = 0 + for (provider_name, model_name), rows in claims.items(): + if any( + not _is_valid_price_component(value) + for row in rows + for value in row[:2] + ) or len(set(rows)) != 1: continue + prompt_price, completion_price, currency_code = rows[0] price_book.set_price( PriceEntry( - provider_name=model.provider_name, - model_name=model.model_id, - prompt_price_per_1k=model.prompt_price_per_1k or 0.0, - completion_price_per_1k=model.completion_price_per_1k or 0.0, - currency_code=model.currency_code, + provider_name=provider_name, + model_name=model_name, + prompt_price_per_1k=prompt_price, + completion_price_per_1k=completion_price, + currency_code=currency_code, ) ) written += 1 return written +def _is_valid_price_component(value: Any) -> bool: + """Return whether one provider price is finite and non-negative.""" + return ( + value is not None + and not isinstance(value, bool) + and isinstance(value, (int, float)) + and math.isfinite(value) + and value >= 0 + ) + + +def _unique_discovered_models( + discovered: list[DiscoveredModel], +) -> list[DiscoveredModel]: + """Keep one deterministic row per provider/model serving identity.""" + unique: dict[tuple[str, str], DiscoveredModel] = {} + for model in discovered: + unique.setdefault((model.provider_name, model.model_id), model) + return list(unique.values()) + + def _discovery_price_key( model: DiscoveredModel, price_book: "PriceBook", ) -> tuple[int, float, str, str]: """Rank known prices first, then deterministically order unknown prices.""" entry = price_book.get_price(model.provider_name, model.model_id) - if entry is None: + if entry is None or not _is_trustworthy_price_entry(entry, price_book): return (1, 0.0, model.provider_name, model.model_id) cost, _currency = price_book.compute_cost( model.provider_name, @@ -287,6 +328,15 @@ def _discovery_price_key( return (0, cost, model.provider_name, model.model_id) +def _is_trustworthy_price_entry(entry: Any, price_book: "PriceBook") -> bool: + """Accept only complete, finite prices in the book's comparison currency.""" + return ( + entry.currency_code == price_book.default_currency + and _is_valid_price_component(entry.prompt_price_per_1k) + and _is_valid_price_component(entry.completion_price_per_1k) + ) + + def _provider_family(provider_name: str) -> str: """Collapse credentials that share one upstream provider outage domain.""" if provider_name in {"nvidia_nim", "nvidia_nim_sub"}: @@ -303,9 +353,10 @@ def select_cheapest_discovered_agent( sort first; when every candidate is unpriced, provider and model identifiers provide deterministic fallback ordering without inventing a monetary value. """ - if not discovered: + unique = _unique_discovered_models(discovered) + if not unique: return None - return min(discovered, key=lambda model: _discovery_price_key(model, price_book)) + return min(unique, key=lambda model: _discovery_price_key(model, price_book)) def select_top_n_cheapest_discovered_agents( @@ -314,6 +365,7 @@ def select_top_n_cheapest_discovered_agents( """Return up to ``limit`` candidates with known prices before unknown ones.""" if limit <= 0 or not discovered: return [] + discovered = _unique_discovered_models(discovered) return sorted( discovered, key=lambda model: _discovery_price_key(model, price_book), @@ -335,6 +387,7 @@ def select_bootstrap_discovered_agents( """ if limit <= 0 or not discovered: return [] + discovered = _unique_discovered_models(discovered) ranked = sorted( discovered, From a78d3a60842360d861ae4fcafb225e763963fe24 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:36:14 +0900 Subject: [PATCH 58/89] fix(discovery): keep duplicate evidence comparison total --- contextual_orchestrator/model_discovery.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 329798559..25746b30a 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -274,9 +274,11 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook not _is_valid_price_component(value) for row in rows for value in row[:2] - ) or len(set(rows)) != 1: + ) or any(row != rows[0] for row in rows[1:]): continue prompt_price, completion_price, currency_code = rows[0] + if prompt_price is None or completion_price is None: + continue price_book.set_price( PriceEntry( provider_name=provider_name, From 75e15afecc9d4227097e0891c6c14aed208c2339 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 00:38:35 -0700 Subject: [PATCH 59/89] test(discovery): expose overflow, corrupt-row, and currency gaps --- tests/test_discovery_bootstrap_selection.py | 59 +++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py index cb4abba2c..e6a3c0ade 100644 --- a/tests/test_discovery_bootstrap_selection.py +++ b/tests/test_discovery_bootstrap_selection.py @@ -109,6 +109,65 @@ def test_invalid_catalog_prices_are_unknown_not_trusted_cost_evidence() -> None: assert model_discovery._price_per_1k("0") == 0.0 +def test_huge_price_values_remain_unknown_without_crashing_discovery_or_ranking() -> None: + """Unbounded JSON or KV integers must not terminate bootstrap selection.""" + huge_price = 10**10000 + assert model_discovery._price_per_1k(huge_price) is None + + price_book = PriceBook(InMemoryConfigStore()) + huge = _model("huge_vendor", "huge-model") + valid = _model("openrouter", "valid-model") + _set_price(price_book, huge, huge_price) + _set_price(price_book, valid, 1.0) + + assert select_cheapest_discovered_agent([huge, valid], price_book) is valid + + +def test_malformed_price_book_row_is_unknown_instead_of_crashing_selection() -> None: + """A corrupt persisted price row must not take down the serving bootstrap.""" + store = InMemoryConfigStore() + store.set( + "llm_price_entries", + "broken_vendor:broken-model", + { + "provider_name": "broken_vendor", + "model_name": "broken-model", + "prompt_price_per_1k": "not-a-number", + "completion_price_per_1k": 0.001, + "currency_code": "USD", + }, + ) + price_book = PriceBook(store) + broken = _model("broken_vendor", "broken-model") + valid = _model("openrouter", "valid-model") + _set_price(price_book, valid, 1.0) + + assert select_cheapest_discovered_agent([broken, valid], price_book) is valid + + +def test_refresh_counts_only_complete_prices_in_the_comparison_currency() -> None: + """Cross-currency evidence is unknown until an explicit conversion exists.""" + price_book = PriceBook(InMemoryConfigStore(), default_currency="USD") + usd = _priced_model( + "openrouter", + "usd-model", + prompt_price_per_1k=1.0, + completion_price_per_1k=1.0, + currency_code="USD", + ) + eur = _priced_model( + "eur_vendor", + "eur-model", + prompt_price_per_1k=0.001, + completion_price_per_1k=0.001, + currency_code="EUR", + ) + + assert refresh_price_book([eur, usd], price_book) == 1 + assert price_book.get_price("eur_vendor", "eur-model") is None + assert price_book.get_price("openrouter", "usd-model") is not None + + def test_invalid_or_cross_currency_price_rows_do_not_outrank_comparable_usd_cost() -> None: """Only finite non-negative prices in the configured currency are comparable.""" price_book = PriceBook(InMemoryConfigStore(), default_currency="USD") From 183e7a8be1d6ee455843e2737869f75e61059d93 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 00:44:53 -0700 Subject: [PATCH 60/89] fix(discovery): fail closed on malformed and incomparable price evidence --- contextual_orchestrator/model_discovery.py | 224 +++++++++++++-------- 1 file changed, 136 insertions(+), 88 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 25746b30a..e5f171d91 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -18,7 +18,7 @@ import re import urllib.error import urllib.request -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any from .credentials import get_credential @@ -121,17 +121,70 @@ def _fetch_json(url: str, *, api_key: str, auth_scheme: str, timeout: float) -> return json.loads(response.read().decode("utf-8")) +def _valid_price_component(value: object) -> bool: + """Return whether one price component is finite, numeric, and non-negative.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + numeric = float(value) + except (TypeError, ValueError, OverflowError): + return False + return math.isfinite(numeric) and numeric >= 0.0 + + def _price_per_1k(value: Any) -> float | None: - """OpenAI-compatible providers report USD price per single token; convert to per-1K.""" + """Convert a trustworthy per-token USD price to per-1K, else return unknown.""" if value is None or isinstance(value, bool): return None try: - price = float(value) - except (TypeError, ValueError): - return None - if not math.isfinite(price) or price < 0: + per_1k = float(value) * 1000 + except (TypeError, ValueError, OverflowError): return None - return price * 1000 + return per_1k if _valid_price_component(per_1k) else None + + +def _serving_identity(model: DiscoveredModel) -> tuple[str, str]: + """Return the durable agent identity used by discovery synchronization.""" + return (model.provider_name, model.model_id) + + +def _source_tiebreaker(model: DiscoveredModel) -> tuple[str, str, str, str]: + """Choose deterministic transport metadata for an ambiguous duplicate row.""" + return ( + model.credential_name, + model.chat_base_url, + model.auth_scheme, + model.currency_code, + ) + + +def _deduplicate_discovered_models( + discovered: list[DiscoveredModel], +) -> list[DiscoveredModel]: + """Collapse duplicate agent identities and withhold conflicting price evidence. + + Exact duplicate catalog rows become one candidate. When the same provider/model + identity is repeated with conflicting metadata or prices, one deterministic + transport record is retained but its prices become unknown. Provider row order + therefore cannot fabricate a cheaper bootstrap candidate or consume failover + capacity twice. + """ + unique: dict[tuple[str, str], DiscoveredModel] = {} + for model in discovered: + identity = _serving_identity(model) + previous = unique.get(identity) + if previous is None: + unique[identity] = model + continue + if previous == model: + continue + chosen = min((previous, model), key=_source_tiebreaker) + unique[identity] = replace( + chosen, + prompt_price_per_1k=None, + completion_price_per_1k=None, + ) + return list(unique.values()) def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: @@ -155,7 +208,7 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ completion_price_per_1k=_price_per_1k(pricing.get("completion")), ) ) - return discovered + return _deduplicate_discovered_models(discovered) def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: @@ -178,7 +231,7 @@ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredMo # per-1k pricing unset is more honest than a misleading estimate. ) ) - return discovered + return _deduplicate_discovered_models(discovered) def discover_provider_models( @@ -217,7 +270,7 @@ def discover_all_models( discovered.extend(discover_provider_models(source, timeout=timeout)) except ProviderDiscoveryError as exc: errors.append(exc) - return discovered, errors + return _deduplicate_discovered_models(discovered), errors _SLUG_RE = re.compile(r"[^a-z0-9]+") @@ -248,97 +301,87 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> ) +def _currency_is_comparable(currency_code: object, default_currency: object) -> bool: + """Return whether two ISO-style currency codes can be compared directly.""" + return ( + isinstance(currency_code, str) + and isinstance(default_currency, str) + and currency_code.strip().upper() == default_currency.strip().upper() + and bool(currency_code.strip()) + ) + + def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook") -> int: - """Write every discovered model's known pricing into the price book. + """Write complete, comparable provider pricing into the discovery price book. - Returns the number of price rows written. A model without provider-reported - pricing is skipped rather than defaulted to zero. Discovery ranking treats - that absence as unknown price evidence, never as proof that the model is free. + Both prompt and completion prices are required for the fixed 1K+1K ranking + workload. Partial, conflicting, non-finite, negative, or cross-currency + evidence remains unknown rather than acquiring an invented zero component. """ from .cost_ledger import PriceEntry - claims: dict[tuple[str, str], list[tuple[float | None, float | None, str]]] = {} - for model in discovered: - identity = (model.provider_name, model.model_id) - claims.setdefault(identity, []).append( - ( - model.prompt_price_per_1k, - model.completion_price_per_1k, + written = 0 + for model in _deduplicate_discovered_models(discovered): + if not ( + _valid_price_component(model.prompt_price_per_1k) + and _valid_price_component(model.completion_price_per_1k) + and _currency_is_comparable( model.currency_code, + price_book.default_currency, ) - ) - - written = 0 - for (provider_name, model_name), rows in claims.items(): - if any( - not _is_valid_price_component(value) - for row in rows - for value in row[:2] - ) or any(row != rows[0] for row in rows[1:]): - continue - prompt_price, completion_price, currency_code = rows[0] - if prompt_price is None or completion_price is None: + ): continue price_book.set_price( PriceEntry( - provider_name=provider_name, - model_name=model_name, - prompt_price_per_1k=prompt_price, - completion_price_per_1k=completion_price, - currency_code=currency_code, + provider_name=model.provider_name, + model_name=model.model_id, + prompt_price_per_1k=float(model.prompt_price_per_1k), + completion_price_per_1k=float(model.completion_price_per_1k), + currency_code=model.currency_code.strip().upper(), ) ) written += 1 return written -def _is_valid_price_component(value: Any) -> bool: - """Return whether one provider price is finite and non-negative.""" - return ( - value is not None - and not isinstance(value, bool) - and isinstance(value, (int, float)) - and math.isfinite(value) - and value >= 0 - ) - - -def _unique_discovered_models( - discovered: list[DiscoveredModel], -) -> list[DiscoveredModel]: - """Keep one deterministic row per provider/model serving identity.""" - unique: dict[tuple[str, str], DiscoveredModel] = {} - for model in discovered: - unique.setdefault((model.provider_name, model.model_id), model) - return list(unique.values()) - - def _discovery_price_key( model: DiscoveredModel, price_book: "PriceBook", ) -> tuple[int, float, str, str]: - """Rank known prices first, then deterministically order unknown prices.""" - entry = price_book.get_price(model.provider_name, model.model_id) - if entry is None or not _is_trustworthy_price_entry(entry, price_book): - return (1, 0.0, model.provider_name, model.model_id) - cost, _currency = price_book.compute_cost( - model.provider_name, - model.model_id, - 1000, - 1000, - ) + """Rank comparable trustworthy prices first, then deterministic unknowns.""" + unknown = (1, 0.0, model.provider_name, model.model_id) + try: + entry = price_book.get_price(model.provider_name, model.model_id) + except (TypeError, ValueError, OverflowError): + return unknown + if entry is None: + return unknown + if not ( + _valid_price_component(entry.prompt_price_per_1k) + and _valid_price_component(entry.completion_price_per_1k) + and _currency_is_comparable( + entry.currency_code, + price_book.default_currency, + ) + ): + return unknown + try: + cost, currency = price_book.compute_cost( + model.provider_name, + model.model_id, + 1000, + 1000, + ) + except (TypeError, ValueError, OverflowError): + return unknown + if not ( + _valid_price_component(cost) + and _currency_is_comparable(currency, price_book.default_currency) + ): + return unknown return (0, cost, model.provider_name, model.model_id) -def _is_trustworthy_price_entry(entry: Any, price_book: "PriceBook") -> bool: - """Accept only complete, finite prices in the book's comparison currency.""" - return ( - entry.currency_code == price_book.default_currency - and _is_valid_price_component(entry.prompt_price_per_1k) - and _is_valid_price_component(entry.completion_price_per_1k) - ) - - def _provider_family(provider_name: str) -> str: """Collapse credentials that share one upstream provider outage domain.""" if provider_name in {"nvidia_nim", "nvidia_nim_sub"}: @@ -355,21 +398,23 @@ def select_cheapest_discovered_agent( sort first; when every candidate is unpriced, provider and model identifiers provide deterministic fallback ordering without inventing a monetary value. """ - unique = _unique_discovered_models(discovered) - if not unique: + eligible = _deduplicate_discovered_models(discovered) + if not eligible: return None - return min(unique, key=lambda model: _discovery_price_key(model, price_book)) + return min(eligible, key=lambda model: _discovery_price_key(model, price_book)) def select_top_n_cheapest_discovered_agents( discovered: list[DiscoveredModel], price_book: "PriceBook", limit: int ) -> list[DiscoveredModel]: - """Return up to ``limit`` candidates with known prices before unknown ones.""" - if limit <= 0 or not discovered: + """Return up to ``limit`` unique candidates, known-priced before unknown.""" + if limit <= 0: + return [] + eligible = _deduplicate_discovered_models(discovered) + if not eligible: return [] - discovered = _unique_discovered_models(discovered) return sorted( - discovered, + eligible, key=lambda model: _discovery_price_key(model, price_book), )[:limit] @@ -386,13 +431,16 @@ def select_bootstrap_discovered_agents( capacity is filled in the same deterministic cost order. NVIDIA NIM primary and sub credentials are one outage domain, so they participate in the second pass only after independently hosted providers have had a chance to enter. + Duplicate serving identities never consume capacity twice. """ - if limit <= 0 or not discovered: + if limit <= 0: + return [] + eligible = _deduplicate_discovered_models(discovered) + if not eligible: return [] - discovered = _unique_discovered_models(discovered) ranked = sorted( - discovered, + eligible, key=lambda model: _discovery_price_key(model, price_book), ) selected: list[DiscoveredModel] = [] From c01733b4813e161493ea73a6bde8510a547d87e7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:47:37 +0900 Subject: [PATCH 61/89] fix(cost): reject corrupt persisted price rows --- contextual_orchestrator/cost_ledger.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index 3866396dc..8c1a0dde1 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -31,6 +31,7 @@ import threading from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Decimal +import math import time from typing import Any, Dict, List, Optional, Protocol import uuid @@ -162,11 +163,25 @@ def get_price(self, provider: str, model: str) -> Optional[PriceEntry]: raw = self._config.get(_PRICE_CATEGORY, _price_key(provider, "*"), None) if raw is None: return None + if not isinstance(raw, dict): + return None + try: + prompt_price = float(raw.get("prompt_price_per_1k", 0.0)) + completion_price = float(raw.get("completion_price_per_1k", 0.0)) + except (OverflowError, TypeError, ValueError): + return None + if ( + not math.isfinite(prompt_price) + or not math.isfinite(completion_price) + or prompt_price < 0 + or completion_price < 0 + ): + return None return PriceEntry( provider_name=raw.get("provider_name", provider), model_name=raw.get("model_name", model), - prompt_price_per_1k=float(raw.get("prompt_price_per_1k", 0.0)), - completion_price_per_1k=float(raw.get("completion_price_per_1k", 0.0)), + prompt_price_per_1k=prompt_price, + completion_price_per_1k=completion_price, currency_code=raw.get("currency_code", self.default_currency), ) From 669d97f13d2057b066116c5dbcc6ae1398d391df Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 00:54:50 -0700 Subject: [PATCH 62/89] ci: finalize chat transport and role capability boundaries --- .../repair-chat-capability-finalize-v2.yml | 395 ++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100644 .github/workflows/repair-chat-capability-finalize-v2.yml diff --git a/.github/workflows/repair-chat-capability-finalize-v2.yml b/.github/workflows/repair-chat-capability-finalize-v2.yml new file mode 100644 index 000000000..de6490d8b --- /dev/null +++ b/.github/workflows/repair-chat-capability-finalize-v2.yml @@ -0,0 +1,395 @@ +name: Finalize chat transport and role capability boundaries + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/repair-chat-capability-finalize-v2.yml + +permissions: + contents: write + +concurrency: + group: repair-chat-transport-role-separation + cancel-in-progress: true + +jobs: + finalize: + name: Patch, verify, and self-delete + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout exact branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Finalize transport and ordinary-role classification + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + import re + + CHAT_CAPABILITY = '''"""Classify chat transport compatibility and ordinary agent-role eligibility. + +Provider catalogs mix endpoint-only models with models served through an +OpenAI-compatible chat transport. Transport compatibility is not the same as +fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and +policy-classification models can use chat transport, while embedding, reranking, +transcription, moderation-endpoint, image-generation, realtime, and speech-only +models cannot. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "clip", + "dall", + "e5", + "embed", + "embedding", + "embeddings", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "siglip", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_TRANSPORT_INCOMPATIBLE_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def is_chat_transport_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use the ordinary chat transport. + + The classifier rejects only identifiers that clearly advertise an endpoint + family incompatible with chat messages. Audio-capable and safety-classifier + models remain transport-compatible because providers serve some of them over + ``/chat/completions``. + """ + tokens = _model_tokens(model_id) + if not tokens: + return False + for token in tokens: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: + return False + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): + return False + return True + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Backward-compatible name for chat transport compatibility.""" + return is_chat_transport_compatible_model_id(model_id) + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models can use chat transport but are specialized + policy classifiers, not general answer synthesizers. This negative role gate + does not infer reasoning, coding, vision, or verification capabilities. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_transport_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) +''' + Path("contextual_orchestrator/chat_capability.py").write_text( + CHAT_CAPABILITY, + encoding="utf-8", + ) + + discovery_path = Path("contextual_orchestrator/model_discovery.py") + discovery = discovery_path.read_text(encoding="utf-8") + import_pattern = re.compile( + r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", + re.DOTALL, + ) + discovery, count = import_pattern.subn( + "from .chat_capability import is_general_chat_agent_model_id\n", + discovery, + count=1, + ) + if count != 1: + raise SystemExit(f"model discovery import: expected one match, found {count}") + for name in ( + "is_chat_transport_compatible_model_id", + "is_chat_compatible_model_id", + "is_general_chat_agent_model_id", + ): + discovery = discovery.replace( + f"{name}(", + "is_general_chat_agent_model_id(", + ) + discovery = discovery.replace( + "One chat-compatible model found on a provider, with reported pricing.", + "One general chat-agent candidate found on a provider, with reported pricing.", + ) + discovery = discovery.replace( + "Build a disabled chat ModelAgent or reject a non-chat discovery record.", + "Build a disabled general chat agent or reject an ineligible discovery record.", + ) + discovery = discovery.replace( + 'raise ValueError("non-chat model cannot be converted into a chat agent")', + 'raise ValueError("model is not eligible for the general chat agent pool")', + ) + discovery_path.write_text(discovery, encoding="utf-8") + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator, count = import_pattern.subn( + "from .chat_capability import (\n" + " is_chat_transport_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + orchestrator, + count=1, + ) + if count != 1: + raise SystemExit(f"orchestrator import: expected one match, found {count}") + for name in ( + "is_chat_compatible_model_id", + "is_chat_transport_compatible_model_id", + "is_general_chat_agent_model_id", + ): + orchestrator = orchestrator.replace( + f"not {name}(agent.model)", + "not is_chat_transport_compatible_model_id(agent.model)", + ) + for name in ( + "is_chat_compatible_model_id", + "is_chat_transport_compatible_model_id", + "is_general_chat_agent_model_id", + ): + orchestrator = orchestrator.replace( + f"if {name}(agent.model)", + "if is_general_chat_agent_model_id(agent.model)", + ) + orchestrator = orchestrator.replace( + f"not {name}(assigned.model)", + "not is_general_chat_agent_model_id(assigned.model)", + ) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + existing_tests_path = Path("tests/test_chat_model_capability_isolation.py") + existing_tests = existing_tests_path.read_text(encoding="utf-8") + if "from contextual_orchestrator.chat_capability import" not in existing_tests: + anchor = "from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402\n" + replacement = ( + anchor + + "from contextual_orchestrator.chat_capability import ( # noqa: E402\n" + + " is_chat_compatible_model_id,\n" + + ")\n" + ) + if anchor not in existing_tests: + raise SystemExit("existing test import anchor missing") + existing_tests = existing_tests.replace(anchor, replacement, 1) + existing_tests = re.sub( + r"\n is_chat_compatible_model_id,\n", + "\n", + existing_tests, + count=1, + ) if "from contextual_orchestrator.model_discovery import" in existing_tests else existing_tests + existing_tests = existing_tests.replace( + 'with pytest.raises(ValueError, match="non-chat model"):', + 'with pytest.raises(ValueError, match="general chat agent"):', + ) + existing_tests_path.write_text(existing_tests, encoding="utf-8") + + Path("tests/test_chat_transport_role_separation.py").write_text( + '''"""Regression tests for chat transport versus ordinary agent roles.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_transport_compatible_model_id, + is_general_chat_agent_model_id, +) +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 + + +def _agent(agent_id: str, model_id: str, *, priority: int = 0) -> ModelAgent: + """Build one mock-backed agent for transport and role regressions.""" + return ModelAgent( + id=agent_id, + model=model_id, + base_url="mock://local", + priority=priority, + tags=("writing",), + ) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-4o-mini-audio-preview", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: + """Do not reject models that providers serve through chat transport.""" + assert is_chat_transport_compatible_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: + """Keep chat-served policy classifiers out of ordinary worker roles.""" + assert not is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ], +) +def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: + """Preserve capable chat generation models for ordinary agent selection.""" + assert is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "azure/text-embedding-3-large", + "text_embedding_3_large", + "company/reranker-v2", + "gpt-4o-mini-transcribe", + "omni-moderation-latest", + "gpt-image-1", + "dall-e-3", + "gpt-realtime", + "tts-1", + ], +) +def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: + """Reject clear non-chat endpoint families before transport or routing.""" + assert not is_chat_transport_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) + + +def test_direct_transport_can_use_guard_but_role_selection_cannot() -> None: + """Separate an intentional direct policy call from generic synthesis.""" + guard_agent = _agent( + "guard_agent", + "meta-llama/Llama-Guard-4-12B", + priority=10_000, + ) + chat_agent = _agent("chat_agent", "gpt-5.2") + client = ModelClient() + + assert client.chat( + guard_agent, + [{"role": "user", "content": "Classify this policy input."}], + ) + orchestrator = TaskOrchestrator([guard_agent, chat_agent], client=client) + assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent +''', + encoding="utf-8", + ) + + doctoring_path = Path("docs/doctoring/embedding-chat-capability-isolation.md") + doctoring = doctoring_path.read_text(encoding="utf-8") + heading = "## Transport compatibility and ordinary-role eligibility" + if heading not in doctoring: + doctoring += f'''\n\n{heading}\n\nModel identifiers are now evaluated at two different boundaries. The transport\nboundary rejects endpoint-only families such as embeddings, reranking,\ntranscription, moderation endpoints, image generation, realtime, and speech-only\nmodels. The ordinary-agent boundary is stricter: chat-served guard and safety\nclassifiers remain available for intentional direct policy calls but cannot become\nthinkers, workers, verifiers, synthesizers, or generic failovers. Audio-capable\nchat models are not rejected merely because their identifier contains `audio`.\nThis avoids both the original embedding-to-chat failure and the inverse failure of\nblocking legitimate chat endpoints.\n\n### Additional references\n\nNVIDIA. (n.d.). *OpenAI-compatible APIs for NVIDIA NIM*. NVIDIA Documentation.\nRetrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/openai-api.html\n\nOpenAI. (n.d.). *Audio and speech*. OpenAI API documentation. Retrieved August 20,\n2026, from https://platform.openai.com/docs/guides/audio\n''' + doctoring_path.write_text(doctoring, encoding="utf-8") + + for temporary in ( + ".github/workflows/repair-chat-capability-explicit-integration.yml", + ".github/workflows/repair-chat-capability-final.yml", + ".github/workflows/repair-chat-transport-role-separation.yml", + ".github/workflows/repair-chat-capability-finalize-v2.yml", + ): + path = Path(temporary) + if path.exists(): + path.unlink() + PY + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Verify focused and full suites + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py + python -m pytest -q + git diff --check + + - name: Commit verified final state + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + git commit -m "fix: distinguish chat transport from synthesis roles" + git push origin HEAD:fix/chat-capability-isolation-embedding-models From c08a8a68b5bae7a68dfa65e62f6a736fba1bb42b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 00:57:30 -0700 Subject: [PATCH 63/89] ci: finalize endpoint transport and agent role separation --- .../repair-chat-capability-finalize-v3.yml | 383 ++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 .github/workflows/repair-chat-capability-finalize-v3.yml diff --git a/.github/workflows/repair-chat-capability-finalize-v3.yml b/.github/workflows/repair-chat-capability-finalize-v3.yml new file mode 100644 index 000000000..9513cf747 --- /dev/null +++ b/.github/workflows/repair-chat-capability-finalize-v3.yml @@ -0,0 +1,383 @@ +name: Finalize endpoint transport and agent role separation + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/repair-chat-capability-finalize-v3.yml + +permissions: + contents: write + +concurrency: + group: repair-chat-transport-role-separation + cancel-in-progress: true + +jobs: + finalize: + name: Patch, verify, and self-delete + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout exact branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Finalize capability boundaries + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + import re + + Path("contextual_orchestrator/chat_capability.py").write_text( + '''"""Classify chat transport compatibility and ordinary agent-role eligibility. + +Provider catalogs mix endpoint-only models with models served through an +OpenAI-compatible chat transport. Transport compatibility is not the same as +fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and +policy-classification models can use chat transport, while embedding, reranking, +transcription, moderation-endpoint, image-generation, realtime, and speech-only +models cannot. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "clip", + "dall", + "e5", + "embed", + "embedding", + "embeddings", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "siglip", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_TRANSPORT_INCOMPATIBLE_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def is_chat_transport_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use the ordinary chat transport. + + The classifier rejects only identifiers that clearly advertise an endpoint + family incompatible with chat messages. Audio-capable and safety-classifier + models remain transport-compatible because providers serve some of them over + ``/chat/completions``. + """ + tokens = _model_tokens(model_id) + if not tokens: + return False + for token in tokens: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: + return False + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): + return False + return True + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Backward-compatible name for chat transport compatibility.""" + return is_chat_transport_compatible_model_id(model_id) + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models can use chat transport but are specialized + policy classifiers, not general answer synthesizers. This negative role gate + does not infer reasoning, coding, vision, or verification capabilities. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_transport_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) +''', + encoding="utf-8", + ) + + import_pattern = re.compile( + r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", + re.DOTALL, + ) + + discovery_path = Path("contextual_orchestrator/model_discovery.py") + discovery = discovery_path.read_text(encoding="utf-8") + discovery, count = import_pattern.subn( + "from .chat_capability import (\n" + " is_chat_compatible_model_id, # compatibility re-export\n" + " is_general_chat_agent_model_id,\n" + ")\n", + discovery, + count=1, + ) + if count != 1: + raise SystemExit(f"model discovery import: expected one match, found {count}") + for name in ( + "is_chat_transport_compatible_model_id", + "is_chat_compatible_model_id", + "is_general_chat_agent_model_id", + ): + discovery = discovery.replace( + f"{name}(", + "is_general_chat_agent_model_id(", + ) + discovery = discovery.replace( + "One chat-compatible model found on a provider, with reported pricing.", + "One general chat-agent candidate found on a provider, with reported pricing.", + ) + discovery = discovery.replace( + "Build a disabled chat ModelAgent or reject a non-chat discovery record.", + "Build a disabled general chat agent or reject an ineligible discovery record.", + ) + discovery = discovery.replace( + 'raise ValueError("non-chat model cannot be converted into a chat agent")', + 'raise ValueError("model is not eligible for the general chat agent pool")', + ) + discovery_path.write_text(discovery, encoding="utf-8") + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator, count = import_pattern.subn( + "from .chat_capability import (\n" + " is_chat_transport_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + orchestrator, + count=1, + ) + if count != 1: + raise SystemExit(f"orchestrator import: expected one match, found {count}") + + for name in ( + "is_chat_compatible_model_id", + "is_chat_transport_compatible_model_id", + "is_general_chat_agent_model_id", + ): + orchestrator = orchestrator.replace( + f"not {name}(agent.model)", + "not is_chat_transport_compatible_model_id(agent.model)", + ) + for name in ( + "is_chat_compatible_model_id", + "is_chat_transport_compatible_model_id", + "is_general_chat_agent_model_id", + ): + orchestrator = orchestrator.replace( + f"if {name}(agent.model)", + "if is_general_chat_agent_model_id(agent.model)", + ) + orchestrator = orchestrator.replace( + f"not {name}(assigned.model)", + "not is_general_chat_agent_model_id(assigned.model)", + ) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + existing_tests_path = Path("tests/test_chat_model_capability_isolation.py") + existing_tests = existing_tests_path.read_text(encoding="utf-8").replace( + 'with pytest.raises(ValueError, match="non-chat model"):', + 'with pytest.raises(ValueError, match="general chat agent"):', + ) + existing_tests_path.write_text(existing_tests, encoding="utf-8") + + Path("tests/test_chat_transport_role_separation.py").write_text( + '''"""Regression tests for chat transport versus ordinary agent roles.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_transport_compatible_model_id, + is_general_chat_agent_model_id, +) +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 + + +def _agent(agent_id: str, model_id: str, *, priority: int = 0) -> ModelAgent: + """Build one mock-backed agent for transport and role regressions.""" + return ModelAgent( + id=agent_id, + model=model_id, + base_url="mock://local", + priority=priority, + tags=("writing",), + ) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-4o-mini-audio-preview", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: + """Do not reject models that providers serve through chat transport.""" + assert is_chat_transport_compatible_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: + """Keep chat-served policy classifiers out of ordinary worker roles.""" + assert not is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ], +) +def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: + """Preserve capable chat generation models for ordinary agent selection.""" + assert is_general_chat_agent_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "azure/text-embedding-3-large", + "text_embedding_3_large", + "company/reranker-v2", + "gpt-4o-mini-transcribe", + "omni-moderation-latest", + "gpt-image-1", + "dall-e-3", + "gpt-realtime", + "tts-1", + ], +) +def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: + """Reject clear non-chat endpoint families before transport or routing.""" + assert not is_chat_transport_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) + + +def test_direct_transport_can_use_guard_but_role_selection_cannot() -> None: + """Separate an intentional direct policy call from generic synthesis.""" + guard_agent = _agent( + "guard_agent", + "meta-llama/Llama-Guard-4-12B", + priority=10_000, + ) + chat_agent = _agent("chat_agent", "gpt-5.2") + client = ModelClient() + + output = client.chat( + guard_agent, + [{"role": "user", "content": "Classify this policy input."}], + ) + assert isinstance(output, str) + orchestrator = TaskOrchestrator([guard_agent, chat_agent], client=client) + assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent +''', + encoding="utf-8", + ) + + doctoring_path = Path("docs/doctoring/embedding-chat-capability-isolation.md") + doctoring = doctoring_path.read_text(encoding="utf-8") + heading = "## Transport compatibility and ordinary-role eligibility" + if heading not in doctoring: + doctoring += f'''\n\n{heading}\n\nModel identifiers are evaluated at two different boundaries. The transport\nboundary rejects endpoint-only families such as embeddings, reranking,\ntranscription, moderation endpoints, image generation, realtime, and speech-only\nmodels. The ordinary-agent boundary is stricter: chat-served guard and safety\nclassifiers remain available for intentional direct policy calls but cannot become\nthinkers, workers, verifiers, synthesizers, or generic failovers. Audio-capable\nchat models are not rejected merely because their identifier contains `audio`.\nThis avoids both the original embedding-to-chat failure and the inverse failure of\nblocking legitimate chat endpoints.\n\n### Additional references\n\nNVIDIA. (n.d.). *OpenAI-compatible APIs for NVIDIA NIM*. NVIDIA Documentation.\nRetrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/openai-api.html\n\nOpenAI. (n.d.). *Audio and speech*. OpenAI API documentation. Retrieved August 20,\n2026, from https://platform.openai.com/docs/guides/audio\n''' + doctoring_path.write_text(doctoring, encoding="utf-8") + + for temporary in ( + ".github/workflows/repair-chat-capability-explicit-integration.yml", + ".github/workflows/repair-chat-capability-final.yml", + ".github/workflows/repair-chat-transport-role-separation.yml", + ".github/workflows/repair-chat-capability-finalize-v2.yml", + ".github/workflows/repair-chat-capability-finalize-v3.yml", + ): + path = Path(temporary) + if path.exists(): + path.unlink() + PY + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Verify focused and full suites + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py + python -m pytest -q + git diff --check + + - name: Commit verified final state + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + git commit -m "fix: separate chat transport from synthesis roles" + git push origin HEAD:fix/chat-capability-isolation-embedding-models From 70757f6667ab97841850b6e81257e06f9c7bd586 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 01:04:05 -0700 Subject: [PATCH 64/89] chore: remove superseded capability repair workflow v2 --- .../repair-chat-capability-finalize-v2.yml | 395 ------------------ 1 file changed, 395 deletions(-) delete mode 100644 .github/workflows/repair-chat-capability-finalize-v2.yml diff --git a/.github/workflows/repair-chat-capability-finalize-v2.yml b/.github/workflows/repair-chat-capability-finalize-v2.yml deleted file mode 100644 index de6490d8b..000000000 --- a/.github/workflows/repair-chat-capability-finalize-v2.yml +++ /dev/null @@ -1,395 +0,0 @@ -name: Finalize chat transport and role capability boundaries - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/repair-chat-capability-finalize-v2.yml - -permissions: - contents: write - -concurrency: - group: repair-chat-transport-role-separation - cancel-in-progress: true - -jobs: - finalize: - name: Patch, verify, and self-delete - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout exact branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: true - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Finalize transport and ordinary-role classification - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - import re - - CHAT_CAPABILITY = '''"""Classify chat transport compatibility and ordinary agent-role eligibility. - -Provider catalogs mix endpoint-only models with models served through an -OpenAI-compatible chat transport. Transport compatibility is not the same as -fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and -policy-classification models can use chat transport, while embedding, reranking, -transcription, moderation-endpoint, image-generation, realtime, and speech-only -models cannot. -""" - -from __future__ import annotations - -import re - -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( - { - "bge", - "clip", - "dall", - "e5", - "embed", - "embedding", - "embeddings", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "siglip", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_TRANSPORT_INCOMPATIBLE_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - - -def _model_tokens(model_id: str) -> tuple[str, ...]: - """Normalize one provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: - return () - return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - - -def is_chat_transport_compatible_model_id(model_id: str) -> bool: - """Return whether an identifier can use the ordinary chat transport. - - The classifier rejects only identifiers that clearly advertise an endpoint - family incompatible with chat messages. Audio-capable and safety-classifier - models remain transport-compatible because providers serve some of them over - ``/chat/completions``. - """ - tokens = _model_tokens(model_id) - if not tokens: - return False - for token in tokens: - if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: - return False - if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): - return False - return True - - -def is_chat_compatible_model_id(model_id: str) -> bool: - """Backward-compatible name for chat transport compatibility.""" - return is_chat_transport_compatible_model_id(model_id) - - -def is_general_chat_agent_model_id(model_id: str) -> bool: - """Return whether a chat model may enter ordinary orchestration roles. - - Explicit guard and safety models can use chat transport but are specialized - policy classifiers, not general answer synthesizers. This negative role gate - does not infer reasoning, coding, vision, or verification capabilities. - """ - tokens = _model_tokens(model_id) - if not tokens or not is_chat_transport_compatible_model_id(model_id): - return False - return not any( - token == "safety" - or token == "guard" - or token.startswith("nemoguard") - or token.endswith("guard") - for token in tokens - ) -''' - Path("contextual_orchestrator/chat_capability.py").write_text( - CHAT_CAPABILITY, - encoding="utf-8", - ) - - discovery_path = Path("contextual_orchestrator/model_discovery.py") - discovery = discovery_path.read_text(encoding="utf-8") - import_pattern = re.compile( - r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", - re.DOTALL, - ) - discovery, count = import_pattern.subn( - "from .chat_capability import is_general_chat_agent_model_id\n", - discovery, - count=1, - ) - if count != 1: - raise SystemExit(f"model discovery import: expected one match, found {count}") - for name in ( - "is_chat_transport_compatible_model_id", - "is_chat_compatible_model_id", - "is_general_chat_agent_model_id", - ): - discovery = discovery.replace( - f"{name}(", - "is_general_chat_agent_model_id(", - ) - discovery = discovery.replace( - "One chat-compatible model found on a provider, with reported pricing.", - "One general chat-agent candidate found on a provider, with reported pricing.", - ) - discovery = discovery.replace( - "Build a disabled chat ModelAgent or reject a non-chat discovery record.", - "Build a disabled general chat agent or reject an ineligible discovery record.", - ) - discovery = discovery.replace( - 'raise ValueError("non-chat model cannot be converted into a chat agent")', - 'raise ValueError("model is not eligible for the general chat agent pool")', - ) - discovery_path.write_text(discovery, encoding="utf-8") - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - orchestrator, count = import_pattern.subn( - "from .chat_capability import (\n" - " is_chat_transport_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - orchestrator, - count=1, - ) - if count != 1: - raise SystemExit(f"orchestrator import: expected one match, found {count}") - for name in ( - "is_chat_compatible_model_id", - "is_chat_transport_compatible_model_id", - "is_general_chat_agent_model_id", - ): - orchestrator = orchestrator.replace( - f"not {name}(agent.model)", - "not is_chat_transport_compatible_model_id(agent.model)", - ) - for name in ( - "is_chat_compatible_model_id", - "is_chat_transport_compatible_model_id", - "is_general_chat_agent_model_id", - ): - orchestrator = orchestrator.replace( - f"if {name}(agent.model)", - "if is_general_chat_agent_model_id(agent.model)", - ) - orchestrator = orchestrator.replace( - f"not {name}(assigned.model)", - "not is_general_chat_agent_model_id(assigned.model)", - ) - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - existing_tests_path = Path("tests/test_chat_model_capability_isolation.py") - existing_tests = existing_tests_path.read_text(encoding="utf-8") - if "from contextual_orchestrator.chat_capability import" not in existing_tests: - anchor = "from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402\n" - replacement = ( - anchor - + "from contextual_orchestrator.chat_capability import ( # noqa: E402\n" - + " is_chat_compatible_model_id,\n" - + ")\n" - ) - if anchor not in existing_tests: - raise SystemExit("existing test import anchor missing") - existing_tests = existing_tests.replace(anchor, replacement, 1) - existing_tests = re.sub( - r"\n is_chat_compatible_model_id,\n", - "\n", - existing_tests, - count=1, - ) if "from contextual_orchestrator.model_discovery import" in existing_tests else existing_tests - existing_tests = existing_tests.replace( - 'with pytest.raises(ValueError, match="non-chat model"):', - 'with pytest.raises(ValueError, match="general chat agent"):', - ) - existing_tests_path.write_text(existing_tests, encoding="utf-8") - - Path("tests/test_chat_transport_role_separation.py").write_text( - '''"""Regression tests for chat transport versus ordinary agent roles.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.chat_capability import ( # noqa: E402 - is_chat_transport_compatible_model_id, - is_general_chat_agent_model_id, -) -from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 - - -def _agent(agent_id: str, model_id: str, *, priority: int = 0) -> ModelAgent: - """Build one mock-backed agent for transport and role regressions.""" - return ModelAgent( - id=agent_id, - model=model_id, - base_url="mock://local", - priority=priority, - tags=("writing",), - ) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-4o-mini-audio-preview", - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: - """Do not reject models that providers serve through chat transport.""" - assert is_chat_transport_compatible_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "nvidia/llama-3.1-nemoguard-8b-content-safety", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: - """Keep chat-served policy classifiers out of ordinary worker roles.""" - assert not is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-5.2", - "qwen/qwen3-235b-a22b-instruct", - ], -) -def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: - """Preserve capable chat generation models for ordinary agent selection.""" - assert is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "azure/text-embedding-3-large", - "text_embedding_3_large", - "company/reranker-v2", - "gpt-4o-mini-transcribe", - "omni-moderation-latest", - "gpt-image-1", - "dall-e-3", - "gpt-realtime", - "tts-1", - ], -) -def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: - """Reject clear non-chat endpoint families before transport or routing.""" - assert not is_chat_transport_compatible_model_id(model_id) - assert not is_general_chat_agent_model_id(model_id) - - -def test_direct_transport_can_use_guard_but_role_selection_cannot() -> None: - """Separate an intentional direct policy call from generic synthesis.""" - guard_agent = _agent( - "guard_agent", - "meta-llama/Llama-Guard-4-12B", - priority=10_000, - ) - chat_agent = _agent("chat_agent", "gpt-5.2") - client = ModelClient() - - assert client.chat( - guard_agent, - [{"role": "user", "content": "Classify this policy input."}], - ) - orchestrator = TaskOrchestrator([guard_agent, chat_agent], client=client) - assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent -''', - encoding="utf-8", - ) - - doctoring_path = Path("docs/doctoring/embedding-chat-capability-isolation.md") - doctoring = doctoring_path.read_text(encoding="utf-8") - heading = "## Transport compatibility and ordinary-role eligibility" - if heading not in doctoring: - doctoring += f'''\n\n{heading}\n\nModel identifiers are now evaluated at two different boundaries. The transport\nboundary rejects endpoint-only families such as embeddings, reranking,\ntranscription, moderation endpoints, image generation, realtime, and speech-only\nmodels. The ordinary-agent boundary is stricter: chat-served guard and safety\nclassifiers remain available for intentional direct policy calls but cannot become\nthinkers, workers, verifiers, synthesizers, or generic failovers. Audio-capable\nchat models are not rejected merely because their identifier contains `audio`.\nThis avoids both the original embedding-to-chat failure and the inverse failure of\nblocking legitimate chat endpoints.\n\n### Additional references\n\nNVIDIA. (n.d.). *OpenAI-compatible APIs for NVIDIA NIM*. NVIDIA Documentation.\nRetrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/openai-api.html\n\nOpenAI. (n.d.). *Audio and speech*. OpenAI API documentation. Retrieved August 20,\n2026, from https://platform.openai.com/docs/guides/audio\n''' - doctoring_path.write_text(doctoring, encoding="utf-8") - - for temporary in ( - ".github/workflows/repair-chat-capability-explicit-integration.yml", - ".github/workflows/repair-chat-capability-final.yml", - ".github/workflows/repair-chat-transport-role-separation.yml", - ".github/workflows/repair-chat-capability-finalize-v2.yml", - ): - path = Path(temporary) - if path.exists(): - path.unlink() - PY - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Verify focused and full suites - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py - python -m pytest -q - git diff --check - - - name: Commit verified final state - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git diff --cached --check - git commit -m "fix: distinguish chat transport from synthesis roles" - git push origin HEAD:fix/chat-capability-isolation-embedding-models From 8c06c74e4d36961d512ac4e8498ccc4dddcfa0ec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 01:04:17 -0700 Subject: [PATCH 65/89] chore: remove superseded capability repair workflow v3 --- .../repair-chat-capability-finalize-v3.yml | 383 ------------------ 1 file changed, 383 deletions(-) delete mode 100644 .github/workflows/repair-chat-capability-finalize-v3.yml diff --git a/.github/workflows/repair-chat-capability-finalize-v3.yml b/.github/workflows/repair-chat-capability-finalize-v3.yml deleted file mode 100644 index 9513cf747..000000000 --- a/.github/workflows/repair-chat-capability-finalize-v3.yml +++ /dev/null @@ -1,383 +0,0 @@ -name: Finalize endpoint transport and agent role separation - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/repair-chat-capability-finalize-v3.yml - -permissions: - contents: write - -concurrency: - group: repair-chat-transport-role-separation - cancel-in-progress: true - -jobs: - finalize: - name: Patch, verify, and self-delete - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout exact branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: true - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Finalize capability boundaries - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from pathlib import Path - import re - - Path("contextual_orchestrator/chat_capability.py").write_text( - '''"""Classify chat transport compatibility and ordinary agent-role eligibility. - -Provider catalogs mix endpoint-only models with models served through an -OpenAI-compatible chat transport. Transport compatibility is not the same as -fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and -policy-classification models can use chat transport, while embedding, reranking, -transcription, moderation-endpoint, image-generation, realtime, and speech-only -models cannot. -""" - -from __future__ import annotations - -import re - -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( - { - "bge", - "clip", - "dall", - "e5", - "embed", - "embedding", - "embeddings", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "siglip", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_TRANSPORT_INCOMPATIBLE_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - - -def _model_tokens(model_id: str) -> tuple[str, ...]: - """Normalize one provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: - return () - return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - - -def is_chat_transport_compatible_model_id(model_id: str) -> bool: - """Return whether an identifier can use the ordinary chat transport. - - The classifier rejects only identifiers that clearly advertise an endpoint - family incompatible with chat messages. Audio-capable and safety-classifier - models remain transport-compatible because providers serve some of them over - ``/chat/completions``. - """ - tokens = _model_tokens(model_id) - if not tokens: - return False - for token in tokens: - if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: - return False - if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): - return False - return True - - -def is_chat_compatible_model_id(model_id: str) -> bool: - """Backward-compatible name for chat transport compatibility.""" - return is_chat_transport_compatible_model_id(model_id) - - -def is_general_chat_agent_model_id(model_id: str) -> bool: - """Return whether a chat model may enter ordinary orchestration roles. - - Explicit guard and safety models can use chat transport but are specialized - policy classifiers, not general answer synthesizers. This negative role gate - does not infer reasoning, coding, vision, or verification capabilities. - """ - tokens = _model_tokens(model_id) - if not tokens or not is_chat_transport_compatible_model_id(model_id): - return False - return not any( - token == "safety" - or token == "guard" - or token.startswith("nemoguard") - or token.endswith("guard") - for token in tokens - ) -''', - encoding="utf-8", - ) - - import_pattern = re.compile( - r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", - re.DOTALL, - ) - - discovery_path = Path("contextual_orchestrator/model_discovery.py") - discovery = discovery_path.read_text(encoding="utf-8") - discovery, count = import_pattern.subn( - "from .chat_capability import (\n" - " is_chat_compatible_model_id, # compatibility re-export\n" - " is_general_chat_agent_model_id,\n" - ")\n", - discovery, - count=1, - ) - if count != 1: - raise SystemExit(f"model discovery import: expected one match, found {count}") - for name in ( - "is_chat_transport_compatible_model_id", - "is_chat_compatible_model_id", - "is_general_chat_agent_model_id", - ): - discovery = discovery.replace( - f"{name}(", - "is_general_chat_agent_model_id(", - ) - discovery = discovery.replace( - "One chat-compatible model found on a provider, with reported pricing.", - "One general chat-agent candidate found on a provider, with reported pricing.", - ) - discovery = discovery.replace( - "Build a disabled chat ModelAgent or reject a non-chat discovery record.", - "Build a disabled general chat agent or reject an ineligible discovery record.", - ) - discovery = discovery.replace( - 'raise ValueError("non-chat model cannot be converted into a chat agent")', - 'raise ValueError("model is not eligible for the general chat agent pool")', - ) - discovery_path.write_text(discovery, encoding="utf-8") - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - orchestrator, count = import_pattern.subn( - "from .chat_capability import (\n" - " is_chat_transport_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - orchestrator, - count=1, - ) - if count != 1: - raise SystemExit(f"orchestrator import: expected one match, found {count}") - - for name in ( - "is_chat_compatible_model_id", - "is_chat_transport_compatible_model_id", - "is_general_chat_agent_model_id", - ): - orchestrator = orchestrator.replace( - f"not {name}(agent.model)", - "not is_chat_transport_compatible_model_id(agent.model)", - ) - for name in ( - "is_chat_compatible_model_id", - "is_chat_transport_compatible_model_id", - "is_general_chat_agent_model_id", - ): - orchestrator = orchestrator.replace( - f"if {name}(agent.model)", - "if is_general_chat_agent_model_id(agent.model)", - ) - orchestrator = orchestrator.replace( - f"not {name}(assigned.model)", - "not is_general_chat_agent_model_id(assigned.model)", - ) - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - existing_tests_path = Path("tests/test_chat_model_capability_isolation.py") - existing_tests = existing_tests_path.read_text(encoding="utf-8").replace( - 'with pytest.raises(ValueError, match="non-chat model"):', - 'with pytest.raises(ValueError, match="general chat agent"):', - ) - existing_tests_path.write_text(existing_tests, encoding="utf-8") - - Path("tests/test_chat_transport_role_separation.py").write_text( - '''"""Regression tests for chat transport versus ordinary agent roles.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.chat_capability import ( # noqa: E402 - is_chat_transport_compatible_model_id, - is_general_chat_agent_model_id, -) -from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 - - -def _agent(agent_id: str, model_id: str, *, priority: int = 0) -> ModelAgent: - """Build one mock-backed agent for transport and role regressions.""" - return ModelAgent( - id=agent_id, - model=model_id, - base_url="mock://local", - priority=priority, - tags=("writing",), - ) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-4o-mini-audio-preview", - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: - """Do not reject models that providers serve through chat transport.""" - assert is_chat_transport_compatible_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "nvidia/llama-3.1-nemoguard-8b-content-safety", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: - """Keep chat-served policy classifiers out of ordinary worker roles.""" - assert not is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-5.2", - "qwen/qwen3-235b-a22b-instruct", - ], -) -def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: - """Preserve capable chat generation models for ordinary agent selection.""" - assert is_general_chat_agent_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "azure/text-embedding-3-large", - "text_embedding_3_large", - "company/reranker-v2", - "gpt-4o-mini-transcribe", - "omni-moderation-latest", - "gpt-image-1", - "dall-e-3", - "gpt-realtime", - "tts-1", - ], -) -def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: - """Reject clear non-chat endpoint families before transport or routing.""" - assert not is_chat_transport_compatible_model_id(model_id) - assert not is_general_chat_agent_model_id(model_id) - - -def test_direct_transport_can_use_guard_but_role_selection_cannot() -> None: - """Separate an intentional direct policy call from generic synthesis.""" - guard_agent = _agent( - "guard_agent", - "meta-llama/Llama-Guard-4-12B", - priority=10_000, - ) - chat_agent = _agent("chat_agent", "gpt-5.2") - client = ModelClient() - - output = client.chat( - guard_agent, - [{"role": "user", "content": "Classify this policy input."}], - ) - assert isinstance(output, str) - orchestrator = TaskOrchestrator([guard_agent, chat_agent], client=client) - assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent -''', - encoding="utf-8", - ) - - doctoring_path = Path("docs/doctoring/embedding-chat-capability-isolation.md") - doctoring = doctoring_path.read_text(encoding="utf-8") - heading = "## Transport compatibility and ordinary-role eligibility" - if heading not in doctoring: - doctoring += f'''\n\n{heading}\n\nModel identifiers are evaluated at two different boundaries. The transport\nboundary rejects endpoint-only families such as embeddings, reranking,\ntranscription, moderation endpoints, image generation, realtime, and speech-only\nmodels. The ordinary-agent boundary is stricter: chat-served guard and safety\nclassifiers remain available for intentional direct policy calls but cannot become\nthinkers, workers, verifiers, synthesizers, or generic failovers. Audio-capable\nchat models are not rejected merely because their identifier contains `audio`.\nThis avoids both the original embedding-to-chat failure and the inverse failure of\nblocking legitimate chat endpoints.\n\n### Additional references\n\nNVIDIA. (n.d.). *OpenAI-compatible APIs for NVIDIA NIM*. NVIDIA Documentation.\nRetrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/openai-api.html\n\nOpenAI. (n.d.). *Audio and speech*. OpenAI API documentation. Retrieved August 20,\n2026, from https://platform.openai.com/docs/guides/audio\n''' - doctoring_path.write_text(doctoring, encoding="utf-8") - - for temporary in ( - ".github/workflows/repair-chat-capability-explicit-integration.yml", - ".github/workflows/repair-chat-capability-final.yml", - ".github/workflows/repair-chat-transport-role-separation.yml", - ".github/workflows/repair-chat-capability-finalize-v2.yml", - ".github/workflows/repair-chat-capability-finalize-v3.yml", - ): - path = Path(temporary) - if path.exists(): - path.unlink() - PY - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Verify focused and full suites - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py - python -m pytest -q - git diff --check - - - name: Commit verified final state - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git diff --cached --check - git commit -m "fix: separate chat transport from synthesis roles" - git push origin HEAD:fix/chat-capability-isolation-embedding-models From 8885ef23907143fa5414a2a08632090a29d31259 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 01:17:21 -0700 Subject: [PATCH 66/89] ci: finalize chat capability boundaries on exact branch --- ...finalize-chat-capability-boundaries-v4.yml | 426 ++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 .github/workflows/finalize-chat-capability-boundaries-v4.yml diff --git a/.github/workflows/finalize-chat-capability-boundaries-v4.yml b/.github/workflows/finalize-chat-capability-boundaries-v4.yml new file mode 100644 index 000000000..c7cd3eb27 --- /dev/null +++ b/.github/workflows/finalize-chat-capability-boundaries-v4.yml @@ -0,0 +1,426 @@ +name: Finalize chat capability boundaries v4 + +on: + push: + branches: + - fix/chat-capability-isolation-embedding-models + paths: + - .github/workflows/finalize-chat-capability-boundaries-v4.yml + +permissions: + contents: write + +concurrency: + group: finalize-chat-capability-boundaries + cancel-in-progress: true + +jobs: + finalize: + name: Normalize, verify, and self-delete + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Checkout exact branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: fix/chat-capability-isolation-embedding-models + fetch-depth: 0 + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Normalize transport and ordinary-role boundaries + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from __future__ import annotations + + from pathlib import Path + import re + + CHAT_CAPABILITY = '''"""Classify chat transport compatibility and ordinary agent-role eligibility. + +Provider catalogs mix endpoint-only models with models served through an +OpenAI-compatible chat transport. Transport compatibility is not the same as +fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and +policy-classification models can use chat transport, while embedding, reranking, +transcription, moderation-endpoint, image-generation, realtime, and speech-only +models cannot. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "clip", + "dall", + "e5", + "embed", + "embedding", + "embeddings", + "gte", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "siglip", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_TRANSPORT_INCOMPATIBLE_PREFIXES = ( + "embed", + "moderat", + "rerank", + "transcrib", +) + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider-prefixed model identifier into lowercase tokens.""" + if type(model_id) is not str: + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def is_chat_transport_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use the ordinary chat transport. + + The classifier rejects only identifiers that clearly advertise an endpoint + family incompatible with chat messages. Audio-capable and safety-classifier + models remain transport-compatible because providers serve some of them over + ``/chat/completions``. + """ + tokens = _model_tokens(model_id) + if not tokens: + return False + for token in tokens: + if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: + return False + if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): + return False + return True + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Backward-compatible alias for chat transport compatibility.""" + return is_chat_transport_compatible_model_id(model_id) + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary orchestration roles. + + Explicit guard and safety models can use chat transport but are specialized + policy classifiers, not general answer synthesizers. This negative role gate + does not infer reasoning, coding, vision, or verification capabilities. + """ + tokens = _model_tokens(model_id) + if not tokens or not is_chat_transport_compatible_model_id(model_id): + return False + return not any( + token == "safety" + or token == "guard" + or token.startswith("nemoguard") + or token.endswith("guard") + for token in tokens + ) +''' + + def replace_import(text: str, replacement: str, *, label: str) -> str: + pattern = re.compile( + r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", + re.DOTALL, + ) + updated, count = pattern.subn(replacement, text, count=1) + if count != 1: + raise SystemExit(f"{label}: expected one chat_capability import, found {count}") + return updated + + def function_region(text: str, name: str) -> tuple[int, int]: + marker = re.search(rf"^ def {re.escape(name)}\(", text, re.MULTILINE) + if marker is None: + raise SystemExit(f"missing function: {name}") + next_marker = re.search(r"^ def [A-Za-z_][A-Za-z0-9_]*\(", text[marker.end():], re.MULTILINE) + end = len(text) if next_marker is None else marker.end() + next_marker.start() + return marker.start(), end + + def normalize_region_calls( + text: str, + function_name: str, + target_name: str, + *, + expected_minimum: int, + ) -> str: + start, end = function_region(text, function_name) + region = text[start:end] + pattern = re.compile( + r"(?:is_chat_compatible_model_id|is_chat_transport_compatible_model_id|is_general_chat_agent_model_id)\(" + ) + region, count = pattern.subn(f"{target_name}(", region) + if count < expected_minimum: + raise SystemExit( + f"{function_name}: expected at least {expected_minimum} capability call(s), found {count}" + ) + return text[:start] + region + text[end:] + + Path("contextual_orchestrator/chat_capability.py").write_text( + CHAT_CAPABILITY, + encoding="utf-8", + ) + + discovery_path = Path("contextual_orchestrator/model_discovery.py") + discovery = discovery_path.read_text(encoding="utf-8") + discovery = replace_import( + discovery, + "from .chat_capability import (\n" + " is_chat_compatible_model_id, # compatibility re-export\n" + " is_general_chat_agent_model_id,\n" + ")\n", + label="model_discovery import", + ) + discovery = re.sub( + r"(?:is_chat_compatible_model_id|is_chat_transport_compatible_model_id|is_general_chat_agent_model_id)\(", + "is_general_chat_agent_model_id(", + discovery, + ) + discovery = discovery.replace( + "One chat-compatible model found on a provider, with reported pricing.", + "One general chat-agent candidate found on a provider, with reported pricing.", + ) + discovery = discovery.replace( + "Build a disabled chat ModelAgent or reject a non-chat discovery record.", + "Build a disabled general chat agent or reject an ineligible discovery record.", + ) + discovery = discovery.replace( + 'raise ValueError("non-chat model cannot be converted into a chat agent")', + 'raise ValueError("model is not eligible for the general chat agent pool")', + ) + discovery_path.write_text(discovery, encoding="utf-8") + + orchestrator_path = Path("contextual_orchestrator/orchestrator.py") + orchestrator = orchestrator_path.read_text(encoding="utf-8") + orchestrator = replace_import( + orchestrator, + "from .chat_capability import (\n" + " is_chat_transport_compatible_model_id,\n" + " is_general_chat_agent_model_id,\n" + ")\n", + label="orchestrator import", + ) + for name in ("chat", "probe", "stream_chat"): + orchestrator = normalize_region_calls( + orchestrator, + name, + "is_chat_transport_compatible_model_id", + expected_minimum=1, + ) + for name in ( + "_plan_generated", + "_parse_workflow_plan", + "_ranked_agents", + "_failover_candidates", + ): + orchestrator = normalize_region_calls( + orchestrator, + name, + "is_general_chat_agent_model_id", + expected_minimum=1, + ) + orchestrator_path.write_text(orchestrator, encoding="utf-8") + + incident_tests_path = Path("tests/test_chat_model_capability_isolation.py") + incident_tests = incident_tests_path.read_text(encoding="utf-8") + incident_tests = incident_tests.replace( + 'with pytest.raises(ValueError, match="non-chat model"):', + 'with pytest.raises(ValueError, match="general chat agent"):', + ) + incident_tests_path.write_text(incident_tests, encoding="utf-8") + + Path("tests/test_chat_transport_role_separation.py").write_text( + '''"""Regression tests for chat transport versus ordinary agent roles.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_transport_compatible_model_id, + is_general_chat_agent_model_id, +) +from contextual_orchestrator.model_discovery import ( # noqa: E402 + DiscoveredModel, + agent_from_discovered, +) +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 + + +def _agent(agent_id: str, model_id: str, *, priority: int = 0) -> ModelAgent: + """Build one mock-backed agent for transport and role regressions.""" + return ModelAgent( + id=agent_id, + model=model_id, + base_url="mock://local", + priority=priority, + tags=("writing",), + ) + + +def _discovered(model_id: str) -> DiscoveredModel: + """Build one synthetic discovered provider model.""" + return DiscoveredModel( + provider_name="provider_account", + model_id=model_id, + credential_name="PROVIDER_API_KEY", + chat_base_url="https://provider.example.test/v1", + ) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-4o-mini-audio-preview", + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: + """Do not reject models that providers serve through chat transport.""" + assert is_chat_transport_compatible_model_id(model_id) + + +@pytest.mark.parametrize( + "model_id", + [ + "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "nvidia/llama-3.1-nemoguard-8b-content-safety", + "meta-llama/Llama-Guard-4-12B", + ], +) +def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: + """Keep chat-served policy classifiers out of ordinary worker roles.""" + assert not is_general_chat_agent_model_id(model_id) + with pytest.raises(ValueError, match="general chat agent"): + agent_from_discovered(_discovered(model_id)) + + +@pytest.mark.parametrize( + "model_id", + [ + "gpt-audio", + "gpt-4o-audio-preview", + "gpt-5.2", + "qwen/qwen3-235b-a22b-instruct", + ], +) +def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: + """Preserve capable chat generation models for ordinary agent selection.""" + assert is_general_chat_agent_model_id(model_id) + assert agent_from_discovered(_discovered(model_id)).model == model_id + + +@pytest.mark.parametrize( + "model_id", + [ + "azure/text-embedding-3-large", + "text_embedding_3_large", + "company/reranker-v2", + "gpt-4o-mini-transcribe", + "omni-moderation-latest", + "gpt-image-1", + "dall-e-3", + "gpt-realtime", + "tts-1", + ], +) +def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: + """Reject clear non-chat endpoint families before transport or routing.""" + assert not is_chat_transport_compatible_model_id(model_id) + assert not is_general_chat_agent_model_id(model_id) + + +def test_direct_transport_can_use_guard_but_role_selection_cannot() -> None: + """Separate an intentional direct policy call from generic synthesis.""" + guard_agent = _agent( + "guard_agent", + "meta-llama/Llama-Guard-4-12B", + priority=10_000, + ) + chat_agent = _agent("chat_agent", "gpt-5.2") + client = ModelClient() + + output = client.chat( + guard_agent, + [{"role": "user", "content": "Classify this policy input."}], + ) + assert isinstance(output, str) + orchestrator = TaskOrchestrator([guard_agent, chat_agent], client=client) + assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent +''', + encoding="utf-8", + ) + + doctoring_path = Path("docs/doctoring/embedding-chat-capability-isolation.md") + doctoring = doctoring_path.read_text(encoding="utf-8") + heading = "## Transport compatibility and ordinary-role eligibility" + if heading not in doctoring: + doctoring += f'''\n\n{heading}\n\nModel identifiers are evaluated at two different boundaries. The transport\nboundary rejects endpoint-only families such as embeddings, reranking,\ntranscription, moderation endpoints, image generation, realtime, and speech-only\nmodels. The ordinary-agent boundary is stricter: chat-served guard and safety\nclassifiers remain available for intentional direct policy calls but cannot become\nthinkers, workers, verifiers, synthesizers, or generic failovers. Audio-capable\nchat models are not rejected merely because their identifier contains `audio`.\nThis avoids both the original embedding-to-chat failure and the inverse failure of\nblocking legitimate chat endpoints.\n\n### Additional references\n\nNVIDIA. (n.d.). *OpenAI-compatible APIs for NVIDIA NIM*. NVIDIA Documentation.\nRetrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/openai-api.html\n\nOpenAI. (n.d.). *Audio and speech*. OpenAI API documentation. Retrieved August 20,\n2026, from https://platform.openai.com/docs/guides/audio\n''' + doctoring_path.write_text(doctoring, encoding="utf-8") + + for temporary in ( + ".github/workflows/repair-chat-capability-explicit-integration.yml", + ".github/workflows/repair-chat-capability-final.yml", + ".github/workflows/repair-chat-transport-role-separation.yml", + ".github/workflows/repair-chat-capability-finalize-v2.yml", + ".github/workflows/repair-chat-capability-finalize-v3.yml", + ".github/workflows/finalize-chat-capability-boundaries-v4.yml", + ): + path = Path(temporary) + if path.exists(): + path.unlink() + PY + + - name: Install hash-locked test dependencies + run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt + + - name: Verify focused and full suites + shell: bash + run: | + set -euo pipefail + python -m compileall -q contextual_orchestrator + python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py + python -m pytest -q + git diff --check + + - name: Commit verified final state + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + git commit -m "fix: separate chat transport from synthesis roles" + git push origin HEAD:fix/chat-capability-isolation-embedding-models From d1e06d59be525569c77da17405d6a2a5819b6d27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:04:41 +0900 Subject: [PATCH 67/89] fix: close chat capability review gaps --- ...finalize-chat-capability-boundaries-v4.yml | 426 ------------------ contextual_orchestrator/chat_capability.py | 12 +- contextual_orchestrator/model_discovery.py | 36 +- contextual_orchestrator/orchestrator.py | 8 +- .../embedding-chat-capability-isolation.md | 8 +- tests/test_chat_model_capability_isolation.py | 14 + ...t_chat_passthrough_capability_isolation.py | 2 +- tests/test_chat_transport_role_separation.py | 3 + tests/test_local_mlx.py | 5 +- 9 files changed, 58 insertions(+), 456 deletions(-) delete mode 100644 .github/workflows/finalize-chat-capability-boundaries-v4.yml diff --git a/.github/workflows/finalize-chat-capability-boundaries-v4.yml b/.github/workflows/finalize-chat-capability-boundaries-v4.yml deleted file mode 100644 index c7cd3eb27..000000000 --- a/.github/workflows/finalize-chat-capability-boundaries-v4.yml +++ /dev/null @@ -1,426 +0,0 @@ -name: Finalize chat capability boundaries v4 - -on: - push: - branches: - - fix/chat-capability-isolation-embedding-models - paths: - - .github/workflows/finalize-chat-capability-boundaries-v4.yml - -permissions: - contents: write - -concurrency: - group: finalize-chat-capability-boundaries - cancel-in-progress: true - -jobs: - finalize: - name: Normalize, verify, and self-delete - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - timeout-minutes: 40 - steps: - - name: Checkout exact branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 - with: - ref: fix/chat-capability-isolation-embedding-models - fetch-depth: 0 - persist-credentials: true - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Normalize transport and ordinary-role boundaries - shell: bash - run: | - set -euo pipefail - python - <<'PY' - from __future__ import annotations - - from pathlib import Path - import re - - CHAT_CAPABILITY = '''"""Classify chat transport compatibility and ordinary agent-role eligibility. - -Provider catalogs mix endpoint-only models with models served through an -OpenAI-compatible chat transport. Transport compatibility is not the same as -fitness for an ordinary thinker, worker, verifier, or synthesizer role: audio and -policy-classification models can use chat transport, while embedding, reranking, -transcription, moderation-endpoint, image-generation, realtime, and speech-only -models cannot. -""" - -from __future__ import annotations - -import re - -_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") -_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( - { - "bge", - "clip", - "dall", - "e5", - "embed", - "embedding", - "embeddings", - "gte", - "image", - "images", - "moderation", - "realtime", - "rerank", - "reranker", - "siglip", - "sora", - "speech", - "transcribe", - "transcription", - "tts", - "whisper", - } -) -_TRANSPORT_INCOMPATIBLE_PREFIXES = ( - "embed", - "moderat", - "rerank", - "transcrib", -) - - -def _model_tokens(model_id: str) -> tuple[str, ...]: - """Normalize one provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: - return () - return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) - - -def is_chat_transport_compatible_model_id(model_id: str) -> bool: - """Return whether an identifier can use the ordinary chat transport. - - The classifier rejects only identifiers that clearly advertise an endpoint - family incompatible with chat messages. Audio-capable and safety-classifier - models remain transport-compatible because providers serve some of them over - ``/chat/completions``. - """ - tokens = _model_tokens(model_id) - if not tokens: - return False - for token in tokens: - if token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS: - return False - if token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES): - return False - return True - - -def is_chat_compatible_model_id(model_id: str) -> bool: - """Backward-compatible alias for chat transport compatibility.""" - return is_chat_transport_compatible_model_id(model_id) - - -def is_general_chat_agent_model_id(model_id: str) -> bool: - """Return whether a chat model may enter ordinary orchestration roles. - - Explicit guard and safety models can use chat transport but are specialized - policy classifiers, not general answer synthesizers. This negative role gate - does not infer reasoning, coding, vision, or verification capabilities. - """ - tokens = _model_tokens(model_id) - if not tokens or not is_chat_transport_compatible_model_id(model_id): - return False - return not any( - token == "safety" - or token == "guard" - or token.startswith("nemoguard") - or token.endswith("guard") - for token in tokens - ) -''' - - def replace_import(text: str, replacement: str, *, label: str) -> str: - pattern = re.compile( - r"from \.chat_capability import (?:\([^)]*\)|[^\n]+)\n", - re.DOTALL, - ) - updated, count = pattern.subn(replacement, text, count=1) - if count != 1: - raise SystemExit(f"{label}: expected one chat_capability import, found {count}") - return updated - - def function_region(text: str, name: str) -> tuple[int, int]: - marker = re.search(rf"^ def {re.escape(name)}\(", text, re.MULTILINE) - if marker is None: - raise SystemExit(f"missing function: {name}") - next_marker = re.search(r"^ def [A-Za-z_][A-Za-z0-9_]*\(", text[marker.end():], re.MULTILINE) - end = len(text) if next_marker is None else marker.end() + next_marker.start() - return marker.start(), end - - def normalize_region_calls( - text: str, - function_name: str, - target_name: str, - *, - expected_minimum: int, - ) -> str: - start, end = function_region(text, function_name) - region = text[start:end] - pattern = re.compile( - r"(?:is_chat_compatible_model_id|is_chat_transport_compatible_model_id|is_general_chat_agent_model_id)\(" - ) - region, count = pattern.subn(f"{target_name}(", region) - if count < expected_minimum: - raise SystemExit( - f"{function_name}: expected at least {expected_minimum} capability call(s), found {count}" - ) - return text[:start] + region + text[end:] - - Path("contextual_orchestrator/chat_capability.py").write_text( - CHAT_CAPABILITY, - encoding="utf-8", - ) - - discovery_path = Path("contextual_orchestrator/model_discovery.py") - discovery = discovery_path.read_text(encoding="utf-8") - discovery = replace_import( - discovery, - "from .chat_capability import (\n" - " is_chat_compatible_model_id, # compatibility re-export\n" - " is_general_chat_agent_model_id,\n" - ")\n", - label="model_discovery import", - ) - discovery = re.sub( - r"(?:is_chat_compatible_model_id|is_chat_transport_compatible_model_id|is_general_chat_agent_model_id)\(", - "is_general_chat_agent_model_id(", - discovery, - ) - discovery = discovery.replace( - "One chat-compatible model found on a provider, with reported pricing.", - "One general chat-agent candidate found on a provider, with reported pricing.", - ) - discovery = discovery.replace( - "Build a disabled chat ModelAgent or reject a non-chat discovery record.", - "Build a disabled general chat agent or reject an ineligible discovery record.", - ) - discovery = discovery.replace( - 'raise ValueError("non-chat model cannot be converted into a chat agent")', - 'raise ValueError("model is not eligible for the general chat agent pool")', - ) - discovery_path.write_text(discovery, encoding="utf-8") - - orchestrator_path = Path("contextual_orchestrator/orchestrator.py") - orchestrator = orchestrator_path.read_text(encoding="utf-8") - orchestrator = replace_import( - orchestrator, - "from .chat_capability import (\n" - " is_chat_transport_compatible_model_id,\n" - " is_general_chat_agent_model_id,\n" - ")\n", - label="orchestrator import", - ) - for name in ("chat", "probe", "stream_chat"): - orchestrator = normalize_region_calls( - orchestrator, - name, - "is_chat_transport_compatible_model_id", - expected_minimum=1, - ) - for name in ( - "_plan_generated", - "_parse_workflow_plan", - "_ranked_agents", - "_failover_candidates", - ): - orchestrator = normalize_region_calls( - orchestrator, - name, - "is_general_chat_agent_model_id", - expected_minimum=1, - ) - orchestrator_path.write_text(orchestrator, encoding="utf-8") - - incident_tests_path = Path("tests/test_chat_model_capability_isolation.py") - incident_tests = incident_tests_path.read_text(encoding="utf-8") - incident_tests = incident_tests.replace( - 'with pytest.raises(ValueError, match="non-chat model"):', - 'with pytest.raises(ValueError, match="general chat agent"):', - ) - incident_tests_path.write_text(incident_tests, encoding="utf-8") - - Path("tests/test_chat_transport_role_separation.py").write_text( - '''"""Regression tests for chat transport versus ordinary agent roles.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.chat_capability import ( # noqa: E402 - is_chat_transport_compatible_model_id, - is_general_chat_agent_model_id, -) -from contextual_orchestrator.model_discovery import ( # noqa: E402 - DiscoveredModel, - agent_from_discovered, -) -from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 - - -def _agent(agent_id: str, model_id: str, *, priority: int = 0) -> ModelAgent: - """Build one mock-backed agent for transport and role regressions.""" - return ModelAgent( - id=agent_id, - model=model_id, - base_url="mock://local", - priority=priority, - tags=("writing",), - ) - - -def _discovered(model_id: str) -> DiscoveredModel: - """Build one synthetic discovered provider model.""" - return DiscoveredModel( - provider_name="provider_account", - model_id=model_id, - credential_name="PROVIDER_API_KEY", - chat_base_url="https://provider.example.test/v1", - ) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-4o-mini-audio-preview", - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_chat_endpoint_models_remain_transport_compatible(model_id: str) -> None: - """Do not reject models that providers serve through chat transport.""" - assert is_chat_transport_compatible_model_id(model_id) - - -@pytest.mark.parametrize( - "model_id", - [ - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "nvidia/llama-3.1-nemoguard-8b-content-safety", - "meta-llama/Llama-Guard-4-12B", - ], -) -def test_guard_models_do_not_enter_general_synthesis_pool(model_id: str) -> None: - """Keep chat-served policy classifiers out of ordinary worker roles.""" - assert not is_general_chat_agent_model_id(model_id) - with pytest.raises(ValueError, match="general chat agent"): - agent_from_discovered(_discovered(model_id)) - - -@pytest.mark.parametrize( - "model_id", - [ - "gpt-audio", - "gpt-4o-audio-preview", - "gpt-5.2", - "qwen/qwen3-235b-a22b-instruct", - ], -) -def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: - """Preserve capable chat generation models for ordinary agent selection.""" - assert is_general_chat_agent_model_id(model_id) - assert agent_from_discovered(_discovered(model_id)).model == model_id - - -@pytest.mark.parametrize( - "model_id", - [ - "azure/text-embedding-3-large", - "text_embedding_3_large", - "company/reranker-v2", - "gpt-4o-mini-transcribe", - "omni-moderation-latest", - "gpt-image-1", - "dall-e-3", - "gpt-realtime", - "tts-1", - ], -) -def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: - """Reject clear non-chat endpoint families before transport or routing.""" - assert not is_chat_transport_compatible_model_id(model_id) - assert not is_general_chat_agent_model_id(model_id) - - -def test_direct_transport_can_use_guard_but_role_selection_cannot() -> None: - """Separate an intentional direct policy call from generic synthesis.""" - guard_agent = _agent( - "guard_agent", - "meta-llama/Llama-Guard-4-12B", - priority=10_000, - ) - chat_agent = _agent("chat_agent", "gpt-5.2") - client = ModelClient() - - output = client.chat( - guard_agent, - [{"role": "user", "content": "Classify this policy input."}], - ) - assert isinstance(output, str) - orchestrator = TaskOrchestrator([guard_agent, chat_agent], client=client) - assert orchestrator._select_agent("Produce the final answer.", "synthesizer") is chat_agent -''', - encoding="utf-8", - ) - - doctoring_path = Path("docs/doctoring/embedding-chat-capability-isolation.md") - doctoring = doctoring_path.read_text(encoding="utf-8") - heading = "## Transport compatibility and ordinary-role eligibility" - if heading not in doctoring: - doctoring += f'''\n\n{heading}\n\nModel identifiers are evaluated at two different boundaries. The transport\nboundary rejects endpoint-only families such as embeddings, reranking,\ntranscription, moderation endpoints, image generation, realtime, and speech-only\nmodels. The ordinary-agent boundary is stricter: chat-served guard and safety\nclassifiers remain available for intentional direct policy calls but cannot become\nthinkers, workers, verifiers, synthesizers, or generic failovers. Audio-capable\nchat models are not rejected merely because their identifier contains `audio`.\nThis avoids both the original embedding-to-chat failure and the inverse failure of\nblocking legitimate chat endpoints.\n\n### Additional references\n\nNVIDIA. (n.d.). *OpenAI-compatible APIs for NVIDIA NIM*. NVIDIA Documentation.\nRetrieved August 20, 2026, from https://docs.nvidia.com/nim/large-language-models/latest/openai-api.html\n\nOpenAI. (n.d.). *Audio and speech*. OpenAI API documentation. Retrieved August 20,\n2026, from https://platform.openai.com/docs/guides/audio\n''' - doctoring_path.write_text(doctoring, encoding="utf-8") - - for temporary in ( - ".github/workflows/repair-chat-capability-explicit-integration.yml", - ".github/workflows/repair-chat-capability-final.yml", - ".github/workflows/repair-chat-transport-role-separation.yml", - ".github/workflows/repair-chat-capability-finalize-v2.yml", - ".github/workflows/repair-chat-capability-finalize-v3.yml", - ".github/workflows/finalize-chat-capability-boundaries-v4.yml", - ): - path = Path(temporary) - if path.exists(): - path.unlink() - PY - - - name: Install hash-locked test dependencies - run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r fuzz/requirements-property.txt - - - name: Verify focused and full suites - shell: bash - run: | - set -euo pipefail - python -m compileall -q contextual_orchestrator - python -m pytest -q tests/test_chat_transport_role_separation.py tests/test_chat_model_capability_isolation.py - python -m pytest -q - git diff --check - - - name: Commit verified final state - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git diff --cached --check - git commit -m "fix: separate chat transport from synthesis roles" - git push origin HEAD:fix/chat-capability-isolation-embedding-models diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index d149a3427..fdc6fbf4e 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -16,6 +16,8 @@ _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( { "bge", + "clip", + "dall", "e5", "embed", "embedding", @@ -27,6 +29,7 @@ "realtime", "rerank", "reranker", + "siglip", "sora", "speech", "transcribe", @@ -52,6 +55,11 @@ def is_chat_compatible_model_id(model_id: str) -> bool: ``/chat/completions``. """ tokens = _model_tokens(model_id) + return _is_transport_compatible_tokens(tokens) + + +def _is_transport_compatible_tokens(tokens: tuple[str, ...]) -> bool: + """Judge transport compatibility from already-normalized model tokens.""" if not tokens: return False for token in tokens: @@ -64,7 +72,7 @@ def is_chat_compatible_model_id(model_id: str) -> bool: def _model_tokens(model_id: str) -> tuple[str, ...]: """Normalize one provider-prefixed model identifier into lowercase tokens.""" - if type(model_id) is not str: + if not isinstance(model_id, str): return () return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) @@ -77,7 +85,7 @@ def is_general_chat_agent_model_id(model_id: str) -> bool: does not infer reasoning, coding, vision, or verification capabilities. """ tokens = _model_tokens(model_id) - if not tokens or not is_chat_compatible_model_id(model_id): + if not tokens or not _is_transport_compatible_tokens(tokens): return False return not any( token == "safety" diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 00064f575..738111039 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -26,7 +26,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from .batch_routing import cheapest_upstream from .chat_capability import is_general_chat_agent_model_id from .credentials import get_credential from .orchestrator import ModelAgent @@ -91,7 +90,7 @@ class ProviderModelSource: @dataclass(frozen=True) class DiscoveredModel: - """One chat-compatible model found on a provider, with reported pricing.""" + """One general chat-agent eligible model found on a provider, with pricing.""" provider_name: str model_id: str @@ -139,7 +138,7 @@ def _price_per_1k(value: Any) -> float | None: def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: - """Parse one OpenAI-compatible catalog into chat-compatible candidates.""" + """Parse one OpenAI-compatible catalog into general chat-agent candidates.""" rows = payload.get("data") if isinstance(payload, dict) else None discovered: list[DiscoveredModel] = [] for row in rows if isinstance(rows, list) else []: @@ -164,7 +163,7 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredModel]: - """Parse one Bytez chat catalog without admitting non-chat identifiers.""" + """Parse one Bytez chat catalog without admitting ineligible identifiers.""" rows = payload.get("output") if isinstance(payload, dict) else None discovered: list[DiscoveredModel] = [] for row in rows if isinstance(rows, list) else []: @@ -288,11 +287,11 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook def select_cheapest_discovered_agent( discovered: list[DiscoveredModel], price_book: "PriceBook" ) -> DiscoveredModel | None: - """Pick the lowest-cost chat-compatible model per the price book. + """Pick the lowest-cost general chat-agent model per the price book. - Reuses :func:`~contextual_orchestrator.batch_routing.cheapest_upstream`, the - existing cost-optimizing upstream selector. Call :func:`refresh_price_book` - first so discovered pricing is visible; an unpriced candidate costs ``0`` + Uses the same representative request cost as the top-N selector. Call + :func:`refresh_price_book` first so discovered pricing is visible; an + unpriced candidate costs ``0`` under that selector's documented contract and is treated as free, not unknown -- so a genuinely unpriced provider (e.g. Bytez, priced by GPU-second rather than per token) will always look cheapest here. Fine for @@ -302,28 +301,23 @@ def select_cheapest_discovered_agent( eligible = [model for model in discovered if is_general_chat_agent_model_id(model.model_id)] if not eligible: return None - candidates = [{"provider": model.provider_name, "model": model.model_id} for model in eligible] - winner = cheapest_upstream(candidates, price_book) - if winner is None: - return None - for model in eligible: - if model.provider_name == winner["provider"] and model.model_id == winner["model"]: - return model - return None # pragma: no cover - winner always comes from candidates + return min(eligible, key=lambda model: _discovered_cost(model, price_book)) def select_top_n_cheapest_discovered_agents( discovered: list[DiscoveredModel], price_book: "PriceBook", limit: int ) -> list[DiscoveredModel]: - """Return the ``limit`` cheapest chat-compatible models in ascending cost.""" + """Return the ``limit`` cheapest general chat-agent models in ascending cost.""" if limit <= 0: return [] eligible = [model for model in discovered if is_general_chat_agent_model_id(model.model_id)] if not eligible: return [] - def _cost(model: DiscoveredModel) -> float: - cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) - return cost + return sorted(eligible, key=lambda model: _discovered_cost(model, price_book))[:limit] + - return sorted(eligible, key=_cost)[:limit] +def _discovered_cost(model: DiscoveredModel, price_book: "PriceBook") -> float: + """Price the representative discovery request used by both selectors.""" + cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) + return cost diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0cc65e404..254d593be 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1159,9 +1159,9 @@ def proxy_send( f"model {agent.model!r} is not chat-compatible and cannot serve {agent.id!r}" ) if agent.base_url.startswith("mock://"): - return self._mock_raw(agent, endpoint, payload) + return self._mock_raw(agent, normalized_endpoint, payload) destination = self._validate_provider(agent) # pragma: no cover - if endpoint.strip("/") == "responses" and _is_local_provider_url(agent.base_url): + if normalized_endpoint == "responses" and _is_local_provider_url(agent.base_url): chat_payload = _responses_to_chat_payload(payload) chat_payload.setdefault("max_tokens", self.max_output_tokens) if _is_direct_mlx_provider_url(agent.base_url) and self.chat_template_args: @@ -1172,7 +1172,7 @@ def proxy_send( ) return _chat_to_responses_payload(chat_response, payload) with _local_provider_slot(agent, self.local_concurrency, self.timeout): # pragma: no cover - return self._send_raw_with_retry(agent, endpoint, payload, destination) + return self._send_raw_with_retry(agent, normalized_endpoint, payload, destination) def _send_raw_with_retry( self, @@ -2576,6 +2576,8 @@ def _invoke( usage when available (else None), so spend analytics can prefer it. """ candidates = self._failover_candidates(primary, text, role) + if not candidates: + raise RuntimeError(f"no chat-compatible agent available for role={role}") last_error: Exception | None = None for agent in candidates: try: diff --git a/docs/doctoring/embedding-chat-capability-isolation.md b/docs/doctoring/embedding-chat-capability-isolation.md index 93429d30b..a1bdcf76b 100644 --- a/docs/doctoring/embedding-chat-capability-isolation.md +++ b/docs/doctoring/embedding-chat-capability-isolation.md @@ -93,7 +93,10 @@ persistent configuration. ## Verification evidence `tests/test_chat_model_capability_isolation.py` reproduces the exact Azure model ID -and provider/separator aliases. It verifies: +and provider/separator aliases. Together with +`tests/test_chat_capability_unknown_identifiers.py`, +`tests/test_chat_transport_role_separation.py`, and +`tests/test_chat_passthrough_capability_isolation.py`, it verifies: - OpenAI-compatible and Bytez catalog filtering; - malformed and prefix-only identifier handling; @@ -107,6 +110,9 @@ and provider/separator aliases. It verifies: - readiness failure with a stable non-chat code before provider access; - planner inventory and generated-plan isolation; - distinction between chat-served audio/policy models and general agent roles. +- conservative unknown-identifier handling, including unrelated `vanguard` names; +- endpoint-family exclusions for image-generation (`dall-e`), CLIP, and SigLIP; +- normalized `/v1/responses` passthrough and pre-transport rejection of embedding models. ## References diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index 23e17d3da..ba0f6916c 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -276,6 +276,20 @@ def test_failover_candidates_exclude_stale_embedding_agents() -> None: assert candidates == [chat_agent] +def test_invoke_fails_clearly_when_no_general_chat_agent_remains() -> None: + """Report the role boundary instead of claiming that zero candidates failed.""" + embedding_agent = _agent("embedding_agent", "azure/text-embedding-3-large") + orchestrator = TaskOrchestrator([embedding_agent]) + + with pytest.raises(RuntimeError, match="no chat-compatible agent available"): + orchestrator._invoke( + embedding_agent, + [{"role": "user", "content": "Produce the final answer."}], + text="Produce the final answer.", + role="worker", + ) + + def test_model_client_rejects_non_chat_model_before_mock_or_network_call() -> None: """Keep the provider boundary fail-closed even when selection is bypassed.""" client = ModelClient() diff --git a/tests/test_chat_passthrough_capability_isolation.py b/tests/test_chat_passthrough_capability_isolation.py index d200257c5..32026e5cf 100644 --- a/tests/test_chat_passthrough_capability_isolation.py +++ b/tests/test_chat_passthrough_capability_isolation.py @@ -32,7 +32,7 @@ def _chat_agent() -> ModelAgent: ) -@pytest.mark.parametrize("endpoint", ["chat/completions", "/v1/chat/completions", "responses"]) +@pytest.mark.parametrize("endpoint", ["chat/completions", "/v1/chat/completions", "responses", "/v1/responses"]) def test_proxy_send_rejects_embedding_before_mock_or_network_transport(endpoint: str) -> None: """Keep raw OpenAI passthrough from bypassing the chat transport invariant.""" client = ModelClient() diff --git a/tests/test_chat_transport_role_separation.py b/tests/test_chat_transport_role_separation.py index 57655f1e3..02aff53a7 100644 --- a/tests/test_chat_transport_role_separation.py +++ b/tests/test_chat_transport_role_separation.py @@ -66,6 +66,9 @@ def test_general_generation_models_remain_agent_eligible(model_id: str) -> None: "gpt-4o-mini-transcribe", "omni-moderation-latest", "gpt-image-1", + "dall-e-3", + "openai/clip-vit-large-patch14", + "google/siglip-so400m-patch14-384", "sora-2", "gpt-realtime", "tts-1", diff --git a/tests/test_local_mlx.py b/tests/test_local_mlx.py index 8c2b3d815..f1eca6acd 100644 --- a/tests/test_local_mlx.py +++ b/tests/test_local_mlx.py @@ -350,7 +350,8 @@ def test_response_without_content_or_reasoning_fails_clearly() -> None: ModelClient()._response_content(agent, {"choices": [{"message": {}}]}) -def test_local_responses_passthrough_adapts_to_chat_transport() -> None: +@pytest.mark.parametrize("endpoint", ["responses", "/v1/responses"]) +def test_local_responses_passthrough_adapts_to_chat_transport(endpoint: str) -> None: agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") client = ModelClient(max_retries=0, chat_template_args={"enable_thinking": False}) with patch.object(client, "_validate_provider", return_value=None), patch.object( @@ -369,7 +370,7 @@ def test_local_responses_passthrough_adapts_to_chat_transport() -> None: ) as send: response = client.proxy_send( agent, - "responses", + endpoint, { "model": "local-model", "instructions": "Be concise.", From 22a3270a27ef19d15254eb83948ec6f3e18b56ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 02:42:58 -0700 Subject: [PATCH 68/89] test: make unknown-identifier regression directly runnable --- tests/test_chat_capability_unknown_identifiers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_chat_capability_unknown_identifiers.py b/tests/test_chat_capability_unknown_identifiers.py index 1e317cbc6..8dad26129 100644 --- a/tests/test_chat_capability_unknown_identifiers.py +++ b/tests/test_chat_capability_unknown_identifiers.py @@ -40,3 +40,7 @@ def test_explicit_policy_classifier_markers_remain_role_ineligible(model_id: str """Keep exact guard, safety, and NemoGuard markers out of general synthesis roles.""" assert is_chat_compatible_model_id(model_id) assert not is_general_chat_agent_model_id(model_id) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__])) From 60be2e46a42f25fe4975a701124ae3def5aa9056 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 02:43:23 -0700 Subject: [PATCH 69/89] test: make transport-role regression directly runnable --- tests/test_chat_transport_role_separation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_chat_transport_role_separation.py b/tests/test_chat_transport_role_separation.py index 02aff53a7..17f5b7277 100644 --- a/tests/test_chat_transport_role_separation.py +++ b/tests/test_chat_transport_role_separation.py @@ -78,3 +78,7 @@ def test_endpoint_only_models_fail_both_boundaries(model_id: str) -> None: """Reject endpoint-only model families before transport or ordinary role routing.""" assert not is_chat_compatible_model_id(model_id) assert not is_general_chat_agent_model_id(model_id) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__])) From e91a108603b13545965fbb11b9427c2772b8dd33 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 02:43:56 -0700 Subject: [PATCH 70/89] test: cover legacy completions and direct execution --- ...test_chat_passthrough_capability_isolation.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_chat_passthrough_capability_isolation.py b/tests/test_chat_passthrough_capability_isolation.py index 32026e5cf..927c21324 100644 --- a/tests/test_chat_passthrough_capability_isolation.py +++ b/tests/test_chat_passthrough_capability_isolation.py @@ -32,7 +32,17 @@ def _chat_agent() -> ModelAgent: ) -@pytest.mark.parametrize("endpoint", ["chat/completions", "/v1/chat/completions", "responses", "/v1/responses"]) +@pytest.mark.parametrize( + "endpoint", + [ + "chat/completions", + "/v1/chat/completions", + "completions", + "/v1/completions", + "responses", + "/v1/responses", + ], +) def test_proxy_send_rejects_embedding_before_mock_or_network_transport(endpoint: str) -> None: """Keep raw OpenAI passthrough from bypassing the chat transport invariant.""" client = ModelClient() @@ -111,3 +121,7 @@ def test_chat_served_specialized_models_remain_valid_passthrough_transports(mode assert response["object"] == "chat.completion" assert response["model"] == model_id + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__])) From a4aef39c45299264849d91d4a9734d337520b1d7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 02:45:36 -0700 Subject: [PATCH 71/89] test: make chat-capability regression directly runnable --- tests/test_chat_model_capability_isolation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index ba0f6916c..249240e24 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -385,3 +385,7 @@ def chat(self, _agent, messages, **_kwargs): assert "embedding_agent" not in client.system_prompt assert "azure/text-embedding-3-large" not in client.system_prompt assert "chat_agent" in client.system_prompt + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__])) From f13ee66e571ee2a99213104f8dc091ae4f63c7f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 02:49:12 -0700 Subject: [PATCH 72/89] docs: register chat capability regression checks --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f10818c6c..518cef880 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,6 @@ Run an evaluation against that server with `--temperature 0` for repeatable judg Model-based conduct verification requires `fast-mlsirm` in the same runtime and fails closed when it is absent or broken; fast-mlsirm sends its judge completion through this contextual-orchestrator gateway, so no direct provider fallback is used. “Same runtime” means that the exact interpreter used for the live run can import both packages: install both checkouts into one environment (prefer editable installs), or expose both source roots with `PYTHONPATH` during a source run. Before a live judge benchmark, run `python -m contextual_orchestrator check-fast-mlsirm` with that exact interpreter. It prints the interpreter, package version, transitive-import status, and contextual contract check, and exits nonzero on a missing dependency or contract mismatch. Do not run the preflight in one virtual environment and the judge in another. See [ADR 0001](docs/planning/adrs/0001-fail-closed-model-judgment.md). The agent pool is manageable at runtime: `POST`/`PATCH`/`DELETE` on `/api/v1/agent_pools/default/worker_agents[/{id}]` add, govern, and remove model-group members. Pass `--agents-db PATH` (or `CONTEXTUAL_ORCHESTRATOR_AGENTS_DB`) to persist those changes to a stdlib sqlite file — stored changes overlay the seed agents file at startup, and removals write disabled tombstones so they survive restarts; without it the pool is in-memory as before. - Beyond the local MLX/llama.cpp discovery above, `python -m contextual_orchestrator discover-models [--agents-db PATH]` discovers models from remote providers (OpenAI, OpenRouter, NVIDIA NIM ×2 keys, Bytez) for any subset of their KV-registered credentials, and can persist them into the same `--agents-db` sqlite file, added disabled by default. See [docs/kv-credentials.md](docs/kv-credentials.md#multi-provider-auto-discovery) for the credential-name table and cost-based auto-selection. Seed the credential into the KV once at bootstrap: @@ -304,6 +303,10 @@ python tests/test_admin_contract.py python tests/test_conventions.py python tests/test_api_contract.py python tests/test_security_hardening.py +python tests/test_chat_model_capability_isolation.py +python tests/test_chat_transport_role_separation.py +python tests/test_chat_capability_unknown_identifiers.py +python tests/test_chat_passthrough_capability_isolation.py python tests/test_repository_security_metadata.py python tests/test_product_planning_contract.py python tests/test_plugin_driven_artifacts.py From 88fee976ca4222309f625058a6f95f09e66744ec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:59:09 +0900 Subject: [PATCH 73/89] fix: exclude shieldgemma from general chat roles --- contextual_orchestrator/chat_capability.py | 1 + tests/test_chat_capability_unknown_identifiers.py | 1 + 2 files changed, 2 insertions(+) diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index fdc6fbf4e..4a2a96cd5 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -90,6 +90,7 @@ def is_general_chat_agent_model_id(model_id: str) -> bool: return not any( token == "safety" or token == "guard" + or token == "shieldgemma" or token.startswith("nemoguard") for token in tokens ) diff --git a/tests/test_chat_capability_unknown_identifiers.py b/tests/test_chat_capability_unknown_identifiers.py index 8dad26129..d9e72b6ca 100644 --- a/tests/test_chat_capability_unknown_identifiers.py +++ b/tests/test_chat_capability_unknown_identifiers.py @@ -34,6 +34,7 @@ def test_unknown_names_that_merely_end_with_guard_remain_eligible(model_id: str) "meta-llama/llama-guard-4-12b", "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", "nvidia/llama-3.1-nemoguard-8b-topic-control", + "google/shieldgemma-2b-it", ], ) def test_explicit_policy_classifier_markers_remain_role_ineligible(model_id: str) -> None: From 0777e1494b2aca6313572fe125cea93439439eff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 03:02:38 -0700 Subject: [PATCH 74/89] test(cost): reject persisted partial price rows --- tests/test_discovery_bootstrap_selection.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py index e6a3c0ade..e12eeb0b6 100644 --- a/tests/test_discovery_bootstrap_selection.py +++ b/tests/test_discovery_bootstrap_selection.py @@ -100,6 +100,24 @@ def test_partial_provider_price_is_unknown_instead_of_fabricating_a_free_compone assert select_cheapest_discovered_agent([partial, complete], price_book) is complete +def test_persisted_price_row_missing_one_component_remains_unknown() -> None: + """KV corruption must not silently manufacture a zero-priced component.""" + store = InMemoryConfigStore() + store.set( + "llm_price_entries", + "partial_vendor:partial-model", + { + "provider_name": "partial_vendor", + "model_name": "partial-model", + "prompt_price_per_1k": 0.001, + "currency_code": "USD", + }, + ) + price_book = PriceBook(store) + + assert price_book.get_price("partial_vendor", "partial-model") is None + + def test_invalid_catalog_prices_are_unknown_not_trusted_cost_evidence() -> None: """Reject negative, non-finite, and boolean provider price values.""" assert model_discovery._price_per_1k("-0.000001") is None From 386c9a03d1d7076106e4061776e81ffd6dac4d6f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:44:36 +0900 Subject: [PATCH 75/89] fix: reject incomplete persisted price rows --- contextual_orchestrator/cost_ledger.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index 8c1a0dde1..daee00c62 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -166,8 +166,13 @@ def get_price(self, provider: str, model: str) -> Optional[PriceEntry]: if not isinstance(raw, dict): return None try: - prompt_price = float(raw.get("prompt_price_per_1k", 0.0)) - completion_price = float(raw.get("completion_price_per_1k", 0.0)) + if ( + "prompt_price_per_1k" not in raw + or "completion_price_per_1k" not in raw + ): + return None + prompt_price = float(raw["prompt_price_per_1k"]) + completion_price = float(raw["completion_price_per_1k"]) except (OverflowError, TypeError, ValueError): return None if ( From 5d989c6177514d30dace2afa2cf61c64a7d41f67 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:49:00 +0900 Subject: [PATCH 76/89] feat: add contextual review gateway bootstrap --- contextual_orchestrator/review_gateway.py | 127 ++++++++++++++++++++++ tests/test_review_gateway.py | 126 +++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 contextual_orchestrator/review_gateway.py create mode 100644 tests/test_review_gateway.py diff --git a/contextual_orchestrator/review_gateway.py b/contextual_orchestrator/review_gateway.py new file mode 100644 index 000000000..7de5a7b28 --- /dev/null +++ b/contextual_orchestrator/review_gateway.py @@ -0,0 +1,127 @@ +"""Bootstrap a local authenticated gateway for trusted CI model reviews. + +The five provider keys arrive from a deployment environment only as bootstrap +transport. They are immediately registered in the process-local KV, after +which model discovery and every provider request use the normal credential +registry path. This keeps the review sidecar useful without pretending that a +short-lived runner has a durable production credential store. +""" + +from __future__ import annotations + +import argparse +import os +from dataclasses import replace +from typing import Mapping + +from .credentials import NotConfigured, register_credential +from .cost_ledger import PriceBook +from .kv_config import InMemoryConfigStore +from .model_discovery import ( + agent_from_discovered, + discover_all_models, + refresh_price_book, + select_top_n_cheapest_discovered_agents, +) +from .orchestrator import ModelClient, TaskOrchestrator +from .server import SecurityConfig, serve + +REVIEW_CREDENTIAL_NAMES: tuple[str, ...] = ( + "BYTEZ_API_KEY", + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", +) +DEFAULT_REVIEW_AGENT_LIMIT = 12 + + +def register_review_credentials(environment: Mapping[str, str]) -> tuple[str, ...]: + """Register non-empty CI provider keys in the process-local KV. + + ``environment`` is bootstrap input only. The returned names contain no + secret values and are suitable for diagnostics or tests. + """ + registered: list[str] = [] + for name in REVIEW_CREDENTIAL_NAMES: + value = environment.get(name, "").strip() + if value: + register_credential(name, value) + registered.append(name) + return tuple(registered) + + +def build_review_orchestrator( + environment: Mapping[str, str] | None = None, + *, + max_agents: int = DEFAULT_REVIEW_AGENT_LIMIT, +) -> TaskOrchestrator: + """Build an enabled, cost-ranked orchestrator from discovered providers. + + A missing provider key is allowed so the sidecar can use an available + subset. Starting with no provider key or no discovered model fails closed + rather than silently falling back to a mock or fabricated model. + """ + if type(max_agents) is not int or max_agents < 1: + raise ValueError("max_agents must be a positive integer") + source_environment = os.environ if environment is None else environment + registered = register_review_credentials(source_environment) + if not registered: + raise NotConfigured("review gateway requires at least one provider credential") + + discovered, errors = discover_all_models() + if not discovered: + providers = ", ".join(sorted({error.provider_name for error in errors})) + detail = f"; failed providers: {providers}" if providers else "" + raise NotConfigured(f"review gateway discovered no provider models{detail}") + + price_book = PriceBook(InMemoryConfigStore()) + refresh_price_book(discovered, price_book) + selected = select_top_n_cheapest_discovered_agents(discovered, price_book, max_agents) + if not selected: + raise NotConfigured("review gateway selected no provider models") + + agents = [ + replace( + agent_from_discovered(model, priority=index), + disabled=False, + tags=("review", "coding", "security", "reasoning", "verification"), + ) + for index, model in enumerate(selected) + ] + return TaskOrchestrator( + agents, + client=ModelClient(max_output_tokens=32768), + ) + + +def _build_parser() -> argparse.ArgumentParser: + """Create the sidecar CLI parser with loopback-safe defaults.""" + parser = argparse.ArgumentParser(description="Run the contextual review gateway sidecar.") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=18080) + parser.add_argument("--max-agents", type=int, default=DEFAULT_REVIEW_AGENT_LIMIT) + parser.add_argument( + "--auth-token", + default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", ""), + help="Bearer token for the local review gateway (prefer the environment bootstrap).", + ) + return parser + + +def main() -> None: + """Discover providers and serve the authenticated OpenAI-compatible sidecar.""" + args = _build_parser().parse_args() + if not args.auth_token: + raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN or --auth-token is required") + orchestrator = build_review_orchestrator(max_agents=args.max_agents) + serve( + orchestrator, + host=args.host, + port=args.port, + security=SecurityConfig(auth_token=args.auth_token), + ) + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py new file mode 100644 index 000000000..4d152bf17 --- /dev/null +++ b/tests/test_review_gateway.py @@ -0,0 +1,126 @@ +"""Tests for the trusted CI review gateway bootstrap.""" + +from __future__ import annotations + +import sys + +import pytest + +from contextual_orchestrator.credentials import ( + InMemoryCredentialBackend, + NotConfigured, + get_credential, + set_backend, +) +from contextual_orchestrator.model_discovery import DiscoveredModel +from contextual_orchestrator import review_gateway + + +@pytest.fixture(autouse=True) +def _fresh_backend(): + """Give every gateway test an isolated in-memory credential registry.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def _discovered(provider: str, model: str, credential: str, price: float) -> DiscoveredModel: + """Build one deterministic discovered chat candidate for the tests.""" + return DiscoveredModel( + provider_name=provider, + model_id=model, + credential_name=credential, + chat_base_url=f"https://{provider}.example/v1", + auth_scheme="Bearer", + prompt_price_per_1k=price, + completion_price_per_1k=price, + ) + + +def test_build_review_orchestrator_registers_all_provider_credentials(monkeypatch): + """The bootstrap registers every configured provider key before discovery.""" + discovered = [ + _discovered("openai", "gpt-review", "OPENAI_API_KEY", 0.01), + _discovered("openrouter", "router-review", "OPENROUTER_API_KEY", 0.02), + _discovered("nvidia_nim", "nim-review", "NVIDIA_NIM_API_KEY", 0.03), + ] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + environment = { + name: f"secret-{name.lower()}" + for name in review_gateway.REVIEW_CREDENTIAL_NAMES + } + + orchestrator = review_gateway.build_review_orchestrator(environment, max_agents=2) + + assert len(orchestrator.agents) == 2 + assert all(not agent.disabled for agent in orchestrator.agents) + assert all("review" in agent.tags for agent in orchestrator.agents) + assert all(get_credential(name) == environment[name] for name in review_gateway.REVIEW_CREDENTIAL_NAMES) + + +def test_build_review_orchestrator_fails_closed_without_credentials(): + """A sidecar cannot start without at least one registered provider key.""" + with pytest.raises(NotConfigured, match="provider credential"): + review_gateway.build_review_orchestrator({}) + + +def test_build_review_orchestrator_rejects_invalid_agent_limit(): + """The sidecar refuses an invalid routing-pool bound before using credentials.""" + with pytest.raises(ValueError, match="max_agents"): + review_gateway.build_review_orchestrator({}, max_agents=0) + + +def test_build_review_orchestrator_fails_closed_without_discovered_models(monkeypatch): + """A configured key without a usable model list cannot start the sidecar.""" + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: ([], [])) + with pytest.raises(NotConfigured, match="no provider models"): + review_gateway.build_review_orchestrator({"OPENAI_API_KEY": "openai-secret"}) + + +def test_build_review_orchestrator_fails_closed_when_selection_is_empty(monkeypatch): + """A discovery result must still produce at least one selected candidate.""" + monkeypatch.setattr( + review_gateway, + "discover_all_models", + lambda: ([_discovered("openai", "gpt-review", "OPENAI_API_KEY", 0.01)], []), + ) + monkeypatch.setattr(review_gateway, "select_top_n_cheapest_discovered_agents", lambda *args: []) + + with pytest.raises(NotConfigured, match="selected no provider models"): + review_gateway.build_review_orchestrator({"OPENAI_API_KEY": "openai-secret"}) + + +def test_main_starts_authenticated_gateway(monkeypatch): + """The CLI passes the generated local token into the protected server.""" + discovered = [_discovered("openai", "gpt-review", "OPENAI_API_KEY", 0.01)] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + captured: dict[str, object] = {} + + def fake_serve(orchestrator, **kwargs): + """Capture server startup without binding a test port.""" + captured["orchestrator"] = orchestrator + captured.update(kwargs) + + monkeypatch.setattr(review_gateway, "serve", fake_serve) + monkeypatch.setenv("OPENAI_API_KEY", "openai-secret") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "local-review-token") + monkeypatch.setattr(sys, "argv", ["review_gateway", "--port", "18181"]) + + review_gateway.main() + + assert captured["host"] == "127.0.0.1" + assert captured["port"] == 18181 + security = captured["security"] + assert security.auth_token == "local-review-token" + assert security.allow_public_bind is False + + +def test_main_requires_authentication(monkeypatch): + """The sidecar never starts an unauthenticated inference endpoint.""" + monkeypatch.delenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", raising=False) + monkeypatch.setattr(sys, "argv", ["review_gateway"]) + + with pytest.raises(SystemExit, match="CONTEXTUAL_ORCHESTRATOR_TOKEN"): + review_gateway.main() From 04761e412083e02ef6ccc0a888ceec4c0be71553 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:16:39 +0900 Subject: [PATCH 77/89] fix: preserve priced discovery and review routing --- contextual_orchestrator/model_discovery.py | 42 +++++++--------------- contextual_orchestrator/review_gateway.py | 8 ++++- tests/test_model_discovery.py | 13 +++++++ tests/test_review_gateway.py | 18 +++++++++- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index bd76100c0..57ad83e4f 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -20,7 +20,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from .batch_routing import cheapest_upstream from .credentials import get_credential from .orchestrator import ModelAgent @@ -275,43 +274,26 @@ def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook def select_cheapest_discovered_agent( discovered: list[DiscoveredModel], price_book: "PriceBook" ) -> DiscoveredModel | None: - """Pick the lowest-cost discovered model per the price book (auto-optimization). - - Reuses :func:`~contextual_orchestrator.batch_routing.cheapest_upstream`, the - existing cost-optimizing upstream selector. Call :func:`refresh_price_book` - first so discovered pricing is visible; an unpriced candidate costs ``0`` - under that selector's documented contract and is treated as free, not - unknown -- so a genuinely unpriced provider (e.g. Bytez, priced by - GPU-second rather than per token) will always look cheapest here. Fine for - "auto-pick something free to try," but callers doing real cost comparison - should refresh pricing for every candidate they care about first. - """ + """Pick the lowest-cost discovered model, with unknown prices as fallback.""" if not discovered: return None - candidates = [{"provider": model.provider_name, "model": model.model_id} for model in discovered] - winner = cheapest_upstream(candidates, price_book) - if winner is None: - return None - for model in discovered: - if model.provider_name == winner["provider"] and model.model_id == winner["model"]: - return model - return None # pragma: no cover - winner always comes from candidates + return min(discovered, key=lambda model: _discovered_sort_key(model, price_book)) def select_top_n_cheapest_discovered_agents( discovered: list[DiscoveredModel], price_book: "PriceBook", limit: int ) -> list[DiscoveredModel]: - """Return the ``limit`` lowest-cost discovered models, cheapest first. - - For bootstrapping a CI sidecar (or any first-boot pool) with more than one - enabled agent for failover, without hand-picking which discovered models to - trust. Same pricing contract as :func:`select_cheapest_discovered_agent`. - """ + """Return the ``limit`` cheapest discovered models, with unknown prices last.""" if limit <= 0 or not discovered: return [] - def _cost(model: DiscoveredModel) -> float: - cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) - return cost + return sorted(discovered, key=lambda model: _discovered_sort_key(model, price_book))[:limit] + - return sorted(discovered, key=_cost)[:limit] +def _discovered_sort_key(model: DiscoveredModel, price_book: "PriceBook") -> tuple[int, float, str, str]: + """Sort priced models first and keep unknown-price fallback ordering stable.""" + entry = price_book.get_price(model.provider_name, model.model_id) + if entry is None: + return (1, 0.0, model.provider_name, model.model_id) + cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) + return (0, cost, model.provider_name, model.model_id) diff --git a/contextual_orchestrator/review_gateway.py b/contextual_orchestrator/review_gateway.py index 7de5a7b28..7a493dbc2 100644 --- a/contextual_orchestrator/review_gateway.py +++ b/contextual_orchestrator/review_gateway.py @@ -85,7 +85,13 @@ def build_review_orchestrator( replace( agent_from_discovered(model, priority=index), disabled=False, - tags=("review", "coding", "security", "reasoning", "verification"), + # The discovery catalog does not advertise reasoning, coding, or + # verification support. ``review`` is the gateway purpose, not a + # fabricated provider capability. + tags=("review",), + # TaskOrchestrator ranks larger priorities first. Discovery is + # already cheapest-first, so preserve that order for routing. + priority=-index, ) for index, model in enumerate(selected) ] diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index fd69e6b3e..d5a7e4aa7 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -255,6 +255,19 @@ def test_select_top_n_cheapest_discovered_agents_orders_by_cost() -> None: assert top_two == [cheapest, middle] +def test_unknown_price_is_fallback_after_priced_models() -> None: + """Discovery never treats an unknown provider price as a free winner.""" + from contextual_orchestrator.cost_ledger import PriceEntry + + price_book = PriceBook(InMemoryConfigStore()) + priced = DiscoveredModel("openai", "priced_model", "OPENAI_API_KEY", "https://api.openai.com/v1", "Bearer") + unpriced = DiscoveredModel("bytez", "unpriced_model", "BYTEZ_API_KEY", "https://api.bytez.com/models/v2/openai/v1", "Key") + price_book.set_price(PriceEntry("openai", "priced_model", 0.01, 0.01)) + + assert select_cheapest_discovered_agent([unpriced, priced], price_book) is priced + assert select_top_n_cheapest_discovered_agents([unpriced, priced], price_book, 1) == [priced] + + def test_select_top_n_cheapest_discovered_agents_zero_limit_returns_empty() -> None: price_book = PriceBook(InMemoryConfigStore()) model = DiscoveredModel("openai", "a", "OPENAI_API_KEY", "https://api.openai.com/v1", "Bearer") diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py index 4d152bf17..1328d06ba 100644 --- a/tests/test_review_gateway.py +++ b/tests/test_review_gateway.py @@ -56,10 +56,26 @@ def test_build_review_orchestrator_registers_all_provider_credentials(monkeypatc assert len(orchestrator.agents) == 2 assert all(not agent.disabled for agent in orchestrator.agents) - assert all("review" in agent.tags for agent in orchestrator.agents) + assert all(agent.tags == ("review",) for agent in orchestrator.agents) assert all(get_credential(name) == environment[name] for name in review_gateway.REVIEW_CREDENTIAL_NAMES) +def test_build_review_orchestrator_routes_to_cheapest_selected_agent(monkeypatch): + """Cost-ranked discovery remains the routing order after agent construction.""" + discovered = [ + _discovered("openai", "expensive_review", "OPENAI_API_KEY", 2.0), + _discovered("openrouter", "cheap_review", "OPENROUTER_API_KEY", 0.01), + ] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + + orchestrator = review_gateway.build_review_orchestrator( + {"OPENAI_API_KEY": "openai-secret", "OPENROUTER_API_KEY": "router-secret"}, + max_agents=2, + ) + + assert orchestrator._select_agent("review this change", "worker").model == "cheap_review" + + def test_build_review_orchestrator_fails_closed_without_credentials(): """A sidecar cannot start without at least one registered provider key.""" with pytest.raises(NotConfigured, match="provider credential"): From 3a0e9f49933a646a39deeb25c32e812f1ab3a4c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:40:52 +0900 Subject: [PATCH 78/89] fix: isolate review gateway chat candidates --- contextual_orchestrator/chat_capability.py | 75 ++++++++++++++++++++++ contextual_orchestrator/review_gateway.py | 8 ++- tests/test_review_gateway.py | 47 ++++++++++++++ 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 contextual_orchestrator/chat_capability.py diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py new file mode 100644 index 000000000..6d6520a22 --- /dev/null +++ b/contextual_orchestrator/chat_capability.py @@ -0,0 +1,75 @@ +"""Classify model identifiers that cannot serve ordinary chat-agent roles. + +Provider catalogs mix chat, embedding, reranking, transcription, moderation, +image, realtime, and speech-only models. This negative classifier rejects only +families that clearly advertise a non-chat endpoint; it never infers reasoning, +coding, vision, or verification support from a model name. +""" + +from __future__ import annotations + +import re + +_MODEL_TOKEN_RE = re.compile(r"[a-z0-9]+") +_TRANSPORT_INCOMPATIBLE_EXACT_TOKENS = frozenset( + { + "bge", + "clip", + "dall", + "e5", + "embed", + "embedding", + "embeddings", + "image", + "images", + "moderation", + "realtime", + "rerank", + "reranker", + "siglip", + "sora", + "speech", + "transcribe", + "transcription", + "tts", + "whisper", + } +) +_TRANSPORT_INCOMPATIBLE_PREFIXES = ("embed", "moderat", "rerank", "transcrib") + + +def _model_tokens(model_id: str) -> tuple[str, ...]: + """Normalize one provider model identifier into lowercase tokens.""" + if not isinstance(model_id, str): + return () + return tuple(_MODEL_TOKEN_RE.findall(model_id.casefold())) + + +def _is_transport_compatible_tokens(tokens: tuple[str, ...]) -> bool: + """Reject already-tokenized identifiers for endpoint-only model families.""" + if not tokens: + return False + return not any( + token in _TRANSPORT_INCOMPATIBLE_EXACT_TOKENS + or token.startswith(_TRANSPORT_INCOMPATIBLE_PREFIXES) + for token in tokens + ) + + +def is_chat_compatible_model_id(model_id: str) -> bool: + """Return whether an identifier can use the ordinary chat transport.""" + return _is_transport_compatible_tokens(_model_tokens(model_id)) + + +def is_general_chat_agent_model_id(model_id: str) -> bool: + """Return whether a chat model may enter ordinary review-agent roles.""" + tokens = _model_tokens(model_id) + if not _is_transport_compatible_tokens(tokens): + return False + return not any( + token == "safety" + or token == "guard" + or token == "shieldgemma" + or token.startswith("nemoguard") + for token in tokens + ) diff --git a/contextual_orchestrator/review_gateway.py b/contextual_orchestrator/review_gateway.py index 7a493dbc2..e40feaa6b 100644 --- a/contextual_orchestrator/review_gateway.py +++ b/contextual_orchestrator/review_gateway.py @@ -16,6 +16,7 @@ from .credentials import NotConfigured, register_credential from .cost_ledger import PriceBook +from .chat_capability import is_general_chat_agent_model_id from .kv_config import InMemoryConfigStore from .model_discovery import ( agent_from_discovered, @@ -75,9 +76,12 @@ def build_review_orchestrator( detail = f"; failed providers: {providers}" if providers else "" raise NotConfigured(f"review gateway discovered no provider models{detail}") + chat_discovered = [model for model in discovered if is_general_chat_agent_model_id(model.model_id)] + if not chat_discovered: + raise NotConfigured("review gateway discovered no general chat models") price_book = PriceBook(InMemoryConfigStore()) - refresh_price_book(discovered, price_book) - selected = select_top_n_cheapest_discovered_agents(discovered, price_book, max_agents) + refresh_price_book(chat_discovered, price_book) + selected = select_top_n_cheapest_discovered_agents(chat_discovered, price_book, max_agents) if not selected: raise NotConfigured("review gateway selected no provider models") diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py index 1328d06ba..bec13dc44 100644 --- a/tests/test_review_gateway.py +++ b/tests/test_review_gateway.py @@ -12,6 +12,10 @@ get_credential, set_backend, ) +from contextual_orchestrator.chat_capability import ( + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) from contextual_orchestrator.model_discovery import DiscoveredModel from contextual_orchestrator import review_gateway @@ -76,6 +80,49 @@ def test_build_review_orchestrator_routes_to_cheapest_selected_agent(monkeypatch assert orchestrator._select_agent("review this change", "worker").model == "cheap_review" +def test_build_review_orchestrator_excludes_endpoint_only_models(monkeypatch): + """Embedding and image catalog rows never enter the review-agent pool.""" + discovered = [ + _discovered("openai", "text-embedding-3-large", "OPENAI_API_KEY", 0.001), + _discovered("openai", "gpt-review", "OPENAI_API_KEY", 2.0), + ] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + + orchestrator = review_gateway.build_review_orchestrator( + {"OPENAI_API_KEY": "openai-secret"}, max_agents=1 + ) + + assert [agent.model for agent in orchestrator.agents] == ["gpt-review"] + + +def test_build_review_orchestrator_fails_closed_without_general_chat_models(monkeypatch): + """A catalog containing only endpoint-specific models cannot start review.""" + discovered = [_discovered("openai", "text-embedding-3-large", "OPENAI_API_KEY", 0.001)] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + + with pytest.raises(NotConfigured, match="general chat models"): + review_gateway.build_review_orchestrator({"OPENAI_API_KEY": "openai-secret"}) + + +@pytest.mark.parametrize( + ("model_id", "chat_compatible", "general_agent"), + [ + ("text-embedding-3-large", False, False), + ("gpt-image-1", False, False), + ("gpt-review", True, True), + ("nvidia/llama-3.1-nemoguard-8b-content-safety", True, False), + (None, False, False), + ("", False, False), + ], +) +def test_chat_capability_boundary_is_explicit( + model_id, chat_compatible: bool, general_agent: bool +) -> None: + """The gateway rejects endpoint-only and specialized role identifiers.""" + assert is_chat_compatible_model_id(model_id) is chat_compatible + assert is_general_chat_agent_model_id(model_id) is general_agent + + def test_build_review_orchestrator_fails_closed_without_credentials(): """A sidecar cannot start without at least one registered provider key.""" with pytest.raises(NotConfigured, match="provider credential"): From 84a8999a38009a94c532f6e6c87e1390eebd63f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:44:32 +0900 Subject: [PATCH 79/89] fix: keep partial discovery prices unknown --- contextual_orchestrator/model_discovery.py | 15 ++++++++------- tests/test_model_discovery.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 57ad83e4f..91e1ffebc 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -247,23 +247,24 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> def refresh_price_book(discovered: list[DiscoveredModel], price_book: "PriceBook") -> int: """Write every discovered model's known pricing into the price book. - Returns the number of price rows written. A model without provider-reported - pricing is skipped rather than defaulted to 0 -- an unpriced model already - costs 0 under ``PriceBook.compute_cost``'s "explicit, not silently expensive" - contract, so writing a fabricated 0 row here would just hide that signal. + Returns the number of price rows written. A model without complete + provider-reported pricing is skipped rather than defaulted to 0. An + unpriced model already costs 0 under ``PriceBook.compute_cost``'s + "explicit, not silently expensive" contract, so writing a fabricated + partial row would just hide that signal. """ from .cost_ledger import PriceEntry written = 0 for model in discovered: - if model.prompt_price_per_1k is None and model.completion_price_per_1k is None: + if model.prompt_price_per_1k is None or model.completion_price_per_1k is None: continue price_book.set_price( PriceEntry( provider_name=model.provider_name, model_name=model.model_id, - prompt_price_per_1k=model.prompt_price_per_1k or 0.0, - completion_price_per_1k=model.completion_price_per_1k or 0.0, + prompt_price_per_1k=model.prompt_price_per_1k, + completion_price_per_1k=model.completion_price_per_1k, currency_code=model.currency_code, ) ) diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index d5a7e4aa7..c0a1e6f5b 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -210,6 +210,23 @@ def test_refresh_price_book_writes_known_pricing_and_skips_unpriced() -> None: assert price_book.get_price("bytez", "some/model") is None +def test_refresh_price_book_skips_partial_pricing() -> None: + """A missing prompt or completion price remains an unknown-price fallback.""" + price_book = PriceBook(InMemoryConfigStore()) + partial = DiscoveredModel( + provider_name="openai", + model_id="partial-price-model", + credential_name="OPENAI_API_KEY", + chat_base_url="https://api.openai.com/v1", + auth_scheme="Bearer", + prompt_price_per_1k=0.01, + completion_price_per_1k=None, + ) + + assert refresh_price_book([partial], price_book) == 0 + assert price_book.get_price("openai", "partial-price-model") is None + + def test_select_cheapest_discovered_agent_picks_the_lower_priced_candidate() -> None: from contextual_orchestrator.cost_ledger import PriceEntry From 9041e78f54d0be9625b9cb1b6f5bf0b0154d9266 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:56:27 +0900 Subject: [PATCH 80/89] fix: resolve review gateway auth through KV --- contextual_orchestrator/review_gateway.py | 22 +++++++++++++++------- tests/test_review_gateway.py | 1 + 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/contextual_orchestrator/review_gateway.py b/contextual_orchestrator/review_gateway.py index e40feaa6b..83edcdb4e 100644 --- a/contextual_orchestrator/review_gateway.py +++ b/contextual_orchestrator/review_gateway.py @@ -14,7 +14,7 @@ from dataclasses import replace from typing import Mapping -from .credentials import NotConfigured, register_credential +from .credentials import NotConfigured, get_credential, register_credential from .cost_ledger import PriceBook from .chat_capability import is_general_chat_agent_model_id from .kv_config import InMemoryConfigStore @@ -35,10 +35,11 @@ "OPENAI_API_KEY", ) DEFAULT_REVIEW_AGENT_LIMIT = 12 +REVIEW_AUTH_CREDENTIAL_NAME = "CONTEXTUAL_ORCHESTRATOR_TOKEN" def register_review_credentials(environment: Mapping[str, str]) -> tuple[str, ...]: - """Register non-empty CI provider keys in the process-local KV. + """Register non-empty CI provider and gateway credentials in the KV. ``environment`` is bootstrap input only. The returned names contain no secret values and are suitable for diagnostics or tests. @@ -49,6 +50,10 @@ def register_review_credentials(environment: Mapping[str, str]) -> tuple[str, .. if value: register_credential(name, value) registered.append(name) + auth_value = environment.get(REVIEW_AUTH_CREDENTIAL_NAME, "").strip() + if auth_value: + register_credential(REVIEW_AUTH_CREDENTIAL_NAME, auth_value) + registered.append(REVIEW_AUTH_CREDENTIAL_NAME) return tuple(registered) @@ -113,23 +118,26 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--max-agents", type=int, default=DEFAULT_REVIEW_AGENT_LIMIT) parser.add_argument( "--auth-token", - default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", ""), - help="Bearer token for the local review gateway (prefer the environment bootstrap).", + default="", + help="Explicit local bearer token; otherwise resolve --auth-token-key from the KV.", ) + parser.add_argument("--auth-token-key", default=REVIEW_AUTH_CREDENTIAL_NAME) return parser def main() -> None: """Discover providers and serve the authenticated OpenAI-compatible sidecar.""" args = _build_parser().parse_args() - if not args.auth_token: - raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN or --auth-token is required") + register_review_credentials(os.environ) + auth_token = args.auth_token or get_credential(args.auth_token_key) + if not auth_token: + raise SystemExit(f"KV credential {args.auth_token_key!r} or --auth-token is required") orchestrator = build_review_orchestrator(max_agents=args.max_agents) serve( orchestrator, host=args.host, port=args.port, - security=SecurityConfig(auth_token=args.auth_token), + security=SecurityConfig(auth_token=auth_token), ) diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py index bec13dc44..00aa89d2e 100644 --- a/tests/test_review_gateway.py +++ b/tests/test_review_gateway.py @@ -178,6 +178,7 @@ def fake_serve(orchestrator, **kwargs): security = captured["security"] assert security.auth_token == "local-review-token" assert security.allow_public_bind is False + assert get_credential(review_gateway.REVIEW_AUTH_CREDENTIAL_NAME) == "local-review-token" def test_main_requires_authentication(monkeypatch): From 0071751782ae535721e71785c3037989d2d27b77 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:59:50 +0900 Subject: [PATCH 81/89] fix: keep auth outside provider bootstrap gate --- contextual_orchestrator/review_gateway.py | 2 +- tests/test_review_gateway.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/review_gateway.py b/contextual_orchestrator/review_gateway.py index 83edcdb4e..277b8e8ea 100644 --- a/contextual_orchestrator/review_gateway.py +++ b/contextual_orchestrator/review_gateway.py @@ -72,7 +72,7 @@ def build_review_orchestrator( raise ValueError("max_agents must be a positive integer") source_environment = os.environ if environment is None else environment registered = register_review_credentials(source_environment) - if not registered: + if not any(name in registered for name in REVIEW_CREDENTIAL_NAMES): raise NotConfigured("review gateway requires at least one provider credential") discovered, errors = discover_all_models() diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py index 00aa89d2e..c9ba8815a 100644 --- a/tests/test_review_gateway.py +++ b/tests/test_review_gateway.py @@ -129,6 +129,14 @@ def test_build_review_orchestrator_fails_closed_without_credentials(): review_gateway.build_review_orchestrator({}) +def test_build_review_orchestrator_does_not_count_auth_as_provider_credential(): + """The gateway auth token cannot satisfy the provider bootstrap gate.""" + with pytest.raises(NotConfigured, match="provider credential"): + review_gateway.build_review_orchestrator( + {review_gateway.REVIEW_AUTH_CREDENTIAL_NAME: "local-review-token"} + ) + + def test_build_review_orchestrator_rejects_invalid_agent_limit(): """The sidecar refuses an invalid routing-pool bound before using credentials.""" with pytest.raises(ValueError, match="max_agents"): From 6b603efeb9728d7c142f090153925948c0f1248f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 05:10:48 +0900 Subject: [PATCH 82/89] docs: ground provider-diverse routing in research --- .../provider-diverse-discovery-routing.md | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/doctoring/provider-diverse-discovery-routing.md diff --git a/docs/doctoring/provider-diverse-discovery-routing.md b/docs/doctoring/provider-diverse-discovery-routing.md new file mode 100644 index 000000000..541bf6f93 --- /dev/null +++ b/docs/doctoring/provider-diverse-discovery-routing.md @@ -0,0 +1,49 @@ +--- +title: "Provider-diverse discovery and cost-honest failover routing" +status: "implemented" +date: "2026-08-21" +scope: "PR #770" +--- + +# Provider-diverse discovery and cost-honest failover routing + +## Decision + +PR #770 makes model discovery fail closed for invalid or unpriced catalog +rows, retains candidates whose prices are unknown only as an explicit +fallback, and selects a provider-diverse bootstrap pool before ordinary +chat routing. The selector is deterministic eligibility and cost accounting; +it is not a learned answer-quality judge and does not claim to reproduce the +learning systems in the cited work. + +## Research-to-code mapping + +| Implementation boundary | Evidence-informed reason | Acceptance evidence | +| --- | --- | --- | +| Reject malformed, negative, or non-finite price rows | A cost-aware router must not treat missing or invalid evidence as zero cost. | Discovery and persisted-price tests reject the row before selection. | +| Keep unknown-price candidates only as an explicit fallback | Cost optimization must remain honest when price evidence is incomplete. | Selection tests never rank an unknown price above a valid priced candidate. | +| Prefer distinct providers in the bootstrap pool | A gateway needs an upstream failover set rather than several aliases for one provider. | Provider-diversity tests assert the configured pool spans available providers. | +| Leave quality judgment to evaluation/review policy | Routing signals and answer-quality judgment have different failure modes. | Existing model-judge and fail-closed routing tests remain the quality boundary. | + +The routing papers and OA PDFs are already committed in the prerequisite +stack base under `docs/papers/` (`routellm-routing-2406.18665.pdf`, +`hybrid-llm-query-routing-2404.14618.pdf`, and +`frugalgpt-cost-2305.05176.pdf`). This doctoring record makes their relevance +to the exact discovery selector explicit instead of treating inherited files +as incidental documentation. + +## APA 7 references + +Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large +language models while reducing cost and improving performance*. arXiv. +https://arxiv.org/abs/2305.05176 + +Ding, D., Mallick, A., Wang, C., Sim, R., Mukherjee, S., Rühle, V., +Lakshmanan, L. V. S., & Awadallah, A. H. (2024). *Hybrid LLM: +Cost-efficient and quality-aware query routing*. International Conference on +Learning Representations. https://arxiv.org/abs/2404.14618 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., +Kadous, M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with +preference data*. arXiv. https://arxiv.org/abs/2406.18665 + From 172a8901f6fa6cad500917095e92878ed9ffcf04 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:52:19 +0900 Subject: [PATCH 83/89] test: cover model discovery rejection paths --- tests/test_model_discovery.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index c0a1e6f5b..52378b6c1 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -28,6 +28,7 @@ agent_id_for, discover_all_models, discover_provider_models, + ProviderDiscoveryError, refresh_price_book, select_cheapest_discovered_agent, select_top_n_cheapest_discovered_agents, @@ -86,6 +87,20 @@ def test_discover_provider_models_skips_when_credential_missing() -> None: assert discover_provider_models(OPENAI_SOURCE) == [] +def test_discovery_rejects_non_https_model_catalog_url() -> None: + """Never allow a model catalog source to reach a non-HTTPS URL opener.""" + register_credential("UNSAFE_CATALOG_KEY", "catalog-secret") + unsafe_source = ProviderModelSource( + provider_name="unsafe_catalog", + credential_name="UNSAFE_CATALOG_KEY", + list_url="file:///tmp/models.json", + chat_base_url="https://example.invalid/v1", + ) + + with pytest.raises(ProviderDiscoveryError, match="non-https"): + discover_provider_models(unsafe_source) + + def test_discover_openai_compatible_parses_models_and_pricing() -> None: register_credential("OPENROUTER_API_KEY", "sk-router") payload = { @@ -93,6 +108,8 @@ def test_discover_openai_compatible_parses_models_and_pricing() -> None: {"id": "meta/llama-3.3", "pricing": {"prompt": "0.0000006", "completion": "0.0000012"}}, {"id": "no-pricing-model"}, {"missing": "id-field"}, + ["not-a-model-row"], + {"id": "invalid-pricing-model", "pricing": {"prompt": "unknown", "completion": []}}, ] } seen_requests = [] @@ -106,11 +123,17 @@ def urlopen(request, timeout=None): assert seen_requests[0].get_header("Authorization") == "Bearer sk-router" assert seen_requests[0].full_url == "https://openrouter.ai/api/v1/models" - assert [m.model_id for m in discovered] == ["meta/llama-3.3", "no-pricing-model"] + assert [m.model_id for m in discovered] == [ + "meta/llama-3.3", + "no-pricing-model", + "invalid-pricing-model", + ] priced = discovered[0] assert priced.prompt_price_per_1k == pytest.approx(0.0006) assert priced.completion_price_per_1k == pytest.approx(0.0012) assert discovered[1].prompt_price_per_1k is None + assert discovered[2].prompt_price_per_1k is None + assert discovered[2].completion_price_per_1k is None def test_discover_bytez_parses_models_with_key_auth_scheme() -> None: @@ -119,6 +142,8 @@ def test_discover_bytez_parses_models_with_key_auth_scheme() -> None: "error": None, "output": [ {"modelId": "0-hero/Matter-0.1-Slim-7B-C", "task": "chat", "meterPrice": "0.0006 / sec"}, + "not-a-model-row", + {"modelId": 12345}, ], } seen_requests = [] From bfdcdd006365f6af2930ca4798806273e980bef6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:56:06 +0900 Subject: [PATCH 84/89] fix: reject gte embedding models from chat roles --- contextual_orchestrator/chat_capability.py | 1 + tests/test_chat_capability.py | 73 ++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tests/test_chat_capability.py diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index 6d6520a22..91dbaeae2 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -20,6 +20,7 @@ "embed", "embedding", "embeddings", + "gte", "image", "images", "moderation", diff --git a/tests/test_chat_capability.py b/tests/test_chat_capability.py new file mode 100644 index 000000000..0660d5ddf --- /dev/null +++ b/tests/test_chat_capability.py @@ -0,0 +1,73 @@ +"""Boundary tests for model-family chat and orchestration-role capability gates.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) + + +@pytest.mark.parametrize( + "model_id", + ( + "bge-small-en", + "clip-vit", + "dall-e-3", + "e5-large", + "embed-model", + "embedding-model", + "embeddings-v2", + "gte-base", + "image-model", + "images-v2", + "moderation-latest", + "realtime-preview", + "rerank-v3", + "reranker-large", + "siglip-base", + "sora-video", + "speech-model", + "transcribe-large", + "transcription-v2", + "tts-1", + "whisper-1", + ), +) +def test_endpoint_only_model_families_cannot_use_chat_transport(model_id: str) -> None: + """Reject every explicitly known endpoint-only model family.""" + assert is_chat_compatible_model_id(model_id) is False + assert is_general_chat_agent_model_id(model_id) is False + + +@pytest.mark.parametrize("model_id", ("embedder-v2", "moderator-v1", "reranking-v1", "transcriber-v1")) +def test_endpoint_prefixes_reject_unlisted_model_variants(model_id: str) -> None: + """Reject future-looking family variants without matching exact tokens.""" + assert is_chat_compatible_model_id(model_id) is False + + +def test_empty_and_non_string_identifiers_fail_closed() -> None: + """Unknown identifier shapes never become eligible by accident.""" + assert is_chat_compatible_model_id("") is False + assert is_chat_compatible_model_id(None) is False # type: ignore[arg-type] + assert is_general_chat_agent_model_id("") is False + + +@pytest.mark.parametrize("model_id", ("safety-classifier", "guard-model", "shieldgemma-2b", "nemo_guard_v2")) +def test_transport_compatible_policy_models_stay_out_of_general_agent_roles(model_id: str) -> None: + """Safety and guard models may use chat transport but are not thinkers.""" + assert is_chat_compatible_model_id(model_id) is True + assert is_general_chat_agent_model_id(model_id) is False + + +def test_normal_chat_identifier_remains_eligible() -> None: + """A normal provider model remains eligible for both ordinary gates.""" + assert is_chat_compatible_model_id("provider/gpt-5.5") is True + assert is_general_chat_agent_model_id("provider/gpt-5.5") is True From d3a27db0a69f09f245a19a189ec41d3aa2f6b2fc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 10:00:08 +0900 Subject: [PATCH 85/89] fix: keep foreign currency prices out of ranking --- contextual_orchestrator/model_discovery.py | 11 +++++++++-- tests/test_model_discovery.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 91e1ffebc..48425abc8 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -292,9 +292,16 @@ def select_top_n_cheapest_discovered_agents( def _discovered_sort_key(model: DiscoveredModel, price_book: "PriceBook") -> tuple[int, float, str, str]: - """Sort priced models first and keep unknown-price fallback ordering stable.""" + """Sort comparable prices first and keep unknown-currency models as fallback. + + The price book has no exchange-rate authority. A non-default currency is + therefore not comparable to the gateway's default currency and must never + win merely because its numeric amount is smaller. + """ entry = price_book.get_price(model.provider_name, model.model_id) if entry is None: return (1, 0.0, model.provider_name, model.model_id) - cost, _currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) + cost, currency = price_book.compute_cost(model.provider_name, model.model_id, 1000, 1000) + if currency != price_book.default_currency: + return (1, 0.0, model.provider_name, model.model_id) return (0, cost, model.provider_name, model.model_id) diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 52378b6c1..f60c114ca 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -310,6 +310,23 @@ def test_unknown_price_is_fallback_after_priced_models() -> None: assert select_top_n_cheapest_discovered_agents([unpriced, priced], price_book, 1) == [priced] +def test_foreign_currency_is_not_compared_as_default_currency() -> None: + """Treat an unconvertible currency as unknown instead of a cheap winner.""" + from contextual_orchestrator.cost_ledger import PriceEntry + + price_book = PriceBook(InMemoryConfigStore(), default_currency="USD") + usd_model = DiscoveredModel("openai", "usd_model", "OPENAI_API_KEY", "https://api.openai.com/v1", "Bearer") + eur_model = DiscoveredModel("european_provider", "eur_model", "EUROPEAN_API_KEY", "https://example.invalid/v1", "Bearer") + price_book.set_price(PriceEntry("openai", "usd_model", 0.01, 0.01, "USD")) + price_book.set_price(PriceEntry("european_provider", "eur_model", 0.0001, 0.0001, "EUR")) + + assert select_cheapest_discovered_agent([eur_model, usd_model], price_book) is usd_model + assert select_top_n_cheapest_discovered_agents([eur_model, usd_model], price_book, 2) == [ + usd_model, + eur_model, + ] + + def test_select_top_n_cheapest_discovered_agents_zero_limit_returns_empty() -> None: price_book = PriceBook(InMemoryConfigStore()) model = DiscoveredModel("openai", "a", "OPENAI_API_KEY", "https://api.openai.com/v1", "Bearer") From c649cdb876feef9a8cf3f392ea7002de7bf93e76 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 18:15:36 +0900 Subject: [PATCH 86/89] fix: preserve non-chat capability routing and price integrity --- README.md | 2 ++ contextual_orchestrator/cost_ledger.py | 2 ++ contextual_orchestrator/orchestrator.py | 6 +++++- docs/doctoring/provider-diverse-discovery-routing.md | 6 +++--- docs/kv-credentials.md | 10 ++++------ tests/test_discovery_bootstrap_selection.py | 12 ++++++++++++ tests/test_embeddings_model_pool_http_honesty.py | 2 +- 7 files changed, 29 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4e7576b9f..a46767084 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,8 @@ python tests/test_security_hardening.py python tests/test_chat_model_capability_isolation.py python tests/test_chat_transport_role_separation.py python tests/test_chat_capability_unknown_identifiers.py +python tests/test_discovery_bootstrap_selection.py +python tests/test_review_gateway.py python tests/test_chat_passthrough_capability_isolation.py python tests/test_tool_execution_fallback.py python tests/test_repository_security_metadata.py diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index d54fd187e..7991d6a20 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -169,6 +169,8 @@ def get_price(self, provider: str, model: str) -> Optional[PriceEntry]: if ( "prompt_price_per_1k" not in raw or "completion_price_per_1k" not in raw + or isinstance(raw["prompt_price_per_1k"], bool) + or isinstance(raw["completion_price_per_1k"], bool) ): return None prompt_price = float(raw["prompt_price_per_1k"]) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 15d56abe8..affc19ea9 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -2801,7 +2801,11 @@ def select_capability_agent(self, capability: str) -> ModelAgent: raise ValueError("capability must be a non-empty string") ranked = [ agent - for agent in self._ranked_agents("", capability) + for agent in sorted( + self.agents, + key=lambda candidate: self._score_agent(candidate, capability, ""), + reverse=True, + ) if not agent.disabled and capability in agent.tags and capability not in agent.provider_exclusions diff --git a/docs/doctoring/provider-diverse-discovery-routing.md b/docs/doctoring/provider-diverse-discovery-routing.md index 9eb03e0ef..bbb62d5cf 100644 --- a/docs/doctoring/provider-diverse-discovery-routing.md +++ b/docs/doctoring/provider-diverse-discovery-routing.md @@ -9,9 +9,9 @@ scope: "PR #770" ## Decision -PR #770 makes model discovery fail closed for invalid or unpriced catalog -rows, retains candidates whose prices are unknown only as an explicit -fallback, and selects a provider-diverse bootstrap pool before ordinary +PR #770 makes the pricing ledger and direct price ranking fail closed for +invalid or unpriced catalog rows, retains those candidates only as an explicit +unknown-price fallback, and selects a provider-diverse bootstrap pool before ordinary chat routing. The selector is deterministic eligibility and cost accounting; it is not a learned answer-quality judge and does not claim to reproduce the learning systems in the cited work. diff --git a/docs/kv-credentials.md b/docs/kv-credentials.md index d63113c1f..1b40fbafe 100644 --- a/docs/kv-credentials.md +++ b/docs/kv-credentials.md @@ -234,12 +234,10 @@ uses (`TaskOrchestrator.sync_discovered_agents`, an idempotent upsert of added **disabled**, so a newly found model never starts serving traffic before an operator opts it in. -Cost-based auto-optimization reuses the existing price-table selector rather -than adding a new one: `model_discovery.refresh_price_book` writes any -provider-reported per-token pricing into `PriceBook`, and -`model_discovery.select_cheapest_discovered_agent` calls -`batch_routing.cheapest_upstream` to pick the lowest-cost candidate among -discovered models for a representative request shape. +Cost-based auto-optimization uses `model_discovery.refresh_price_book` to write +provider-reported per-token pricing into `PriceBook`. Discovery selectors rank +complete, valid price evidence first and keep unknown-priced candidates as a +deterministic fallback. ## Gateway direction diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py index e12eeb0b6..ae34c4910 100644 --- a/tests/test_discovery_bootstrap_selection.py +++ b/tests/test_discovery_bootstrap_selection.py @@ -118,6 +118,18 @@ def test_persisted_price_row_missing_one_component_remains_unknown() -> None: assert price_book.get_price("partial_vendor", "partial-model") is None +def test_persisted_boolean_price_row_remains_unknown() -> None: + """Do not coerce corrupt KV booleans into zero-cost price evidence.""" + store = InMemoryConfigStore() + store.set( + "llm_price_entries", + "broken_vendor:broken-model", + {"prompt_price_per_1k": False, "completion_price_per_1k": False}, + ) + + assert PriceBook(store).get_price("broken_vendor", "broken-model") is None + + def test_invalid_catalog_prices_are_unknown_not_trusted_cost_evidence() -> None: """Reject negative, non-finite, and boolean provider price values.""" assert model_discovery._price_per_1k("-0.000001") is None diff --git a/tests/test_embeddings_model_pool_http_honesty.py b/tests/test_embeddings_model_pool_http_honesty.py index e01a5e9ca..7769a78f0 100644 --- a/tests/test_embeddings_model_pool_http_honesty.py +++ b/tests/test_embeddings_model_pool_http_honesty.py @@ -62,7 +62,7 @@ def _server_without_embedding(): def test_select_capability_agent_normalizes_and_rejects_empty_capability() -> None: """Capability selection normalizes names and rejects an empty capability.""" orchestrator = TaskOrchestrator( - [ModelAgent("embedding_agent", "mock-planner", tags=("embedding",))] + [ModelAgent("embedding_agent", "text-embedding-3-large", tags=("embedding",))] ) assert orchestrator.select_capability_agent(" EMBEDDING ").id == "embedding_agent" try: From ee216d0e4588cd23d4d7bcb93e6864e4db4a4461 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 18:29:47 +0900 Subject: [PATCH 87/89] fix: keep one canonical price parser --- contextual_orchestrator/cost_ledger.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index 05acd0c26..d3efb462b 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -203,27 +203,6 @@ def _parse_price_entry( completion_price = _decimal_safe_price(raw["completion_price_per_1k"]) if prompt_price is None or completion_price is None: return None - if not isinstance(raw, dict): - return None - try: - if ( - "prompt_price_per_1k" not in raw - or "completion_price_per_1k" not in raw - or isinstance(raw["prompt_price_per_1k"], bool) - or isinstance(raw["completion_price_per_1k"], bool) - ): - return None - prompt_price = float(raw["prompt_price_per_1k"]) - completion_price = float(raw["completion_price_per_1k"]) - except (OverflowError, TypeError, ValueError): - return None - if ( - not math.isfinite(prompt_price) - or not math.isfinite(completion_price) - or prompt_price < 0 - or completion_price < 0 - ): - return None return PriceEntry( provider_name=raw.get("provider_name", provider), model_name=raw.get("model_name", model), From 13bc93985d7d2b9800a05e6a7dc6059ab0b59845 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 19:22:19 +0900 Subject: [PATCH 88/89] docs: list chat capability check --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 03f58c661..bb62b2a47 100644 --- a/README.md +++ b/README.md @@ -318,6 +318,7 @@ python tests/test_chat_transport_role_separation.py python tests/test_chat_capability_unknown_identifiers.py python tests/test_chat_passthrough_capability_isolation.py python tests/test_discovery_bootstrap_selection.py +python tests/test_chat_capability.py python tests/test_review_gateway.py python tests/test_provider_bootstrap.py python tests/test_provider_bootstrap_secret_normalization.py From 6acd0fb5dd1ec9dc5680f28b2c84941dbd45a864 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 19:30:47 +0900 Subject: [PATCH 89/89] fix: validate review gateway agent limit at CLI --- contextual_orchestrator/review_gateway.py | 5 ++++- tests/test_review_gateway.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/review_gateway.py b/contextual_orchestrator/review_gateway.py index f806129e8..ef2e6438c 100644 --- a/contextual_orchestrator/review_gateway.py +++ b/contextual_orchestrator/review_gateway.py @@ -127,7 +127,10 @@ def _build_parser() -> argparse.ArgumentParser: def main() -> None: """Discover providers and serve the authenticated OpenAI-compatible sidecar.""" - args = _build_parser().parse_args() + parser = _build_parser() + args = parser.parse_args() + if args.max_agents < 1: + parser.error("--max-agents must be a positive integer") register_review_credentials(os.environ) auth_token = args.auth_token or get_credential(args.auth_token_key) if not auth_token: diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py index fb4674015..995fb8f9e 100644 --- a/tests/test_review_gateway.py +++ b/tests/test_review_gateway.py @@ -216,3 +216,12 @@ def test_main_requires_authentication(monkeypatch): with pytest.raises(SystemExit, match="CONTEXTUAL_ORCHESTRATOR_TOKEN"): review_gateway.main() + + +def test_main_rejects_invalid_agent_limit_without_traceback(monkeypatch, capsys): + """Argparse reports an invalid routing-pool bound as a CLI usage error.""" + monkeypatch.setattr(sys, "argv", ["review_gateway", "--max-agents", "0"]) + with pytest.raises(SystemExit) as exc_info: + review_gateway.main() + assert exc_info.value.code == 2 + assert "--max-agents must be a positive integer" in capsys.readouterr().err