From 4093c4b9c59483e1f39df5ca4ef7e769b44ebb86 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:57:22 +0000 Subject: [PATCH 01/13] feat(discovery): record parallel tool-call capability and exclude single-tool models from orchestrator/free - Add supports_parallel_tool_calls to DiscoveredModel (True/False/None). - Populate from provider supported_parameters listing parallel_tool_calls. - Add probe_discovered_model_tool_call_capability for live 400 evidence. - Wire field through is_general_chat_candidate, is_discovered_chat_candidate, is_routable_discovered_model, general_free_serving_candidates, agent_from_discovered, serving_tags_for_discovered, _is_general_chat_agent, and _is_general_free_agent. - Emit tool_call:multi/tool_call:single tags and reject tool_call:single agents from the general free pool. - Add tests and ADR 0039; update product-technical-gap-baseline.md. Fixes ContextualWisdomLab/contextual-orchestrator#940. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- contextual_orchestrator/chat_capability.py | 8 + contextual_orchestrator/model_discovery.py | 164 +++++++++++++- contextual_orchestrator/orchestrator.py | 33 ++- contextual_orchestrator/provider_bootstrap.py | 7 + .../0039-parallel-tool-call-capability.md | 93 ++++++++ docs/product-technical-gap-baseline.md | 36 ++++ tests/test_chat_capability.py | 11 + tests/test_model_discovery.py | 203 ++++++++++++++++++ tests/test_provider_bootstrap.py | 53 +++++ 9 files changed, 597 insertions(+), 11 deletions(-) create mode 100644 docs/planning/adrs/0039-parallel-tool-call-capability.md diff --git a/contextual_orchestrator/chat_capability.py b/contextual_orchestrator/chat_capability.py index f10c1ba63..b9825f10d 100644 --- a/contextual_orchestrator/chat_capability.py +++ b/contextual_orchestrator/chat_capability.py @@ -127,13 +127,21 @@ def is_general_chat_candidate( *, capabilities: Iterable[str] = (), output_modalities: Iterable[str] = (), + supports_parallel_tool_calls: bool | None = None, ) -> bool: """Apply explicit catalog evidence before falling back to the model name. A generic model identifier cannot identify a media-only endpoint. When a provider supplies capability or output-modality metadata, that metadata is therefore authoritative; absent metadata keeps the legacy name heuristic. + + General chat agents may receive multi-tool-call requests, so a model whose + catalog or probe evidence shows it only supports one tool call at a time is + not a general chat candidate. ``None`` means no evidence either way and + keeps the existing eligibility decision. """ + if supports_parallel_tool_calls is False: + return False outputs = { value.strip().casefold() for value in output_modalities diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index b95d01758..bac8b2f13 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -62,6 +62,132 @@ # safe to send on every request, authenticated or not. _HTTP_USER_AGENT = "contextual-orchestrator/0.2.0 (+https://github.com/ContextualWisdomLab/contextual-orchestrator)" _CAPABILITY_NAMES = {"embeddings": "embedding"} + + +def _parallel_tool_call_evidence(supported_parameters: list[Any]) -> bool | None: + """Return the strongest tool-call parallelism signal in a parameter list. + + Provider ``supported_parameters`` entries (e.g. from OpenRouter or a + LiteLLM-model-info proxy) name request fields the model accepts. A literal + ``"parallel_tool_calls"`` parameter is direct evidence the model can receive + multiple tool-call requests at once. A ``"tools"`` parameter alone tells us + only that some form of tool calling is accepted, not whether multiple calls + may be requested simultaneously; without an explicit ``parallel_tool_calls`` + signal we stay honest and report ``None`` rather than guessing ``False``. + """ + if not isinstance(supported_parameters, list): + return None + params = { + value.strip().casefold() + for value in supported_parameters + if isinstance(value, str) and value.strip() + } + if "parallel_tool_calls" in params: + return True + return None + + +def _tool_call_parallelism_from_error(error_payload: Any) -> bool | None: + """Return ``False`` when a provider's 400 clearly rejects multi-tool calls. + + The only negative signal this function trusts is a provider error whose + message explicitly says the model accepts only one tool call at a time, or + that ``parallel_tool_calls`` is not supported. Ambiguous 400s (malformed + payload, auth, rate limits, etc.) return ``None`` so the pool stays open + rather than excluding a model on a misunderstood error. + """ + if isinstance(error_payload, dict): + error = error_payload.get("error", {}) + if not isinstance(error, dict): + error = error_payload + message = str(error.get("message", "")) + if not message and isinstance(error_payload.get("message"), str): + message = error_payload["message"] + elif isinstance(error_payload, str): + message = error_payload + else: + return None + text = message.casefold() + if "single tool" in text or "one tool" in text or "parallel_tool_calls" in text: + return False + return None + + +def probe_discovered_model_tool_call_capability( + discovered: DiscoveredModel, + *, + timeout: float = 30.0, +) -> bool | None: + """Probe whether a discovered model accepts multi-tool-call requests. + + Sends a minimal ``/chat/completions`` request with ``parallel_tool_calls: true`` + and two tool definitions, using the provider credential registered in the KV. + A successful response means the model accepted the multi-tool shape + (``True``). A 400 whose error text clearly says the model only supports a + single tool call means it does not (``False``). Any network, auth, or + ambiguous error returns ``None`` so the pool stays open rather than + excluding a model on a flaky probe. + + This is real runtime evidence, not a model-name heuristic. It is deliberately + separate from :func:`discover_all_models` so callers decide when the extra + latency and token cost are justified. + """ + api_key = get_credential(discovered.credential_name) + if not api_key: + return None + url = discovered.chat_base_url.rstrip("/") + "/chat/completions" + if not url.startswith("https://"): + return None + payload = { + "model": discovered.model_id, + "messages": [{"role": "user", "content": "Call both functions."}], + "tools": [ + { + "type": "function", + "function": { + "name": "probe_a", + "description": "Probe function A", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "probe_b", + "description": "Probe function B", + "parameters": {"type": "object", "properties": {}}, + }, + }, + ], + "parallel_tool_calls": True, + "max_tokens": 1, + "temperature": 0.0, + "stream": False, + } + headers = { + "content-type": "application/json", + "user-agent": _HTTP_USER_AGENT, + "authorization": format_authorization_header(discovered.auth_scheme, api_key), + } + data = json.dumps(payload, separators=(",", ":")).encode("utf-8") + request = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - scoped provider probe + response.read() + except urllib.error.HTTPError as exc: + if exc.code != 400: + return None + body = exc.read().decode("utf-8", errors="replace") + try: + error_payload = json.loads(body) + except json.JSONDecodeError: + error_payload = {"message": body} + return _tool_call_parallelism_from_error(error_payload) + except (urllib.error.URLError, OSError, TimeoutError, ValueError): + return None + return True + + _MODELS_DEV_URL = "https://models.dev/api.json" # Small bounded retry budget for the one shared, unauthenticated, third-party # Models.dev fetch that every ``models_dev_provider_id``-joined source's @@ -285,6 +411,7 @@ class DiscoveredModel: zdr_capable: bool = False evidence_only: bool = False spend_admitted: bool = True + supports_parallel_tool_calls: bool | None = None class ProviderDiscoveryError(RuntimeError): @@ -941,6 +1068,7 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ if isinstance(row.get("supported_parameters"), list) else [] ) + parallel_tool_calls = _parallel_tool_call_evidence(supported_parameters) raw_inputs = architecture.get("input_modalities") raw_outputs = architecture.get("output_modalities") inputs = tuple(value for value in raw_inputs if isinstance(value, str)) if isinstance(raw_inputs, list) else () @@ -1029,6 +1157,7 @@ def _parse_openai_compatible(payload: Any, source: ProviderModelSource) -> list[ else None ), privacy_policy_urls=_privacy_policy_urls(source, row), + supports_parallel_tool_calls=parallel_tool_calls, ) ) return _deduplicate_discovered_models(discovered) @@ -1437,6 +1566,7 @@ def is_discovered_chat_candidate(discovered: DiscoveredModel) -> bool: discovered.model_id, capabilities=discovered.capabilities, output_modalities=discovered.output_modalities, + supports_parallel_tool_calls=discovered.supports_parallel_tool_calls, ) @@ -1461,9 +1591,7 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> if not any( capability not in {"chat", "response_format"} for capability in discovered.capabilities - ) and not ( - is_general_chat_agent_model_id(discovered.model_id) - ): + ) and not is_discovered_chat_candidate(discovered): raise ValueError("model is not eligible for a general chat agent") return ModelAgent( id=agent_id_for(discovered), @@ -1481,6 +1609,13 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> *(f"capability:{value}" for value in discovered.capabilities), *(f"input:{value}" for value in discovered.input_modalities), *(f"output:{value}" for value in discovered.output_modalities), + *( + ("tool_call:multi",) + if discovered.supports_parallel_tool_calls is True + else ("tool_call:single",) + if discovered.supports_parallel_tool_calls is False + else () + ), ), priority=priority, disabled=True, @@ -1523,6 +1658,16 @@ def _requires_non_text_input(discovered: DiscoveredModel) -> bool: return requires_non_text_input(discovered.input_modalities) +def _requires_single_tool_call(discovered: DiscoveredModel) -> bool: + """Return whether catalog/probe evidence shows this model only supports one tool call at a time. + + The general blind serving pool may send multi-tool-call requests, so a model + whose evidence says it cannot handle ``parallel_tool_calls`` is disqualified + from that pool. ``None`` (no evidence) never triggers exclusion. + """ + return discovered.supports_parallel_tool_calls is False + + def free_discovered_models(discovered: list[DiscoveredModel]) -> list[DiscoveredModel]: """Return the complete zero-cost model inventory (price evidence only). @@ -1603,10 +1748,15 @@ def general_free_serving_candidates( an extra input modality (e.g. an image) could ever use meaningfully; :func:`_requires_non_text_input` excludes exactly those rows here, using catalog evidence discovery already records, not a per-model name rule. + Similarly, a model whose evidence says it only accepts a single tool call + at a time is not fit for arbitrary tool-calling requests; + :func:`_requires_single_tool_call` excludes those rows, while ``None`` + (no evidence) keeps the pool open. Such a model remains fully discovered, fully counted in :func:`free_discovered_models`'s price-based inventory, and eligible for - a pool that is not modality-blind (e.g. one built for vision/multimodal - tasks) -- it is only withheld from *this* general-purpose free selector. + a pool that is not modality-blind or tool-call-blind (e.g. one built for + vision/multimodal or single-tool tasks) -- it is only withheld from *this* + general-purpose free selector. Reproduces ContextualWisdomLab/.github#1198's required Strix Security Scan failure (run 33325907333, job 99295892400): NVIDIA NIM's free @@ -1641,7 +1791,9 @@ def general_free_serving_candidates( candidates = [ model for model in free_discovered_models(discovered) - if is_routable_discovered_model(model) and not _requires_non_text_input(model) + if is_routable_discovered_model(model) + and not _requires_non_text_input(model) + and not _requires_single_tool_call(model) ] _log_zero_free_serving_contribution(discovered, candidates) return candidates diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 1f99f5fac..613f6c644 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -565,6 +565,11 @@ def from_dict(cls, value: dict[str, Any]) -> "ModelAgent": # pragma: no cover def _is_general_chat_agent(agent: ModelAgent) -> bool: """Apply persisted provider capability tags before model-name fallback.""" + supports_parallel_tool_calls: bool | None = None + if "tool_call:multi" in agent.tags: + supports_parallel_tool_calls = True + elif "tool_call:single" in agent.tags: + supports_parallel_tool_calls = False return is_general_chat_candidate( agent.model, capabilities=( @@ -577,6 +582,7 @@ def _is_general_chat_agent(agent: ModelAgent) -> bool: for tag in agent.tags if tag.startswith("output:") ), + supports_parallel_tool_calls=supports_parallel_tool_calls, ) @@ -5920,6 +5926,18 @@ def _agent_requires_non_text_input(agent: ModelAgent) -> bool: tag[len("input:"):] for tag in agent.tags if tag.startswith("input:") ) + @staticmethod + def _agent_requires_single_tool_call(agent: ModelAgent) -> bool: + """Return whether an agent's discovery-derived tags declare single-tool-call only. + + Tool-call capability is carried as ``tool_call:multi`` or + ``tool_call:single`` tags. The general blind chat pool may send + multi-tool-call requests, so an agent tagged ``tool_call:single`` is + not eligible there. Absence of either tag means no evidence either way + and keeps the agent eligible. + """ + return "tool_call:single" in agent.tags + def _is_free_agent(self, agent: ModelAgent) -> bool: """Return true only for explicitly zero-priced configured models. @@ -5946,10 +5964,11 @@ def _is_general_free_agent(self, agent: ModelAgent) -> bool: ``orchestrator/free`` chat pool: that pool serves every role and request shape -- including tool-calling requests -- without knowing in advance which capability a request will need. An agent whose tags - declare a non-text input modality (e.g. a vision-input deployment) is - therefore never treated as free *here*, even when it is honestly - tagged ``cost:free`` for price inventory purposes and for its own - capability-scoped free route (see :meth:`_is_free_agent`, and + declare a non-text input modality (e.g. a vision-input deployment) or + single-tool-call-only capability is therefore never treated as free + *here*, even when it is honestly tagged ``cost:free`` for price + inventory purposes and for its own capability-scoped free route (see + :meth:`_is_free_agent`, and ``contextual_orchestrator.model_discovery.general_free_serving_candidates`` for the equivalent discovery-time selector and its incident writeup). This is the single choke point every *general chat* ``FREE_MODEL`` @@ -5957,7 +5976,11 @@ def _is_general_free_agent(self, agent: ModelAgent) -> bool: pool store that was written before this exclusion existed, or one activated by a pool-construction path this repository adds later. """ - return self._is_free_agent(agent) and not self._agent_requires_non_text_input(agent) + return ( + self._is_free_agent(agent) + and not self._agent_requires_non_text_input(agent) + and not self._agent_requires_single_tool_call(agent) + ) # --- semantic-affinity evidence (cosine similarity; no keyword lists) --- diff --git a/contextual_orchestrator/provider_bootstrap.py b/contextual_orchestrator/provider_bootstrap.py index 5de695124..b705266a2 100644 --- a/contextual_orchestrator/provider_bootstrap.py +++ b/contextual_orchestrator/provider_bootstrap.py @@ -193,6 +193,13 @@ def serving_tags_for_discovered(model: DiscoveredModel) -> tuple[str, ...]: *(f"capability:{value}" for value in model.capabilities), *(f"input:{value}" for value in model.input_modalities), *(f"output:{value}" for value in model.output_modalities), + *( + ("tool_call:multi",) + if model.supports_parallel_tool_calls is True + else ("tool_call:single",) + if model.supports_parallel_tool_calls is False + else () + ), ) ) ) diff --git a/docs/planning/adrs/0039-parallel-tool-call-capability.md b/docs/planning/adrs/0039-parallel-tool-call-capability.md new file mode 100644 index 000000000..f7eb49d66 --- /dev/null +++ b/docs/planning/adrs/0039-parallel-tool-call-capability.md @@ -0,0 +1,93 @@ +--- +id: "0039" +title: "Model discovery must record parallel tool-call capability" +status: accepted +proposed_date: "2026-08-31" +accepted_date: "2026-08-31" +deciders: + - "repository maintainer" +affected_components: + - "contextual_orchestrator/model_discovery.py" + - "contextual_orchestrator/chat_capability.py" + - "contextual_orchestrator/orchestrator.py" + - "contextual_orchestrator/provider_bootstrap.py" +related: + - path: "docs/planning/adrs/0035-structured-provider-orchestration.md" + relation: extends + - path: "docs/planning/adrs/0034-anti-heuristic-routing-evidence.md" + relation: constrained-by +success_criteria: + - metric: "single-tool-call exclusion from orchestrator/free" + target: "a model whose evidence says it accepts only one tool call at a time is not selected by the general free pool" + source: "tests/test_model_discovery.py" + - metric: "discovery-orchestrator agreement" + target: "general_free_serving_candidates and TaskOrchestrator._is_general_free_agent reach the same conclusion for the same evidence" + source: "tests/test_model_discovery.py" + - metric: "runtime capability tag propagation" + target: "ModelAgent tags carry tool_call:multi or tool_call:single when discovery provides the evidence" + source: "tests/test_provider_bootstrap.py" +--- + +# Model discovery must record parallel tool-call capability + +## Context + +The `orchestrator/free` pool is used by the ContextualWisdomLab central review agents +(OpenCode, Noema, Strix) because it is the zero-cost, capability-blind pool. A model +that enters this pool must be able to handle arbitrary chat requests, including +multi-tool-call requests, without the caller knowing the model's limitations in +advance. + +Issue [#940](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/940) +records a live NIM failure: `meta/llama-3.2-11b-vision-instruct` rejected a request +with `openai.BadRequestError: 400 ... This model only supports single tool-calls at +once!`. The model had already been excluded from the general free pool on vision-input +grounds by ADR 0035/0034, but the failure surfaced a broader gap: `DiscoveredModel` +carried no tool-call parallelism signal at all, so there was no honest way to keep a +single-tool-call model out of the general free pool even when the evidence was known. + +## Decision + +Add a `supports_parallel_tool_calls: bool | None = None` field to `DiscoveredModel`. + +- `True` means the provider catalog explicitly lists `parallel_tool_calls` in + `supported_parameters` (or a live probe returns a successful multi-tool response). +- `False` means the provider catalog or a probe clearly reports the model accepts only + one tool call at a time. +- `None` means no evidence either way, preserving the "positive declarations" contract + from ADR 0035: absence of a declaration must not be treated as a false claim of + incompatibility. + +The field is wired through the same capability-tag pipeline as input modalities and +privacy evidence: + +- `is_general_chat_candidate` excludes a model whose evidence is `False`. +- `is_discovered_chat_candidate` passes the field from `DiscoveredModel`. +- `is_routable_discovered_model` rejects single-tool rows. +- `general_free_serving_candidates` excludes them from the blind `orchestrator/free` pool. +- `agent_from_discovered` and `provider_bootstrap.serving_tags_for_discovered` emit + `tool_call:multi` or `tool_call:single` tags. +- `TaskOrchestrator._is_general_chat_agent` derives the value from the runtime tags and + passes it to `is_general_chat_candidate`. +- `TaskOrchestrator._is_general_free_agent` additionally refuses agents tagged + `tool_call:single`. + +A new `probe_discovered_model_tool_call_capability` function performs a minimal live +`POST /chat/completions` probe with `parallel_tool_calls: true` and two tool +definitions. It is deliberately separate from `discover_all_models` so callers decide +when the extra latency and token cost are justified. A successful response returns +`True`; a 400 whose body contains an explicit single-tool-call message returns `False`; +any network, auth, or ambiguous error returns `None` so the pool stays open rather +than excluding a model on a flaky probe. + +## Consequences + +- The general `orchestrator/free` pool can no longer route arbitrary multi-tool requests + to a model known to support only single tool calls. +- Provider catalog evidence is the primary source; runtime probing is a deliberate, + opt-in secondary source for providers that do not publish the parameter. +- Existing models whose evidence is `None` keep their previous eligibility, avoiding + false negatives. +- The `.github` sidecar must still be updated to consume `general_free_serving_candidates` + and preserve `input:` and `tool_call:` tags when it builds its own CI review catalog; + that is a follow-up change in `ContextualWisdomLab/.github`. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index cbad42f7f..ddeb1089d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2405,3 +2405,39 @@ shows this is now occasional, not the dominant failure mode (most is an overall deadline on `_invoke`'s candidate/retry loop, not another timeout increase on the sidecar's client side — deferred rather than rushed into this heavily-tested core file without dedicated validation. + +## 2026-08-31 orchestrator/free: single-tool-call models enter the general free pool and fail at runtime (#940) + +Live NIM failure (`openai.BadRequestError: 400 ... This model only supports single +tool-calls at once!`) on `meta/llama-3.2-11b-vision-instruct` showed that +`DiscoveredModel` had no tool-call parallelism signal. Even after the vision-input +exclusion from ADR 0035, there was no honest way to keep a single-tool-call model +out of the capability-blind `orchestrator/free` pool used by OpenCode, Noema, and +Strix. + +**Gap:** model discovery and runtime agent tags did not carry parallel vs. single +tool-call capability, so `general_free_serving_candidates` and +`_is_general_free_agent` could not reject single-tool models. + +**Fix (PR [#940](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/940)):** +added `supports_parallel_tool_calls: bool | None = None` to `DiscoveredModel`: + +- `True` when provider `supported_parameters` lists `parallel_tool_calls` or a live + multi-tool probe succeeds. +- `False` when provider evidence or a probe 400 explicitly reports single-tool-call + only. +- `None` when there is no evidence, preserving positive-declaration semantics. + +Wired through `is_general_chat_candidate`, `is_discovered_chat_candidate`, +`is_routable_discovered_model`, `general_free_serving_candidates`, +`agent_from_discovered`, `serving_tags_for_discovered`, `_is_general_chat_agent`, +and `_is_general_free_agent`. Added runtime `tool_call:multi`/`tool_call:single` +tags and `_agent_requires_single_tool_call`. Added `probe_discovered_model_tool_call_capability` +for opt-in live probing. Added ADR 0039. + +**Remaining follow-up:** `ContextualWisdomLab/.github`'s +`scripts/ci/contextual_orchestrator_review_launcher.py` and +`contextual_orchestrator_review_policy.py` still build their own CI review catalog +and do not consume `general_free_serving_candidates` or preserve `input:` and +`tool_call:` tags, so the NIM vision/single-tool models can still enter the +`orchestrator/free` CI selection unless the vendored sidecar is updated. diff --git a/tests/test_chat_capability.py b/tests/test_chat_capability.py index a1e82732f..8090677b1 100644 --- a/tests/test_chat_capability.py +++ b/tests/test_chat_capability.py @@ -83,3 +83,14 @@ def test_normal_chat_identifier_remains_eligible() -> None: ) def test_explicit_chat_metadata_does_not_admit_safety_models(metadata: dict) -> None: assert is_general_chat_candidate("vendor/safety-guard", **metadata) is False + + +def test_single_tool_call_evidence_excludes_general_chat_candidate() -> None: + """A model that only supports one tool call at a time is not a general chat agent.""" + assert is_general_chat_candidate("vendor/model", supports_parallel_tool_calls=False) is False + + +def test_unproven_tool_call_parallelism_keeps_existing_eligibility() -> None: + """No tool-call evidence neither adds nor removes eligibility.""" + assert is_general_chat_candidate("vendor/model", supports_parallel_tool_calls=None) is True + assert is_general_chat_candidate("vendor/model", supports_parallel_tool_calls=True) is True diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index c753a3d6a..2198505cf 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -6,6 +6,7 @@ import sys import urllib.error import urllib.parse +from io import BytesIO from dataclasses import replace from pathlib import Path from unittest.mock import patch @@ -36,10 +37,13 @@ _merge_configured_gateway_metadata, _merge_openrouter_provider_privacy, _merge_openrouter_zdr_metadata, + _parallel_tool_call_evidence, _price_per_1k, _parse_openai_compatible, + _tool_call_parallelism_from_error, agent_from_discovered, agent_id_for, + probe_discovered_model_tool_call_capability, discover_all_models, discover_provider_models, free_discovered_models, @@ -2363,3 +2367,202 @@ def test_sync_discovered_agents_persists_when_agents_db_is_set(tmp_path) -> None second = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) assert any(a.id == "openai_gpt_5_5" for a in second.candidates) + + +def _single_tool_discovered_model() -> DiscoveredModel: + return DiscoveredModel( + provider_name="nvidia_nim", + model_id="generic/single-tool-model", + credential_name="NVIDIA_NIM_API_KEY", + chat_base_url="https://integrate.api.nvidia.com/v1", + auth_scheme="Bearer", + capabilities=("chat",), + input_modalities=("text",), + output_modalities=("text",), + is_free=True, + supports_parallel_tool_calls=False, + ) + + +def test_parallel_tool_call_evidence_from_supported_parameters() -> None: + """Direct ``parallel_tool_calls`` parameter is positive evidence; absence is not.""" + assert _parallel_tool_call_evidence(["parallel_tool_calls", "temperature"]) is True + assert _parallel_tool_call_evidence(["tools", "temperature"]) is None + assert _parallel_tool_call_evidence([]) is None + assert _parallel_tool_call_evidence(None) is None + assert _parallel_tool_call_evidence([" PARALLEL_TOOL_CALLS "]) is True + + +def test_tool_call_parallelism_from_error_recognizes_single_tool_rejection() -> None: + """Only explicit single-tool-call messages are treated as negative evidence.""" + assert ( + _tool_call_parallelism_from_error( + {"error": {"message": "This model only supports single tool-calls at once!"}} + ) + is False + ) + assert ( + _tool_call_parallelism_from_error( + {"message": "parallel_tool_calls is not supported for this model."} + ) + is False + ) + assert ( + _tool_call_parallelism_from_error({"error": {"message": "Invalid API key"}}) + is None + ) + assert _tool_call_parallelism_from_error("only supports one tool call at a time.") is False + assert _tool_call_parallelism_from_error({"error": {}}) is None + + +def test_probe_discovered_model_tool_call_capability_success_returns_true() -> None: + """A 200 response to a multi-tool probe means the model accepted the shape.""" + discovered = _single_tool_discovered_model() + response = urllib.request.addinfourl( + BytesIO(b'{"id":"chatcmpl"}'), + {"content-type": "application/json"}, + "", + ) + response.code = 200 + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "urllib.request.urlopen", return_value=response + ) as mocked: + result = probe_discovered_model_tool_call_capability(discovered, timeout=5.0) + assert result is True + request = mocked.call_args[0][0] + assert request.get_full_url() == "https://integrate.api.nvidia.com/v1/chat/completions" + assert request.method == "POST" + payload = json.loads(request.data) + assert payload["parallel_tool_calls"] is True + assert len(payload["tools"]) == 2 + + +def test_probe_discovered_model_tool_call_capability_single_tool_400_returns_false() -> None: + """NIM's explicit single-tool 400 is captured as negative evidence.""" + discovered = _single_tool_discovered_model() + body = json.dumps( + {"error": {"message": "This model only supports single tool-calls at once!"}} + ).encode() + exc = urllib.error.HTTPError( + discovered.chat_base_url, 400, "Bad Request", {}, BytesIO(body) + ) + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "urllib.request.urlopen", side_effect=exc + ): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is False + + +def test_probe_returns_none_without_credential() -> None: + """No credential means no probe; absence of evidence stays open.""" + discovered = _single_tool_discovered_model() + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None + + +def test_probe_returns_none_on_network_error() -> None: + """Network failures are not evidence one way or the other.""" + discovered = _single_tool_discovered_model() + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "urllib.request.urlopen", side_effect=urllib.error.URLError("timeout") + ): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None + + +def test_tool_call_parallelism_from_error_handles_non_dict_error_value() -> None: + """A string ``error`` value still lets us read ``message`` from the payload.""" + assert ( + _tool_call_parallelism_from_error( + {"error": "string-value", "message": "This model only supports single tool-calls"} + ) + is False + ) + + +def test_tool_call_parallelism_from_error_rejects_unrecognised_types() -> None: + """Non-dict, non-string payloads carry no usable signal.""" + assert _tool_call_parallelism_from_error(123) is None + assert _tool_call_parallelism_from_error(None) is None + + +def test_probe_returns_none_for_non_https_url() -> None: + """A non-HTTPS base URL is not probed for safety.""" + discovered = replace(_single_tool_discovered_model(), chat_base_url="http://insecure.example/v1") + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None + + +def test_probe_returns_none_for_non_400_http_error() -> None: + """Only 400 responses are interpreted as capability evidence.""" + discovered = _single_tool_discovered_model() + exc = urllib.error.HTTPError( + discovered.chat_base_url, 500, "Internal Error", {}, BytesIO(b"oops") + ) + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "urllib.request.urlopen", side_effect=exc + ): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None + + +def test_probe_reads_non_json_400_body_as_plain_message() -> None: + """A plain-text 400 body is still searched for a single-tool signal.""" + discovered = _single_tool_discovered_model() + exc = urllib.error.HTTPError( + discovered.chat_base_url, 400, "Bad Request", {}, BytesIO(b"only supports single tool-calls") + ) + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "urllib.request.urlopen", side_effect=exc + ): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is False + + +def test_is_routable_discovered_model_false_when_single_tool_only() -> None: + """Single-tool-call evidence makes a discovered row not a general chat candidate.""" + discovered = _single_tool_discovered_model() + assert is_routable_discovered_model(discovered) is False + + +def test_general_free_serving_candidates_excludes_single_tool_call_model() -> None: + """The general free pool excludes a model known to only accept one tool call.""" + single = _single_tool_discovered_model() + multi = replace(single, model_id="generic/multi-tool-model", supports_parallel_tool_calls=True) + unknown = replace(single, model_id="generic/unknown-tool-model", supports_parallel_tool_calls=None) + candidates = general_free_serving_candidates([single, multi, unknown]) + assert [model.model_id for model in candidates] == [ + "generic/multi-tool-model", + "generic/unknown-tool-model", + ] + + +def test_discovery_and_orchestrator_tool_call_eligibility_cannot_drift() -> None: + """``general_free_serving_candidates`` and ``_is_general_free_agent`` agree on tool calls.""" + single = _single_tool_discovered_model() + multi = replace(single, model_id="generic/multi-tool-model", supports_parallel_tool_calls=True) + unknown = replace(single, model_id="generic/unknown-tool-model", supports_parallel_tool_calls=None) + discovered = [single, multi, unknown] + serving_model_ids = { + model.model_id for model in general_free_serving_candidates(discovered) + } + agent_id_translation = str.maketrans("/.-", "___") + agents = { + model.model_id: ModelAgent( + model.model_id.casefold().translate(agent_id_translation), + model.model_id, + tags=( + "cost:free", + *(f"input:{value}" for value in model.input_modalities), + *( + ("tool_call:multi",) + if model.supports_parallel_tool_calls is True + else ("tool_call:single",) + if model.supports_parallel_tool_calls is False + else () + ), + ), + ) + for model in discovered + } + orchestrator = TaskOrchestrator(list(agents.values())) + for model in discovered: + agent = agents[model.model_id] + assert orchestrator._is_general_free_agent(agent) == ( + model.model_id in serving_model_ids + ), model.model_id diff --git a/tests/test_provider_bootstrap.py b/tests/test_provider_bootstrap.py index 082151901..4a9ed9ecb 100644 --- a/tests/test_provider_bootstrap.py +++ b/tests/test_provider_bootstrap.py @@ -325,6 +325,59 @@ def test_active_agent_from_discovered_free_vision_model_is_not_free_pool_eligibl assert orchestrator._is_general_free_agent(agent) is False +def test_serving_tags_preserve_tool_call_parallelism_evidence(): + """Explicit parallel/single tool-call evidence survives bootstrap tag normalization.""" + multi = replace( + _model("openai", "OPENAI_API_KEY", "gpt-4o-mini", 0.0), + capabilities=("chat",), + supports_parallel_tool_calls=True, + ) + single = replace( + _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "meta/llama-3.2-11b-vision-instruct", 0.0), + capabilities=("chat",), + supports_parallel_tool_calls=False, + ) + assert "tool_call:multi" in provider_bootstrap.serving_tags_for_discovered(multi) + assert "tool_call:single" in provider_bootstrap.serving_tags_for_discovered(single) + + +def test_single_tool_call_model_is_not_chat_serving_candidate(): + """A model known to accept only one tool call at a time is not a general chat candidate.""" + single = replace( + _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "generic/single-tool-model", 0.0), + capabilities=("chat",), + input_modalities=("text",), + output_modalities=("text",), + supports_parallel_tool_calls=False, + ) + assert provider_bootstrap.is_chat_serving_candidate(single) is False + + +def test_bootstrap_active_free_single_tool_agent_is_not_free_pool_eligible(): + """A bootstrap-activated free single-tool-call agent stays out of orchestrator/free.""" + single_tool = replace( + _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "generic/single-tool-model", 0.0), + capabilities=("chat",), + input_modalities=("text",), + output_modalities=("text",), + is_free=True, + supports_parallel_tool_calls=False, + ) + + tags = provider_bootstrap.serving_tags_for_discovered(single_tool) + agent = ModelAgent( + "generic_single_tool_model", + "generic/single-tool-model", + tags=tags, + ) + orchestrator = TaskOrchestrator([agent]) + + assert "cost:free" in agent.tags + assert "tool_call:single" in agent.tags + assert orchestrator._is_free_agent(agent) is True + assert orchestrator._is_general_free_agent(agent) is False + + def test_serving_tags_preserve_explicit_no_zdr_evidence(): """Explicit unsupported zero-data retention survives tag normalization.""" model = replace( From b7842bec0e1fb28f658cbc58f03fdf449daff188 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 19:41:35 +0900 Subject: [PATCH 02/13] fix(discovery): preserve parallel tool-call evidence end to end --- contextual_orchestrator/model_discovery.py | 79 ++++++++- .../provider_catalog_store.py | 8 + .../0039-parallel-tool-call-capability.md | 20 +++ tests/test_model_discovery.py | 153 +++++++++++++++++- tests/test_provider_catalog_store.py | 21 +++ 5 files changed, 271 insertions(+), 10 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index bac8b2f13..c6f0bf094 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -113,6 +113,35 @@ def _tool_call_parallelism_from_error(error_payload: Any) -> bool | None: return None +def _response_contains_parallel_probe_tool_calls(payload: Any) -> bool: + """Return whether a probe response clearly contains both requested tool calls.""" + if not isinstance(payload, dict): + return False + seen: set[str] = set() + + def collect(value: Any) -> None: + if isinstance(value, dict): + if value.get("type") == "function": + function = value.get("function") + if isinstance(function, dict): + name = function.get("name") + if isinstance(name, str) and name in {"probe_a", "probe_b"}: + seen.add(name) + if value.get("type") == "function_call": + name = value.get("name") + if isinstance(name, str) and name in {"probe_a", "probe_b"}: + seen.add(name) + for child in value.values(): + collect(child) + return + if isinstance(value, list): + for child in value: + collect(child) + + collect(payload) + return seen == {"probe_a", "probe_b"} + + def probe_discovered_model_tool_call_capability( discovered: DiscoveredModel, *, @@ -122,11 +151,12 @@ def probe_discovered_model_tool_call_capability( Sends a minimal ``/chat/completions`` request with ``parallel_tool_calls: true`` and two tool definitions, using the provider credential registered in the KV. - A successful response means the model accepted the multi-tool shape - (``True``). A 400 whose error text clearly says the model only supports a - single tool call means it does not (``False``). Any network, auth, or - ambiguous error returns ``None`` so the pool stays open rather than - excluding a model on a flaky probe. + A successful response counts as positive evidence only when the body + demonstrably contains tool calls to both probe functions. A 400 whose error + text clearly says the model only supports a single tool call means it does + not (``False``). Any network, auth, malformed, or ambiguous response + returns ``None`` so the pool stays open rather than excluding a model on a + flaky probe. This is real runtime evidence, not a model-name heuristic. It is deliberately separate from :func:`discover_all_models` so callers decide when the extra @@ -160,7 +190,7 @@ def probe_discovered_model_tool_call_capability( }, ], "parallel_tool_calls": True, - "max_tokens": 1, + "max_tokens": 32, "temperature": 0.0, "stream": False, } @@ -173,7 +203,7 @@ def probe_discovered_model_tool_call_capability( request = urllib.request.Request(url, data=data, headers=headers, method="POST") try: with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - scoped provider probe - response.read() + body = response.read().decode("utf-8", errors="replace") except urllib.error.HTTPError as exc: if exc.code != 400: return None @@ -185,7 +215,11 @@ def probe_discovered_model_tool_call_capability( return _tool_call_parallelism_from_error(error_payload) except (urllib.error.URLError, OSError, TimeoutError, ValueError): return None - return True + try: + response_payload = json.loads(body) + except json.JSONDecodeError: + return None + return True if _response_contains_parallel_probe_tool_calls(response_payload) else None _MODELS_DEV_URL = "https://models.dev/api.json" @@ -638,6 +672,12 @@ def _deduplicate_discovered_models( supports_no_training=None, supports_no_prompt_retention=None, zdr_capable=False, + supports_parallel_tool_calls=( + chosen.supports_parallel_tool_calls + if previous.supports_parallel_tool_calls + == model.supports_parallel_tool_calls + else None + ), ) return list(unique.values()) @@ -810,9 +850,11 @@ def _merge_configured_gateway_metadata(payload: Any, metadata: Any) -> Any: "privacy_policy_urls", ): row.pop(key, None) + row.pop("supported_parameters", None) model_details = by_name.get(row["id"], []) deployment_outputs: list[tuple[str, ...]] = [] deployment_inputs: list[tuple[str, ...]] = [] + deployment_supported_parameters: list[tuple[str, ...]] = [] prices: set[tuple[object, object]] = set() pricing_complete = bool(model_details) unit_price_maps: list[tuple[tuple[str, object], ...]] = [] @@ -869,6 +911,23 @@ def _merge_configured_gateway_metadata(payload: Any, metadata: Any) -> Any: if info.get("supports_vision") is True and "image" not in inputs: inputs = (*inputs, "image") deployment_inputs.append(inputs) + supported_parameters = info.get( + "supported_openai_params", + info.get("supported_parameters", detail.get("supported_parameters")), + ) + if isinstance(params.get("supported_openai_params"), list): + supported_parameters = params["supported_openai_params"] + elif isinstance(params.get("supported_parameters"), list): + supported_parameters = params["supported_parameters"] + deployment_supported_parameters.append( + tuple( + value.strip() + for value in supported_parameters + if isinstance(value, str) and value.strip() + ) + if isinstance(supported_parameters, list) + else () + ) prompt = info.get("input_cost_per_token", params.get("input_cost_per_token")) completion = info.get( "output_cost_per_token", params.get("output_cost_per_token") @@ -904,6 +963,10 @@ def _merge_configured_gateway_metadata(payload: Any, metadata: Any) -> Any: "input_modalities": list(deployment_inputs[0]), "output_modalities": list(deployment_outputs[0]), } + if deployment_supported_parameters and all( + deployment_supported_parameters + ) and len(set(deployment_supported_parameters)) == 1: + row["supported_parameters"] = list(deployment_supported_parameters[0]) if pricing_complete and len(prices) == 1: prompt, completion = prices.pop() if prompt is not None and completion is not None: diff --git a/contextual_orchestrator/provider_catalog_store.py b/contextual_orchestrator/provider_catalog_store.py index ee76c53a6..63e022c2e 100644 --- a/contextual_orchestrator/provider_catalog_store.py +++ b/contextual_orchestrator/provider_catalog_store.py @@ -328,6 +328,7 @@ def normalize_discovered_model( privacy_policy_urls=tuple(model.privacy_policy_urls), zdr_capable=bool(model.zdr_capable), spend_admitted=bool(model.spend_admitted), + supports_parallel_tool_calls=model.supports_parallel_tool_calls, ) @@ -366,6 +367,13 @@ def _restore_model_semantics( ), privacy_policy_urls=tuple(model.privacy_policy_urls), zdr_capable=bool(model.zdr_capable), + supports_parallel_tool_calls=( + True + if "tool_call:multi" in normalized + else False + if "tool_call:single" in normalized + else None + ), ) diff --git a/docs/planning/adrs/0039-parallel-tool-call-capability.md b/docs/planning/adrs/0039-parallel-tool-call-capability.md index f7eb49d66..baa56b894 100644 --- a/docs/planning/adrs/0039-parallel-tool-call-capability.md +++ b/docs/planning/adrs/0039-parallel-tool-call-capability.md @@ -11,6 +11,10 @@ affected_components: - "contextual_orchestrator/chat_capability.py" - "contextual_orchestrator/orchestrator.py" - "contextual_orchestrator/provider_bootstrap.py" +consulted: + - "docs/papers/frugalgpt-cost-2305.05176.pdf" + - "docs/papers/hybrid-llm-query-routing-2404.14618.pdf" + - "docs/papers/routellm-routing-2406.18665.pdf" related: - path: "docs/planning/adrs/0035-structured-provider-orchestration.md" relation: extends @@ -91,3 +95,19 @@ than excluding a model on a flaky probe. - The `.github` sidecar must still be updated to consume `general_free_serving_candidates` and preserve `input:` and `tool_call:` tags when it builds its own CI review catalog; that is a follow-up change in `ContextualWisdomLab/.github`. + +## Research grounding + +This decision is a capability-constrained routing safeguard, not a learned quality claim. +It reuses the repository's existing vendored routing literature: + +- Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language models while reducing cost and improving performance*. arXiv. https://arxiv.org/abs/2305.05176 +- Ding, D., Mallick, A., Wang, C., Sim, R., Mukherjee, S., Rühle, V., Lakshmanan, L. V. S., & Awadallah, A. H. (2024). *Hybrid LLM: Cost-efficient and quality-aware query routing*. International Conference on Learning Representations. https://arxiv.org/abs/2404.14618 +- Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with preference data*. arXiv. https://arxiv.org/abs/2406.18665 + +These papers justify preserving explicit capability evidence at the routing boundary. +They do not justify inferring multi-tool support from model names or from a bare 200 +response, so this ADR keeps the field fail-closed on ambiguity. + +The cited PDFs are already vendored in `docs/papers/`; no additional restricted paper is +copied in this run. diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 2198505cf..2d993a74b 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -39,6 +39,7 @@ _merge_openrouter_zdr_metadata, _parallel_tool_call_evidence, _price_per_1k, + _response_contains_parallel_probe_tool_calls, _parse_openai_compatible, _tool_call_parallelism_from_error, agent_from_discovered, @@ -199,6 +200,45 @@ def test_configured_gateway_preserves_consensus_privacy_evidence() -> None: ] +def test_configured_gateway_preserves_consensus_supported_parameters() -> None: + payload = {"data": [{"id": "tool-model"}]} + detail = { + "model_name": "tool-model", + "model_info": { + "mode": "chat", + "supported_parameters": ["parallel_tool_calls", "tools"], + }, + } + + merged = _merge_configured_gateway_metadata(payload, {"data": [detail, detail]}) + + assert merged["data"][0]["supported_parameters"] == [ + "parallel_tool_calls", + "tools", + ] + + +def test_configured_gateway_withholds_conflicting_supported_parameters() -> None: + payload = {"data": [{"id": "tool-model"}]} + merged = _merge_configured_gateway_metadata( + payload, + { + "data": [ + { + "model_name": "tool-model", + "model_info": {"supported_parameters": ["parallel_tool_calls"]}, + }, + { + "model_name": "tool-model", + "model_info": {"supported_parameters": ["tools"]}, + }, + ] + }, + ) + + assert "supported_parameters" not in merged["data"][0] + + def test_configured_gateway_keeps_ambiguous_privacy_strings_unknown() -> None: """Only explicit boolean strings can become provider privacy evidence.""" payload = {"data": [{"id": "chat-model"}]} @@ -321,6 +361,39 @@ def test_duplicate_discovery_withholds_conflicting_zdr_capability() -> None: assert discovered[0].zdr_capable is False +@pytest.mark.parametrize( + ("left", "right", "expected"), + [ + (True, False, None), + (True, None, None), + (False, None, None), + (True, True, True), + (False, False, False), + (None, None, None), + ], +) +def test_duplicate_discovery_parallel_tool_call_evidence_is_order_invariant( + left: bool | None, + right: bool | None, + expected: bool | None, +) -> None: + first = DiscoveredModel( + provider_name="gateway", + model_id="shared-model", + credential_name="KEY_A", + chat_base_url="https://gateway.example/v1", + auth_scheme="Bearer", + supports_parallel_tool_calls=left, + ) + second = replace(first, supports_parallel_tool_calls=right) + + forward = _deduplicate_discovered_models([first, second])[0] + reverse = _deduplicate_discovered_models([second, first])[0] + + assert forward.supports_parallel_tool_calls is expected + assert reverse.supports_parallel_tool_calls is expected + + def test_same_provider_model_under_different_credentials_remains_independent() -> None: """Credential accounts may expose different evidence for the same model id.""" discovered = _deduplicate_discovered_models( @@ -2416,10 +2489,31 @@ def test_tool_call_parallelism_from_error_recognizes_single_tool_rejection() -> def test_probe_discovered_model_tool_call_capability_success_returns_true() -> None: - """A 200 response to a multi-tool probe means the model accepted the shape.""" + """A 200 probe counts only when the response actually contains both tool calls.""" discovered = _single_tool_discovered_model() response = urllib.request.addinfourl( - BytesIO(b'{"id":"chatcmpl"}'), + BytesIO( + json.dumps( + { + "choices": [ + { + "message": { + "tool_calls": [ + { + "type": "function", + "function": {"name": "probe_a", "arguments": "{}"}, + }, + { + "type": "function", + "function": {"name": "probe_b", "arguments": "{}"}, + }, + ] + } + } + ] + } + ).encode() + ), {"content-type": "application/json"}, "", ) @@ -2434,9 +2528,64 @@ def test_probe_discovered_model_tool_call_capability_success_returns_true() -> N assert request.method == "POST" payload = json.loads(request.data) assert payload["parallel_tool_calls"] is True + assert payload["max_tokens"] == 32 assert len(payload["tools"]) == 2 +@pytest.mark.parametrize( + "body", + [ + {"choices": [{"message": {"content": "no tool calls"}}]}, + { + "choices": [ + { + "message": { + "tool_calls": [ + { + "type": "function", + "function": {"name": "probe_a", "arguments": "{}"}, + } + ] + } + } + ] + }, + {}, + ], +) +def test_probe_discovered_model_tool_call_capability_success_without_two_calls_returns_none( + body: dict[str, object] +) -> None: + discovered = _single_tool_discovered_model() + response = urllib.request.addinfourl( + BytesIO(json.dumps(body).encode()), + {"content-type": "application/json"}, + "", + ) + response.code = 200 + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "urllib.request.urlopen", return_value=response + ): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None + + +def test_response_contains_parallel_probe_tool_calls_recognizes_responses_api_shape() -> None: + assert _response_contains_parallel_probe_tool_calls( + { + "output": [ + { + "type": "function_call", + "name": "probe_a", + }, + { + "type": "function_call", + "name": "probe_b", + }, + ] + } + ) + + def test_probe_discovered_model_tool_call_capability_single_tool_400_returns_false() -> None: """NIM's explicit single-tool 400 is captured as negative evidence.""" discovered = _single_tool_discovered_model() diff --git a/tests/test_provider_catalog_store.py b/tests/test_provider_catalog_store.py index 9ba7b6a57..5c0b790a8 100644 --- a/tests/test_provider_catalog_store.py +++ b/tests/test_provider_catalog_store.py @@ -238,6 +238,25 @@ def test_last_known_good_restores_explicit_no_zdr_evidence() -> None: assert restored[0].supports_zero_data_retention is False +def test_last_known_good_restores_tool_call_parallelism_evidence() -> None: + source = _source(provider="openai", credential="OPENAI_API_KEY") + model = replace( + _model(source, "tool-model", 0), + supports_parallel_tool_calls=True, + ) + store = InMemoryProviderCatalogStore() + store.record_success( + source, + [model], + eligible_model_ids={model.model_id}, + serving_tags={model.model_id: ("discovered", "tool_call:multi")}, + ) + + restored = store.serving_models(source) + assert len(restored) == 1 + assert restored[0].supports_parallel_tool_calls is True + + def _assessment( source: ProviderModelSource, *, @@ -504,6 +523,7 @@ def test_postgres_serving_models_reconstructs_account_scoped_rows() -> None: ("model-b", "capability:chat"), ("model-b", "cost:free"), ("model-b", "input:text"), + ("model-b", "tool_call:single"), ], [("model-b", "https://provider.example/privacy")], unit_price_rows=[ @@ -527,6 +547,7 @@ def test_postgres_serving_models_reconstructs_account_scoped_rows() -> None: capabilities=("chat",), input_modalities=("text",), is_free=True, + supports_parallel_tool_calls=False, privacy_policy_urls=("https://provider.example/privacy",), unit_prices=(ModelUnitPrice("output_cost_per_image", 0.04),), ) From 3a8168cc1d0f38097e99e55a7f0e2f2dc2129107 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:19:52 +0000 Subject: [PATCH 03/13] fix(discovery): suppress Semgrep dynamic-urllib rule on scoped provider probe Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- contextual_orchestrator/model_discovery.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index c6f0bf094..79a410ed2 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -202,7 +202,9 @@ def probe_discovered_model_tool_call_capability( data = json.dumps(payload, separators=(",", ":")).encode("utf-8") request = urllib.request.Request(url, data=data, headers=headers, method="POST") try: - with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - scoped provider probe + with urllib.request.urlopen( # noqa: S310 - scoped provider probe # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected + request, timeout=timeout + ) as response: body = response.read().decode("utf-8", errors="replace") except urllib.error.HTTPError as exc: if exc.code != 400: From 74cea16334b442abc6bd2ba3e5c4a6c59b07a931 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 20:23:42 +0900 Subject: [PATCH 04/13] fix(discovery): harden capability reconciliation --- contextual_orchestrator/__main__.py | 5 ++- contextual_orchestrator/model_discovery.py | 48 +++++++++++++++++----- tests/test_auto_discovery_server.py | 29 +++++++++++++ tests/test_model_discovery.py | 41 +++++++++++++++--- 4 files changed, 106 insertions(+), 17 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index ffd82536d..8efbc9d37 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -416,7 +416,10 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l chat_models = [ model for model in discovered - if not model.evidence_only and is_discovered_chat_candidate(model) + if not model.evidence_only + and is_discovered_chat_candidate( + replace(model, supports_parallel_tool_calls=None) + ) ] existing_by_id = {agent.id: agent for agent in orchestrator.candidates} agents = [] diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 79a410ed2..0835040fe 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -25,7 +25,7 @@ import urllib.request import certifi from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Any, Literal, Mapping +from typing import TYPE_CHECKING, Any, Literal, Mapping, Sequence from urllib.parse import quote, urlsplit, urlunsplit from .chat_capability import ( @@ -33,7 +33,7 @@ is_general_chat_candidate, requires_non_text_input, ) -from .credentials import get_credential +from .credentials import NotConfigured, get_credential from .orchestrator import ( AUTH_SCHEME_RAW_TOKEN, ModelAgent, @@ -108,7 +108,14 @@ def _tool_call_parallelism_from_error(error_payload: Any) -> bool | None: else: return None text = message.casefold() - if "single tool" in text or "one tool" in text or "parallel_tool_calls" in text: + if ( + "single tool" in text + or "one tool" in text + or ( + "parallel_tool_calls" in text + and any(phrase in text for phrase in ("not supported", "unsupported")) + ) + ): return False return None @@ -165,8 +172,31 @@ def probe_discovered_model_tool_call_capability( api_key = get_credential(discovered.credential_name) if not api_key: return None + parsed = urlsplit(discovered.chat_base_url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + return None url = discovered.chat_base_url.rstrip("/") + "/chat/completions" - if not url.startswith("https://"): + client = ModelClient( + timeout=max(1, math.ceil(timeout)), + allowed_provider_hosts={parsed.hostname}, + ) + agent = ModelAgent( + "tool_call_capability_probe", + discovered.model_id, + base_url=discovered.chat_base_url, + credential_key=discovered.credential_name, + auth_scheme=discovered.auth_scheme, + ) + try: + destination = client._validate_provider(agent) + except (NotConfigured, RuntimeError, ValueError): return None payload = { "model": discovered.model_id, @@ -202,9 +232,7 @@ def probe_discovered_model_tool_call_capability( data = json.dumps(payload, separators=(",", ":")).encode("utf-8") request = urllib.request.Request(url, data=data, headers=headers, method="POST") try: - with urllib.request.urlopen( # noqa: S310 - scoped provider probe # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - request, timeout=timeout - ) as response: + with client._open_provider(request, destination, timeout=timeout) as response: body = response.read().decode("utf-8", errors="replace") except urllib.error.HTTPError as exc: if exc.code != 400: @@ -922,11 +950,11 @@ def _merge_configured_gateway_metadata(payload: Any, metadata: Any) -> Any: elif isinstance(params.get("supported_parameters"), list): supported_parameters = params["supported_parameters"] deployment_supported_parameters.append( - tuple( - value.strip() + tuple(sorted({ + value.strip().casefold() for value in supported_parameters if isinstance(value, str) and value.strip() - ) + })) if isinstance(supported_parameters, list) else () ) diff --git a/tests/test_auto_discovery_server.py b/tests/test_auto_discovery_server.py index 86102624d..21d5f36b6 100644 --- a/tests/test_auto_discovery_server.py +++ b/tests/test_auto_discovery_server.py @@ -48,6 +48,35 @@ def test_auto_discovery_activates_only_chat_capable_agents(monkeypatch) -> None: assert "bootstrap_agent" in result["updated"] +def test_auto_discovery_disables_active_model_after_capability_downgrade(monkeypatch) -> None: + discovered = DiscoveredModel( + provider_name="openai", + model_id="chat-capable-model", + credential_name="OPENAI_API_KEY", + chat_base_url="https://api.openai.com/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=False, + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([discovered], []), + ) + existing = ModelAgent( + "openai_chat_capable_model", + discovered.model_id, + base_url=discovered.chat_base_url, + provider_name=discovered.provider_name, + tags=("discovered", "chat", "tool_calls:parallel"), + ) + orchestrator = TaskOrchestrator([existing]) + + result = _auto_discover_runtime_agents(orchestrator) + + assert result == {"added": [], "updated": [existing.id]} + assert orchestrator.candidates[0].disabled is True + + def test_auto_discovery_activates_a_free_vision_model_but_free_pool_excludes_it( monkeypatch, ) -> None: diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 2d993a74b..91f94ecb6 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -218,6 +218,19 @@ def test_configured_gateway_preserves_consensus_supported_parameters() -> None: ] +def test_configured_gateway_parameter_consensus_ignores_order_and_case() -> None: + payload = {"data": [{"id": "tool-model"}]} + merged = _merge_configured_gateway_metadata( + payload, + {"data": [ + {"model_name": "tool-model", "model_info": {"supported_parameters": ["tools", "PARALLEL_TOOL_CALLS"]}}, + {"model_name": "tool-model", "model_info": {"supported_parameters": ["parallel_tool_calls", "tools"]}}, + ]}, + ) + + assert merged["data"][0]["supported_parameters"] == ["parallel_tool_calls", "tools"] + + def test_configured_gateway_withholds_conflicting_supported_parameters() -> None: payload = {"data": [{"id": "tool-model"}]} merged = _merge_configured_gateway_metadata( @@ -2484,6 +2497,9 @@ def test_tool_call_parallelism_from_error_recognizes_single_tool_rejection() -> _tool_call_parallelism_from_error({"error": {"message": "Invalid API key"}}) is None ) + assert _tool_call_parallelism_from_error( + {"error": {"message": "parallel_tool_calls must be a boolean"}} + ) is None assert _tool_call_parallelism_from_error("only supports one tool call at a time.") is False assert _tool_call_parallelism_from_error({"error": {}}) is None @@ -2519,7 +2535,9 @@ def test_probe_discovered_model_tool_call_capability_success_returns_true() -> N ) response.code = 200 with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", return_value=response + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", return_value=response ) as mocked: result = probe_discovered_model_tool_call_capability(discovered, timeout=5.0) assert result is True @@ -2564,7 +2582,9 @@ def test_probe_discovered_model_tool_call_capability_success_without_two_calls_r ) response.code = 200 with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", return_value=response + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", return_value=response ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None @@ -2596,7 +2616,9 @@ def test_probe_discovered_model_tool_call_capability_single_tool_400_returns_fal discovered.chat_base_url, 400, "Bad Request", {}, BytesIO(body) ) with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=exc + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", side_effect=exc ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is False @@ -2611,7 +2633,10 @@ def test_probe_returns_none_on_network_error() -> None: """Network failures are not evidence one way or the other.""" discovered = _single_tool_discovered_model() with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=urllib.error.URLError("timeout") + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", + side_effect=urllib.error.URLError("timeout"), ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None @@ -2646,7 +2671,9 @@ def test_probe_returns_none_for_non_400_http_error() -> None: discovered.chat_base_url, 500, "Internal Error", {}, BytesIO(b"oops") ) with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=exc + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", side_effect=exc ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None @@ -2658,7 +2685,9 @@ def test_probe_reads_non_json_400_body_as_plain_message() -> None: discovered.chat_base_url, 400, "Bad Request", {}, BytesIO(b"only supports single tool-calls") ) with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=exc + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", side_effect=exc ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is False From af03bf0fb219101e821bd69d07571787a0303005 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 20:23:42 +0900 Subject: [PATCH 05/13] fix(discovery): harden capability reconciliation --- contextual_orchestrator/__main__.py | 5 ++- contextual_orchestrator/model_discovery.py | 48 +++++++++++++++++----- contextual_orchestrator/orchestrator.py | 6 +-- tests/test_auto_discovery_server.py | 29 +++++++++++++ tests/test_chat_capability.py | 12 ++++++ tests/test_model_discovery.py | 41 +++++++++++++++--- 6 files changed, 121 insertions(+), 20 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index ffd82536d..8efbc9d37 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -416,7 +416,10 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l chat_models = [ model for model in discovered - if not model.evidence_only and is_discovered_chat_candidate(model) + if not model.evidence_only + and is_discovered_chat_candidate( + replace(model, supports_parallel_tool_calls=None) + ) ] existing_by_id = {agent.id: agent for agent in orchestrator.candidates} agents = [] diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 79a410ed2..0835040fe 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -25,7 +25,7 @@ import urllib.request import certifi from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Any, Literal, Mapping +from typing import TYPE_CHECKING, Any, Literal, Mapping, Sequence from urllib.parse import quote, urlsplit, urlunsplit from .chat_capability import ( @@ -33,7 +33,7 @@ is_general_chat_candidate, requires_non_text_input, ) -from .credentials import get_credential +from .credentials import NotConfigured, get_credential from .orchestrator import ( AUTH_SCHEME_RAW_TOKEN, ModelAgent, @@ -108,7 +108,14 @@ def _tool_call_parallelism_from_error(error_payload: Any) -> bool | None: else: return None text = message.casefold() - if "single tool" in text or "one tool" in text or "parallel_tool_calls" in text: + if ( + "single tool" in text + or "one tool" in text + or ( + "parallel_tool_calls" in text + and any(phrase in text for phrase in ("not supported", "unsupported")) + ) + ): return False return None @@ -165,8 +172,31 @@ def probe_discovered_model_tool_call_capability( api_key = get_credential(discovered.credential_name) if not api_key: return None + parsed = urlsplit(discovered.chat_base_url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + return None url = discovered.chat_base_url.rstrip("/") + "/chat/completions" - if not url.startswith("https://"): + client = ModelClient( + timeout=max(1, math.ceil(timeout)), + allowed_provider_hosts={parsed.hostname}, + ) + agent = ModelAgent( + "tool_call_capability_probe", + discovered.model_id, + base_url=discovered.chat_base_url, + credential_key=discovered.credential_name, + auth_scheme=discovered.auth_scheme, + ) + try: + destination = client._validate_provider(agent) + except (NotConfigured, RuntimeError, ValueError): return None payload = { "model": discovered.model_id, @@ -202,9 +232,7 @@ def probe_discovered_model_tool_call_capability( data = json.dumps(payload, separators=(",", ":")).encode("utf-8") request = urllib.request.Request(url, data=data, headers=headers, method="POST") try: - with urllib.request.urlopen( # noqa: S310 - scoped provider probe # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - request, timeout=timeout - ) as response: + with client._open_provider(request, destination, timeout=timeout) as response: body = response.read().decode("utf-8", errors="replace") except urllib.error.HTTPError as exc: if exc.code != 400: @@ -922,11 +950,11 @@ def _merge_configured_gateway_metadata(payload: Any, metadata: Any) -> Any: elif isinstance(params.get("supported_parameters"), list): supported_parameters = params["supported_parameters"] deployment_supported_parameters.append( - tuple( - value.strip() + tuple(sorted({ + value.strip().casefold() for value in supported_parameters if isinstance(value, str) and value.strip() - ) + })) if isinstance(supported_parameters, list) else () ) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 613f6c644..887c6e46b 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -566,10 +566,10 @@ def from_dict(cls, value: dict[str, Any]) -> "ModelAgent": # pragma: no cover def _is_general_chat_agent(agent: ModelAgent) -> bool: """Apply persisted provider capability tags before model-name fallback.""" supports_parallel_tool_calls: bool | None = None - if "tool_call:multi" in agent.tags: - supports_parallel_tool_calls = True - elif "tool_call:single" in agent.tags: + if "tool_call:single" in agent.tags: supports_parallel_tool_calls = False + elif "tool_call:multi" in agent.tags: + supports_parallel_tool_calls = True return is_general_chat_candidate( agent.model, capabilities=( diff --git a/tests/test_auto_discovery_server.py b/tests/test_auto_discovery_server.py index 86102624d..b8dee2471 100644 --- a/tests/test_auto_discovery_server.py +++ b/tests/test_auto_discovery_server.py @@ -48,6 +48,35 @@ def test_auto_discovery_activates_only_chat_capable_agents(monkeypatch) -> None: assert "bootstrap_agent" in result["updated"] +def test_auto_discovery_disables_active_model_after_capability_downgrade(monkeypatch) -> None: + discovered = DiscoveredModel( + provider_name="openai", + model_id="chat-capable-model", + credential_name="OPENAI_API_KEY", + chat_base_url="https://api.openai.com/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=False, + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([discovered], []), + ) + existing = ModelAgent( + "openai_chat_capable_model", + discovered.model_id, + base_url=discovered.chat_base_url, + provider_name=discovered.provider_name, + tags=("discovered", "chat", "tool_call:multi"), + ) + orchestrator = TaskOrchestrator([existing]) + + result = _auto_discover_runtime_agents(orchestrator) + + assert result == {"added": [], "updated": [existing.id]} + assert orchestrator.candidates[0].disabled is True + + def test_auto_discovery_activates_a_free_vision_model_but_free_pool_excludes_it( monkeypatch, ) -> None: diff --git a/tests/test_chat_capability.py b/tests/test_chat_capability.py index 8090677b1..7bff1294e 100644 --- a/tests/test_chat_capability.py +++ b/tests/test_chat_capability.py @@ -14,6 +14,7 @@ is_general_chat_candidate, is_general_chat_agent_model_id, ) +from contextual_orchestrator.orchestrator import ModelAgent, _is_general_chat_agent # noqa: E402 @pytest.mark.parametrize( @@ -94,3 +95,14 @@ def test_unproven_tool_call_parallelism_keeps_existing_eligibility() -> None: """No tool-call evidence neither adds nor removes eligibility.""" assert is_general_chat_candidate("vendor/model", supports_parallel_tool_calls=None) is True assert is_general_chat_candidate("vendor/model", supports_parallel_tool_calls=True) is True + + +def test_conflicting_tool_call_tags_fail_closed() -> None: + """Malformed operator tags cannot override explicit single-call evidence.""" + agent = ModelAgent( + "conflicting_tool_agent", + "vendor/model", + tags=("tool_call:multi", "tool_call:single"), + ) + + assert _is_general_chat_agent(agent) is False diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 2d993a74b..91f94ecb6 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -218,6 +218,19 @@ def test_configured_gateway_preserves_consensus_supported_parameters() -> None: ] +def test_configured_gateway_parameter_consensus_ignores_order_and_case() -> None: + payload = {"data": [{"id": "tool-model"}]} + merged = _merge_configured_gateway_metadata( + payload, + {"data": [ + {"model_name": "tool-model", "model_info": {"supported_parameters": ["tools", "PARALLEL_TOOL_CALLS"]}}, + {"model_name": "tool-model", "model_info": {"supported_parameters": ["parallel_tool_calls", "tools"]}}, + ]}, + ) + + assert merged["data"][0]["supported_parameters"] == ["parallel_tool_calls", "tools"] + + def test_configured_gateway_withholds_conflicting_supported_parameters() -> None: payload = {"data": [{"id": "tool-model"}]} merged = _merge_configured_gateway_metadata( @@ -2484,6 +2497,9 @@ def test_tool_call_parallelism_from_error_recognizes_single_tool_rejection() -> _tool_call_parallelism_from_error({"error": {"message": "Invalid API key"}}) is None ) + assert _tool_call_parallelism_from_error( + {"error": {"message": "parallel_tool_calls must be a boolean"}} + ) is None assert _tool_call_parallelism_from_error("only supports one tool call at a time.") is False assert _tool_call_parallelism_from_error({"error": {}}) is None @@ -2519,7 +2535,9 @@ def test_probe_discovered_model_tool_call_capability_success_returns_true() -> N ) response.code = 200 with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", return_value=response + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", return_value=response ) as mocked: result = probe_discovered_model_tool_call_capability(discovered, timeout=5.0) assert result is True @@ -2564,7 +2582,9 @@ def test_probe_discovered_model_tool_call_capability_success_without_two_calls_r ) response.code = 200 with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", return_value=response + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", return_value=response ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None @@ -2596,7 +2616,9 @@ def test_probe_discovered_model_tool_call_capability_single_tool_400_returns_fal discovered.chat_base_url, 400, "Bad Request", {}, BytesIO(body) ) with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=exc + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", side_effect=exc ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is False @@ -2611,7 +2633,10 @@ def test_probe_returns_none_on_network_error() -> None: """Network failures are not evidence one way or the other.""" discovered = _single_tool_discovered_model() with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=urllib.error.URLError("timeout") + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", + side_effect=urllib.error.URLError("timeout"), ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None @@ -2646,7 +2671,9 @@ def test_probe_returns_none_for_non_400_http_error() -> None: discovered.chat_base_url, 500, "Internal Error", {}, BytesIO(b"oops") ) with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=exc + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", side_effect=exc ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None @@ -2658,7 +2685,9 @@ def test_probe_reads_non_json_400_body_as_plain_message() -> None: discovered.chat_base_url, 400, "Bad Request", {}, BytesIO(b"only supports single tool-calls") ) with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( - "urllib.request.urlopen", side_effect=exc + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", side_effect=exc ): assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is False From b60401e571e2c7c5f40af69ce63ad5a1e2fceb28 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 20:50:18 +0900 Subject: [PATCH 06/13] fix(discovery): recover capability-blocked runtime agents --- contextual_orchestrator/__main__.py | 84 +++++++++++++++++++++++++--- tests/test_auto_discovery_server.py | 87 +++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 8 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 8efbc9d37..45fc57bd7 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -402,6 +402,33 @@ def _discover_models_command(argv: list[str]) -> None: raise SystemExit(1) +_DISCOVERY_CAPABILITY_BLOCKED_TAG = "discovery:blocked:capability" +_DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG = ( + "discovery:blocked:capability:preserve-disabled" +) + + +def _discovered_tool_call_tags(model: DiscoveredModel) -> tuple[str, ...]: + """Return the discovery-derived tool-call tags for one model.""" + if model.supports_parallel_tool_calls is True: + return ("tool_call:multi",) + if model.supports_parallel_tool_calls is False: + return ("tool_call:single",) + return () + + +def _refresh_discovered_tool_call_tags( + tags: tuple[str, ...], + model: DiscoveredModel, +) -> tuple[str, ...]: + """Replace stale tool-call evidence with the current discovery result.""" + return tuple( + dict.fromkeys( + tag for tag in tags if not tag.startswith("tool_call:") + ) + ) + _discovered_tool_call_tags(model) + + def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, list[str]]: """Discover and activate routable chat models without runtime env transport. @@ -426,14 +453,39 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l for model in chat_models: existing = existing_by_id.get(agent_id_for(model)) routable = is_routable_discovered_model(model) + capability_blocked = model.supports_parallel_tool_calls is False if existing is None: - agents.append(replace(agent_from_discovered(model), disabled=not routable)) + agent = replace( + agent_from_discovered( + replace(model, supports_parallel_tool_calls=None) + ), + disabled=not routable, + ) + tags = _refresh_discovered_tool_call_tags(agent.tags, model) + if capability_blocked: + tags = (*tags, _DISCOVERY_CAPABILITY_BLOCKED_TAG) + agent = replace(agent, tags=tuple(dict.fromkeys(tags))) + agents.append(agent) elif "discovered" not in existing.tags: continue elif not routable: - tags = (*existing.tags, "spend:blocked") - if existing.disabled and "spend:blocked" not in existing.tags: - tags = (*tags, "spend:blocked:preserve-disabled") + tags = tuple( + tag + for tag in _refresh_discovered_tool_call_tags(existing.tags, model) + if tag + not in { + _DISCOVERY_CAPABILITY_BLOCKED_TAG, + _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG, + } + ) + if not model.spend_admitted: + tags = (*tags, "spend:blocked") + if existing.disabled and "spend:blocked" not in existing.tags: + tags = (*tags, "spend:blocked:preserve-disabled") + if capability_blocked: + tags = (*tags, _DISCOVERY_CAPABILITY_BLOCKED_TAG) + if existing.disabled and _DISCOVERY_CAPABILITY_BLOCKED_TAG not in existing.tags: + tags = (*tags, _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG) agents.append( replace( existing, @@ -441,18 +493,34 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l tags=tuple(dict.fromkeys(tags)), ) ) - elif "spend:blocked" in existing.tags: + elif ( + "spend:blocked" in existing.tags + or _DISCOVERY_CAPABILITY_BLOCKED_TAG in existing.tags + ): agents.append( replace( existing, - disabled="spend:blocked:preserve-disabled" in existing.tags, + disabled=( + "spend:blocked:preserve-disabled" in existing.tags + or _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG in existing.tags + ), tags=tuple( tag - for tag in existing.tags - if tag not in {"spend:blocked", "spend:blocked:preserve-disabled"} + for tag in _refresh_discovered_tool_call_tags(existing.tags, model) + if tag + not in { + "spend:blocked", + "spend:blocked:preserve-disabled", + _DISCOVERY_CAPABILITY_BLOCKED_TAG, + _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG, + } ), ) ) + else: + refreshed_tags = _refresh_discovered_tool_call_tags(existing.tags, model) + if refreshed_tags != existing.tags: + agents.append(replace(existing, tags=refreshed_tags)) result = ( orchestrator.sync_discovered_agents(agents) if agents diff --git a/tests/test_auto_discovery_server.py b/tests/test_auto_discovery_server.py index b8dee2471..10599dba2 100644 --- a/tests/test_auto_discovery_server.py +++ b/tests/test_auto_discovery_server.py @@ -527,6 +527,93 @@ def test_auto_discovery_recovers_model_first_discovered_while_spend_blocked( assert "spend:blocked" not in orchestrator.candidates[0].tags +def test_auto_discovery_recovers_model_first_discovered_while_capability_blocked( + monkeypatch, +) -> None: + blocked = DiscoveredModel( + provider_name="openrouter", + model_id="provider/parallel", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=False, + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([blocked], []), + ) + orchestrator = TaskOrchestrator([], allow_empty_agents=True) + + _auto_discover_runtime_agents(orchestrator) + + initially_blocked = orchestrator.candidates[0] + assert initially_blocked.disabled is True + assert "tool_call:single" in initially_blocked.tags + assert "discovery:blocked:capability" in initially_blocked.tags + + recovered = replace(blocked, supports_parallel_tool_calls=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([recovered], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + recovered_agent = orchestrator.candidates[0] + assert recovered_agent.disabled is False + assert "tool_call:multi" in recovered_agent.tags + assert "tool_call:single" not in recovered_agent.tags + assert "discovery:blocked:capability" not in recovered_agent.tags + + +def test_auto_discovery_preserves_operator_disable_across_capability_recovery( + monkeypatch, +) -> None: + blocked = DiscoveredModel( + provider_name="openrouter", + model_id="provider/parallel", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=False, + ) + existing = ModelAgent( + "openrouter_provider_parallel", + blocked.model_id, + provider_name="openrouter", + tags=("discovered", "chat"), + disabled=True, + ) + orchestrator = TaskOrchestrator([existing], allow_empty_agents=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([blocked], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + blocked_agent = orchestrator.candidates[0] + assert blocked_agent.disabled is True + assert "discovery:blocked:capability" in blocked_agent.tags + assert "discovery:blocked:capability:preserve-disabled" in blocked_agent.tags + + recovered = replace(blocked, supports_parallel_tool_calls=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([recovered], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + recovered_agent = orchestrator.candidates[0] + assert recovered_agent.disabled is True + assert "tool_call:multi" in recovered_agent.tags + assert "discovery:blocked:capability" not in recovered_agent.tags + assert "discovery:blocked:capability:preserve-disabled" not in recovered_agent.tags + + def test_auto_discovery_preserves_operator_disable_across_spend_recovery( monkeypatch, ) -> None: From fce941d502af17999fb57301d87035309ea3d60f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 21:54:50 +0900 Subject: [PATCH 07/13] fix(discovery): avoid sticky blocker transitions --- contextual_orchestrator/__main__.py | 37 ++++-- contextual_orchestrator/model_discovery.py | 9 +- tests/test_auto_discovery_server.py | 124 +++++++++++++++++++++ tests/test_model_discovery.py | 10 ++ 4 files changed, 170 insertions(+), 10 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 45fc57bd7..d18100a3e 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -406,6 +406,21 @@ def _discover_models_command(argv: list[str]) -> None: _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG = ( "discovery:blocked:capability:preserve-disabled" ) +_DISCOVERY_SPEND_BLOCKED_TAG = "spend:blocked" +_DISCOVERY_SPEND_PRESERVE_DISABLED_TAG = "spend:blocked:preserve-disabled" + + +def _should_preserve_operator_disabled_state(existing: ModelAgent) -> bool: + """Return whether a discovery blocker should preserve operator disablement.""" + if not existing.disabled: + return False + tags = set(existing.tags) + if tags & { + _DISCOVERY_SPEND_PRESERVE_DISABLED_TAG, + _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG, + }: + return True + return not bool(tags & {_DISCOVERY_SPEND_BLOCKED_TAG, _DISCOVERY_CAPABILITY_BLOCKED_TAG}) def _discovered_tool_call_tags(model: DiscoveredModel) -> tuple[str, ...]: @@ -479,12 +494,18 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l } ) if not model.spend_admitted: - tags = (*tags, "spend:blocked") - if existing.disabled and "spend:blocked" not in existing.tags: - tags = (*tags, "spend:blocked:preserve-disabled") + tags = (*tags, _DISCOVERY_SPEND_BLOCKED_TAG) + if ( + _should_preserve_operator_disabled_state(existing) + and _DISCOVERY_SPEND_BLOCKED_TAG not in existing.tags + ): + tags = (*tags, _DISCOVERY_SPEND_PRESERVE_DISABLED_TAG) if capability_blocked: tags = (*tags, _DISCOVERY_CAPABILITY_BLOCKED_TAG) - if existing.disabled and _DISCOVERY_CAPABILITY_BLOCKED_TAG not in existing.tags: + if ( + _should_preserve_operator_disabled_state(existing) + and _DISCOVERY_CAPABILITY_BLOCKED_TAG not in existing.tags + ): tags = (*tags, _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG) agents.append( replace( @@ -494,14 +515,14 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l ) ) elif ( - "spend:blocked" in existing.tags + _DISCOVERY_SPEND_BLOCKED_TAG in existing.tags or _DISCOVERY_CAPABILITY_BLOCKED_TAG in existing.tags ): agents.append( replace( existing, disabled=( - "spend:blocked:preserve-disabled" in existing.tags + _DISCOVERY_SPEND_PRESERVE_DISABLED_TAG in existing.tags or _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG in existing.tags ), tags=tuple( @@ -509,8 +530,8 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l for tag in _refresh_discovered_tool_call_tags(existing.tags, model) if tag not in { - "spend:blocked", - "spend:blocked:preserve-disabled", + _DISCOVERY_SPEND_BLOCKED_TAG, + _DISCOVERY_SPEND_PRESERVE_DISABLED_TAG, _DISCOVERY_CAPABILITY_BLOCKED_TAG, _DISCOVERY_CAPABILITY_PRESERVE_DISABLED_TAG, } diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 0835040fe..7a60b8d3a 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -108,9 +108,14 @@ def _tool_call_parallelism_from_error(error_payload: Any) -> bool | None: else: return None text = message.casefold() + single_tool_limit_patterns = ( + r"\bonly supports?\s+(?:a\s+)?(?:single|one)\s+tool(?:-?calls?)?(?:\s+at\s+(?:once|a\s+time))?\b", + r"\b(?:single|one)\s+tool(?:-?calls?)?\s+at\s+(?:once|a\s+time)\b", + r"\b(?:accepts?|allows?)\s+only\s+(?:a\s+)?(?:single|one)\s+tool(?:-?calls?)?\b", + r"\bmax(?:imum)?\s+of\s+one\s+tool(?:-?calls?)?\b", + ) if ( - "single tool" in text - or "one tool" in text + any(re.search(pattern, text) for pattern in single_tool_limit_patterns) or ( "parallel_tool_calls" in text and any(phrase in text for phrase in ("not supported", "unsupported")) diff --git a/tests/test_auto_discovery_server.py b/tests/test_auto_discovery_server.py index 10599dba2..1810b76f8 100644 --- a/tests/test_auto_discovery_server.py +++ b/tests/test_auto_discovery_server.py @@ -650,6 +650,130 @@ def test_auto_discovery_preserves_operator_disable_across_spend_recovery( assert orchestrator.candidates[0] == existing +def test_auto_discovery_does_not_make_spend_blocker_sticky_after_capability_swap( + monkeypatch, +) -> None: + active = ModelAgent( + "openrouter_provider_parallel", + "provider/parallel", + provider_name="openrouter", + tags=("discovered", "chat"), + ) + spend_blocked = DiscoveredModel( + provider_name="openrouter", + model_id="provider/parallel", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + capabilities=("chat",), + spend_admitted=False, + ) + orchestrator = TaskOrchestrator([active], allow_empty_agents=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([spend_blocked], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + blocked = orchestrator.candidates[0] + assert blocked.disabled is True + assert "spend:blocked" in blocked.tags + assert "spend:blocked:preserve-disabled" not in blocked.tags + + capability_blocked = replace( + spend_blocked, + spend_admitted=True, + supports_parallel_tool_calls=False, + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([capability_blocked], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + swapped = orchestrator.candidates[0] + assert swapped.disabled is True + assert "discovery:blocked:capability" in swapped.tags + assert "discovery:blocked:capability:preserve-disabled" not in swapped.tags + + recovered = replace(capability_blocked, supports_parallel_tool_calls=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([recovered], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + final = orchestrator.candidates[0] + assert final.disabled is False + assert "spend:blocked" not in final.tags + assert "discovery:blocked:capability" not in final.tags + + +def test_auto_discovery_does_not_make_capability_blocker_sticky_after_spend_swap( + monkeypatch, +) -> None: + active = ModelAgent( + "openrouter_provider_parallel", + "provider/parallel", + provider_name="openrouter", + tags=("discovered", "chat"), + ) + capability_blocked = DiscoveredModel( + provider_name="openrouter", + model_id="provider/parallel", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=False, + ) + orchestrator = TaskOrchestrator([active], allow_empty_agents=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([capability_blocked], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + blocked = orchestrator.candidates[0] + assert blocked.disabled is True + assert "discovery:blocked:capability" in blocked.tags + assert "discovery:blocked:capability:preserve-disabled" not in blocked.tags + + spend_blocked = replace( + capability_blocked, + supports_parallel_tool_calls=True, + spend_admitted=False, + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([spend_blocked], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + swapped = orchestrator.candidates[0] + assert swapped.disabled is True + assert "spend:blocked" in swapped.tags + assert "spend:blocked:preserve-disabled" not in swapped.tags + + recovered = replace(spend_blocked, spend_admitted=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([recovered], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + final = orchestrator.candidates[0] + assert final.disabled is False + assert "spend:blocked" not in final.tags + assert "discovery:blocked:capability" not in final.tags + + def test_runtime_auto_discovery_does_not_read_gateway_environment(monkeypatch) -> None: captured = [] monkeypatch.setattr( diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 91f94ecb6..bdab2406e 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -2504,6 +2504,16 @@ def test_tool_call_parallelism_from_error_recognizes_single_tool_rejection() -> assert _tool_call_parallelism_from_error({"error": {}}) is None +def test_tool_call_parallelism_from_error_ignores_generic_one_tool_validation() -> None: + """Generic one-tool validation errors are not evidence of single-tool-only support.""" + assert _tool_call_parallelism_from_error( + {"error": {"message": "At least one tool is required for this request."}} + ) is None + assert _tool_call_parallelism_from_error( + {"error": {"message": "One tool has an invalid schema."}} + ) is None + + def test_probe_discovered_model_tool_call_capability_success_returns_true() -> None: """A 200 probe counts only when the response actually contains both tool calls.""" discovered = _single_tool_discovered_model() From 91e8145981b28702b15b2a8184dda1359e103010 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 23:01:02 +0900 Subject: [PATCH 08/13] fix(discovery): preserve single-tool agent evidence --- contextual_orchestrator/__main__.py | 27 +++++++++---- contextual_orchestrator/model_discovery.py | 4 +- tests/test_auto_discovery_server.py | 31 +++++++++++++++ tests/test_discover_models_cli.py | 44 ++++++++++++++++++++++ tests/test_model_discovery.py | 14 +++++++ 5 files changed, 112 insertions(+), 8 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index d18100a3e..5c2f7c62e 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -408,6 +408,8 @@ def _discover_models_command(argv: list[str]) -> None: ) _DISCOVERY_SPEND_BLOCKED_TAG = "spend:blocked" _DISCOVERY_SPEND_PRESERVE_DISABLED_TAG = "spend:blocked:preserve-disabled" +_DISCOVERY_TOOL_CALL_SINGLE_TAG = "discovery:tool_call:single" +_DISCOVERY_TOOL_CALL_MULTI_TAG = "discovery:tool_call:multi" def _should_preserve_operator_disabled_state(existing: ModelAgent) -> bool: @@ -426,9 +428,9 @@ def _should_preserve_operator_disabled_state(existing: ModelAgent) -> bool: def _discovered_tool_call_tags(model: DiscoveredModel) -> tuple[str, ...]: """Return the discovery-derived tool-call tags for one model.""" if model.supports_parallel_tool_calls is True: - return ("tool_call:multi",) + return ("tool_call:multi", _DISCOVERY_TOOL_CALL_MULTI_TAG) if model.supports_parallel_tool_calls is False: - return ("tool_call:single",) + return ("tool_call:single", _DISCOVERY_TOOL_CALL_SINGLE_TAG) return () @@ -437,11 +439,22 @@ def _refresh_discovered_tool_call_tags( model: DiscoveredModel, ) -> tuple[str, ...]: """Replace stale tool-call evidence with the current discovery result.""" - return tuple( - dict.fromkeys( - tag for tag in tags if not tag.startswith("tool_call:") - ) - ) + _discovered_tool_call_tags(model) + hidden_tags = { + _DISCOVERY_TOOL_CALL_SINGLE_TAG, + _DISCOVERY_TOOL_CALL_MULTI_TAG, + } + has_discovery_provenance = bool(hidden_tags.intersection(tags)) + refreshed = [] + for tag in tags: + if tag in hidden_tags: + continue + if ( + has_discovery_provenance + and tag in {"tool_call:single", "tool_call:multi"} + ): + continue + refreshed.append(tag) + return tuple(dict.fromkeys(refreshed)) + _discovered_tool_call_tags(model) def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, list[str]]: diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 7a60b8d3a..ed43e2b28 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -1689,7 +1689,9 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> if not any( capability not in {"chat", "response_format"} for capability in discovered.capabilities - ) and not is_discovered_chat_candidate(discovered): + ) and not is_discovered_chat_candidate( + replace(discovered, supports_parallel_tool_calls=None) + ): raise ValueError("model is not eligible for a general chat agent") return ModelAgent( id=agent_id_for(discovered), diff --git a/tests/test_auto_discovery_server.py b/tests/test_auto_discovery_server.py index 1810b76f8..5f9de061f 100644 --- a/tests/test_auto_discovery_server.py +++ b/tests/test_auto_discovery_server.py @@ -650,6 +650,37 @@ def test_auto_discovery_preserves_operator_disable_across_spend_recovery( assert orchestrator.candidates[0] == existing +def test_auto_discovery_unknown_capability_keeps_operator_tool_call_override( + monkeypatch, +) -> None: + discovered = DiscoveredModel( + provider_name="openrouter", + model_id="provider/parallel", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=None, + ) + existing = ModelAgent( + "openrouter_provider_parallel", + discovered.model_id, + provider_name="openrouter", + tags=("discovered", "chat", "tool_call:single", "operator-tag"), + ) + orchestrator = TaskOrchestrator([existing], allow_empty_agents=True) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([discovered], []), + ) + + _auto_discover_runtime_agents(orchestrator) + + refreshed = orchestrator.candidates[0] + assert "tool_call:single" in refreshed.tags + assert "operator-tag" in refreshed.tags + + def test_auto_discovery_does_not_make_spend_blocker_sticky_after_capability_swap( monkeypatch, ) -> None: diff --git a/tests/test_discover_models_cli.py b/tests/test_discover_models_cli.py index 3b369fac7..280bc9cc8 100644 --- a/tests/test_discover_models_cli.py +++ b/tests/test_discover_models_cli.py @@ -326,6 +326,50 @@ def urlopen(request, timeout=None): assert any(agent.id == "openai_gpt_5_5" for agent in reloaded.candidates) +def test_discover_models_persists_single_tool_chat_rows_to_agents_db(tmp_path) -> None: + from contextual_orchestrator import TaskOrchestrator + from contextual_orchestrator.model_discovery import DiscoveredModel + from contextual_orchestrator.orchestrator import ModelAgent + + set_backend(InMemoryCredentialBackend()) + db_path = str(tmp_path / "pool.db") + stdout = StringIO() + discovered = DiscoveredModel( + provider_name="nvidia_nim", + model_id="generic/single-tool-model", + credential_name="NVIDIA_NIM_API_KEY", + chat_base_url="https://integrate.api.nvidia.com/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=False, + is_free=True, + ) + + try: + with ( + patch.object( + sys, + "argv", + ["contextual-orchestrator", "discover-models", "--agents-db", db_path], + ), + patch.object(sys, "stdout", stdout), + patch( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([discovered], []), + ), + ): + main() + finally: + set_backend(None) + + report = json.loads(stdout.getvalue()) + assert report["discovered_count"] == 1 + reloaded = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) + stored = next(agent for agent in reloaded.candidates if agent.model == discovered.model_id) + assert stored.disabled is True + assert "tool_call:single" in stored.tags + + def test_discover_models_closes_temporary_agents_db_orchestrator(tmp_path) -> None: from contextual_orchestrator import TaskOrchestrator from contextual_orchestrator import __main__ as cli diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index bdab2406e..f383888e0 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -2279,6 +2279,20 @@ def test_agent_from_discovered_preserves_explicit_privacy_evidence() -> None: "privacy:no_training", "privacy:retention_only", } <= set(agent_from_discovered(discovered).tags) + + +def test_agent_from_discovered_allows_single_tool_chat_rows_as_disabled_agents() -> None: + """Single-tool chat evidence blocks general routing, not durable representation.""" + discovered = _single_tool_discovered_model() + + agent = agent_from_discovered(discovered) + + assert agent.disabled is True + assert agent.model == discovered.model_id + assert "chat" in agent.tags + assert "tool_call:single" in agent.tags + + def test_response_format_metadata_does_not_make_non_chat_model_eligible() -> None: discovered = DiscoveredModel( provider_name="openai", From 61607bafedb982c24acd894378af762d17fb0afa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 23:45:44 +0900 Subject: [PATCH 09/13] fix(discovery): preserve tool capability provenance --- contextual_orchestrator/__main__.py | 24 +++++++------------ contextual_orchestrator/model_discovery.py | 19 +++++++++------ contextual_orchestrator/provider_bootstrap.py | 9 ++----- tests/test_discover_models_cli.py | 15 ++++++++++++ 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 5c2f7c62e..2407d5ca6 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -15,12 +15,16 @@ from .kv_config import InMemoryConfigStore from .model_discovery import ( CONFIGURED_GATEWAY_CREDENTIAL_NAME, + DISCOVERY_TOOL_CALL_MULTI_TAG, + DISCOVERY_TOOL_CALL_SINGLE_TAG, + DiscoveredModel, PROVIDER_MODEL_SOURCES, ProviderModelSource, agent_from_discovered, agent_id_for, configured_gateway_source, discover_all_models, + discovery_tool_call_tags, free_discovered_models, general_free_serving_candidates, is_discovered_chat_candidate, @@ -31,6 +35,7 @@ from .orchestrator import ( CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1, MAX_LOCAL_CONCURRENCY, + ModelAgent, ModelClient, TaskOrchestrator, load_agents, @@ -408,10 +413,6 @@ def _discover_models_command(argv: list[str]) -> None: ) _DISCOVERY_SPEND_BLOCKED_TAG = "spend:blocked" _DISCOVERY_SPEND_PRESERVE_DISABLED_TAG = "spend:blocked:preserve-disabled" -_DISCOVERY_TOOL_CALL_SINGLE_TAG = "discovery:tool_call:single" -_DISCOVERY_TOOL_CALL_MULTI_TAG = "discovery:tool_call:multi" - - def _should_preserve_operator_disabled_state(existing: ModelAgent) -> bool: """Return whether a discovery blocker should preserve operator disablement.""" if not existing.disabled: @@ -425,23 +426,14 @@ def _should_preserve_operator_disabled_state(existing: ModelAgent) -> bool: return not bool(tags & {_DISCOVERY_SPEND_BLOCKED_TAG, _DISCOVERY_CAPABILITY_BLOCKED_TAG}) -def _discovered_tool_call_tags(model: DiscoveredModel) -> tuple[str, ...]: - """Return the discovery-derived tool-call tags for one model.""" - if model.supports_parallel_tool_calls is True: - return ("tool_call:multi", _DISCOVERY_TOOL_CALL_MULTI_TAG) - if model.supports_parallel_tool_calls is False: - return ("tool_call:single", _DISCOVERY_TOOL_CALL_SINGLE_TAG) - return () - - def _refresh_discovered_tool_call_tags( tags: tuple[str, ...], model: DiscoveredModel, ) -> tuple[str, ...]: """Replace stale tool-call evidence with the current discovery result.""" hidden_tags = { - _DISCOVERY_TOOL_CALL_SINGLE_TAG, - _DISCOVERY_TOOL_CALL_MULTI_TAG, + DISCOVERY_TOOL_CALL_SINGLE_TAG, + DISCOVERY_TOOL_CALL_MULTI_TAG, } has_discovery_provenance = bool(hidden_tags.intersection(tags)) refreshed = [] @@ -454,7 +446,7 @@ def _refresh_discovered_tool_call_tags( ): continue refreshed.append(tag) - return tuple(dict.fromkeys(refreshed)) + _discovered_tool_call_tags(model) + return tuple(dict.fromkeys(refreshed)) + discovery_tool_call_tags(model) def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, list[str]]: diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index ed43e2b28..8fff90bd5 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -62,6 +62,17 @@ # safe to send on every request, authenticated or not. _HTTP_USER_AGENT = "contextual-orchestrator/0.2.0 (+https://github.com/ContextualWisdomLab/contextual-orchestrator)" _CAPABILITY_NAMES = {"embeddings": "embedding"} +DISCOVERY_TOOL_CALL_SINGLE_TAG = "discovery:tool_call:single" +DISCOVERY_TOOL_CALL_MULTI_TAG = "discovery:tool_call:multi" + + +def discovery_tool_call_tags(model: DiscoveredModel) -> tuple[str, ...]: + """Return public capability evidence with its discovery-ownership marker.""" + if model.supports_parallel_tool_calls is True: + return ("tool_call:multi", DISCOVERY_TOOL_CALL_MULTI_TAG) + if model.supports_parallel_tool_calls is False: + return ("tool_call:single", DISCOVERY_TOOL_CALL_SINGLE_TAG) + return () def _parallel_tool_call_evidence(supported_parameters: list[Any]) -> bool | None: @@ -1709,13 +1720,7 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> *(f"capability:{value}" for value in discovered.capabilities), *(f"input:{value}" for value in discovered.input_modalities), *(f"output:{value}" for value in discovered.output_modalities), - *( - ("tool_call:multi",) - if discovered.supports_parallel_tool_calls is True - else ("tool_call:single",) - if discovered.supports_parallel_tool_calls is False - else () - ), + *discovery_tool_call_tags(discovered), ), priority=priority, disabled=True, diff --git a/contextual_orchestrator/provider_bootstrap.py b/contextual_orchestrator/provider_bootstrap.py index b705266a2..255341cd1 100644 --- a/contextual_orchestrator/provider_bootstrap.py +++ b/contextual_orchestrator/provider_bootstrap.py @@ -33,6 +33,7 @@ agent_from_discovered, agent_id_for, discover_all_models, + discovery_tool_call_tags, privacy_tags_for_discovered, is_routable_discovered_model, refresh_price_book, @@ -193,13 +194,7 @@ def serving_tags_for_discovered(model: DiscoveredModel) -> tuple[str, ...]: *(f"capability:{value}" for value in model.capabilities), *(f"input:{value}" for value in model.input_modalities), *(f"output:{value}" for value in model.output_modalities), - *( - ("tool_call:multi",) - if model.supports_parallel_tool_calls is True - else ("tool_call:single",) - if model.supports_parallel_tool_calls is False - else () - ), + *discovery_tool_call_tags(model), ) ) ) diff --git a/tests/test_discover_models_cli.py b/tests/test_discover_models_cli.py index 280bc9cc8..0a2d30bca 100644 --- a/tests/test_discover_models_cli.py +++ b/tests/test_discover_models_cli.py @@ -6,6 +6,7 @@ import os import sys import urllib.parse +from dataclasses import replace from io import StringIO from pathlib import Path from unittest.mock import patch @@ -368,6 +369,20 @@ def test_discover_models_persists_single_tool_chat_rows_to_agents_db(tmp_path) - stored = next(agent for agent in reloaded.candidates if agent.model == discovered.model_id) assert stored.disabled is True assert "tool_call:single" in stored.tags + assert "discovery:tool_call:single" in stored.tags + + unknown = replace(discovered, supports_parallel_tool_calls=None) + with patch( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([unknown], []), + ): + from contextual_orchestrator.__main__ import _auto_discover_runtime_agents + + _auto_discover_runtime_agents(reloaded) + + refreshed = next(agent for agent in reloaded.candidates if agent.model == discovered.model_id) + assert "tool_call:single" not in refreshed.tags + assert "discovery:tool_call:single" not in refreshed.tags def test_discover_models_closes_temporary_agents_db_orchestrator(tmp_path) -> None: From a68fc5c12f879c6f620d1fff4c85bac721517aa5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 11:31:03 +0900 Subject: [PATCH 10/13] fix(discovery): reject echoed tool probe definitions --- contextual_orchestrator/model_discovery.py | 41 +++++++++++----------- tests/test_model_discovery.py | 14 ++++++++ 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index 8fff90bd5..f21ade44d 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -141,27 +141,28 @@ def _response_contains_parallel_probe_tool_calls(payload: Any) -> bool: if not isinstance(payload, dict): return False seen: set[str] = set() - - def collect(value: Any) -> None: - if isinstance(value, dict): - if value.get("type") == "function": - function = value.get("function") - if isinstance(function, dict): - name = function.get("name") - if isinstance(name, str) and name in {"probe_a", "probe_b"}: - seen.add(name) - if value.get("type") == "function_call": - name = value.get("name") - if isinstance(name, str) and name in {"probe_a", "probe_b"}: + choices = payload.get("choices") + if isinstance(choices, list): + for choice in choices: + message = choice.get("message") if isinstance(choice, dict) else None + tool_calls = message.get("tool_calls") if isinstance(message, dict) else None + if not isinstance(tool_calls, list): + continue + for tool_call in tool_calls: + if not isinstance(tool_call, dict) or tool_call.get("type") != "function": + continue + function = tool_call.get("function") + name = function.get("name") if isinstance(function, dict) else None + if name in {"probe_a", "probe_b"}: seen.add(name) - for child in value.values(): - collect(child) - return - if isinstance(value, list): - for child in value: - collect(child) - - collect(payload) + output = payload.get("output") + if isinstance(output, list): + for item in output: + if not isinstance(item, dict) or item.get("type") != "function_call": + continue + name = item.get("name") + if name in {"probe_a", "probe_b"}: + seen.add(name) return seen == {"probe_a", "probe_b"} diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index f383888e0..6a8924f6d 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -2630,6 +2630,20 @@ def test_response_contains_parallel_probe_tool_calls_recognizes_responses_api_sh ) +def test_parallel_probe_does_not_treat_echoed_tool_definitions_as_calls() -> None: + echoed_tools = [ + {"type": "function", "function": {"name": name}} + for name in ("probe_a", "probe_b") + ] + + assert not _response_contains_parallel_probe_tool_calls( + { + "request": {"tools": echoed_tools}, + "choices": [{"message": {"content": "no tool calls"}}], + } + ) + + def test_probe_discovered_model_tool_call_capability_single_tool_400_returns_false() -> None: """NIM's explicit single-tool 400 is captured as negative evidence.""" discovered = _single_tool_discovered_model() From bd422bdaa90f517c35e9f587e2d3872b859601e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 23:04:04 +0000 Subject: [PATCH 11/13] fix(972): address Devin review findings on the merge (operator tool-call tag loss, unbounded probe read) Two real findings from Devin's review of a1103da4 (the main-merge commit for this PR): 1. _refresh_discovered_tool_call_tags (contextual_orchestrator/__main__.py) conflated "a discovery marker for EITHER polarity has ever been seen" with "this specific visible tag is discovery-owned". Once discovery supplied evidence once, a later refresh would strip an operator's pre-existing tool_call:single/multi override too, even though it predated any discovery marker and belongs to the opposite polarity. Now only the visible tag paired with its own hidden discovery marker is treated as discovery-owned; an operator-authored tag of the other polarity survives evidence going from known back to unknown. 2. probe_discovered_model_tool_call_capability (model_discovery.py) read the entire provider response body with an unbounded response.read()/ exc.read(). Applied the same bounded-read-then-check pattern already used elsewhere in this module (MAX_DISCOVERY_RESPONSE_BYTES): an oversized body now returns None (ambiguous evidence) instead of buffering an unbounded amount of memory. Added regression tests for both: - test_auto_discovery_preserves_operator_tool_call_override_when_evidence_goes_stale - test_probe_discovered_model_tool_call_capability_rejects_oversized_response - test_probe_discovered_model_tool_call_capability_rejects_oversized_400_body The third finding (ADR 0042 already cites its research PDFs under docs/papers/) was informational, no action needed. Verified: targeted test files pass (194 passed), interrogate 100% on touched files, git diff --check clean; full suite rerun in progress. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- contextual_orchestrator/__main__.py | 39 +++++++++++-------- contextual_orchestrator/model_discovery.py | 16 +++++++- tests/test_auto_discovery_server.py | 45 ++++++++++++++++++++++ tests/test_model_discovery.py | 36 +++++++++++++++++ 4 files changed, 119 insertions(+), 17 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 7cabbabec..fa7bb7afe 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -703,22 +703,31 @@ def _refresh_discovered_tool_call_tags( tags: tuple[str, ...], model: DiscoveredModel, ) -> tuple[str, ...]: - """Replace stale tool-call evidence with the current discovery result.""" - hidden_tags = { - DISCOVERY_TOOL_CALL_SINGLE_TAG, - DISCOVERY_TOOL_CALL_MULTI_TAG, + """Replace stale tool-call evidence with the current discovery result. + + Only the specific visible tag paired with a discovery marker currently + present is discovery-owned. A prior discovery pass always writes its + visible tag and hidden marker together (see + :func:`~contextual_orchestrator.model_discovery.discovery_tool_call_tags`), + so a marker for one polarity (e.g. ``discovery:tool_call:multi``) never + implies ownership of the *other* visible tag. This preserves an + operator-authored ``tool_call:single``/``tool_call:multi`` override that + predates any discovery evidence: it is never removed just because + discovery later supplies -- and then withdraws -- unrelated evidence. + """ + discovery_owned_visible_tag = { + DISCOVERY_TOOL_CALL_SINGLE_TAG: "tool_call:single", + DISCOVERY_TOOL_CALL_MULTI_TAG: "tool_call:multi", } - has_discovery_provenance = bool(hidden_tags.intersection(tags)) - refreshed = [] - for tag in tags: - if tag in hidden_tags: - continue - if ( - has_discovery_provenance - and tag in {"tool_call:single", "tool_call:multi"} - ): - continue - refreshed.append(tag) + hidden_tags = set(discovery_owned_visible_tag) + owned_visible_tags = { + discovery_owned_visible_tag[tag] for tag in tags if tag in hidden_tags + } + refreshed = [ + tag + for tag in tags + if tag not in hidden_tags and tag not in owned_visible_tags + ] return tuple(dict.fromkeys(refreshed)) + discovery_tool_call_tags(model) diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index f32a1fe77..97dd1bde4 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -186,6 +186,12 @@ def probe_discovered_model_tool_call_capability( This is real runtime evidence, not a model-name heuristic. It is deliberately separate from :func:`discover_all_models` so callers decide when the extra latency and token cost are justified. + + The response body (success or 400 error) is capped at + :data:`MAX_DISCOVERY_RESPONSE_BYTES`, the same bounded-read-then-check + pattern used elsewhere in this module, so an oversized or misbehaving + provider response cannot exhaust memory; an oversized body is treated as + ambiguous evidence (``None``) rather than raising. """ api_key = get_credential(discovered.credential_name) if not api_key: @@ -251,11 +257,17 @@ def probe_discovered_model_tool_call_capability( request = urllib.request.Request(url, data=data, headers=headers, method="POST") try: with client._open_provider(request, destination, timeout=timeout) as response: - body = response.read().decode("utf-8", errors="replace") + raw = response.read(MAX_DISCOVERY_RESPONSE_BYTES + 1) + if len(raw) > MAX_DISCOVERY_RESPONSE_BYTES: + return None + body = raw.decode("utf-8", errors="replace") except urllib.error.HTTPError as exc: if exc.code != 400: return None - body = exc.read().decode("utf-8", errors="replace") + raw = exc.read(MAX_DISCOVERY_RESPONSE_BYTES + 1) + if len(raw) > MAX_DISCOVERY_RESPONSE_BYTES: + return None + body = raw.decode("utf-8", errors="replace") try: error_payload = json.loads(body) except json.JSONDecodeError: diff --git a/tests/test_auto_discovery_server.py b/tests/test_auto_discovery_server.py index dd6357a50..c7c5857a5 100644 --- a/tests/test_auto_discovery_server.py +++ b/tests/test_auto_discovery_server.py @@ -89,6 +89,51 @@ def test_auto_discovery_disables_active_model_after_capability_downgrade(monkeyp assert orchestrator.candidates[0].disabled is True +def test_auto_discovery_preserves_operator_tool_call_override_when_evidence_goes_stale( + monkeypatch, +) -> None: + """An operator's pre-existing tool_call tag outlives discovery evidence that + later arrives and then withdraws again -- it must never be treated as + discovery-owned just because a discovery marker for a *different* tool-call + polarity is present.""" + existing = ModelAgent( + "openai_chat_capable_model", + "chat-capable-model", + base_url="https://api.openai.com/v1", + provider_name="openai", + tags=("discovered", "chat", "tool_call:single"), + ) + orchestrator = TaskOrchestrator([existing]) + + multi_evidence = DiscoveredModel( + provider_name="openai", + model_id="chat-capable-model", + credential_name="OPENAI_API_KEY", + chat_base_url="https://api.openai.com/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=True, + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([multi_evidence], []), + ) + _auto_discover_runtime_agents(orchestrator) + after_multi_evidence = orchestrator.candidates[0] + assert "tool_call:single" in after_multi_evidence.tags + assert "tool_call:multi" in after_multi_evidence.tags + + unknown_evidence = replace(multi_evidence, supports_parallel_tool_calls=None) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([unknown_evidence], []), + ) + _auto_discover_runtime_agents(orchestrator) + after_unknown_evidence = orchestrator.candidates[0] + assert "tool_call:single" in after_unknown_evidence.tags + assert "tool_call:multi" not in after_unknown_evidence.tags + + def test_embedding_only_discovery_keeps_chat_fallbacks(monkeypatch) -> None: embedding = DiscoveredModel( provider_name="configured_gateway", diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 4e2ccc145..e263f0df9 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -31,6 +31,7 @@ from contextual_orchestrator.model_discovery import ( # noqa: E402 PROVIDER_MODEL_SOURCES, DiscoveredModel, + MAX_DISCOVERY_RESPONSE_BYTES, ModelUnitPrice, ProviderDiscoveryError, ProviderModelSource, @@ -2917,6 +2918,41 @@ def test_probe_discovered_model_tool_call_capability_success_returns_true() -> N assert len(payload["tools"]) == 2 +def test_probe_discovered_model_tool_call_capability_rejects_oversized_response() -> None: + """A provider cannot exhaust memory by streaming an unbounded probe body.""" + discovered = _single_tool_discovered_model() + oversized = urllib.request.addinfourl( + BytesIO(b"x" * (MAX_DISCOVERY_RESPONSE_BYTES + 1)), + {"content-type": "application/json"}, + "", + ) + oversized.code = 200 + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", return_value=oversized + ): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None + + +def test_probe_discovered_model_tool_call_capability_rejects_oversized_400_body() -> None: + """An oversized 400 error body is also treated as ambiguous, not evidence.""" + discovered = _single_tool_discovered_model() + exc = urllib.error.HTTPError( + discovered.chat_base_url, + 400, + "Bad Request", + {}, + BytesIO(b"only supports single tool-calls " + b"x" * (MAX_DISCOVERY_RESPONSE_BYTES + 1)), + ) + with patch("contextual_orchestrator.model_discovery.get_credential", return_value="test-key"), patch( + "contextual_orchestrator.model_discovery.ModelClient._validate_provider", return_value=object() + ), patch( + "contextual_orchestrator.model_discovery.ModelClient._open_provider", side_effect=exc + ): + assert probe_discovered_model_tool_call_capability(discovered, timeout=5.0) is None + + @pytest.mark.parametrize( "body", [ From 62d59fc1576129455190a2af7f420101599eb21e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:14:20 +0900 Subject: [PATCH 12/13] test(discovery): preserve matching operator tool-call overrides --- tests/test_discovery_tool_call_ownership.py | 50 +++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_discovery_tool_call_ownership.py diff --git a/tests/test_discovery_tool_call_ownership.py b/tests/test_discovery_tool_call_ownership.py new file mode 100644 index 000000000..b12b8ecc7 --- /dev/null +++ b/tests/test_discovery_tool_call_ownership.py @@ -0,0 +1,50 @@ +"""Regression coverage for operator/discovery tool-call tag ownership.""" + +from dataclasses import replace + +from contextual_orchestrator.__main__ import _refresh_discovered_tool_call_tags +from contextual_orchestrator.model_discovery import DiscoveredModel + + +def _model(parallel: bool | None) -> DiscoveredModel: + return DiscoveredModel( + provider_name="openrouter", + model_id="provider/parallel", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + capabilities=("chat",), + supports_parallel_tool_calls=parallel, + ) + + +def test_same_single_operator_override_survives_discovery_withdrawal() -> None: + """Matching negative discovery evidence must not consume operator authority.""" + operator_tags = ("discovered", "chat", "tool_call:single", "operator-tag") + + with_discovery = _refresh_discovered_tool_call_tags(operator_tags, _model(False)) + after_unknown = _refresh_discovered_tool_call_tags( + with_discovery, + replace(_model(False), supports_parallel_tool_calls=None), + ) + + assert "tool_call:single" in after_unknown + assert "tool_call:multi" not in after_unknown + assert "operator-tag" in after_unknown + assert not any(tag.startswith("discovery:tool_call:") for tag in after_unknown) + + +def test_same_multi_operator_override_survives_discovery_withdrawal() -> None: + """Matching positive discovery evidence must not consume operator authority.""" + operator_tags = ("discovered", "chat", "tool_call:multi", "operator-tag") + + with_discovery = _refresh_discovered_tool_call_tags(operator_tags, _model(True)) + after_unknown = _refresh_discovered_tool_call_tags( + with_discovery, + replace(_model(True), supports_parallel_tool_calls=None), + ) + + assert "tool_call:multi" in after_unknown + assert "tool_call:single" not in after_unknown + assert "operator-tag" in after_unknown + assert not any(tag.startswith("discovery:tool_call:") for tag in after_unknown) From b542003ff8d5080b8a136175f8fe348c6a0a3162 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 23:19:20 +0000 Subject: [PATCH 13/13] fix(972): resolve same-polarity tool-call tag ownership collision Seonghobae pushed test_discovery_tool_call_ownership.py directly to this branch, specifying the desired same-polarity behavior as failing acceptance tests (the different-polarity case was already fixed in bd422bda; this closes the remaining gap Devin's re-review confirmed was distinct). The gap: when an operator's pre-existing tool_call:single/multi tag happens to match discovery's own value, the two are string-identical, so _refresh_discovered_tool_call_tags had no way to tell them apart once discovery's hidden marker got attached -- a later refresh would treat both occurrences as discovery-owned and strip the operator's tag along with discovery's when evidence went stale. Fix: discovery never claims ownership (by adding its hidden marker) of a visible tag that is already present without one. Since the two tags are indistinguishable strings, the only safe rule is "if it was already there unclaimed, it stays unclaimed" -- discovery's matching evidence is still effectively honored (the correct value is already showing), but the marker that would let a later refresh treat it as discovery's own (and thus remove it in the same-polarity case) is never added. Verified by exhaustive trace across every existing and new scenario (different-polarity known->known->unknown, same-polarity known-> known->unknown, fresh-discovery no-operator, discovery value flips, repeated identical discovery evidence) before implementing, then confirmed: tests/test_discovery_tool_call_ownership.py (the new acceptance tests) + test_auto_discovery_server.py + test_model_discovery.py all pass (196 passed), plus the broader chat_capability/provider_bootstrap/ discover_models_cli/provider_catalog_store/multimodal_model_group_http suites (147 passed). interrogate 100% on touched files. git diff --check clean. Full suite rerun in progress. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- contextual_orchestrator/__main__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index fa7bb7afe..0b1634615 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -714,6 +714,15 @@ def _refresh_discovered_tool_call_tags( operator-authored ``tool_call:single``/``tool_call:multi`` override that predates any discovery evidence: it is never removed just because discovery later supplies -- and then withdraws -- unrelated evidence. + + A same-polarity collision (an operator's plain tag already reads the + same value discovery now independently reports) is handled the same + way: discovery never claims ownership -- by adding its hidden marker -- + of a visible tag that is already present without one. The two tags are + string-identical, so there would otherwise be no way to tell them apart + on a later refresh; leaving the marker off means the pre-existing tag + is never mistaken for discovery's own and never gets removed just + because discovery's evidence later goes stale. """ discovery_owned_visible_tag = { DISCOVERY_TOOL_CALL_SINGLE_TAG: "tool_call:single", @@ -728,7 +737,10 @@ def _refresh_discovered_tool_call_tags( for tag in tags if tag not in hidden_tags and tag not in owned_visible_tags ] - return tuple(dict.fromkeys(refreshed)) + discovery_tool_call_tags(model) + new_evidence_tags = discovery_tool_call_tags(model) + if new_evidence_tags and new_evidence_tags[0] in refreshed: + return tuple(dict.fromkeys(refreshed)) + return tuple(dict.fromkeys(refreshed)) + new_evidence_tags def _probe_configured_gateway_structured_chat(