From a62e1ca3559ea2c606012049fd15fd48afdb6fd2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 20:08:47 +0900 Subject: [PATCH 01/33] feat(errors): classify provider failures and enrich telemetry evidence Every upstream model failure used to collapse into one generic internal_error: _send_with_retry/_send_raw_with_retry raised bare RuntimeErrors and the server catch-all mapped everything to 500. Telemetry spans recorded only transport metadata, and orchestration traces omitted which model served a step. - add contextual_orchestrator.provider_errors: upstream HTTP status, network, TLS, and transport failures classify into OpenAI-compatible codes with client status, retryability, and bounded redacted messages (CWE-209); chat/passthrough/stream/batch transports surface the classification; agent failover preserves the final classified cause; server error payloads gain next-step guidance per failure family. - telemetry spans now record GenAI semantic-convention usage counts, served response model, finish reason, request latency, and classified error.type plus upstream status on failures. - route/stream/batch/conduct trace steps carry model, provider, and latency_ms alongside usage. Full suite: 2279 passed. Docstrings 100% (interrogate); statement coverage 100% on provider_errors.py and telemetry.py. --- CHANGELOG.md | 20 ++ contextual_orchestrator/orchestrator.py | 97 +++++- contextual_orchestrator/provider_errors.py | 222 +++++++++++++ contextual_orchestrator/server.py | 52 +++ contextual_orchestrator/telemetry.py | 68 +++- docs/product-technical-gap-baseline.md | 28 ++ tests/test_orchestrator_client_boundaries.py | 9 +- tests/test_provider_error_taxonomy.py | 329 +++++++++++++++++++ tests/test_provider_reliability.py | 4 +- tests/test_telemetry.py | 134 +++++++- tests/test_true_streaming.py | 8 +- 11 files changed, 951 insertions(+), 20 deletions(-) create mode 100644 contextual_orchestrator/provider_errors.py create mode 100644 tests/test_provider_error_taxonomy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e43d559db..50771f6f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,26 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Provider/model failures no longer collapse into a generic `internal_error`. + A typed provider-error taxonomy (`contextual_orchestrator.provider_errors`) + classifies every upstream HTTP status, network, TLS, and transport failure + into OpenAI-compatible error codes (`rate_limit_exceeded`, + `authentication_error`, `model_not_found`, `provider_timeout`, ...) with the + client status, retryability, and one bounded redacted message (CWE-209). + Chat, passthrough, stream, and batch transports all surface the classified + cause; a fully-failed agent pool surfaces the final classified failure + after measured failover instead of an opaque collapse. Server error + payloads carry actionable next-step guidance per failure family. +- Telemetry spans now carry concrete GenAI semantic-convention evidence: + `gen_ai.usage.input_tokens/output_tokens/total_tokens` from provider-reported + counts, served `gen_ai.response.model`, `gen_ai.response.finish_reasons`, + request latency, and classified `error.type` plus upstream status on + failures — replacing exception-class-only error labels. +- Orchestration traces now include per-step telemetry evidence: streamed, + batched, routed, and conducted steps record `model`, `provider`, and + `latency_ms` alongside usage so workflow runs answer which model served a + step, how long it took, and what it cost. + - Make per-request budget checks constant time while preserving exact parity with full spend analytics across persisted, replaced, estimated, and provider-reported workflow runs. diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 91cd05062..4b2e07cfe 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -39,7 +39,13 @@ from .conventions import require_object_name from .credentials import NotConfigured, get_credential from .model_group import ModelGroupRouter, canonical_group_name -from .telemetry import inject_trace_context, traced +from .provider_errors import ProviderUpstreamError, classify_provider_failure +from .telemetry import ( + annotate_current_span, + inject_trace_context, + record_provider_usage, + traced, +) from .pii_protection import ( DEFAULT_PII_KEY_NAME, ENCRYPTED_FIELDS_KEY, @@ -832,6 +838,31 @@ def is_transient_error(exc: BaseException) -> bool: return False +def _record_provider_response_telemetry(data: Any, started_monotonic: float) -> None: + """Annotate the active provider span with one response's concrete evidence. + + Records GenAI semantic-convention usage counts, the served model name, + the finish reason, and request latency so traces carry real per-call + telemetry instead of transport metadata alone. + """ + if not isinstance(data, dict): + return + attributes: dict[str, Any] = { + "contextual_orchestrator.latency_ms": round((time.monotonic() - started_monotonic) * 1000, 2) + } + served_model = data.get("model") + if isinstance(served_model, str) and served_model: + attributes["gen_ai.response.model"] = served_model + choices = data.get("choices") + finish_reason = ( + choices[0].get("finish_reason") if isinstance(choices, list) and isinstance(choices[0], dict) else None + ) + if isinstance(finish_reason, str) and finish_reason: + attributes["gen_ai.response.finish_reasons"] = finish_reason + annotate_current_span(attributes) + record_provider_usage(data.get("usage")) + + class ModelClient: """Small chat-completions client with retry, backoff, and mock support.""" @@ -1184,7 +1215,9 @@ def _send_with_retry( raise _provider_tool_execution_stopped(agent) from None if isinstance(last_error, ProviderResponseError): raise last_error - raise RuntimeError(f"provider {agent.id} request failed") from None + # Classify instead of collapsing: a 401/404/429 upstream failure is + # caller-actionable and must not surface as one opaque internal error. + raise classify_provider_failure(last_error, agent_id=agent.id, model=agent.model) def _retry_limit(self, agent: ModelAgent) -> int: """Return a retry budget without multiplying an expensive local queue by default.""" @@ -1215,6 +1248,7 @@ def _send( headers=headers, method="POST", ) + started = time.monotonic() opened = ( self._open_provider(request, destination) if timeout is None @@ -1222,6 +1256,7 @@ def _send( ) with opened as response: data = json.loads(response.read().decode("utf-8")) + _record_provider_response_telemetry(data, started) usage = data.get("usage") if isinstance(usage, dict): self._local.usage = usage @@ -1241,7 +1276,6 @@ def _response_content(agent: ModelAgent, data: dict[str, Any]) -> str: "for mlx-lm set chat_template_args={\"enable_thinking\": false} or increase max_output_tokens" ) raise ProviderResponseError(f"provider {agent.id} response did not contain assistant content") - @staticmethod def _connect_validated( destination: ProviderDestination, timeout: float | None, source_address: tuple[str, int] | None @@ -1403,6 +1437,8 @@ def _stream_send( method="POST", ) stream_error: RuntimeError | None = None + started = time.monotonic() + stream_usage: dict[str, Any] | None = None try: with self._open_provider(request, destination) as response: for raw in response: @@ -1416,10 +1452,15 @@ def _stream_send( chunk = json.loads(data) except json.JSONDecodeError: continue + if isinstance(chunk, dict) and isinstance(chunk.get("usage"), dict): + stream_usage = chunk["usage"] choices = chunk.get("choices") or [{}] delta = (choices[0] or {}).get("delta", {}).get("content") if delta: yield delta + _record_provider_response_telemetry( + {"usage": stream_usage} if stream_usage else {}, started + ) except Exception as exc: # noqa: BLE001 - provider error boundary (CWE-209) # The gateway's own terminal tool-stop contract must survive the # boundary: convert the provider HTTP shape into the package-owned @@ -1431,8 +1472,10 @@ def _stream_send( # A stream may already have emitted bytes, so it can neither be retried # nor failed over to another provider. Keep the provider status, body, # and exception cause inside the gateway; callers get one stable, - # package-owned error instead of raw provider diagnostics. - stream_error = RuntimeError(f"provider {agent.id} streaming request failed") + # classified, package-owned error instead of raw provider diagnostics. + stream_error = classify_provider_failure( + exc, agent_id=agent.id, model=agent.model, transport="stream" + ) if stream_error is not None: raise stream_error @@ -1528,7 +1571,11 @@ def _send_raw_with_retry( self._sleep(self._backoff_delay(attempt)) if isinstance(last_error, urllib.error.HTTPError) and _is_tool_execution_stopped(last_error): raise _provider_tool_execution_stopped(agent) from None - raise RuntimeError(f"provider {agent.id} passthrough request failed") from None + if last_error is None: # pragma: no cover - the loop always attempts once + raise RuntimeError(f"provider {agent.id} passthrough request failed") + raise classify_provider_failure( + last_error, agent_id=agent.id, model=agent.model, transport="passthrough" + ) from None def _send_raw( self, @@ -1696,16 +1743,19 @@ def batch_chat( results = self._local_batch_chat(agent, requests, temperature, effort_profile) else: destination = self._validate_provider(agent) # pragma: no cover - batch_error: RuntimeError | None = None + batch_error: ProviderUpstreamError | None = None try: results = self._batch_run( # pragma: no cover agent, requests, temperature, poll_interval, poll_timeout, destination, effort_profile ) - except Exception: # noqa: BLE001 - provider batch boundary (CWE-209) + except Exception as exc: # noqa: BLE001 - provider batch boundary (CWE-209) # Batch upload, polling, and output retrieval all cross the same # public gateway boundary; provider bodies and exception text stay - # inside the authorized provider observability system. - batch_error = RuntimeError(f"provider {agent.id} batch request failed") + # inside the authorized provider observability system. The failure + # is still classified so callers can act on the cause. + batch_error = classify_provider_failure( + exc, agent_id=agent.id, model=agent.model, transport="batch" + ) if batch_error is not None: raise batch_error return _validate_batch_results(requests, results) @@ -2905,8 +2955,17 @@ def stream_route( "prompt_text": text, "answer": answer, "trace": [ - {"id": 0, "role": "worker", "agent_id": agent.id, "subtask": "Direct route (streamed)", - "access": [], "output": answer} + { + "id": 0, + "role": "worker", + "agent_id": agent.id, + "model": agent.model, + "provider": agent.provider_name or agent.base_url.split("://", 1)[-1], + "subtask": "Direct route (streamed)", + "access": [], + "latency_ms": round(latency_seconds * 1000, 2), + "output": answer, + } ], "policy_snapshot": self.policy.as_dict(), "verification": {**verification, "verifier_output": answer}, @@ -3076,6 +3135,8 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]: result = answers[index] row: dict[str, Any] = { "id": 0, "role": "worker", "agent_id": agent.id, + "model": agent.model, + "provider": agent.provider_name or agent.base_url.split("://", 1)[-1], "subtask": "Direct route (batched)", "access": [], "output": result["content"], } if result.get("usage") is not None: @@ -3582,6 +3643,8 @@ def route_once( "id": attempt_index, "role": "worker", "agent_id": candidate.id, + "model": candidate.model, + "provider": candidate.provider_name or candidate.base_url.split("://", 1)[-1], "subtask": "Direct route", "access": [], "latency_ms": round(latency_seconds * 1000, 2), @@ -3758,6 +3821,8 @@ def conduct( outputs[step.id] = output row = step.as_dict() row["latency_ms"] = round(elapsed, 2) + row["model"] = agent.model + row["provider"] = agent.provider_name or agent.base_url.split("://", 1)[-1] row["output"] = output if usage is not None: row["usage"] = usage @@ -4338,6 +4403,10 @@ def _invoke( if not candidates: raise RuntimeError(f"no chat-compatible agent available for role={role}") retry_limit = min(self.tool_retry_attempts, MAX_TOOL_RETRY_ATTEMPTS) + # The final classified upstream failure survives the candidate loop so a + # fully-failed pool surfaces *why* (rate limit, auth, timeout) instead of + # one opaque collapse message. + last_upstream_error: ProviderUpstreamError | None = None for agent in candidates: retry_attempt = 0 while True: @@ -4354,6 +4423,8 @@ def _invoke( self._group_router.observe_failure(agent.id) if isinstance(exc, (ProviderResponseError, ToolFallbackStoppedError)): raise + if isinstance(exc, ProviderUpstreamError): + last_upstream_error = exc decision = classify_tool_failure(exc) action = decision.action # A failed attempt is one Bernoulli stability observation @@ -4398,6 +4469,8 @@ def _invoke( ) self._record_success(agent.id) return output, agent.id, usage + if last_upstream_error is not None: + raise last_upstream_error raise RuntimeError(f"all {len(candidates)} candidate agents failed for role={role}") from None def _record_tool_fallback( diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py new file mode 100644 index 000000000..1a672c408 --- /dev/null +++ b/contextual_orchestrator/provider_errors.py @@ -0,0 +1,222 @@ +"""Provider failure taxonomy: typed, caller-actionable model errors. + +Every upstream provider/model failure is classified into one OpenAI-compatible +error surface so callers learn *which* model failed, *why*, and *whether to +retry* — instead of receiving one opaque ``internal_error`` for every cause. + +The classification derives from the upstream HTTP status (RFC 9110 semantics) +and the OpenAI error-code conventions used across compatible providers. +Provider response bodies are never surfaced raw: only a bounded, control-free +message field is kept, because provider diagnostics can embed secrets, +prompts, or internal topology (CWE-209). + +References +---------- +- Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* + (RFC 9110). IETF. https://doi.org/10.17487/RFC9110 +- OpenAI. (2026). *Error codes*. https://platform.openai.com/docs/guides/error-codes +""" + +from __future__ import annotations + +import json as _json +import socket +import ssl +import urllib.error +from typing import Any + +__all__ = [ + "MAX_SAFE_MESSAGE_CHARS", + "PROVIDER_STATUS_SURFACES", + "ProviderUpstreamError", + "classify_provider_failure", + "safe_provider_message", +] + +#: Upper bound for any provider-supplied message that reaches a caller. +MAX_SAFE_MESSAGE_CHARS = 300 + +#: Upstream HTTP status -> ``(client_status, error_code, retryable)`` surface. +#: The client status is what this gateway returns; ``error_code`` follows the +#: OpenAI-compatible vocabulary; ``retryable`` mirrors the gateway's own +#: transient-retry policy so callers can make consistent retry decisions. +PROVIDER_STATUS_SURFACES: dict[int, tuple[int, str, bool]] = { + 400: (400, "invalid_request_error", False), + 401: (401, "authentication_error", False), + 402: (402, "payment_required", False), + 403: (403, "permission_error", False), + 404: (404, "model_not_found", False), + 405: (502, "api_error", False), + 408: (504, "provider_timeout", True), + 409: (409, "conflict", False), + 410: (404, "model_not_found", False), + 413: (413, "request_too_large", False), + 415: (400, "invalid_request_error", False), + 422: (400, "invalid_request_error", False), + 425: (429, "rate_limit_exceeded", True), + 429: (429, "rate_limit_exceeded", True), + 500: (502, "api_error", True), + 501: (502, "api_error", False), + 502: (502, "api_error", True), + 503: (503, "service_unavailable", True), + 504: (504, "provider_timeout", True), + 529: (503, "service_unavailable", True), +} + +#: Fallback surface for an upstream status that has no explicit mapping. +_UNMAPPED_UPSTREAM_SURFACE: tuple[int, str, bool] = (502, "api_error", False) + + +def safe_provider_message(exc: BaseException) -> str | None: + """Extract one bounded, control-free diagnostic sentence from a failure. + + Only an HTTP error's JSON ``error.message``-style fields pass through; + every other cause returns ``None`` so callers fall back to package-owned + sentences. Provider diagnostics can embed URLs, secrets, prompts, or + internal topology, so raw exception text never reaches API callers. + Control characters are collapsed so no body can smuggle log-formatting + or header content downstream. + """ + if isinstance(exc, urllib.error.HTTPError): + try: + payload = _json.loads(exc.read().decode("utf-8", errors="replace")) + except Exception: # noqa: BLE001 - bodies are untrusted input + return None + raw: Any = None + if isinstance(payload, dict): + error_field = payload.get("error") + if isinstance(error_field, dict): + raw = error_field.get("message") or error_field.get("code") + elif isinstance(error_field, str): + raw = error_field + else: + raw = payload.get("message") or payload.get("detail") or payload.get("code") + if not isinstance(raw, (str, int, float)) or raw == "": + return None + else: + first_arg = next((part for part in exc.args[:1] if isinstance(part, str)), "") + raw = first_arg or type(exc).__name__ + collapsed = "".join( + char if ord(char) >= 0x20 or char in "\t" else " " for char in str(raw) + ).strip() + return collapsed[:MAX_SAFE_MESSAGE_CHARS] or None + + +class ProviderUpstreamError(RuntimeError): + """One classified upstream model/provider failure. + + Attributes carry the evidence a caller needs to act: which agent/model + failed, the upstream status when one existed, the OpenAI-compatible code, + whether retrying can help, and one bounded redacted message. + """ + + def __init__( + self, + *, + agent_id: str, + model: str, + error_code: str, + message: str, + client_status: int, + provider_status: int | None = None, + retryable: bool = False, + transport: str = "chat", + ) -> None: + self.agent_id = agent_id + self.model = model + self.error_code = error_code + self.client_status = client_status + self.provider_status = provider_status + self.retryable = retryable + self.transport = transport + super().__init__(message) + + @property + def detail(self) -> dict[str, Any]: + """Return the structured evidence attached to API error payloads.""" + return { + "agent_id": self.agent_id, + "model": self.model, + "provider_status": self.provider_status, + "retryable": self.retryable, + "transport": self.transport, + } + + +def classify_provider_failure( + exc: BaseException | None, + *, + agent_id: str, + model: str, + transport: str = "chat", +) -> ProviderUpstreamError: + """Map one provider-call exception onto the caller-facing error taxonomy. + + Messages stay package-owned and bounded: only an upstream JSON body's + ``error.message``-style field may pass through (redacted); every other + cause gets a fixed sentence so raw provider diagnostics (URLs, prompts, + secrets) can never reach callers or logs. + """ + if isinstance(exc, ProviderUpstreamError): + return exc + if isinstance(exc, urllib.error.HTTPError): + status = exc.code + client_status, error_code, retryable = PROVIDER_STATUS_SURFACES.get( + status, _UNMAPPED_UPSTREAM_SURFACE + ) + return ProviderUpstreamError( + agent_id=agent_id, + model=model, + error_code=error_code, + message=( + safe_provider_message(exc) + or f"provider rejected the request with HTTP {status}" + ), + client_status=client_status, + provider_status=status, + retryable=retryable, + transport=transport, + ) + if isinstance(exc, ssl.SSLCertVerificationError): + return ProviderUpstreamError( + agent_id=agent_id, + model=model, + error_code="tls_verification_failed", + message=f"provider {agent_id} endpoint failed TLS certificate verification", + client_status=502, + provider_status=None, + retryable=False, + transport=transport, + ) + if isinstance(exc, ssl.SSLError): + return ProviderUpstreamError( + agent_id=agent_id, + model=model, + error_code="tls_failure", + message=f"a TLS error interrupted the provider {agent_id} connection", + client_status=502, + provider_status=None, + retryable=True, + transport=transport, + ) + if isinstance(exc, (urllib.error.URLError, TimeoutError, ConnectionError, socket.timeout)): + return ProviderUpstreamError( + agent_id=agent_id, + model=model, + error_code="provider_connection_error", + message=f"the provider {agent_id} connection failed or did not finish in time", + client_status=502, + provider_status=None, + retryable=True, + transport=transport, + ) + return ProviderUpstreamError( + agent_id=agent_id, + model=model, + error_code="api_error", + message=f"provider {agent_id} request failed", + client_status=502, + provider_status=None, + retryable=False, + transport=transport, + ) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 6221f8932..12a8a0bd2 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -35,6 +35,7 @@ sse_stream_body, ) from .pii_protection import DEFAULT_PURPOSE_BY_SCOPE, PURPOSES_BY_SCOPE +from .provider_errors import ProviderUpstreamError from .tool_fallback import ToolFallbackStoppedError from .model_group import canonical_group_name from .telemetry import ( @@ -510,6 +511,36 @@ def _error_payload(error_code: str, error_message: str, error_detail: dict[str, } +_PROVIDER_FAILURE_GUIDANCE: dict[str, str] = { + "invalid_request_error": "Adjust the request parameters and retry.", + "authentication_error": "Verify the credential registered for this model in the credential registry.", + "payment_required": "Add payment capacity to the provider account that serves this model.", + "permission_error": "Request provider access for this model from the operator.", + "model_not_found": "Check the model id against the provider's model list and retry with a valid one.", + "request_too_large": "Reduce the request size (shorter input or lower max tokens).", + "rate_limit_exceeded": "Retry after a short delay; the provider is throttling this model.", + "conflict": "Resolve the conflicting in-flight operation before retrying.", + "provider_timeout": "The provider accepted but did not finish in time; retrying may succeed.", + "provider_connection_error": "The provider was unreachable; check network reachability before retrying.", + "tls_failure": "A transport-layer security error interrupted the provider connection; retrying may succeed.", + "tls_verification_failed": "Verify the provider endpoint certificate chain before retrying.", + "api_error": "The provider reported an internal failure; retrying may succeed.", + "service_unavailable": "The provider is temporarily unavailable; retry after a short delay.", +} + + +def _provider_upstream_message(exc: ProviderUpstreamError) -> str: + """Build one caller-actionable sentence for a classified upstream failure. + + The message names which model/agent failed and what to do next; it never + echoes raw provider diagnostics beyond the bounded redacted sentence. + """ + guidance = _PROVIDER_FAILURE_GUIDANCE.get( + exc.error_code, "Review the request or contact the operator." + ) + return f"Model '{exc.model}' via agent '{exc.agent_id}': {exc}. {guidance}" + + def _cache_bypass_header(value: str | None) -> bool: """Parse the opt-in cache bypass header without accepting ambiguous values.""" if value is None or not value.strip(): @@ -5384,6 +5415,13 @@ def do_GET(self) -> None: # noqa: N802 self._send_error(exc.status, exc.code, exc.message, exc.detail) except (TypeError, ValueError) as exc: self._send_error(400, "invalid_request", str(exc)) + except ProviderUpstreamError as exc: + self._send_error( + exc.client_status, + exc.error_code, + _provider_upstream_message(exc), + exc.detail, + ) except Exception: self._send_error(500, "internal_error", "internal server error") @@ -5420,6 +5458,13 @@ def do_PATCH(self) -> None: # noqa: N802 self._send_error(400, "invalid_request", str(exc)) except KeyError as exc: self._send_error(404, "agent_not_found", str(exc)) + except ProviderUpstreamError as exc: + self._send_error( + exc.client_status, + exc.error_code, + _provider_upstream_message(exc), + exc.detail, + ) except Exception: self._send_error(500, "internal_error", "internal server error") @@ -6418,6 +6463,13 @@ def do_POST(self) -> None: # noqa: N802 self._send_error(exc.status, exc.code, exc.message, exc.detail) except (TypeError, ValueError) as exc: self._send_error(400, "invalid_request", str(exc)) + except ProviderUpstreamError as exc: + self._send_error( + exc.client_status, + exc.error_code, + _provider_upstream_message(exc), + exc.detail, + ) except Exception: self._send_error(500, "internal_error", "internal server error") diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index 1c45e9439..09bf6a059 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -41,7 +41,15 @@ "gen_ai.operation.name", "gen_ai.provider.name", "gen_ai.request.model", + "gen_ai.response.finish_reasons", + "gen_ai.response.model", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", "contextual_orchestrator.agent_id", + "contextual_orchestrator.error_code", + "contextual_orchestrator.latency_ms", + "contextual_orchestrator.provider_status_code", "contextual_orchestrator.session_id_hash", "server.address", "server.port", @@ -168,6 +176,45 @@ def _safe_attributes( return result +def annotate_current_span(attributes: Mapping[str, Any]) -> None: + """Apply the allow-list filter to one attribute set on the ACTIVE span. + + Callers inside a ``traced`` block use this to attach per-call evidence + (token counts, latency, served model) without bypassing redaction. + """ + if trace is None: # pragma: no cover - dependency is declared by the project + return + span = trace.get_current_span() + if span is None or not span.is_recording(): + return + for key, value in _safe_attributes(attributes).items(): + span.set_attribute(key, value) + + +def record_provider_usage(usage: Mapping[str, Any] | None) -> None: + """Record OpenAI-compatible token usage as GenAI semantic-convention counts.""" + if trace is None or not isinstance(usage, Mapping): # pragma: no cover - guarded above + return + + def _count(key: str) -> int | None: + value = usage.get(key) + return value if isinstance(value, int) and not isinstance(value, bool) else None + + attributes: dict[str, Any] = {} + input_tokens = _count("prompt_tokens") if "prompt_tokens" in usage else _count("input_tokens") + output_tokens = ( + _count("completion_tokens") if "completion_tokens" in usage else _count("output_tokens") + ) + total_tokens = _count("total_tokens") + if input_tokens is not None: + attributes["gen_ai.usage.input_tokens"] = input_tokens + if output_tokens is not None: + attributes["gen_ai.usage.output_tokens"] = output_tokens + if total_tokens is not None: + attributes["gen_ai.usage.total_tokens"] = total_tokens + annotate_current_span(attributes) + + def configure_telemetry( service_name: str = "contextual-orchestrator", *, @@ -219,7 +266,13 @@ def traced( name: str, attributes: Mapping[str, Any] | None = None, ) -> Iterator[Any]: - """Trace one provider CLIENT operation and preserve all failures.""" + """Trace one provider CLIENT operation and preserve all failures. + + On failure the span records the classified error code (not just the + Python exception class), the upstream status when one existed, and a + non-recording-safe ERROR status — so exported traces answer *why* the + model call failed. + """ if trace is None: # pragma: no cover - dependency is declared by the project yield None return @@ -235,12 +288,21 @@ def traced( try: yield span except Exception as exc: + from .provider_errors import classify_provider_failure + + classified = classify_provider_failure(exc, agent_id="", model="") + failure_code = classified.error_code + provider_status = classified.provider_status if Status is not None and StatusCode is not None: - span.set_attribute("error.type", type(exc).__name__) + span.set_attribute("error.type", failure_code) + if provider_status is not None: + span.set_attribute( + "contextual_orchestrator.provider_status_code", provider_status + ) span.set_status(Status(StatusCode.ERROR)) _LOGGER.warning( "telemetry.operation_failed operation=%s error_type=%s", name, - type(exc).__name__, + failure_code, ) raise diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 710737976..be46b29e4 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,33 @@ # Product and Technical Gap Baseline +## 2026-08-26 18:05 KST error-surface and telemetry-gap snapshot + +Customer-visible gap: every upstream model failure surfaced as one opaque +`internal_error`, telemetry spans carried no per-call evidence (no token +counts, latency, served model, or classified cause), and workflow traces +omitted which model/provider served each step. Branch +`fix/provider-error-taxonomy-telemetry` closes all three at once: + +- `contextual_orchestrator.provider_errors` classifies upstream HTTP status, + network, TLS, and transport failures into OpenAI-compatible codes with + client status, retryability, and bounded redacted messages (CWE-209); + chat/passthrough/stream/batch transports surface the classification and a + fully-failed pool surfaces the final classified cause after measured + failover. Server payloads add next-step guidance per failure family. +- Provider spans now record GenAI semantic-convention usage counts, served + response model, finish reason, request latency, and classified `error.type` + plus upstream status on failures (`docs/adr/0005` deadline evidence gains a + concrete span counterpart). +- Route/stream/batch/conduct trace steps now carry `model`, `provider`, and + `latency_ms`; streamed steps previously had none. + +Evidence: full suite `2279 passed` locally on the exact head; docstring +coverage `100%` (interrogate); `provider_errors.py`/`telemetry.py` statement +coverage `100%`. Remaining open gaps tracked below stay unchanged until their +PRs merge: #849 keep-alive session correlation + k6 gate, #851 structured +passthrough batch routing, #857 streaming deadline + baseline SHA row, #858 +measurement-status i18n. The standing hourly loop re-checks this queue. + ## 2026-08-26 11:46 KST exact-head queue snapshot Protected `main` remains `762f7a345b1d8c82584023a7ff05b4660d628cab`. diff --git a/tests/test_orchestrator_client_boundaries.py b/tests/test_orchestrator_client_boundaries.py index d60d63b10..bffba605f 100644 --- a/tests/test_orchestrator_client_boundaries.py +++ b/tests/test_orchestrator_client_boundaries.py @@ -25,6 +25,7 @@ _validate_batch_results, _validate_provider_probe_timeout, ) +from contextual_orchestrator.provider_errors import ProviderUpstreamError def _orch(*agents: ModelAgent, **kwargs) -> TaskOrchestrator: @@ -456,8 +457,10 @@ def __iter__(self): ): iterator = client._stream_send(agent, {}) assert next(iterator) == "par" - with pytest.raises(RuntimeError, match="streaming request failed"): + with pytest.raises(ProviderUpstreamError) as excinfo: next(iterator) + assert excinfo.value.error_code == "provider_connection_error" + assert "connection reset" not in str(excinfo.value) # -- batch success paths ------------------------------------------------------------ @@ -494,8 +497,10 @@ def test_batch_chat_wraps_provider_failures_without_provider_text() -> None: with patch.object(client, "_validate_provider", return_value=None), patch.object( client, "_batch_run", side_effect=ValueError("provider secret detail") ): - with pytest.raises(RuntimeError, match="batch request failed") as excinfo: + with pytest.raises(ProviderUpstreamError) as excinfo: client.batch_chat(agent, requests) + assert excinfo.value.error_code == "api_error" + assert excinfo.value.transport == "batch" assert "secret detail" not in str(excinfo.value) diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py new file mode 100644 index 000000000..6265f3285 --- /dev/null +++ b/tests/test_provider_error_taxonomy.py @@ -0,0 +1,329 @@ +"""Provider failure taxonomy: classification, redaction, and API error surface. + +Every upstream provider/model failure must reach the caller as one classified, +OpenAI-compatible error — naming the model, the cause family, and whether a +retry can help — instead of collapsing into a generic ``internal_error``. +""" + +from __future__ import annotations + +import io +import json +import socket +import ssl +import sys +import threading +import urllib.error +import urllib.request +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 +from contextual_orchestrator.provider_errors import ( # noqa: E402 + MAX_SAFE_MESSAGE_CHARS, + PROVIDER_STATUS_SURFACES, + ProviderUpstreamError, + classify_provider_failure, + safe_provider_message, +) +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + + +def _http_error(code: int, body: bytes | None = None) -> urllib.error.HTTPError: + payload = io.BytesIO(body) if body is not None else None + return urllib.error.HTTPError( + "https://provider.example/chat/completions", code, "error", None, payload + ) + + +def _body_http_error(code: int, payload: dict) -> urllib.error.HTTPError: + return _http_error(code, json.dumps(payload).encode("utf-8")) + + +# -- message redaction -------------------------------------------------------- + + +def test_safe_message_prefers_nested_provider_error_fields() -> None: + """``error.message`` / ``error.code`` / top-level fields are the only pass-through.""" + nested = safe_provider_message(_body_http_error(400, {"error": {"message": "max_tokens too large"}})) + assert nested == "max_tokens too large" + coded = safe_provider_message(_body_http_error(400, {"error": {"code": "context_length_exceeded"}})) + assert coded == "context_length_exceeded" + plain_string = safe_provider_message(_body_http_error(400, {"error": "invalid api key"})) + assert plain_string == "invalid api key" + top_level = safe_provider_message(_body_http_error(429, {"message": "rate limit reached"})) + assert top_level == "rate limit reached" + detail = safe_provider_message(_body_http_error(422, {"detail": "validation failed"})) + assert detail == "validation failed" + + +def test_safe_message_hides_unparseable_bodies_and_urls() -> None: + """Non-JSON bodies return None so URLs/reasons never leak through fallback text.""" + assert safe_provider_message(_http_error(500, b"upstream-secret http://10.0.0.9/internal")) is None + assert safe_provider_message(_http_error(502)) is None + + +def test_safe_message_collapses_control_characters_and_bounds_length() -> None: + """Control characters cannot smuggle log or header content; length is bounded.""" + long = safe_provider_message( + _body_http_error(400, {"error": {"message": "x" * 500}}) + ) + assert long is not None + assert len(long) == MAX_SAFE_MESSAGE_CHARS + raw = "line1\nline2\ttabbed\x00nul" + collapsed = safe_provider_message(_body_http_error(400, {"error": {"message": raw}})) + assert collapsed is not None + assert "\n" not in collapsed and "\x00" not in collapsed + assert "\t" in collapsed # tab is preserved for readability + + +def test_safe_message_non_http_exception_uses_first_arg_text() -> None: + """Generic exceptions keep their own first argument as bounded context.""" + assert safe_provider_message(RuntimeError("plain failure")) == "plain failure" + assert safe_provider_message(ValueError()) == "ValueError" + + +# -- classification table ------------------------------------------------------ + + +def test_classification_maps_every_upstream_status_to_openai_surface() -> None: + """Each upstream status yields its documented client status/code/retryable.""" + expected = { + 400: (400, "invalid_request_error", False), + 401: (401, "authentication_error", False), + 402: (402, "payment_required", False), + 403: (403, "permission_error", False), + 404: (404, "model_not_found", False), + 408: (504, "provider_timeout", True), + 409: (409, "conflict", False), + 413: (413, "request_too_large", False), + 422: (400, "invalid_request_error", False), + 425: (429, "rate_limit_exceeded", True), + 429: (429, "rate_limit_exceeded", True), + 500: (502, "api_error", True), + 503: (503, "service_unavailable", True), + 504: (504, "provider_timeout", True), + 529: (503, "service_unavailable", True), + } + for status, surface in expected.items(): + assert PROVIDER_STATUS_SURFACES[status] == surface, f"status {status}" + classified = classify_provider_failure(_http_error(status), agent_id="a", model="m") + assert isinstance(classified, ProviderUpstreamError) + assert classified.client_status == surface[0], f"status {status}" + assert classified.error_code == surface[1], f"status {status}" + assert classified.retryable == surface[2], f"status {status}" + assert classified.provider_status == status + assert classified.model == "m" + + +def test_classification_handles_network_tls_and_unknown_causes() -> None: + """Network, TLS, unmapped-status, and generic failures each classify distinctly.""" + network = classify_provider_failure(urllib.error.URLError("dns"), agent_id="a", model="m") + assert network.error_code == "provider_connection_error" + assert network.client_status == 502 and network.retryable + for exc in (socket.timeout("slow"), TimeoutError("deadline"), ConnectionError("reset")): + assert classify_provider_failure(exc, agent_id="a", model="m").retryable + + cert = ssl.SSLCertVerificationError("bad chain") + tls_verify = classify_provider_failure(cert, agent_id="a", model="m") + assert tls_verify.error_code == "tls_verification_failed" + assert not tls_verify.retryable + + handshake = ssl.SSLError("handshake eof") + assert classify_provider_failure(handshake, agent_id="a", model="m").retryable + + unmapped = classify_provider_failure(_http_error(418), agent_id="a", model="m") + assert unmapped.error_code == "api_error" + assert not unmapped.retryable + + generic = classify_provider_failure(RuntimeError("internal"), agent_id="agent_x", model="m") + assert generic.error_code == "api_error" + assert "internal" not in str(generic) + assert str(generic).endswith("request failed") + + original = ProviderUpstreamError( + agent_id="a", + model="m", + error_code="rate_limit_exceeded", + message="kept", + client_status=429, + retryable=True, + ) + assert classify_provider_failure(original, agent_id="b", model="n") is original + + +def test_detail_and_transport_are_preserved_for_callers() -> None: + """The structured detail names agent/model/status/retryability/transport.""" + classified = classify_provider_failure( + _http_error(429), agent_id="worker_agent", model="gpt-x", transport="passthrough" + ) + assert classified.detail == { + "agent_id": "worker_agent", + "model": "gpt-x", + "provider_status": 429, + "retryable": True, + "transport": "passthrough", + } + assert "HTTP 429" in str(classified) + + +# -- client wiring ------------------------------------------------------------- + + +class _StatusFailureClient(ModelClient): + """Raises one fixed HTTP status from every transport send.""" + + def __init__(self, code: int) -> None: + super().__init__(max_retries=2, retry_backoff=0.0) + self.code = code + self.attempts = 0 + + def _send(self, agent: ModelAgent, payload: dict, destination=None) -> str: # type: ignore[override] + self.attempts += 1 + raise _http_error(self.code) + + def _send_raw(self, agent: ModelAgent, endpoint: str, payload: dict, destination=None): # type: ignore[override] + self.attempts += 1 + raise _http_error(self.code) + + +def test_chat_retry_layer_surfaces_classified_rate_limit() -> None: + """A 429 exhausts retries then raises rate_limit_exceeded with retryable=True.""" + client = _StatusFailureClient(429) + agent = ModelAgent("worker_agent", "gpt-x", base_url="https://provider.example/v1") + with patch.object(client, "_validate_provider", lambda unused: None): + try: + client._send_with_retry(agent, {"model": "gpt-x"}) + except ProviderUpstreamError as exc: + assert exc.error_code == "rate_limit_exceeded" + assert exc.client_status == 429 + assert exc.provider_status == 429 + assert exc.retryable is True + else: # pragma: no cover + raise AssertionError("classified failure must propagate") + assert client.attempts == 3 # initial + 2 retries + + +def test_passthrough_retry_layer_surfaces_model_not_found_without_retry() -> None: + """A 404 on passthrough fails immediately as model_not_found.""" + client = _StatusFailureClient(404) + agent = ModelAgent("proxy_agent", "missing-model", base_url="https://provider.example/v1") + with patch.object(client, "_validate_provider", lambda unused: None): + try: + client._send_raw_with_retry(agent, "responses", {}) + except ProviderUpstreamError as exc: + assert exc.error_code == "model_not_found" + assert exc.transport == "passthrough" + assert client.attempts == 1 # caller errors are never retried + else: # pragma: no cover + raise AssertionError("classified failure must propagate") + + +def test_invoke_preserves_final_classified_failure_across_candidates() -> None: + """Failover still runs, but an all-failed pool surfaces the last cause. + + The fake client raises exactly what ``ModelClient._send_with_retry`` now + produces — a classified ``ProviderUpstreamError`` — so this exercises the + real boundary contract between the transport layer and agent failover. + """ + + def _rate_limited(agent: ModelAgent) -> ProviderUpstreamError: + return classify_provider_failure(_http_error(429), agent_id=agent.id, model=agent.model) + + class RateLimited(ModelClient): + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + raise _rate_limited(agent) + + agents = [ + ModelAgent("primary_worker", "mock-a", tags=("reasoning",)), + ModelAgent("backup_worker", "mock-b", tags=("reasoning",)), + ] + orchestrator = TaskOrchestrator(agents, client=RateLimited()) + orchestrator._triage_fn = lambda text: False # single-step route accounting + try: + orchestrator.route_once([{"role": "user", "content": "route this"}]) + except ProviderUpstreamError as exc: + assert exc.error_code == "rate_limit_exceeded" + assert exc.client_status == 429 + assert exc.agent_id in {"primary_worker", "backup_worker"} + else: # pragma: no cover + raise AssertionError("the final classified failure must survive failover") + + +# -- server error surface ------------------------------------------------------ + + +class _UpstreamDown(ModelClient): + """Chat client whose provider always answers 429, classified as the real layer does.""" + + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + raise classify_provider_failure(_http_error(429), agent_id=agent.id, model=agent.model) + + +def _post(url: str, payload: dict, token: str) -> tuple[int, dict]: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={"content-type": "application/json", "authorization": f"Bearer {token}", "connection": "close"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def test_chat_completions_returns_openai_compatible_rate_limit_error() -> None: + """A throttled model answers 429/rate_limit_exceeded, never internal_error.""" + orchestrator = TaskOrchestrator( + [ModelAgent("worker_agent", "gpt-x", tags=("reasoning",))], + client=_UpstreamDown(), + ) + orchestrator._triage_fn = lambda text: False + token = "taxonomy_token" + server = build_server(orchestrator, port=0, security=SecurityConfig(auth_token=token)) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + status, body = _post( + f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions", + {"model": "gpt-x", "messages": [{"role": "user", "content": "hello"}]}, + token, + ) + finally: + server.shutdown() + + assert status == 429 + error = body["error"] + assert error["code"] == "rate_limit_exceeded" + assert "gpt-x" in error["message"] and "worker_agent" in error["message"] + assert "Retry after a short delay" in error["message"] + assert error["detail"]["provider_status"] == 429 + assert error["detail"]["model"] == "gpt-x" + assert error["detail"]["retryable"] is True + + +def test_guidance_table_covers_every_documented_code() -> None: + """Every classified code has caller guidance; unknown codes get a default.""" + from contextual_orchestrator import server as server_module + + codes = {surface[1] for surface in PROVIDER_STATUS_SURFACES.values()} + codes.update({"tls_verification_failed", "tls_failure", "provider_connection_error"}) + for code in codes: + assert code in server_module._PROVIDER_FAILURE_GUIDANCE, f"missing guidance for {code}" + + class UnknownCode(ProviderUpstreamError): + pass + + exotic = UnknownCode( + agent_id="a", + model="m", + error_code="exotic_future_code", + message="unknown", + client_status=500, + ) + message = server_module._provider_upstream_message(exotic) + assert "contact the operator" in message diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index d92b7c511..ae720d30a 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -28,6 +28,7 @@ ProviderResponseError, is_transient_error, ) +from contextual_orchestrator.provider_errors import ProviderUpstreamError # noqa: E402 from contextual_orchestrator.tool_fallback import ToolFallbackStoppedError @@ -455,7 +456,8 @@ def _batch_run(self, agent, requests, temperature, poll_interval, poll_timeout, try: client.batch_chat(agent, {"task_0": [{"role": "user", "content": "ping"}]}) except RuntimeError as error: - assert "batch request failed" in str(error) + assert isinstance(error, ProviderUpstreamError) + assert error.transport == "batch" assert "provider-secret-batch-body" not in str(error) assert error.__cause__ is None else: # pragma: no cover diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index d5963c1c3..00266bd08 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,11 +1,15 @@ """Tests for request session binding and prompt-safe telemetry.""" import hashlib +import io +import json from contextlib import contextmanager from http.server import BaseHTTPRequestHandler from types import SimpleNamespace from unittest.mock import MagicMock +import pytest + import contextual_orchestrator.orchestrator as orchestrator_module import contextual_orchestrator.server as server_module import contextual_orchestrator.telemetry as telemetry_module @@ -435,6 +439,134 @@ def test_traced_starts_safe_client_span_with_error_type_and_no_raw_exception(mon set_status_on_exception=False, ) span.record_exception.assert_not_called() - span.set_attribute.assert_called_once_with("error.type", "TimeoutError") + # Failures record the CLASSIFIED cause family (network timeout here), not + # the Python exception class, and never the exception text. + span.set_attribute.assert_called_once_with("error.type", "provider_connection_error") assert "provider-response-secret" not in caplog.text assert "session-secret" not in caplog.text + + +def test_traced_records_upstream_status_for_http_failures(monkeypatch): + """An upstream HTTP failure exposes its semantic code and original status.""" + import urllib.error + + tracer = MagicMock() + span = tracer.start_as_current_span.return_value.__enter__.return_value + monkeypatch.setattr(telemetry_module.trace, "get_tracer", lambda unused_name: tracer) + + with pytest.raises(urllib.error.HTTPError) as excinfo: + with traced("chat model-x"): + raise urllib.error.HTTPError( + "https://p.example", 429, "err", None, io.BytesIO(b"{}") + ) + assert excinfo.value.code == 429 + span.set_attribute.assert_any_call("error.type", "rate_limit_exceeded") + span.set_attribute.assert_any_call( + "contextual_orchestrator.provider_status_code", 429 + ) + + +def test_annotate_and_usage_helpers_filter_to_allowed_genai_attributes(): + """Span annotation keeps approved scalars only; prompts never enter spans.""" + span = MagicMock() + span.is_recording.return_value = True + fake_trace = SimpleNamespace(get_current_span=lambda: span) + original_trace = telemetry_module.trace + telemetry_module.trace = fake_trace + try: + telemetry_module.annotate_current_span( + { + "gen_ai.response.model": "gpt-x", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.total_tokens": 18, + "prompt": "secret", + "nested": {"a": 1}, + "bad_ratio": object(), + } + ) + telemetry_module.record_provider_usage( + {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} + ) + telemetry_module.record_provider_usage(None) + finally: + telemetry_module.trace = original_trace + + calls = [tuple(call.args) for call in span.set_attribute.call_args_list] + assert ("gen_ai.response.model", "gpt-x") in calls + assert ("gen_ai.usage.input_tokens", 11) in calls + assert ("gen_ai.usage.output_tokens", 7) in calls + assert ("gen_ai.usage.total_tokens", 18) in calls + # The second record_provider_usage call re-annotates the OpenAI-keyed counts. + assert ("gen_ai.usage.input_tokens", 3) in calls + assert all(key not in {"prompt", "nested"} for key, _value in calls) + + +def test_record_provider_usage_accepts_responses_style_keys(monkeypatch): + """``input_tokens``/``output_tokens`` aliases map onto GenAI counts.""" + span = MagicMock() + span.is_recording.return_value = True + fake_trace = SimpleNamespace(get_current_span=lambda: span) + original_trace = telemetry_module.trace + telemetry_module.trace = fake_trace + try: + telemetry_module.record_provider_usage({"input_tokens": 5, "output_tokens": 2}) + # A recording-less span must be a silent no-op. + empty_span = MagicMock() + empty_span.is_recording.return_value = False + telemetry_module.trace = SimpleNamespace(get_current_span=lambda: empty_span) + telemetry_module.record_provider_usage({"total_tokens": 9}) + finally: + telemetry_module.trace = original_trace + + calls = [call.args[0] for call in span.set_attribute.call_args_list] + assert calls == [ + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + ] + + +def test_provider_response_telemetry_records_latency_model_finish_reason(monkeypatch): + """One completed provider response annotates usage, model, reason, latency.""" + captured: dict[str, list] = {"annotate": [], "usage": []} + + @contextmanager + def capture(name, attributes): # noqa: ARG001 - signature parity + yield None + + class FakeResponse(io.BytesIO): + pass + + client = ModelClient() + agent = ModelAgent( + "worker_agent", + "gpt-x", + base_url="https://provider.example/v1", + credential_key="", + ) + + @contextmanager + def fake_open(request, destination, timeout=None): # noqa: ARG001 + yield FakeResponse( + json.dumps({ + "model": "gpt-x-served", + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 6, "total_tokens": 11}, + }).encode("utf-8") + ) + + monkeypatch.setattr(client, "_validate_provider", lambda unused: None) + monkeypatch.setattr(client, "_open_provider", fake_open) + monkeypatch.setattr(orchestrator_module, "traced", capture) + monkeypatch.setattr(orchestrator_module, "annotate_current_span", lambda a: captured["annotate"].append(a)) + monkeypatch.setattr(orchestrator_module, "record_provider_usage", lambda u: captured["usage"].append(u)) + + content = client.chat(agent, [{"role": "user", "content": "hi"}]) + assert content == "ok" + annotated = captured["annotate"][0] + assert annotated["gen_ai.response.model"] == "gpt-x-served" + assert annotated["gen_ai.response.finish_reasons"] == "stop" + assert isinstance(annotated["contextual_orchestrator.latency_ms"], float) + assert captured["usage"] == [ + {"prompt_tokens": 5, "completion_tokens": 6, "total_tokens": 11} + ] diff --git a/tests/test_true_streaming.py b/tests/test_true_streaming.py index ff9bf0911..ba03c380d 100644 --- a/tests/test_true_streaming.py +++ b/tests/test_true_streaming.py @@ -18,6 +18,7 @@ from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 +from contextual_orchestrator.provider_errors import ProviderUpstreamError # noqa: E402 from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 @@ -132,7 +133,12 @@ def log_message(self, *args: object) -> None: try: list(client._stream_send(agent, {"model": "gpt-x", "stream": True})) except RuntimeError as error: - assert "streaming request failed" in str(error) + # Classified, package-owned failure: the upstream status is kept, + # while the provider URL, body text, and cause stay inside. + assert isinstance(error, ProviderUpstreamError) + assert error.error_code == "api_error" + assert error.client_status == 502 + assert "HTTP 500" in str(error) assert "http://" not in str(error) # provider URL stays inside assert "upstream-secret-diagnostic" not in str(error) assert error.__cause__ is None From 7101227f40ec275c0282eeaeea9d908432d35323 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 20:36:30 +0900 Subject: [PATCH 02/33] fix: preserve provider error and telemetry contracts --- contextual_orchestrator/orchestrator.py | 18 ++++-- contextual_orchestrator/provider_errors.py | 22 +++++-- contextual_orchestrator/server.py | 25 ++++++++ contextual_orchestrator/telemetry.py | 21 ++++-- tests/test_provider_error_taxonomy.py | 21 +++++- tests/test_telemetry.py | 45 ++++++++++++- tests/test_true_streaming.py | 75 ++++++++++++++++++++++ 7 files changed, 209 insertions(+), 18 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 4b2e07cfe..c55267423 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -530,7 +530,7 @@ def as_dict(self) -> dict[str, Any]: # HTTP statuses worth retrying: request timeout, conflict, too-early, rate limit, # and the standard upstream/gateway failures. Everything else (400/401/403/404 ...) # is a caller or configuration error and must not be retried. -TRANSIENT_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504}) +TRANSIENT_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504, 529}) LOCAL_PROVIDER_SCHEMES = frozenset({"mlx", "local"}) LOCAL_PROVIDER_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) @@ -854,11 +854,19 @@ def _record_provider_response_telemetry(data: Any, started_monotonic: float) -> if isinstance(served_model, str) and served_model: attributes["gen_ai.response.model"] = served_model choices = data.get("choices") - finish_reason = ( - choices[0].get("finish_reason") if isinstance(choices, list) and isinstance(choices[0], dict) else None + finish_reasons = ( + [ + choice["finish_reason"] + for choice in choices + if isinstance(choice, dict) + and isinstance(choice.get("finish_reason"), str) + and choice["finish_reason"] + ] + if isinstance(choices, list) + else [] ) - if isinstance(finish_reason, str) and finish_reason: - attributes["gen_ai.response.finish_reasons"] = finish_reason + if finish_reasons: + attributes["gen_ai.response.finish_reasons"] = finish_reasons annotate_current_span(attributes) record_provider_usage(data.get("usage")) diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index 1a672c408..ce84753f5 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -26,6 +26,7 @@ from typing import Any __all__ = [ + "MAX_PROVIDER_ERROR_BODY_BYTES", "MAX_SAFE_MESSAGE_CHARS", "PROVIDER_STATUS_SURFACES", "ProviderUpstreamError", @@ -36,6 +37,9 @@ #: Upper bound for any provider-supplied message that reaches a caller. MAX_SAFE_MESSAGE_CHARS = 300 +#: Maximum provider response bytes inspected for one caller-safe diagnostic. +MAX_PROVIDER_ERROR_BODY_BYTES = 65_536 + #: Upstream HTTP status -> ``(client_status, error_code, retryable)`` surface. #: The client status is what this gateway returns; ``error_code`` follows the #: OpenAI-compatible vocabulary; ``retryable`` mirrors the gateway's own @@ -48,7 +52,7 @@ 404: (404, "model_not_found", False), 405: (502, "api_error", False), 408: (504, "provider_timeout", True), - 409: (409, "conflict", False), + 409: (409, "conflict", True), 410: (404, "model_not_found", False), 413: (413, "request_too_large", False), 415: (400, "invalid_request_error", False), @@ -70,16 +74,22 @@ def safe_provider_message(exc: BaseException) -> str | None: """Extract one bounded, control-free diagnostic sentence from a failure. - Only an HTTP error's JSON ``error.message``-style fields pass through; - every other cause returns ``None`` so callers fall back to package-owned - sentences. Provider diagnostics can embed URLs, secrets, prompts, or - internal topology, so raw exception text never reaches API callers. + An HTTP error contributes only JSON ``error.message``-style fields. For + non-HTTP exceptions, this low-level helper returns a bounded first argument + or exception type for internal diagnostics; caller-facing classification + deliberately replaces those values with package-owned sentences. + Provider diagnostics can embed URLs, secrets, prompts, or internal topology, + so raw exception text never reaches API callers. Control characters are collapsed so no body can smuggle log-formatting or header content downstream. """ if isinstance(exc, urllib.error.HTTPError): try: - payload = _json.loads(exc.read().decode("utf-8", errors="replace")) + payload = _json.loads( + exc.read(MAX_PROVIDER_ERROR_BODY_BYTES + 1)[ + :MAX_PROVIDER_ERROR_BODY_BYTES + ].decode("utf-8", errors="replace") + ) except Exception: # noqa: BLE001 - bodies are untrusted input return None raw: Any = None diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 12a8a0bd2..b244c93d6 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -6857,6 +6857,19 @@ def progress(role: str, status: str) -> None: ) except ConnectionAbortedError: raise + except ProviderUpstreamError as exc: + failed = { + **created_response, + "status": "failed", + "error": _error_payload( + exc.error_code, + _provider_upstream_message(exc), + {"request_id": uuid.uuid4().hex, **exc.detail}, + )["error"], + } + emit("response.failed", response=failed) + self._write_sse("data: [DONE]\n\n") + return False except Exception: # noqa: BLE001 - headers sent; terminate with a valid Responses event failed = { **created_response, @@ -6980,6 +6993,18 @@ def frame(delta: dict[str, Any], finish: str | None = None) -> str: return if not self._write_sse(frame({}, finish="error")): return + except ProviderUpstreamError as exc: + payload = _error_payload( + exc.error_code, + _provider_upstream_message(exc), + {"request_id": uuid.uuid4().hex, **exc.detail}, + ) + if not self._write_sse( + f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + ): + return + if not self._write_sse(frame({}, finish="error")): + return except Exception: # noqa: BLE001 - headers already sent; surface as a terminal error frame if not self._write_sse(frame({}, finish="error")): return diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index 09bf6a059..1dc149130 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -153,17 +153,24 @@ def inject_trace_context(headers: dict[str, str]) -> None: def _safe_attributes( attributes: Mapping[str, Any] | None, -) -> dict[str, str | int | float | bool]: - """Keep only approved scalar span attributes; prompts and secrets never enter OTLP.""" - result: dict[str, str | int | float | bool] = {} +) -> dict[str, str | int | float | bool | list[str]]: + """Keep approved bounded span attributes; prompts and secrets never enter OTLP.""" + result: dict[str, str | int | float | bool | list[str]] = {} for key, value in (attributes or {}).items(): if ( not isinstance(key, str) or not key or key not in _ALLOWED_ATTRIBUTE_KEYS - or isinstance(value, (dict, list, tuple, set)) + or isinstance(value, (dict, set)) ): continue + if key == "gen_ai.response.finish_reasons" and isinstance(value, (list, tuple)): + reasons = [item[:256] for item in value if isinstance(item, str) and item] + if reasons: + result[key] = reasons + continue + if isinstance(value, (list, tuple)): + continue if isinstance(value, str): result[key] = value[:256] elif isinstance(value, (bool, int, float)): @@ -198,7 +205,11 @@ def record_provider_usage(usage: Mapping[str, Any] | None) -> None: def _count(key: str) -> int | None: value = usage.get(key) - return value if isinstance(value, int) and not isinstance(value, bool) else None + return ( + value + if isinstance(value, int) and not isinstance(value, bool) and value >= 0 + else None + ) attributes: dict[str, Any] = {} input_tokens = _count("prompt_tokens") if "prompt_tokens" in usage else _count("input_tokens") diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index 6265f3285..a0bfd0018 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -23,6 +23,7 @@ from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 from contextual_orchestrator.provider_errors import ( # noqa: E402 + MAX_PROVIDER_ERROR_BODY_BYTES, MAX_SAFE_MESSAGE_CHARS, PROVIDER_STATUS_SURFACES, ProviderUpstreamError, @@ -66,6 +67,24 @@ def test_safe_message_hides_unparseable_bodies_and_urls() -> None: assert safe_provider_message(_http_error(502)) is None +def test_safe_message_reads_only_a_bounded_provider_body() -> None: + """Untrusted provider bodies are bounded before JSON parsing.""" + + class RecordingBody(io.BytesIO): + requested_size: int | None = None + + def read(self, size: int = -1) -> bytes: + self.requested_size = size + return super().read(size) + + body = RecordingBody(b"{}") + error = urllib.error.HTTPError( + "https://provider.example/chat/completions", 500, "error", None, body + ) + assert safe_provider_message(error) is None + assert body.requested_size == MAX_PROVIDER_ERROR_BODY_BYTES + 1 + + def test_safe_message_collapses_control_characters_and_bounds_length() -> None: """Control characters cannot smuggle log or header content; length is bounded.""" long = safe_provider_message( @@ -98,7 +117,7 @@ def test_classification_maps_every_upstream_status_to_openai_surface() -> None: 403: (403, "permission_error", False), 404: (404, "model_not_found", False), 408: (504, "provider_timeout", True), - 409: (409, "conflict", False), + 409: (409, "conflict", True), 413: (413, "request_too_large", False), 422: (400, "invalid_request_error", False), 425: (429, "rate_limit_exceeded", True), diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 00266bd08..2dc55bd15 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -94,6 +94,16 @@ def test_session_and_attribute_boundaries_reject_unsafe_values(): reset_session_id(token) +def test_finish_reason_attribute_accepts_only_bounded_string_arrays(): + """The standard finish-reasons array remains bounded and prompt-safe.""" + assert telemetry_module._safe_attributes( + {"gen_ai.response.finish_reasons": ["stop", "x" * 300, 2, ""]} + ) == {"gen_ai.response.finish_reasons": ["stop", "x" * 256]} + assert telemetry_module._safe_attributes( + {"gen_ai.response.finish_reasons": {"stop": True}} + ) == {} + + def test_session_binding_is_reset(): """A request cannot leak its session into a later request context.""" token = set_session_id("session-2") @@ -526,6 +536,16 @@ def test_record_provider_usage_accepts_responses_style_keys(monkeypatch): ] +def test_record_provider_usage_rejects_negative_counts(monkeypatch): + """Impossible negative token counts never enter GenAI telemetry.""" + captured: list[dict] = [] + monkeypatch.setattr(telemetry_module, "annotate_current_span", captured.append) + telemetry_module.record_provider_usage( + {"prompt_tokens": -1, "completion_tokens": 2, "total_tokens": -3} + ) + assert captured == [{"gen_ai.usage.output_tokens": 2}] + + def test_provider_response_telemetry_records_latency_model_finish_reason(monkeypatch): """One completed provider response annotates usage, model, reason, latency.""" captured: dict[str, list] = {"annotate": [], "usage": []} @@ -565,8 +585,31 @@ def fake_open(request, destination, timeout=None): # noqa: ARG001 assert content == "ok" annotated = captured["annotate"][0] assert annotated["gen_ai.response.model"] == "gpt-x-served" - assert annotated["gen_ai.response.finish_reasons"] == "stop" + assert annotated["gen_ai.response.finish_reasons"] == ["stop"] assert isinstance(annotated["contextual_orchestrator.latency_ms"], float) assert captured["usage"] == [ {"prompt_tokens": 5, "completion_tokens": 6, "total_tokens": 11} ] + + +def test_provider_response_telemetry_accepts_empty_and_multiple_choices(monkeypatch): + """Usage-only responses do not crash and all valid finish reasons are recorded.""" + captured: list[dict] = [] + monkeypatch.setattr(orchestrator_module, "annotate_current_span", captured.append) + monkeypatch.setattr(orchestrator_module, "record_provider_usage", lambda unused: None) + + orchestrator_module._record_provider_response_telemetry({"choices": []}, 0.0) + orchestrator_module._record_provider_response_telemetry( + { + "choices": [ + {"finish_reason": "stop"}, + {"finish_reason": "length"}, + {"finish_reason": None}, + "malformed", + ] + }, + 0.0, + ) + + assert "gen_ai.response.finish_reasons" not in captured[0] + assert captured[1]["gen_ai.response.finish_reasons"] == ["stop", "length"] diff --git a/tests/test_true_streaming.py b/tests/test_true_streaming.py index ba03c380d..7a16ef082 100644 --- a/tests/test_true_streaming.py +++ b/tests/test_true_streaming.py @@ -274,6 +274,81 @@ def post(payload: dict) -> tuple[str, str]: assert streamed == reference # live-streamed deltas equal the non-streamed route answer +def test_chat_stream_preserves_classified_provider_error_payload() -> None: + """A terminal chat SSE frame keeps the actionable upstream taxonomy.""" + server = build_server(TaskOrchestrator([ModelAgent("general_agent", "m-model")]), port=0) + handler = server.RequestHandlerClass.__new__(server.RequestHandlerClass) + frames: list[str] = [] + + class Orchestrator: + def stream_route(self, *_args, **_kwargs): + raise ProviderUpstreamError( + agent_id="worker_agent", + model="gpt-x", + error_code="rate_limit_exceeded", + message="provider throttled the request", + client_status=429, + provider_status=429, + retryable=True, + ) + yield "unreachable" + + handler._begin_sse = lambda: True + handler._write_sse = lambda frame: frames.append(frame) is None or True + try: + handler._stream_route_completion( + Orchestrator(), SecurityConfig(auth_token="stream-token"), [], "gpt-x" + ) + finally: + server.server_close() + + errors = [ + json.loads(frame.removeprefix("data: ")) + for frame in frames + if frame.startswith("data: {") and '"error_code"' in frame + ] + assert errors[0]["error_code"] == "rate_limit_exceeded" + assert errors[0]["error_detail"]["provider_status"] == 429 + assert errors[0]["error_detail"]["retryable"] is True + + +def test_responses_stream_preserves_classified_provider_error_payload() -> None: + """A failed Responses SSE event keeps the actionable upstream taxonomy.""" + server = build_server(TaskOrchestrator([ModelAgent("general_agent", "m-model")]), port=0) + handler = server.RequestHandlerClass.__new__(server.RequestHandlerClass) + frames: list[str] = [] + + class Orchestrator: + def would_route(self, *_args, **_kwargs): + return False + + def conduct(self, *_args, **_kwargs): + raise ProviderUpstreamError( + agent_id="worker_agent", + model="gpt-x", + error_code="rate_limit_exceeded", + message="provider throttled the request", + client_status=429, + provider_status=429, + retryable=True, + ) + + handler._begin_sse = lambda: True + handler._write_sse = lambda frame: frames.append(frame) is None or True + try: + assert handler._stream_orchestrated_response( + Orchestrator(), SecurityConfig(auth_token="stream-token"), [], "gpt-x" + ) is False + finally: + server.server_close() + + failed_frame = next(frame for frame in frames if "response.failed" in frame) + payload = json.loads(failed_frame.split("data: ", 1)[1]) + assert payload["response"]["error"]["code"] == "rate_limit_exceeded" + assert payload["response"]["error"]["detail"]["provider_status"] == 429 + assert payload["response"]["error"]["detail"]["retryable"] is True + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_") and callable(fn): From ed0690e3e2ff3e017ab17d21b3e8d455ef703b05 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 21:06:08 +0900 Subject: [PATCH 03/33] fix(routing): honor nonretryable provider failures --- contextual_orchestrator/orchestrator.py | 7 ++++++ tests/test_provider_error_taxonomy.py | 31 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 3d9ea8287..a72fc8cf6 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -5395,6 +5395,13 @@ def call(agent: ModelAgent) -> tuple[str, str, dict[str, Any] | None]: last_upstream_error = exc decision = classify_tool_failure(exc) action = decision.action + if ( + isinstance(exc, ProviderUpstreamError) + and not exc.retryable + and action is ToolFallbackAction.RETRY_SAME_AGENT + ): + decision = downgrade_to_failover(decision) + action = decision.action # A failed attempt is one Bernoulli stability observation # for measured group routing regardless of what happens next. if ( diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index a0bfd0018..38850e02a 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -272,6 +272,37 @@ def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> s raise AssertionError("the final classified failure must survive failover") +def test_invoke_does_not_retry_nonretryable_provider_failure_on_same_agent() -> None: + """A classified auth failure advances to the backup without repeating it.""" + + class AuthThenBackup(ModelClient): + def __init__(self) -> None: + super().__init__() + self.calls: list[str] = [] + + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + self.calls.append(agent.id) + if agent.id == "primary_worker": + raise classify_provider_failure( + _http_error(401), agent_id=agent.id, model=agent.model + ) + return "backup answer" + + client = AuthThenBackup() + agents = [ + ModelAgent("primary_worker", "mock-a", tags=("reasoning",), priority=2), + ModelAgent("backup_worker", "mock-b", tags=("reasoning",), priority=1), + ] + orchestrator = TaskOrchestrator(agents, client=client, tool_retry_attempts=2) + orchestrator._triage_fn = lambda text: False + + result = orchestrator.route_once([{"role": "user", "content": "route this"}]) + + assert result["answer"] == "backup answer" + assert client.calls[0:2] == ["primary_worker", "backup_worker"] + assert client.calls.count("primary_worker") == 1 + + # -- server error surface ------------------------------------------------------ From 44d38cc8857abfd2044a92535e50ad3e3b181ad9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 21:51:33 +0900 Subject: [PATCH 04/33] fix: preserve provider error bodies across classification --- contextual_orchestrator/orchestrator.py | 8 ++++++-- contextual_orchestrator/provider_errors.py | 19 ++++++++++++++++--- tests/test_provider_error_taxonomy.py | 10 +++++++++- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index a72fc8cf6..fb3800087 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -40,7 +40,11 @@ from .credentials import NotConfigured, get_credential from .model_group import ModelGroupRouter, canonical_group_name from .endpoint_race import EndpointAttempt, EndpointEquivalenceContract, race_first_valid -from .provider_errors import ProviderUpstreamError, classify_provider_failure +from .provider_errors import ( + ProviderUpstreamError, + classify_provider_failure, + provider_error_body, +) from .telemetry import ( annotate_current_span, inject_trace_context, @@ -559,7 +563,7 @@ def _is_tool_execution_stopped(error: urllib.error.HTTPError) -> bool: if isinstance(cached, bool): return cached try: - payload = json.loads(error.read(65536).decode("utf-8")) + payload = json.loads(provider_error_body(error).decode("utf-8")) except (AttributeError, OSError, UnicodeDecodeError, json.JSONDecodeError): result = False else: diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index ce84753f5..9f463861f 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -31,6 +31,7 @@ "PROVIDER_STATUS_SURFACES", "ProviderUpstreamError", "classify_provider_failure", + "provider_error_body", "safe_provider_message", ] @@ -71,6 +72,20 @@ _UNMAPPED_UPSTREAM_SURFACE: tuple[int, str, bool] = (502, "api_error", False) +def provider_error_body(exc: urllib.error.HTTPError) -> bytes: + """Read and cache one bounded upstream error body for all classifiers.""" + cache_key = "_contextual_orchestrator_provider_error_body" + cached = getattr(exc, cache_key, None) + if isinstance(cached, bytes): + return cached + body = exc.read(MAX_PROVIDER_ERROR_BODY_BYTES + 1)[:MAX_PROVIDER_ERROR_BODY_BYTES] + try: + setattr(exc, cache_key, body) + except (AttributeError, TypeError): # pragma: no cover - HTTPError is mutable + pass + return body + + def safe_provider_message(exc: BaseException) -> str | None: """Extract one bounded, control-free diagnostic sentence from a failure. @@ -86,9 +101,7 @@ def safe_provider_message(exc: BaseException) -> str | None: if isinstance(exc, urllib.error.HTTPError): try: payload = _json.loads( - exc.read(MAX_PROVIDER_ERROR_BODY_BYTES + 1)[ - :MAX_PROVIDER_ERROR_BODY_BYTES - ].decode("utf-8", errors="replace") + provider_error_body(exc).decode("utf-8", errors="replace") ) except Exception: # noqa: BLE001 - bodies are untrusted input return None diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index 38850e02a..a1e41e276 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -21,7 +21,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 +from contextual_orchestrator.orchestrator import ModelClient, is_transient_error # noqa: E402 from contextual_orchestrator.provider_errors import ( # noqa: E402 MAX_PROVIDER_ERROR_BODY_BYTES, MAX_SAFE_MESSAGE_CHARS, @@ -85,6 +85,14 @@ def read(self, size: int = -1) -> bytes: assert body.requested_size == MAX_PROVIDER_ERROR_BODY_BYTES + 1 +def test_safe_message_reuses_body_after_retryability_inspection() -> None: + """Tool-stop and caller-safe classification share one bounded body read.""" + error = _body_http_error(401, {"error": {"message": "invalid credential"}}) + + assert is_transient_error(error) is False + assert safe_provider_message(error) == "invalid credential" + + def test_safe_message_collapses_control_characters_and_bounds_length() -> None: """Control characters cannot smuggle log or header content; length is bounded.""" long = safe_provider_message( From bcd327329fb23b3a4e07fe7b48259da6c0025334 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 22:06:24 +0900 Subject: [PATCH 05/33] fix: classify exhausted passthrough providers --- contextual_orchestrator/orchestrator.py | 23 ++++++++++++++++----- tests/test_passthrough_provider_failover.py | 17 ++++++++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index fb3800087..be3a88d26 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1356,7 +1356,7 @@ def _send_with_retry( timeout: float | None = None, ) -> str: """Call the provider, retrying transient failures with exponential backoff + jitter.""" - last_error: Exception | None = None + last_failure: tuple[Exception, ModelAgent] | None = None retry_limit = self._retry_limit(agent) for attempt in range(retry_limit + 1): # pragma: no branch - retry limits are validated non-negative try: @@ -3169,8 +3169,15 @@ def proxy_completion( result = send_once(candidate, endpoint, candidate_payload) except Exception as exc: # noqa: BLE001 - provider trust boundary if not _is_passthrough_failover_error(exc): + if isinstance(exc, (urllib.error.HTTPError, ProviderUpstreamError)): + raise classify_provider_failure( + exc, + agent_id=candidate.id, + model=candidate.model, + transport="passthrough", + ) from exc raise - last_error = exc + last_failure = (exc, candidate) self._record_failure(candidate.id) if candidate.group_name: self._group_router.observe_failure(candidate.id) @@ -3181,9 +3188,15 @@ def proxy_completion( candidate.id, time.perf_counter() - started_at ) return result - raise RuntimeError( - f"all {len(candidates)} candidate agents failed for passthrough endpoint={endpoint}" - ) from last_error + if last_failure is not None: + last_error, failed_candidate = last_failure + raise classify_provider_failure( + last_error, + agent_id=failed_candidate.id, + model=failed_candidate.model, + transport="passthrough", + ) from last_error + raise RuntimeError("passthrough has no eligible provider candidate") def _orchestrated_provider_completion( self, diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 38c522c26..afa296d4d 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -16,6 +16,7 @@ TaskOrchestrator, ) from contextual_orchestrator.orchestrator import ModelClient +from contextual_orchestrator.provider_errors import ProviderUpstreamError class SequencedProxyClient: @@ -198,7 +199,7 @@ def test_free_virtual_model_never_fails_over_to_a_paid_agent() -> None: client=client, ) - with pytest.raises(RuntimeError, match="all 1 candidate agents failed"): + with pytest.raises(ProviderUpstreamError) as caught: orchestrator.proxy_completion( { "model": TaskOrchestrator.FREE_MODEL, @@ -206,6 +207,8 @@ def test_free_virtual_model_never_fails_over_to_a_paid_agent() -> None: } ) + assert caught.value.agent_id == "free_agent" + assert [agent_id for agent_id, _ in client.calls] == ["free_agent"] @@ -219,10 +222,11 @@ def test_non_transient_error_is_not_replayed() -> None: } ) - with pytest.raises(urllib.error.HTTPError) as caught: + with pytest.raises(ProviderUpstreamError) as caught: _build(client).proxy_completion({"messages": [{"role": "user", "content": "x"}]}) - assert caught.value is failure + assert caught.value.provider_status == 400 + assert caught.value.__cause__ is failure assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] @@ -274,9 +278,11 @@ def test_all_candidates_chain_the_last_failure() -> None: SequencedProxyClient({"primary_agent": _http_error(429), "fallback_agent": final}) ) - with pytest.raises(RuntimeError, match="all 2 candidate agents failed") as caught: + with pytest.raises(ProviderUpstreamError) as caught: orchestrator.proxy_completion({"messages": [{"role": "user", "content": "x"}]}) + assert caught.value.agent_id == "fallback_agent" + assert caught.value.provider_status == 503 assert caught.value.__cause__ is final @@ -491,9 +497,10 @@ def test_default_mock_endpoint_represents_one_fixture_provider() -> None: client=client, ) - with pytest.raises(RuntimeError, match="all 1 candidate agents failed"): + with pytest.raises(ProviderUpstreamError) as caught: orchestrator.proxy_completion( {"model": orchestrator.AUTO_MODEL, "messages": [{"role": "user", "content": "x"}]} ) + assert caught.value.agent_id == "first_mock" assert [agent_id for agent_id, _ in client.calls] == ["first_mock"] From 032ad4158045c7f8f98e14d60a5e859abf7e8f95 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 22:19:23 +0900 Subject: [PATCH 06/33] fix: initialize provider retry failure state --- contextual_orchestrator/orchestrator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index be3a88d26..075707d3e 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1356,7 +1356,7 @@ def _send_with_retry( timeout: float | None = None, ) -> str: """Call the provider, retrying transient failures with exponential backoff + jitter.""" - last_failure: tuple[Exception, ModelAgent] | None = None + last_error: Exception | None = None retry_limit = self._retry_limit(agent) for attempt in range(retry_limit + 1): # pragma: no branch - retry limits are validated non-negative try: @@ -1765,7 +1765,7 @@ def _send_raw_with_retry( allow_transient_retries: bool = True, ) -> dict[str, Any]: # pragma: no cover """Passthrough transport with the same transient-failure retry policy as _send.""" - last_error: Exception | None = None + last_failure: tuple[Exception, ModelAgent] | None = None retry_limit = self._retry_limit(agent) if allow_transient_retries else 0 for attempt in range(retry_limit + 1): try: From b6ba2560890d9f19a8511537f81f3c093497d491 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 01:08:44 +0900 Subject: [PATCH 07/33] fix(telemetry): normalize trace provider names --- contextual_orchestrator/orchestrator.py | 8 ++++---- tests/test_true_streaming.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 075707d3e..6d022e3cd 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3594,7 +3594,7 @@ def stream_route( "role": "worker", "agent_id": agent.id, "model": agent.model, - "provider": agent.provider_name or agent.base_url.split("://", 1)[-1], + "provider": agent.provider_name or self._infer_provider_name(agent.base_url), "subtask": "Direct route (streamed)", "access": [], "latency_ms": round(latency_seconds * 1000, 2), @@ -3815,7 +3815,7 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]: row: dict[str, Any] = { "id": 0, "role": "worker", "agent_id": agent.id, "model": agent.model, - "provider": agent.provider_name or agent.base_url.split("://", 1)[-1], + "provider": agent.provider_name or self._infer_provider_name(agent.base_url), "subtask": "Direct route (batched)", "access": [], "output": result["content"], } if result.get("usage") is not None: @@ -4328,7 +4328,7 @@ def route_once( "role": "worker", "agent_id": candidate.id, "model": candidate.model, - "provider": candidate.provider_name or candidate.base_url.split("://", 1)[-1], + "provider": candidate.provider_name or self._infer_provider_name(candidate.base_url), "subtask": "Direct route", "access": [], "latency_ms": round(latency_seconds * 1000, 2), @@ -4540,7 +4540,7 @@ def conduct( row["agent_id"] = agent.id row["latency_ms"] = round(elapsed, 2) row["model"] = agent.model - row["provider"] = agent.provider_name or agent.base_url.split("://", 1)[-1] + row["provider"] = agent.provider_name or self._infer_provider_name(agent.base_url) row["output"] = output if usage is not None: row["usage"] = usage diff --git a/tests/test_true_streaming.py b/tests/test_true_streaming.py index 7a16ef082..99dd959ea 100644 --- a/tests/test_true_streaming.py +++ b/tests/test_true_streaming.py @@ -172,6 +172,17 @@ def test_stream_route_yields_and_persists() -> None: assert len(orchestrator._workflow_runs) == 1 # streamed run still persisted for observability +def test_stream_route_uses_canonical_provider_name_in_trace() -> None: + orchestrator = TaskOrchestrator( + [ModelAgent("general_agent", "m-model", base_url="mock://provider/path")] + ) + + list(orchestrator.stream_route([{"role": "user", "content": "stream"}])) + + trace = next(iter(orchestrator._workflow_runs.values()))["trace"] + assert trace[0]["provider"] == "mock-provider/path" + + def test_stream_route_persists_owner() -> None: """Authenticated streaming lineage retains its owner boundary.""" orchestrator = TaskOrchestrator( From 6482cb975028c830c952397eb641ec9bfa929323 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 01:17:14 +0900 Subject: [PATCH 08/33] fix(telemetry): complete provider failure evidence --- contextual_orchestrator/orchestrator.py | 18 +++++++++++++++--- tests/test_passthrough_provider_failover.py | 12 ++++++++++++ tests/test_true_streaming.py | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 6d022e3cd..6e02e8214 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1598,6 +1598,8 @@ def _stream_send( stream_error: RuntimeError | None = None started = time.monotonic() stream_usage: dict[str, Any] | None = None + stream_model: str | None = None + stream_choices: list[dict[str, str]] = [] try: with self._open_provider(request, destination) as response: for raw in response: @@ -1613,12 +1615,22 @@ def _stream_send( continue if isinstance(chunk, dict) and isinstance(chunk.get("usage"), dict): stream_usage = chunk["usage"] + if isinstance(chunk, dict) and isinstance(chunk.get("model"), str): + stream_model = chunk["model"] choices = chunk.get("choices") or [{}] + stream_choices.extend( + {"finish_reason": choice["finish_reason"]} + for choice in choices + if isinstance(choice, dict) + and isinstance(choice.get("finish_reason"), str) + and choice["finish_reason"] + ) delta = (choices[0] or {}).get("delta", {}).get("content") if delta: yield delta _record_provider_response_telemetry( - {"usage": stream_usage} if stream_usage else {}, started + {"usage": stream_usage, "model": stream_model, "choices": stream_choices}, + started, ) except Exception as exc: # noqa: BLE001 - provider error boundary (CWE-209) # The gateway's own terminal tool-stop contract must survive the @@ -1765,7 +1777,7 @@ def _send_raw_with_retry( allow_transient_retries: bool = True, ) -> dict[str, Any]: # pragma: no cover """Passthrough transport with the same transient-failure retry policy as _send.""" - last_failure: tuple[Exception, ModelAgent] | None = None + last_error: Exception | None = None retry_limit = self._retry_limit(agent) if allow_transient_retries else 0 for attempt in range(retry_limit + 1): try: @@ -3153,7 +3165,7 @@ def proxy_completion( continue seen_providers.add(provider_key) candidates.append(candidate) - last_error: Exception | None = None + last_failure: tuple[Exception, ModelAgent] | None = None for candidate in candidates: started_at = time.perf_counter() candidate_payload = dict(upstream) diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index afa296d4d..5fa303947 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -444,6 +444,18 @@ def test_virtual_effort_profile_selects_a_supported_provider() -> None: assert [agent_id for agent_id, _ in client.calls] == ["supported_agent"] +def test_passthrough_with_no_ranked_provider_fails_cleanly(monkeypatch) -> None: + """An empty filtered pool reports unavailability instead of reading stale state.""" + client = SequencedProxyClient({"primary_agent": {"model": "primary-model"}}) + orchestrator = _build(client) + monkeypatch.setattr(orchestrator, "_failover_candidates", lambda *args, **kwargs: []) + + with pytest.raises(RuntimeError, match="no eligible provider candidate"): + orchestrator.proxy_completion({"messages": [{"role": "user", "content": "x"}]}) + + assert client.calls == [] + + def test_effort_support_filter_precedes_same_provider_deduplication() -> None: """A lower-ranked supported alias remains eligible for its provider.""" client = SequencedProxyClient( diff --git a/tests/test_true_streaming.py b/tests/test_true_streaming.py index 99dd959ea..1bc7b6327 100644 --- a/tests/test_true_streaming.py +++ b/tests/test_true_streaming.py @@ -100,6 +100,26 @@ def test_stream_send_ignores_empty_and_missing_choices() -> None: assert deltas == ["before", "after"] +def test_stream_send_records_response_model_and_finish_reason(monkeypatch) -> None: + """Streaming spans retain the same response identity as non-streaming calls.""" + captured: list[dict] = [] + monkeypatch.setattr( + "contextual_orchestrator.orchestrator.annotate_current_span", captured.append + ) + frames = [ + 'data: {"model":"served-model","choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n', + 'data: {"model":"served-model","choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', + "data: [DONE]\n\n", + ] + with _FakeSSEProvider(frames) as provider: + client = ModelClient() + agent = ModelAgent("worker_agent", "gpt-x", base_url=provider.base_url) + assert list(client._stream_send(agent, {"model": "gpt-x", "stream": True})) == ["ok"] + + assert captured[-1]["gen_ai.response.model"] == "served-model" + assert captured[-1]["gen_ai.response.finish_reasons"] == ["stop"] + + def test_stream_send_hides_raw_provider_error_text_and_cause() -> None: """A mid-stream provider failure surfaces one package-owned error (CWE-209). From f2c444aa4d279fa75a19212a12a84e327f382e69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 01:40:26 +0900 Subject: [PATCH 09/33] fix(errors): suppress raw passthrough causes --- contextual_orchestrator/orchestrator.py | 4 ++-- tests/test_passthrough_provider_failover.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 6e02e8214..c117bb957 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3187,7 +3187,7 @@ def proxy_completion( agent_id=candidate.id, model=candidate.model, transport="passthrough", - ) from exc + ) from None raise last_failure = (exc, candidate) self._record_failure(candidate.id) @@ -3207,7 +3207,7 @@ def proxy_completion( agent_id=failed_candidate.id, model=failed_candidate.model, transport="passthrough", - ) from last_error + ) from None raise RuntimeError("passthrough has no eligible provider candidate") def _orchestrated_provider_completion( diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 5fa303947..487234458 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -226,7 +226,7 @@ def test_non_transient_error_is_not_replayed() -> None: _build(client).proxy_completion({"messages": [{"role": "user", "content": "x"}]}) assert caught.value.provider_status == 400 - assert caught.value.__cause__ is failure + assert caught.value.__cause__ is None assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] @@ -283,7 +283,7 @@ def test_all_candidates_chain_the_last_failure() -> None: assert caught.value.agent_id == "fallback_agent" assert caught.value.provider_status == 503 - assert caught.value.__cause__ is final + assert caught.value.__cause__ is None @pytest.mark.parametrize( From ecf8c1e49eb53773e91b63c92832c741ce088870 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:19:11 +0900 Subject: [PATCH 10/33] fix(errors): preserve capability and DNS taxonomy --- contextual_orchestrator/orchestrator.py | 2 ++ contextual_orchestrator/provider_errors.py | 12 +++++++ tests/test_provider_error_taxonomy.py | 42 ++++++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index c117bb957..44e1479b7 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -5304,6 +5304,8 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: continue self._group_router.observe_success(agent.id, time.perf_counter() - started_at) return result + if isinstance(last_error, ProviderUpstreamError): + raise last_error raise RuntimeError(f"all {capability} providers failed") from last_error def _invoke( diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index 9f463861f..b02178507 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -222,6 +222,18 @@ def classify_provider_failure( retryable=True, transport=transport, ) + dns_error = exc if isinstance(exc, socket.gaierror) else exc.__cause__ + if isinstance(dns_error, socket.gaierror): + return ProviderUpstreamError( + agent_id=agent_id, + model=model, + error_code="provider_connection_error", + message=f"the provider {agent_id} host could not be resolved", + client_status=502, + provider_status=None, + retryable=dns_error.errno == socket.EAI_AGAIN, + transport=transport, + ) if isinstance(exc, (urllib.error.URLError, TimeoutError, ConnectionError, socket.timeout)): return ProviderUpstreamError( agent_id=agent_id, diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index a1e41e276..0268e8580 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -154,6 +154,20 @@ def test_classification_handles_network_tls_and_unknown_causes() -> None: for exc in (socket.timeout("slow"), TimeoutError("deadline"), ConnectionError("reset")): assert classify_provider_failure(exc, agent_id="a", model="m").retryable + temporary_dns = socket.gaierror(socket.EAI_AGAIN, "temporary DNS failure") + wrapped_dns = RuntimeError("provider host could not be resolved") + wrapped_dns.__cause__ = temporary_dns + for exc in (temporary_dns, wrapped_dns): + dns = classify_provider_failure(exc, agent_id="a", model="m") + assert dns.error_code == "provider_connection_error" + assert dns.retryable + + permanent_dns = classify_provider_failure( + socket.gaierror(socket.EAI_NONAME, "unknown host"), agent_id="a", model="m" + ) + assert permanent_dns.error_code == "provider_connection_error" + assert not permanent_dns.retryable + cert = ssl.SSLCertVerificationError("bad chain") tls_verify = classify_provider_failure(cert, agent_id="a", model="m") assert tls_verify.error_code == "tls_verification_failed" @@ -182,6 +196,34 @@ def test_classification_handles_network_tls_and_unknown_causes() -> None: assert classify_provider_failure(original, agent_id="b", model="n") is original +def test_capability_dispatch_preserves_last_classified_provider_failure() -> None: + """Non-chat capability exhaustion keeps the actionable upstream taxonomy.""" + orchestrator = TaskOrchestrator( + [ModelAgent("image_agent", "image-model", tags=("image",))] + ) + expected = ProviderUpstreamError( + agent_id="image_agent", + model="image-model", + error_code="rate_limit_exceeded", + message="provider rate limit reached", + client_status=429, + retryable=True, + ) + def fail(*_args: object, **_kwargs: object) -> dict: + raise expected + + orchestrator.client.proxy_send = fail # type: ignore[method-assign] + + try: + orchestrator.proxy_capability( + {"prompt": "demo"}, capability="image", endpoint="images/generations" + ) + except ProviderUpstreamError as raised: + assert raised is expected + else: + raise AssertionError("classified capability failure was not raised") + + def test_detail_and_transport_are_preserved_for_callers() -> None: """The structured detail names agent/model/status/retryability/transport.""" classified = classify_provider_failure( From be6972c43cbf6506121633f9a3880c2c9567137c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 03:08:24 +0900 Subject: [PATCH 11/33] fix(telemetry): bound and complete provider evidence --- CHANGELOG.md | 4 ++- contextual_orchestrator/orchestrator.py | 5 ++- contextual_orchestrator/telemetry.py | 9 ++++- tests/test_telemetry.py | 44 +++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fdcb596e..de63155de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,7 +75,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) `gen_ai.usage.input_tokens/output_tokens/total_tokens` from provider-reported counts, served `gen_ai.response.model`, `gen_ai.response.finish_reasons`, request latency, and classified `error.type` plus upstream status on - failures — replacing exception-class-only error labels. + failures — replacing exception-class-only error labels. Chat, streaming, + and passthrough responses share this evidence path, and finish-reason arrays + are bounded to the OpenTelemetry default span-attribute budget. - Orchestration traces now include per-step telemetry evidence: streamed, batched, routed, and conducted steps record `model`, `provider`, and `latency_ms` alongside usage so workflow runs answer which model served a diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 44e1479b7..fd13abb49 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1816,8 +1816,11 @@ def _send_raw( headers=headers, method="POST", ) + started = time.monotonic() with self._open_provider(request, destination) as response: - return json.loads(response.read().decode("utf-8")) + data = json.loads(response.read().decode("utf-8")) + _record_provider_response_telemetry(data, started) + return data def _mock_raw( self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index 1dc149130..3b9fdccb6 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -36,6 +36,9 @@ "contextual_orchestrator_session_id", default=None ) _CONFIGURED = False +# Match the OpenTelemetry SDK's default span-attribute budget so a single +# sequence-valued attribute cannot exceed the span's default evidence budget. +_MAX_ATTRIBUTE_SEQUENCE_ITEMS = 128 _ALLOWED_ATTRIBUTE_KEYS = frozenset( { "gen_ai.operation.name", @@ -165,7 +168,11 @@ def _safe_attributes( ): continue if key == "gen_ai.response.finish_reasons" and isinstance(value, (list, tuple)): - reasons = [item[:256] for item in value if isinstance(item, str) and item] + reasons = [ + item[:256] + for item in value[:_MAX_ATTRIBUTE_SEQUENCE_ITEMS] + if isinstance(item, str) and item + ] if reasons: result[key] = reasons continue diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 1aa93e7dc..91e744318 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -615,3 +615,47 @@ def test_provider_response_telemetry_accepts_empty_and_multiple_choices(monkeypa assert "gen_ai.response.finish_reasons" not in captured[0] assert captured[1]["gen_ai.response.finish_reasons"] == ["stop", "length"] + + +def test_finish_reason_sequence_respects_span_attribute_budget() -> None: + """One provider response cannot create an unbounded sequence attribute.""" + reasons = [f"reason-{index}" for index in range(256)] + attributes = telemetry_module._safe_attributes( + {"gen_ai.response.finish_reasons": reasons} + ) + assert attributes["gen_ai.response.finish_reasons"] == reasons[:128] + + +def test_passthrough_response_records_provider_telemetry(monkeypatch) -> None: + """Feature-rich passthrough responses emit the same evidence as chat.""" + captured: list[dict] = [] + client = ModelClient() + agent = ModelAgent( + "worker_agent", + "gpt-x", + base_url="https://provider.example/v1", + credential_key="", + ) + + @contextmanager + def fake_open(request, destination): # noqa: ARG001 + yield io.BytesIO( + json.dumps( + { + "model": "gpt-x-served", + "choices": [{"finish_reason": "stop"}], + "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5}, + } + ).encode("utf-8") + ) + + monkeypatch.setattr(client, "_open_provider", fake_open) + monkeypatch.setattr(orchestrator_module, "annotate_current_span", captured.append) + monkeypatch.setattr(orchestrator_module, "record_provider_usage", lambda usage: captured.append(dict(usage))) + + response = client._send_raw(agent, "responses", {"input": "hello"}) + + assert response["model"] == "gpt-x-served" + assert captured[0]["gen_ai.response.model"] == "gpt-x-served" + assert captured[0]["gen_ai.response.finish_reasons"] == ["stop"] + assert captured[1] == {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5} From ce537b167966d40332a23ca00a3cc07b9f3536f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 07:06:47 +0900 Subject: [PATCH 12/33] fix(server): restore structured trace contracts after main merge --- CHANGELOG.md | 4 ---- contextual_orchestrator/server.py | 29 ++++++++++++++++------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc17e3780..1f249d80c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,6 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed -<<<<<<< HEAD - Provider/model failures no longer collapse into a generic `internal_error`. A typed provider-error taxonomy (`contextual_orchestrator.provider_errors`) classifies every upstream HTTP status, network, TLS, and transport failure @@ -88,11 +87,8 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) batched, routed, and conducted steps record `model`, `provider`, and `latency_ms` alongside usage so workflow runs answer which model served a step, how long it took, and what it cost. -======= - Validate orchestration-trace requests before every chat execution branch and require trace-purpose authorization before access-report lookup. - ->>>>>>> main - Mixed structured workflows now retain a cost-ledger row for calls whose provider omitted usage, using the existing token-counting fallback while preserving reported counts for the other calls in the same workflow. diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 56231dc37..96084e8ca 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -5956,12 +5956,6 @@ def do_POST(self) -> None: # noqa: N802 # Explicit JSON null on trigger keys is omit-equivalent (SDK optional # defaults) — do not force single-agent passthrough for null-only keys. if body.get("response_format") or tools_list: - if explicit_trace: - raise RequestError( - 400, - "unsupported_trace_disclosure", - "remove include_orchestration_trace or use chat without tools or response_format", - ) tool_loop = bool(tools_list) if ( tool_loop @@ -5977,16 +5971,25 @@ def do_POST(self) -> None: # noqa: N802 tool_loop and body.get("include_orchestration_trace") is True ): + if security.bearer_verifier is None: + raise RequestError( + 400, + "trace_unavailable", + "orchestration trace is unavailable for single-agent tool passthrough", + ) raise RequestError( 400, - "trace_unavailable", - "orchestration trace is unavailable for single-agent tool passthrough", + "unsupported_trace_disclosure", + "remove include_orchestration_trace or use chat without tools or response_format", ) - include_trace = ( - False - if tool_loop - else self._trace_requested(body, "/v1/chat/completions") - ) + if include_trace: + if security.bearer_verifier is not None: + raise RequestError( + 400, + "unsupported_trace_disclosure", + "remove include_orchestration_trace or use chat without tools or response_format", + ) + self._authorize_trace_access() started_at = time.perf_counter() if tool_loop: proxied = self._run( From 93f2b86e8f4f2abbb8a9e4a1eef8ff44bb48d45f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 10:11:13 +0900 Subject: [PATCH 13/33] fix(errors): preserve passthrough failover on classified retries --- contextual_orchestrator/orchestrator.py | 7 ++++ tests/test_passthrough_provider_failover.py | 41 +++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 4b84a04de..a0adbf58a 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1032,6 +1032,13 @@ def _is_passthrough_failover_error(exc: BaseException) -> bool: if current is None or id(current) in seen: return False seen.add(id(current)) + if isinstance(current, ProviderUpstreamError): + if current.provider_status in ( + _PASSTHROUGH_UNAVAILABLE_STATUS | TRANSIENT_HTTP_STATUS + ): + return True + if current.retryable: + return True if ( isinstance(current, urllib.error.HTTPError) and current.code diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 487234458..f5c193371 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -248,6 +248,47 @@ def test_wrapped_transient_error_can_fail_over() -> None: )["model"] == "fallback-model" +def test_proxy_send_only_client_preserves_classified_failover_signal() -> None: + """Virtual passthrough still fails over when only classified proxy_send exists.""" + + class ProxySendOnlyClient: + def __init__(self) -> None: + self.calls: list[tuple[str, dict[str, Any]]] = [] + + def proxy_send( + self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + ) -> dict[str, Any]: + del endpoint + self.calls.append((agent.id, deepcopy(payload))) + if agent.id == "primary_agent": + raise ProviderUpstreamError( + agent_id=agent.id, + model=agent.model, + error_code="model_not_found", + message="provider rejected the request with HTTP 404", + client_status=404, + provider_status=404, + retryable=False, + transport="passthrough", + ) + return {"model": "fallback-model", "choices": []} + + def apply_effort_profile( + self, + agent: ModelAgent, + payload: dict[str, Any], + profile: ReasoningEffortProfile, + ) -> dict[str, Any]: + return ModelClient().apply_effort_profile(agent, payload, profile) + + client = ProxySendOnlyClient() + + assert _build(client).proxy_completion( + {"model": TaskOrchestrator.AUTO_MODEL, "messages": [{"role": "user", "content": "x"}]} + )["model"] == "fallback-model" + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent", "fallback_agent"] + + def test_suppressed_transient_context_does_not_authorize_failover() -> None: """A deliberately hidden exception context cannot become a routing signal.""" try: From 2b1829a81ea79e012480c680a7ef5683dc13c3bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 11:33:00 +0900 Subject: [PATCH 14/33] fix: avoid replaying ambiguous passthrough failures --- contextual_orchestrator/orchestrator.py | 2 -- tests/test_passthrough_provider_failover.py | 25 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index e72bff500..5bf866a17 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1038,8 +1038,6 @@ def _is_passthrough_failover_error(exc: BaseException) -> bool: _PASSTHROUGH_UNAVAILABLE_STATUS | TRANSIENT_HTTP_STATUS ): return True - if current.retryable: - return True if ( isinstance(current, urllib.error.HTTPError) and current.code diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index f5c193371..438b149fa 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -289,6 +289,31 @@ def apply_effort_profile( assert [agent_id for agent_id, _ in client.calls] == ["primary_agent", "fallback_agent"] +def test_classified_ambiguous_connection_error_does_not_fail_over() -> None: + """A retryable connection error without status must not replay passthrough.""" + failure = ProviderUpstreamError( + agent_id="primary_agent", + model="primary-model", + error_code="provider_connection_error", + message="the provider primary_agent connection failed or did not finish in time", + client_status=502, + provider_status=None, + retryable=True, + transport="passthrough", + ) + client = SequencedProxyClient( + {"primary_agent": failure, "fallback_agent": {"model": "fallback-model"}} + ) + + with pytest.raises(ProviderUpstreamError) as caught: + _build(client).proxy_completion( + {"model": TaskOrchestrator.AUTO_MODEL, "messages": [{"role": "user", "content": "x"}]} + ) + + assert caught.value is failure + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] + + def test_suppressed_transient_context_does_not_authorize_failover() -> None: """A deliberately hidden exception context cannot become a routing signal.""" try: From ec17d4e0b77fe10c8087c587cb748d4027fe4d0f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 02:34:05 +0900 Subject: [PATCH 15/33] fix(errors): scrub extended control characters Signed-off-by: Seongho Bae --- contextual_orchestrator/provider_errors.py | 5 ++++- tests/test_provider_error_taxonomy.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index 8d3859a63..5fe8ebdd5 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -120,7 +120,10 @@ def safe_provider_message(exc: BaseException) -> str | None: first_arg = next((part for part in exc.args[:1] if isinstance(part, str)), "") raw = first_arg or type(exc).__name__ collapsed = "".join( - char if ord(char) >= 0x20 or char in "\t" else " " for char in str(raw) + char + if char == "\t" or not (ord(char) < 0x20 or 0x7F <= ord(char) <= 0x9F) + else " " + for char in str(raw) ).strip() return collapsed[:MAX_SAFE_MESSAGE_CHARS] or None diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index 72f1b7e00..925a1696e 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -100,10 +100,10 @@ def test_safe_message_collapses_control_characters_and_bounds_length() -> None: ) assert long is not None assert len(long) == MAX_SAFE_MESSAGE_CHARS - raw = "line1\nline2\ttabbed\x00nul" + raw = "line1\nline2\ttabbed\x00nul\x7fdel\x85next" collapsed = safe_provider_message(_body_http_error(400, {"error": {"message": raw}})) assert collapsed is not None - assert "\n" not in collapsed and "\x00" not in collapsed + assert all(control not in collapsed for control in ("\n", "\x00", "\x7f", "\x85")) assert "\t" in collapsed # tab is preserved for readability From a49a39c9be931c7662bbb2f1141074c0a329a577 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 01:39:46 -0700 Subject: [PATCH 16/33] chore: run one-shot conflict resolver for PR #879 --- .github/workflows/resolve-pr-879.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/resolve-pr-879.yml diff --git a/.github/workflows/resolve-pr-879.yml b/.github/workflows/resolve-pr-879.yml new file mode 100644 index 000000000..8d860706e --- /dev/null +++ b/.github/workflows/resolve-pr-879.yml @@ -0,0 +1,17 @@ +name: Resolve PR 879 against protected main + +on: + push: + branches: [fix/provider-error-taxonomy-telemetry] + +permissions: + contents: write + pull-requests: write + +jobs: + resolve: + uses: ContextualWisdomLab/contextual-orchestrator/.github/workflows/reusable-pr-conflict-resolver.yml@automation/one-shot-pr-conflict-resolver + with: + pr_number: 879 + branch: fix/provider-error-taxonomy-telemetry + caller_file: .github/workflows/resolve-pr-879.yml From f064bebef652faa4b68b0485ddfd8d9cf5629feb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 02:15:33 -0700 Subject: [PATCH 17/33] chore: rerun hardened conflict inventory for PR #879 --- .github/workflows/resolve-pr-879.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/resolve-pr-879.yml b/.github/workflows/resolve-pr-879.yml index 8d860706e..5cb9f1e4c 100644 --- a/.github/workflows/resolve-pr-879.yml +++ b/.github/workflows/resolve-pr-879.yml @@ -1,5 +1,6 @@ name: Resolve PR 879 against protected main +# Hardened diagnostic retry after resolver correction. on: push: branches: [fix/provider-error-taxonomy-telemetry] From ef6faa1b6f45abfd7e2a54b139afd1bf08fbf40a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:28:51 -0700 Subject: [PATCH 18/33] ci: semantically resolve PR 879 against protected main --- .github/workflows/resolve-pr-879.yml | 274 ++++++++++++++++++++++++++- 1 file changed, 267 insertions(+), 7 deletions(-) diff --git a/.github/workflows/resolve-pr-879.yml b/.github/workflows/resolve-pr-879.yml index 5cb9f1e4c..08a96a18a 100644 --- a/.github/workflows/resolve-pr-879.yml +++ b/.github/workflows/resolve-pr-879.yml @@ -1,18 +1,278 @@ name: Resolve PR 879 against protected main -# Hardened diagnostic retry after resolver correction. on: push: - branches: [fix/provider-error-taxonomy-telemetry] + branches: + - fix/provider-error-taxonomy-telemetry permissions: contents: write pull-requests: write +concurrency: + group: resolve-pr-879 + cancel-in-progress: false + jobs: resolve: - uses: ContextualWisdomLab/contextual-orchestrator/.github/workflows/reusable-pr-conflict-resolver.yml@automation/one-shot-pr-conflict-resolver - with: - pr_number: 879 - branch: fix/provider-error-taxonomy-telemetry - caller_file: .github/workflows/resolve-pr-879.yml + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Check out the exact PR branch + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: fix/provider-error-taxonomy-telemetry + fetch-depth: 0 + + - name: Merge protected main and resolve the known semantic conflicts + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git fetch --no-tags origin main + original_head="$(git rev-parse HEAD)" + main_head="$(git rev-parse origin/main)" + + if git merge-base --is-ancestor origin/main HEAD; then + echo "Protected main is already an ancestor of ${original_head}." + else + set +e + git merge --no-ff --no-commit origin/main + merge_rc=$? + set -e + + if [[ $merge_rc -ne 0 ]]; then + mapfile -t conflicts < <(git diff --name-only --diff-filter=U | sort) + expected=( + CHANGELOG.md + contextual_orchestrator/orchestrator.py + contextual_orchestrator/server.py + tests/test_passthrough_provider_failover.py + ) + if [[ "${conflicts[*]}" != "${expected[*]}" ]]; then + printf 'Unexpected conflict set while merging %s into %s:\n' "$main_head" "$original_head" >&2 + printf ' %s\n' "${conflicts[@]}" >&2 + exit 1 + fi + + tmpdir="$(mktemp -d)" + git show :2:CHANGELOG.md > "$tmpdir/changelog.ours" + git show :1:CHANGELOG.md > "$tmpdir/changelog.base" + git show :3:CHANGELOG.md > "$tmpdir/changelog.theirs" + git merge-file --union \ + "$tmpdir/changelog.ours" \ + "$tmpdir/changelog.base" \ + "$tmpdir/changelog.theirs" || true + cp "$tmpdir/changelog.ours" CHANGELOG.md + rm -rf "$tmpdir" + git add CHANGELOG.md + + git checkout --conflict=diff3 -- \ + contextual_orchestrator/orchestrator.py \ + contextual_orchestrator/server.py \ + tests/test_passthrough_provider_failover.py + + python - <<'PY' + from __future__ import annotations + + from pathlib import Path + + + def resolve_file(path: str) -> None: + source = Path(path).read_text(encoding="utf-8") + lines = source.splitlines(keepends=True) + output: list[str] = [] + index = 0 + block_index = 0 + + while index < len(lines): + if not lines[index].startswith("<<<<<<<"): + output.append(lines[index]) + index += 1 + continue + + ours_start = index + 1 + base_marker = ours_start + while base_marker < len(lines) and not lines[base_marker].startswith("|||||||"): + base_marker += 1 + if base_marker >= len(lines): + raise RuntimeError(f"{path}: expected diff3 base marker") + + separator = base_marker + 1 + while separator < len(lines) and not lines[separator].startswith("======="): + separator += 1 + if separator >= len(lines): + raise RuntimeError(f"{path}: expected conflict separator") + + end_marker = separator + 1 + while end_marker < len(lines) and not lines[end_marker].startswith(">>>>>>>"): + end_marker += 1 + if end_marker >= len(lines): + raise RuntimeError(f"{path}: expected conflict end marker") + + ours = lines[ours_start:base_marker] + base = lines[base_marker + 1:separator] + theirs = lines[separator + 1:end_marker] + ours_text = "".join(ours) + base_text = "".join(base) + theirs_text = "".join(theirs) + + if path == "contextual_orchestrator/orchestrator.py": + if block_index == 0: + required_ours = ("stream_usage", "stream_model", "stream_choices.extend") + required_theirs = ("self._local.usage", "choices == []") + if not all(token in ours_text for token in required_ours): + raise RuntimeError("unexpected orchestrator streaming telemetry conflict") + if not all(token in theirs_text for token in required_theirs): + raise RuntimeError("unexpected orchestrator streaming usage conflict") + resolution = ''' if not isinstance(chunk, dict): + continue + usage = chunk.get("usage") + if isinstance(usage, dict): + self._local.usage = usage + stream_usage = usage + if isinstance(chunk.get("model"), str): + stream_model = chunk["model"] + choices = chunk.get("choices") + if choices == []: + continue + if not isinstance(choices, list) or not choices: + continue + stream_choices.extend( + {"finish_reason": choice["finish_reason"]} + for choice in choices + if isinstance(choice, dict) + and isinstance(choice.get("finish_reason"), str) + and choice["finish_reason"] + ) +'''.splitlines(keepends=True) + elif block_index == 1: + if '"model": agent.model' not in ours_text or "trace_step" not in theirs_text: + raise RuntimeError("unexpected streamed trace conflict") + resolution = [" trace_step\n"] + else: + raise RuntimeError(f"unexpected orchestrator conflict block {block_index}") + + elif path == "contextual_orchestrator/server.py": + if block_index == 0: + if "invalid_stream_options" not in theirs_text or "explicit_trace" not in theirs_text: + raise RuntimeError("unexpected structured stream-options conflict") + resolution = theirs + elif block_index == 1: + if "ProviderUpstreamError" not in ours_text or "except Exception" not in ours_text: + raise RuntimeError("unexpected SSE provider-error conflict") + resolution = ours + else: + raise RuntimeError(f"unexpected server conflict block {block_index}") + + elif path == "tests/test_passthrough_provider_failover.py": + if block_index != 0: + raise RuntimeError(f"unexpected passthrough-test conflict block {block_index}") + if "test_passthrough_with_no_ranked_provider_fails_cleanly" not in ours_text: + raise RuntimeError("missing provider-exhaustion regression") + if "test_virtual_responses_effort_profile_uses_responses_wire_shape" not in theirs_text: + raise RuntimeError("missing Responses effort-profile regression") + resolution = ours + theirs + + else: + raise RuntimeError(f"unexpected conflict path {path}") + + output.extend(resolution) + block_index += 1 + index = end_marker + 1 + + expected_blocks = { + "contextual_orchestrator/orchestrator.py": 2, + "contextual_orchestrator/server.py": 2, + "tests/test_passthrough_provider_failover.py": 1, + }[path] + if block_index != expected_blocks: + raise RuntimeError( + f"{path}: resolved {block_index} blocks; expected {expected_blocks}" + ) + + merged = "".join(output) + if any(marker in merged for marker in ("<<<<<<<", "|||||||", "=======", ">>>>>>>")): + raise RuntimeError(f"{path}: unresolved conflict marker remains") + + if path == "contextual_orchestrator/orchestrator.py": + old_trace = ''' trace_step = { + "id": 0, + "role": "worker", + "agent_id": agent.id, + "subtask": "Direct route (streamed)", + "access": [], + "output": answer, + } +''' + new_trace = ''' trace_step = { + "id": 0, + "role": "worker", + "agent_id": agent.id, + "model": agent.model, + "provider": agent.provider_name or self._infer_provider_name(agent.base_url), + "subtask": "Direct route (streamed)", + "access": [], + "latency_ms": round(latency_seconds * 1000, 2), + "output": answer, + } +''' + if merged.count(old_trace) != 1: + raise RuntimeError("streamed trace-step shape changed unexpectedly") + merged = merged.replace(old_trace, new_trace, 1) + + Path(path).write_text(merged, encoding="utf-8") + + + for conflict_path in ( + "contextual_orchestrator/orchestrator.py", + "contextual_orchestrator/server.py", + "tests/test_passthrough_provider_failover.py", + ): + resolve_file(conflict_path) + PY + + git add \ + contextual_orchestrator/orchestrator.py \ + contextual_orchestrator/server.py \ + tests/test_passthrough_provider_failover.py + fi + fi + + test -z "$(git diff --name-only --diff-filter=U)" + rm -f .github/workflows/resolve-pr-879.yml + git add -A + git diff --cached --check + python -m compileall -q contextual_orchestrator + printf 'ORIGINAL_HEAD=%s\nMAIN_HEAD=%s\n' "$original_head" "$main_head" >> "$GITHUB_ENV" + + - name: Set up uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + version: "0.12.5" + + - name: Run conflict-boundary regressions + shell: bash + run: | + set -euo pipefail + uv run --locked --extra api --extra db --extra queue --group dev \ + python -m pytest -q \ + tests/test_provider_error_taxonomy.py \ + tests/test_telemetry.py \ + tests/test_passthrough_provider_failover.py \ + tests/test_streaming.py \ + tests/test_true_streaming.py \ + tests/test_orchestrator_client_boundaries.py \ + tests/test_openai_passthrough.py \ + tests/test_provider_reliability.py + + - name: Commit and push the verified merge result + shell: bash + run: | + set -euo pipefail + test -z "$(git diff --name-only --diff-filter=U)" + git diff --cached --check + git commit -m "chore: merge protected main into PR #879" + git push origin HEAD:fix/provider-error-taxonomy-telemetry + echo "Merged protected main ${MAIN_HEAD} into PR head ${ORIGINAL_HEAD}." From da8393c7640ed0b4bc4a0bff192c75ba0c48c452 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:30:14 -0700 Subject: [PATCH 19/33] ci: isolate PR 879 semantic conflict resolver --- .github/scripts/resolve_pr_879_conflicts.py | 197 ++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 .github/scripts/resolve_pr_879_conflicts.py diff --git a/.github/scripts/resolve_pr_879_conflicts.py b/.github/scripts/resolve_pr_879_conflicts.py new file mode 100644 index 000000000..d2e4463d8 --- /dev/null +++ b/.github/scripts/resolve_pr_879_conflicts.py @@ -0,0 +1,197 @@ +"""Resolve the known semantic conflicts between PR #879 and protected main. + +The script runs only in the one-shot branch workflow. It preserves current-main +streaming usage and Responses contracts while retaining PR #879's typed provider +errors, telemetry, and trace evidence. Every conflict shape is asserted so a +future drift fails closed rather than selecting a side implicitly. +""" + +from __future__ import annotations + +from pathlib import Path + + +ORCHESTRATOR_PATH = "contextual_orchestrator/orchestrator.py" +SERVER_PATH = "contextual_orchestrator/server.py" +PASSTHROUGH_TEST_PATH = "tests/test_passthrough_provider_failover.py" + + +def _resolve_file(path: str) -> None: + """Resolve all strictly recognized diff3 blocks in one conflicted file.""" + source = Path(path).read_text(encoding="utf-8") + lines = source.splitlines(keepends=True) + output: list[str] = [] + index = 0 + block_index = 0 + + while index < len(lines): + if not lines[index].startswith("<<<<<<<"): + output.append(lines[index]) + index += 1 + continue + + ours_start = index + 1 + base_marker = ours_start + while base_marker < len(lines) and not lines[base_marker].startswith("|||||||"): + base_marker += 1 + if base_marker >= len(lines): + raise RuntimeError(f"{path}: expected diff3 base marker") + + separator = base_marker + 1 + while separator < len(lines) and not lines[separator].startswith("======="): + separator += 1 + if separator >= len(lines): + raise RuntimeError(f"{path}: expected conflict separator") + + end_marker = separator + 1 + while end_marker < len(lines) and not lines[end_marker].startswith(">>>>>>>"): + end_marker += 1 + if end_marker >= len(lines): + raise RuntimeError(f"{path}: expected conflict end marker") + + ours = lines[ours_start:base_marker] + base = lines[base_marker + 1 : separator] + theirs = lines[separator + 1 : end_marker] + ours_text = "".join(ours) + base_text = "".join(base) + theirs_text = "".join(theirs) + del base_text # The asserted ours/theirs contracts determine the resolution. + + resolution = _resolve_block( + path, + block_index, + ours=ours, + theirs=theirs, + ours_text=ours_text, + theirs_text=theirs_text, + ) + output.extend(resolution) + block_index += 1 + index = end_marker + 1 + + expected_blocks = { + ORCHESTRATOR_PATH: 2, + SERVER_PATH: 2, + PASSTHROUGH_TEST_PATH: 1, + }[path] + if block_index != expected_blocks: + raise RuntimeError(f"{path}: resolved {block_index} blocks; expected {expected_blocks}") + + merged = "".join(output) + if any(marker in merged for marker in ("<<<<<<<", "|||||||", "=======", ">>>>>>>")): + raise RuntimeError(f"{path}: unresolved conflict marker remains") + + if path == ORCHESTRATOR_PATH: + merged = _enrich_streamed_trace_step(merged) + + Path(path).write_text(merged, encoding="utf-8") + + +def _resolve_block( + path: str, + block_index: int, + *, + ours: list[str], + theirs: list[str], + ours_text: str, + theirs_text: str, +) -> list[str]: + """Return the semantic resolution for one strictly recognized block.""" + if path == ORCHESTRATOR_PATH: + if block_index == 0: + required_ours = ("stream_usage", "stream_model", "stream_choices.extend") + required_theirs = ("self._local.usage", "choices == []") + if not all(token in ours_text for token in required_ours): + raise RuntimeError("unexpected orchestrator streaming telemetry conflict") + if not all(token in theirs_text for token in required_theirs): + raise RuntimeError("unexpected orchestrator streaming usage conflict") + return [ + " if not isinstance(chunk, dict):\n", + " continue\n", + " usage = chunk.get(\"usage\")\n", + " if isinstance(usage, dict):\n", + " self._local.usage = usage\n", + " stream_usage = usage\n", + " if isinstance(chunk.get(\"model\"), str):\n", + " stream_model = chunk[\"model\"]\n", + " choices = chunk.get(\"choices\")\n", + " if choices == []:\n", + " continue\n", + " if not isinstance(choices, list) or not choices:\n", + " continue\n", + " stream_choices.extend(\n", + " {\"finish_reason\": choice[\"finish_reason\"]}\n", + " for choice in choices\n", + " if isinstance(choice, dict)\n", + " and isinstance(choice.get(\"finish_reason\"), str)\n", + " and choice[\"finish_reason\"]\n", + " )\n", + ] + if block_index == 1: + if '"model": agent.model' not in ours_text or "trace_step" not in theirs_text: + raise RuntimeError("unexpected streamed trace conflict") + return [" trace_step\n"] + raise RuntimeError(f"unexpected orchestrator conflict block {block_index}") + + if path == SERVER_PATH: + if block_index == 0: + if "invalid_stream_options" not in theirs_text or "explicit_trace" not in theirs_text: + raise RuntimeError("unexpected structured stream-options conflict") + return theirs + if block_index == 1: + if "ProviderUpstreamError" not in ours_text or "except Exception" not in ours_text: + raise RuntimeError("unexpected SSE provider-error conflict") + return ours + raise RuntimeError(f"unexpected server conflict block {block_index}") + + if path == PASSTHROUGH_TEST_PATH: + if block_index != 0: + raise RuntimeError(f"unexpected passthrough-test conflict block {block_index}") + if "test_passthrough_with_no_ranked_provider_fails_cleanly" not in ours_text: + raise RuntimeError("missing provider-exhaustion regression") + if "test_virtual_responses_effort_profile_uses_responses_wire_shape" not in theirs_text: + raise RuntimeError("missing Responses effort-profile regression") + return ours + theirs + + raise RuntimeError(f"unexpected conflict path {path}") + + +def _enrich_streamed_trace_step(merged: str) -> str: + """Combine main's usage-bearing trace object with PR #879 telemetry fields.""" + old_trace = ( + " trace_step = {\n" + " \"id\": 0,\n" + " \"role\": \"worker\",\n" + " \"agent_id\": agent.id,\n" + " \"subtask\": \"Direct route (streamed)\",\n" + " \"access\": [],\n" + " \"output\": answer,\n" + " }\n" + ) + new_trace = ( + " trace_step = {\n" + " \"id\": 0,\n" + " \"role\": \"worker\",\n" + " \"agent_id\": agent.id,\n" + " \"model\": agent.model,\n" + " \"provider\": agent.provider_name or self._infer_provider_name(agent.base_url),\n" + " \"subtask\": \"Direct route (streamed)\",\n" + " \"access\": [],\n" + " \"latency_ms\": round(latency_seconds * 1000, 2),\n" + " \"output\": answer,\n" + " }\n" + ) + if merged.count(old_trace) != 1: + raise RuntimeError("streamed trace-step shape changed unexpectedly") + return merged.replace(old_trace, new_trace, 1) + + +def main() -> int: + """Resolve all expected executable conflicts and fail on any shape drift.""" + for path in (ORCHESTRATOR_PATH, SERVER_PATH, PASSTHROUGH_TEST_PATH): + _resolve_file(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From c48f77b9f267c6993ca17cd559717ab46b1b5118 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:30:34 -0700 Subject: [PATCH 20/33] ci: execute isolated PR 879 conflict resolver --- .github/workflows/resolve-pr-879.yml | 170 +-------------------------- 1 file changed, 5 insertions(+), 165 deletions(-) diff --git a/.github/workflows/resolve-pr-879.yml b/.github/workflows/resolve-pr-879.yml index 08a96a18a..e4c996afb 100644 --- a/.github/workflows/resolve-pr-879.yml +++ b/.github/workflows/resolve-pr-879.yml @@ -34,9 +34,7 @@ jobs: original_head="$(git rev-parse HEAD)" main_head="$(git rev-parse origin/main)" - if git merge-base --is-ancestor origin/main HEAD; then - echo "Protected main is already an ancestor of ${original_head}." - else + if ! git merge-base --is-ancestor origin/main HEAD; then set +e git merge --no-ff --no-commit origin/main merge_rc=$? @@ -72,167 +70,7 @@ jobs: contextual_orchestrator/orchestrator.py \ contextual_orchestrator/server.py \ tests/test_passthrough_provider_failover.py - - python - <<'PY' - from __future__ import annotations - - from pathlib import Path - - - def resolve_file(path: str) -> None: - source = Path(path).read_text(encoding="utf-8") - lines = source.splitlines(keepends=True) - output: list[str] = [] - index = 0 - block_index = 0 - - while index < len(lines): - if not lines[index].startswith("<<<<<<<"): - output.append(lines[index]) - index += 1 - continue - - ours_start = index + 1 - base_marker = ours_start - while base_marker < len(lines) and not lines[base_marker].startswith("|||||||"): - base_marker += 1 - if base_marker >= len(lines): - raise RuntimeError(f"{path}: expected diff3 base marker") - - separator = base_marker + 1 - while separator < len(lines) and not lines[separator].startswith("======="): - separator += 1 - if separator >= len(lines): - raise RuntimeError(f"{path}: expected conflict separator") - - end_marker = separator + 1 - while end_marker < len(lines) and not lines[end_marker].startswith(">>>>>>>"): - end_marker += 1 - if end_marker >= len(lines): - raise RuntimeError(f"{path}: expected conflict end marker") - - ours = lines[ours_start:base_marker] - base = lines[base_marker + 1:separator] - theirs = lines[separator + 1:end_marker] - ours_text = "".join(ours) - base_text = "".join(base) - theirs_text = "".join(theirs) - - if path == "contextual_orchestrator/orchestrator.py": - if block_index == 0: - required_ours = ("stream_usage", "stream_model", "stream_choices.extend") - required_theirs = ("self._local.usage", "choices == []") - if not all(token in ours_text for token in required_ours): - raise RuntimeError("unexpected orchestrator streaming telemetry conflict") - if not all(token in theirs_text for token in required_theirs): - raise RuntimeError("unexpected orchestrator streaming usage conflict") - resolution = ''' if not isinstance(chunk, dict): - continue - usage = chunk.get("usage") - if isinstance(usage, dict): - self._local.usage = usage - stream_usage = usage - if isinstance(chunk.get("model"), str): - stream_model = chunk["model"] - choices = chunk.get("choices") - if choices == []: - continue - if not isinstance(choices, list) or not choices: - continue - stream_choices.extend( - {"finish_reason": choice["finish_reason"]} - for choice in choices - if isinstance(choice, dict) - and isinstance(choice.get("finish_reason"), str) - and choice["finish_reason"] - ) -'''.splitlines(keepends=True) - elif block_index == 1: - if '"model": agent.model' not in ours_text or "trace_step" not in theirs_text: - raise RuntimeError("unexpected streamed trace conflict") - resolution = [" trace_step\n"] - else: - raise RuntimeError(f"unexpected orchestrator conflict block {block_index}") - - elif path == "contextual_orchestrator/server.py": - if block_index == 0: - if "invalid_stream_options" not in theirs_text or "explicit_trace" not in theirs_text: - raise RuntimeError("unexpected structured stream-options conflict") - resolution = theirs - elif block_index == 1: - if "ProviderUpstreamError" not in ours_text or "except Exception" not in ours_text: - raise RuntimeError("unexpected SSE provider-error conflict") - resolution = ours - else: - raise RuntimeError(f"unexpected server conflict block {block_index}") - - elif path == "tests/test_passthrough_provider_failover.py": - if block_index != 0: - raise RuntimeError(f"unexpected passthrough-test conflict block {block_index}") - if "test_passthrough_with_no_ranked_provider_fails_cleanly" not in ours_text: - raise RuntimeError("missing provider-exhaustion regression") - if "test_virtual_responses_effort_profile_uses_responses_wire_shape" not in theirs_text: - raise RuntimeError("missing Responses effort-profile regression") - resolution = ours + theirs - - else: - raise RuntimeError(f"unexpected conflict path {path}") - - output.extend(resolution) - block_index += 1 - index = end_marker + 1 - - expected_blocks = { - "contextual_orchestrator/orchestrator.py": 2, - "contextual_orchestrator/server.py": 2, - "tests/test_passthrough_provider_failover.py": 1, - }[path] - if block_index != expected_blocks: - raise RuntimeError( - f"{path}: resolved {block_index} blocks; expected {expected_blocks}" - ) - - merged = "".join(output) - if any(marker in merged for marker in ("<<<<<<<", "|||||||", "=======", ">>>>>>>")): - raise RuntimeError(f"{path}: unresolved conflict marker remains") - - if path == "contextual_orchestrator/orchestrator.py": - old_trace = ''' trace_step = { - "id": 0, - "role": "worker", - "agent_id": agent.id, - "subtask": "Direct route (streamed)", - "access": [], - "output": answer, - } -''' - new_trace = ''' trace_step = { - "id": 0, - "role": "worker", - "agent_id": agent.id, - "model": agent.model, - "provider": agent.provider_name or self._infer_provider_name(agent.base_url), - "subtask": "Direct route (streamed)", - "access": [], - "latency_ms": round(latency_seconds * 1000, 2), - "output": answer, - } -''' - if merged.count(old_trace) != 1: - raise RuntimeError("streamed trace-step shape changed unexpectedly") - merged = merged.replace(old_trace, new_trace, 1) - - Path(path).write_text(merged, encoding="utf-8") - - - for conflict_path in ( - "contextual_orchestrator/orchestrator.py", - "contextual_orchestrator/server.py", - "tests/test_passthrough_provider_failover.py", - ): - resolve_file(conflict_path) - PY - + python .github/scripts/resolve_pr_879_conflicts.py git add \ contextual_orchestrator/orchestrator.py \ contextual_orchestrator/server.py \ @@ -241,7 +79,9 @@ jobs: fi test -z "$(git diff --name-only --diff-filter=U)" - rm -f .github/workflows/resolve-pr-879.yml + rm -f \ + .github/workflows/resolve-pr-879.yml \ + .github/scripts/resolve_pr_879_conflicts.py git add -A git diff --cached --check python -m compileall -q contextual_orchestrator From 2649fc72ad79389f5a2cfdc265cac1195ab53dbe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:36:27 -0700 Subject: [PATCH 21/33] ci: retrigger protected checks for PR #879 --- .github/workflow-trigger-879.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/workflow-trigger-879.txt diff --git a/.github/workflow-trigger-879.txt b/.github/workflow-trigger-879.txt new file mode 100644 index 000000000..3daa365d9 --- /dev/null +++ b/.github/workflow-trigger-879.txt @@ -0,0 +1 @@ +Transient check trigger for PR #879 after the conflict resolver's GITHUB_TOKEN-authored merge commit. From ff1783cf2d357e3f8193f958d0f0079b89da579f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:36:38 -0700 Subject: [PATCH 22/33] ci: remove transient PR #879 check trigger --- .github/workflow-trigger-879.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .github/workflow-trigger-879.txt diff --git a/.github/workflow-trigger-879.txt b/.github/workflow-trigger-879.txt deleted file mode 100644 index 3daa365d9..000000000 --- a/.github/workflow-trigger-879.txt +++ /dev/null @@ -1 +0,0 @@ -Transient check trigger for PR #879 after the conflict resolver's GITHUB_TOKEN-authored merge commit. From e084a2f93b18df8af9715eb90bd9ae6100c5e6c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:55:03 -0700 Subject: [PATCH 23/33] ci: rerun PR #879 against provider-diverse Strix --- .github/workflow-trigger-879-auto.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/workflow-trigger-879-auto.txt diff --git a/.github/workflow-trigger-879-auto.txt b/.github/workflow-trigger-879-auto.txt new file mode 100644 index 000000000..83f79bf83 --- /dev/null +++ b/.github/workflow-trigger-879-auto.txt @@ -0,0 +1 @@ +Transient trigger after central Strix provider-diverse routing reached protected main. From 3fa7d7ea128f20939dc9aa993216cdcb9eefd0c0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:55:20 -0700 Subject: [PATCH 24/33] ci: remove transient PR #879 auto-pool trigger --- .github/workflow-trigger-879-auto.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .github/workflow-trigger-879-auto.txt diff --git a/.github/workflow-trigger-879-auto.txt b/.github/workflow-trigger-879-auto.txt deleted file mode 100644 index 83f79bf83..000000000 --- a/.github/workflow-trigger-879-auto.txt +++ /dev/null @@ -1 +0,0 @@ -Transient trigger after central Strix provider-diverse routing reached protected main. From 2572c8daf8e54092603d0fdfebf57e54a5601920 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:55:43 -0700 Subject: [PATCH 25/33] chore: add bounded PR #879 trace contract repair --- scripts/ci/repair_pr_879_trace_contract.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 scripts/ci/repair_pr_879_trace_contract.py diff --git a/scripts/ci/repair_pr_879_trace_contract.py b/scripts/ci/repair_pr_879_trace_contract.py new file mode 100644 index 000000000..106cd2852 --- /dev/null +++ b/scripts/ci/repair_pr_879_trace_contract.py @@ -0,0 +1,29 @@ +"""One-shot repair for structured and tool-passthrough trace contracts in PR #879.""" + +from __future__ import annotations + +from pathlib import Path + + +def main() -> None: + """Remove the over-broad trace rejection that shadows specific contracts.""" + path = Path("contextual_orchestrator/server.py") + text = path.read_text(encoding="utf-8") + old = ''' if explicit_trace: + raise RequestError( + 400, + "unsupported_trace_disclosure", + "remove include_orchestration_trace or use chat without tools or response_format", + ) + tool_loop = bool(tools_list) +''' + new = ''' tool_loop = bool(tools_list) +''' + count = text.count(old) + if count != 1: + raise SystemExit(f"expected one over-broad trace guard; found {count}") + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + +if __name__ == "__main__": + main() From 87bf8d1296ea22bba6eb7a391ec2a17e23418010 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:55:58 -0700 Subject: [PATCH 26/33] chore: run bounded PR #879 trace contract repair --- .../repair-pr-879-trace-contract.yml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/repair-pr-879-trace-contract.yml diff --git a/.github/workflows/repair-pr-879-trace-contract.yml b/.github/workflows/repair-pr-879-trace-contract.yml new file mode 100644 index 000000000..650037b07 --- /dev/null +++ b/.github/workflows/repair-pr-879-trace-contract.yml @@ -0,0 +1,57 @@ +name: Repair PR 879 trace contracts + +on: + push: + branches: + - fix/provider-error-taxonomy-telemetry + +permissions: + contents: write + +concurrency: + group: repair-pr-879-trace-contract + cancel-in-progress: false + +jobs: + repair: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out exact PR head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e + with: + version: "0.12.5" + + - name: Apply, validate, and commit the trace repair + env: + EXPECTED_PARENT: 2572c8daf8e54092603d0fdfebf57e54a5601920 + BRANCH: fix/provider-error-taxonomy-telemetry + shell: bash + run: | + set -euo pipefail + test "$(git branch --show-current)" = "$BRANCH" + test "$(git rev-parse HEAD^)" = "$EXPECTED_PARENT" + python scripts/ci/repair_pr_879_trace_contract.py + rm -f \ + scripts/ci/repair_pr_879_trace_contract.py \ + .github/workflows/repair-pr-879-trace-contract.yml + git add -A + git diff --cached --check + uv run --locked --extra api --extra db --extra queue --group dev \ + python -m pytest -q \ + tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_structured_chat_discloses_only_an_authorized_conduct_trace \ + tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_tool_passthrough_rejects_a_trace_it_cannot_return + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit -m 'fix: preserve structured and tool trace contracts' + git push origin "HEAD:$BRANCH" From c2e80425bc2be74e4ba9279a48acd31c38580b91 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:56:40 +0000 Subject: [PATCH 27/33] fix: preserve structured and tool trace contracts --- .../repair-pr-879-trace-contract.yml | 57 ------------------- contextual_orchestrator/server.py | 6 -- scripts/ci/repair_pr_879_trace_contract.py | 29 ---------- 3 files changed, 92 deletions(-) delete mode 100644 .github/workflows/repair-pr-879-trace-contract.yml delete mode 100644 scripts/ci/repair_pr_879_trace_contract.py diff --git a/.github/workflows/repair-pr-879-trace-contract.yml b/.github/workflows/repair-pr-879-trace-contract.yml deleted file mode 100644 index 650037b07..000000000 --- a/.github/workflows/repair-pr-879-trace-contract.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Repair PR 879 trace contracts - -on: - push: - branches: - - fix/provider-error-taxonomy-telemetry - -permissions: - contents: write - -concurrency: - group: repair-pr-879-trace-contract - cancel-in-progress: false - -jobs: - repair: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Check out exact PR head - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e - with: - version: "0.12.5" - - - name: Apply, validate, and commit the trace repair - env: - EXPECTED_PARENT: 2572c8daf8e54092603d0fdfebf57e54a5601920 - BRANCH: fix/provider-error-taxonomy-telemetry - shell: bash - run: | - set -euo pipefail - test "$(git branch --show-current)" = "$BRANCH" - test "$(git rev-parse HEAD^)" = "$EXPECTED_PARENT" - python scripts/ci/repair_pr_879_trace_contract.py - rm -f \ - scripts/ci/repair_pr_879_trace_contract.py \ - .github/workflows/repair-pr-879-trace-contract.yml - git add -A - git diff --cached --check - uv run --locked --extra api --extra db --extra queue --group dev \ - python -m pytest -q \ - tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_structured_chat_discloses_only_an_authorized_conduct_trace \ - tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_tool_passthrough_rejects_a_trace_it_cannot_return - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git commit -m 'fix: preserve structured and tool trace contracts' - git push origin "HEAD:$BRANCH" diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index c20aa78c6..a462fd645 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -6568,12 +6568,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di "invalid_stream_options", "stream_options.include_usage=true is not supported with tools or response_format", ) - if explicit_trace: - raise RequestError( - 400, - "unsupported_trace_disclosure", - "remove include_orchestration_trace or use chat without tools or response_format", - ) tool_loop = bool(tools_list) if ( tool_loop diff --git a/scripts/ci/repair_pr_879_trace_contract.py b/scripts/ci/repair_pr_879_trace_contract.py deleted file mode 100644 index 106cd2852..000000000 --- a/scripts/ci/repair_pr_879_trace_contract.py +++ /dev/null @@ -1,29 +0,0 @@ -"""One-shot repair for structured and tool-passthrough trace contracts in PR #879.""" - -from __future__ import annotations - -from pathlib import Path - - -def main() -> None: - """Remove the over-broad trace rejection that shadows specific contracts.""" - path = Path("contextual_orchestrator/server.py") - text = path.read_text(encoding="utf-8") - old = ''' if explicit_trace: - raise RequestError( - 400, - "unsupported_trace_disclosure", - "remove include_orchestration_trace or use chat without tools or response_format", - ) - tool_loop = bool(tools_list) -''' - new = ''' tool_loop = bool(tools_list) -''' - count = text.count(old) - if count != 1: - raise SystemExit(f"expected one over-broad trace guard; found {count}") - path.write_text(text.replace(old, new, 1), encoding="utf-8") - - -if __name__ == "__main__": - main() From 0fe060fbee86230e1a69519f4153dc769c6afe29 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:57:34 -0700 Subject: [PATCH 28/33] ci: run final protected checks for PR #879 --- .github/workflow-trigger-879-final.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/workflow-trigger-879-final.txt diff --git a/.github/workflow-trigger-879-final.txt b/.github/workflow-trigger-879-final.txt new file mode 100644 index 000000000..fae154c21 --- /dev/null +++ b/.github/workflow-trigger-879-final.txt @@ -0,0 +1 @@ +Transient final check trigger after conflict and trace-contract repairs. From fbfef43bedbba6a53ec148bc0247431c21166945 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 03:57:48 -0700 Subject: [PATCH 29/33] ci: remove final PR #879 check trigger --- .github/workflow-trigger-879-final.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .github/workflow-trigger-879-final.txt diff --git a/.github/workflow-trigger-879-final.txt b/.github/workflow-trigger-879-final.txt deleted file mode 100644 index fae154c21..000000000 --- a/.github/workflow-trigger-879-final.txt +++ /dev/null @@ -1 +0,0 @@ -Transient final check trigger after conflict and trace-contract repairs. From fd828d934ae434e308766bd6003925736ed79986 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 04:00:46 -0700 Subject: [PATCH 30/33] chore: add bounded PR #879 taxonomy evidence repair --- scripts/ci/repair_pr_879_taxonomy_evidence.py | 298 ++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 scripts/ci/repair_pr_879_taxonomy_evidence.py diff --git a/scripts/ci/repair_pr_879_taxonomy_evidence.py b/scripts/ci/repair_pr_879_taxonomy_evidence.py new file mode 100644 index 000000000..7e95df8d9 --- /dev/null +++ b/scripts/ci/repair_pr_879_taxonomy_evidence.py @@ -0,0 +1,298 @@ +"""One-shot exact-head repair for PR #879 taxonomy and trace evidence.""" + +from __future__ import annotations + +from pathlib import Path + + +def replace_once(path: str, old: str, new: str) -> None: + """Replace one reviewed block or stop without modifying the branch.""" + target = Path(path) + text = target.read_text(encoding="utf-8") + count = text.count(old) + if count != 1: + raise SystemExit(f"expected one replacement in {path}; found {count}") + target.write_text(text.replace(old, new, 1), encoding="utf-8") + + +def main() -> None: + """Apply request-size, redaction, and shared batch-latency contracts.""" + replace_once( + "contextual_orchestrator/provider_errors.py", + "import json as _json\nimport socket\n", + "import json as _json\nimport re as _re\nimport socket\n", + ) + replace_once( + "contextual_orchestrator/provider_errors.py", + '''#: Maximum provider response bytes inspected for one caller-safe diagnostic. +MAX_PROVIDER_ERROR_BODY_BYTES = 65_536 + +#: Upstream HTTP status -> ``(client_status, error_code, retryable)`` surface. +''', + '''#: Maximum provider response bytes inspected for one caller-safe diagnostic. +MAX_PROVIDER_ERROR_BODY_BYTES = 65_536 + +#: Provider-controlled text containing any likely secret, request content, URL, +#: or network address is discarded wholesale. Partial masking is unsafe because +#: adjacent diagnostic text can still identify credentials or private topology. +_SENSITIVE_PROVIDER_MESSAGE = _re.compile( + r"(?ix)(?:" + r"https?://|" + r"(?:^|[^0-9])(?:[0-9]{1,3}\\.){3}[0-9]{1,3}(?:[^0-9]|$)|" + r"\\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input|messages?)\\b" + r")" +) + +#: Upstream HTTP status -> ``(client_status, error_code, retryable)`` surface. +''', + ) + replace_once( + "contextual_orchestrator/provider_errors.py", + ''' ).strip() + return collapsed[:MAX_SAFE_MESSAGE_CHARS] or None +''', + ''' ).strip() + collapsed = collapsed[:MAX_SAFE_MESSAGE_CHARS] + if not collapsed or _SENSITIVE_PROVIDER_MESSAGE.search(collapsed): + return None + return collapsed +''', + ) + replace_once( + "contextual_orchestrator/orchestrator.py", + '''class ProviderRequestTooLargeError(RuntimeError): + """Raised after every eligible provider rejects the request as too large.""" +''', + '''class ProviderRequestTooLargeError(ProviderUpstreamError): + """Preserve request-size taxonomy across transport and telemetry boundaries.""" + + def __init__( + self, + message: str, + *, + agent_id: str = "", + model: str = "", + provider_status: int | None = None, + transport: str = "chat", + ) -> None: + super().__init__( + agent_id=agent_id, + model=model, + error_code="request_too_large", + message=message, + client_status=413, + provider_status=provider_status, + retryable=False, + transport=transport, + ) +''', + ) + + orchestrator = Path("contextual_orchestrator/orchestrator.py") + text = orchestrator.read_text(encoding="utf-8") + old_raise = ''' raise ProviderRequestTooLargeError("provider request body is too large") from None +''' + if text.count(old_raise) != 2: + raise SystemExit(f"expected two direct request-size raises; found {text.count(old_raise)}") + chat_raise = ''' raise ProviderRequestTooLargeError( + "provider request body is too large", + agent_id=agent.id, + model=agent.model, + provider_status=last_error.code, + transport="chat", + ) from None +''' + passthrough_raise = ''' raise ProviderRequestTooLargeError( + "provider request body is too large", + agent_id=agent.id, + model=agent.model, + provider_status=last_error.code, + transport="passthrough", + ) from None +''' + text = text.replace(old_raise, chat_raise, 1).replace(old_raise, passthrough_raise, 1) + orchestrator.write_text(text, encoding="utf-8") + + replace_once( + "contextual_orchestrator/orchestrator.py", + ''' answers: dict[int, dict[str, Any]] = {} + for agent_id, requests in requests_by_agent.items(): + effort_profile = self._role_effort_profile("worker") + batch = ( + self.client.batch_chat( + agents_by_id[agent_id], requests, effort_profile=effort_profile + ) + if effort_profile is not None + else self.client.batch_chat(agents_by_id[agent_id], requests) + ) + results = _validate_batch_results(requests, batch) +''', + ''' answers: dict[int, dict[str, Any]] = {} + batch_latency_ms_by_agent: dict[str, float] = {} + for agent_id, requests in requests_by_agent.items(): + effort_profile = self._role_effort_profile("worker") + batch_started_at = time.perf_counter() + batch = ( + self.client.batch_chat( + agents_by_id[agent_id], requests, effort_profile=effort_profile + ) + if effort_profile is not None + else self.client.batch_chat(agents_by_id[agent_id], requests) + ) + # One provider batch call covers every request in this group. Record + # its shared elapsed time on each trace row without claiming + # unavailable per-request timing precision. + batch_latency_ms_by_agent[agent_id] = round( + (time.perf_counter() - batch_started_at) * 1000, + 2, + ) + results = _validate_batch_results(requests, batch) +''', + ) + replace_once( + "contextual_orchestrator/orchestrator.py", + ''' "provider": agent.provider_name or self._infer_provider_name(agent.base_url), + "subtask": "Direct route (batched)", "access": [], "output": result["content"], +''', + ''' "provider": agent.provider_name or self._infer_provider_name(agent.base_url), + "latency_ms": batch_latency_ms_by_agent[agent.id], + "subtask": "Direct route (batched)", "access": [], "output": result["content"], +''', + ) + replace_once( + "tests/test_provider_error_taxonomy.py", + ''' plain_string = safe_provider_message(_body_http_error(400, {"error": "invalid api key"})) + assert plain_string == "invalid api key" +''', + ''' plain_string = safe_provider_message(_body_http_error(400, {"error": "invalid api key"})) + assert plain_string is None +''', + ) + + taxonomy_tests = Path("tests/test_provider_error_taxonomy.py") + taxonomy_text = taxonomy_tests.read_text(encoding="utf-8") + security_test = ''' + +def test_safe_message_discards_sensitive_provider_diagnostics() -> None: + """Credentials, request content, URLs, and private topology never reach callers.""" + diagnostics = ( + "authorization: Bearer provider-secret-value", + "prompt=private customer message", + "request failed at http://10.0.0.9/internal", + "token=abc123456789012345", + ) + for diagnostic in diagnostics: + error = _body_http_error(400, {"error": {"message": diagnostic}}) + assert safe_provider_message(error) is None + classified = classify_provider_failure(error, agent_id="a", model="m") + assert diagnostic not in str(classified) + assert str(classified) == "provider rejected the request with HTTP 400" +''' + if "test_safe_message_discards_sensitive_provider_diagnostics" in taxonomy_text: + raise SystemExit("sensitive-provider-message regression already exists") + taxonomy_tests.write_text(taxonomy_text + security_test, encoding="utf-8") + + regression = Path("tests/test_provider_error_merge_regressions.py") + if regression.exists(): + raise SystemExit(f"{regression} already exists") + regression.write_text( + '''"""Merge-result regressions for provider taxonomy and trace evidence.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +import contextual_orchestrator.orchestrator as orchestrator_module +import contextual_orchestrator.telemetry as telemetry_module +from contextual_orchestrator.orchestrator import ( + ModelAgent, + ProviderRequestTooLargeError, + TaskOrchestrator, +) +from contextual_orchestrator.telemetry import traced + + +class _SpanContext(AbstractContextManager): + def __init__(self, span: MagicMock) -> None: + self._span = span + + def __enter__(self) -> MagicMock: + return self._span + + def __exit__(self, exc_type, exc, traceback) -> bool: + del exc_type, exc, traceback + return False + + +class _Tracer: + def __init__(self, span: MagicMock) -> None: + self._span = span + + def start_as_current_span(self, *_args, **_kwargs) -> _SpanContext: + return _SpanContext(self._span) + + +class _Trace: + def __init__(self, span: MagicMock) -> None: + self._span = span + + def get_tracer(self, _name: str) -> _Tracer: + return _Tracer(self._span) + + +@pytest.mark.parametrize("transport", ["chat", "passthrough"]) +def test_traced_preserves_request_too_large_taxonomy(monkeypatch, transport: str) -> None: + """HTTP 413 remains request_too_large with its upstream status on every transport.""" + span = MagicMock() + monkeypatch.setattr(telemetry_module, "trace", _Trace(span)) + monkeypatch.setattr(telemetry_module, "Status", lambda code: code) + monkeypatch.setattr(telemetry_module, "StatusCode", SimpleNamespace(ERROR="error")) + failure = ProviderRequestTooLargeError( + "provider request body is too large", + agent_id="worker_agent", + model="model-x", + provider_status=413, + transport=transport, + ) + + with pytest.raises(ProviderRequestTooLargeError): + with traced(f"{transport} model-x"): + raise failure + + span.set_attribute.assert_any_call("error.type", "request_too_large") + span.set_attribute.assert_any_call( + "contextual_orchestrator.provider_status_code", 413 + ) + + +def test_batch_trace_uses_one_honest_shared_provider_duration(monkeypatch) -> None: + """Every result in one provider batch reports the same measured batch duration.""" + agent = ModelAgent("batch_agent", "batch-model", provider_name="provider-x") + orchestrator = TaskOrchestrator([agent]) + + def batch_chat(_agent, requests, **_kwargs): + return { + custom_id: {"content": f"answer:{custom_id}"} + for custom_id in requests + } + + monkeypatch.setattr(orchestrator.client, "batch_chat", batch_chat) + with patch.object( + orchestrator_module.time, + "perf_counter", + side_effect=[10.0, 10.25], + ): + records = orchestrator.batch_route(["one", "two"]) + + assert [record["trace"][0]["latency_ms"] for record in records] == [250.0, 250.0] +''', + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() From 63560e85d8fee90c82a34204a86be934c17503a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 04:01:03 -0700 Subject: [PATCH 31/33] chore: run bounded PR #879 taxonomy evidence repair --- .../repair-pr-879-taxonomy-evidence.yml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/repair-pr-879-taxonomy-evidence.yml diff --git a/.github/workflows/repair-pr-879-taxonomy-evidence.yml b/.github/workflows/repair-pr-879-taxonomy-evidence.yml new file mode 100644 index 000000000..aa433d784 --- /dev/null +++ b/.github/workflows/repair-pr-879-taxonomy-evidence.yml @@ -0,0 +1,59 @@ +name: Repair PR 879 taxonomy evidence + +on: + push: + branches: + - fix/provider-error-taxonomy-telemetry + +permissions: + contents: write + +concurrency: + group: repair-pr-879-taxonomy-evidence + cancel-in-progress: false + +jobs: + repair: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out exact PR head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e + with: + version: "0.12.5" + + - name: Apply, validate, and commit the evidence repair + env: + EXPECTED_PARENT: fd828d934ae434e308766bd6003925736ed79986 + BRANCH: fix/provider-error-taxonomy-telemetry + shell: bash + run: | + set -euo pipefail + test "$(git branch --show-current)" = "$BRANCH" + test "$(git rev-parse HEAD^)" = "$EXPECTED_PARENT" + python scripts/ci/repair_pr_879_taxonomy_evidence.py + rm -f \ + scripts/ci/repair_pr_879_taxonomy_evidence.py \ + .github/workflows/repair-pr-879-taxonomy-evidence.yml + git add -A + git diff --cached --check + uv run --locked --extra api --extra db --extra queue --group dev \ + python -m pytest -q \ + tests/test_provider_error_taxonomy.py \ + tests/test_provider_error_merge_regressions.py \ + tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_structured_chat_discloses_only_an_authorized_conduct_trace \ + tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_tool_passthrough_rejects_a_trace_it_cannot_return + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit -m 'fix: preserve provider taxonomy and batch trace evidence' + git push origin "HEAD:$BRANCH" From 7ba4bb945b72463595442f1292d00fd85d660c99 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:01:27 +0000 Subject: [PATCH 32/33] fix: preserve provider taxonomy and batch trace evidence --- .../repair-pr-879-taxonomy-evidence.yml | 59 ---- contextual_orchestrator/orchestrator.py | 50 ++- contextual_orchestrator/provider_errors.py | 17 +- scripts/ci/repair_pr_879_taxonomy_evidence.py | 298 ------------------ .../test_provider_error_merge_regressions.py | 93 ++++++ tests/test_provider_error_taxonomy.py | 18 +- 6 files changed, 172 insertions(+), 363 deletions(-) delete mode 100644 .github/workflows/repair-pr-879-taxonomy-evidence.yml delete mode 100644 scripts/ci/repair_pr_879_taxonomy_evidence.py create mode 100644 tests/test_provider_error_merge_regressions.py diff --git a/.github/workflows/repair-pr-879-taxonomy-evidence.yml b/.github/workflows/repair-pr-879-taxonomy-evidence.yml deleted file mode 100644 index aa433d784..000000000 --- a/.github/workflows/repair-pr-879-taxonomy-evidence.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Repair PR 879 taxonomy evidence - -on: - push: - branches: - - fix/provider-error-taxonomy-telemetry - -permissions: - contents: write - -concurrency: - group: repair-pr-879-taxonomy-evidence - cancel-in-progress: false - -jobs: - repair: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Check out exact PR head - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e - with: - version: "0.12.5" - - - name: Apply, validate, and commit the evidence repair - env: - EXPECTED_PARENT: fd828d934ae434e308766bd6003925736ed79986 - BRANCH: fix/provider-error-taxonomy-telemetry - shell: bash - run: | - set -euo pipefail - test "$(git branch --show-current)" = "$BRANCH" - test "$(git rev-parse HEAD^)" = "$EXPECTED_PARENT" - python scripts/ci/repair_pr_879_taxonomy_evidence.py - rm -f \ - scripts/ci/repair_pr_879_taxonomy_evidence.py \ - .github/workflows/repair-pr-879-taxonomy-evidence.yml - git add -A - git diff --cached --check - uv run --locked --extra api --extra db --extra queue --group dev \ - python -m pytest -q \ - tests/test_provider_error_taxonomy.py \ - tests/test_provider_error_merge_regressions.py \ - tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_structured_chat_discloses_only_an_authorized_conduct_trace \ - tests/test_chat_include_orchestration_trace_http_honesty.py::test_http_tool_passthrough_rejects_a_trace_it_cannot_return - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git commit -m 'fix: preserve provider taxonomy and batch trace evidence' - git push origin "HEAD:$BRANCH" diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 7e5a69f77..5b9b0832e 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -141,8 +141,28 @@ class ProviderResponseError(RuntimeError): """Raised for a provider response that cannot become a safe completion.""" -class ProviderRequestTooLargeError(RuntimeError): - """Raised after every eligible provider rejects the request as too large.""" +class ProviderRequestTooLargeError(ProviderUpstreamError): + """Preserve request-size taxonomy across transport and telemetry boundaries.""" + + def __init__( + self, + message: str, + *, + agent_id: str = "", + model: str = "", + provider_status: int | None = None, + transport: str = "chat", + ) -> None: + super().__init__( + agent_id=agent_id, + model=model, + error_code="request_too_large", + message=message, + client_status=413, + provider_status=provider_status, + retryable=False, + transport=transport, + ) def _structured_output_error( @@ -1564,7 +1584,13 @@ def _send_with_retry( if isinstance(last_error, urllib.error.HTTPError) and ( last_error.code == 413 or _is_oversized_tool_description_error(last_error) ): - raise ProviderRequestTooLargeError("provider request body is too large") from None + raise ProviderRequestTooLargeError( + "provider request body is too large", + agent_id=agent.id, + model=agent.model, + provider_status=last_error.code, + transport="chat", + ) from None if isinstance(last_error, ProviderResponseError): raise last_error # Classify instead of collapsing: a 401/404/429 upstream failure is @@ -2092,7 +2118,13 @@ def _send_raw_with_retry( if isinstance(last_error, urllib.error.HTTPError) and ( last_error.code == 413 or _is_oversized_tool_description_error(last_error) ): - raise ProviderRequestTooLargeError("provider request body is too large") from None + raise ProviderRequestTooLargeError( + "provider request body is too large", + agent_id=agent.id, + model=agent.model, + provider_status=last_error.code, + transport="passthrough", + ) from None if last_error is None: # pragma: no cover - the loop always attempts once raise RuntimeError(f"provider {agent.id} passthrough request failed") if not allow_transient_retries: @@ -4578,8 +4610,10 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]: requests_by_agent.setdefault(agent.id, {})[f"task_{index}"] = [{"role": "user", "content": prompt}] answers: dict[int, dict[str, Any]] = {} + batch_latency_ms_by_agent: dict[str, float] = {} for agent_id, requests in requests_by_agent.items(): effort_profile = self._role_effort_profile("worker") + batch_started_at = time.perf_counter() batch = ( self.client.batch_chat( agents_by_id[agent_id], requests, effort_profile=effort_profile @@ -4587,6 +4621,13 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]: if effort_profile is not None else self.client.batch_chat(agents_by_id[agent_id], requests) ) + # One provider batch call covers every request in this group. Record + # its shared elapsed time on each trace row without claiming + # unavailable per-request timing precision. + batch_latency_ms_by_agent[agent_id] = round( + (time.perf_counter() - batch_started_at) * 1000, + 2, + ) results = _validate_batch_results(requests, batch) for custom_id, result in results.items(): # _validate_batch_results already pinned every result key to the @@ -4612,6 +4653,7 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]: "id": 0, "role": "worker", "agent_id": agent.id, "model": agent.model, "provider": agent.provider_name or self._infer_provider_name(agent.base_url), + "latency_ms": batch_latency_ms_by_agent[agent.id], "subtask": "Direct route (batched)", "access": [], "output": result["content"], } if result.get("usage") is not None: diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index 5fe8ebdd5..9657b8644 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -20,6 +20,7 @@ from __future__ import annotations import json as _json +import re as _re import socket import ssl import urllib.error @@ -41,6 +42,17 @@ #: Maximum provider response bytes inspected for one caller-safe diagnostic. MAX_PROVIDER_ERROR_BODY_BYTES = 65_536 +#: Provider-controlled text containing any likely secret, request content, URL, +#: or network address is discarded wholesale. Partial masking is unsafe because +#: adjacent diagnostic text can still identify credentials or private topology. +_SENSITIVE_PROVIDER_MESSAGE = _re.compile( + r"(?ix)(?:" + r"https?://|" + r"(?:^|[^0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?:[^0-9]|$)|" + r"\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input|messages?)\b" + r")" +) + #: Upstream HTTP status -> ``(client_status, error_code, retryable)`` surface. #: The client status is what this gateway returns; ``error_code`` follows the #: OpenAI-compatible vocabulary; ``retryable`` mirrors the gateway's own @@ -125,7 +137,10 @@ def safe_provider_message(exc: BaseException) -> str | None: else " " for char in str(raw) ).strip() - return collapsed[:MAX_SAFE_MESSAGE_CHARS] or None + collapsed = collapsed[:MAX_SAFE_MESSAGE_CHARS] + if not collapsed or _SENSITIVE_PROVIDER_MESSAGE.search(collapsed): + return None + return collapsed class ProviderUpstreamError(RuntimeError): diff --git a/scripts/ci/repair_pr_879_taxonomy_evidence.py b/scripts/ci/repair_pr_879_taxonomy_evidence.py deleted file mode 100644 index 7e95df8d9..000000000 --- a/scripts/ci/repair_pr_879_taxonomy_evidence.py +++ /dev/null @@ -1,298 +0,0 @@ -"""One-shot exact-head repair for PR #879 taxonomy and trace evidence.""" - -from __future__ import annotations - -from pathlib import Path - - -def replace_once(path: str, old: str, new: str) -> None: - """Replace one reviewed block or stop without modifying the branch.""" - target = Path(path) - text = target.read_text(encoding="utf-8") - count = text.count(old) - if count != 1: - raise SystemExit(f"expected one replacement in {path}; found {count}") - target.write_text(text.replace(old, new, 1), encoding="utf-8") - - -def main() -> None: - """Apply request-size, redaction, and shared batch-latency contracts.""" - replace_once( - "contextual_orchestrator/provider_errors.py", - "import json as _json\nimport socket\n", - "import json as _json\nimport re as _re\nimport socket\n", - ) - replace_once( - "contextual_orchestrator/provider_errors.py", - '''#: Maximum provider response bytes inspected for one caller-safe diagnostic. -MAX_PROVIDER_ERROR_BODY_BYTES = 65_536 - -#: Upstream HTTP status -> ``(client_status, error_code, retryable)`` surface. -''', - '''#: Maximum provider response bytes inspected for one caller-safe diagnostic. -MAX_PROVIDER_ERROR_BODY_BYTES = 65_536 - -#: Provider-controlled text containing any likely secret, request content, URL, -#: or network address is discarded wholesale. Partial masking is unsafe because -#: adjacent diagnostic text can still identify credentials or private topology. -_SENSITIVE_PROVIDER_MESSAGE = _re.compile( - r"(?ix)(?:" - r"https?://|" - r"(?:^|[^0-9])(?:[0-9]{1,3}\\.){3}[0-9]{1,3}(?:[^0-9]|$)|" - r"\\b(?:api[_ -]?key|authorization|bearer|password|secret|token|prompt|input|messages?)\\b" - r")" -) - -#: Upstream HTTP status -> ``(client_status, error_code, retryable)`` surface. -''', - ) - replace_once( - "contextual_orchestrator/provider_errors.py", - ''' ).strip() - return collapsed[:MAX_SAFE_MESSAGE_CHARS] or None -''', - ''' ).strip() - collapsed = collapsed[:MAX_SAFE_MESSAGE_CHARS] - if not collapsed or _SENSITIVE_PROVIDER_MESSAGE.search(collapsed): - return None - return collapsed -''', - ) - replace_once( - "contextual_orchestrator/orchestrator.py", - '''class ProviderRequestTooLargeError(RuntimeError): - """Raised after every eligible provider rejects the request as too large.""" -''', - '''class ProviderRequestTooLargeError(ProviderUpstreamError): - """Preserve request-size taxonomy across transport and telemetry boundaries.""" - - def __init__( - self, - message: str, - *, - agent_id: str = "", - model: str = "", - provider_status: int | None = None, - transport: str = "chat", - ) -> None: - super().__init__( - agent_id=agent_id, - model=model, - error_code="request_too_large", - message=message, - client_status=413, - provider_status=provider_status, - retryable=False, - transport=transport, - ) -''', - ) - - orchestrator = Path("contextual_orchestrator/orchestrator.py") - text = orchestrator.read_text(encoding="utf-8") - old_raise = ''' raise ProviderRequestTooLargeError("provider request body is too large") from None -''' - if text.count(old_raise) != 2: - raise SystemExit(f"expected two direct request-size raises; found {text.count(old_raise)}") - chat_raise = ''' raise ProviderRequestTooLargeError( - "provider request body is too large", - agent_id=agent.id, - model=agent.model, - provider_status=last_error.code, - transport="chat", - ) from None -''' - passthrough_raise = ''' raise ProviderRequestTooLargeError( - "provider request body is too large", - agent_id=agent.id, - model=agent.model, - provider_status=last_error.code, - transport="passthrough", - ) from None -''' - text = text.replace(old_raise, chat_raise, 1).replace(old_raise, passthrough_raise, 1) - orchestrator.write_text(text, encoding="utf-8") - - replace_once( - "contextual_orchestrator/orchestrator.py", - ''' answers: dict[int, dict[str, Any]] = {} - for agent_id, requests in requests_by_agent.items(): - effort_profile = self._role_effort_profile("worker") - batch = ( - self.client.batch_chat( - agents_by_id[agent_id], requests, effort_profile=effort_profile - ) - if effort_profile is not None - else self.client.batch_chat(agents_by_id[agent_id], requests) - ) - results = _validate_batch_results(requests, batch) -''', - ''' answers: dict[int, dict[str, Any]] = {} - batch_latency_ms_by_agent: dict[str, float] = {} - for agent_id, requests in requests_by_agent.items(): - effort_profile = self._role_effort_profile("worker") - batch_started_at = time.perf_counter() - batch = ( - self.client.batch_chat( - agents_by_id[agent_id], requests, effort_profile=effort_profile - ) - if effort_profile is not None - else self.client.batch_chat(agents_by_id[agent_id], requests) - ) - # One provider batch call covers every request in this group. Record - # its shared elapsed time on each trace row without claiming - # unavailable per-request timing precision. - batch_latency_ms_by_agent[agent_id] = round( - (time.perf_counter() - batch_started_at) * 1000, - 2, - ) - results = _validate_batch_results(requests, batch) -''', - ) - replace_once( - "contextual_orchestrator/orchestrator.py", - ''' "provider": agent.provider_name or self._infer_provider_name(agent.base_url), - "subtask": "Direct route (batched)", "access": [], "output": result["content"], -''', - ''' "provider": agent.provider_name or self._infer_provider_name(agent.base_url), - "latency_ms": batch_latency_ms_by_agent[agent.id], - "subtask": "Direct route (batched)", "access": [], "output": result["content"], -''', - ) - replace_once( - "tests/test_provider_error_taxonomy.py", - ''' plain_string = safe_provider_message(_body_http_error(400, {"error": "invalid api key"})) - assert plain_string == "invalid api key" -''', - ''' plain_string = safe_provider_message(_body_http_error(400, {"error": "invalid api key"})) - assert plain_string is None -''', - ) - - taxonomy_tests = Path("tests/test_provider_error_taxonomy.py") - taxonomy_text = taxonomy_tests.read_text(encoding="utf-8") - security_test = ''' - -def test_safe_message_discards_sensitive_provider_diagnostics() -> None: - """Credentials, request content, URLs, and private topology never reach callers.""" - diagnostics = ( - "authorization: Bearer provider-secret-value", - "prompt=private customer message", - "request failed at http://10.0.0.9/internal", - "token=abc123456789012345", - ) - for diagnostic in diagnostics: - error = _body_http_error(400, {"error": {"message": diagnostic}}) - assert safe_provider_message(error) is None - classified = classify_provider_failure(error, agent_id="a", model="m") - assert diagnostic not in str(classified) - assert str(classified) == "provider rejected the request with HTTP 400" -''' - if "test_safe_message_discards_sensitive_provider_diagnostics" in taxonomy_text: - raise SystemExit("sensitive-provider-message regression already exists") - taxonomy_tests.write_text(taxonomy_text + security_test, encoding="utf-8") - - regression = Path("tests/test_provider_error_merge_regressions.py") - if regression.exists(): - raise SystemExit(f"{regression} already exists") - regression.write_text( - '''"""Merge-result regressions for provider taxonomy and trace evidence.""" - -from __future__ import annotations - -from contextlib import AbstractContextManager -from types import SimpleNamespace -from unittest.mock import MagicMock, patch - -import pytest - -import contextual_orchestrator.orchestrator as orchestrator_module -import contextual_orchestrator.telemetry as telemetry_module -from contextual_orchestrator.orchestrator import ( - ModelAgent, - ProviderRequestTooLargeError, - TaskOrchestrator, -) -from contextual_orchestrator.telemetry import traced - - -class _SpanContext(AbstractContextManager): - def __init__(self, span: MagicMock) -> None: - self._span = span - - def __enter__(self) -> MagicMock: - return self._span - - def __exit__(self, exc_type, exc, traceback) -> bool: - del exc_type, exc, traceback - return False - - -class _Tracer: - def __init__(self, span: MagicMock) -> None: - self._span = span - - def start_as_current_span(self, *_args, **_kwargs) -> _SpanContext: - return _SpanContext(self._span) - - -class _Trace: - def __init__(self, span: MagicMock) -> None: - self._span = span - - def get_tracer(self, _name: str) -> _Tracer: - return _Tracer(self._span) - - -@pytest.mark.parametrize("transport", ["chat", "passthrough"]) -def test_traced_preserves_request_too_large_taxonomy(monkeypatch, transport: str) -> None: - """HTTP 413 remains request_too_large with its upstream status on every transport.""" - span = MagicMock() - monkeypatch.setattr(telemetry_module, "trace", _Trace(span)) - monkeypatch.setattr(telemetry_module, "Status", lambda code: code) - monkeypatch.setattr(telemetry_module, "StatusCode", SimpleNamespace(ERROR="error")) - failure = ProviderRequestTooLargeError( - "provider request body is too large", - agent_id="worker_agent", - model="model-x", - provider_status=413, - transport=transport, - ) - - with pytest.raises(ProviderRequestTooLargeError): - with traced(f"{transport} model-x"): - raise failure - - span.set_attribute.assert_any_call("error.type", "request_too_large") - span.set_attribute.assert_any_call( - "contextual_orchestrator.provider_status_code", 413 - ) - - -def test_batch_trace_uses_one_honest_shared_provider_duration(monkeypatch) -> None: - """Every result in one provider batch reports the same measured batch duration.""" - agent = ModelAgent("batch_agent", "batch-model", provider_name="provider-x") - orchestrator = TaskOrchestrator([agent]) - - def batch_chat(_agent, requests, **_kwargs): - return { - custom_id: {"content": f"answer:{custom_id}"} - for custom_id in requests - } - - monkeypatch.setattr(orchestrator.client, "batch_chat", batch_chat) - with patch.object( - orchestrator_module.time, - "perf_counter", - side_effect=[10.0, 10.25], - ): - records = orchestrator.batch_route(["one", "two"]) - - assert [record["trace"][0]["latency_ms"] for record in records] == [250.0, 250.0] -''', - encoding="utf-8", - ) - - -if __name__ == "__main__": - main() diff --git a/tests/test_provider_error_merge_regressions.py b/tests/test_provider_error_merge_regressions.py new file mode 100644 index 000000000..318a5be7a --- /dev/null +++ b/tests/test_provider_error_merge_regressions.py @@ -0,0 +1,93 @@ +"""Merge-result regressions for provider taxonomy and trace evidence.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +import contextual_orchestrator.orchestrator as orchestrator_module +import contextual_orchestrator.telemetry as telemetry_module +from contextual_orchestrator.orchestrator import ( + ModelAgent, + ProviderRequestTooLargeError, + TaskOrchestrator, +) +from contextual_orchestrator.telemetry import traced + + +class _SpanContext(AbstractContextManager): + def __init__(self, span: MagicMock) -> None: + self._span = span + + def __enter__(self) -> MagicMock: + return self._span + + def __exit__(self, exc_type, exc, traceback) -> bool: + del exc_type, exc, traceback + return False + + +class _Tracer: + def __init__(self, span: MagicMock) -> None: + self._span = span + + def start_as_current_span(self, *_args, **_kwargs) -> _SpanContext: + return _SpanContext(self._span) + + +class _Trace: + def __init__(self, span: MagicMock) -> None: + self._span = span + + def get_tracer(self, _name: str) -> _Tracer: + return _Tracer(self._span) + + +@pytest.mark.parametrize("transport", ["chat", "passthrough"]) +def test_traced_preserves_request_too_large_taxonomy(monkeypatch, transport: str) -> None: + """HTTP 413 remains request_too_large with its upstream status on every transport.""" + span = MagicMock() + monkeypatch.setattr(telemetry_module, "trace", _Trace(span)) + monkeypatch.setattr(telemetry_module, "Status", lambda code: code) + monkeypatch.setattr(telemetry_module, "StatusCode", SimpleNamespace(ERROR="error")) + failure = ProviderRequestTooLargeError( + "provider request body is too large", + agent_id="worker_agent", + model="model-x", + provider_status=413, + transport=transport, + ) + + with pytest.raises(ProviderRequestTooLargeError): + with traced(f"{transport} model-x"): + raise failure + + span.set_attribute.assert_any_call("error.type", "request_too_large") + span.set_attribute.assert_any_call( + "contextual_orchestrator.provider_status_code", 413 + ) + + +def test_batch_trace_uses_one_honest_shared_provider_duration(monkeypatch) -> None: + """Every result in one provider batch reports the same measured batch duration.""" + agent = ModelAgent("batch_agent", "batch-model", provider_name="provider-x") + orchestrator = TaskOrchestrator([agent]) + + def batch_chat(_agent, requests, **_kwargs): + return { + custom_id: {"content": f"answer:{custom_id}"} + for custom_id in requests + } + + monkeypatch.setattr(orchestrator.client, "batch_chat", batch_chat) + with patch.object( + orchestrator_module.time, + "perf_counter", + side_effect=[10.0, 10.25], + ): + records = orchestrator.batch_route(["one", "two"]) + + assert [record["trace"][0]["latency_ms"] for record in records] == [250.0, 250.0] diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index 925a1696e..008391fb1 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -54,7 +54,7 @@ def test_safe_message_prefers_nested_provider_error_fields() -> None: coded = safe_provider_message(_body_http_error(400, {"error": {"code": "context_length_exceeded"}})) assert coded == "context_length_exceeded" plain_string = safe_provider_message(_body_http_error(400, {"error": "invalid api key"})) - assert plain_string == "invalid api key" + assert plain_string is None top_level = safe_provider_message(_body_http_error(429, {"message": "rate limit reached"})) assert top_level == "rate limit reached" detail = safe_provider_message(_body_http_error(422, {"detail": "validation failed"})) @@ -450,3 +450,19 @@ class UnknownCode(ProviderUpstreamError): ) message = server_module._provider_upstream_message(exotic) assert "contact the operator" in message + + +def test_safe_message_discards_sensitive_provider_diagnostics() -> None: + """Credentials, request content, URLs, and private topology never reach callers.""" + diagnostics = ( + "authorization: Bearer provider-secret-value", + "prompt=private customer message", + "request failed at http://10.0.0.9/internal", + "token=abc123456789012345", + ) + for diagnostic in diagnostics: + error = _body_http_error(400, {"error": {"message": diagnostic}}) + assert safe_provider_message(error) is None + classified = classify_provider_failure(error, agent_id="a", model="m") + assert diagnostic not in str(classified) + assert str(classified) == "provider rejected the request with HTTP 400" From 59f80a4248a3d996fda6deecfbdae474cd2ba06b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 04:02:52 -0700 Subject: [PATCH 33/33] docs: record provider taxonomy and trace evidence contracts --- CHANGELOG.d/provider-error-taxonomy-evidence.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGELOG.d/provider-error-taxonomy-evidence.md diff --git a/CHANGELOG.d/provider-error-taxonomy-evidence.md b/CHANGELOG.d/provider-error-taxonomy-evidence.md new file mode 100644 index 000000000..a76231064 --- /dev/null +++ b/CHANGELOG.d/provider-error-taxonomy-evidence.md @@ -0,0 +1 @@ +Classify provider failures by actionable HTTP/network cause without exposing provider-controlled diagnostics; preserve request-size taxonomy in telemetry, and attach measured shared provider-batch latency to each affected workflow trace row without claiming unavailable per-request timing precision.