Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions contextual_orchestrator/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -4352,6 +4352,14 @@ def send_synthesis(
self._record_failure(final_agent.id)
if final_agent.group_name and not _is_request_too_large_error(exc):
self._group_router.observe_failure(final_agent.id)
if (
virtual_model
and response_format_requested
and not isinstance(exc, ProviderRequestTooLargeError)
Comment thread
seonghobae marked this conversation as resolved.
):
raise NoViableAgentError(
retry_after_seconds=max(1, math.ceil(self.circuit_reset_seconds))
) from None
Comment thread
seonghobae marked this conversation as resolved.
raise

def provider_output(response: Mapping[str, Any]) -> str:
Expand Down
52 changes: 52 additions & 0 deletions tests/test_openai_passthrough.py
Original file line number Diff line number Diff line change
Expand Up @@ -779,6 +779,58 @@ def proxy_send_once(self, agent, endpoint, payload):
assert body["error_message"] == "request body exceeds every eligible provider limit"


def test_http_virtual_structured_synthesis_failure_returns_retryable_503() -> None:
"""A final provider failure is availability, not an internal server bug."""

class FailingSynthesisClient(ModelClient):
def proxy_send_once(self, agent, endpoint, payload):
del agent, endpoint, payload
raise urllib.error.URLError("synthetic provider outage")

proxy_send = proxy_send_once

orchestrator = TaskOrchestrator(
[ModelAgent("worker_agent", "worker-model", tags=("response_format",))],
client=FailingSynthesisClient(), # type: ignore[arg-type]
)
orchestrator.conduct = lambda *args, **kwargs: { # type: ignore[method-assign]
"mode": "conduct",
"answer": "evidence",
"trace": [{
"id": 0,
"role": "worker",
"agent_id": "worker_agent",
"subtask": "Evidence",
"access": [],
"output": "evidence",
}],
"verification": {"accepted": True, "reason": "test", "verifier_output": ""},
}
token = "passthrough_token"
server = build_server(
orchestrator, port=0, security=SecurityConfig(auth_token=token)
)
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
status, body = _post(
f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions",
{
"model": TaskOrchestrator.AUTO_MODEL,
"messages": [{"role": "user", "content": "return JSON"}],
"response_format": {"type": "json_object"},
},
token,
)
finally:
server.shutdown()
server.server_close()

assert status == 503
assert body["error"]["code"] == "no_viable_agent"
assert body["error"]["detail"]["retry_after_seconds"] == 30
assert "synthetic provider outage" not in json.dumps(body)


def test_http_chat_completions_accepts_response_format_and_passes_through() -> None:
server, port, token = _serve()
url = f"http://127.0.0.1:{port}/v1/chat/completions"
Expand Down
4 changes: 3 additions & 1 deletion tests/test_passthrough_provider_failover.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
)
from contextual_orchestrator.orchestrator import (
ModelClient,
NoViableAgentError,
ProviderRequestTooLargeError,
)

Expand Down Expand Up @@ -506,7 +507,7 @@ def test_structured_synthesis_records_non_413_failure_on_actual_provider() -> No
"verification": {"accepted": True, "reason": "test", "verifier_output": ""},
}

with pytest.raises(RuntimeError, match="fallback unavailable"):
with pytest.raises(NoViableAgentError) as exc_info:
orchestrator.proxy_completion(
{
"model": TaskOrchestrator.AUTO_MODEL,
Expand All @@ -516,6 +517,7 @@ def test_structured_synthesis_records_non_413_failure_on_actual_provider() -> No
single_agent=False,
)

assert "fallback unavailable" not in str(exc_info.value)
assert orchestrator._circuit["fallback_agent"]["failures"] == 1.0
assert "primary_agent" not in orchestrator._circuit

Expand Down