Skip to content
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
334f0c6
fix: add safe tool execution fallback policy
Aug 15, 2026
7e1e7dd
fix: harden HTTP tool failure classification
Aug 15, 2026
4e6512c
fix: address tool fallback review findings
Aug 15, 2026
c796661
docs: specify fail-closed HTTP and SSE contract
seonghobae Aug 15, 2026
fbe50d3
fix: address follow-up tool fallback review
github-actions[bot] Aug 16, 2026
4407046
fix: bound tool fallback retries
seonghobae Aug 20, 2026
6cc4466
docs: remove plan whitespace
seonghobae Aug 20, 2026
2351cab
fix(tools): honor suppressed exception contexts
seonghobae Aug 20, 2026
ac7d546
fix(security): require object policy for external admin reads
seonghobae Aug 20, 2026
9794cff
fix(security): pass principal token to resource policy
seonghobae Aug 20, 2026
b762782
test(tool): preserve structured metadata through wrappers
seonghobae Aug 20, 2026
b706952
fix(tools): preserve structured failures through wrappers
seonghobae Aug 20, 2026
f60adbc
docs: record external admin resource policy
seonghobae Aug 20, 2026
5bbec44
refactor: keep tool failure PR focused
seonghobae Aug 20, 2026
e1d4170
test: repair fallback regression checks
seonghobae Aug 20, 2026
1329dc7
fix: contain raw provider failures
seonghobae Aug 21, 2026
1ccb868
fix: sever fail closed tool error causes
seonghobae Aug 21, 2026
e258875
fix: harden tool failure retry boundaries
seonghobae Aug 21, 2026
cc806cd
fix: stop malformed provider failover
seonghobae Aug 21, 2026
276ed4f
fix: preserve terminal provider tool failures
seonghobae Aug 21, 2026
d7b4f68
fix: preserve provider tool-stop failures (#811)
seonghobae Aug 21, 2026
035d08a
fix: count safe retries in circuit health
seonghobae Aug 21, 2026
b24b3a3
refactor: remove unreachable provider error branch
seonghobae Aug 21, 2026
c05ab7e
fix: remove stale fallback error state
seonghobae Aug 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Changelog

All notable changes to this project are documented in this file.

## [Unreleased]

### Added

- Structured tool failure categories, stable fallback actions, and public adapter exceptions.
- Secret-free `tool_fallback_decision` audit events.
- Exact regression coverage for the Strix `Tool execute_command not found in agent strix` failure.

### Changed

- Agent invocation now retries explicitly idempotent transient tool failures with bounded exponential backoff within a per-agent budget.
- A shared four-attempt ceiling now bounds the configured same-agent tool retry budget.
- Fail-closed tool decisions now have dedicated JSON and SSE error contracts, and preserve the observed failure kind in secret-free audit evidence.
- Missing or unavailable tools move to the next eligible agent instead of terminating the workflow immediately.

### Security

- Ambiguous non-idempotent outcomes, invalid arguments, permission denial, and policy denial fail closed.
- Fallback errors and audit events do not copy provider exception text, tool arguments, outputs, or credentials; fail-closed exceptions also sever the original cause chain so later traceback logging cannot recover them.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ python tests/test_admin_contract.py
python tests/test_conventions.py
python tests/test_api_contract.py
python tests/test_security_hardening.py
python tests/test_tool_execution_fallback.py
python tests/test_repository_security_metadata.py
python tests/test_product_planning_contract.py
python tests/test_plugin_driven_artifacts.py
Expand Down
17 changes: 17 additions & 0 deletions contextual_orchestrator/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,15 @@
from .kv_config import InMemoryConfigStore, get_config_store
from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents
from .token_counting import HeuristicTokenCounter, build_token_counter
from .tool_fallback import (
MAX_TOOL_RETRY_ATTEMPTS,
ToolExecutionError,
ToolFallbackAction,
ToolFallbackStoppedError,
ToolFailureDecision,
ToolFailureKind,
classify_tool_failure,
)

__all__ = [
"ModelAgent",
Expand All @@ -48,6 +57,14 @@
"get_credential",
"register_credential",
"NotConfigured",
"MAX_TOOL_RETRY_ATTEMPTS",
# tool fallback
"ToolExecutionError",
"ToolFallbackAction",
"ToolFallbackStoppedError",
"ToolFailureDecision",
"ToolFailureKind",
"classify_tool_failure",
# cost review
"ATTRIBUTION_DIMENSIONS",
"AttributionDimensions",
Expand Down
20 changes: 17 additions & 3 deletions contextual_orchestrator/model_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,19 @@
DISCOVERY_TIMEOUT_SECONDS = 15.0


def _provider_discovery_error_code(exc: Exception) -> str:
"""Map provider failures to stable codes without retaining provider response text."""
if isinstance(exc, urllib.error.HTTPError):
return f"http_status_{exc.code}"
if isinstance(exc, TimeoutError):
return "timeout"
if isinstance(exc, urllib.error.URLError):
return "transport_error"
if isinstance(exc, ValueError):
return "invalid_response"
return "provider_error"


@dataclass(frozen=True)
class ProviderModelSource:
"""Where and how to discover one provider's models."""
Expand Down Expand Up @@ -99,9 +112,10 @@ class DiscoveredModel:
class ProviderDiscoveryError(RuntimeError):
"""Raised when a provider's model list could not be fetched (network/auth failure)."""

def __init__(self, provider_name: str, detail: str) -> None:
def __init__(self, provider_name: str, error_code: str) -> None:
self.provider_name = provider_name
super().__init__(f"model discovery failed for provider {provider_name!r}: {detail}")
self.error_code = error_code
super().__init__(f"model discovery failed for provider {provider_name!r}: {error_code}")


def _fetch_json(url: str, *, api_key: str, auth_scheme: str, timeout: float) -> Any:
Expand Down Expand Up @@ -191,7 +205,7 @@ def discover_provider_models(
try:
payload = _fetch_json(url, api_key=api_key, auth_scheme=source.auth_scheme, timeout=timeout)
except (urllib.error.URLError, TimeoutError, ValueError) as exc: # pragma: no cover - network path
raise ProviderDiscoveryError(source.provider_name, str(exc)) from exc
raise ProviderDiscoveryError(source.provider_name, _provider_discovery_error_code(exc)) from None
if source.style == "bytez":
return _parse_bytez(payload, source)
return _parse_openai_compatible(payload, source)
Expand Down
154 changes: 130 additions & 24 deletions contextual_orchestrator/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@

from .conventions import require_object_name
from .credentials import NotConfigured, get_credential
from .tool_fallback import (
MAX_TOOL_RETRY_ATTEMPTS,
ToolFallbackAction,
ToolFallbackStoppedError,
ToolFailureDecision,
classify_tool_failure,
downgrade_to_failover,
)


# content is usually str; multimodal vision messages use OpenAI content-parts lists.
Expand Down Expand Up @@ -81,6 +89,10 @@ def __init__(self, message: str, detail: dict[str, Any] | None = None) -> None:
self.detail = detail or {}


class ProviderResponseError(RuntimeError):
"""Raised for a provider response that cannot become a safe completion."""


def estimate_tokens(text: str) -> int:
"""Rough token estimate (~4 chars/token). ponytail: heuristic, not a real tokenizer.

Expand Down Expand Up @@ -220,8 +232,8 @@ def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:

try:
decision = json.loads(reply.strip(), object_pairs_hook=reject_duplicate_keys)
except (json.JSONDecodeError, RecursionError, TypeError) as exc:
raise ValueError("judge response is not valid JSON") from exc
except (json.JSONDecodeError, RecursionError, TypeError):
raise ValueError("judge response is not valid JSON") from None
if not isinstance(decision, dict) or set(decision) != {"decision", "reason"}:
raise ValueError("judge response must match the exact verdict schema")
decision_value = decision["decision"]
Expand Down Expand Up @@ -407,6 +419,18 @@ def as_dict(self) -> dict[str, Any]:
LOCAL_PROVIDER_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})


def _is_tool_execution_stopped(error: urllib.error.HTTPError) -> bool:
"""Return whether an HTTP error carries the terminal tool-stop contract."""
try:
payload = json.loads(error.read(65536).decode("utf-8"))
except (AttributeError, OSError, UnicodeDecodeError, json.JSONDecodeError):
return False
if not isinstance(payload, dict):
return False
details = payload.get("error")
return isinstance(details, dict) and details.get("code") == "tool_execution_stopped"


def _is_local_provider_url(base_url: str) -> bool:
"""Return whether a provider uses the explicit loopback-only local scheme."""
parsed = urlparse(base_url)
Expand Down Expand Up @@ -663,6 +687,8 @@ def _chat_to_responses_payload(data: dict[str, Any], request: dict[str, Any]) ->
def is_transient_error(exc: BaseException) -> bool:
"""Return True when a provider call failure is worth retrying with backoff."""
if isinstance(exc, urllib.error.HTTPError):
if _is_tool_execution_stopped(exc):
return False
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return exc.code in TRANSIENT_HTTP_STATUS
Comment thread
seonghobae marked this conversation as resolved.
Resolved
# Network-level failures (DNS, connection reset, read timeout) are transient.
if isinstance(exc, (urllib.error.URLError, TimeoutError, ConnectionError, socket.timeout)):
Expand Down Expand Up @@ -909,8 +935,9 @@ def _send_with_retry(
if attempt >= retry_limit or not is_transient_error(exc):
break
self._sleep(self._backoff_delay(attempt))
detail = f": {last_error}" if last_error else ""
raise RuntimeError(f"provider {agent.id} request failed{detail}") from last_error
if isinstance(last_error, ProviderResponseError):
raise last_error
raise RuntimeError(f"provider {agent.id} request failed") from None
Comment thread
seonghobae marked this conversation as resolved.
Resolved

def _retry_limit(self, agent: ModelAgent) -> int:
"""Return a retry budget without multiplying an expensive local queue by default."""
Expand Down Expand Up @@ -961,11 +988,11 @@ def _response_content(agent: ModelAgent, data: dict[str, Any]) -> str:
if isinstance(content, str):
return content
if isinstance(message, dict) and message.get("reasoning"):
raise RuntimeError(
raise ProviderResponseError(
f"provider {agent.id} returned reasoning without content; "
"for mlx-lm set chat_template_args={\"enable_thinking\": false} or increase max_output_tokens"
)
raise RuntimeError(f"provider {agent.id} response did not contain assistant content")
raise ProviderResponseError(f"provider {agent.id} response did not contain assistant content")

@staticmethod
def _connect_validated(
Expand Down Expand Up @@ -1163,7 +1190,7 @@ def _send_raw_with_retry(
if attempt >= retry_limit or not is_transient_error(exc):
break
self._sleep(self._backoff_delay(attempt))
raise RuntimeError(f"provider {agent.id} passthrough request failed") from last_error
raise RuntimeError(f"provider {agent.id} passthrough request failed") from None

def _send_raw(
self,
Expand Down Expand Up @@ -1667,6 +1694,8 @@ def __init__(
agents_db: str | None = None,
cache_ttl: float = 0.0,
cache_max_entries: int = 256,
tool_retry_attempts: int = 1,
tool_retry_backoff_seconds: float = 0.25,
) -> None:
# Optional durable model-group management: stored operator changes overlay the
# seed agents file at startup (stored rows win by id; stored-new rows append).
Expand All @@ -1679,6 +1708,31 @@ def __init__(
if not self.agents: # pragma: no cover
raise ValueError("at least one enabled agent is required")
self.client = client or ModelClient()
if (
isinstance(tool_retry_attempts, bool)
or not isinstance(tool_retry_attempts, int)
or tool_retry_attempts < 0
or tool_retry_attempts > MAX_TOOL_RETRY_ATTEMPTS
):
raise ValueError(
"tool_retry_attempts must be a nonnegative integer at most "
f"{MAX_TOOL_RETRY_ATTEMPTS}"
)
self.tool_retry_attempts = tool_retry_attempts
if (
isinstance(tool_retry_backoff_seconds, bool)
or not isinstance(tool_retry_backoff_seconds, (int, float))
or not math.isfinite(float(tool_retry_backoff_seconds))
or tool_retry_backoff_seconds < 0
):
raise ValueError(
"tool_retry_backoff_seconds must be a finite nonnegative number"
)
self.tool_retry_backoff_seconds = float(tool_retry_backoff_seconds)
# Injectable seams keep retry timing deterministic in tests while
# production uses full jitter to avoid synchronized retry bursts.
self._tool_retry_sleep = time.sleep
self._tool_retry_jitter = random.uniform
self.policy = OrchestrationPolicy()
# Operator-supplied USD price per 1M tokens, keyed by model. Empty => cost not computed.
self.price_per_million = dict(price_per_million or {})
Expand Down Expand Up @@ -1981,8 +2035,8 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]:
try:
prefix, suffix = custom_id.rsplit("_", 1)
index = int(suffix)
except (AttributeError, ValueError) as exc:
raise RuntimeError("batch provider returned an invalid request identifier") from exc
except (AttributeError, ValueError):
raise RuntimeError("batch provider returned an invalid request identifier") from None
if prefix != "task" or custom_id != f"task_{index}" or not 0 <= index < len(selected):
raise RuntimeError("batch provider returned an invalid request identifier")
if index in answers:
Expand Down Expand Up @@ -2522,26 +2576,78 @@ def _select_agent(self, text: str, role: str) -> ModelAgent:
def _invoke(
self, primary: ModelAgent, messages: list[ChatMessage], *, text: str, role: str
) -> tuple[str, str, dict[str, Any] | None]:
"""Call the primary agent, failing over across capability-matched agents on error.
"""Call an agent with bounded, safety-aware tool retry and failover.

Transient retry/backoff happens inside ``ModelClient``; this layer adds
cross-agent failover plus a per-agent circuit breaker, and returns
``(output, served_agent_id, usage)`` — usage is the provider-reported token
usage when available (else None), so spend analytics can prefer it.
``ModelClient`` handles provider transport retries. This layer classifies
agent/tool-runtime failures: missing tools move to a compatible agent,
explicitly idempotent transient calls retry the same agent, and ambiguous
side effects or policy/permission/argument errors fail closed.
"""
candidates = self._failover_candidates(primary, text, role)
retry_limit = min(self.tool_retry_attempts, MAX_TOOL_RETRY_ATTEMPTS)
last_error: Exception | None = None
for agent in candidates:
try:
output = self.client.chat(agent, messages)
except Exception as exc: # noqa: BLE001 - one agent failing routes to the next
last_error = exc
self._record_failure(agent.id)
continue
self._record_success(agent.id)
usage = self.client.take_usage() if hasattr(self.client, "take_usage") else None
return output, agent.id, usage
raise RuntimeError(f"all {len(candidates)} candidate agents failed for role={role}") from last_error
retry_attempt = 0
while True:
try:
output = self.client.chat(agent, messages)
except Exception as exc: # noqa: BLE001 - classify before fallback
last_error = exc
decision = classify_tool_failure(exc)
action = decision.action
if (
action is ToolFallbackAction.RETRY_SAME_AGENT
and retry_attempt < retry_limit
):
retry_attempt += 1
self._record_tool_fallback(agent.id, decision, retry_attempt)
if self.tool_retry_backoff_seconds:
retry_ceiling = min(
self.tool_retry_backoff_seconds
* (2.0 ** min(retry_attempt - 1, 16)),
30.0,
)
retry_delay = self._tool_retry_jitter(0.0, retry_ceiling)
self._tool_retry_sleep(retry_delay)
continue
Comment thread
seonghobae marked this conversation as resolved.
Resolved
Comment thread
seonghobae marked this conversation as resolved.
Resolved
Comment thread
seonghobae marked this conversation as resolved.
Resolved
if action is ToolFallbackAction.RETRY_SAME_AGENT:
decision = downgrade_to_failover(decision)
action = decision.action
self._record_tool_fallback(agent.id, decision, retry_attempt)
if decision.circuit_failure:
self._record_failure(agent.id)
Comment thread
seonghobae marked this conversation as resolved.
Resolved
if action is ToolFallbackAction.FAIL_CLOSED:
raise ToolFallbackStoppedError(agent.id, decision) from None
break
self._record_success(agent.id)
usage = self.client.take_usage() if hasattr(self.client, "take_usage") else None
return output, agent.id, usage
if isinstance(last_error, ProviderResponseError):
raise last_error
raise RuntimeError(f"all {len(candidates)} candidate agents failed for role={role}") from None

def _record_tool_fallback(
self,
agent_id: str,
decision: ToolFailureDecision,
retry_attempt: int,
) -> None:
"""Record a secret-free audit event for one tool fallback decision."""
event_detail = {
"agent_id": agent_id,
"action": decision.action.value,
"failure_kind": decision.kind.value,
"reason_code": decision.reason_code,
"retry_attempt": retry_attempt,
}
observed_kind = (
decision.kind
if decision.observed_kind is None
else decision.observed_kind
)
if observed_kind is not decision.kind:
event_detail["observed_failure_kind"] = observed_kind.value
self._append_audit_event("tool_fallback_decision", event_detail)
Comment thread
seonghobae marked this conversation as resolved.
Resolved

def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]:
ranked = self._ranked_agents(text, role)
Expand Down
Loading
Loading