Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion agent/agent_runtime_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -930,6 +930,7 @@ def recover_with_credential_pool(
has_retried_429: bool,
classified_reason: Optional[FailoverReason] = None,
error_context: Optional[Dict[str, Any]] = None,
billing_unverified: bool = False,
) -> tuple[bool, bool]:
"""Attempt credential recovery via pool rotation.

Expand All @@ -944,6 +945,12 @@ def recover_with_credential_pool(
providers that surface billing/rate-limit/auth conditions under a
different status code, such as Anthropic returning HTTP 400 for
"out of extra usage".

`billing_unverified` marks a billing verdict that rests on an ambiguous
body (``ClassifiedError.billing_unverified``, #82154): the pool persists
it as ``billing_unverified`` so the exhausted entry gets a short cooldown
instead of the one-hour billing bench — the same 400 can be a
content-filter rejection that leaves the credential healthy.
"""
pool = agent._credential_pool
if pool is None:
Expand Down Expand Up @@ -1036,7 +1043,13 @@ def _rotate_failed_credential(rotate_status: int):
# cooldowns — the pool can only tell them apart if we say which.
# ``effective_reason`` is resolved below; this closure runs after.
if effective_reason is not None:
kwargs["failure_reason"] = effective_reason.value
_failure_reason = effective_reason.value
if effective_reason == FailoverReason.billing and billing_unverified:
# Ambiguous billing body (#82154): persist the ambiguity so
# the cooldown is sized as transient, not a 1-hour bench.
from agent.credential_pool import FAILURE_REASON_BILLING_UNVERIFIED
_failure_reason = FAILURE_REASON_BILLING_UNVERIFIED
kwargs["failure_reason"] = _failure_reason
return pool.mark_exhausted_and_rotate(**kwargs)

effective_reason = classified_reason
Expand Down
188 changes: 153 additions & 35 deletions agent/conversation_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,7 @@ def _billing_or_entitlement_message(
provider: str,
base_url: str,
model: str,
unverified: bool = False,
) -> str:
if _is_nous_inference_route(provider, base_url):
return _nous_entitlement_message(capability)
Expand All @@ -526,16 +527,45 @@ def _billing_or_entitlement_message(
# apply to a subscription — the user waits for the reset or switches to an
# API key.
if (provider or "").strip().lower() == "anthropic":
lines = [
(
f"{provider_label} reported that your Claude subscription usage is "
f"exhausted for {model_label} (included quota + extra-usage credits)."
),
"Options: wait for the billing cycle to reset, or add extra usage at "
"https://claude.ai/settings/usage",
"You can also switch to an Anthropic API key or another provider with "
"/model <model> --provider <provider>.",
]
# ``unverified`` (ClassifiedError.billing_unverified, #82154): the
# "out of extra usage" 400 is ambiguous — Anthropic returns the same
# body when its server-side content filter rejects part of the request
# on a subscription OAuth token, so the message reliably misdirects
# diagnosis toward buying quota. Hedge the claim and name the other
# cause. A confirmed verdict (e.g. a real 402 or an API-key credit
# depletion) keeps the assertive wording.
if unverified:
lines = [
(
f"{provider_label} reported that your Claude subscription usage may be "
f"exhausted for {model_label} (included quota + extra-usage credits) — "
"but this specific error is not proof of a billing problem."
),
"If https://claude.ai/settings/usage still shows quota remaining, this is "
"probably NOT a billing problem: on a Claude subscription (OAuth) token "
"Anthropic returns this same message when its content filter rejects part "
"of the request — typically a phrase in the system prompt.",
"If usage really is exhausted: wait for the billing cycle to reset, or add "
"extra usage at https://claude.ai/settings/usage",
"You can also switch to an Anthropic API key or another provider with "
"/model <model> --provider <provider>.",
# The exhaustion latch replays the stored error without issuing
# a request, so a real fix looks like it didn't work.
"Retry with a fresh credential state: `hermes auth reset anthropic`. Until "
"that cooldown clears, this error can be replayed from cache without "
"contacting the API.",
]
else:
lines = [
(
f"{provider_label} reported that your Claude subscription usage is "
f"exhausted for {model_label} (included quota + extra-usage credits)."
),
"Options: wait for the billing cycle to reset, or add extra usage at "
"https://claude.ai/settings/usage",
"You can also switch to an Anthropic API key or another provider with "
"/model <model> --provider <provider>.",
]
return "\n".join(lines)

# Provider-agnostic billing URL derivation (OpenAI, DeepSeek, xAI, Groq,
Expand Down Expand Up @@ -564,16 +594,84 @@ def _billing_or_entitlement_message(
return "\n".join(lines)


def _billing_block_dict(provider, base_url, model, message="") -> Optional[dict]:
def _billing_block_dict(
provider, base_url, model, message="", *, unverified: bool = False
) -> Optional[dict]:
"""Best-effort structured billing descriptor (None if billing_links is unavailable)."""
try:
from agent.billing_links import build_billing_block

return build_billing_block(
block = build_billing_block(
provider=provider, base_url=str(base_url), model=model, message=message
).to_dict()
except Exception:
return None
if block is not None and unverified:
# Carry the classifier's ambiguity into the structured descriptor so
# every surface rendering the block can hedge too (#82154).
block["unverified"] = True
return block


def _billing_terminal_label(summary: str, unverified: bool) -> str:
"""Terminal-failure prefix for a billing-classified error.

``unverified`` (#82154): the Anthropic "out of extra usage" 400 can be a
content-filter rejection, so the terminal line must not assert billing
exhaustion as fact.
"""
if unverified:
return (
"Provider reported usage/credit exhaustion (unverified — the same "
f"error can be a content-filter rejection, not billing): {summary}"
)
return f"Billing or credits exhausted: {summary}"


def _billing_failure_result(
*,
classified,
summary: str,
messages,
api_call_count: int,
provider: str,
base_url,
model: str,
guidance: Optional[str] = None,
) -> dict:
"""Structured terminal result for a billing-classified failure.

Single construction point for the returned terminal response so the
label, guidance, structured block, and ambiguity flag stay consistent
across the non-retryable abort and max-retries paths (#82154).
"""
unverified = bool(getattr(classified, "billing_unverified", False))
if guidance is None:
guidance = _billing_or_entitlement_message(
capability="model access",
provider=provider,
base_url=str(base_url),
model=model,
unverified=unverified,
)
final = _billing_terminal_label(summary, unverified)
if guidance:
final += f"\n\n{guidance}"
return {
"final_response": final,
"messages": messages,
"api_calls": api_call_count,
"completed": False,
"failed": True,
"error": summary,
"failure_reason": classified.reason.value,
# The billing verdict may rest on an ambiguous body (#82154) — carry
# that through the structured result, not just the prose.
"billing_unverified": unverified,
"billing_block": _billing_block_dict(
provider, base_url, model, guidance, unverified=unverified
),
}


def _print_billing_or_entitlement_guidance(
Expand All @@ -583,12 +681,14 @@ def _print_billing_or_entitlement_guidance(
provider: str,
base_url: str,
model: str,
unverified: bool = False,
) -> bool:
message = _billing_or_entitlement_message(
capability=capability,
provider=provider,
base_url=base_url,
model=model,
unverified=unverified,
)
if not message:
return False
Expand Down Expand Up @@ -4334,6 +4434,7 @@ def _perform_api_call(next_api_kwargs):
has_retried_429=_retry.has_retried_429,
classified_reason=classified.reason,
error_context=error_context,
billing_unverified=classified.billing_unverified,
)
if recovered_with_pool:
continue
Expand Down Expand Up @@ -4978,9 +5079,17 @@ def _perform_api_call(next_api_kwargs):
"switching to fallback model..."
)
elif classified.reason == FailoverReason.billing:
agent._buffer_status(
"⚠️ Billing or credits exhausted — switching to fallback provider..."
)
if classified.billing_unverified:
# Ambiguous body (#82154) — don't assert billing.
agent._buffer_status(
"⚠️ Provider reported usage/credit exhaustion "
"(unverified — may be a content-filter rejection) "
"— switching to fallback provider..."
)
else:
agent._buffer_status(
"⚠️ Billing or credits exhausted — switching to fallback provider..."
)
elif _is_transport_failure:
agent._buffer_status(
"⚠️ Provider unreachable — switching to fallback provider..."
Expand Down Expand Up @@ -5677,6 +5786,7 @@ def _perform_api_call(next_api_kwargs):
provider=_provider,
base_url=str(_base),
model=_model,
unverified=classified.billing_unverified,
):
pass
elif _provider == "nous" and _print_nous_entitlement_guidance(
Expand Down Expand Up @@ -5803,26 +5913,15 @@ def _perform_api_call(next_api_kwargs):
# the max-retries path so every surface (CLI, TUI, desktop)
# renders one consistent billing signal.
if classified.reason == FailoverReason.billing:
_ce_guidance = _billing_or_entitlement_message(
capability="model access",
return _billing_failure_result(
classified=classified,
summary=_nonretryable_summary,
messages=messages,
api_call_count=api_call_count,
provider=_provider,
base_url=str(_base),
base_url=_base,
model=_model,
)
_ce_final = f"Billing or credits exhausted: {_nonretryable_summary}"
if _ce_guidance:
_ce_final += f"\n\n{_ce_guidance}"
_ce_block = _billing_block_dict(_provider, _base, _model, _ce_guidance)
return {
"final_response": _ce_final,
"messages": messages,
"api_calls": api_call_count,
"completed": False,
"failed": True,
"error": _nonretryable_summary,
"failure_reason": classified.reason.value,
"billing_block": _ce_block,
}
return {
"final_response": _nonretryable_summary,
"messages": messages,
Expand Down Expand Up @@ -5866,19 +5965,28 @@ def _perform_api_call(next_api_kwargs):
_final_summary = agent._summarize_api_error(api_error)
_billing_guidance = ""
if classified.reason == FailoverReason.billing:
agent._emit_status(f"❌ Billing or credits exhausted — {_final_summary}")
if classified.billing_unverified:
# Ambiguous body (#82154) — hedge the terminal line.
agent._emit_status(
"❌ Provider reported usage/credit exhaustion "
f"(unverified — may be a content-filter rejection) — {_final_summary}"
)
else:
agent._emit_status(f"❌ Billing or credits exhausted — {_final_summary}")
_billing_guidance = _billing_or_entitlement_message(
capability="model access",
provider=_provider,
base_url=str(_base),
model=_model,
unverified=classified.billing_unverified,
)
_print_billing_or_entitlement_guidance(
agent,
capability="model access",
provider=_provider,
base_url=str(_base),
model=_model,
unverified=classified.billing_unverified,
)
elif is_rate_limited:
agent._emit_status(f"❌ Rate limited after {max_retries} retries — {_final_summary}")
Expand Down Expand Up @@ -5985,13 +6093,20 @@ def _perform_api_call(next_api_kwargs):
)
agent._persist_session(messages, conversation_history)
_billing_block = None
_billing_unverified = False
if classified.reason == FailoverReason.billing:
_final_response = f"Billing or credits exhausted: {_final_summary}"
_billing_unverified = classified.billing_unverified
_final_response = _billing_terminal_label(
_final_summary, _billing_unverified
)
if _billing_guidance:
_final_response += f"\n\n{_billing_guidance}"
# Structured recovery descriptor so every surface renders
# the same link + label from one signal (see helper).
_billing_block = _billing_block_dict(_provider, _base, _model, _billing_guidance)
_billing_block = _billing_block_dict(
_provider, _base, _model, _billing_guidance,
unverified=_billing_unverified,
)
else:
_final_response = f"API call failed after {max_retries} retries: {_final_summary}"
if _is_thinking_timeout:
Expand Down Expand Up @@ -6031,6 +6146,9 @@ def _perform_api_call(next_api_kwargs):
# different exit code. ``rate_limit`` / ``billing`` here
# mean "quota wall, not a task error".
"failure_reason": classified.reason.value,
# True when the billing verdict rests on an ambiguous
# body (#82154) — may be a content-filter rejection.
"billing_unverified": _billing_unverified,
# Present only for billing walls: structured recovery
# descriptor (provider, billing_url, is_nous, message).
"billing_block": _billing_block,
Expand Down
17 changes: 17 additions & 0 deletions agent/credential_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,14 @@ def _load_config_safe() -> Optional[dict]:
# the classifier), so the value is duplicated here rather than referenced.
FAILURE_REASON_BILLING = "billing"

# Billing verdict that rests on an ambiguous body (#82154): Anthropic's
# "out of extra usage" 400 is returned both for genuine overage depletion and
# for a server-side content-filter rejection of the request. The latter leaves
# the credential perfectly healthy, so an unverified billing exhaustion gets
# the short transient cooldown instead of the one-hour billing bench — a
# genuine depletion simply re-latches on the next attempt.
FAILURE_REASON_BILLING_UNVERIFIED = "billing_unverified"

# Throttle window for the "no available entries" INFO line. Credential
# selection runs on a hot path (every model call, plus auxiliary tasks like
# compression/moa/titles), so when a pool is empty or fully exhausted the
Expand Down Expand Up @@ -332,6 +340,15 @@ def _exhausted_ttl(
if error_code == 401:
return EXHAUSTED_TTL_401_SECONDS
base = EXHAUSTED_TTL_429_SECONDS if error_code == 429 else EXHAUSTED_TTL_DEFAULT_SECONDS
# Unverified billing (#82154): the same 400 body can be a content-filter
# rejection of the request itself, in which case the credential is healthy
# and an hour-long bench just blocks it (and, for a sole credential,
# replays the stored error for the full hour — making a real fix look like
# it did not work). Short cooldown regardless of pool size; a genuine
# depletion re-latches on the next attempt. A true 402 stays a full bench
# even if something mislabeled it unverified.
if failure_reason == FAILURE_REASON_BILLING_UNVERIFIED and error_code != 402:
return min(base, EXHAUSTED_TTL_SOLE_CREDENTIAL_SECONDS)
# Sole credential: shorten only TRANSIENT throttles (429 rate-limit, 403
# edge-throttle, 5xx server, or unknown). Billing exhaustion — whether
# classified as such or self-evident from a 402 — is a genuine depletion
Expand Down
Loading
Loading