feat: add provider rotation cooldowns

agent/agent_runtime_helpers.py

@@ -860,11 +860,69 @@ def restore_primary_runtime(agent) -> bool:
         # entirely, stranding the index and silently blocking all future
         # fallback attempts for the session.  Fixes #20465.
         agent._fallback_index = 0
+        try:
+            from hermes_cli.config import load_config
+            from agent.provider_rotation import ProviderRotationState, is_rotation_enabled
+
+            rotation_config = load_config()
+            primary_provider = ((agent._primary_runtime or {}).get("provider") or getattr(agent, "provider", "") or "").strip()
+            primary_model = ((agent._primary_runtime or {}).get("model") or getattr(agent, "model", "") or "").strip()
+            primary_base_url = str(
+                (agent._primary_runtime or {}).get("base_url")
+                or getattr(agent, "base_url", "")
+                or ""
+            ).strip()
+            if (
+                is_rotation_enabled(rotation_config)
+                and primary_provider
+                and primary_model
+                and ProviderRotationState.load().is_unavailable(
+                    primary_provider,
+                    primary_model,
+                    base_url=primary_base_url,
+                )
+            ):
+                logging.info(
+                    "Provider rotation: primary %s (%s) is cooling down; trying fallback",
+                    primary_model,
+                    primary_provider,
+                )
+                return bool(agent._try_activate_fallback())
+        except Exception:
+            logging.debug("Provider rotation turn-start check skipped", exc_info=True)
         return False
 
     if getattr(agent, "_rate_limited_until", 0) > time.monotonic():
         return False  # primary still in rate-limit cooldown, stay on fallback
 
+    try:
+        from hermes_cli.config import load_config
+        from agent.provider_rotation import ProviderRotationState, is_rotation_enabled
+
+        rotation_config = load_config()
+        rt = agent._primary_runtime
+        primary_provider = (rt.get("provider") or "").strip()
+        primary_model = (rt.get("model") or "").strip()
+        primary_base_url = str(rt.get("base_url") or "").strip()
+        if (
+            is_rotation_enabled(rotation_config)
+            and primary_provider
+            and primary_model
+            and ProviderRotationState.load().is_unavailable(
+                primary_provider,
+                primary_model,
+                base_url=primary_base_url,
+            )
+        ):
+            logging.info(
+                "Provider rotation: primary %s (%s) still cooling down after transient gate; staying on fallback",
+                primary_model,
+                primary_provider,
+            )
+            return False
+    except Exception:
+        logging.debug("Provider rotation restore-path check skipped", exc_info=True)
+
     rt = agent._primary_runtime
     try:
         # ── Core runtime state ──

agent/chat_completion_helpers.py

@@ -717,7 +717,13 @@ def build_assistant_message(agent, assistant_message, finish_reason: str) -> dic
 
 
 
-def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool:
+def try_activate_fallback(
+    agent,
+    reason: "FailoverReason | None" = None,
+    *,
+    rate_limit_headers: Any = None,
+    error_context: dict[str, Any] | None = None,
+) -> bool:
     """Switch to the next fallback model/provider in the chain.
 
     Called when the current model is failing after retries.  Swaps the
@@ -729,6 +735,8 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
     auth resolution and client construction — no duplicated provider→key
     mappings.
     """
+    rotation_enabled = False
+    rotation_config = {}
     if reason in {FailoverReason.rate_limit, FailoverReason.billing}:
         # Only start cooldown when leaving the primary provider.  If we're
         # already on a fallback and chain-switching, the primary wasn't the
@@ -738,11 +746,69 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
         primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower()
         if (not fallback_already_active) or (primary_provider and current_provider == primary_provider):
             agent._rate_limited_until = time.monotonic() + 60
+    try:
+        from hermes_cli.config import load_config
+        from agent.provider_rotation import (
+            ProviderRotationState,
+            cooldown_for_reason,
+            has_durable_rate_limit_evidence,
+            is_rotation_enabled,
+        )
+
+        rotation_config = load_config()
+        rotation_enabled = is_rotation_enabled(rotation_config)
+        if rotation_enabled and reason in {FailoverReason.rate_limit, FailoverReason.billing}:
+            should_persist_cooldown = reason == FailoverReason.billing or has_durable_rate_limit_evidence(
+                headers=rate_limit_headers,
+                last_known_state=getattr(agent, "_rate_limit_state", None),
+                error_context=error_context,
+            )
+            if should_persist_cooldown:
+                current_provider_for_state = (getattr(agent, "provider", "") or "").strip()
+                current_model_for_state = (getattr(agent, "model", "") or "").strip()
+                current_base_url_for_state = str(getattr(agent, "base_url", "") or "").strip()
+                if current_provider_for_state and current_model_for_state:
+                    ProviderRotationState.load().mark_unavailable(
+                        provider=current_provider_for_state,
+                        model=current_model_for_state,
+                        base_url=current_base_url_for_state,
+                        reason=getattr(reason, "value", str(reason)),
+                        cooldown_seconds=cooldown_for_reason(
+                            rotation_config,
+                            getattr(reason, "value", str(reason)),
+                        ),
+                    )
+    except Exception:
+        logger.debug("Provider rotation state update skipped", exc_info=True)
+
     if agent._fallback_index >= len(agent._fallback_chain):
         return False
 
     fb = agent._fallback_chain[agent._fallback_index]
     agent._fallback_index += 1
+    if rotation_enabled:
+        try:
+            from agent.provider_rotation import ProviderRotationState
+
+            while (
+                isinstance(fb, dict)
+                and ProviderRotationState.load().is_unavailable(
+                    fb.get("provider") or "",
+                    fb.get("model") or "",
+                    base_url=fb.get("base_url") or "",
+                )
+                and agent._fallback_index < len(agent._fallback_chain)
+            ):
+                fb = agent._fallback_chain[agent._fallback_index]
+                agent._fallback_index += 1
+            if isinstance(fb, dict) and ProviderRotationState.load().is_unavailable(
+                fb.get("provider") or "",
+                fb.get("model") or "",
+                base_url=fb.get("base_url") or "",
+            ):
+                return False
+        except Exception:
+            logger.debug("Provider rotation filtering skipped", exc_info=True)
     fb_provider = (fb.get("provider") or "").strip().lower()
     fb_model = (fb.get("model") or "").strip()
     if not fb_provider or not fb_model:

agent/conversation_loop.py

@@ -2430,7 +2430,16 @@ def _stop_spinner():
                     )
                     if not pool_may_recover:
                         agent._emit_status("⚠️ Rate limited — switching to fallback provider...")
-                        if agent._try_activate_fallback(reason=classified.reason):
+                        _err_resp = getattr(api_error, "response", None)
+                        _err_hdrs = (
+                            getattr(_err_resp, "headers", None)
+                            if _err_resp else None
+                        )
+                        if agent._try_activate_fallback(
+                            reason=classified.reason,
+                            rate_limit_headers=_err_hdrs,
+                            error_context=error_context,
+                        ):
                             retry_count = 0
                             compression_attempts = 0
                             primary_recovery_attempted = False