diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 0043c70ca2968..12d0034ef63a8 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -29,6 +29,7 @@ _resolve_zai_base_url, _save_auth_store, _save_provider_state, + detect_zai_endpoint, read_credential_pool, write_credential_pool, ) @@ -594,6 +595,55 @@ def _sync_nous_entry_from_auth_store(self, entry: PooledCredential) -> PooledCre logger.debug("Failed to sync Nous entry from auth.json: %s", exc) return entry + def _sync_zai_entry_from_auth_store(self, entry: PooledCredential) -> PooledCredential: + """Sync a Z.AI pool entry from auth.json if the detected endpoint changed. + + Z.AI has two API surfaces (regular /api/paas/v4 and coding /api/coding/paas/v4) + and an account may only have quota on one. When ``detect_zai_endpoint`` probes + during runtime resolution it caches the working endpoint in + ``provider_state.zai.detected_endpoint``. If the pool entry was seeded earlier + with a different endpoint, or the account's available quota shifted, the pool + entry's ``base_url`` becomes stale. This method detects that and adopts the + newer endpoint, clearing exhaustion so the entry can be retried immediately. + """ + if self.provider != "zai": + return entry + try: + with _auth_store_lock(): + auth_store = _load_auth_store() + state = _load_provider_state(auth_store, "zai") + if not isinstance(state, dict): + return entry + detected = state.get("detected_endpoint") + if not isinstance(detected, dict): + return entry + detected_url = detected.get("base_url", "").rstrip("/") + entry_url = (entry.base_url or "").rstrip("/") + if detected_url and detected_url != entry_url: + logger.debug( + "Pool entry %s: syncing Z.AI endpoint from auth.json " + "%s -> %s", + entry.id, + entry_url, + detected_url, + ) + updated = replace( + entry, + base_url=detected_url, + last_status=None, + last_status_at=None, + last_error_code=None, + last_error_reason=None, + last_error_message=None, + last_error_reset_at=None, + ) + self._replace_entry(entry, updated) + self._persist() + return updated + except Exception as exc: + logger.debug("Failed to sync Z.AI entry from auth.json: %s", exc) + return entry + def _sync_device_code_entry_to_auth_store(self, entry: PooledCredential) -> None: """Write refreshed pool entry tokens back to auth.json providers. @@ -883,11 +933,24 @@ def _available_entries(self, *, clear_expired: bool = False, refresh: bool = Fal if synced is not entry: entry = synced cleared_any = True + # For zai entries, sync the detected endpoint from auth.json. + # Z.AI has regular and coding endpoints; the account may only + # have quota on one. If runtime resolution detected a different + # working endpoint, adopt it and clear exhaustion immediately. + if (self.provider == "zai" + and entry.last_status == STATUS_EXHAUSTED): + synced = self._sync_zai_entry_from_auth_store(entry) + if synced is not entry: + entry = synced + cleared_any = True if entry.last_status == STATUS_EXHAUSTED: exhausted_until = _exhausted_until(entry) if exhausted_until is not None and now < exhausted_until: continue if clear_expired: + previous_error_code = entry.last_error_code + previous_error_reason = entry.last_error_reason + previous_error_message = entry.last_error_message cleared = replace( entry, last_status=STATUS_OK, @@ -900,6 +963,60 @@ def _available_entries(self, *, clear_expired: bool = False, refresh: bool = Fal self._replace_entry(entry, cleared) entry = cleared cleared_any = True + # For zai, re-probe endpoints after cooldown clears to + # ensure the cached base_url is still valid. Quota may + # have shifted between regular and coding endpoints. + if self.provider == "zai": + api_key = entry.access_token or "" + if api_key: + try: + detected = detect_zai_endpoint(api_key, timeout=8.0) + if detected and detected.get("base_url"): + new_url = detected["base_url"].rstrip("/") + old_url = (entry.base_url or "").rstrip("/") + if new_url != old_url: + logger.debug( + "Pool entry %s: re-probed Z.AI endpoint %s -> %s", + entry.id, old_url, new_url, + ) + entry = replace(entry, base_url=new_url) + self._replace_entry(cleared, entry) + cleared_any = True + else: + # Probe failed — no endpoint works, keep exhausted + logger.debug( + "Pool entry %s: Z.AI re-probe failed, keeping exhausted", + entry.id, + ) + entry = replace( + entry, + last_status=STATUS_EXHAUSTED, + last_status_at=now, + last_error_code=previous_error_code, + last_error_reason=previous_error_reason, + last_error_message=previous_error_message, + last_error_reset_at=None, + ) + self._replace_entry(cleared, entry) + cleared_any = True + continue + except Exception as exc: + logger.debug( + "Pool entry %s: Z.AI re-probe error: %s", + entry.id, exc, + ) + entry = replace( + entry, + last_status=STATUS_EXHAUSTED, + last_status_at=now, + last_error_code=previous_error_code, + last_error_reason=previous_error_reason, + last_error_message=previous_error_message, + last_error_reset_at=None, + ) + self._replace_entry(cleared, entry) + cleared_any = True + continue if refresh and self._entry_needs_refresh(entry): refreshed = self._refresh_entry(entry, force=False) if refreshed is None: diff --git a/cli.py b/cli.py index b85ee0ee91678..30ecba38014bf 100644 --- a/cli.py +++ b/cli.py @@ -6975,7 +6975,7 @@ def process_command(self, command: str) -> bool: self._handle_fast_command(cmd_original) elif canonical == "compress": self._manual_compress(cmd_original) - elif canonical == "usage": + elif canonical == "costs": self._show_usage() elif canonical == "insights": self._show_insights(cmd_original) diff --git a/gateway/run.py b/gateway/run.py index 15bfe53c66d2d..a40a881b55585 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -5581,7 +5581,7 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: # running-agent guard. Reject gracefully rather than falling # through to interrupt + discard. Without this, commands # like /model, /reasoning, /voice, /insights, /title, - # /resume, /retry, /undo, /compress, /usage, + # /resume, /retry, /undo, /compress, /costs, # /reload-mcp, /sethome, /reset (all registered as Discord # slash commands) would interrupt the agent AND get # silently discarded by the slash-command safety net, @@ -5857,7 +5857,7 @@ async def _do_undo(): if canonical == "compress": return await self._handle_compress_command(event) - if canonical == "usage": + if canonical == "costs": return await self._handle_usage_command(event) if canonical == "insights": diff --git a/hermes_cli/commands.py b/hermes_cli/commands.py index de41bcfae7e8c..049e252c9c24a 100644 --- a/hermes_cli/commands.py +++ b/hermes_cli/commands.py @@ -186,7 +186,8 @@ class CommandDef: CommandDef("help", "Show available commands", "Info"), CommandDef("restart", "Gracefully restart the gateway after draining active runs", "Session", gateway_only=True), - CommandDef("usage", "Show token usage and rate limits for the current session", "Info"), + CommandDef("costs", "Show token usage, costs, and provider rate limits", "Info", + aliases=("usage", "credits")), CommandDef("insights", "Show usage insights and analytics", "Info", args_hint="[days]"), CommandDef("platforms", "Show gateway/messaging platform status", "Info", @@ -349,7 +350,7 @@ def should_bypass_active_session(command_name: str | None) -> bool: safety net in gateway.run discards any command text that reaches the pending queue — which meant a mid-run /model (or /reasoning, /voice, /insights, /title, /resume, /retry, /undo, /compress, - /usage, /reload-mcp, /sethome, /reset) would silently + /costs, /reload-mcp, /sethome, /reset) would silently interrupt the agent AND get discarded, producing a zero-char response. See issue #5057 / PRs #6252, #10370, #4665. diff --git a/tests/agent/test_credential_pool.py b/tests/agent/test_credential_pool.py index 299567a9a6ff9..41f9ad406e1e4 100644 --- a/tests/agent/test_credential_pool.py +++ b/tests/agent/test_credential_pool.py @@ -1641,3 +1641,160 @@ def test_codex_exhausted_entry_stays_stuck_without_auth_store_update(tmp_path, m # still skips it. available = pool._available_entries(clear_expired=True, refresh=False) assert available == [] + + +def test_zai_exhausted_entry_recovers_when_detected_endpoint_changes(tmp_path, monkeypatch): + """Z.AI entries can be stuck on the wrong billing surface. + + Regression for GLM/Z.AI coding-plan accounts: the pool entry may have + been seeded with the regular endpoint, then exhausted with 429 + "insufficient balance" even though the same key has quota on the coding + endpoint. Once endpoint detection has found the coding endpoint, an + exhausted profile entry should adopt it and become selectable immediately. + """ + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) + monkeypatch.setenv("ZAI_API_KEY", "zai-key") + now = time.time() + _write_auth_store( + tmp_path, + { + "version": 1, + "providers": { + "zai": { + "detected_endpoint": { + "base_url": "https://api.z.ai/api/coding/paas/v4", + "endpoint_id": "coding-global", + "model": "glm-5.1", + "label": "Global (Coding Plan)", + "key_hash": "unused-in-pool-sync", + } + } + }, + "credential_pool": { + "zai": [ + { + "id": "zai-1", + "label": "ZAI_API_KEY", + "auth_type": "api_key", + "priority": 0, + "source": "env:ZAI_API_KEY", + "access_token": "zai-key", + "base_url": "https://api.z.ai/api/paas/v4", + "last_status": "exhausted", + "last_status_at": now, + "last_error_code": 429, + "last_error_reason": "1305", + "last_error_message": "Insufficient balance or no resource package", + } + ] + }, + }, + ) + + from agent.credential_pool import load_pool + + pool = load_pool("zai") + available = pool._available_entries(clear_expired=True, refresh=False) + + assert len(available) == 1 + assert available[0].base_url == "https://api.z.ai/api/coding/paas/v4" + assert available[0].last_status is None + assert available[0].last_error_code is None + + +def test_zai_expired_entry_reprobes_endpoint_before_reuse(tmp_path, monkeypatch): + """After Z.AI cooldown, re-probe before sending traffic to a stale URL.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) + monkeypatch.setenv("ZAI_API_KEY", "zai-key") + _write_auth_store( + tmp_path, + { + "version": 1, + "credential_pool": { + "zai": [ + { + "id": "zai-1", + "label": "ZAI_API_KEY", + "auth_type": "api_key", + "priority": 0, + "source": "env:ZAI_API_KEY", + "access_token": "zai-key", + "base_url": "https://api.z.ai/api/paas/v4", + "last_status": "exhausted", + "last_status_at": time.time() - 3700, + "last_error_code": 429, + "last_error_reason": "1305", + } + ] + }, + }, + ) + + from agent import credential_pool as credential_pool_mod + from agent.credential_pool import load_pool + + monkeypatch.setattr( + credential_pool_mod, + "detect_zai_endpoint", + lambda api_key, timeout=8.0: { + "base_url": "https://api.z.ai/api/coding/paas/v4", + "id": "coding-global", + "model": "glm-5.1", + "label": "Global (Coding Plan)", + }, + ) + + pool = load_pool("zai") + available = pool._available_entries(clear_expired=True, refresh=False) + + assert len(available) == 1 + assert available[0].base_url == "https://api.z.ai/api/coding/paas/v4" + assert available[0].last_status == "ok" + + +def test_zai_failed_reprobe_keeps_entry_unavailable(tmp_path, monkeypatch): + """Do not resurrect an expired Z.AI entry when no endpoint currently works.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes")) + monkeypatch.setenv("ZAI_API_KEY", "zai-key") + _write_auth_store( + tmp_path, + { + "version": 1, + "credential_pool": { + "zai": [ + { + "id": "zai-1", + "label": "ZAI_API_KEY", + "auth_type": "api_key", + "priority": 0, + "source": "env:ZAI_API_KEY", + "access_token": "zai-key", + "base_url": "https://api.z.ai/api/paas/v4", + "last_status": "exhausted", + "last_status_at": time.time() - 3700, + "last_error_code": 429, + "last_error_reason": "1305", + } + ] + }, + }, + ) + + from agent import credential_pool as credential_pool_mod + from agent.credential_pool import load_pool + + monkeypatch.setattr( + credential_pool_mod, + "detect_zai_endpoint", + lambda api_key, timeout=8.0: None, + ) + + pool = load_pool("zai") + available = pool._available_entries(clear_expired=True, refresh=False) + + assert available == [] + persisted = json.loads((tmp_path / "hermes" / "auth.json").read_text()) + entry = persisted["credential_pool"]["zai"][0] + assert entry["last_status"] == "exhausted" + assert entry["last_error_code"] == 429 + diff --git a/tests/hermes_cli/test_commands.py b/tests/hermes_cli/test_commands.py index ad4c7d5c6386d..ee0e1ab10364d 100644 --- a/tests/hermes_cli/test_commands.py +++ b/tests/hermes_cli/test_commands.py @@ -108,6 +108,17 @@ def test_alias_resolves_to_canonical(self): assert resolve_command("set-home").name == "sethome" assert resolve_command("reload_mcp").name == "reload-mcp" assert resolve_command("tasks").name == "agents" + assert resolve_command("usage").name == "costs" + assert resolve_command("credits").name == "costs" + + def test_costs_is_canonical_gateway_usage_command(self): + costs = resolve_command("costs") + assert costs is not None + assert costs.name == "costs" + assert resolve_command("/costs").name == "costs" + assert "costs" in GATEWAY_KNOWN_COMMANDS + assert "usage" in GATEWAY_KNOWN_COMMANDS + assert "credits" in GATEWAY_KNOWN_COMMANDS def test_topic_is_gateway_command(self): topic = resolve_command("topic") @@ -249,6 +260,12 @@ def test_excludes_commands_with_required_args(self): assert "steer" not in names assert "background" in GATEWAY_KNOWN_COMMANDS + def test_costs_is_telegram_menu_command(self): + names = {name for name, _ in telegram_bot_commands()} + assert "costs" in names + assert "usage" not in names # aliases are accepted but not menu entries + assert "credits" not in names + class TestSlackSubcommandMap: def test_returns_dict(self):