fix: recover Z.AI endpoints and register costs command

agent/credential_pool.py

@@ -29,6 +29,7 @@
     _resolve_zai_base_url,
     _save_auth_store,
     _save_provider_state,
+    detect_zai_endpoint,
     read_credential_pool,
     write_credential_pool,
 )
@@ -594,6 +595,55 @@ def _sync_nous_entry_from_auth_store(self, entry: PooledCredential) -> PooledCre
             logger.debug("Failed to sync Nous entry from auth.json: %s", exc)
         return entry
 
+    def _sync_zai_entry_from_auth_store(self, entry: PooledCredential) -> PooledCredential:
+        """Sync a Z.AI pool entry from auth.json if the detected endpoint changed.
+
+        Z.AI has two API surfaces (regular /api/paas/v4 and coding /api/coding/paas/v4)
+        and an account may only have quota on one.  When ``detect_zai_endpoint`` probes
+        during runtime resolution it caches the working endpoint in
+        ``provider_state.zai.detected_endpoint``.  If the pool entry was seeded earlier
+        with a different endpoint, or the account's available quota shifted, the pool
+        entry's ``base_url`` becomes stale.  This method detects that and adopts the
+        newer endpoint, clearing exhaustion so the entry can be retried immediately.
+        """
+        if self.provider != "zai":
+            return entry
+        try:
+            with _auth_store_lock():
+                auth_store = _load_auth_store()
+                state = _load_provider_state(auth_store, "zai")
+            if not isinstance(state, dict):
+                return entry
+            detected = state.get("detected_endpoint")
+            if not isinstance(detected, dict):
+                return entry
+            detected_url = detected.get("base_url", "").rstrip("/")
+            entry_url = (entry.base_url or "").rstrip("/")
+            if detected_url and detected_url != entry_url:
+                logger.debug(
+                    "Pool entry %s: syncing Z.AI endpoint from auth.json "
+                    "%s -> %s",
+                    entry.id,
+                    entry_url,
+                    detected_url,
+                )
+                updated = replace(
+                    entry,
+                    base_url=detected_url,
+                    last_status=None,
+                    last_status_at=None,
+                    last_error_code=None,
+                    last_error_reason=None,
+                    last_error_message=None,
+                    last_error_reset_at=None,
+                )
+                self._replace_entry(entry, updated)
+                self._persist()
+                return updated
+        except Exception as exc:
+            logger.debug("Failed to sync Z.AI entry from auth.json: %s", exc)
+        return entry
+
     def _sync_device_code_entry_to_auth_store(self, entry: PooledCredential) -> None:
         """Write refreshed pool entry tokens back to auth.json providers.
 
@@ -883,11 +933,24 @@ def _available_entries(self, *, clear_expired: bool = False, refresh: bool = Fal
                 if synced is not entry:
                     entry = synced
                     cleared_any = True
+            # For zai entries, sync the detected endpoint from auth.json.
+            # Z.AI has regular and coding endpoints; the account may only
+            # have quota on one.  If runtime resolution detected a different
+            # working endpoint, adopt it and clear exhaustion immediately.
+            if (self.provider == "zai"
+                    and entry.last_status == STATUS_EXHAUSTED):
+                synced = self._sync_zai_entry_from_auth_store(entry)
+                if synced is not entry:
+                    entry = synced
+                    cleared_any = True
             if entry.last_status == STATUS_EXHAUSTED:
                 exhausted_until = _exhausted_until(entry)
                 if exhausted_until is not None and now < exhausted_until:
                     continue
                 if clear_expired:
+                    previous_error_code = entry.last_error_code
+                    previous_error_reason = entry.last_error_reason
+                    previous_error_message = entry.last_error_message
                     cleared = replace(
                         entry,
                         last_status=STATUS_OK,
@@ -900,6 +963,60 @@ def _available_entries(self, *, clear_expired: bool = False, refresh: bool = Fal
                     self._replace_entry(entry, cleared)
                     entry = cleared
                     cleared_any = True
+                    # For zai, re-probe endpoints after cooldown clears to
+                    # ensure the cached base_url is still valid.  Quota may
+                    # have shifted between regular and coding endpoints.
+                    if self.provider == "zai":
+                        api_key = entry.access_token or ""
+                        if api_key:
+                            try:
+                                detected = detect_zai_endpoint(api_key, timeout=8.0)
+                                if detected and detected.get("base_url"):
+                                    new_url = detected["base_url"].rstrip("/")
+                                    old_url = (entry.base_url or "").rstrip("/")
+                                    if new_url != old_url:
+                                        logger.debug(
+                                            "Pool entry %s: re-probed Z.AI endpoint %s -> %s",
+                                            entry.id, old_url, new_url,
+                                        )
+                                        entry = replace(entry, base_url=new_url)
+                                        self._replace_entry(cleared, entry)
+                                        cleared_any = True
+                                else:
+                                    # Probe failed — no endpoint works, keep exhausted
+                                    logger.debug(
+                                        "Pool entry %s: Z.AI re-probe failed, keeping exhausted",
+                                        entry.id,
+                                    )
+                                    entry = replace(
+                                        entry,
+                                        last_status=STATUS_EXHAUSTED,
+                                        last_status_at=now,
+                                        last_error_code=previous_error_code,
+                                        last_error_reason=previous_error_reason,
+                                        last_error_message=previous_error_message,
+                                        last_error_reset_at=None,
+                                    )
+                                    self._replace_entry(cleared, entry)
+                                    cleared_any = True
+                                    continue
+                            except Exception as exc:
+                                logger.debug(
+                                    "Pool entry %s: Z.AI re-probe error: %s",
+                                    entry.id, exc,
+                                )
+                                entry = replace(
+                                    entry,
+                                    last_status=STATUS_EXHAUSTED,
+                                    last_status_at=now,
+                                    last_error_code=previous_error_code,
+                                    last_error_reason=previous_error_reason,
+                                    last_error_message=previous_error_message,
+                                    last_error_reset_at=None,
+                                )
+                                self._replace_entry(cleared, entry)
+                                cleared_any = True
+                                continue
             if refresh and self._entry_needs_refresh(entry):
                 refreshed = self._refresh_entry(entry, force=False)
                 if refreshed is None:

cli.py

@@ -6975,7 +6975,7 @@ def process_command(self, command: str) -> bool:
             self._handle_fast_command(cmd_original)
         elif canonical == "compress":
             self._manual_compress(cmd_original)
-        elif canonical == "usage":
+        elif canonical == "costs":
             self._show_usage()
         elif canonical == "insights":
             self._show_insights(cmd_original)

gateway/run.py

@@ -5581,7 +5581,7 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]:
             # running-agent guard. Reject gracefully rather than falling
             # through to interrupt + discard. Without this, commands
             # like /model, /reasoning, /voice, /insights, /title,
-            # /resume, /retry, /undo, /compress, /usage,
+            # /resume, /retry, /undo, /compress, /costs,
             # /reload-mcp, /sethome, /reset (all registered as Discord
             # slash commands) would interrupt the agent AND get
             # silently discarded by the slash-command safety net,
@@ -5857,7 +5857,7 @@ async def _do_undo():
         if canonical == "compress":
             return await self._handle_compress_command(event)
 
-        if canonical == "usage":
+        if canonical == "costs":
             return await self._handle_usage_command(event)
 
         if canonical == "insights":

hermes_cli/commands.py

@@ -186,7 +186,8 @@ class CommandDef:
     CommandDef("help", "Show available commands", "Info"),
     CommandDef("restart", "Gracefully restart the gateway after draining active runs", "Session",
                gateway_only=True),
-    CommandDef("usage", "Show token usage and rate limits for the current session", "Info"),
+    CommandDef("costs", "Show token usage, costs, and provider rate limits", "Info",
+               aliases=("usage", "credits")),
     CommandDef("insights", "Show usage insights and analytics", "Info",
                args_hint="[days]"),
     CommandDef("platforms", "Show gateway/messaging platform status", "Info",
@@ -349,7 +350,7 @@ def should_bypass_active_session(command_name: str | None) -> bool:
     safety net in gateway.run discards any command text that reaches
     the pending queue — which meant a mid-run /model (or /reasoning,
     /voice, /insights, /title, /resume, /retry, /undo, /compress,
-    /usage, /reload-mcp, /sethome, /reset) would silently
+    /costs, /reload-mcp, /sethome, /reset) would silently
     interrupt the agent AND get discarded, producing a zero-char
     response. See issue #5057 / PRs #6252, #10370, #4665.
 

tests/agent/test_credential_pool.py

@@ -1641,3 +1641,160 @@ def test_codex_exhausted_entry_stays_stuck_without_auth_store_update(tmp_path, m
     # still skips it.
     available = pool._available_entries(clear_expired=True, refresh=False)
     assert available == []
+
+
+def test_zai_exhausted_entry_recovers_when_detected_endpoint_changes(tmp_path, monkeypatch):
+    """Z.AI entries can be stuck on the wrong billing surface.
+
+    Regression for GLM/Z.AI coding-plan accounts: the pool entry may have
+    been seeded with the regular endpoint, then exhausted with 429
+    "insufficient balance" even though the same key has quota on the coding
+    endpoint. Once endpoint detection has found the coding endpoint, an
+    exhausted profile entry should adopt it and become selectable immediately.
+    """
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
+    monkeypatch.setenv("ZAI_API_KEY", "zai-key")
+    now = time.time()
+    _write_auth_store(
+        tmp_path,
+        {
+            "version": 1,
+            "providers": {
+                "zai": {
+                    "detected_endpoint": {
+                        "base_url": "https://api.z.ai/api/coding/paas/v4",
+                        "endpoint_id": "coding-global",
+                        "model": "glm-5.1",
+                        "label": "Global (Coding Plan)",
+                        "key_hash": "unused-in-pool-sync",
+                    }
+                }
+            },
+            "credential_pool": {
+                "zai": [
+                    {
+                        "id": "zai-1",
+                        "label": "ZAI_API_KEY",
+                        "auth_type": "api_key",
+                        "priority": 0,
+                        "source": "env:ZAI_API_KEY",
+                        "access_token": "zai-key",
+                        "base_url": "https://api.z.ai/api/paas/v4",
+                        "last_status": "exhausted",
+                        "last_status_at": now,
+                        "last_error_code": 429,
+                        "last_error_reason": "1305",
+                        "last_error_message": "Insufficient balance or no resource package",
+                    }
+                ]
+            },
+        },
+    )
+
+    from agent.credential_pool import load_pool
+
+    pool = load_pool("zai")
+    available = pool._available_entries(clear_expired=True, refresh=False)
+
+    assert len(available) == 1
+    assert available[0].base_url == "https://api.z.ai/api/coding/paas/v4"
+    assert available[0].last_status is None
+    assert available[0].last_error_code is None
+
+
+def test_zai_expired_entry_reprobes_endpoint_before_reuse(tmp_path, monkeypatch):
+    """After Z.AI cooldown, re-probe before sending traffic to a stale URL."""
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
+    monkeypatch.setenv("ZAI_API_KEY", "zai-key")
+    _write_auth_store(
+        tmp_path,
+        {
+            "version": 1,
+            "credential_pool": {
+                "zai": [
+                    {
+                        "id": "zai-1",
+                        "label": "ZAI_API_KEY",
+                        "auth_type": "api_key",
+                        "priority": 0,
+                        "source": "env:ZAI_API_KEY",
+                        "access_token": "zai-key",
+                        "base_url": "https://api.z.ai/api/paas/v4",
+                        "last_status": "exhausted",
+                        "last_status_at": time.time() - 3700,
+                        "last_error_code": 429,
+                        "last_error_reason": "1305",
+                    }
+                ]
+            },
+        },
+    )
+
+    from agent import credential_pool as credential_pool_mod
+    from agent.credential_pool import load_pool
+
+    monkeypatch.setattr(
+        credential_pool_mod,
+        "detect_zai_endpoint",
+        lambda api_key, timeout=8.0: {
+            "base_url": "https://api.z.ai/api/coding/paas/v4",
+            "id": "coding-global",
+            "model": "glm-5.1",
+            "label": "Global (Coding Plan)",
+        },
+    )
+
+    pool = load_pool("zai")
+    available = pool._available_entries(clear_expired=True, refresh=False)
+
+    assert len(available) == 1
+    assert available[0].base_url == "https://api.z.ai/api/coding/paas/v4"
+    assert available[0].last_status == "ok"
+
+
+def test_zai_failed_reprobe_keeps_entry_unavailable(tmp_path, monkeypatch):
+    """Do not resurrect an expired Z.AI entry when no endpoint currently works."""
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
+    monkeypatch.setenv("ZAI_API_KEY", "zai-key")
+    _write_auth_store(
+        tmp_path,
+        {
+            "version": 1,
+            "credential_pool": {
+                "zai": [
+                    {
+                        "id": "zai-1",
+                        "label": "ZAI_API_KEY",
+                        "auth_type": "api_key",
+                        "priority": 0,
+                        "source": "env:ZAI_API_KEY",
+                        "access_token": "zai-key",
+                        "base_url": "https://api.z.ai/api/paas/v4",
+                        "last_status": "exhausted",
+                        "last_status_at": time.time() - 3700,
+                        "last_error_code": 429,
+                        "last_error_reason": "1305",
+                    }
+                ]
+            },
+        },
+    )
+
+    from agent import credential_pool as credential_pool_mod
+    from agent.credential_pool import load_pool
+
+    monkeypatch.setattr(
+        credential_pool_mod,
+        "detect_zai_endpoint",
+        lambda api_key, timeout=8.0: None,
+    )
+
+    pool = load_pool("zai")
+    available = pool._available_entries(clear_expired=True, refresh=False)
+
+    assert available == []
+    persisted = json.loads((tmp_path / "hermes" / "auth.json").read_text())
+    entry = persisted["credential_pool"]["zai"][0]
+    assert entry["last_status"] == "exhausted"
+    assert entry["last_error_code"] == 429
+

tests/hermes_cli/test_commands.py

@@ -108,6 +108,17 @@ def test_alias_resolves_to_canonical(self):
         assert resolve_command("set-home").name == "sethome"
         assert resolve_command("reload_mcp").name == "reload-mcp"
         assert resolve_command("tasks").name == "agents"
+        assert resolve_command("usage").name == "costs"
+        assert resolve_command("credits").name == "costs"
+
+    def test_costs_is_canonical_gateway_usage_command(self):
+        costs = resolve_command("costs")
+        assert costs is not None
+        assert costs.name == "costs"
+        assert resolve_command("/costs").name == "costs"
+        assert "costs" in GATEWAY_KNOWN_COMMANDS
+        assert "usage" in GATEWAY_KNOWN_COMMANDS
+        assert "credits" in GATEWAY_KNOWN_COMMANDS
 
     def test_topic_is_gateway_command(self):
         topic = resolve_command("topic")
@@ -249,6 +260,12 @@ def test_excludes_commands_with_required_args(self):
         assert "steer" not in names
         assert "background" in GATEWAY_KNOWN_COMMANDS
 
+    def test_costs_is_telegram_menu_command(self):
+        names = {name for name, _ in telegram_bot_commands()}
+        assert "costs" in names
+        assert "usage" not in names  # aliases are accepted but not menu entries
+        assert "credits" not in names
+
 
 class TestSlackSubcommandMap:
     def test_returns_dict(self):