From 5fe3f72b63ada11fc9cf7878d6561ed4a2901346 Mon Sep 17 00:00:00 2001 From: MichaelRunchangYang Date: Mon, 8 Jun 2026 10:44:30 +0800 Subject: [PATCH] fix(auth): resync copied Codex profile tokens --- agent/credential_pool.py | 96 +++++++---- hermes_cli/auth.py | 139 ++++++++++++++-- tests/agent/test_credential_pool.py | 157 ++++++++++++++++++ .../hermes_cli/test_auth_profile_fallback.py | 31 ++++ 4 files changed, 382 insertions(+), 41 deletions(-) diff --git a/agent/credential_pool.py b/agent/credential_pool.py index e5b473ec525c..8cc7ebf40691 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -575,7 +575,7 @@ def _sync_anthropic_entry_from_credentials_file(self, entry: PooledCredential) - return entry def _sync_codex_entry_from_auth_store(self, entry: PooledCredential) -> PooledCredential: - """Sync a Codex device_code pool entry from auth.json if tokens differ. + """Sync a Codex device-code-backed pool entry if fresher tokens exist. When a Codex OAuth access token expires (or the ChatGPT account hits its 5h/weekly quota), the pool entry gets marked ``STATUS_EXHAUSTED`` @@ -587,40 +587,74 @@ def _sync_codex_entry_from_auth_store(self, entry: PooledCredential) -> PooledCr though fresh credentials are sitting on disk — and every request fails with "no available entries (all exhausted or empty)". - Mirrors the Nous/Anthropic resync paths above. Only applies to - device_code-sourced entries; env/API-key-sourced entries have no - auth.json shadow to sync from. + Mirrors the Nous/Anthropic resync paths above. Singleton ``device_code`` + entries sync from the local auth singleton. Device-code-backed manual + entries also sync from matching shared profile pool entries so copied + profile auth stores do not spend an already-rotated refresh token. + Env/API-key-sourced entries have no auth.json shadow to sync from. """ - if self.provider != "openai-codex" or entry.source != "device_code": + if ( + self.provider != "openai-codex" + or entry.source not in auth_mod.CODEX_DEVICE_CODE_POOL_SOURCES + ): return entry try: - with _auth_store_lock(): - auth_store = _load_auth_store() - state = _load_provider_state(auth_store, "openai-codex") - if not isinstance(state, dict): - return entry - tokens = state.get("tokens") - if not isinstance(tokens, dict): - return entry - store_access = tokens.get("access_token", "") - store_refresh = tokens.get("refresh_token", "") - # Adopt auth.json tokens when either side differs. Codex refresh - # tokens are single-use too, so a fresh refresh_token from - # another process means our entry's pair is consumed/stale. - entry_access = entry.access_token or "" - entry_refresh = entry.refresh_token or "" - if store_access and ( - store_access != entry_access - or (store_refresh and store_refresh != entry_refresh) - ): + if entry.source == "device_code": + with _auth_store_lock(): + auth_store = _load_auth_store() + state = _load_provider_state(auth_store, "openai-codex") + if isinstance(state, dict): + tokens = state.get("tokens") + if isinstance(tokens, dict): + store_access = tokens.get("access_token", "") + store_refresh = tokens.get("refresh_token", "") + # Adopt auth.json tokens when either side differs. Codex refresh + # tokens are single-use too, so a fresh refresh_token from + # another process means our entry's pair is consumed/stale. + entry_access = entry.access_token or "" + entry_refresh = entry.refresh_token or "" + store_time = auth_mod._parse_auth_timestamp(state.get("last_refresh")) + entry_time = auth_mod._parse_auth_timestamp(entry.last_refresh) + if store_access and ( + store_access != entry_access + or (store_refresh and store_refresh != entry_refresh) + ) and not (store_time and entry_time and store_time <= entry_time): + logger.debug( + "Pool entry %s: syncing Codex tokens from auth.json " + "(refreshed by another process)", + entry.id, + ) + field_updates: Dict[str, Any] = { + "access_token": store_access, + "refresh_token": store_refresh or entry.refresh_token, + "last_status": None, + "last_status_at": None, + "last_error_code": None, + "last_error_reason": None, + "last_error_message": None, + "last_error_reset_at": None, + } + if state.get("last_refresh"): + field_updates["last_refresh"] = state["last_refresh"] + updated = replace(entry, **field_updates) + self._replace_entry(entry, updated) + self._persist() + return updated + + shared_entry = auth_mod.find_shared_codex_pool_entry( + entry_id=entry.id, + source=entry.source, + current_refresh_token=entry.refresh_token or "", + current_last_refresh=entry.last_refresh, + ) + if shared_entry: logger.debug( - "Pool entry %s: syncing Codex tokens from auth.json " - "(refreshed by another process)", + "Pool entry %s: syncing Codex tokens from shared profile pool", entry.id, ) field_updates: Dict[str, Any] = { - "access_token": store_access, - "refresh_token": store_refresh or entry.refresh_token, + "access_token": shared_entry["access_token"], + "refresh_token": shared_entry["refresh_token"], "last_status": None, "last_status_at": None, "last_error_code": None, @@ -628,8 +662,8 @@ def _sync_codex_entry_from_auth_store(self, entry: PooledCredential) -> PooledCr "last_error_message": None, "last_error_reset_at": None, } - if state.get("last_refresh"): - field_updates["last_refresh"] = state["last_refresh"] + if shared_entry.get("last_refresh"): + field_updates["last_refresh"] = shared_entry["last_refresh"] updated = replace(entry, **field_updates) self._replace_entry(entry, updated) self._persist() @@ -1261,7 +1295,7 @@ def _available_entries(self, *, clear_expired: bool = False, refresh: bool = Fal # frozen behind last_error_reset_at (can be hours in the # future for ChatGPT weekly windows). if (self.provider == "openai-codex" - and entry.source == "device_code" + and entry.source in auth_mod.CODEX_DEVICE_CODE_POOL_SOURCES and entry.last_status in {STATUS_EXHAUSTED, STATUS_DEAD}): synced = self._sync_codex_entry_from_auth_store(entry) if synced is not entry: diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 021905c3ec05..7c419e7b72ed 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -38,7 +38,7 @@ from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer from pathlib import Path -from typing import Any, Callable, Dict, FrozenSet, List, Optional, Tuple +from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple from urllib.parse import parse_qs, urlencode, urlparse import httpx @@ -96,6 +96,7 @@ CODEX_OAUTH_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann" CODEX_OAUTH_TOKEN_URL = "https://auth.openai.com/oauth/token" CODEX_ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120 +CODEX_DEVICE_CODE_POOL_SOURCES = frozenset({"device_code", "manual:device_code"}) XAI_OAUTH_ISSUER = "https://auth.x.ai" XAI_OAUTH_DISCOVERY_URL = f"{XAI_OAUTH_ISSUER}/.well-known/openid-configuration" XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828" @@ -1241,6 +1242,127 @@ def read_credential_pool(provider_id: Optional[str] = None) -> Dict[str, Any]: return list(global_entries) if isinstance(global_entries, list) else [] +def _shared_auth_store_paths() -> List[Path]: + """Return other Hermes auth stores that may hold fresher rotating OAuth state.""" + try: + current = _auth_file_path().resolve(strict=False) + except Exception: + current = _auth_file_path() + seen: Set[str] = set() + paths: List[Path] = [] + + def _add(path: Path) -> None: + try: + resolved = path.resolve(strict=False) + except Exception: + resolved = path + key = str(resolved) + if key in seen or resolved == current: + return + seen.add(key) + paths.append(path) + + global_path = _global_auth_file_path() + if global_path is not None: + _add(global_path) + + try: + from hermes_constants import get_default_hermes_root + _add(get_default_hermes_root() / "auth.json") + except Exception: + pass + + profiles_root = Path.home() / ".hermes" / "profiles" + try: + if profiles_root.is_dir(): + for child in sorted(profiles_root.iterdir()): + if child.is_dir(): + _add(child / "auth.json") + except Exception: + pass + + return paths + + +def _parse_auth_timestamp(value: Any) -> float: + if not isinstance(value, str) or not value.strip(): + return 0.0 + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + return datetime.fromisoformat(text).timestamp() + except Exception: + return 0.0 + + +def find_shared_codex_pool_entry( + *, + entry_id: str, + source: str, + current_refresh_token: str = "", + current_last_refresh: Any = None, +) -> Optional[Dict[str, Any]]: + """Find a matching Codex pool entry with a fresher rotating refresh token. + + Codex device-code refresh tokens are single-use. When users clone profiles + or copy auth.json as a recovery step, multiple profile stores can hold the + same pool entry id/source. If one profile refreshes first, siblings must + adopt that rotated pair before spending their stale copied refresh token. + """ + entry_id = str(entry_id or "").strip() + source = str(source or "").strip() + current_refresh_token = str(current_refresh_token or "").strip() + if not entry_id or source not in CODEX_DEVICE_CODE_POOL_SOURCES: + return None + + current_time = _parse_auth_timestamp(current_last_refresh) + candidates: List[Tuple[float, float, Dict[str, Any]]] = [] + for path in _shared_auth_store_paths(): + if not path.exists(): + continue + try: + auth_store = _load_auth_store(path) + except Exception: + continue + pool = auth_store.get("credential_pool") + if not isinstance(pool, dict): + continue + entries = pool.get("openai-codex") + if not isinstance(entries, list): + continue + for raw_entry in entries: + if not isinstance(raw_entry, dict): + continue + if str(raw_entry.get("id") or "").strip() != entry_id: + continue + if str(raw_entry.get("source") or "").strip() != source: + continue + access_token = str(raw_entry.get("access_token") or "").strip() + refresh_token = str(raw_entry.get("refresh_token") or "").strip() + if not access_token or not refresh_token: + continue + if current_refresh_token and refresh_token == current_refresh_token: + continue + candidate_time = _parse_auth_timestamp(raw_entry.get("last_refresh")) + if current_time and candidate_time and candidate_time <= current_time: + continue + try: + mtime = path.stat().st_mtime + except Exception: + mtime = 0.0 + candidates.append(( + candidate_time, + mtime, + dict(raw_entry), + )) + + if not candidates: + return None + candidates.sort(key=lambda item: (item[0], item[1]), reverse=True) + return candidates[0][2] + + def write_credential_pool(provider_id: str, entries: List[Dict[str, Any]]) -> Path: """Persist one provider's credential pool under auth.json. @@ -3684,19 +3806,16 @@ def _pool_codex_access_token() -> str: """Return the most-recent usable access_token from the openai-codex pool. Used as a fallback by ``resolve_codex_runtime_credentials`` when the - singleton has no creds. Reads ``credential_pool.openai-codex`` entries - directly from auth.json and picks the first non-empty access_token, - preferring entries that are not currently in an exhaustion cooldown. + singleton has no creds. Reads through ``read_credential_pool`` instead of + directly from the active profile store so named profiles with an empty + local Codex pool still inherit the global-root credential pool. Picks the + first non-empty access_token, preferring entries that are not currently in + an exhaustion cooldown. Returns ``""`` when no usable entry is found (caller handles by raising the original AuthError). """ try: - with _auth_store_lock(): - auth_store = _load_auth_store() - pool = auth_store.get("credential_pool") - if not isinstance(pool, dict): - return "" - entries = pool.get("openai-codex") + entries = read_credential_pool("openai-codex") if not isinstance(entries, list): return "" diff --git a/tests/agent/test_credential_pool.py b/tests/agent/test_credential_pool.py index 22a4de6d5071..bd2d706da289 100644 --- a/tests/agent/test_credential_pool.py +++ b/tests/agent/test_credential_pool.py @@ -2721,6 +2721,163 @@ def test_codex_exhausted_entry_stays_stuck_without_auth_store_update(tmp_path, m assert available == [] +def test_codex_manual_device_code_adopts_sibling_profile_refresh_before_spending_token( + tmp_path, monkeypatch +): + """Copied profile Codex pools must resync before spending stale refresh tokens. + + Operators commonly clone profiles or copy auth.json to recover a named + gateway. That duplicates ``manual:device_code`` entries across profile + auth stores. Codex refresh tokens rotate on every refresh, so once one + profile refreshes, sibling profiles holding the copied refresh token must + adopt the newer sibling entry before calling the token endpoint. + """ + home = tmp_path + global_root = home / ".hermes" + profile_a = global_root / "profiles" / "alpha" + profile_b = global_root / "profiles" / "beta" + profile_a.mkdir(parents=True) + profile_b.mkdir(parents=True) + monkeypatch.setattr("pathlib.Path.home", lambda: home) + monkeypatch.setenv("HERMES_HOME", str(profile_b)) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("CODEX_OAUTH_ACCESS_TOKEN", raising=False) + + entry_id = "copied" + source = "manual:device_code" + profile_b_store = { + "version": 1, + "credential_pool": { + "openai-codex": [{ + "id": entry_id, + "label": "copied-codex", + "source": source, + "auth_type": "oauth", + "access_token": "stale-access", + "refresh_token": "stale-refresh", + "last_refresh": "2026-06-01T00:00:00Z", + }], + }, + } + profile_a_store = { + "version": 1, + "credential_pool": { + "openai-codex": [{ + "id": entry_id, + "label": "copied-codex", + "source": source, + "auth_type": "oauth", + "access_token": "fresh-access", + "refresh_token": "fresh-refresh", + "last_refresh": "2026-06-02T00:00:00Z", + }], + }, + } + (profile_b / "auth.json").write_text(json.dumps(profile_b_store, indent=2)) + (profile_a / "auth.json").write_text(json.dumps(profile_a_store, indent=2)) + + from agent.credential_pool import load_pool + import hermes_cli.auth as auth_mod + + refresh_args = [] + + def _fake_refresh(access_token, refresh_token, **_kwargs): + refresh_args.append((access_token, refresh_token)) + assert refresh_token != "stale-refresh" + return { + "access_token": "rotated-access", + "refresh_token": "rotated-refresh", + "last_refresh": "2026-06-03T00:00:00Z", + } + + monkeypatch.setattr(auth_mod, "refresh_codex_oauth_pure", _fake_refresh) + + pool = load_pool("openai-codex") + selected = pool.select() + assert selected is not None + + refreshed = pool.try_refresh_current() + + assert refreshed is not None + assert refresh_args == [("fresh-access", "fresh-refresh")] + assert refreshed.access_token == "rotated-access" + assert refreshed.refresh_token == "rotated-refresh" + + persisted = json.loads((profile_b / "auth.json").read_text()) + persisted_entry = persisted["credential_pool"]["openai-codex"][0] + assert persisted_entry["access_token"] == "rotated-access" + assert persisted_entry["refresh_token"] == "rotated-refresh" + + +def test_codex_manual_device_code_ignores_older_sibling_profile_refresh( + tmp_path, monkeypatch +): + """A matching copied profile entry must not roll the current profile backward.""" + home = tmp_path + global_root = home / ".hermes" + profile_a = global_root / "profiles" / "alpha" + profile_b = global_root / "profiles" / "beta" + profile_a.mkdir(parents=True) + profile_b.mkdir(parents=True) + monkeypatch.setattr("pathlib.Path.home", lambda: home) + monkeypatch.setenv("HERMES_HOME", str(profile_b)) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("CODEX_OAUTH_ACCESS_TOKEN", raising=False) + + entry_id = "copied" + source = "manual:device_code" + (profile_b / "auth.json").write_text(json.dumps({ + "version": 1, + "credential_pool": { + "openai-codex": [{ + "id": entry_id, + "label": "copied-codex", + "source": source, + "auth_type": "oauth", + "access_token": "current-access", + "refresh_token": "current-refresh", + "last_refresh": "2026-06-03T00:00:00Z", + }], + }, + }, indent=2)) + (profile_a / "auth.json").write_text(json.dumps({ + "version": 1, + "credential_pool": { + "openai-codex": [{ + "id": entry_id, + "label": "copied-codex", + "source": source, + "auth_type": "oauth", + "access_token": "older-access", + "refresh_token": "older-refresh", + "last_refresh": "2026-06-02T00:00:00Z", + }], + }, + }, indent=2)) + + from agent.credential_pool import load_pool + import hermes_cli.auth as auth_mod + + refresh_args = [] + + def _fake_refresh(access_token, refresh_token, **_kwargs): + refresh_args.append((access_token, refresh_token)) + return { + "access_token": "rotated-access", + "refresh_token": "rotated-refresh", + "last_refresh": "2026-06-04T00:00:00Z", + } + + monkeypatch.setattr(auth_mod, "refresh_codex_oauth_pure", _fake_refresh) + + pool = load_pool("openai-codex") + assert pool.select() is not None + refreshed = pool.try_refresh_current() + + assert refreshed is not None + assert refresh_args == [("current-access", "current-refresh")] + + # --------------------------------------------------------------------------- # xAI OAuth terminal error quarantine # --------------------------------------------------------------------------- diff --git a/tests/hermes_cli/test_auth_profile_fallback.py b/tests/hermes_cli/test_auth_profile_fallback.py index 5210404c40ea..84259ac60765 100644 --- a/tests/hermes_cli/test_auth_profile_fallback.py +++ b/tests/hermes_cli/test_auth_profile_fallback.py @@ -367,6 +367,37 @@ def test_load_provider_state_malformed_global_does_not_break_profile(profile_env assert state["access_token"] == "profile-token" +def test_codex_runtime_uses_global_pool_when_profile_singleton_is_empty(profile_env): + """Stale empty profile Codex state must not block the global credential pool.""" + from hermes_cli.auth import resolve_codex_runtime_credentials + + _write(profile_env["global"] / "auth.json", _make_auth_store(pool={ + "openai-codex": [{ + "id": "glob-codex", + "label": "global-codex", + "auth_type": "oauth", + "priority": 0, + "source": "manual:device_code", + "access_token": "global-codex-access-token", + "refresh_token": "global-codex-refresh-token", + }], + })) + _write(profile_env["profile"] / "auth.json", _make_auth_store( + providers={ + "openai-codex": { + "auth_mode": "chatgpt", + "tokens": {"access_token": "", "refresh_token": ""}, + }, + }, + pool={"openai-codex": []}, + )) + + creds = resolve_codex_runtime_credentials(refresh_if_expiring=False) + + assert creds["source"] == "credential_pool" + assert creds["api_key"] == "global-codex-access-token" + + # --------------------------------------------------------------------------- # Classic mode — no fallback path should ever trigger # ---------------------------------------------------------------------------