diff --git a/agent/agent_init.py b/agent/agent_init.py index bf90925a061e..5b96995e1093 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1082,6 +1082,11 @@ def init_agent( # Bedrock + Claude β†’ use AnthropicBedrock SDK for full feature parity # (prompt caching, thinking budgets, adaptive thinking). _is_bedrock_anthropic = agent.provider == "bedrock" + # Vertex + Claude β†’ use AnthropicVertex SDK. The SDK holds the + # google-auth Credentials object and refreshes the OAuth2 token + # itself, so the client survives long-lived sessions without a + # per-turn refresh hook. + _is_vertex_anthropic = agent.provider == "vertex" if _is_bedrock_anthropic: from agent.anthropic_adapter import build_anthropic_bedrock_client _region_match = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "") @@ -1096,6 +1101,36 @@ def init_agent( agent._client_kwargs = {} if not agent.quiet_mode: print(f"πŸ€– AI Agent initialized with model: {agent.model} (AWS Bedrock + AnthropicBedrock SDK, {_br_region})") + elif _is_vertex_anthropic: + from agent.anthropic_adapter import build_anthropic_vertex_client + from agent.vertex_adapter import get_vertex_anthropic_config + + # Cached resolve (runtime_provider already built + validated the + # Credentials object moments ago); this is a cheap cache read. + _vx_creds, _vx_project, _vx_region = get_vertex_anthropic_config() + if not _vx_project: + raise RuntimeError( + "Claude-on-Vertex selected but Vertex credentials could " + "not be resolved. Provide a service-account JSON via " + "GOOGLE_APPLICATION_CREDENTIALS / VERTEX_CREDENTIALS_PATH, " + "or run 'gcloud auth application-default login', and set " + "the GCP project/region under vertex: in config.yaml. " + "Install with: pip install 'hermes-agent[vertex]'." + ) + agent._vertex_project_id = _vx_project + agent._vertex_region = _vx_region or "global" + agent._vertex_credentials = _vx_creds + agent._anthropic_client = build_anthropic_vertex_client( + _vx_project, agent._vertex_region, credentials=_vx_creds, + ) + agent._anthropic_api_key = "vertex-oauth" + agent._anthropic_base_url = base_url + agent._is_anthropic_oauth = False + agent.api_key = "vertex-oauth" + agent.client = None + agent._client_kwargs = {} + if not agent.quiet_mode: + print(f"πŸ€– AI Agent initialized with model: {agent.model} (Google Vertex AI + AnthropicVertex SDK, project={_vx_project}, {agent._vertex_region})") else: # Only fall back to ANTHROPIC_TOKEN when the provider is actually Anthropic. # Other anthropic_messages providers (MiniMax, Alibaba, etc.) must use their own API key. diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index e0c51fb35388..dce15a85b7a8 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1351,12 +1351,19 @@ def try_recover_primary_transport( agent.api_key = rt["api_key"] if agent.api_mode == "anthropic_messages": - from agent.anthropic_adapter import build_anthropic_client + # Provider-aware: a Bedrock/Vertex primary must be rebuilt with + # its SDK client (AnthropicBedrock / AnthropicVertex) β€” rebuilding + # with build_anthropic_client() would point the recovered session + # at api.anthropic.com with the "aws-sdk"/"vertex-oauth" + # placeholder key and 401 every subsequent request. + from agent.anthropic_adapter import build_anthropic_client_for_provider agent._anthropic_api_key = rt["anthropic_api_key"] agent._anthropic_base_url = rt["anthropic_base_url"] - agent._anthropic_client = build_anthropic_client( + agent._anthropic_client = build_anthropic_client_for_provider( + agent.provider, rt["anthropic_api_key"], rt["anthropic_base_url"], timeout=get_provider_request_timeout(agent.provider, agent.model), + agent=agent, ) agent._is_anthropic_oauth = rt["is_anthropic_oauth"] agent.client = None @@ -1607,12 +1614,18 @@ def restore_primary_runtime(agent) -> bool: agent.client = build_moa_facade(agent, agent.model) agent._anthropic_client = None elif agent.api_mode == "anthropic_messages": - from agent.anthropic_adapter import build_anthropic_client + # Provider-aware: restoring a Bedrock/Vertex primary must rebuild + # its SDK client (AnthropicBedrock / AnthropicVertex); the plain + # Anthropic client + "aws-sdk"/"vertex-oauth" placeholder key + # would 401 against api.anthropic.com on the next turn. + from agent.anthropic_adapter import build_anthropic_client_for_provider agent._anthropic_api_key = rt["anthropic_api_key"] agent._anthropic_base_url = rt["anthropic_base_url"] - agent._anthropic_client = build_anthropic_client( + agent._anthropic_client = build_anthropic_client_for_provider( + agent.provider, rt["anthropic_api_key"], rt["anthropic_base_url"], timeout=get_provider_request_timeout(agent.provider, agent.model), + agent=agent, ) agent._is_anthropic_oauth = rt["is_anthropic_oauth"] agent.client = None @@ -2681,7 +2694,7 @@ def _restore_snapshot() -> None: agent.client = build_moa_facade(agent, agent.model) elif api_mode == "anthropic_messages": from agent.anthropic_adapter import ( - build_anthropic_client, + build_anthropic_client_for_provider, resolve_anthropic_token, _is_oauth_token, ) @@ -2691,6 +2704,14 @@ def _restore_snapshot() -> None: _is_native_anthropic = new_provider == "anthropic" effective_key = (api_key or agent.api_key or resolve_anthropic_token() or "") if _is_native_anthropic else (api_key or agent.api_key or "") + # SDK-auth providers carry no bearer key: Bedrock signs with + # SigV4, Vertex with a google-auth Credentials object. Pin the + # same placeholder keys agent_init uses so a stale key from the + # previous provider can't leak into logs/snapshots. + _sdk_placeholder_keys = {"bedrock": "aws-sdk", "vertex": "vertex-oauth"} + if new_provider in _sdk_placeholder_keys: + effective_key = _sdk_placeholder_keys[new_provider] + # MiniMax OAuth: swap static string for a per-request callable token # provider so the rebuilt client survives 15-min token expiry. See # the matching block in agent_init.py for the full rationale. @@ -2709,9 +2730,14 @@ def _restore_snapshot() -> None: agent.api_key = effective_key agent._anthropic_api_key = effective_key agent._anthropic_base_url = base_url or getattr(agent, "_anthropic_base_url", None) - agent._anthropic_client = build_anthropic_client( + # Provider-aware: switching to Claude-on-Vertex (or Bedrock) must + # build the SDK client β€” the plain Anthropic client would send the + # placeholder key to api.anthropic.com and 401. + agent._anthropic_client = build_anthropic_client_for_provider( + new_provider, effective_key, agent._anthropic_base_url, timeout=get_provider_request_timeout(agent.provider, agent.model), + agent=agent, ) agent._is_anthropic_oauth = _is_oauth_token(effective_key) if (_is_native_anthropic and isinstance(effective_key, str)) else False agent.client = None diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 68b8ca228dd4..86b19588156b 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -981,6 +981,137 @@ def build_anthropic_bedrock_client(region: str): ) +def build_anthropic_vertex_client( + project_id: Optional[str], + region: str, + credentials=None, + base_url: Optional[str] = None, +): + """Create an AnthropicVertex client for Claude-on-Vertex (Google Cloud). + + Uses the Anthropic SDK's native Vertex adapter, which speaks the + Anthropic Messages protocol over Vertex's rawPredict / streamRawPredict + endpoints. This gives Claude on Google Cloud the same enhanced features + as native Anthropic β€” prompt caching, thinking budgets, adaptive + thinking, fine-grained tool streaming β€” that the OpenAI-compatible + Gemini endpoint cannot express. + + Auth: passes the google-auth ``credentials`` object straight through so + the SDK mints and refreshes short-lived OAuth2 access tokens itself + (see anthropic.lib.vertex._client._ensure_access_token). Long-lived + gateway sessions therefore survive the ~1-hour token lifetime without a + per-turn refresh hook. When ``credentials`` is None the SDK falls back to + Application Default Credentials. + + The 1M-context beta is intentionally NOT attached: Vertex Claude does not + honor the ``context-1m-2025-08-07`` beta the way Bedrock does, and sending + it can trigger a 400 on some model/region combos. Callers that want it can + add it per-request once Google enables it. + """ + _anthropic_sdk = _get_anthropic_sdk() + if _anthropic_sdk is None: + raise ImportError( + "The 'anthropic' package is required for the Vertex provider. " + "Install it with: pip install 'anthropic>=0.39.0'" + ) + if not hasattr(_anthropic_sdk, "AnthropicVertex"): + raise ImportError( + "anthropic.AnthropicVertex not available. " + "Upgrade with: pip install 'anthropic>=0.39.0'" + ) + from httpx import Timeout + + _headers = {"anthropic-beta": ",".join(_COMMON_BETAS)} + # User ADC (authorized_user) requires the quota-project header on every + # aiplatform request β€” without it Vertex returns 403 "requires a quota + # project". Service accounts don't need it but tolerate it. google-auth's + # own transports attach this automatically; the Anthropic SDK uses its + # own httpx client, so we must set it explicitly. + if project_id: + _headers["x-goog-user-project"] = project_id + _kwargs = dict( + region=region, + credentials=credentials, + timeout=Timeout(timeout=900.0, connect=10.0), + # Delegate retry to hermes's outer loop (honors Retry-After); the SDK + # default max_retries=2 ignores it and double-retries. Mirrors the + # Bedrock client (#26293). + max_retries=0, + default_headers=_headers, + ) + # Only pin project_id when we actually have one; otherwise let the SDK + # resolve it from the credentials / ADC (passing None would override that). + if project_id: + _kwargs["project_id"] = project_id + if base_url: + _kwargs["base_url"] = base_url + return _anthropic_sdk.AnthropicVertex(**_kwargs) + + +def build_anthropic_client_for_provider( + provider: Optional[str], + api_key, + base_url: Optional[str], + *, + timeout: Optional[float] = None, + drop_context_1m_beta: bool = False, + agent=None, +): + """Provider-aware Anthropic client construction β€” the single chokepoint + for every path that (re)builds an ``anthropic_messages`` primary client. + + ``api_mode == "anthropic_messages"`` does not imply a direct Anthropic + endpoint: Bedrock needs ``AnthropicBedrock`` (SigV4) and Vertex needs + ``AnthropicVertex`` (self-refreshing OAuth2 Credentials). Recovery, + restore, and provider-switch paths must build through here β€” calling + ``build_anthropic_client()`` directly for those providers silently + points the session at api.anthropic.com with a placeholder key + ("aws-sdk" / "vertex-oauth") and every subsequent request 401s. + + For vertex, credentials are re-resolved (so a Credentials object + refreshed by another path is picked up) with the agent's cached + ``_vertex_*`` attributes as fallback; the caches are refreshed when an + ``agent`` is supplied. For bedrock, the region comes from the agent + cache, then the base_url, then us-east-1 β€” mirroring agent_init. + """ + provider_norm = (provider or "").strip().lower() + + if provider_norm == "bedrock": + import re + + _region = getattr(agent, "_bedrock_region", None) if agent is not None else None + if not _region: + _match = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "") + _region = _match.group(1) if _match else "us-east-1" + if agent is not None: + agent._bedrock_region = _region + return build_anthropic_bedrock_client(_region) + + if provider_norm == "vertex": + from agent.vertex_adapter import get_vertex_anthropic_config + + from agent.vertex_adapter import get_vertex_anthropic_base_url + _creds, _project, _region = get_vertex_anthropic_config() + _base_url = get_vertex_anthropic_base_url() + if agent is not None: + _project = _project or getattr(agent, "_vertex_project_id", None) + _region = _region or getattr(agent, "_vertex_region", None) + _creds = _creds or getattr(agent, "_vertex_credentials", None) + _region = _region or "global" + if agent is not None: + agent._vertex_project_id = _project + agent._vertex_region = _region + agent._vertex_credentials = _creds + return build_anthropic_vertex_client(_project, _region, credentials=_creds, base_url=_base_url) + + return build_anthropic_client( + api_key, + base_url, + timeout=timeout, + drop_context_1m_beta=drop_context_1m_beta, + ) + + def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]: """Read Claude Code OAuth credentials from the macOS Keychain. diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index d1a513c7ec12..ed8aa5b321e7 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -545,6 +545,14 @@ def _extract_url_query_params(url: str): # Module-level flag: only warn once per process about stale OPENAI_BASE_URL. _stale_base_url_warned = False +# Vertex AI provider spellings accepted across config surfaces (mirrors the +# alias tuple in hermes_cli.runtime_provider). Used by the auxiliary 401 +# refresh path β€” Vertex is OAuth2-token-based, so its stale clients must be +# re-minted rather than re-read from env keys. +_VERTEX_PROVIDER_NAMES = frozenset( + {"vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai"} +) + _PROVIDER_ALIASES = { "google": "gemini", "google-gemini": "gemini", @@ -775,6 +783,7 @@ def _compression_threshold_for_model( "gpt-5.4-mini", "gpt-5-mini", "haiku-4.5", + "gemini-3.7-flash", "gemini-3.6-flash", "flash-lite", "-nano", @@ -2103,13 +2112,32 @@ def create(self, **kwargs) -> Any: usage = None if hasattr(response, "usage") and response.usage: - prompt_tokens = getattr(response.usage, "input_tokens", 0) or 0 + input_tokens = getattr(response.usage, "input_tokens", 0) or 0 completion_tokens = getattr(response.usage, "output_tokens", 0) or 0 - total_tokens = getattr(response.usage, "total_tokens", 0) or (prompt_tokens + completion_tokens) + # Anthropic's input_tokens EXCLUDES cache reads/writes; preserve the + # native fields so normalize_usage(api_mode="anthropic_messages") + # sees real values. Without them, every MoA advisor / auxiliary call + # routed through this adapter normalized to all-zero usage (the + # SimpleNamespace only carried the renamed prompt_tokens shape), so + # the entire reference fan-out was invisible to cost tracking β€” the + # exact blind spot moa_loop's advisor accounting exists to close. + cache_read = getattr(response.usage, "cache_read_input_tokens", 0) or 0 + cache_write = getattr(response.usage, "cache_creation_input_tokens", 0) or 0 + # OpenAI convention: prompt_tokens INCLUDES cached tokens (the + # details fields separate them), so OpenAI-shape consumers see the + # true prompt size and normalize_usage's OpenAI branch β€” which + # subtracts the top-level cache fields we also expose β€” still + # recovers the correct uncached input count. + prompt_tokens = input_tokens + cache_read + cache_write + total_tokens = prompt_tokens + completion_tokens usage = SimpleNamespace( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, + input_tokens=input_tokens, + output_tokens=completion_tokens, + cache_read_input_tokens=cache_read, + cache_creation_input_tokens=cache_write, ) choice = SimpleNamespace( @@ -4226,6 +4254,45 @@ def _is_transient_transport_error(exc: Exception) -> bool: return isinstance(status, int) and (status == 408 or 500 <= status < 600) +def _is_transient_concurrency_throttle(exc: Exception) -> bool: + """Detect a *transient* 429/RESOURCE_EXHAUSTED worth a same-provider retry. + + Distinct from ``_is_payment_error`` (which owns quota/billing exhaustion): + some Vertex shared-capacity preview MaaS endpoints (e.g. the + ``deepseek-*-maas`` pool) return RESOURCE_EXHAUSTED "too many concurrent + requests" / "please try again later" on a cold-start burst, then serve + normally within seconds. That is a concurrency blip, not depleted quota β€” + retrying with backoff recovers it, whereas the payment path would drop the + target (and for a pinned auxiliary call like a MoA reference advisor there + is no meaningful provider fallback, so the advisor is silently lost for the + turn). + + Deliberately narrow: fires only on explicit transient "try again" / + "concurrent" language, and NEVER on daily/per-day/quota-exceeded/billing + wording, which ``_is_payment_error`` handles by switching provider. + """ + status = getattr(exc, "status_code", None) or getattr( + getattr(exc, "response", None), "status_code", None + ) + err_lower = str(exc).lower() + if status not in {429, None} and "resource_exhausted" not in err_lower \ + and "resource exhausted" not in err_lower: + return False + # Never treat genuine quota/billing exhaustion as a transient blip β€” that + # is depleted-until-reset capacity, not a cold-start concurrency bounce. + if any(kw in err_lower for kw in ( + "quota exceeded", "quota_exceeded", "daily", "per day", + "tokens per day", "weekly", "billing", "credits", + "insufficient", "out of funds", "payment required", + )): + return False + return any(kw in err_lower for kw in ( + "too many concurrent", "concurrent request", + "please try again later", "try again later", + "please retry", "retry later", + )) + + _DEFAULT_TRANSIENT_RETRIES = 2 # Base for exponential backoff between transient retries (seconds). Overridable # so tests can zero it out and not sleep real wall-clock time. @@ -4443,6 +4510,43 @@ def _evict_cached_clients(provider: str) -> None: _client_cache.pop(key, None) +def _is_vertex_host(base_url: str) -> bool: + """True when *base_url* points at a Vertex AI endpoint. + + Vertex hosts: bare ``aiplatform.googleapis.com`` for the global location, + ``{region}-aiplatform.googleapis.com`` for regional ones. The regional + form is a hyphenated prefix (NOT a subdomain), so ``base_url_host_matches`` + alone would miss it. ``base_url_hostname`` returns ``""`` (never None) and + lowercases, so the endswith check is safe. + """ + host = base_url_hostname(base_url) + return bool(host) and ( + host == "aiplatform.googleapis.com" + or host.endswith("-aiplatform.googleapis.com") + ) + + +def _evict_cached_vertex_clients() -> None: + """Drop every cached client whose base_url is a Vertex endpoint. + + Vertex 401 recovery must evict by HOST, not provider label: an + auto-routed or task-aliased vertex client is cached under a key whose + provider element isn't "vertex", so name-based eviction leaves it holding + the dead frozen token and the retry keeps failing until process restart. + """ + with _client_cache_lock: + stale_keys = [ + key for key, entry in _client_cache.items() + if entry and entry[0] is not None + and _is_vertex_host(str(getattr(entry[0], "base_url", "") or "")) + ] + for key in stale_keys: + client = _client_cache.get(key, (None, None, None))[0] + if client is not None: + _close_cached_client(client) + _client_cache.pop(key, None) + + def _evict_cached_client_instance(target: Any) -> bool: """Drop the cache entry whose stored client is *target*. @@ -4798,6 +4902,25 @@ def _refresh_provider_credentials(provider: str) -> bool: return False _evict_cached_clients(normalized) return True + if normalized in _VERTEX_PROVIDER_NAMES: + # Vertex Gemini/openapi clients carry a frozen OAuth2 bearer token + # baked in at build time (unlike the Claude-on-Vertex path, where + # the AnthropicVertex SDK self-refreshes). After the ~1h token + # lifetime every call 401s (ACCESS_TOKEN_TYPE_UNSUPPORTED) β€” seen + # live as compression/title_generation dying for hours in + # long-lived desktop sessions (Jul 2026). Force a re-mint and + # evict the stale clients so the retry builds against a fresh + # token. + from agent.vertex_adapter import refresh_vertex_credentials + + if not refresh_vertex_credentials(): + return False + _evict_cached_clients(normalized) + # Provider-name eviction misses vertex clients cached under a + # different label ("auto", a task alias, ...). Sweep by base_url + # host so every client holding the dead token is dropped. + _evict_cached_vertex_clients() + return True if normalized == "xai-oauth": # Preference: pool-level refresh (uses refresh_token from pool entry), # then fall back to singleton auth-store resolver. @@ -4816,24 +4939,6 @@ def _refresh_provider_credentials(provider: str) -> bool: return False _evict_cached_clients(normalized) return True - if normalized == "vertex": - # Mirrors run_agent.py's _try_refresh_vertex_client_credentials - # for the main conversation loop. Without this branch, an - # auxiliary Vertex client (vision, title generation, reflection, - # context compression, ...) that 401s on its ~1h token expiry - # falls through to the final `return False` below: the stale - # client is never evicted from _client_cache (whose cache key - # ignores the rotating bearer token), so every subsequent - # auxiliary Vertex call keeps 401ing until process restart. - from agent.vertex_adapter import get_vertex_config - - token, base_url = get_vertex_config() - if not isinstance(token, str) or not token.strip(): - return False - if not isinstance(base_url, str) or not base_url.strip(): - return False - _evict_cached_clients(normalized) - return True except Exception as exc: logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc) return False @@ -4862,6 +4967,8 @@ def _auth_refresh_provider_for_route( return "anthropic" if base_url_host_matches(client_base_url, "inference-api.nousresearch.com"): return "nous" + if _is_vertex_host(client_base_url): + return "vertex" return normalized @@ -6803,12 +6910,22 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", return None, None elif pconfig.auth_type == "vertex": - # Google Vertex AI β€” Gemini via the OpenAI-compatible endpoint with an - # OAuth2 bearer token (NOT a static key). We build a standard OpenAI - # client pointed at the runtime-computed Vertex base_url with a fresh - # token; no custom SDK or message translation needed. + # Google Vertex AI β€” dual path, mirroring the aws_sdk/bedrock branch: + # - Claude models β†’ AnthropicVertex SDK (Anthropic Messages over + # rawPredict; prompt caching + thinking parity). The SDK holds the + # google-auth Credentials OBJECT and self-refreshes tokens, so + # long-lived gateways don't 401 after ~1h. + # - Gemini + partner MaaS β†’ OpenAI-compatible endpoint with a + # short-lived OAuth2 bearer token (NOT a static key). + # Without the Claude arm, a vertex/claude-* slot in auxiliary.* or a + # MoA preset built an OpenAI client against the openapi endpoint and + # 404'd ("Malformed publisher model" / /v1/messages not found). try: - from agent.vertex_adapter import get_vertex_config, has_vertex_credentials + from agent.vertex_adapter import ( + get_vertex_config, + has_vertex_credentials, + is_anthropic_vertex_model, + ) except ImportError: logger.warning("resolve_provider_client: vertex requested but " "google-auth not installed") @@ -6819,17 +6936,55 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", "no GCP credentials found") return None, None + default_model = "google/gemini-3-flash-preview" + final_model = _normalize_resolved_model(model or default_model, provider) or default_model + + if is_anthropic_vertex_model(final_model): + try: + from agent.vertex_adapter import get_vertex_anthropic_config + from agent.anthropic_adapter import build_anthropic_vertex_client + except ImportError as exc: + logger.warning("resolve_provider_client: vertex Claude " + "requested but anthropic SDK unavailable: %s", exc) + return None, None + creds, project_id, region = get_vertex_anthropic_config() + if creds is None or not region: + logger.warning("resolve_provider_client: vertex Claude " + "requested but could not resolve GCP credentials") + return None, None + try: + real_client = build_anthropic_vertex_client( + project_id, region, credentials=creds, + ) + except Exception as exc: + logger.warning("resolve_provider_client: cannot create " + "AnthropicVertex client: %s", exc) + return None, None + # Strip any anthropic/ prefix; Vertex publisher IDs are bare + # (claude-fable-5, claude-sonnet-5, optionally @YYYYMMDD). + _vx_model = final_model + if _vx_model.lower().startswith("anthropic/"): + _vx_model = _vx_model[len("anthropic/"):] + client = AnthropicAuxiliaryClient( + real_client, _vx_model, api_key="vertex-oauth", + base_url="https://aiplatform.googleapis.com/v1", + ) + logger.debug("resolve_provider_client: vertex anthropic (%s, %s)", + _vx_model, region) + return (_to_async_client(client, _vx_model, is_vision=is_vision) if async_mode + else (client, _vx_model)) + token, base_url = get_vertex_config() if not token or not base_url: logger.warning("resolve_provider_client: vertex requested but " "could not mint token / resolve project") return None, None - default_model = "google/gemini-3-flash-preview" - final_model = _normalize_resolved_model(model or default_model, provider) try: - from openai import OpenAI - client = OpenAI(api_key=token, base_url=base_url) + # _create_openai_client (not bare OpenAI) so max_retries defaults + # to 0 β€” Hermes's call_llm owns retry/backoff policy (#54465), and + # the vertex transient-429 handling relies on seeing the error. + client = _create_openai_client(api_key=token, base_url=base_url) except Exception as exc: logger.warning("resolve_provider_client: cannot create Vertex " "client: %s", exc) @@ -9403,7 +9558,8 @@ def _call_llm_impl( task, provider=request_provider, base_url=_base_info) except Exception as transient_err: - if not _is_transient_transport_error(transient_err): + if not (_is_transient_transport_error(transient_err) + or _is_transient_concurrency_throttle(transient_err)): raise # Compression is on the critical preflight path: a user cannot # continue or resume an oversized session until it compacts. A @@ -9450,7 +9606,8 @@ def _call_llm_impl( ), task) except Exception as retry_transient: - if not _is_transient_transport_error(retry_transient): + if not (_is_transient_transport_error(retry_transient) + or _is_transient_concurrency_throttle(retry_transient)): raise _last_transient = retry_transient # Retries exhausted β€” fall through to first_err fallback handling. @@ -10145,7 +10302,8 @@ async def _acreate(_kwargs: Dict[str, Any]) -> Any: task, provider=request_provider, base_url=_client_base) except Exception as transient_err: - if not _is_transient_transport_error(transient_err): + if not (_is_transient_transport_error(transient_err) + or _is_transient_concurrency_throttle(transient_err)): raise # See call_llm(): compression is on the critical preflight path, # so skip the same-provider retry on a full-budget timeout and @@ -10157,20 +10315,35 @@ async def _acreate(_kwargs: Dict[str, Any]) -> Any: transient_err, ) raise - logger.info( - "Auxiliary %s (async): transient transport error; retrying " - "once on the same provider before fallback: %s", - task or "call", transient_err, - ) - return _validate_llm_response( - await _relay_async_completion( - client, - kwargs, - provider=request_provider, - api_mode=resolved_api_mode, - create=_acreate, - ), - task) + _max_transient_retries = _transient_retry_count() + _last_transient = transient_err + import asyncio as _asyncio + for _attempt in range(1, _max_transient_retries + 1): + _backoff = min(_TRANSIENT_RETRY_BACKOFF_BASE * (2.0 ** (_attempt - 1)), 8.0) + logger.info( + "Auxiliary %s (async): transient error (attempt %d/%d); " + "retrying same provider after %.1fs before fallback: %s", + task or "call", _attempt, _max_transient_retries, _backoff, + _last_transient, + ) + await _asyncio.sleep(_backoff) + try: + return _validate_llm_response( + await _relay_async_completion( + client, + kwargs, + provider=request_provider, + api_mode=resolved_api_mode, + create=_acreate, + ), + task) + except Exception as retry_transient: + if not (_is_transient_transport_error(retry_transient) + or _is_transient_concurrency_throttle(retry_transient)): + raise + _last_transient = retry_transient + # Retries exhausted β€” fall through to first_err fallback handling. + raise _last_transient except Exception as first_err: if "temperature" in kwargs and _is_unsupported_temperature_error(first_err): retry_kwargs = dict(kwargs) diff --git a/agent/bedrock_adapter.py b/agent/bedrock_adapter.py index 8d63323fd299..6307499e0857 100644 --- a/agent/bedrock_adapter.py +++ b/agent/bedrock_adapter.py @@ -1425,6 +1425,7 @@ def classify_bedrock_error(error_message: str) -> str: "anthropic.claude-fable-5": 1_000_000, "anthropic.claude-fable": 1_000_000, "anthropic.claude-sonnet-5": 1_000_000, + "anthropic.claude-opus-5": 1_000_000, "anthropic.claude-opus-4-8": 1_000_000, "anthropic.claude-opus-4-7": 1_000_000, "anthropic.claude-opus-4-6": 1_000_000, diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index e998f1a85d01..3016defb9cd9 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -2560,6 +2560,13 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool # the pre-computed pass above landed on the default and the user did # not pin api_mode explicitly. An explicit fb.api_mode (even # "chat_completions") must never be overridden here. + try: + from agent.vertex_adapter import ( + is_anthropic_vertex_model as _fb_is_anthropic_vertex_model, + ) + except Exception: + def _fb_is_anthropic_vertex_model(_m): # noqa: ANN001 + return False fb_base_url = str(fb_client.base_url) _fb_is_azure = agent._is_azure_openai_url(fb_base_url) @@ -2574,6 +2581,18 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool from hermes_cli.providers import nous_api_mode fb_api_mode = nous_api_mode(fb_model) + elif fb_provider in ( + "vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai" + ) and _fb_is_anthropic_vertex_model(fb_model): + # Vertex is a mixed surface: Claude speaks anthropic_messages via + # the AnthropicVertex SDK, Gemini/MaaS speak chat_completions via + # the OpenAI-compat endpoint. The host is aiplatform.googleapis.com + # for both, so the hostname/suffix checks below can't tell them + # apart β€” without this, a Claude-on-Vertex fallback defaults to + # chat_completions, skips the anthropic_messages client build, and + # every request 404s. Mirrors determine_api_mode()'s vertex split + # on the primary path. + fb_api_mode = "anthropic_messages" elif ( fb_base_url.rstrip("/").lower().endswith("/anthropic") or base_url_hostname(fb_base_url) == "api.anthropic.com" @@ -2664,14 +2683,27 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool _fb_timeout = get_provider_request_timeout(fb_provider, fb_model) if fb_api_mode == "anthropic_messages": - # Build native Anthropic client instead of using OpenAI client - from agent.anthropic_adapter import build_anthropic_client, resolve_anthropic_token, _is_oauth_token + # Build native Anthropic client instead of using OpenAI client. + # Provider-aware: a vertex/bedrock fallback target needs its SDK + # client (AnthropicVertex / AnthropicBedrock) β€” the plain client + # would POST {host}/v1/messages and 404, so the fallback dies the + # same way the primary just did. + from agent.anthropic_adapter import ( + build_anthropic_client_for_provider, + resolve_anthropic_token, + _is_oauth_token, + ) effective_key = (fb_client.api_key or resolve_anthropic_token() or "") if fb_provider == "anthropic" else (fb_client.api_key or "") + if fb_provider in ("bedrock", "vertex"): + # SDK-auth placeholder keys, matching agent_init. + effective_key = {"bedrock": "aws-sdk", "vertex": "vertex-oauth"}[fb_provider] agent.api_key = effective_key agent._anthropic_api_key = effective_key agent._anthropic_base_url = fb_base_url - agent._anthropic_client = build_anthropic_client( + agent._anthropic_client = build_anthropic_client_for_provider( + fb_provider, effective_key, agent._anthropic_base_url, timeout=_fb_timeout, + agent=agent, ) agent._is_anthropic_oauth = _is_oauth_token(effective_key) if fb_provider == "anthropic" else False agent.client = None diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 84c5b6834b96..e3e16565b1eb 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -2548,7 +2548,24 @@ def _env_val(key: str) -> str: anthropic_oauth_env = ( _env_val("ANTHROPIC_TOKEN") or _env_val("CLAUDE_CODE_OAUTH_TOKEN") ) - api_key_path_explicit = bool(anthropic_api_key and not anthropic_oauth_env) + # A SUPPRESSED ``env:ANTHROPIC_API_KEY`` means the user ran + # ``hermes auth remove anthropic `` against that source: the key may + # still sit in .env, but it is not an active credential. It therefore + # must NOT count as the "user picked the API-key path" signal. + # + # Without this guard the two seeders disagree and the pool ends up + # EMPTY: ``_seed_from_env`` honours the suppression and skips seeding + # the key, while this function saw the raw value, took the branch + # below, and pruned the ``claude_code`` / ``hermes_pkce`` OAuth entries + # on its way out. The result is a hard "no credentials" auth failure + # even though a valid key is on disk AND a valid OAuth login exists β€” + # silent, because each half looks locally correct. + api_key_suppressed = _is_suppressed(provider, "env:ANTHROPIC_API_KEY") + api_key_path_explicit = bool( + anthropic_api_key + and not anthropic_oauth_env + and not api_key_suppressed + ) if api_key_path_explicit: # Prune any stale autodiscovered OAuth entries that may have been diff --git a/agent/model_metadata.py b/agent/model_metadata.py index c3a0fc4b414a..c8c949e412cd 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -426,6 +426,7 @@ def _warn_context_length_fallback(model: str, base_url: str) -> None: "claude-fable": 1000000, "claude-opus-5": 1000000, "claude-sonnet-5": 1000000, + "claude-opus-5": 1000000, "claude-opus-4-8": 1000000, "claude-opus-4.8": 1000000, "claude-opus-4-7": 1000000, diff --git a/agent/usage_pricing.py b/agent/usage_pricing.py index c955bdc7f048..dfff9162279d 100644 --- a/agent/usage_pricing.py +++ b/agent/usage_pricing.py @@ -195,6 +195,34 @@ class CostResult: pricing_version="openai-gpt-5.6-2026-07", ), # ── Anthropic Claude 4.8 ───────────────────────────────────────────── + # Claude Opus 5 β€” current flagship. $5/$25 per 1M, same base rates as the + # 4.5–4.8 Opus generation (cache read 0.1x input, cache write 1.25x input). + # Served identically on Anthropic direct and Vertex Model Garden. + ( + "anthropic", + "claude-opus-5", + ): PricingEntry( + input_cost_per_million=Decimal("5.00"), + output_cost_per_million=Decimal("25.00"), + cache_read_cost_per_million=Decimal("0.50"), + cache_write_cost_per_million=Decimal("6.25"), + source="official_docs_snapshot", + source_url="https://platform.claude.com/docs/en/about-claude/pricing", + pricing_version="anthropic-pricing-2026-05", + ), + # Claude Fable 5 β€” premium tier, $10/$50 per 1M. + ( + "anthropic", + "claude-fable-5", + ): PricingEntry( + input_cost_per_million=Decimal("10.00"), + output_cost_per_million=Decimal("50.00"), + cache_read_cost_per_million=Decimal("1.00"), + cache_write_cost_per_million=Decimal("12.50"), + source="official_docs_snapshot", + source_url="https://platform.claude.com/docs/en/about-claude/pricing", + pricing_version="anthropic-pricing-2026-05", + ), # Same $5/$25 base pricing as 4.6/4.7. Fast-mode variant is a separate # model ID with 2x premium (vs the 6x premium on older Opus generations). # Source: https://openrouter.ai/anthropic/claude-opus-4.8 @@ -553,7 +581,78 @@ class CostResult: source_url="https://api-docs.deepseek.com/quick_start/pricing", pricing_version="deepseek-pricing-2026-07", ), + # ── xAI Grok on Vertex Model Garden ────────────────────────────────── + # The 4.1-fast tier is the cheap high-throughput SKU; the 4.20/4.3 tier is + # the frontier SKU. Public sources disagree on the Vertex rate for the + # 4.20/4.3 tier ($1.25/$2.50 xAI-direct vs ~$2/$6 quoted for Vertex), so the + # HIGHER figure is recorded here β€” under-reporting spend is the worse error. + # NOTE: Vertex bills Grok reasoning tokens as output, but (unlike Gemini + # 3.x) they do NOT count against max_tokens β€” verified 2026-08-15. + ( + "xai", + "grok-4.1-fast-non-reasoning", + ): PricingEntry( + input_cost_per_million=Decimal("0.20"), + output_cost_per_million=Decimal("0.50"), + source="official_docs_snapshot", + source_url="https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + pricing_version="xai-vertex-2026-08", + ), + ( + "xai", + "grok-4.1-fast-reasoning", + ): PricingEntry( + input_cost_per_million=Decimal("0.20"), + output_cost_per_million=Decimal("0.50"), + source="official_docs_snapshot", + source_url="https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + pricing_version="xai-vertex-2026-08", + ), + ( + "xai", + "grok-4.20-reasoning", + ): PricingEntry( + input_cost_per_million=Decimal("2.00"), + output_cost_per_million=Decimal("6.00"), + source="official_docs_snapshot", + source_url="https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + pricing_version="xai-vertex-2026-08", + ), + ( + "xai", + "grok-4.20-non-reasoning", + ): PricingEntry( + input_cost_per_million=Decimal("2.00"), + output_cost_per_million=Decimal("6.00"), + source="official_docs_snapshot", + source_url="https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + pricing_version="xai-vertex-2026-08", + ), + ( + "xai", + "grok-4.3", + ): PricingEntry( + input_cost_per_million=Decimal("2.00"), + output_cost_per_million=Decimal("6.00"), + source="official_docs_snapshot", + source_url="https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + pricing_version="xai-vertex-2026-08", + ), # Google Gemini + # Gemini 3.7 Flash β€” introductory pricing $0.75/$3.75 per 1M through + # 2026-12-31 (half of 3.6 Flash); standard rates take effect 2027-01-01. + # Cache read follows the family's 0.1x-input convention. + ( + "google", + "gemini-3.7-flash", + ): PricingEntry( + input_cost_per_million=Decimal("0.75"), + output_cost_per_million=Decimal("3.75"), + cache_read_cost_per_million=Decimal("0.075"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/gemini-api/docs/pricing", + pricing_version="google-pricing-2026-08-13-intro", + ), ( "google", "gemini-3.6-flash", @@ -1093,7 +1192,25 @@ def resolve_billing_route( or base_url_host_matches(base_url or "", "aiplatform.googleapis.com") or base_url_host_matches(base_url or "", "generativelanguage.googleapis.com") ): - return BillingRoute(provider="google", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") + # Vertex Model Garden also serves Anthropic Claude (via the + # AnthropicVertex rawPredict path) at the SAME per-token rates as + # Anthropic direct β€” Claude on Vertex is a Marketplace product, so + # list price matches. Route those to provider='anthropic' so they hit + # the existing Claude pricing keys; otherwise every Claude-on-Vertex + # call looks up a ("google", "claude-*") key that does not exist and + # is silently recorded as $0.00 with cost_status='unknown'. + _bare = model.split("/")[-1] + if _bare.startswith("claude"): + return BillingRoute(provider="anthropic", model=_bare, base_url=base_url or "", billing_mode="official_docs_snapshot") + # Vertex Model Garden likewise serves xAI Grok under the "xai/" + # publisher prefix. Routing those to provider='google' looks up a + # ("google", "grok-*") key that does not exist, so every Grok call on + # Vertex is silently recorded as $0.00 / cost_status='unknown' β€” the + # same bug class the Claude branch above fixes. Keep the vendor + # identity so the ("xai", "grok-*") pricing keys match. + if model.lower().startswith("xai/") or _bare.startswith("grok"): + return BillingRoute(provider="xai", model=_bare, base_url=base_url or "", billing_mode="official_docs_snapshot") + return BillingRoute(provider="google", model=_bare, base_url=base_url or "", billing_mode="official_docs_snapshot") if provider_name == "fireworks" or base_url_host_matches(base_url or "", "api.fireworks.ai"): # Fireworks model ids look like accounts/fireworks/models/; # rsplit("/", 1)[-1] yields just which is what the dict keys on. diff --git a/agent/vertex_adapter.py b/agent/vertex_adapter.py index 6e425753f053..bfc29474c3f2 100644 --- a/agent/vertex_adapter.py +++ b/agent/vertex_adapter.py @@ -18,6 +18,7 @@ import logging import os +import threading import time from typing import Optional, Tuple @@ -187,6 +188,63 @@ def get_vertex_credentials(credentials_path: Optional[str] = None) -> Tuple[Opti return None, None +_refresh_lock = threading.Lock() +_last_forced_refresh: dict = {"ts": 0.0, "ok": False} +_FORCED_REFRESH_COOLDOWN_S = 30.0 + + +def refresh_vertex_credentials(credentials_path: Optional[str] = None) -> bool: + """Force a token re-mint, bypassing the cached Credentials object. + + Used by the auxiliary 401-recovery path (:func:`agent.auxiliary_client. + _refresh_provider_credentials`). An auxiliary OpenAI client for the + Gemini/openapi endpoint carries a FROZEN bearer token baked in at + client-build time; once that token passes the ~1h lifetime the endpoint + returns 401 UNAUTHENTICATED (ACCESS_TOKEN_TYPE_UNSUPPORTED) on every + call. Clearing the credentials cache before re-minting guarantees the + next client build gets a genuinely fresh token even when the cached + object's local expiry check disagrees with the server (clock skew, + wedged google-auth Credentials state). + + Thread-safe with a single-flight cooldown: when the token expires, + several auxiliary tasks (compression, title_generation, + background_review) tend to 401 in the same second β€” only the first + caller mints; the rest reuse its outcome for + ``_FORCED_REFRESH_COOLDOWN_S`` seconds. The cooldown also stops a + non-expiry 401 (wrong credential *type*, where re-minting never helps) + from hammering the token endpoint. + + Returns True when a fresh (token, project_id) pair was minted. + """ + with _refresh_lock: + now = time.time() + if now - _last_forced_refresh["ts"] < _FORCED_REFRESH_COOLDOWN_S: + return bool(_last_forced_refresh["ok"]) + # Capture the (possibly wedged) old token so we can detect a no-op + # re-mint β€” same token back means recovery will keep failing. + resolved_path = _resolve_credentials_path(credentials_path) + cache_key = resolved_path or "__adc__" + old_entry = _creds_cache.get(cache_key) + old_token = getattr(old_entry[0], "token", None) if old_entry else None + # Clear ALL cached credentials (not just this key): a task-level + # credentials_path override caches under a different key, and a + # stale sibling entry would hand the dead token to the next client + # build. Objects already handed to the AnthropicVertex SDK keep + # their own reference and self-refresh, so this is safe. + _creds_cache.clear() + token, project_id = get_vertex_credentials(credentials_path) + ok = bool(token and project_id) + if ok and old_token and token == old_token: + logger.warning( + "Vertex forced token refresh returned the SAME access token β€” " + "the credential source may be wedged (revoked/cached upstream); " + "the retry will likely 401 again." + ) + _last_forced_refresh["ts"] = now + _last_forced_refresh["ok"] = ok + return ok + + def build_vertex_base_url(project_id: str, region: str = DEFAULT_REGION) -> str: """Build the OpenAI-compatible base URL for Vertex AI. @@ -226,3 +284,88 @@ def has_vertex_credentials() -> bool: if _resolve_project_override(): return True return False + + +def is_anthropic_vertex_model(model_id: str) -> bool: + """Return True if the model is an Anthropic Claude model on Vertex AI. + + Claude-on-Vertex speaks the Anthropic Messages protocol (rawPredict / + streamRawPredict) β€” NOT the OpenAI-compatible endpoint that Gemini and + partner MaaS models use. It must be routed through the AnthropicVertex + SDK for full feature parity (prompt caching, thinking budgets, + fine-grained tool streaming), so it needs its own detection. + + Matches Vertex Claude model IDs such as: + - ``claude-sonnet-4-5@20250929`` + - ``claude-opus-4-1@20250805`` + - ``claude-3-5-sonnet-v2@20241022`` + and the OpenRouter-style ``anthropic/claude-*`` alias. + """ + lower = (model_id or "").strip().lower() + if lower.startswith("anthropic/"): + lower = lower[len("anthropic/"):] + return lower.startswith("claude") + + +def get_vertex_anthropic_config( + credentials_path: Optional[str] = None, + region: Optional[str] = None, +) -> Tuple[Optional[object], Optional[str], Optional[str]]: + """Resolve (google-auth Credentials, project_id, region) for Claude-on-Vertex. + + Unlike ``get_vertex_config`` β€” which mints a short-lived bearer token for + the OpenAI-compatible Gemini endpoint β€” the Anthropic SDK's + ``AnthropicVertex`` client takes the google-auth Credentials object + directly and refreshes the OAuth2 access token itself on expiry. Handing + over the Credentials object (rather than a frozen token) means long-lived + gateway sessions don't 401 after the ~1-hour token lifetime, with no + per-turn refresh hack needed on Hermes's side. + + Returns ``(None, None, None)`` on any failure (missing google-auth, + unresolvable credentials, no project ID). + """ + if google is None: + logger.warning( + "google-auth package not installed. Cannot use Claude on Vertex AI. " + "Install with: pip install 'hermes-agent[vertex]'." + ) + return None, None, None + + # Reuse get_vertex_credentials so the Credentials object is built, cached, + # and validated (it also refreshes when near expiry). We only need the + # side effect of populating _creds_cache plus the resolved project_id. + token, project_id = get_vertex_credentials(credentials_path) + if not token or not project_id: + return None, None, None + + resolved_path = _resolve_credentials_path(credentials_path) + cache_key = resolved_path or "__adc__" + cached = _creds_cache.get(cache_key) + creds = cached[0] if cached else None + if creds is None: + return None, None, None + + effective_region = _resolve_region(region) + return creds, project_id, effective_region + + +def get_vertex_anthropic_base_url() -> str | None: + """Resolve the base URL for AnthropicVertex (corporate proxies/gateways). + + Resolves from ANTHROPIC_VERTEX_BASE_URL env var, then vertex.anthropic_base_url + config. Returns None if unset (falls back to Google's public endpoint). + """ + import os + env_url = os.environ.get("ANTHROPIC_VERTEX_BASE_URL") + if env_url: + return env_url + try: + from hermes_cli.config import load_config + config = load_config() + if config and isinstance(config, dict): + vertex_conf = config.get("vertex") + if isinstance(vertex_conf, dict): + return vertex_conf.get("anthropic_base_url") + except Exception: + pass + return None diff --git a/contributors/emails/nick.poon@irrigreen.com b/contributors/emails/nick.poon@irrigreen.com new file mode 100644 index 000000000000..1237135f9221 --- /dev/null +++ b/contributors/emails/nick.poon@irrigreen.com @@ -0,0 +1,2 @@ +nickkpoon +# PR #66522 (feat/claude-on-vertex) diff --git a/gateway/run.py b/gateway/run.py index e59af635503a..a0b29f4ff7c3 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2724,6 +2724,50 @@ def _credential_pool_for_provider(provider: Optional[str]): return None +_VERTEX_PROVIDER_ALIASES = frozenset( + {"vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai"} +) + + +def _sanitize_vertex_claude_runtime(model: Optional[str], runtime: dict) -> None: + """Repair a Claude-on-Vertex runtime dict poisoned by stale stored fields. + + Claude-on-Vertex runtime fields are DERIVED, not stored: the wire + protocol is always anthropic_messages (AnthropicVertex SDK, rawPredict) + and auth is a self-refreshing OAuth Credentials object resolved at + client-build time. Session model overrides persisted by older builds β€” + or echoed back by desktop clients from cached UI state β€” carry + ``api_mode: "chat_completions"`` and sometimes a bearer ``api_key`` + minted for the OpenAI-compat endpoint. Honoring those builds an OpenAI + client against the Anthropic base URL and every request 404s (HTML + "Error 404 (Not Found)!!1"). Mutates ``runtime`` in place; no-op for + everything that isn't Claude-on-Vertex. + """ + provider = str(runtime.get("provider") or "").strip().lower() + if provider not in _VERTEX_PROVIDER_ALIASES: + return + try: + from agent.vertex_adapter import is_anthropic_vertex_model + except Exception: + return + if not is_anthropic_vertex_model(str(model or "")): + return + if runtime.get("api_mode") != "anthropic_messages" or ( + runtime.get("api_key") not in (None, "", "vertex-oauth") + ): + logger.info( + "Sanitized stale Claude-on-Vertex override: model=%s " + "api_mode=%s->anthropic_messages", + model, + runtime.get("api_mode"), + ) + runtime["api_mode"] = "anthropic_messages" + # Placeholder key, matching agent_init β€” a stored bearer token here was + # minted for the OpenAI-compat endpoint and must not gate the fast path + # into building a chat_completions client with it. + runtime["api_key"] = "vertex-oauth" + + def _try_resolve_fallback_provider() -> dict | None: """Attempt to resolve credentials from the fallback_model/fallback_providers config.""" from hermes_cli.runtime_provider import resolve_runtime_provider @@ -7601,6 +7645,9 @@ def _resolve_session_agent_runtime( "max_tokens": override.get("max_tokens"), "credential_pool": override.get("credential_pool"), } + # Stored Claude-on-Vertex overrides may carry a stale + # chat_completions api_mode / bearer key (see helper docstring). + _sanitize_vertex_claude_runtime(override_model, override_runtime) if override_runtime.get("api_key"): if override_runtime.get("credential_pool") is None: override_runtime["credential_pool"] = _credential_pool_for_provider( @@ -25614,6 +25661,9 @@ def _apply_session_model_override( val = override.get(key) if val is not None: runtime_kwargs[key] = val + # Stored Claude-on-Vertex overrides may carry a stale chat_completions + # api_mode / bearer key (see _sanitize_vertex_claude_runtime). + _sanitize_vertex_claude_runtime(model, runtime_kwargs) if ( runtime_kwargs.get("api_key") and runtime_kwargs.get("credential_pool") is None diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index c10032c3052e..1cbd6d69e7b1 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -1937,6 +1937,23 @@ def _slot_matches_provider(slot): except Exception: pass + # 2b. Vertex: auth_type "vertex" has no API-key env vars (auth is ADC / + # a service-account JSON path), so check 3 below can never fire for it. + # The non-secret ``vertex:`` config section (project_id written by + # `hermes setup`) or an explicit credentials-path env var is the user's + # explicit opt-in. + if normalized == "vertex": + try: + from hermes_cli.config import load_config + vertex_cfg = load_config().get("vertex") + if isinstance(vertex_cfg, dict) and str(vertex_cfg.get("project_id") or "").strip(): + return True + except Exception: + pass + for env_var in ("VERTEX_CREDENTIALS_PATH", "GOOGLE_APPLICATION_CREDENTIALS"): + if os.getenv(env_var, "").strip(): + return True + # 3. Check provider-specific env vars # Exclude CLAUDE_CODE_OAUTH_TOKEN β€” it's set by Claude Code itself, # not by the user explicitly configuring anthropic in Hermes. diff --git a/hermes_cli/model_normalize.py b/hermes_cli/model_normalize.py index d2dfe132a8a4..832e67b3e5a2 100644 --- a/hermes_cli/model_normalize.py +++ b/hermes_cli/model_normalize.py @@ -553,6 +553,18 @@ def normalize_model_for_provider(model_input: str, target_provider: str) -> str: return bare return _normalize_for_deepseek(bare) + # --- Vertex: mixed model space, two surfaces --- + # Claude rides the AnthropicVertex SDK, which URL-injects + # ``publishers/anthropic/models/`` β€” the id must be BARE + # (``anthropic/claude-fable-5`` would 404). Every other Vertex model + # (Gemini, partner MaaS) goes through the OpenAI-compatible endpoint, + # which REQUIRES the ``publisher/`` prefix. Strip only the anthropic/ + # prefix so both the primary and auxiliary paths agree on the wire ID. + if provider == "vertex": + if name.lower().startswith("anthropic/"): + return name.split("/", 1)[1] + return name + # --- Direct providers: repair matching provider prefixes only --- if provider in _MATCHING_PREFIX_STRIP_PROVIDERS: result = _strip_matching_provider_prefix(name, provider) diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index f424ef3b6b59..3ae69b9dd714 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -2539,7 +2539,13 @@ def _sort_key(m): def _model_flow_vertex(config, current_model=""): - """Google Vertex AI provider: Gemini via the OpenAI-compatible endpoint. + """Google Vertex AI provider: Gemini (OpenAI-compat) or Claude (Anthropic). + + Two model families are reachable on Vertex through one provider: + β€’ Gemini + partner MaaS models β†’ the OpenAI-compatible endpoint. + β€’ Claude (``claude-*@YYYYMMDD``) β†’ the Anthropic Messages protocol via + the AnthropicVertex SDK, with full prompt-caching / thinking parity. + The runtime picks the path automatically from the selected model ID. Auth is OAuth2 β€” short-lived tokens minted from a service-account JSON or Application Default Credentials (ADC). No static API key. The credential @@ -2553,6 +2559,7 @@ def _model_flow_vertex(config, current_model=""): ) from hermes_cli.config import load_config, save_config, get_env_value from hermes_cli.models import _PROVIDER_MODELS + from agent.vertex_adapter import is_anthropic_vertex_model # 1. Credential source detection (fast, no network / no google-auth import). sa_path = ( @@ -2568,6 +2575,12 @@ def _model_flow_vertex(config, current_model=""): print(" β€’ run 'gcloud auth application-default login', or") print(" β€’ set VERTEX_CREDENTIALS_PATH in ~/.hermes/.env to a service account JSON") print() + print(" Vertex serves two model families through this provider:") + print(" β€’ Gemini / partner models β†’ OpenAI-compatible endpoint") + print(" β€’ Claude (claude-*@date) β†’ AnthropicVertex SDK (full prompt caching)") + print(" Claude models must be enabled in the GCP Vertex Model Garden for") + print(" your project + region first, or requests 404.") + print() cfg = load_config() vertex_cfg = cfg.get("vertex") @@ -2585,10 +2598,14 @@ def _model_flow_vertex(config, current_model=""): return project_id = project_input or current_project - # 3. Region (default global β€” required for the Gemini 3.x previews). + # 3. Region. Default global (required for Gemini 3.x previews); Claude is + # typically pinned to a regional endpoint (e.g. us-east5, europe-west1). current_region = str(vertex_cfg.get("region") or "global").strip() or "global" try: - region_input = input(f" Vertex region [{current_region}]: ").strip() + region_input = input( + f" Vertex region [{current_region}] " + f"(Claude models usually need a regional value, e.g. us-east5): " + ).strip() except (KeyboardInterrupt, EOFError): print() return @@ -2599,18 +2616,32 @@ def _model_flow_vertex(config, current_model=""): "google/gemini-3-pro-preview", "google/gemini-3-flash-preview", ] - base_url_preview = ( - "https://aiplatform.googleapis.com/v1beta1/projects//" - f"locations/{region}/endpoints/openapi" - if region == "global" - else f"https://{region}-aiplatform.googleapis.com/v1beta1/projects//" - f"locations/{region}/endpoints/openapi" - ) + + def _vertex_base_url_preview(model_id: str) -> str: + """Endpoint preview differs by protocol: Claude uses the Anthropic + rawPredict path (SDK-managed, shown as /v1); Gemini uses OpenAI-compat.""" + if is_anthropic_vertex_model(model_id): + host = ( + "aiplatform.googleapis.com" + if region == "global" + else f"{region}-aiplatform.googleapis.com" + ) + return f"https://{host}/v1 (Claude via AnthropicVertex SDK)" + host = ( + "aiplatform.googleapis.com" + if region == "global" + else f"{region}-aiplatform.googleapis.com" + ) + return ( + f"https://{host}/v1beta1/projects//" + f"locations/{region}/endpoints/openapi" + ) + selected = _prompt_model_selection( model_list, current_model=current_model, confirm_provider="vertex", - confirm_base_url=base_url_preview, + confirm_base_url=_vertex_base_url_preview(current_model or (model_list[0] if model_list else "")), ) if selected: @@ -2624,7 +2655,7 @@ def _model_flow_vertex(config, current_model=""): model["provider"] = "vertex" # base_url is computed at runtime from project+region; do not pin it. model.pop("base_url", None) - model.pop("api_mode", None) # chat_completions is the profile default + model.pop("api_mode", None) # api_mode is derived per-model at runtime clear_model_endpoint_credentials(model, clear_api_mode=False) vcfg = cfg.get("vertex") @@ -2637,7 +2668,8 @@ def _model_flow_vertex(config, current_model=""): save_config(cfg) deactivate_provider() - print(f" Default model set to: {selected} (via Google Vertex AI, {region})") + _family = "Claude via AnthropicVertex SDK" if is_anthropic_vertex_model(selected) else "Gemini / OpenAI-compat" + print(f" Default model set to: {selected} (via Google Vertex AI, {region} β€” {_family})") else: print(" No change.") diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index 9b88f8465fb3..950ec0882796 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -308,6 +308,7 @@ class ModelIdentity(NamedTuple): "sonnet": ModelIdentity("anthropic", "claude-sonnet"), "opus": ModelIdentity("anthropic", "claude-opus"), "haiku": ModelIdentity("anthropic", "claude-haiku"), + "fable": ModelIdentity("anthropic", "claude-fable"), "claude": ModelIdentity("anthropic", "claude"), # OpenAI diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 2050fb53934c..da4c30a0abeb 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -627,19 +627,32 @@ def _xai_curated_models() -> list[str]: # Google Vertex AI β€” static curated list. Vertex's OpenAI-compatible # endpoint has no /models listing route, so without this entry the # /model picker only ever shows the currently-configured model. - # Model IDs use the "google/" publisher prefix Vertex's openapi - # endpoint expects (see hermes_cli/model_setup_flows.py). - # Entries validated live against a GCP project (global region, - # HTTP 200) as of 2026-07-21 (PR #68767). + # Gemini/MaaS ids use the publisher prefix Vertex's openapi endpoint + # expects (see hermes_cli/model_setup_flows.py); Claude ids are bare + # because they route through the AnthropicVertex SDK (rawPredict). + # Exactly one model per Claude family: resolve_alias() raises + # AmbiguousAliasError rather than guessing, so listing two opus + # generations here would break the bare `/model opus` shorthand. + # Gemini entries validated live against a GCP project (global region, + # HTTP 200) as of 2026-07-21 (PR #68767); gemini-3.7-flash added and + # re-validated live 2026-08-15 (global only β€” us-central1/us-east5 404). "vertex": [ + "claude-opus-5", + "claude-fable-5", + "claude-sonnet-5", "google/gemini-3.1-pro-preview", - "google/gemini-3-pro-preview", + "google/gemini-3.7-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3-flash-preview", - "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-flash-lite", + "deepseek-ai/deepseek-v3.2-maas", + "zai-org/glm-5-maas", + "moonshotai/kimi-k2-thinking-maas", + "xai/grok-4.20-reasoning", + "xai/grok-4.1-fast-reasoning", + "xai/grok-4.1-fast-non-reasoning", ], "novita": [ "moonshotai/kimi-k2.5", @@ -1168,7 +1181,7 @@ class ProviderEntry(NamedTuple): ProviderEntry("copilot-acp", "GitHub Copilot ACP", "GitHub Copilot ACP (Spawns copilot --acp --stdio)"), ProviderEntry("huggingface", "Hugging Face", "Hugging Face Inference Providers"), ProviderEntry("gemini", "Google AI Studio", "Google AI Studio (Native Gemini API)"), - ProviderEntry("vertex", "Google Vertex AI", "Google Vertex AI (Gemini via GCP; OAuth2 service account or ADC, GCP billing/quotas)"), + ProviderEntry("vertex", "Google Vertex AI", "Google Vertex AI (Gemini + Claude via GCP; OAuth2 service account or ADC, GCP billing/quotas)"), ProviderEntry("deepseek", "DeepSeek", "DeepSeek (V3, R1, coder, direct API)"), ProviderEntry("xai", "xAI", "xAI Grok (Direct API)"), ProviderEntry("zai", "Z.AI / GLM", "Z.AI / GLM (Zhipu direct API)"), diff --git a/hermes_cli/providers.py b/hermes_cli/providers.py index 20884cb3aee5..c4f2729d8b0d 100644 --- a/hermes_cli/providers.py +++ b/hermes_cli/providers.py @@ -674,12 +674,14 @@ def determine_api_mode(provider: str, base_url: str = "", model: str = "") -> st Resolution order: 1. Host-mandated mode (special endpoints that only accept one protocol). 2. Nous Portal dual-wire (model-derived; overlay alone is openai_chat). - 3. Known provider β†’ transport β†’ TRANSPORT_TO_API_MODE. - 4. Direct provider checks (bedrock). - 5. Default: 'chat_completions'. - - *model* is optional but required for dual-wire providers (Nous) whose - transport depends on the catalog id, not just the provider/host. + 3. Model-aware provider splits (vertex: Claude vs Gemini/MaaS). + 4. Known provider β†’ transport β†’ TRANSPORT_TO_API_MODE. + 5. Direct provider checks (bedrock). + 6. Default: 'chat_completions'. + + ``model`` is optional but callers that have one in hand should pass it: + some providers serve more than one wire protocol (Nous, Vertex) and the + provider-level transport alone misroutes the exceptions. """ mandated = host_mandated_api_mode(base_url) if mandated is not None: @@ -693,6 +695,21 @@ def determine_api_mode(provider: str, base_url: str = "", model: str = "") -> st if provider_norm in {"nous", "nous-portal", "nousresearch"}: return nous_api_mode(model) + # Vertex is a mixed surface: Claude speaks anthropic_messages through the + # AnthropicVertex SDK (rawPredict), while Gemini and partner MaaS models + # speak chat_completions through the OpenAI-compat endpoint. The + # provider-level transport says chat_completions, which silently points a + # Claude primary at the OpenAI surface (HTTP 404 / "Malformed publisher + # model"). See resolve_runtime_provider's vertex branch for the same split. + # Alias tuple mirrors resolve_runtime_provider's vertex branch. + if model and normalize_provider(provider) in ( + "vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai" + ): + from agent.vertex_adapter import is_anthropic_vertex_model + + if is_anthropic_vertex_model(model): + return "anthropic_messages" + pdef = get_provider(provider) if pdef is not None: return TRANSPORT_TO_API_MODE.get(pdef.transport, "chat_completions") diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index a7da0ae7f1ce..8b7feff15316 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -1799,6 +1799,50 @@ def resolve_runtime_provider( # margin) by get_vertex_config(); mid-session expiry is additionally # recovered on 401 by run_agent._try_refresh_vertex_client_credentials(). if requested_provider in ("vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai"): + _vx_model = str(target_model or _get_model_config().get("default") or "").strip() + + # Claude-on-Vertex speaks the Anthropic Messages protocol (rawPredict / + # streamRawPredict), NOT the OpenAI-compat endpoint used by Gemini and + # partner MaaS models. Route Claude models through the AnthropicVertex + # SDK for full feature parity (prompt caching, thinking budgets, + # fine-grained tool streaming). The SDK holds the google-auth + # Credentials object and refreshes the OAuth2 token itself, so no + # per-turn 401 refresh hack is needed (unlike the Gemini path below). + from agent.vertex_adapter import is_anthropic_vertex_model + + if is_anthropic_vertex_model(_vx_model): + from agent.vertex_adapter import get_vertex_anthropic_config + + creds, project_id, region = get_vertex_anthropic_config() + if not project_id: + raise AuthError( + "Vertex AI credentials could not be resolved for Claude. " + "Vertex uses OAuth2 (not a static API key): provide a " + "service-account JSON via GOOGLE_APPLICATION_CREDENTIALS " + "(or VERTEX_CREDENTIALS_PATH) in ~/.hermes/.env, or run " + "'gcloud auth application-default login' for ADC. Set the " + "GCP project/region under vertex: in config.yaml if they " + "aren't embedded in the credentials. Install the extra " + "with: pip install 'hermes-agent[vertex]'." + ) + host = ( + "aiplatform.googleapis.com" + if region == "global" + else f"{region}-aiplatform.googleapis.com" + ) + return { + "provider": "vertex", + "api_mode": "anthropic_messages", + "base_url": f"https://{host}/v1", + "api_key": "vertex-oauth", + "source": "vertex-oauth", + "region": region, + "vertex_project_id": project_id, + "vertex_anthropic": True, # Signal to use AnthropicVertex client + "vertex_credentials": creds, # google-auth Credentials (self-refresh) + "requested_provider": requested_provider, + } + from agent.vertex_adapter import get_vertex_config token, base_url = get_vertex_config() diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 3c65981cbb06..96512c828b9b 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -95,9 +95,13 @@ def _supports_same_provider_pool_setup(provider: str) -> bool: "gemini-3.6-flash", "gemini-3.1-flash-lite-preview", ], "vertex": [ - "google/gemini-3.1-pro-preview", "google/gemini-3-pro-preview", - "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", + "google/gemini-3.1-pro-preview", + "google/gemini-3.7-flash", + "google/gemini-3-flash-preview", "google/gemini-2.5-pro", "google/gemini-2.5-flash", + "claude-opus-4-1@20250805", "claude-sonnet-5", + "claude-haiku-4-5@20251001", "zai-org/glm-5-maas", + "moonshotai/kimi-k2-thinking-maas", "xai/grok-4.1-fast-reasoning", ], "zai": ["glm-5.2", "glm-5.1", "glm-5", "glm-4.7", "glm-4.5", "glm-4.5-flash"], "kimi-coding": ["kimi-k3", "kimi-k2.6", "kimi-k2.5", "kimi-k2-thinking", "kimi-k2-turbo-preview"], diff --git a/hermes_state.py b/hermes_state.py index d229bfcb1b9d..d2155788c4d2 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -10492,6 +10492,17 @@ def get_conversation_root(self, session_id: str) -> str: return (chain[0] if chain and chain[0] else session_id) def _session_lineage_root_to_tip(self, session_id: str) -> List[str]: + """Walk ``parent_session_id`` links from *session_id* up to the lineage root. + + Stops at (and includes) a /branch child: branches persist a full COPY + of the pre-branch transcript at creation (the branch seed), so + following the parent link past a ``_branched_from`` marker would + concatenate the parent's rows on top of the branch's own copy and + every pre-branch message would render twice in resume/display + projections (the "repeating queries in sessions" bug). Compression + continuations persist NO copy, so they keep walking to their + ancestors as before. + """ if not session_id: return [session_id] @@ -10505,11 +10516,21 @@ def _session_lineage_root_to_tip(self, session_id: str) -> List[str]: seen.add(current) chain.append(current) row = conn.execute( - "SELECT parent_session_id FROM sessions WHERE id = ?", + "SELECT parent_session_id, model_config FROM sessions WHERE id = ?", (current,), ).fetchone() if row is None: break + # A branch is self-contained (it owns a seeded copy of its + # pre-branch history) β€” never cross into the parent lineage. + raw_cfg = row["model_config"] if hasattr(row, "keys") else row[1] + if raw_cfg: + try: + cfg = json.loads(raw_cfg) if isinstance(raw_cfg, str) else raw_cfg + if isinstance(cfg, dict) and cfg.get("_branched_from") is not None: + break + except (TypeError, json.JSONDecodeError): + pass current = row["parent_session_id"] if hasattr(row, "keys") else row[0] return list(reversed(chain)) or [session_id] diff --git a/plugins/model-providers/vertex/__init__.py b/plugins/model-providers/vertex/__init__.py index a63aa2b68c27..c569d97d587f 100644 --- a/plugins/model-providers/vertex/__init__.py +++ b/plugins/model-providers/vertex/__init__.py @@ -69,7 +69,7 @@ def fetch_models( env_vars=(), # OAuth2 via service account / ADC β€” not a static key env var base_url="https://aiplatform.googleapis.com", # real base_url computed at runtime auth_type="vertex", - default_aux_model="google/gemini-3.6-flash", + default_aux_model="google/gemini-3.7-flash", ) register_provider(vertex) diff --git a/run_agent.py b/run_agent.py index 257aab534767..0e0220391c4c 100644 --- a/run_agent.py +++ b/run_agent.py @@ -5474,9 +5474,17 @@ def _create_request_anthropic_client(self, *, reason: str) -> Any: keeps a second concurrent call from sharing one pool's close/abort lifecycle β€” it gets a fresh untracked client instead. - Mirrors ``_rebuild_anthropic_client`` construction (direct + Bedrock, - 1M-beta drop) but returns a fresh/cached client instead of swapping - the shared one. + Mirrors ``_rebuild_anthropic_client`` construction (direct + Bedrock + + Vertex, 1M-beta drop) but returns a fresh/cached client instead of + swapping the shared one. Construction goes through the provider-aware + chokepoint ``build_anthropic_client_for_provider``: building the plain + client for a Vertex Claude session sends the request to + ``{aiplatform_host}/v1/messages`` β€” Google's HTML 404 β€” on the very + first streamed call of a session. The chokepoint dispatches Bedrock to + ``AnthropicBedrock`` and Vertex to ``AnthropicVertex`` itself, so no + ``key[0]`` branch is needed here; the 1M-beta flag is read from the + agent rather than ``key[4]``, which only exists on the "direct" key + shape. """ if self.api_mode == "anthropic_messages": self._try_refresh_anthropic_client_credentials() @@ -5506,17 +5514,16 @@ def _create_request_anthropic_client(self, *, reason: str) -> Any: # thread owns the pool's FDs (same #29507 reasoning as OpenAI). self._close_request_anthropic_client(stale, reason=f"reuse_evict:{reason}") - if key[0] == "bedrock": - from agent.anthropic_adapter import build_anthropic_bedrock_client - client = build_anthropic_bedrock_client(key[1]) - else: - from agent.anthropic_adapter import build_anthropic_client - client = build_anthropic_client( - self._anthropic_api_key, - getattr(self, "_anthropic_base_url", None), - timeout=get_provider_request_timeout(self.provider, self.model), - drop_context_1m_beta=key[4], - ) + from agent.anthropic_adapter import build_anthropic_client_for_provider + + client = build_anthropic_client_for_provider( + getattr(self, "provider", None), + self._anthropic_api_key, + getattr(self, "_anthropic_base_url", None), + timeout=get_provider_request_timeout(self.provider, self.model), + drop_context_1m_beta=bool(getattr(self, "_oauth_1m_beta_disabled", False)), + agent=self, + ) logger.debug( "Anthropic request client created (%s, shared=False) provider=%s model=%s", reason, @@ -6180,10 +6187,15 @@ def _try_refresh_anthropic_client_credentials(self) -> bool: pass try: - self._anthropic_client = build_anthropic_client( + # Provider-aware: vertex/bedrock primaries rebuild their SDK + # client (the refreshed bearer is irrelevant to SDK auth). + from agent.anthropic_adapter import build_anthropic_client_for_provider + self._anthropic_client = build_anthropic_client_for_provider( + getattr(self, "provider", None), new_token, getattr(self, "_anthropic_base_url", None), timeout=get_provider_request_timeout(self.provider, self.model), + agent=self, ) except Exception as exc: logger.warning("Failed to rebuild Anthropic client after credential refresh: %s", exc) @@ -6314,7 +6326,10 @@ def _swap_credential(self, entry) -> None: ) if self.api_mode == "anthropic_messages": - from agent.anthropic_adapter import build_anthropic_client, _is_oauth_token + from agent.anthropic_adapter import ( + build_anthropic_client_for_provider, + _is_oauth_token, + ) try: self._anthropic_client.close() @@ -6323,9 +6338,14 @@ def _swap_credential(self, entry) -> None: self._anthropic_api_key = runtime_key self._anthropic_base_url = runtime_base.rstrip("/") if isinstance(runtime_base, str) else runtime_base - self._anthropic_client = build_anthropic_client( + # Provider-aware: vertex/bedrock never reach here (no credential + # pool), but if they ever do the SDK client is the only valid + # rebuild β€” the plain client 404s on {host}/v1/messages. + self._anthropic_client = build_anthropic_client_for_provider( + getattr(self, "provider", None), runtime_key, self._anthropic_base_url, timeout=get_provider_request_timeout(self.provider, self.model), + agent=self, ) self._is_anthropic_oauth = _is_oauth_token(runtime_key) if self.provider == "anthropic" else False self.api_key = runtime_key @@ -6417,28 +6437,27 @@ def _anthropic_messages_create(self, api_kwargs: dict, *, client: Any = None): def _rebuild_anthropic_client(self) -> None: """Rebuild the Anthropic client after an interrupt or stale call. - Handles both direct Anthropic and Bedrock-hosted Anthropic models - correctly β€” rebuilding with the Bedrock SDK when provider is bedrock, - rather than always falling back to build_anthropic_client() which - requires a direct Anthropic API key. + Delegates to ``build_anthropic_client_for_provider`` β€” the single + provider-aware chokepoint that picks AnthropicBedrock for bedrock, + AnthropicVertex for vertex (re-resolving credentials, with this + agent's cached ``_vertex_*`` attrs as fallback), and the plain + Anthropic client otherwise. Recovery/restore/switch paths in + agent_runtime_helpers build through the same chokepoint. Honors ``self._oauth_1m_beta_disabled`` (set by the reactive recovery path when an OAuth subscription rejects the 1M-context beta) so the rebuilt client carries the reduced beta set. """ - _drop_1m = bool(getattr(self, "_oauth_1m_beta_disabled", False)) - if getattr(self, "provider", None) == "bedrock": - from agent.anthropic_adapter import build_anthropic_bedrock_client - region = getattr(self, "_bedrock_region", "us-east-1") or "us-east-1" - self._anthropic_client = build_anthropic_bedrock_client(region) - else: - from agent.anthropic_adapter import build_anthropic_client - self._anthropic_client = build_anthropic_client( - self._anthropic_api_key, - getattr(self, "_anthropic_base_url", None), - timeout=get_provider_request_timeout(self.provider, self.model), - drop_context_1m_beta=_drop_1m, - ) + from agent.anthropic_adapter import build_anthropic_client_for_provider + + self._anthropic_client = build_anthropic_client_for_provider( + getattr(self, "provider", None), + self._anthropic_api_key, + getattr(self, "_anthropic_base_url", None), + timeout=get_provider_request_timeout(self.provider, self.model), + drop_context_1m_beta=bool(getattr(self, "_oauth_1m_beta_disabled", False)), + agent=self, + ) def _interruptible_api_call(self, api_kwargs: dict): """Forwarder β€” see ``agent.chat_completion_helpers.interruptible_api_call``.""" diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 53cbf795cafe..390d5b4ffbd5 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -2571,26 +2571,46 @@ def test_refresh_provider_credentials_remints_vertex_token_and_evicts_cache(self with ( patch("agent.auxiliary_client._client_cache", {cache_key: (stale_client, "google/gemini-3-flash-preview", None)}), patch( - "agent.vertex_adapter.get_vertex_config", - return_value=("ya29.FRESH", "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"), - ) as mock_get_config, + "agent.vertex_adapter.refresh_vertex_credentials", + return_value=True, + ) as mock_refresh, ): from agent.auxiliary_client import _refresh_provider_credentials assert _refresh_provider_credentials("vertex") is True - mock_get_config.assert_called_once() + mock_refresh.assert_called_once() stale_client.close.assert_called_once() def test_refresh_provider_credentials_vertex_returns_false_when_unminted(self): """No usable token/base_url (e.g. ADC and the service-account file both failed) β€” refresh must report failure, not silently evict and pretend the client is fixed.""" - with patch("agent.vertex_adapter.get_vertex_config", return_value=(None, None)): + with patch("agent.vertex_adapter.refresh_vertex_credentials", return_value=False): from agent.auxiliary_client import _refresh_provider_credentials assert _refresh_provider_credentials("vertex") is False + def test_resolve_provider_client_vertex_builds_client_from_minted_token(self): + """End-to-end: resolve_provider_client("vertex", ...) must reach the + auth_type == "vertex" branch and build a working client, not die at + the PROVIDER_REGISTRY lookup (a plain HERMES_OVERLAYS-only fix would + leave this branch dead code β€” PROVIDER_REGISTRY is what + resolve_provider_client actually gates on).""" + with ( + patch("agent.vertex_adapter.has_vertex_credentials", return_value=True), + patch( + "agent.vertex_adapter.get_vertex_config", + return_value=("ya29.FRESH", "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"), + ), + ): + client, model = resolve_provider_client("vertex", "google/gemini-3-flash-preview") + + assert client is not None + assert model == "google/gemini-3-flash-preview" + assert str(client.base_url).rstrip("/") == ( + "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi" + ) def test_resolve_provider_client_vertex_none_when_no_credentials(self): with patch("agent.vertex_adapter.has_vertex_credentials", return_value=False): diff --git a/tests/agent/test_auxiliary_client_vertex_recovery.py b/tests/agent/test_auxiliary_client_vertex_recovery.py new file mode 100644 index 000000000000..203df6aa2bd0 --- /dev/null +++ b/tests/agent/test_auxiliary_client_vertex_recovery.py @@ -0,0 +1,120 @@ +"""Tests for the auxiliary Vertex 401 recovery path. + +The Gemini/openapi Vertex path bakes a frozen OAuth2 bearer token into the +OpenAI client at build time. After the ~1h token lifetime, every auxiliary +call (compression, title_generation, background_review, ...) fails with +401 UNAUTHENTICATED / ACCESS_TOKEN_TYPE_UNSUPPORTED until the process +restarts. Seen live Jul 2026: hours of context_compressor failures in a +long-lived desktop session. + +Recovery contract: + 1. _auth_refresh_provider_for_route maps aiplatform.googleapis.com hosts + (global AND regional "{region}-aiplatform...") to "vertex". + 2. _refresh_provider_credentials("vertex") force re-mints the token + (via agent.vertex_adapter.refresh_vertex_credentials) and evicts the + stale cached clients so the retry builds against a fresh token. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from agent.auxiliary_client import ( + _auth_refresh_provider_for_route, + _is_vertex_host, + _refresh_provider_credentials, +) + + +# ── Host routing ───────────────────────────────────────────────────────────── + +class TestVertexHostRouting: + def test_global_endpoint_maps_to_vertex(self): + url = ("https://aiplatform.googleapis.com/v1beta1/projects/p/" + "locations/global/endpoints/openapi") + assert _auth_refresh_provider_for_route("auto", url) == "vertex" + + def test_regional_endpoint_maps_to_vertex(self): + url = ("https://us-east4-aiplatform.googleapis.com/v1beta1/projects/p/" + "locations/us-east4/endpoints/openapi") + assert _auth_refresh_provider_for_route("auto", url) == "vertex" + + def test_lookalike_host_does_not_match(self): + # Substring lookalikes must not be treated as Vertex. + url = "https://evilaiplatform.googleapis.com.example/v1" + assert _auth_refresh_provider_for_route("auto", url) != "vertex" + + def test_empty_and_garbage_urls_do_not_crash(self): + for url in ("", None, "not a url", "://"): + assert _is_vertex_host(url or "") is False + assert _auth_refresh_provider_for_route("auto", "") == "auto" + + def test_explicit_vertex_provider_passes_through(self): + assert _auth_refresh_provider_for_route("vertex", "") == "vertex" + + +# ── Host-based cache sweep ─────────────────────────────────────────────────── + +class TestVertexClientCacheSweep: + def test_sweep_evicts_vertex_clients_under_any_label(self): + """Clients cached under 'auto'/task labels (not 'vertex') must still be + evicted β€” they hold the same dead frozen token.""" + import agent.auxiliary_client as ac + + class _FakeClient: + def __init__(self, base_url): + self.base_url = base_url + + vertex_url = ("https://aiplatform.googleapis.com/v1beta1/projects/p/" + "locations/global/endpoints/openapi") + other_url = "https://api.openai.com/v1" + k_auto = ("auto", False, vertex_url, "tok", None, None, False, None, "m") + k_task = ("compression-alias", False, vertex_url, "tok", None, None, False, None, "m") + k_keep = ("openai", False, other_url, "sk", None, None, False, None, "m") + with ac._client_cache_lock: + ac._client_cache[k_auto] = (_FakeClient(vertex_url), "m", None) + ac._client_cache[k_task] = (_FakeClient(vertex_url), "m", None) + ac._client_cache[k_keep] = (_FakeClient(other_url), "m", None) + try: + ac._evict_cached_vertex_clients() + with ac._client_cache_lock: + assert k_auto not in ac._client_cache + assert k_task not in ac._client_cache + assert k_keep in ac._client_cache + finally: + with ac._client_cache_lock: + for k in (k_auto, k_task, k_keep): + ac._client_cache.pop(k, None) + + +# ── Credential refresh branch ──────────────────────────────────────────────── + +class TestVertexCredentialRefresh: + @pytest.mark.parametrize("name", ["vertex", "google-vertex", "vertex-ai", + "gcp-vertex", "vertexai"]) + def test_vertex_spellings_hit_remint_and_evict(self, name): + with patch("agent.vertex_adapter.refresh_vertex_credentials", + return_value=True) as mock_remint, \ + patch("agent.auxiliary_client._evict_cached_clients") as mock_evict, \ + patch("agent.auxiliary_client._evict_cached_vertex_clients") as mock_sweep: + assert _refresh_provider_credentials(name) is True + mock_remint.assert_called_once() + mock_evict.assert_called_once() + mock_sweep.assert_called_once() + + def test_remint_failure_returns_false_without_evicting(self): + with patch("agent.vertex_adapter.refresh_vertex_credentials", + return_value=False), \ + patch("agent.auxiliary_client._evict_cached_clients") as mock_evict, \ + patch("agent.auxiliary_client._evict_cached_vertex_clients") as mock_sweep: + assert _refresh_provider_credentials("vertex") is False + mock_evict.assert_not_called() + mock_sweep.assert_not_called() + + def test_remint_exception_returns_false(self): + """A crash inside the re-mint must be swallowed (best-effort recovery).""" + with patch("agent.vertex_adapter.refresh_vertex_credentials", + side_effect=RuntimeError("boom")): + assert _refresh_provider_credentials("vertex") is False diff --git a/tests/agent/test_vertex_adapter.py b/tests/agent/test_vertex_adapter.py index e0778ef55c40..ebba8deef139 100644 --- a/tests/agent/test_vertex_adapter.py +++ b/tests/agent/test_vertex_adapter.py @@ -159,3 +159,98 @@ def test_adc_refuses_foreign_profile_google_application_credentials( +# --------------------------------------------------------------------------- +# Claude-on-Vertex (Anthropic Messages protocol) β€” added for the +# AnthropicVertex routing path. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize( + "model_id,expected", + [ + ("claude-sonnet-4-5@20250929", True), + ("claude-opus-4-1@20250805", True), + ("claude-3-5-sonnet-v2@20241022", True), + ("anthropic/claude-3-5-haiku@20241022", True), + ("CLAUDE-SONNET-4-5@20250929", True), + ("gemini-2.5-flash", False), + ("gemini-3-pro-preview", False), + ("meta/llama-4-scout", False), + ("", False), + ], +) +def test_is_anthropic_vertex_model(vertex_adapter, model_id, expected): + assert vertex_adapter.is_anthropic_vertex_model(model_id) is expected + + +def test_get_vertex_anthropic_config_returns_credentials_object(vertex_adapter): + """Claude path must hand back the google-auth Credentials object (for the + SDK to self-refresh), plus resolved project_id and region β€” not a frozen + token like the Gemini/OpenAI-compat path.""" + creds, project_id, region = vertex_adapter.get_vertex_anthropic_config() + assert creds is not None + assert hasattr(creds, "refresh") # it's a Credentials object, not a str + assert project_id == "adc-project" + assert region == "global" + + +def test_get_vertex_anthropic_config_honors_region_and_project(vertex_adapter, monkeypatch): + monkeypatch.setattr( + vertex_adapter, "_vertex_config", + lambda: {"project_id": "cfg-proj", "region": "us-east5"}, + ) + creds, project_id, region = vertex_adapter.get_vertex_anthropic_config() + assert creds is not None + assert project_id == "cfg-proj" + assert region == "us-east5" + + +def test_get_vertex_anthropic_config_fails_closed_without_creds(monkeypatch): + """No google-auth installed β†’ (None, None, None), never a partial tuple.""" + monkeypatch.setitem(sys.modules, "google", None) + import agent.vertex_adapter as va + va = importlib.reload(va) + monkeypatch.setattr(va, "_vertex_config", lambda: {}) + assert va.get_vertex_anthropic_config() == (None, None, None) + + +# --------------------------------------------------------------------------- +# Forced token re-mint (auxiliary 401 recovery) β€” refresh_vertex_credentials. +# --------------------------------------------------------------------------- + + +def test_refresh_vertex_credentials_drops_cache_and_remints(vertex_adapter): + """A wedged cached Credentials object (non-expired but rejected server-side) + must be evicted so the re-mint builds a genuinely new token. This is the + ~1h-lifetime 401 (ACCESS_TOKEN_TYPE_UNSUPPORTED) recovery path for the + Gemini/openapi endpoint, where the OpenAI client bakes in a frozen token.""" + import datetime as _dt + + # Prime the cache. + token, project = vertex_adapter.get_vertex_credentials() + assert token == "ya29.FAKE" + cached_creds, _ = vertex_adapter._creds_cache["__adc__"] + # Simulate a wedged credential: looks fresh locally (far-future expiry, + # not expired) but the server rejects its token. Without the forced cache + # drop, get_vertex_credentials() would happily return this same token. + cached_creds.token = "ya29.WEDGED" + cached_creds.expired = False + cached_creds.expiry = _dt.datetime.now() + _dt.timedelta(hours=1) + + assert vertex_adapter.refresh_vertex_credentials() is True + new_creds, _ = vertex_adapter._creds_cache["__adc__"] + assert new_creds is not cached_creds # rebuilt, not reused + new_token, _ = vertex_adapter.get_vertex_credentials() + assert new_token == "ya29.FAKE" # freshly minted + + +def test_refresh_vertex_credentials_false_when_unresolvable(monkeypatch): + """No credentials at all β†’ False, and no exception.""" + for var in ("VERTEX_CREDENTIALS_PATH", "GOOGLE_APPLICATION_CREDENTIALS", + "VERTEX_PROJECT_ID", "VERTEX_REGION"): + monkeypatch.delenv(var, raising=False) + _install_fake_google_auth(monkeypatch, adc_ok=False) + import agent.vertex_adapter as va + va = importlib.reload(va) + va._creds_cache.clear() + monkeypatch.setattr(va, "_vertex_config", lambda: {}) + assert va.refresh_vertex_credentials() is False diff --git a/tests/agent/test_vertex_client_recovery.py b/tests/agent/test_vertex_client_recovery.py new file mode 100644 index 000000000000..43083791ce09 --- /dev/null +++ b/tests/agent/test_vertex_client_recovery.py @@ -0,0 +1,483 @@ +"""Recovery/restore/switch paths must rebuild Claude-on-Vertex primaries +through the provider-aware chokepoint. + +Regression tests for the PR-review finding that transient-transport +recovery and fallback restoration rebuilt every ``anthropic_messages`` +primary with ``build_anthropic_client()`` β€” silently pointing a Vertex +(or Bedrock) Claude session at api.anthropic.com with the placeholder +key, where every subsequent request 401s. All client construction is +monkeypatched at the chokepoint; no SDKs or network involved. +""" + +from __future__ import annotations + +import threading +from unittest.mock import MagicMock + +import pytest + + +# ── the chokepoint itself ──────────────────────────────────────────────────── + + +class TestBuildAnthropicClientForProvider: + def test_vertex_dispatches_to_anthropic_vertex(self, monkeypatch): + import agent.anthropic_adapter as aa + import agent.vertex_adapter as va + + creds = object() + monkeypatch.setattr( + va, "get_vertex_anthropic_config", lambda *a, **k: (creds, "proj-1", "global") + ) + built = {} + monkeypatch.setattr( + aa, "build_anthropic_vertex_client", + lambda project, region, credentials=None, **kwargs: built.update( + project=project, region=region, credentials=credentials + ) or "VERTEX_CLIENT", + ) + client = aa.build_anthropic_client_for_provider("vertex", "vertex-oauth", None) + assert client == "VERTEX_CLIENT" + assert built == {"project": "proj-1", "region": "global", "credentials": creds} + + def test_vertex_falls_back_to_agent_cached_attrs(self, monkeypatch): + import agent.anthropic_adapter as aa + import agent.vertex_adapter as va + + monkeypatch.setattr( + va, "get_vertex_anthropic_config", lambda *a, **k: (None, None, None) + ) + built = {} + monkeypatch.setattr( + aa, "build_anthropic_vertex_client", + lambda project, region, credentials=None, **kwargs: built.update( + project=project, region=region, credentials=credentials + ) or "VERTEX_CLIENT", + ) + cached_creds = object() + agent_stub = MagicMock() + agent_stub._vertex_project_id = "cached-proj" + agent_stub._vertex_region = "us-east5" + agent_stub._vertex_credentials = cached_creds + + client = aa.build_anthropic_client_for_provider( + "vertex", "vertex-oauth", None, agent=agent_stub + ) + assert client == "VERTEX_CLIENT" + assert built == { + "project": "cached-proj", "region": "us-east5", "credentials": cached_creds, + } + + def test_vertex_refreshes_agent_caches(self, monkeypatch): + import agent.anthropic_adapter as aa + import agent.vertex_adapter as va + + creds = object() + monkeypatch.setattr( + va, "get_vertex_anthropic_config", lambda *a, **k: (creds, "proj-2", "europe-west1") + ) + monkeypatch.setattr( + aa, "build_anthropic_vertex_client", lambda *a, **k: "VERTEX_CLIENT" + ) + agent_stub = MagicMock() + aa.build_anthropic_client_for_provider("vertex", "vertex-oauth", None, agent=agent_stub) + assert agent_stub._vertex_project_id == "proj-2" + assert agent_stub._vertex_region == "europe-west1" + assert agent_stub._vertex_credentials is creds + + def test_bedrock_dispatches_with_agent_region(self, monkeypatch): + import agent.anthropic_adapter as aa + + monkeypatch.setattr( + aa, "build_anthropic_bedrock_client", lambda region: f"BEDROCK:{region}" + ) + agent_stub = MagicMock() + agent_stub._bedrock_region = "eu-west-3" + client = aa.build_anthropic_client_for_provider( + "bedrock", "aws-sdk", None, agent=agent_stub + ) + assert client == "BEDROCK:eu-west-3" + + def test_bedrock_parses_region_from_base_url(self, monkeypatch): + import agent.anthropic_adapter as aa + + monkeypatch.setattr( + aa, "build_anthropic_bedrock_client", lambda region: f"BEDROCK:{region}" + ) + client = aa.build_anthropic_client_for_provider( + "bedrock", "aws-sdk", "https://bedrock-runtime.ap-southeast-2.amazonaws.com" + ) + assert client == "BEDROCK:ap-southeast-2" + + def test_other_providers_use_plain_client(self, monkeypatch): + import agent.anthropic_adapter as aa + + seen = {} + monkeypatch.setattr( + aa, "build_anthropic_client", + lambda api_key, base_url, timeout=None, drop_context_1m_beta=False: seen.update( + api_key=api_key, base_url=base_url, timeout=timeout, drop=drop_context_1m_beta + ) or "PLAIN_CLIENT", + ) + client = aa.build_anthropic_client_for_provider( + "anthropic", "sk-ant-x", "https://api.anthropic.com", + timeout=42.0, drop_context_1m_beta=True, + ) + assert client == "PLAIN_CLIENT" + assert seen == { + "api_key": "sk-ant-x", "base_url": "https://api.anthropic.com", + "timeout": 42.0, "drop": True, + } + + +# ── recovery / restore / switch paths route through the chokepoint ────────── + + +def _patch_chokepoint(monkeypatch): + """Record chokepoint invocations; return the recorded-calls list.""" + import agent.anthropic_adapter as aa + + calls = [] + + def _fake(provider, api_key, base_url, *, timeout=None, + drop_context_1m_beta=False, agent=None): + calls.append({"provider": provider, "api_key": api_key}) + return f"CLIENT_FOR:{provider}" + + monkeypatch.setattr(aa, "build_anthropic_client_for_provider", _fake) + return calls + + +class _StubAgent: + """Minimal agent shape for the runtime-helper paths under test.""" + + def __init__(self, provider="vertex"): + self.log_prefix = "" + self.quiet_mode = True + self.model = "claude-fable-5" + self.provider = provider + self.base_url = "https://aiplatform.googleapis.com/v1" + self.api_mode = "anthropic_messages" + self.api_key = "vertex-oauth" + self.client = None + self._client_kwargs = {} + self._transport_cache = {} + self._fallback_activated = False + self._credential_pool = None + self._config_context_length = None + self._anthropic_client = None + self._anthropic_api_key = "vertex-oauth" + self._anthropic_base_url = None + self._is_anthropic_oauth = False + self._use_prompt_caching = False + self._use_native_cache_layout = False + self.context_compressor = MagicMock() + self._primary_runtime = { + "model": self.model, + "provider": provider, + "base_url": self.base_url, + "api_mode": "anthropic_messages", + "api_key": "vertex-oauth", + "client_kwargs": {}, + "anthropic_api_key": "vertex-oauth", + "anthropic_base_url": None, + "is_anthropic_oauth": False, + "use_prompt_caching": False, + "use_native_cache_layout": False, + "compressor_model": "m", + "compressor_context_length": 100000, + "compressor_base_url": "", + "compressor_api_key": "", + "compressor_provider": provider, + } + + def _is_openrouter_url(self): + return False + + def _close_openai_client(self, *a, **k): + pass + + def _create_openai_client(self, *a, **k): + return MagicMock() + + def _vprint(self, *a, **k): + pass + + def _anthropic_prompt_cache_policy(self, **k): + return False, False + + def _ensure_lmstudio_runtime_loaded(self, context_intent=None): + return None + + def _lmstudio_load_was_unverified(self, runtime_context_length=None): + return False + + def _effective_lmstudio_context_length(self, context_intent=None, runtime_context_length=None): + return context_intent + + # -- upstream's per-request Anthropic client single-slot cache ------------ + # ``_create_request_anthropic_client`` gained a warm-client cache upstream + # (key = credentials/base-url/timeout/1M-beta) guarded by the OpenAI-wire + # lock. The stub only exercises the CONSTRUCTION path (does it go through + # the provider chokepoint?), so these mirror the real contract minimally: + # a real re-entrant lock, an empty slot that always misses, and no-op + # close. Without them the stub raises AttributeError before ever reaching + # the chokepoint call the test asserts on. + def _request_anthropic_client_key(self): + return ( + "direct", + self._anthropic_api_key, + getattr(self, "_anthropic_base_url", None), + None, + bool(getattr(self, "_oauth_1m_beta_disabled", False)), + ) + + def _openai_client_lock(self): + if getattr(self, "_stub_client_lock", None) is None: + self._stub_client_lock = threading.RLock() + return self._stub_client_lock + + def _request_anthropic_client_cache_ref(self): + if getattr(self, "_stub_request_anthropic_cache", None) is None: + self._stub_request_anthropic_cache = { + "client": None, "key": None, "poisoned": False, "in_use": False, + } + return self._stub_request_anthropic_cache + + @staticmethod + def _is_openai_client_closed(client): + return False + + def _close_request_anthropic_client(self, client, *, reason: str) -> None: + pass + + +def test_transient_recovery_rebuilds_vertex_via_chokepoint(monkeypatch): + import agent.agent_runtime_helpers as helpers + + calls = _patch_chokepoint(monkeypatch) + monkeypatch.setattr(helpers.time, "sleep", lambda *_: None) + + agent_stub = _StubAgent(provider="vertex") + err = type("APIConnectionError", (Exception,), {})() + ok = helpers.try_recover_primary_transport( + agent_stub, err, retry_count=3, max_retries=3 + ) + assert ok is True + assert agent_stub._anthropic_client == "CLIENT_FOR:vertex" + assert calls and calls[0]["provider"] == "vertex" + + +def test_transient_recovery_keeps_plain_path_for_anthropic(monkeypatch): + import agent.agent_runtime_helpers as helpers + + calls = _patch_chokepoint(monkeypatch) + monkeypatch.setattr(helpers.time, "sleep", lambda *_: None) + + agent_stub = _StubAgent(provider="anthropic") + agent_stub._primary_runtime["anthropic_api_key"] = "sk-ant-y" + err = type("ReadTimeout", (Exception,), {})() + ok = helpers.try_recover_primary_transport( + agent_stub, err, retry_count=3, max_retries=3 + ) + assert ok is True + assert calls and calls[0]["provider"] == "anthropic" + assert calls[0]["api_key"] == "sk-ant-y" + + +def test_restore_primary_runtime_rebuilds_vertex_via_chokepoint(monkeypatch): + import agent.agent_runtime_helpers as helpers + + calls = _patch_chokepoint(monkeypatch) + + agent_stub = _StubAgent(provider="vertex") + agent_stub._fallback_activated = True # restoring FROM a fallback + helpers.restore_primary_runtime(agent_stub) + assert agent_stub._anthropic_client == "CLIENT_FOR:vertex" + assert calls and calls[0]["provider"] == "vertex" + + +def test_switch_model_to_vertex_claude_uses_chokepoint(monkeypatch): + import agent.agent_runtime_helpers as helpers + + calls = _patch_chokepoint(monkeypatch) + + agent_stub = _StubAgent(provider="anthropic") + helpers.switch_model( + agent_stub, "claude-fable-5", "vertex", + base_url="https://aiplatform.googleapis.com/v1", + api_mode="anthropic_messages", + ) + assert agent_stub._anthropic_client == "CLIENT_FOR:vertex" + assert calls and calls[0]["provider"] == "vertex" + # SDK-auth placeholder key pinned (matches agent_init) β€” a stale key from + # the previous provider must not leak into the vertex runtime. + assert agent_stub._anthropic_api_key == "vertex-oauth" + assert agent_stub.api_key == "vertex-oauth" + + +def test_rebuild_anthropic_client_delegates(monkeypatch): + """AIAgent._rebuild_anthropic_client routes through the chokepoint.""" + import run_agent as ra + + calls = _patch_chokepoint(monkeypatch) + + stub = _StubAgent(provider="vertex") + ra.AIAgent._rebuild_anthropic_client(stub) + assert stub._anthropic_client == "CLIENT_FOR:vertex" + assert calls and calls[0]["provider"] == "vertex" + + +# ── model-aware api_mode determination (the desktop-session 404) ───────────── +# +# determine_api_mode("vertex") is provider-level chat_completions; the desktop +# model-set flow persisted that and clobbered the model-aware resolution, +# building an OpenAI client against the Anthropic base_url β†’ HTTP 404 on a +# brand-new session. Claude on vertex must resolve anthropic_messages even +# when only provider+model are known. + +@pytest.mark.parametrize("model,expected", [ + ("claude-fable-5", "anthropic_messages"), + ("anthropic/claude-fable-5", "anthropic_messages"), + ("claude-sonnet-5", "anthropic_messages"), + ("google/gemini-3.1-pro-preview", "chat_completions"), + ("moonshotai/kimi-k2-thinking-maas", "chat_completions"), + ("", "chat_completions"), # no model β†’ provider-level default +]) +def test_determine_api_mode_vertex_is_model_aware(model, expected): + from hermes_cli.providers import determine_api_mode + + assert determine_api_mode("vertex", model=model) == expected + + +def test_determine_api_mode_vertex_alias_is_model_aware(): + from hermes_cli.providers import determine_api_mode + + assert ( + determine_api_mode("google-vertex", model="claude-fable-5") + == "anthropic_messages" + ) + + +def test_switch_model_vertex_claude_autodetects_anthropic_mode(monkeypatch): + """The live failure: switch to vertex claude with api_mode unset must not + fall back to the chat_completions OpenAI client.""" + import agent.agent_runtime_helpers as helpers + + calls = _patch_chokepoint(monkeypatch) + + agent_stub = _StubAgent(provider="anthropic") + helpers.switch_model( + agent_stub, "claude-fable-5", "vertex", + base_url="https://aiplatform.googleapis.com/v1", + api_mode="", # ← unset: must be determined model-aware + ) + assert agent_stub.api_mode == "anthropic_messages" + assert agent_stub._anthropic_client == "CLIENT_FOR:vertex" + assert calls and calls[0]["provider"] == "vertex" + + +def test_request_anthropic_client_routes_vertex(monkeypatch): + """The per-request streaming client (stale/interrupt watchdog ownership + contract) must be provider-aware: this was the live 404 β€” every streamed + request on a vertex Claude session built a plain Anthropic client and + POSTed {aiplatform_host}/v1/messages (Google HTML 404), including the + very first call of a fresh session.""" + import run_agent as ra + + calls = _patch_chokepoint(monkeypatch) + + stub = _StubAgent(provider="vertex") + stub._try_refresh_anthropic_client_credentials = lambda: False + client = ra.AIAgent._create_request_anthropic_client( + stub, reason="chat_completion_stream_request" + ) + assert client == "CLIENT_FOR:vertex" + assert calls and calls[0]["provider"] == "vertex" + + +def test_request_anthropic_client_keeps_plain_for_anthropic(monkeypatch): + import run_agent as ra + + calls = _patch_chokepoint(monkeypatch) + + stub = _StubAgent(provider="anthropic") + stub._anthropic_api_key = "sk-ant-z" + stub._try_refresh_anthropic_client_credentials = lambda: False + client = ra.AIAgent._create_request_anthropic_client(stub, reason="x") + assert client == "CLIENT_FOR:anthropic" + assert calls[0]["api_key"] == "sk-ant-z" + + +# ── fallback activation for Claude-on-Vertex (the second live failure) ──────── +# +# try_activate_fallback has its OWN api_mode detection block, separate from +# determine_api_mode. It had no Claude-on-Vertex case, so a vertex fallback +# model (e.g. fable-5 Overloaded β†’ sonnet-5) defaulted to chat_completions, +# skipped the anthropic_messages client build, and 404'd on {host}/v1/messages. + +class _FallbackStubAgent(_StubAgent): + def __init__(self): + super().__init__(provider="vertex") + self.model = "claude-fable-5" + self._fallback_index = 0 + self._fallback_chain = [{"provider": "vertex", "model": "claude-sonnet-5"}] + self._unavailable_fallback_keys = None + self._rate_limited_until = 0 + self.context_compressor = None # skip the network-y context-length probe + self.reasoning_config = None + self._pending_fallback_notice = None + + def _try_activate_fallback(self, reason=None): # recursion guard: should not fire + raise AssertionError("unexpected chain recursion β€” happy path should not skip") + + def _is_azure_openai_url(self, _u): + return False + + def _is_direct_openai_url(self, _u): + return False + + def _provider_model_requires_responses_api(self, *a, **k): + return False + + def _buffer_status(self, *a, **k): + pass + + +class _FakeFbClient: + base_url = "https://aiplatform.googleapis.com/v1" + api_key = "vertex-oauth" + + +def test_fallback_to_vertex_claude_builds_anthropic_vertex(monkeypatch): + import agent.chat_completion_helpers as helpers + import agent.auxiliary_client as aux + import agent.anthropic_adapter as aa + + calls = [] + + def _fake_choke(provider, api_key, base_url, *, timeout=None, + drop_context_1m_beta=False, agent=None): + calls.append(provider) + return f"CLIENT_FOR:{provider}" + + monkeypatch.setattr(aux, "resolve_provider_client", + lambda *a, **k: (_FakeFbClient(), "claude-sonnet-5")) + monkeypatch.setattr(helpers, "_fallback_entry_unavailable_without_network", + lambda *a, **k: None) + monkeypatch.setattr(aa, "build_anthropic_client_for_provider", _fake_choke) + monkeypatch.setattr(helpers, "rewrite_prompt_model_identity", lambda *a, **k: None) + monkeypatch.setattr(helpers, "_reset_stale_streak", lambda *a, **k: None) + + agent = _FallbackStubAgent() + ok = helpers.try_activate_fallback(agent, reason=None) + + assert ok is True + assert agent.model == "claude-sonnet-5" + assert agent.provider == "vertex" + # The core assertions: detected as anthropic_messages and built via the + # provider chokepoint (AnthropicVertex), NOT left on chat_completions. + assert agent.api_mode == "anthropic_messages" + assert agent._anthropic_client == "CLIENT_FOR:vertex" + assert agent.client is None + assert calls == ["vertex"] diff --git a/tests/hermes_cli/test_vertex_model_picker.py b/tests/hermes_cli/test_vertex_model_picker.py index ccf17e6b4ed9..655bb932ee95 100644 --- a/tests/hermes_cli/test_vertex_model_picker.py +++ b/tests/hermes_cli/test_vertex_model_picker.py @@ -22,11 +22,20 @@ def test_vertex_has_curated_model_list(): - """Vertex has no /models route β€” the picker needs a static curated list.""" + """Vertex has no /models route β€” the picker needs a static curated list. + + Claude entries are bare publisher IDs (AnthropicVertex SDK path); + Gemini/partner-MaaS entries carry a vendor publisher prefix (e.g. + "google/", "deepseek-ai/") that Vertex's openapi endpoint expects. + """ + from agent.vertex_adapter import is_anthropic_vertex_model + models = _PROVIDER_MODELS.get("vertex") assert models, "_PROVIDER_MODELS must have a non-empty 'vertex' entry" - # Vertex's openapi endpoint expects the google/ publisher prefix. - assert all(m.startswith("google/") for m in models) + assert all( + "/" in m or is_anthropic_vertex_model(m) + for m in models + ) def test_vertex_appears_when_credentials_configured(): diff --git a/tests/hermes_cli/test_vertex_provider.py b/tests/hermes_cli/test_vertex_provider.py index 129c44a8eb3a..85713e007b18 100644 --- a/tests/hermes_cli/test_vertex_provider.py +++ b/tests/hermes_cli/test_vertex_provider.py @@ -55,3 +55,244 @@ def test_vertex_registered_in_hermes_overlays(): resolved = get_provider("vertex") assert resolved is not None assert resolved.auth_type == "vertex" + + +# --------------------------------------------------------------------------- +# Claude-on-Vertex: dual-path routing (Anthropic Messages vs OpenAI-compat). +# --------------------------------------------------------------------------- + +def test_claude_on_vertex_routes_to_anthropic_messages(monkeypatch): + """A Claude model on the vertex provider must route through the + AnthropicVertex SDK path (api_mode=anthropic_messages), carrying the + google-auth Credentials object and project/region β€” NOT a static token.""" + import agent.vertex_adapter as va + from hermes_cli import runtime_provider as rp + + class _Creds: + token = "ya29.TOKEN" + + fake_creds = _Creds() + monkeypatch.setattr( + va, "get_vertex_anthropic_config", + lambda *a, **k: (fake_creds, "my-proj", "us-east5"), + ) + rt = rp.resolve_runtime_provider( + requested="vertex", target_model="claude-sonnet-4-5@20250929", + ) + assert rt["provider"] == "vertex" + assert rt["api_mode"] == "anthropic_messages" + assert rt["vertex_anthropic"] is True + assert rt["vertex_project_id"] == "my-proj" + assert rt["region"] == "us-east5" + assert rt["vertex_credentials"] is fake_creds + # regional base_url shape (Anthropic SDK appends the rawPredict path itself) + assert rt["base_url"] == "https://us-east5-aiplatform.googleapis.com/v1" + + +def test_claude_on_vertex_global_region_base_url(monkeypatch): + import agent.vertex_adapter as va + from hermes_cli import runtime_provider as rp + + monkeypatch.setattr( + va, "get_vertex_anthropic_config", + lambda *a, **k: (object(), "my-proj", "global"), + ) + rt = rp.resolve_runtime_provider( + requested="vertex", target_model="claude-opus-4-1@20250805", + ) + assert rt["base_url"] == "https://aiplatform.googleapis.com/v1" + + +def test_gemini_on_vertex_still_uses_openai_compat(monkeypatch): + """Non-Claude models must keep the OpenAI-compat (chat_completions) path.""" + import agent.vertex_adapter as va + from hermes_cli import runtime_provider as rp + + monkeypatch.setattr( + va, "get_vertex_config", + lambda: ("ya29.TOKEN", "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"), + ) + rt = rp.resolve_runtime_provider( + requested="vertex", target_model="gemini-2.5-flash", + ) + assert rt["api_mode"] == "chat_completions" + assert rt.get("vertex_anthropic") is None + assert rt["api_key"] == "ya29.TOKEN" + + +def test_claude_on_vertex_raises_autherror_when_unresolved(monkeypatch): + import agent.vertex_adapter as va + from hermes_cli import runtime_provider as rp + from hermes_cli.auth import AuthError + + monkeypatch.setattr(va, "get_vertex_anthropic_config", lambda *a, **k: (None, None, None)) + with pytest.raises(AuthError) as exc: + rp.resolve_runtime_provider(requested="vertex", target_model="claude-sonnet-4-5@20250929") + assert "Claude" in str(exc.value) + + +def test_build_anthropic_vertex_client_shape(): + """The AnthropicVertex client must be built with self-refreshing creds, + max_retries=0 (hermes owns retry), and NO 1M-context beta.""" + pytest.importorskip("anthropic") + from unittest.mock import MagicMock + from agent.anthropic_adapter import build_anthropic_vertex_client + + creds = MagicMock() + client = build_anthropic_vertex_client("my-proj", "us-east5", credentials=creds) + assert type(client).__name__ == "AnthropicVertex" + assert client.project_id == "my-proj" + assert client.region == "us-east5" + assert client.max_retries == 0 + beta = client._custom_headers.get("anthropic-beta", "") + assert "context-1m" not in beta + assert "interleaved-thinking-2025-05-14" in beta + + +# ── /model picker visibility (list_authenticated_providers) ───────────────── +# +# Vertex has auth_type "vertex" and env_vars=() β€” no API key to detect β€” so +# without a dedicated credential check (mirroring bedrock's aws_sdk special +# case) the picker omits the provider row entirely whenever vertex isn't the +# configured model.provider. Regression: switching model.provider to `moa` +# made "Google Vertex AI" vanish from the desktop picker despite working ADC. + +def test_picker_lists_vertex_when_credentials_present(monkeypatch): + import agent.vertex_adapter as va + from hermes_cli import model_switch as ms + + monkeypatch.setattr(va, "has_vertex_credentials", lambda: True) + rows = ms.list_authenticated_providers(current_provider="moa") + vertex_rows = [r for r in rows if r.get("slug") == "vertex"] + assert vertex_rows, "vertex row missing from picker despite credentials" + models = vertex_rows[0].get("models") or [] + assert "claude-fable-5" in models + + +def test_picker_hides_vertex_without_credentials(monkeypatch): + import agent.vertex_adapter as va + from hermes_cli import model_switch as ms + + monkeypatch.setattr(va, "has_vertex_credentials", lambda: False) + rows = ms.list_authenticated_providers(current_provider="moa") + assert not [r for r in rows if r.get("slug") == "vertex"] + + +def test_vertex_explicitly_configured_via_config_section(monkeypatch): + """A `vertex:` config section with project_id is the explicit opt-in + signal (vertex has no API key for check 3 to find).""" + import hermes_cli.auth as auth + import hermes_cli.config as config + + monkeypatch.setattr(auth, "_load_auth_store", lambda: {}) + monkeypatch.setattr( + config, "load_config", + lambda *a, **k: {"model": {"provider": "moa"}, "vertex": {"project_id": "my-proj"}}, + ) + monkeypatch.delenv("VERTEX_CREDENTIALS_PATH", raising=False) + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + assert auth.is_provider_explicitly_configured("vertex") is True + + +def test_vertex_not_explicitly_configured_when_unset(monkeypatch): + import hermes_cli.auth as auth + import hermes_cli.config as config + + monkeypatch.setattr(auth, "_load_auth_store", lambda: {}) + monkeypatch.setattr( + config, "load_config", + lambda *a, **k: {"model": {"provider": "moa"}}, + ) + monkeypatch.delenv("VERTEX_CREDENTIALS_PATH", raising=False) + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + assert auth.is_provider_explicitly_configured("vertex") is False + + +# ── model-ID normalization (primary path) ──────────────────────────────────── +# +# The vertex model space is mixed: Claude rides the AnthropicVertex SDK +# (bare publisher IDs β€” the SDK URL-injects publishers/anthropic/models/), +# while Gemini/partner-MaaS ride the OpenAI-compat endpoint (REQUIRES the +# publisher/ prefix). Only the anthropic/ prefix may be stripped, and the +# primary path must agree with the auxiliary path's strip. + +@pytest.mark.parametrize("given,expected", [ + ("anthropic/claude-fable-5", "claude-fable-5"), + ("anthropic/claude-sonnet-5", "claude-sonnet-5"), + ("Anthropic/Claude-Fable-5", "Claude-Fable-5"), # case-insensitive prefix + ("anthropic/claude-sonnet-4-5@20250929", "claude-sonnet-4-5@20250929"), + ("claude-fable-5", "claude-fable-5"), # already bare + ("google/gemini-3.1-pro-preview", "google/gemini-3.1-pro-preview"), + ("moonshotai/kimi-k2-thinking-maas", "moonshotai/kimi-k2-thinking-maas"), + ("deepseek-ai/deepseek-v3.2-maas", "deepseek-ai/deepseek-v3.2-maas"), +]) +def test_vertex_normalization_strips_only_anthropic_prefix(given, expected): + from hermes_cli.model_normalize import normalize_model_for_provider + + assert normalize_model_for_provider(given, "vertex") == expected + + +def test_vertex_normalization_agrees_with_anthropic_detection(): + """Whatever the classifier accepts, normalization must reduce to an ID the + AnthropicVertex SDK can serve β€” the primary/auxiliary disagreement bug.""" + from agent.vertex_adapter import is_anthropic_vertex_model + from hermes_cli.model_normalize import normalize_model_for_provider + + for alias in ("anthropic/claude-fable-5", "claude-fable-5"): + assert is_anthropic_vertex_model(alias) + normalized = normalize_model_for_provider(alias, "vertex") + assert normalized == "claude-fable-5" + assert is_anthropic_vertex_model(normalized) + + +@pytest.mark.parametrize("short_alias", ["sonnet", "fable", "opus"]) +def test_short_claude_aliases_resolve_on_vertex(short_alias): + """Family-specific Claude aliases must resolve to a bare, + is_anthropic_vertex_model-recognized ID on vertex. + + A missing MODEL_ALIASES entry (e.g. "fable" was absent) makes + resolve_alias() return None, so switch_model() falls through and passes + the literal short alias straight to the wire. Vertex's OpenAI-compatible + /openapi endpoint then 400s with "Malformed publisher model (model: + 'fable')" because the bare short name was never routed through the + AnthropicVertex SDK path in the first place. + + Invariant: each alias here names exactly ONE family in the curated + vertex catalog, so resolution must be unambiguous. Listing two + generations of the same family (e.g. opus-5 and opus-4-8) would make + resolve_alias raise AmbiguousAliasError instead β€” see + test_bare_claude_alias_is_ambiguous_on_vertex for the deliberate + multi-family case. + + Note: "haiku" is deliberately excluded β€” Vertex's curated catalog + doesn't currently list a haiku model, so that alias legitimately + fails to resolve there. + """ + from agent.vertex_adapter import is_anthropic_vertex_model + from hermes_cli.model_switch import resolve_alias + + result = resolve_alias(short_alias, "vertex") + assert result is not None, f"'{short_alias}' did not resolve on vertex β€” check MODEL_ALIASES" + provider, resolved_model, _alias_name = result + assert provider == "vertex" + assert is_anthropic_vertex_model(resolved_model) + + +def test_bare_claude_alias_is_ambiguous_on_vertex(): + """The bare "claude" alias spans every Claude family in the curated + vertex catalog, so it must raise rather than silently pick one. + + resolve_alias() deliberately refuses to guess among multiple matches: + version-sort heuristics have repeatedly landed on the wrong model + (dated snapshots outranking point releases, suffix tiebreaks picking + the cheapest tier). Asserting the raise pins that contract so a future + catalog edit can't reintroduce silent selection. + """ + from hermes_cli.model_switch import AmbiguousAliasError, resolve_alias + + with pytest.raises(AmbiguousAliasError) as exc: + resolve_alias("claude", "vertex") + + # Every candidate offered to the user must be a real Claude id. + assert all("claude" in m.lower() for m in exc.value.candidates) + assert len(exc.value.candidates) > 1 diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index f7aca6465dcc..8ad24095d5b6 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -2270,6 +2270,54 @@ def test_delete_session_expected_targets_fail_closed_on_new_delegate(self, db): + def test_branch_lineage_stops_at_branch_boundary(self, db): + """A /branch child's lineage walk must NOT cross into its parent. + + Branches persist a full seeded COPY of the pre-branch transcript at + creation (desktop session.create + _persist_branch_seed, gateway + /branch history copy). If _session_lineage_root_to_tip followed the + parent link past the _branched_from marker, resume/display projections + would concatenate the parent's rows on top of the branch's own copy β€” + every pre-branch message rendered twice (the "repeating queries in + sessions" bug). + """ + db.create_session("parent", "desktop") + db.append_message("parent", "user", "q1") + db.append_message("parent", "assistant", "a1") + db.append_message("parent", "user", "q2") + db.append_message("parent", "assistant", "a2") + + db.create_session( + "branch", + "desktop", + model_config={"_branched_from": "parent"}, + parent_session_id="parent", + ) + # The seeded copy of the pre-branch history… + for role, content in [("user", "q1"), ("assistant", "a1"), ("user", "q2"), ("assistant", "a2")]: + db.append_message("branch", role, content) + # …plus the branch's own first turn. + db.append_message("branch", "user", "q3") + db.append_message("branch", "assistant", "a3") + + # Lineage stops at the branch: it is self-contained. + assert db._session_lineage_root_to_tip("branch") == ["branch"] + assert db.get_ancestor_display_prefix("branch") == [] + + model_history, display_history = db.get_resume_conversations("branch") + assert len(display_history) == 6, display_history + user_texts = [m["content"] for m in display_history if m["role"] == "user"] + assert user_texts == ["q1", "q2", "q3"], user_texts + assert len(model_history) == 6 + + # Compression continuations still walk to their ancestors (no copy). + db.end_session("parent", "compression") + db.create_session("cont", "desktop", parent_session_id="parent") + db.append_message("cont", "user", "q5") + assert db._session_lineage_root_to_tip("cont") == ["parent", "cont"] + _, cont_display = db.get_resume_conversations("cont") + assert len(cont_display) == 5 # parent's 4 + cont's 1 + def test_subagent_session_still_hidden(self, db): """Sub-agent children (parent NOT ended with 'branched') remain hidden.""" db.create_session("root", "cli") diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 267a6839a57d..c02bebf4109c 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -6840,12 +6840,20 @@ def _make_agent( # The switch already resolved concrete credentials/endpoint; honor # persisted overrides only while using that original runtime. They # must not leak into a different fallback provider/model pair. - if override_base_url: - runtime["base_url"] = override_base_url - if override_api_key: - runtime["api_key"] = override_api_key - if override_api_mode: - runtime["api_mode"] = override_api_mode + # + # Exception: Claude-on-Vertex. resolve_runtime_provider's + # model-aware vertex branch is authoritative there (AnthropicVertex + # SDK, anthropic_messages, self-refreshing OAuth Credentials) β€” + # session rows persisted by provider-level flows store api_mode + # "chat_completions", and re-applying that clobbers the resolution + # and sends the primary to the OpenAI-compat surface (HTTP 404). + if not runtime.get("vertex_anthropic"): + if override_base_url: + runtime["base_url"] = override_base_url + if override_api_key: + runtime["api_key"] = override_api_key + if override_api_mode: + runtime["api_mode"] = override_api_mode else: model, requested_provider = _resolve_startup_runtime() if isinstance(model_override, str) and model_override: diff --git a/website/docs/guides/google-vertex.md b/website/docs/guides/google-vertex.md index 54923db967d5..fd7eb7fe57b1 100644 --- a/website/docs/guides/google-vertex.md +++ b/website/docs/guides/google-vertex.md @@ -1,12 +1,12 @@ --- sidebar_position: 15 title: "Google Vertex AI" -description: "Use Hermes Agent with Gemini on Google Cloud Vertex AI β€” OAuth2 service account or ADC, GCP billing and quotas, no static API key" +description: "Use Hermes Agent with Gemini and Claude on Google Cloud Vertex AI β€” OAuth2 service account or ADC, GCP billing and quotas, no static API key" --- # Google Vertex AI -Hermes Agent supports **Gemini models on Google Cloud Vertex AI** through Vertex's OpenAI-compatible endpoint. Unlike the [Google AI Studio provider](/guides/google-gemini) (which uses a static API key against `generativelanguage.googleapis.com`), Vertex gives you **enterprise-grade rate limits and GCP billing/credits**, and is the right choice when you want Gemini usage to draw on your Google Cloud account rather than an AI Studio key. +Hermes Agent supports **Gemini and Claude models on Google Cloud Vertex AI**. Gemini (and partner MaaS models such as DeepSeek and Kimi) ride Vertex's OpenAI-compatible endpoint; Claude rides the Anthropic SDK's native Vertex adapter β€” see [Claude on Vertex](#claude-on-vertex) below. Unlike the [Google AI Studio provider](/guides/google-gemini) (which uses a static API key against `generativelanguage.googleapis.com`), Vertex gives you **enterprise-grade rate limits and GCP billing/credits**, and is the right choice when you want Gemini usage to draw on your Google Cloud account rather than an AI Studio key. :::info Vertex authenticates with OAuth2, not an API key Vertex has **no static API key** for the standard endpoint. Every request needs a short-lived **OAuth2 access token** (β‰ˆ1 hour TTL) minted from either a service-account JSON or Application Default Credentials (ADC). Hermes mints and **auto-refreshes** these tokens for you β€” you never paste a token by hand. This is why pasting a temporary token into a custom provider's `api_key` field does not work: it expires mid-session. @@ -99,6 +99,35 @@ Vertex requires the `google/` vendor prefix on model IDs. The `hermes model` pic The Gemini 3.x preview models are served through the `global` endpoint. Regional endpoints (`us-central1`, etc.) may 404 them. Leave `region: global` unless you have a specific reason to pin a region. ::: +### Claude models + +Claude uses **bare publisher IDs** (no `google/`-style prefix β€” the SDK injects `publishers/anthropic/models/` into the URL). The OpenRouter-style `anthropic/claude-*` alias is accepted and normalized to the bare ID. + +| Model | ID | +|-------|----| +| Claude Fable 5 | `claude-fable-5` | +| Claude Sonnet 5 | `claude-sonnet-5` | +| Claude Opus 4.8 | `claude-opus-4-8` | + +## Claude on Vertex + +Claude models on Vertex speak the **Anthropic Messages protocol** (`rawPredict` / `streamRawPredict`), not the OpenAI-compatible endpoint that serves Gemini and partner MaaS models. Hermes detects Claude model IDs and routes them through the Anthropic SDK's native **`AnthropicVertex`** client automatically β€” same config, same `provider: vertex`, full feature parity with direct Anthropic (prompt caching, thinking budgets, fine-grained tool streaming). + +Two things differ from the Gemini path: + +- **Auth**: the google-auth `Credentials` object is handed to the SDK, which mints and refreshes OAuth2 tokens itself β€” long-lived gateway sessions never hit the ~1-hour token expiry. User ADC additionally sends the `x-goog-user-project` header so quota is billed to your configured project. +- **Region**: Claude serves through `global` and a small set of regions (e.g. `us-east5`) β€” it is **not** available in every region Gemini uses, and the OpenAI-compat endpoint cannot serve it anywhere. `region: global` works for both families. + +### Model Garden enablement + +Before first use, in the GCP console (Vertex AI β†’ Model Garden β†’ the Claude model card): + +1. **Enable** the model for your project (accept the terms). +2. Some models additionally require **data sharing** to be enabled for publisher `anthropic` β€” a 403 `"requires data sharing to be enabled"` tells you which. +3. Fresh projects often have **zero token quota** for Claude β€” a 429 `Quota exceeded ... per_base_model` on your first tiny request means you need a quota increase (IAM β†’ Quotas β†’ `online_prediction_input_tokens_per_minute_per_base_model` for the `anthropic-claude-*` base model). + +You can mix families freely β€” e.g. an MoA preset with Gemini/Kimi/DeepSeek reference models and a Claude aggregator, all `provider: vertex`. + ## Switching Models Mid-Session ```text @@ -138,6 +167,18 @@ You are probably on a regional endpoint. Set `region: global` in the `vertex:` s The service account (or your ADC identity) needs the `roles/aiplatform.user` role on the project, and the Vertex AI API must be enabled for that project. +### "Malformed publisher model" or 404 on Claude models + +The request went to the OpenAI-compatible endpoint, which cannot serve Anthropic models. Upgrade Hermes β€” older versions routed all Vertex models through that endpoint β€” and make sure the model ID is a Claude ID Hermes can detect (`claude-*` or `anthropic/claude-*`). + +### 403 "requires data sharing to be enabled for publisher 'anthropic'" + +Enable data sharing for Anthropic models in the GCP console (Model Garden β†’ the model card). Per-model; newer models are more likely to require it. + +### 429 quota exceeded on the first Claude request + +Claude token quota defaults to zero on many projects even after enabling the model. Request a quota increase for `online_prediction_input_tokens_per_minute_per_base_model` (base model `anthropic-claude-`). + ## Related - [Google Gemini (AI Studio)](/guides/google-gemini) β€” static-API-key Gemini without GCP diff --git a/website/docs/integrations/providers.md b/website/docs/integrations/providers.md index 6137f8f55940..657ff490cf0c 100644 --- a/website/docs/integrations/providers.md +++ b/website/docs/integrations/providers.md @@ -410,7 +410,7 @@ See the [AWS Bedrock guide](/guides/aws-bedrock) for a walkthrough of IAM setup, ### Google Vertex AI -Gemini models on Google Cloud Vertex AI via Vertex's OpenAI-compatible endpoint. Authentication is **OAuth2** β€” a short-lived access token (~1 hour) minted from a service-account JSON or Application Default Credentials (ADC). There is **no static API key**; Hermes mints and auto-refreshes the token for you, including re-minting on a mid-session `401`. +Gemini **and Claude** models on Google Cloud Vertex AI. Gemini and partner MaaS models ride Vertex's OpenAI-compatible endpoint; Claude models are detected by ID and routed through the Anthropic SDK's native `AnthropicVertex` client (Anthropic Messages protocol β€” prompt caching, thinking budgets). Authentication is **OAuth2** β€” a short-lived access token (~1 hour) minted from a service-account JSON or Application Default Credentials (ADC). There is **no static API key**; Hermes mints and auto-refreshes the token for you, including re-minting on a mid-session `401`. ```bash # Service account JSON (recommended for servers / gateways) @@ -425,7 +425,7 @@ Or in `config.yaml` (project/region are non-secret and live here; the credential ```yaml model: provider: "vertex" - default: "google/gemini-3-flash-preview" # Vertex requires the google/ prefix + default: "google/gemini-3-flash-preview" # Gemini/MaaS need the publisher/ prefix; Claude IDs are bare (claude-fable-5) vertex: project_id: "my-gcp-project" # blank β†’ use the project embedded in the credentials region: "global" # required for the Gemini 3.x previews