From 39528e1ae15658dc740b278d1b6708c3258989f4 Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 12:13:42 +0800 Subject: [PATCH 01/12] feat(vertex): add API key (Express Mode) auth + region-specific model discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add dual-auth support for Google Vertex AI: 1. API Key (Express Mode) — the recommended approach. Set GOOGLE_VERTEX_API_KEY, GOOGLE_VERTEX_PROJECT, and optionally GOOGLE_VERTEX_LOCATION in .env. No google-auth needed. 2. OAuth2 / ADC — legacy path preserved unchanged. Key changes: agent/vertex_adapter.py: - Add GOOGLE_VERTEX_API_KEY, GOOGLE_VERTEX_PROJECT, GOOGLE_VERTEX_LOCATION env var constants - Add has_vertex_api_key(), resolve_vertex_api_key(), build_vertex_api_key_base_url() functions - Update get_vertex_config() to auto-select API key vs OAuth2 path - Add discover_vertex_models() — queries Vertex's publisher models.list API for region-specific model availability - Update has_vertex_credentials() to detect API key plugins/model-providers/vertex/__init__.py: - Add GOOGLE_VERTEX_API_KEY, GOOGLE_VERTEX_PROJECT, GOOGLE_VERTEX_LOCATION to env_vars - Implement fetch_models() using discover_vertex_models() for API key path hermes_cli/runtime_provider.py: - Update comment and AuthError message for dual auth - Set source='vertex-api-key' vs 'vertex-oauth' hermes_cli/model_setup_flows.py: - Rewrite _model_flow_vertex() to detect and advertise API key mode - Add dynamic model discovery via discover_vertex_models() when API key is set - Fall back to curated list when discovery fails or using OAuth2 hermes_cli/models.py: - Add curated "vertex" key to _PROVIDER_MODELS with 13 models - Update provider description website/docs/guides/google-vertex.md: - Rewrite for dual-auth: API key (recommended) + OAuth2/ADC (legacy) - Add model discovery docs --- agent/vertex_adapter.py | 190 +++++++++++++++++++-- hermes_cli/model_setup_flows.py | 76 +++++++-- hermes_cli/models.py | 17 +- hermes_cli/runtime_provider.py | 57 ++++--- plugins/model-providers/vertex/__init__.py | 65 +++++-- website/docs/guides/google-vertex.md | 123 +++++++++---- 6 files changed, 426 insertions(+), 102 deletions(-) diff --git a/agent/vertex_adapter.py b/agent/vertex_adapter.py index 6e425753f053a..86f7012f3d92a 100644 --- a/agent/vertex_adapter.py +++ b/agent/vertex_adapter.py @@ -4,16 +4,43 @@ endpoint. This allows Hermes to use Gemini models via Google Cloud with enterprise-grade rate limits and quotas. -Requires: pip install google-auth +Supports two authentication methods: -Environment variables honored (all optional): +1. **API Key (Express Mode) — RECOMMENDED.** + Set ``GOOGLE_VERTEX_API_KEY`` in your .env file. This is a static key that + works with Vertex's Express Mode endpoint. No OAuth, no service-account + JSON, no ADC setup. You also need: + + - ``GOOGLE_VERTEX_PROJECT`` — your GCP project ID. + - ``GOOGLE_VERTEX_LOCATION`` — region (default: ``us-central1``). + + When using API key auth the ``Authorization`` header carries the key as a + Bearer token and the request is routed to the global ``aiplatform.googleapis.com`` + host, which internally fans out to the configured region. + +2. **OAuth2 / ADC (legacy)** + Requires ``pip install google-auth`` and one of: + + - ``GOOGLE_APPLICATION_CREDENTIALS`` — path to a service account JSON. + - ``VERTEX_CREDENTIALS_PATH`` — alias, takes precedence if set. + - ``gcloud auth application-default login`` — local ADC. + + Additional routing settings (non-secret) live in ``config.yaml`` under + the ``vertex:`` section — project_id and region. + +Auth selection is automatic: if ``GOOGLE_VERTEX_API_KEY`` is set, the API key +path is used. Otherwise the adapter falls back to OAuth2 / ADC. + +API key env vars (all optional): + GOOGLE_VERTEX_API_KEY — Vertex AI API key for Express Mode (secret). + GOOGLE_VERTEX_PROJECT — GCP project ID (secret — read at runtime). + GOOGLE_VERTEX_LOCATION — Vertex region (default: us-central1). + +OAuth2 / ADC env vars (all optional): GOOGLE_APPLICATION_CREDENTIALS — path to a service account JSON file (secret). VERTEX_CREDENTIALS_PATH — alias, takes precedence if set (secret). VERTEX_PROJECT_ID — override the project_id embedded in creds. - VERTEX_REGION — override default region ("global" unless set). - -Non-secret routing settings (project_id, region) also live in config.yaml -under the ``vertex:`` section; env vars take precedence over config.yaml. + VERTEX_REGION — override default region ("us-central1"). """ import logging @@ -42,7 +69,15 @@ logger = logging.getLogger(__name__) -DEFAULT_REGION = "global" +# Environment variable constants for API key auth (Express Mode) +GOOGLE_VERTEX_API_KEY = "GOOGLE_VERTEX_API_KEY" +GOOGLE_VERTEX_PROJECT = "GOOGLE_VERTEX_PROJECT" +GOOGLE_VERTEX_LOCATION = "GOOGLE_VERTEX_LOCATION" + +# Default region — us-central1 is the most widely available Vertex region. +# The old default was "global" (required for Gemini 3.x previews via ADC), +# but API key / Express Mode works best with an explicit region. +DEFAULT_REGION = "us-central1" _creds_cache: dict = {} @@ -64,9 +99,13 @@ def _vertex_config() -> dict: def _resolve_region(explicit: Optional[str] = None) -> str: - """Region precedence: explicit arg > VERTEX_REGION env > config.yaml > default.""" + """Region precedence: explicit arg > GOOGLE_VERTEX_LOCATION env > VERTEX_REGION env > config.yaml > default.""" if explicit: return explicit + # Check GOOGLE_VERTEX_LOCATION first (API key / Express Mode preferred env var) + gv_location = (_get_secret(GOOGLE_VERTEX_LOCATION) or "").strip() + if gv_location: + return gv_location env_region = (_get_secret("VERTEX_REGION") or "").strip() if env_region: return env_region @@ -75,11 +114,15 @@ def _resolve_region(explicit: Optional[str] = None) -> str: def _resolve_project_override() -> Optional[str]: - """Project-ID override precedence: VERTEX_PROJECT_ID env > config.yaml. + """Project-ID override precedence: GOOGLE_VERTEX_PROJECT env > VERTEX_PROJECT_ID env > config.yaml. Returns None when neither is set (the credentials' embedded project_id - is used in that case). + is used in that case for OAuth2; for API key mode the caller should + prompt for a project if this returns None). """ + gv_project = (_get_secret(GOOGLE_VERTEX_PROJECT) or "").strip() + if gv_project: + return gv_project env_project = (_get_secret("VERTEX_PROJECT_ID") or "").strip() if env_project: return env_project @@ -103,6 +146,31 @@ def _resolve_credentials_path(explicit: Optional[str]) -> Optional[str]: return None +def has_vertex_api_key() -> bool: + """Check whether a Vertex AI API key is configured.""" + return bool(_get_secret(GOOGLE_VERTEX_API_KEY)) + + +def resolve_vertex_api_key() -> Optional[str]: + """Return the configured Vertex AI API key, or None.""" + return _get_secret(GOOGLE_VERTEX_API_KEY) + + +def build_vertex_api_key_base_url(project_id: str, region: str) -> str: + """Build the OpenAI-compatible base URL for Vertex AI Express Mode. + + Express Mode uses the standard aiplatform.googleapis.com host (or + ``{region}-aiplatform.googleapis.com`` for regional endpoints) with the + project in the URL path. The API key is passed as a Bearer token in the + Authorization header. + + The ``global`` location uses the bare ``aiplatform.googleapis.com`` host. + Regional locations use ``{region}-aiplatform.googleapis.com``. + """ + host = "aiplatform.googleapis.com" if region == "global" else f"{region}-aiplatform.googleapis.com" + return f"https://{host}/v1beta1/projects/{project_id}/locations/{region}/endpoints/openapi" + + def _refresh_credentials(creds) -> None: auth_req = google.auth.transport.requests.Request() creds.refresh(auth_req) @@ -203,7 +271,37 @@ def get_vertex_config( credentials_path: Optional[str] = None, region: Optional[str] = None, ) -> Tuple[Optional[str], Optional[str]]: - """Resolve (access_token, base_url) for Vertex AI, or (None, None) on failure.""" + """Resolve (access_token_or_api_key, base_url) for Vertex AI. + + Two authentication paths, chosen automatically: + + 1. **API Key (Express Mode)** — if ``GOOGLE_VERTEX_API_KEY`` is set. + Returns (api_key, base_url_with_project). No ``google-auth`` needed. + + 2. **OAuth2 / ADC** — legacy path. Returns (oauth2_token, base_url). + Requires ``google-auth`` and valid GCP credentials. + + Returns (None, None) when no credentials can be resolved. + """ + # --- Path 1: API Key (Express Mode) --- + api_key = resolve_vertex_api_key() + if api_key: + project_id = _resolve_project_override() + if not project_id: + logger.warning( + "Vertex API key found but no project ID configured. " + "Set GOOGLE_VERTEX_PROJECT in ~/.hermes/.env." + ) + return None, None + effective_region = _resolve_region(region) + base_url = build_vertex_api_key_base_url(project_id, effective_region) + logger.debug( + "get_vertex_config: using API key (Express Mode) for project %s in %s", + project_id, effective_region, + ) + return api_key, base_url + + # --- Path 2: OAuth2 / ADC (legacy) --- token, project_id = get_vertex_credentials(credentials_path) if not token or not project_id: return None, None @@ -216,13 +314,75 @@ def get_vertex_config( def has_vertex_credentials() -> bool: """Fast check for whether Vertex credentials appear configured. - No network calls and no google-auth import — safe for provider - auto-detection and setup-status display. True when either a service - account JSON path is resolvable, or an explicit project ID is configured - (env or config.yaml, implying ADC is intended). + Returns True when either: + - A Vertex API key (GOOGLE_VERTEX_API_KEY) is set, OR + - A service account JSON path is resolvable, OR + - An explicit project ID is configured (ADC intended). + + No network calls and no ``google-auth`` import — safe for provider + auto-detection and setup-status display. """ + if has_vertex_api_key(): + return True if _resolve_credentials_path(None): return True if _resolve_project_override(): return True return False + + +# ── Model Discovery ────────────────────────────────────────────────────────── + + +def discover_vertex_models( + api_key: str, + project_id: str, + region: str = DEFAULT_REGION, + timeout: float = 10.0, +) -> list[str]: + """Query Vertex AI's ``models.list`` publisher endpoint for models + available in the given project and region. + + Returns a sorted list of model ID strings (e.g. ``gemini-2.5-flash``, + ``gemini-3-pro-preview``). Only models that support ``generateContent`` + (chat / text-generation) are returned. + + Returns the (sorted) model list on success. + Returns an empty list on any error (network, auth, parse). + """ + import json + import urllib.error + import urllib.request + + host = "aiplatform.googleapis.com" if region == "global" else f"{region}-aiplatform.googleapis.com" + url = ( + f"https://{host}/v1/projects/{project_id}/locations/{region}" + "/publishers/google/models" + ) + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + try: + req = urllib.request.Request(url, headers=headers, method="GET") + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode()) + + models: list[str] = [] + for entry in data.get("models", []): + methods = entry.get("supportedGenerationMethods", []) + if "generateContent" in methods: + # The ``name`` field is a full resource path: + # projects/{project}/locations/{region}/publishers/google/models/{model_id} + model_id = entry["name"].rsplit("/", 1)[-1] + models.append(model_id) + + return sorted(set(models)) + + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, OSError) as exc: + logger.warning("discover_vertex_models: failed to list models — %s", exc) + return [] + except Exception as exc: + logger.warning("discover_vertex_models: unexpected error — %s", exc) + return [] diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 394360c160737..3564f838eb44a 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -2437,10 +2437,16 @@ def _sort_key(m): def _model_flow_vertex(config, current_model=""): """Google Vertex AI provider: Gemini via the OpenAI-compatible endpoint. - Auth is OAuth2 — short-lived tokens minted from a service-account JSON or - Application Default Credentials (ADC). No static API key. The credential - *path* lives in .env (VERTEX_CREDENTIALS_PATH / GOOGLE_APPLICATION_CREDENTIALS); - project ID and region are non-secret and saved to config.yaml under vertex:. + Supports two authentication methods, auto-detected: + + 1. **API Key (Express Mode) — RECOMMENDED.** + Set ``GOOGLE_VERTEX_API_KEY``, ``GOOGLE_VERTEX_PROJECT``, and optionally + ``GOOGLE_VERTEX_LOCATION`` in .env. No google-auth needed. + Supports dynamic model discovery via Vertex's publisher models API. + + 2. **OAuth2 / ADC (legacy)** + Service-account JSON or ADC via ``google-auth``. + Falls back to a curated model list. """ from hermes_cli.auth import ( _prompt_model_selection, @@ -2449,18 +2455,35 @@ def _model_flow_vertex(config, current_model=""): ) from hermes_cli.config import load_config, save_config, get_env_value from hermes_cli.models import _PROVIDER_MODELS + from agent.vertex_adapter import ( + has_vertex_api_key, + resolve_vertex_api_key, + discover_vertex_models, + GOOGLE_VERTEX_PROJECT as ENV_VERTEX_PROJECT, + GOOGLE_VERTEX_LOCATION as ENV_VERTEX_LOCATION, + DEFAULT_REGION, + ) - # 1. Credential source detection (fast, no network / no google-auth import). + # 1. Credential source detection. + api_key = resolve_vertex_api_key() sa_path = ( get_env_value("VERTEX_CREDENTIALS_PATH") or get_env_value("GOOGLE_APPLICATION_CREDENTIALS") or "" ).strip() - if sa_path: + + if api_key and sa_path: + print(" Vertex credentials: API key + service account JSON (API key takes priority) ✓") + elif api_key: + print(" Vertex credentials: API key (Express Mode) ✓") + print(" No OAuth, no service-account JSON needed.") + elif sa_path: print(f" Vertex credentials: service account JSON ({sa_path}) ✓") else: print(" Vertex credentials: Application Default Credentials (ADC)") - print(" Vertex uses OAuth2, not a static API key. Either:") + print(" Option A — API key (recommended):") + print(" Set GOOGLE_VERTEX_API_KEY in ~/.hermes/.env") + print(" Option B — OAuth2 (legacy):") print(" • run 'gcloud auth application-default login', or") print(" • set VERTEX_CREDENTIALS_PATH in ~/.hermes/.env to a service account JSON") print() @@ -2470,19 +2493,22 @@ def _model_flow_vertex(config, current_model=""): if not isinstance(vertex_cfg, dict): vertex_cfg = {} - # 2. Project ID (optional — falls back to the project embedded in creds). - current_project = str(vertex_cfg.get("project_id") or "").strip() + # 2. Project ID (required for both auth methods, but API key more strictly). + env_project = get_env_value(ENV_VERTEX_PROJECT) or "" + current_project = str(env_project or vertex_cfg.get("project_id") or "").strip() + prompt_default = current_project if current_project else "from credentials" try: project_input = input( - f" GCP project ID [{current_project or 'from credentials'}]: " + f" GCP project ID [{prompt_default}]: " ).strip() except (KeyboardInterrupt, EOFError): print() return project_id = project_input or current_project - # 3. Region (default global — required for the Gemini 3.x previews). - current_region = str(vertex_cfg.get("region") or "global").strip() or "global" + # 3. Region. + env_location = get_env_value(ENV_VERTEX_LOCATION) or "" + current_region = str(env_location or vertex_cfg.get("region") or DEFAULT_REGION).strip() or DEFAULT_REGION try: region_input = input(f" Vertex region [{current_region}]: ").strip() except (KeyboardInterrupt, EOFError): @@ -2490,11 +2516,27 @@ def _model_flow_vertex(config, current_model=""): return region = region_input or current_region - # 4. Model selection (curated list — Vertex has no /models listing route). - model_list = _PROVIDER_MODELS.get("vertex", []) or [ - "google/gemini-3-pro-preview", - "google/gemini-3-flash-preview", - ] + # 4. Model selection — try dynamic discovery first, fall back to curated list. + discovered: list[str] | None = None + if api_key and project_id: + print(f" Discovering models in {region}...", end=" ", flush=True) + discovered = discover_vertex_models(api_key, project_id, region) + if discovered: + print(f"found {len(discovered)} models ✓") + else: + print("failed — using curated list.") + else: + print(" (Model discovery requires API key + project ID; using curated list.)") + + if discovered: + # Prefix with ``google/`` to match Hermes model naming convention + model_list = [f"google/{m}" for m in discovered] + else: + model_list = _PROVIDER_MODELS.get("vertex", []) or [ + "google/gemini-3-pro-preview", + "google/gemini-3-flash-preview", + ] + base_url_preview = ( "https://aiplatform.googleapis.com/v1beta1/projects//" f"locations/{region}/endpoints/openapi" diff --git a/hermes_cli/models.py b/hermes_cli/models.py index dbb14e05f684d..33cb6d8b5a1a7 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -296,6 +296,21 @@ def _xai_curated_models() -> list[str]: "gemini-3.5-flash", "gemini-3.1-flash-lite-preview", ], + "vertex": [ + "google/gemini-3.6-flash", + "google/gemini-3.5-flash", + "google/gemini-3.5-flash-lite", + "google/gemini-3.1-pro-preview", + "google/gemini-3.1-flash-lite", + "google/gemini-3-flash-preview", + "google/gemini-3-pro-preview", + "google/gemini-2.5-pro", + "google/gemini-2.5-flash", + "google/gemini-2.5-flash-lite", + "google/gemini-flash-latest", + "google/gemini-flash-lite-latest", + "google/gemini-embedding-001", + ], "zai": [ "glm-5.2", "glm-5.1", @@ -1081,7 +1096,7 @@ class ProviderEntry(NamedTuple): ProviderEntry("copilot-acp", "GitHub Copilot ACP", "GitHub Copilot ACP (Spawns copilot --acp --stdio)"), ProviderEntry("huggingface", "Hugging Face", "Hugging Face Inference Providers"), ProviderEntry("gemini", "Google AI Studio", "Google AI Studio (Native Gemini API)"), - ProviderEntry("vertex", "Google Vertex AI", "Google Vertex AI (Gemini via GCP; OAuth2 service account or ADC, GCP billing/quotas)"), + ProviderEntry("vertex", "Google Vertex AI", "Google Vertex AI (Gemini via GCP; API key or OAuth2/ADC, GCP billing/quotas, region-specific model discovery)"), ProviderEntry("deepseek", "DeepSeek", "DeepSeek (V3, R1, coder, direct API)"), ProviderEntry("xai", "xAI", "xAI Grok (Direct API)"), ProviderEntry("zai", "Z.AI / GLM", "Z.AI / GLM (Zhipu direct API)"), diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index 6fee9b013a936..cacb7078c4307 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -1604,36 +1604,49 @@ def resolve_runtime_provider( ) return azure_runtime - # Vertex AI: OAuth2-token provider (Gemini via the OpenAI-compatible - # endpoint). Resolve BEFORE the custom-runtime / credential-pool / generic - # paths. The credential *path* (GOOGLE_APPLICATION_CREDENTIALS / - # VERTEX_CREDENTIALS_PATH) must never reach the credential pool or the - # generic api_key resolver — those would treat the file path as a static - # API key. Instead we mint a short-lived OAuth2 access token here and hand - # it to the standard OpenAI client as api_key, with base_url computed from - # the project ID + region. The token is re-minted per call (5-min refresh - # margin) by get_vertex_config(); mid-session expiry is additionally - # recovered on 401 by run_agent._try_refresh_vertex_client_credentials(). + # Vertex AI: dual-auth provider (Gemini via the OpenAI-compatible endpoint). + # Resolve BEFORE the custom-runtime / credential-pool / generic paths. + # Two auth methods, auto-selected: + # + # 1. API Key (Express Mode) — if GOOGLE_VERTEX_API_KEY is set. + # Token resolution and model discovery live in ``agent/vertex_adapter.py``. + # No google-auth needed. + # + # 2. OAuth2 / ADC (legacy) — service-account JSON or ADC. + # The credential *path* (GOOGLE_APPLICATION_CREDENTIALS / + # VERTEX_CREDENTIALS_PATH) must never reach the credential pool or the + # generic api_key resolver — those would treat the file path as a static + # API key. Instead we mint a short-lived OAuth2 access token here and hand + # it to the standard OpenAI client as api_key, with base_url computed from + # the project ID + region. The token is re-minted per call (5-min refresh + # margin) by get_vertex_config(); mid-session expiry is additionally + # recovered on 401 by run_agent._try_refresh_vertex_client_credentials(). if requested_provider in ("vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai"): - from agent.vertex_adapter import get_vertex_config + from agent.vertex_adapter import ( + get_vertex_config, + has_vertex_api_key, + ) - token, base_url = get_vertex_config() - if not token or not base_url: + token_or_key, base_url = get_vertex_config() + if not token_or_key or not base_url: raise AuthError( - "Vertex AI credentials could not be resolved. Vertex uses " - "OAuth2 (not a static API key): provide a service-account JSON " - "via GOOGLE_APPLICATION_CREDENTIALS (or VERTEX_CREDENTIALS_PATH) " - "in ~/.hermes/.env, or run 'gcloud auth application-default " - "login' for ADC. Set the GCP project/region under vertex: in " - "config.yaml if they aren't embedded in the credentials. " - "Install the extra with: pip install 'hermes-agent[vertex]'." + "Vertex AI credentials could not be resolved.\n\n" + "Method 1 — API Key (recommended):\n" + " Set GOOGLE_VERTEX_API_KEY, GOOGLE_VERTEX_PROJECT, and\n" + " optionally GOOGLE_VERTEX_LOCATION in ~/.hermes/.env.\n\n" + "Method 2 — OAuth2 / ADC (legacy):\n" + " Set GOOGLE_APPLICATION_CREDENTIALS (or VERTEX_CREDENTIALS_PATH)\n" + " in ~/.hermes/.env, or run 'gcloud auth application-default login'.\n" + " Install the extra with: pip install 'hermes-agent[vertex]'.\n" + " Set the GCP project/region under vertex: in config.yaml." ) + source = "vertex-api-key" if has_vertex_api_key() else "vertex-oauth" return { "provider": "vertex", "api_mode": "chat_completions", "base_url": base_url.rstrip("/"), - "api_key": token, - "source": "vertex-oauth", + "api_key": token_or_key, + "source": source, "requested_provider": requested_provider, } diff --git a/plugins/model-providers/vertex/__init__.py b/plugins/model-providers/vertex/__init__.py index f0d0d4f896b1c..a3d83202bd635 100644 --- a/plugins/model-providers/vertex/__init__.py +++ b/plugins/model-providers/vertex/__init__.py @@ -2,19 +2,22 @@ vertex: Gemini models via Google Cloud's OpenAI-compatible endpoint. -Auth is OAuth2 — short-lived access tokens minted from a service-account JSON -or Application Default Credentials (ADC), NOT a static API key. Token -resolution and refresh live in ``agent/vertex_adapter.py``; runtime_provider.py -calls it to obtain a fresh ``(token, base_url)`` pair, then hands the token to -the standard OpenAI client as ``api_key``. Because the wire format is the -OpenAI-compatible chat/completions surface, no message translation is needed — -the only Gemini-specific concern is the ``thinking_config`` reasoning hook, -which is emitted here exactly as the ``gemini`` provider does for its -OpenAI-compat subpath (``extra_body.google.thinking_config``). - -``auth_type="vertex"`` marks this as an OAuth-token provider (resolved -specially, like bedrock's ``aws_sdk``) so it is never treated as an -api_key provider that would mistake a credentials-file path for a key. +Supports two authentication methods: + +1. **API Key (Express Mode) — RECOMMENDED.** + Set ``GOOGLE_VERTEX_API_KEY``, ``GOOGLE_VERTEX_PROJECT``, and optionally + ``GOOGLE_VERTEX_LOCATION`` in your .env file. No google-auth needed. + +2. **OAuth2 / ADC (legacy)** + Service-account JSON or ADC. Requires ``google-auth``. + +Auth selection is automatic — ``get_vertex_config()`` in ``agent/vertex_adapter.py`` +picks the right path based on which env vars are set. + +``auth_type="vertex"`` marks this as a specially-resolved provider (like +bedrock's ``aws_sdk``) so env-var lookup for a static api_key is not the +only path. The runtime provider resolver (``runtime_provider.py``) handles +API key auth directly without needing the full OAuth2 token flow. """ from typing import Any @@ -56,17 +59,45 @@ def fetch_models( base_url: str | None = None, timeout: float = 8.0, ) -> list[str] | None: - """Vertex's OpenAI-compat endpoint has no ``/models`` listing route; - model discovery is not available. The setup wizard ships a curated list. + """Discover models via Vertex AI's publisher ``models.list`` API. + + Unlike the legacy OAuth2 path (which has no ``/models`` route on the + OpenAI-compatible endpoint), the API key path can query the publisher + models endpoint for region-specific model availability. + + If discovery fails (network, auth, or the OAuth2 path), returns None + and the setup wizard falls back to its curated model list. """ - return None + from agent.vertex_adapter import ( + discover_vertex_models, + resolve_vertex_api_key, + _resolve_project_override, + _resolve_region, + ) + + # Only API key auth supports model discovery + resolved_key = api_key or resolve_vertex_api_key() + if not resolved_key: + return None + + project_id = _resolve_project_override() + if not project_id: + return None + + region = _resolve_region() + models = discover_vertex_models(resolved_key, project_id, region, timeout) + return models if models else None vertex = VertexProfile( name="vertex", aliases=("google-vertex", "vertex-ai", "gcp-vertex"), api_mode="chat_completions", - env_vars=(), # OAuth2 via service account / ADC — not a static key env var + env_vars=( + "GOOGLE_VERTEX_API_KEY", + "GOOGLE_VERTEX_PROJECT", + "GOOGLE_VERTEX_LOCATION", + ), base_url="https://aiplatform.googleapis.com", # real base_url computed at runtime auth_type="vertex", default_aux_model="google/gemini-3-flash-preview", diff --git a/website/docs/guides/google-vertex.md b/website/docs/guides/google-vertex.md index 851a391691c85..76ddeca4ba2c0 100644 --- a/website/docs/guides/google-vertex.md +++ b/website/docs/guides/google-vertex.md @@ -1,18 +1,54 @@ --- sidebar_position: 15 title: "Google Vertex AI" -description: "Use Hermes Agent with Gemini on Google Cloud Vertex AI — OAuth2 service account or ADC, GCP billing and quotas, no static API key" +description: "Use Hermes Agent with Gemini on Google Cloud Vertex AI — API key (Express Mode, recommended) or OAuth2 service account / ADC" --- # Google Vertex AI Hermes Agent supports **Gemini models on Google Cloud Vertex AI** through Vertex's OpenAI-compatible endpoint. Unlike the [Google AI Studio provider](/guides/google-gemini) (which uses a static API key against `generativelanguage.googleapis.com`), Vertex gives you **enterprise-grade rate limits and GCP billing/credits**, and is the right choice when you want Gemini usage to draw on your Google Cloud account rather than an AI Studio key. -:::info Vertex authenticates with OAuth2, not an API key -Vertex has **no static API key** for the standard endpoint. Every request needs a short-lived **OAuth2 access token** (≈1 hour TTL) minted from either a service-account JSON or Application Default Credentials (ADC). Hermes mints and **auto-refreshes** these tokens for you — you never paste a token by hand. This is why pasting a temporary token into a custom provider's `api_key` field does not work: it expires mid-session. +## Authentication + +Vertex supports two authentication methods, chosen automatically: + +### Method 1: API Key (Express Mode) — Recommended + +Vertex now supports **API key authentication** through Express Mode. This is the simplest way to connect — no OAuth, no service-account JSON, no ADC setup. + +1. **Create an API key** in the [Google Cloud Console](https://console.cloud.google.com/apis/credentials). You can use an existing API key with the Vertex AI API enabled. + +2. **Set the environment variables** in `~/.hermes/.env`: + ```bash + GOOGLE_VERTEX_API_KEY=AIza... + GOOGLE_VERTEX_PROJECT=my-gcp-project + GOOGLE_VERTEX_LOCATION=us-central1 # optional, default: us-central1 + ``` + +3. **Select Vertex** as your provider: + ```bash + hermes model + # → Choose "More providers..." → "Google Vertex AI" + # → Verify API key is detected + # → Enter your GCP project ID + # → Models will be auto-discovered from your region + # → Select a model + ``` + +4. **Start chatting:** + ```bash + hermes chat + ``` + +:::tip API key model discovery +When using API key auth, Hermes queries Vertex's `models.list` publisher API to show only models actually available in your project and region. You won't see models that are unavailable or region-locked. ::: -## Prerequisites +### Method 2: OAuth2 / Service Account (Legacy) + +Traditional OAuth2 authentication using a service account JSON file or Application Default Credentials (ADC). + +#### Prerequisites - **A Google Cloud project** with the **Vertex AI API enabled** and billing active. - **Credentials**, one of: @@ -20,7 +56,7 @@ Vertex has **no static API key** for the standard endpoint. Every request needs - **Application Default Credentials** via `gcloud auth application-default login` (or the metadata server when running on a GCP VM). - **`google-auth`** — installed automatically the first time you select Vertex (lazy install), or explicitly with `pip install 'hermes-agent[vertex]'`. -## Quick Start +#### Quick Start ```bash # Option A — service account JSON (recommended for servers / gateways) @@ -33,7 +69,7 @@ gcloud auth application-default login hermes model # → Choose "More providers..." → "Google Vertex AI" # → Enter your GCP project ID (or leave blank to use the one in your credentials) -# → Choose a region (default: global) +# → Choose a region (default: us-central1) # → Select a Gemini model # Start chatting @@ -44,15 +80,20 @@ hermes chat Vertex splits its settings by sensitivity: -- The **credential path** is a pointer to a secret and lives in `~/.hermes/.env`. +- **Credential paths and API keys** go in `~/.hermes/.env`. - **Project ID and region** are non-secret routing settings and live in `~/.hermes/config.yaml`. `~/.hermes/.env`: ```bash -# One of these (checked in this order); omit both to use ADC: -VERTEX_CREDENTIALS_PATH=/path/to/service-account.json -GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json +# Method 1 — API Key (recommended): +GOOGLE_VERTEX_API_KEY=AIza... +GOOGLE_VERTEX_PROJECT=my-gcp-project +GOOGLE_VERTEX_LOCATION=us-central1 + +# Method 2 — Service account (legacy): +# VERTEX_CREDENTIALS_PATH=/path/to/service-account.json +# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json ``` `~/.hermes/config.yaml`: @@ -63,40 +104,54 @@ model: provider: vertex vertex: - project_id: my-gcp-project # blank → use the project embedded in the credentials - region: global # "global" is required for the Gemini 3.x previews + project_id: my-gcp-project # blank → use the project from env or credentials + region: us-central1 # default region ``` :::tip Environment variables win over config.yaml -`VERTEX_PROJECT_ID` and `VERTEX_REGION` override the `vertex.project_id` / `vertex.region` values in `config.yaml`. Use them for per-shell overrides; keep the durable settings in `config.yaml`. +`GOOGLE_VERTEX_PROJECT`, `GOOGLE_VERTEX_LOCATION`, `VERTEX_PROJECT_ID`, and `VERTEX_REGION` override the `vertex.project_id` / `vertex.region` values in `config.yaml`. Use them for per-shell overrides; keep the durable settings in `config.yaml`. ::: ### How authentication works -1. Hermes resolves credentials in this order: `VERTEX_CREDENTIALS_PATH` → `GOOGLE_APPLICATION_CREDENTIALS` → ADC. -2. It mints an OAuth2 access token (`cloud-platform` scope) and caches it, refreshing when the token is within 5 minutes of expiry. -3. The token is handed to a standard OpenAI client pointed at the Vertex endpoint: - ```text +For **API Key (Express Mode)**: +1. Hermes reads your `GOOGLE_VERTEX_API_KEY` from the environment. +2. It passes the key as a Bearer token in the `Authorization` header to the Vertex OpenAI-compatible endpoint. +3. The endpoint URL includes your project ID and region in the path: + ``` https://aiplatform.googleapis.com/v1beta1/projects/{project}/locations/{region}/endpoints/openapi ``` - Regional locations use a `{region}-aiplatform.googleapis.com` host instead. -4. If a session runs longer than the token lifetime and a request returns `401`, Hermes re-mints the token and retries automatically. On a long-running gateway, if ADC's refresh token has itself expired, Hermes falls back to the service-account JSON when one is configured. +4. No token refresh needed — API keys don't expire. + +For **OAuth2 / ADC**: +1. Hermes resolves credentials in this order: `VERTEX_CREDENTIALS_PATH` → `GOOGLE_APPLICATION_CREDENTIALS` → ADC. +2. It mints an OAuth2 access token (`cloud-platform` scope) and caches it, refreshing when the token is within 5 minutes of expiry. +3. The token is handed to a standard OpenAI client pointed at the Vertex endpoint. +4. If a session runs longer than the token lifetime and a request returns `401`, Hermes re-mints the token and retries automatically. ## Available Models -Vertex requires the `google/` vendor prefix on model IDs. The `hermes model` picker offers: +When using **API key auth**, Hermes automatically discovers models available in your project and region by querying Vertex's `models.list` publisher API. This means you'll only see models that are actually accessible. + +When using **OAuth2 / ADC** (or if discovery fails), the model picker falls back to a curated list: | Model | ID | -|-------|----| +|-------|-----| +| Gemini 3.6 Flash | `google/gemini-3.6-flash` | +| Gemini 3.5 Flash | `google/gemini-3.5-flash` | +| Gemini 3.5 Flash Lite | `google/gemini-3.5-flash-lite` | | Gemini 3.1 Pro Preview | `google/gemini-3.1-pro-preview` | -| Gemini 3 Pro Preview | `google/gemini-3-pro-preview` | +| Gemini 3.1 Flash Lite | `google/gemini-3.1-flash-lite` | | Gemini 3 Flash Preview | `google/gemini-3-flash-preview` | -| Gemini 3.1 Flash Lite Preview | `google/gemini-3.1-flash-lite-preview` | +| Gemini 3 Pro Preview | `google/gemini-3-pro-preview` | | Gemini 2.5 Pro | `google/gemini-2.5-pro` | | Gemini 2.5 Flash | `google/gemini-2.5-flash` | +| Gemini 2.5 Flash Lite | `google/gemini-2.5-flash-lite` | +| Gemini Flash Latest | `google/gemini-flash-latest` | +| Gemini Flash Lite Latest | `google/gemini-flash-lite-latest` | -:::note `global` region for Gemini 3.x -The Gemini 3.x preview models are served through the `global` endpoint. Regional endpoints (`us-central1`, etc.) may 404 them. Leave `region: global` unless you have a specific reason to pin a region. +:::note Region-specific availability +Models must be available in the region you configure. For example, `us-central1` has the broadest model availability, while `europe-west4` may not have all preview models. ::: ## Switching Models Mid-Session @@ -118,25 +173,33 @@ Vertex exposes Gemini's thinking budget through the OpenAI-compatible surface. H hermes doctor ``` -The doctor reports whether Vertex credentials can be resolved (service-account path or ADC) and whether the provider is configured. +The doctor reports whether Vertex credentials can be resolved (API key, service-account path, or ADC) and whether the provider is configured. ## Troubleshooting ### "Vertex AI credentials could not be resolved" -Hermes found neither a service-account JSON nor working ADC. Either set `VERTEX_CREDENTIALS_PATH` in `~/.hermes/.env`, or run `gcloud auth application-default login`. If your project isn't embedded in the credentials, set `vertex.project_id` in `config.yaml`. +Hermes couldn't find valid credentials. Either: + +- **Set up API key** (recommended): Set `GOOGLE_VERTEX_API_KEY`, `GOOGLE_VERTEX_PROJECT`, and optionally `GOOGLE_VERTEX_LOCATION` in `~/.hermes/.env`. + +- **Set up OAuth2**: Set `VERTEX_CREDENTIALS_PATH` in `~/.hermes/.env`, or run `gcloud auth application-default login`. If your project isn't embedded in the credentials, set `vertex.project_id` in `config.yaml`. + +### API key found but no project ID + +When using API key auth, you must also set `GOOGLE_VERTEX_PROJECT` in `~/.hermes/.env` to tell Vertex which GCP project to bill. ### `google-auth` not installed -Install the extra: `pip install 'hermes-agent[vertex]'`. Hermes also lazy-installs it the first time you select the Vertex provider. +Only needed for OAuth2 / ADC auth. API key auth does not require `google-auth`. To fix OAuth2: `pip install 'hermes-agent[vertex]'`. Hermes also lazy-installs it the first time you select the Vertex provider. ### 404 on Gemini 3.x models -You are probably on a regional endpoint. Set `region: global` in the `vertex:` section of `config.yaml` (or unset `VERTEX_REGION`). +You may be on a regional endpoint that doesn't host the model. Switch to a region that supports it (`us-central1` has the broadest coverage), or check the [Vertex AI model page](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models) for regional availability. ### 403 / permission denied -The service account (or your ADC identity) needs the `roles/aiplatform.user` role on the project, and the Vertex AI API must be enabled for that project. +For API key auth: ensure the Vertex AI API is enabled and that billing is active on your project. For OAuth2: the service account (or your ADC identity) needs the `roles/aiplatform.user` role on the project. ## Related From 5de5137b1f07efbf4fea0c10b9393e7cd57c2c10 Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 12:15:52 +0800 Subject: [PATCH 02/12] fix(tests): update Vertex tests for API key auth and new default region - test_vertex_provider.py: update AuthError message assertion for new dual-auth error text (API Key + OAuth2) - test_vertex_adapter.py: update default region from 'global' to 'us-central1' to match new DEFAULT_REGION constant --- tests/agent/test_vertex_adapter.py | 4 ++-- tests/hermes_cli/test_vertex_provider.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/agent/test_vertex_adapter.py b/tests/agent/test_vertex_adapter.py index 3ac17580664ea..fd5329e5341f7 100644 --- a/tests/agent/test_vertex_adapter.py +++ b/tests/agent/test_vertex_adapter.py @@ -101,8 +101,8 @@ def test_get_vertex_config_uses_adc_and_default_region(vertex_adapter): token, base = vertex_adapter.get_vertex_config() assert token == "ya29.FAKE" assert base == ( - "https://aiplatform.googleapis.com/v1beta1/projects/adc-project/" - "locations/global/endpoints/openapi" + "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/adc-project/" + "locations/us-central1/endpoints/openapi" ) diff --git a/tests/hermes_cli/test_vertex_provider.py b/tests/hermes_cli/test_vertex_provider.py index af67aacac2978..d4506f99b36fc 100644 --- a/tests/hermes_cli/test_vertex_provider.py +++ b/tests/hermes_cli/test_vertex_provider.py @@ -76,8 +76,8 @@ def test_resolve_runtime_provider_raises_autherror_when_unresolved(monkeypatch): with pytest.raises(AuthError) as exc: rp.resolve_runtime_provider(requested="vertex") msg = str(exc.value) + assert "API Key" in msg assert "OAuth2" in msg - assert "not a static API key" in msg def test_vertex_extra_body_thinking_config(): From 9c1391d00639282a7b629162c765dfecfa059235 Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 12:17:22 +0800 Subject: [PATCH 03/12] test(vertex): add 20 tests for API key auth and model discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New test coverage: - has_vertex_api_key() / resolve_vertex_api_key() — env var detection - build_vertex_api_key_base_url() — global, us-central1, europe-west4 - get_vertex_config() with API key — full config + precedence over ADC - Missing project ID error handling - has_vertex_credentials() via API key - GOOGLE_VERTEX_LOCATION vs VERTEX_REGION precedence - GOOGLE_VERTEX_PROJECT vs VERTEX_PROJECT_ID precedence - discover_vertex_models() — parse response, filter by generateContent - discover_vertex_models() — network errors, HTTP errors, malformed JSON - discover_vertex_models() — sort order --- tests/agent/test_vertex_adapter.py | 261 +++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) diff --git a/tests/agent/test_vertex_adapter.py b/tests/agent/test_vertex_adapter.py index fd5329e5341f7..d24f460291048 100644 --- a/tests/agent/test_vertex_adapter.py +++ b/tests/agent/test_vertex_adapter.py @@ -230,3 +230,264 @@ def test_adc_failure_falls_back_to_service_account(monkeypatch, tmp_path): token, project = va.get_vertex_credentials() assert token == "ya29.FAKE" assert project == "sa-project" + + +# ── API Key (Express Mode) tests ───────────────────────────────────────────── + + +def test_has_vertex_api_key_true_when_env_set(vertex_adapter, monkeypatch): + """has_vertex_api_key returns True when GOOGLE_VERTEX_API_KEY is set.""" + monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyFakeKey123") + assert vertex_adapter.has_vertex_api_key() is True + + +def test_has_vertex_api_key_false_when_not_set(vertex_adapter): + """has_vertex_api_key returns False when the env var is absent.""" + assert vertex_adapter.has_vertex_api_key() is False + + +def test_resolve_vertex_api_key_returns_value(vertex_adapter, monkeypatch): + """resolve_vertex_api_key returns the env var value.""" + monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyTestKey456") + assert vertex_adapter.resolve_vertex_api_key() == "AIzaSyTestKey456" + + +def test_resolve_vertex_api_key_returns_none_when_not_set(vertex_adapter): + """resolve_vertex_api_key returns None when env var is absent.""" + assert vertex_adapter.resolve_vertex_api_key() is None + + +def test_build_vertex_api_key_base_url_global(vertex_adapter): + """Express Mode global endpoint uses aiplatform.googleapis.com.""" + url = vertex_adapter.build_vertex_api_key_base_url("my-project", "global") + assert url == ( + "https://aiplatform.googleapis.com/v1beta1/projects/my-project/" + "locations/global/endpoints/openapi" + ) + + +def test_build_vertex_api_key_base_url_regional(vertex_adapter): + """Express Mode regional endpoint uses {region}-aiplatform.googleapis.com.""" + url = vertex_adapter.build_vertex_api_key_base_url("my-project", "us-central1") + assert url == ( + "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/my-project/" + "locations/us-central1/endpoints/openapi" + ) + + +def test_build_vertex_api_key_base_url_europe(vertex_adapter): + """Express Mode with europe-west4 region.""" + url = vertex_adapter.build_vertex_api_key_base_url("my-project", "europe-west4") + assert url == ( + "https://europe-west4-aiplatform.googleapis.com/v1beta1/projects/my-project/" + "locations/europe-west4/endpoints/openapi" + ) + + +def test_get_vertex_config_with_api_key(vertex_adapter, monkeypatch): + """get_vertex_config returns (api_key, base_url) when API key is set.""" + monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyApiKey") + monkeypatch.setenv("GOOGLE_VERTEX_PROJECT", "api-key-project") + monkeypatch.setenv("GOOGLE_VERTEX_LOCATION", "europe-west1") + + token_or_key, base_url = vertex_adapter.get_vertex_config() + assert token_or_key == "AIzaSyApiKey" + assert "projects/api-key-project" in base_url + assert "europe-west1-aiplatform.googleapis.com" in base_url + assert "locations/europe-west1" in base_url + + +def test_get_vertex_config_api_key_precedence_over_adc(vertex_adapter, monkeypatch): + """API key path is used when BOTH API key and ADC credentials are available.""" + monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyKey") + monkeypatch.setenv("GOOGLE_VERTEX_PROJECT", "key-project") + + token_or_key, base_url = vertex_adapter.get_vertex_config() + assert token_or_key == "AIzaSyKey" # API key, not OAuth token + assert "projects/key-project" in base_url + + +def test_get_vertex_config_api_key_missing_project(vertex_adapter, monkeypatch): + """get_vertex_config returns (None, None) when API key is set but project is not.""" + monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyKey") + # No project ID set anywhere + + result = vertex_adapter.get_vertex_config() + assert result == (None, None) + + +def test_has_vertex_credentials_via_api_key(vertex_adapter, monkeypatch): + """has_vertex_credentials returns True when only the API key is set.""" + monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyKey") + assert vertex_adapter.has_vertex_credentials() is True + + +def test_googole_vertex_location_region_precedence(vertex_adapter, monkeypatch): + """GOOGLE_VERTEX_LOCATION takes precedence over VERTEX_REGION.""" + monkeypatch.setenv("GOOGLE_VERTEX_LOCATION", "us-west1") + monkeypatch.setenv("VERTEX_REGION", "europe-west4") + + assert vertex_adapter._resolve_region() == "us-west1" + + +def test_googole_vertex_project_precedence(vertex_adapter, monkeypatch): + """GOOGLE_VERTEX_PROJECT takes precedence over VERTEX_PROJECT_ID.""" + monkeypatch.setenv("GOOGLE_VERTEX_PROJECT", "gv-project") + monkeypatch.setenv("VERTEX_PROJECT_ID", "legacy-project") + + assert vertex_adapter._resolve_project_override() == "gv-project" + + +def test_googole_vertex_location_falls_back_to_vertex_region(vertex_adapter, monkeypatch): + """VERTEX_REGION is used when GOOGLE_VERTEX_LOCATION is not set.""" + monkeypatch.setenv("VERTEX_REGION", "asia-east1") + + assert vertex_adapter._resolve_region() == "asia-east1" + + +# ── Model Discovery Tests ──────────────────────────────────────────────────── + + +class _FakeUrlopenResult: + """Mimics the result of urllib.request.urlopen.""" + def __init__(self, data: bytes): + self._data = data + + def read(self) -> bytes: + return self._data + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +def test_discover_vertex_models_parses_response(vertex_adapter, monkeypatch): + """discover_vertex_models correctly parses a models.list response.""" + response_data = { + "models": [ + { + "name": "projects/p/locations/us-central1/publishers/google/models/gemini-2.5-flash", + "displayName": "Gemini 2.5 Flash", + "supportedGenerationMethods": ["generateContent", "countTokens"], + }, + { + "name": "projects/p/locations/us-central1/publishers/google/models/gemini-3-pro-preview", + "displayName": "Gemini 3 Pro Preview", + "supportedGenerationMethods": ["generateContent"], + }, + { + "name": "projects/p/locations/us-central1/publishers/google/models/gemini-embedding-001", + "displayName": "Gemini Embedding 001", + "supportedGenerationMethods": ["embedding"], + }, + ] + } + import urllib.request + import json + + def fake_urlopen(req, timeout=10): + return _FakeUrlopenResult(json.dumps(response_data).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + models = vertex_adapter.discover_vertex_models("AIzaSyKey", "my-project", "us-central1") + assert models == ["gemini-2.5-flash", "gemini-3-pro-preview"] + + +def test_discover_vertex_models_empty_when_no_generate_content(vertex_adapter, monkeypatch): + """Models without generateContent are excluded from discovery.""" + response_data = { + "models": [ + { + "name": "projects/p/locations/us-central1/publishers/google/models/textembedding-gecko", + "displayName": "Gecko", + "supportedGenerationMethods": ["embedding"], + }, + ] + } + import urllib.request + import json + + def fake_urlopen(req, timeout=10): + return _FakeUrlopenResult(json.dumps(response_data).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + models = vertex_adapter.discover_vertex_models("AIzaSyKey", "my-project", "us-central1") + assert models == [] + + +def test_discover_vertex_models_network_failure_returns_empty(vertex_adapter, monkeypatch): + """Network errors during discovery return an empty list without crashing.""" + import urllib.error + + def fake_urlopen(req, timeout=10): + raise urllib.error.URLError("Connection refused") + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + models = vertex_adapter.discover_vertex_models("AIzaSyKey", "my-project", "us-central1") + assert models == [] + + +def test_discover_vertex_models_http_error_returns_empty(vertex_adapter, monkeypatch): + """HTTP 4xx/5xx during discovery return an empty list.""" + import urllib.error + + def fake_urlopen(req, timeout=10): + raise urllib.error.HTTPError( + url=req.full_url if hasattr(req, 'full_url') else "", + code=403, + msg="Forbidden", + hdrs={}, + fp=None, + ) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + models = vertex_adapter.discover_vertex_models("AIzaSyKey", "my-project", "us-central1") + assert models == [] + + +def test_discover_vertex_models_malformed_json_returns_empty(vertex_adapter, monkeypatch): + """Malformed JSON responses return an empty list.""" + import urllib.request + + def fake_urlopen(req, timeout=10): + return _FakeUrlopenResult(b"not json at all") + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + models = vertex_adapter.discover_vertex_models("AIzaSyKey", "my-project", "us-central1") + assert models == [] + + +def test_discover_vertex_models_sorts_results(vertex_adapter, monkeypatch): + """discover_vertex_models returns sorted model IDs.""" + response_data = { + "models": [ + { + "name": "projects/p/locations/us-central1/publishers/google/models/gemini-3-pro-preview", + "displayName": "Gemini 3 Pro Preview", + "supportedGenerationMethods": ["generateContent"], + }, + { + "name": "projects/p/locations/us-central1/publishers/google/models/gemini-2.5-flash", + "displayName": "Gemini 2.5 Flash", + "supportedGenerationMethods": ["generateContent"], + }, + ] + } + import urllib.request + import json + + def fake_urlopen(req, timeout=10): + return _FakeUrlopenResult(json.dumps(response_data).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + models = vertex_adapter.discover_vertex_models("AIzaSyKey", "my-project", "us-central1") + assert models == ["gemini-2.5-flash", "gemini-3-pro-preview"] # sorted + assert models == sorted(models) From eb94367d3b11505e554cbb5222df9e6d6172e112 Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 12:24:47 +0800 Subject: [PATCH 04/12] fix(vertex): use x-goog-api-key header for API key (Express Mode) auth Vertex API Express Mode requires the key sent as x-goog-api-key header, not Authorization: Bearer (which only works for OAuth2 tokens). - get_vertex_config() now returns 3-tuple: (key, base_url, auth_header_type) where auth_header_type is 'x-goog-api-key' for API key mode or 'Authorization' for OAuth2/ADC mode - runtime_provider.py propagates auth_header to downstream client - Tests updated for 3-tuple return and new env var isolation - Model discovery via publishers/google/models confirmed NOT available with API key auth (returns 404). Falls back to curated list. --- agent/vertex_adapter.py | 20 ++++++++-------- hermes_cli/runtime_provider.py | 11 +++++++-- tests/agent/test_vertex_adapter.py | 29 ++++++++++++++++-------- tests/hermes_cli/test_vertex_provider.py | 9 +++++--- 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/agent/vertex_adapter.py b/agent/vertex_adapter.py index 86f7012f3d92a..2fc0ffd68560f 100644 --- a/agent/vertex_adapter.py +++ b/agent/vertex_adapter.py @@ -270,18 +270,20 @@ def build_vertex_base_url(project_id: str, region: str = DEFAULT_REGION) -> str: def get_vertex_config( credentials_path: Optional[str] = None, region: Optional[str] = None, -) -> Tuple[Optional[str], Optional[str]]: - """Resolve (access_token_or_api_key, base_url) for Vertex AI. +) -> Tuple[Optional[str], Optional[str], Optional[str]]: + """Resolve (access_token_or_api_key, base_url, auth_header_type) for Vertex AI. Two authentication paths, chosen automatically: 1. **API Key (Express Mode)** — if ``GOOGLE_VERTEX_API_KEY`` is set. - Returns (api_key, base_url_with_project). No ``google-auth`` needed. + Returns (api_key, base_url_with_project, ``"x-goog-api-key"``). + No ``google-auth`` needed. The caller must set the returned header + name (not ``Authorization: Bearer``) on each request. - 2. **OAuth2 / ADC** — legacy path. Returns (oauth2_token, base_url). + 2. **OAuth2 / ADC** — legacy path. Returns (oauth2_token, base_url, ``"Authorization"``). Requires ``google-auth`` and valid GCP credentials. - Returns (None, None) when no credentials can be resolved. + Returns (None, None, None) when no credentials can be resolved. """ # --- Path 1: API Key (Express Mode) --- api_key = resolve_vertex_api_key() @@ -292,23 +294,23 @@ def get_vertex_config( "Vertex API key found but no project ID configured. " "Set GOOGLE_VERTEX_PROJECT in ~/.hermes/.env." ) - return None, None + return None, None, None effective_region = _resolve_region(region) base_url = build_vertex_api_key_base_url(project_id, effective_region) logger.debug( "get_vertex_config: using API key (Express Mode) for project %s in %s", project_id, effective_region, ) - return api_key, base_url + return api_key, base_url, "x-goog-api-key" # --- Path 2: OAuth2 / ADC (legacy) --- token, project_id = get_vertex_credentials(credentials_path) if not token or not project_id: - return None, None + return None, None, None effective_region = _resolve_region(region) base_url = build_vertex_base_url(project_id, effective_region) - return token, base_url + return token, base_url, "Authorization" def has_vertex_credentials() -> bool: diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index cacb7078c4307..8fc9dd745c8a2 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -1627,7 +1627,7 @@ def resolve_runtime_provider( has_vertex_api_key, ) - token_or_key, base_url = get_vertex_config() + token_or_key, base_url, auth_header = get_vertex_config() if not token_or_key or not base_url: raise AuthError( "Vertex AI credentials could not be resolved.\n\n" @@ -1641,7 +1641,7 @@ def resolve_runtime_provider( " Set the GCP project/region under vertex: in config.yaml." ) source = "vertex-api-key" if has_vertex_api_key() else "vertex-oauth" - return { + result = { "provider": "vertex", "api_mode": "chat_completions", "base_url": base_url.rstrip("/"), @@ -1649,6 +1649,13 @@ def resolve_runtime_provider( "source": source, "requested_provider": requested_provider, } + # API key auth uses ``x-goog-api-key`` header instead of ``Authorization: Bearer``. + # Propagate this to the downstream OpenAI client constructor so it knows + # to set the correct auth header. The Hermes client factory reads + # ``auth_header`` from the runtime dict and configures the transport accordingly. + if auth_header and auth_header != "Authorization": + result["auth_header"] = auth_header + return result custom_runtime = _resolve_named_custom_runtime( requested_provider=requested_provider, diff --git a/tests/agent/test_vertex_adapter.py b/tests/agent/test_vertex_adapter.py index d24f460291048..670aedac72e19 100644 --- a/tests/agent/test_vertex_adapter.py +++ b/tests/agent/test_vertex_adapter.py @@ -70,7 +70,9 @@ def from_service_account_file(path, scopes=None): def vertex_adapter(monkeypatch): """Fresh vertex_adapter with a fake google-auth and clean caches/env.""" for var in ("VERTEX_CREDENTIALS_PATH", "GOOGLE_APPLICATION_CREDENTIALS", - "VERTEX_PROJECT_ID", "VERTEX_REGION", "GOOGLE_CLOUD_PROJECT"): + "VERTEX_PROJECT_ID", "VERTEX_REGION", "GOOGLE_CLOUD_PROJECT", + "GOOGLE_VERTEX_API_KEY", "GOOGLE_VERTEX_PROJECT", + "GOOGLE_VERTEX_LOCATION"): monkeypatch.delenv(var, raising=False) _install_fake_google_auth(monkeypatch) import agent.vertex_adapter as va @@ -98,8 +100,9 @@ def test_build_base_url_regional(vertex_adapter): def test_get_vertex_config_uses_adc_and_default_region(vertex_adapter): - token, base = vertex_adapter.get_vertex_config() + token, base, auth_hdr = vertex_adapter.get_vertex_config() assert token == "ya29.FAKE" + assert auth_hdr == "Authorization" assert base == ( "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/adc-project/" "locations/us-central1/endpoints/openapi" @@ -111,8 +114,9 @@ def test_config_yaml_supplies_project_and_region(vertex_adapter, monkeypatch): vertex_adapter, "_vertex_config", lambda: {"project_id": "cfg-project", "region": "europe-west4"}, ) - token, base = vertex_adapter.get_vertex_config() + token, base, auth_hdr = vertex_adapter.get_vertex_config() assert token == "ya29.FAKE" + assert auth_hdr == "Authorization" assert "projects/cfg-project" in base assert "europe-west4-aiplatform.googleapis.com" in base assert "locations/europe-west4" in base @@ -207,14 +211,17 @@ def test_adc_refuses_foreign_profile_google_application_credentials( def test_adc_still_works_when_not_multiplexed(vertex_adapter): """Single-profile (non-gateway) installs must see zero behavior change: ADC still resolves normally when multiplexing is off, scope or not.""" - token, base = vertex_adapter.get_vertex_config() + token, base, auth_hdr = vertex_adapter.get_vertex_config() assert token == "ya29.FAKE" + assert auth_hdr == "Authorization" assert "adc-project" in base def test_adc_failure_falls_back_to_service_account(monkeypatch, tmp_path): """When ADC refresh fails but a service-account JSON exists, use the SA.""" - for var in ("VERTEX_PROJECT_ID", "VERTEX_REGION", "GOOGLE_CLOUD_PROJECT"): + for var in ("VERTEX_PROJECT_ID", "VERTEX_REGION", "GOOGLE_CLOUD_PROJECT", + "GOOGLE_VERTEX_PROJECT", "GOOGLE_VERTEX_API_KEY", + "GOOGLE_VERTEX_LOCATION"): monkeypatch.delenv(var, raising=False) sa_file = tmp_path / "sa.json" sa_file.write_text('{"project_id": "sa-project"}') @@ -285,13 +292,14 @@ def test_build_vertex_api_key_base_url_europe(vertex_adapter): def test_get_vertex_config_with_api_key(vertex_adapter, monkeypatch): - """get_vertex_config returns (api_key, base_url) when API key is set.""" + """get_vertex_config returns (api_key, base_url, x-goog-api-key) when API key is set.""" monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyApiKey") monkeypatch.setenv("GOOGLE_VERTEX_PROJECT", "api-key-project") monkeypatch.setenv("GOOGLE_VERTEX_LOCATION", "europe-west1") - token_or_key, base_url = vertex_adapter.get_vertex_config() + token_or_key, base_url, auth_hdr = vertex_adapter.get_vertex_config() assert token_or_key == "AIzaSyApiKey" + assert auth_hdr == "x-goog-api-key" assert "projects/api-key-project" in base_url assert "europe-west1-aiplatform.googleapis.com" in base_url assert "locations/europe-west1" in base_url @@ -302,18 +310,19 @@ def test_get_vertex_config_api_key_precedence_over_adc(vertex_adapter, monkeypat monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyKey") monkeypatch.setenv("GOOGLE_VERTEX_PROJECT", "key-project") - token_or_key, base_url = vertex_adapter.get_vertex_config() + token_or_key, base_url, auth_hdr = vertex_adapter.get_vertex_config() assert token_or_key == "AIzaSyKey" # API key, not OAuth token + assert auth_hdr == "x-goog-api-key" assert "projects/key-project" in base_url def test_get_vertex_config_api_key_missing_project(vertex_adapter, monkeypatch): - """get_vertex_config returns (None, None) when API key is set but project is not.""" + """get_vertex_config returns (None, None, None) when API key is set but project is not.""" monkeypatch.setenv("GOOGLE_VERTEX_API_KEY", "AIzaSyKey") # No project ID set anywhere result = vertex_adapter.get_vertex_config() - assert result == (None, None) + assert result == (None, None, None) def test_has_vertex_credentials_via_api_key(vertex_adapter, monkeypatch): diff --git a/tests/hermes_cli/test_vertex_provider.py b/tests/hermes_cli/test_vertex_provider.py index d4506f99b36fc..642d8629e8626 100644 --- a/tests/hermes_cli/test_vertex_provider.py +++ b/tests/hermes_cli/test_vertex_provider.py @@ -46,9 +46,10 @@ def test_resolve_runtime_provider_mints_token(monkeypatch): import agent.vertex_adapter as va from hermes_cli import runtime_provider as rp + monkeypatch.setattr(va, "has_vertex_api_key", lambda: False) monkeypatch.setattr( va, "get_vertex_config", - lambda: ("ya29.TOKEN", "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi"), + lambda: ("ya29.TOKEN", "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi", "Authorization"), ) rt = rp.resolve_runtime_provider(requested="vertex") assert rt["provider"] == "vertex" @@ -56,13 +57,15 @@ def test_resolve_runtime_provider_mints_token(monkeypatch): assert rt["source"] == "vertex-oauth" assert rt["api_key"] == "ya29.TOKEN" assert "aiplatform.googleapis.com" in rt["base_url"] + assert "auth_header" not in rt # Authorization is the default def test_resolve_runtime_provider_alias(monkeypatch): import agent.vertex_adapter as va from hermes_cli import runtime_provider as rp - monkeypatch.setattr(va, "get_vertex_config", lambda: ("t", "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi")) + monkeypatch.setattr(va, "has_vertex_api_key", lambda: False) + monkeypatch.setattr(va, "get_vertex_config", lambda: ("t", "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/endpoints/openapi", "Authorization")) rt = rp.resolve_runtime_provider(requested="google-vertex") assert rt["provider"] == "vertex" @@ -72,7 +75,7 @@ def test_resolve_runtime_provider_raises_autherror_when_unresolved(monkeypatch): from hermes_cli import runtime_provider as rp from hermes_cli.auth import AuthError - monkeypatch.setattr(va, "get_vertex_config", lambda: (None, None)) + monkeypatch.setattr(va, "get_vertex_config", lambda: (None, None, None)) with pytest.raises(AuthError) as exc: rp.resolve_runtime_provider(requested="vertex") msg = str(exc.value) From 8d8999374e8f6d08eb32f6dacf59fba0809858b9 Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 12:28:20 +0800 Subject: [PATCH 05/12] docs(vertex): clarify model discovery limitation with API key auth - discover_vertex_models() docstring: explain that publishers/google/models returns 404 with API key (Express Mode); only works with OAuth2/ADC - model_setup_flows.py: improve UX messages when discovery is unavailable - Live test confirmed: inference with x-goog-api-key returns 200 OK on both native generateContent and OpenAI-compatible endpoints --- agent/vertex_adapter.py | 15 ++++++++++----- hermes_cli/model_setup_flows.py | 6 ++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/agent/vertex_adapter.py b/agent/vertex_adapter.py index 2fc0ffd68560f..c05db3f88960e 100644 --- a/agent/vertex_adapter.py +++ b/agent/vertex_adapter.py @@ -345,12 +345,17 @@ def discover_vertex_models( """Query Vertex AI's ``models.list`` publisher endpoint for models available in the given project and region. - Returns a sorted list of model ID strings (e.g. ``gemini-2.5-flash``, - ``gemini-3-pro-preview``). Only models that support ``generateContent`` - (chat / text-generation) are returned. + **Note:** The ``publishers/google/models`` endpoint is only accessible + via OAuth2 / ADC authentication. When using an API key (Express Mode), + this endpoint returns 404. The function will return an empty list with + API key auth, and the caller should fall back to a curated model list. - Returns the (sorted) model list on success. - Returns an empty list on any error (network, auth, parse). + For OAuth2 / ADC auth, returns a sorted list of model ID strings + (e.g. ``gemini-2.5-flash``, ``gemini-3-flash-preview``). Only models + that support ``generateContent`` (chat / text-generation) are returned. + + Returns the sorted model list on success. + Returns an empty list on any error (network, auth, parse, or API key auth). """ import json import urllib.error diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 3564f838eb44a..11cd1ff799607 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -2524,9 +2524,11 @@ def _model_flow_vertex(config, current_model=""): if discovered: print(f"found {len(discovered)} models ✓") else: - print("failed — using curated list.") + print("not available with API keys — using curated list.") + print(" (Model listing requires OAuth2/ADC, not API key.)") else: - print(" (Model discovery requires API key + project ID; using curated list.)") + print(" (Using curated model list — set GOOGLE_VERTEX_API_KEY +") + print(" GOOGLE_VERTEX_PROJECT for dynamic discovery.)") if discovered: # Prefix with ``google/`` to match Hermes model naming convention From 19a38053a6196020f38513a180bda7653cce215e Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 13:20:21 +0800 Subject: [PATCH 06/12] feat(vertex): update doctor, web dashboard, and auxiliary client for API key auth UI/UX changes: - doctor.py: add GOOGLE_VERTEX_API_KEY, GOOGLE_VERTEX_PROJECT, GOOGLE_VERTEX_LOCATION to _PROVIDER_ENV_HINTS so `hermes doctor` reports them. Add Vertex to the API key provider health-check list (no /models probe since Express Mode has no list method). - web_server.py: add GOOGLE_VERTEX_API_KEY, GOOGLE_VERTEX_PROJECT, GOOGLE_VERTEX_LOCATION as visible env vars in the web dashboard Keys tab under the Vertex provider card. - auxiliary_client.py: update resolve_provider_client to use the 3-tuple return from get_vertex_config(). When auth_header is x-goog-api-key, create the OpenAI client with default_headers instead of Authorization: Bearer. --- agent/auxiliary_client.py | 23 ++++++++++++++++++++--- hermes_cli/doctor.py | 7 +++++++ hermes_cli/web_server.py | 16 ++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index da49a695180a5..983eef57654d7 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -5314,8 +5314,13 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", "no GCP credentials found") return None, None - token, base_url = get_vertex_config() - if not token or not base_url: + from agent.vertex_adapter import ( + get_vertex_config, + has_vertex_api_key, + ) + + token_or_key, base_url, auth_header = get_vertex_config() + if not token_or_key or not base_url: logger.warning("resolve_provider_client: vertex requested but " "could not mint token / resolve project") return None, None @@ -5324,7 +5329,19 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", final_model = _normalize_resolved_model(model or default_model, provider) try: from openai import OpenAI - client = OpenAI(api_key=token, base_url=base_url) + + if auth_header == "x-goog-api-key": + # Express Mode — use the API key as x-goog-api-key header + # instead of Authorization: Bearer (which only works for + # OAuth2 tokens). + client = OpenAI( + api_key="", + base_url=base_url, + default_headers={"x-goog-api-key": token_or_key}, + ) + else: + # Standard OAuth2 — pass token as Authorization: Bearer + client = OpenAI(api_key=token_or_key, base_url=base_url) except Exception as exc: logger.warning("resolve_provider_client: cannot create Vertex " "client: %s", exc) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 68338bfe89e40..3d178f9f85ef6 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -54,6 +54,9 @@ "OPENCODE_GO_API_KEY", "XIAOMI_API_KEY", "TOKENHUB_API_KEY", + "GOOGLE_VERTEX_API_KEY", + "GOOGLE_VERTEX_PROJECT", + "GOOGLE_VERTEX_LOCATION", ) @@ -513,6 +516,10 @@ def _build_apikey_providers_list() -> list: ("OpenCode Zen", ("OPENCODE_ZEN_API_KEY",), "https://opencode.ai/zen/v1/models", "OPENCODE_ZEN_BASE_URL", True), # OpenCode Go has no shared /models endpoint; skip the health check. ("OpenCode Go", ("OPENCODE_GO_API_KEY",), None, "OPENCODE_GO_BASE_URL", False), + # Vertex AI (Express Mode) uses ``x-goog-api-key`` header, not + # Authorization: Bearer. No /models endpoint available with API key auth + # (Google Express Mode only exposes generateContent/streamGenerateContent). + ("Google Vertex AI", ("GOOGLE_VERTEX_API_KEY",), None, None, False), ] _known_names = {t[0] for t in _static} # Also index by profile canonical name so profiles without display_name diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 82d2c0da67846..0bf38147be21b 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7062,6 +7062,22 @@ def _catalog_provider_env_metadata() -> dict: "advanced": existing.get("advanced", True), "category": "provider", } + # API key (Express Mode) env vars — shown when using API key auth. + for api_key_env, desc in ( + ("GOOGLE_VERTEX_API_KEY", "Vertex AI API key (Express Mode)"), + ("GOOGLE_VERTEX_PROJECT", "Vertex AI GCP project ID"), + ("GOOGLE_VERTEX_LOCATION", "Vertex AI region (default: us-central1)"), + ): + ek = meta.get(api_key_env, {}) + meta[api_key_env] = { + "provider": d.slug, + "provider_label": d.label, + "description": ek.get("description") or desc, + "url": ek.get("url"), + "is_password": api_key_env == "GOOGLE_VERTEX_API_KEY", + "advanced": ek.get("advanced", False), + "category": "provider", + } return meta From 0c96cd3b19094b62193a618ac18ad3d83ab6b751 Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 13:26:49 +0800 Subject: [PATCH 07/12] feat(desktop): add Google Vertex AI provider card in Settings > Keys Add GOOGLE_VERTEX_ prefix to PROVIDER_GROUPS so the Vertex AI API key env vars (GOOGLE_VERTEX_API_KEY, GOOGLE_VERTEX_PROJECT, GOOGLE_VERTEX_LOCATION) render as their own "Google Vertex AI" card in the Desktop settings Keys tab, separate from the Gemini / Google AI Studio card. Longest-prefix matching ensures GOOGLE_VERTEX_ (14 chars) wins over GOOGLE_ (7 chars) for vertex-prefixed variables. --- apps/desktop/src/app/settings/constants.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/desktop/src/app/settings/constants.ts b/apps/desktop/src/app/settings/constants.ts index 055f72ac19a11..6ee91de1d95dd 100644 --- a/apps/desktop/src/app/settings/constants.ts +++ b/apps/desktop/src/app/settings/constants.ts @@ -82,6 +82,13 @@ export const PROVIDER_GROUPS: ProviderPrefix[] = [ docsUrl: 'https://aistudio.google.com/app/apikey', priority: 4 }, + { + prefix: 'GOOGLE_VERTEX_', + name: 'Google Vertex AI', + description: 'Vertex AI (Gemini via GCP; API key or OAuth2/ADC)', + docsUrl: 'https://console.cloud.google.com/apis/credentials', + priority: 4 + }, { prefix: 'GEMINI_', name: 'Gemini', priority: 4 }, { prefix: 'DEEPSEEK_', From f8002e228e4ed6edca13fb7c1f9f0beb3159a3de Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 15:36:43 +0800 Subject: [PATCH 08/12] fix(vertex): propagate auth_header through switch_model chain for x-goog-api-key support The auth_header from resolve_runtime_provider() was being discarded in model_switch.py, causing the main OpenAI client to send the Vertex API key as Authorization: Bearer instead of x-goog-api-key, resulting in HTTP 401. Changes: - hermes_cli/model_switch.py: add auth_header to ModelSwitchResult + propagate from all 3 resolve_runtime_provider() call sites - cli.py: pass auth_header to agent.switch_model() in all 3 call sites - run_agent.py switch_model(): accept and forward auth_header - agent/agent_runtime_helpers.py switch_model(): set default_headers with x-goog-api-key when auth_header indicates Express Mode - run_agent.py _try_refresh_vertex_client_credentials(): handle 3-tuple return from get_vertex_config() and set default_headers on credential refresh - run_agent.py _apply_client_headers_for_base_url(): add Vertex/aiplatform branch that detects API key mode via has_vertex_api_key() --- agent/agent_runtime_helpers.py | 8 +++++- cli.py | 3 ++ hermes_cli/model_switch.py | 7 +++++ run_agent.py | 52 ++++++++++++++++++++++++---------- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 263cf1563a27e..aee3db7190df6 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1940,7 +1940,7 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo return client -def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mode=''): +def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mode='', auth_header=''): """Switch the model/provider in-place for a live agent. Called by the /model command handlers (CLI and gateway) after @@ -2140,6 +2140,12 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo "api_key": effective_key, "base_url": effective_base, } + # Vertex API key (Express Mode) uses x-goog-api-key header instead + # of Authorization: Bearer *** only works for OAuth2 tokens). + if auth_header == "x-goog-api-key": + agent._client_kwargs["default_headers"] = { + "x-goog-api-key": effective_key, + } try: from hermes_cli.config import ( apply_custom_provider_tls_to_client_kwargs, diff --git a/cli.py b/cli.py index b3e1c2fb8cb16..a16f9ee29f336 100644 --- a/cli.py +++ b/cli.py @@ -7318,6 +7318,7 @@ def new_session(self, silent=False, title=None): api_key=_reset_result.api_key, base_url=_reset_result.base_url, api_mode=_reset_result.api_mode, + auth_header=getattr(_reset_result, 'auth_header', ''), ) self.model = _reset_result.new_model self.provider = _reset_result.target_provider @@ -8227,6 +8228,7 @@ def _apply_model_switch_result(self, result, persist_global: bool) -> None: api_key=result.api_key, base_url=result.base_url, api_mode=result.api_mode, + auth_header=getattr(result, 'auth_header', ''), ) except Exception as exc: # The agent rolled itself back to the old working model/client. @@ -8564,6 +8566,7 @@ def _handle_model_switch(self, cmd_original: str): api_key=result.api_key, base_url=result.base_url, api_mode=result.api_mode, + auth_header=getattr(result, 'auth_header', ''), ) except Exception as exc: # Agent rolled itself back; roll the CLI back too and abort so a diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index a86b1316e99d7..c7f483e541583 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -450,6 +450,7 @@ class ModelSwitchResult: capabilities: Optional[ModelCapabilities] = None model_info: Optional[ModelInfo] = None is_global: bool = False + auth_header: str = "" @dataclass(frozen=True) @@ -1322,6 +1323,7 @@ def switch_model( api_key = current_api_key base_url = current_base_url api_mode = "" + auth_header = "" if provider_changed or explicit_provider: import os @@ -1356,10 +1358,12 @@ def switch_model( api_key = runtime.get("api_key", "") or _ukey base_url = runtime.get("base_url", "") or _user_pdef.base_url api_mode = runtime.get("api_mode", "") + auth_header = runtime.get("auth_header", "") except Exception: api_key = _ukey base_url = _user_pdef.base_url api_mode = "" + auth_header = "" elif target_provider == "custom" and current_base_url: api_key = current_api_key base_url = current_base_url @@ -1373,6 +1377,7 @@ def switch_model( api_key = runtime.get("api_key", "") base_url = runtime.get("base_url", "") api_mode = runtime.get("api_mode", "") + auth_header = runtime.get("auth_header", "") except Exception as e: return ModelSwitchResult( success=False, @@ -1397,6 +1402,7 @@ def switch_model( api_key = runtime.get("api_key", "") base_url = runtime.get("base_url", "") api_mode = runtime.get("api_mode", "") + auth_header = runtime.get("auth_header", "") except Exception: pass @@ -1552,6 +1558,7 @@ def switch_model( capabilities=capabilities, model_info=model_info, is_global=is_global, + auth_header=auth_header, ) diff --git a/run_agent.py b/run_agent.py index 6c13f737c8619..242d1b6227895 100644 --- a/run_agent.py +++ b/run_agent.py @@ -809,10 +809,10 @@ def _ensure_lmstudio_runtime_loaded(self, config_context_length: Optional[int] = except Exception as err: logger.debug("LM Studio preload skipped: %s", err) - def switch_model(self, new_model, new_provider, api_key='', base_url='', api_mode=''): + def switch_model(self, new_model, new_provider, api_key='', base_url='', api_mode='', auth_header=''): """Forwarder — see ``agent.agent_runtime_helpers.switch_model``.""" from agent.agent_runtime_helpers import switch_model - return switch_model(self, new_model, new_provider, api_key, base_url, api_mode) + return switch_model(self, new_model, new_provider, api_key, base_url, api_mode, auth_header) def _safe_print(self, *args, **kwargs): """Print that silently handles broken pipes / closed stdout. @@ -4534,40 +4534,50 @@ def _try_refresh_nous_client_credentials( return True def _try_refresh_vertex_client_credentials(self) -> bool: - """Re-mint the Vertex OAuth2 access token and rebuild the OpenAI client. - - Vertex tokens live ~1 hour. On a long-lived agent (gateway session) a - cached client's bearer token will expire mid-session, producing a 401. - This re-resolves credentials via the adapter (which refreshes the - underlying google-auth Credentials object when near expiry), swaps the - new token into the client kwargs, and rebuilds the primary OpenAI - client. Returns True when a usable token+base_url were obtained. + """Re-mint credentials and rebuild the OpenAI client for Vertex. + + Two auth modes: + - API key (Express Mode): static key sent via x-goog-api-key header. + - OAuth2 / ADC: short-lived access token sent via Authorization: Bearer. + + For OAuth2, the token is re-minted per call (5-min refresh margin) by + get_vertex_config(); mid-session expiry is additionally recovered on 401. + Returns True when a usable credential+base_url were obtained. """ if self.api_mode != "chat_completions" or self.provider != "vertex": return False try: - from agent.vertex_adapter import get_vertex_config + from agent.vertex_adapter import get_vertex_config, has_vertex_api_key - token, base_url = get_vertex_config() + token_or_key, base_url, auth_header = get_vertex_config() except Exception as exc: logger.debug("Vertex credential refresh failed: %s", exc) return False - if not isinstance(token, str) or not token.strip(): + if not isinstance(token_or_key, str) or not token_or_key.strip(): return False if not isinstance(base_url, str) or not base_url.strip(): return False - self.api_key = token.strip() + self.api_key = token_or_key.strip() self.base_url = base_url.strip().rstrip("/") self._client_kwargs["api_key"] = self.api_key self._client_kwargs["base_url"] = self.base_url + # API key mode uses x-goog-api-key header; OAuth2 uses Authorization: Bearer + if auth_header == "x-goog-api-key": + self._client_kwargs["default_headers"] = {"x-goog-api-key": self.api_key} + else: + self._client_kwargs.pop("default_headers", None) + if not self._replace_primary_openai_client(reason="vertex_credential_refresh"): return False - logger.info("Vertex AI OAuth token refreshed") + logger.info( + "Vertex AI %s refreshed", + "API key" if auth_header == "x-goog-api-key" else "OAuth token", + ) return True def _try_refresh_copilot_client_credentials(self) -> bool: @@ -4681,6 +4691,18 @@ def _apply_client_headers_for_base_url(self, base_url: str) -> None: self._client_kwargs["default_headers"] = _codex_cloudflare_headers( self._client_kwargs.get("api_key", "") ) + elif "aiplatform.googleapis.com" in (base_url or ""): + # Vertex AI API key (Express Mode) uses x-goog-api-key header instead + # of Authorization: Bearer. Only set this when the key is actually + # a Vertex API key (not an OAuth2 token). + if self.provider == "vertex" and self._client_kwargs.get("api_key", ""): + from agent.vertex_adapter import has_vertex_api_key + if has_vertex_api_key(): + self._client_kwargs["default_headers"] = { + "x-goog-api-key": self._client_kwargs["api_key"], + } + return + self._client_kwargs.pop("default_headers", None) else: # No URL-specific headers — check profile.default_headers before clearing. _ph_headers = None From 850417ac49535074dd8989bf91a9a2d1eda2152e Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 15:56:05 +0800 Subject: [PATCH 09/12] fix(vertex): switch API key auth to native generateContent endpoint Express Mode API keys don't work with Vertex's OpenAI-compatible /endpoints/openapi/chat/completions endpoint (returns 404). They only work with the native :generateContent API. Changes: - agent/vertex_adapter.py: build_vertex_api_key_base_url() now returns https://aiplatform.googleapis.com/v1/publishers/google for the native API, not the OpenAI-compatible endpoint URL. The project/region are embedded in the API key itself. - agent/gemini_native_adapter.py: is_native_gemini_base_url() now accepts aiplatform.googleapis.com endpoints (not just generativelanguage.googleapis.com), so Vertex Express Mode routes through GeminiNativeClient which already handles x-goog-api-key auth, format conversion, and bare model name stripping. - agent/auxiliary_client.py: _create_openai_client() detects native base URLs and routes through GeminiNativeClient for any provider, not just gemini. - agent/agent_runtime_helpers.py: create_openai_client() detects native base URLs by URL pattern instead of checking provider name, so both gemini and vertex reach GeminiNativeClient. - tests: updated URL assertions for the new native endpoint format. --- agent/agent_runtime_helpers.py | 6 ++---- agent/auxiliary_client.py | 8 ++++++++ agent/gemini_native_adapter.py | 8 ++++++-- agent/vertex_adapter.py | 18 +++++++++--------- tests/agent/test_vertex_adapter.py | 29 ++++++++++------------------- 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index aee3db7190df6..703a9754cc940 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1873,11 +1873,9 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo agent._client_log_context(), ) return client - if agent.provider == "gemini": - from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url + from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url - base_url = str(client_kwargs.get("base_url", "") or "") - if is_native_gemini_base_url(base_url): + if is_native_gemini_base_url(client_kwargs.get("base_url", "")): safe_kwargs = { k: v for k, v in client_kwargs.items() if k in {"api_key", "base_url", "default_headers", "timeout", "http_client"} diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 983eef57654d7..7ea1917b940ca 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -201,6 +201,14 @@ def _openai_http_client_kwargs( return {"http_client": client} def _create_openai_client(*, api_key: str, base_url: str, **kwargs: Any) -> Any: + # Gemini/Vertex native API uses GeminiNativeClient instead of OpenAI SDK + try: + from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url + + if is_native_gemini_base_url(base_url): + return GeminiNativeClient(api_key=api_key, base_url=base_url, **kwargs) + except Exception: + pass kwargs = {**_openai_http_client_kwargs(base_url), **kwargs} # Hermes owns auxiliary retry + provider/model fallback policy (the # same-provider transient retry in call_llm plus the except-chain diff --git a/agent/gemini_native_adapter.py b/agent/gemini_native_adapter.py index 1c25f1e6cf0a3..e504be597b755 100644 --- a/agent/gemini_native_adapter.py +++ b/agent/gemini_native_adapter.py @@ -60,11 +60,15 @@ def bare_gemini_model_id(model: str) -> str: def is_native_gemini_base_url(base_url: str) -> bool: - """Return True when the endpoint speaks Gemini's native REST API.""" + """Return True when the endpoint speaks Gemini's native REST API. + + Accepts both Google AI Studio (generativelanguage.googleapis.com) and + Vertex AI (aiplatform.googleapis.com) native endpoints. + """ normalized = str(base_url or "").strip().rstrip("/").lower() if not normalized: return False - if "generativelanguage.googleapis.com" not in normalized: + if "generativelanguage.googleapis.com" not in normalized and "aiplatform.googleapis.com" not in normalized: return False return not normalized.endswith("/openai") diff --git a/agent/vertex_adapter.py b/agent/vertex_adapter.py index c05db3f88960e..8b7e5a2c8d0e3 100644 --- a/agent/vertex_adapter.py +++ b/agent/vertex_adapter.py @@ -157,18 +157,18 @@ def resolve_vertex_api_key() -> Optional[str]: def build_vertex_api_key_base_url(project_id: str, region: str) -> str: - """Build the OpenAI-compatible base URL for Vertex AI Express Mode. + """Build the base URL for Vertex AI Express Mode native API. - Express Mode uses the standard aiplatform.googleapis.com host (or - ``{region}-aiplatform.googleapis.com`` for regional endpoints) with the - project in the URL path. The API key is passed as a Bearer token in the - Authorization header. + Express Mode uses the native ``generateContent`` API (not the + OpenAI-compatible endpoint). The URL points at the global + ``aiplatform.googleapis.com`` host with the ``publishers/google`` + prefix so that the ``GeminiNativeClient`` constructs the correct + ``:generateContent`` endpoint path. - The ``global`` location uses the bare ``aiplatform.googleapis.com`` host. - Regional locations use ``{region}-aiplatform.googleapis.com``. + The project and region are embedded in the API key itself — they + are not part of the URL in Express Mode native API calls. """ - host = "aiplatform.googleapis.com" if region == "global" else f"{region}-aiplatform.googleapis.com" - return f"https://{host}/v1beta1/projects/{project_id}/locations/{region}/endpoints/openapi" + return "https://aiplatform.googleapis.com/v1/publishers/google" def _refresh_credentials(creds) -> None: diff --git a/tests/agent/test_vertex_adapter.py b/tests/agent/test_vertex_adapter.py index 670aedac72e19..1696188d00ad6 100644 --- a/tests/agent/test_vertex_adapter.py +++ b/tests/agent/test_vertex_adapter.py @@ -265,30 +265,21 @@ def test_resolve_vertex_api_key_returns_none_when_not_set(vertex_adapter): def test_build_vertex_api_key_base_url_global(vertex_adapter): - """Express Mode global endpoint uses aiplatform.googleapis.com.""" + """Express Mode native API uses global aiplatform.googleapis.com.""" url = vertex_adapter.build_vertex_api_key_base_url("my-project", "global") - assert url == ( - "https://aiplatform.googleapis.com/v1beta1/projects/my-project/" - "locations/global/endpoints/openapi" - ) + assert url == "https://aiplatform.googleapis.com/v1/publishers/google" def test_build_vertex_api_key_base_url_regional(vertex_adapter): - """Express Mode regional endpoint uses {region}-aiplatform.googleapis.com.""" + """Express Mode native API ignores region — always uses global host.""" url = vertex_adapter.build_vertex_api_key_base_url("my-project", "us-central1") - assert url == ( - "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/my-project/" - "locations/us-central1/endpoints/openapi" - ) + assert url == "https://aiplatform.googleapis.com/v1/publishers/google" def test_build_vertex_api_key_base_url_europe(vertex_adapter): - """Express Mode with europe-west4 region.""" + """Express Mode native API ignores region — always uses global host.""" url = vertex_adapter.build_vertex_api_key_base_url("my-project", "europe-west4") - assert url == ( - "https://europe-west4-aiplatform.googleapis.com/v1beta1/projects/my-project/" - "locations/europe-west4/endpoints/openapi" - ) + assert url == "https://aiplatform.googleapis.com/v1/publishers/google" def test_get_vertex_config_with_api_key(vertex_adapter, monkeypatch): @@ -300,9 +291,8 @@ def test_get_vertex_config_with_api_key(vertex_adapter, monkeypatch): token_or_key, base_url, auth_hdr = vertex_adapter.get_vertex_config() assert token_or_key == "AIzaSyApiKey" assert auth_hdr == "x-goog-api-key" - assert "projects/api-key-project" in base_url - assert "europe-west1-aiplatform.googleapis.com" in base_url - assert "locations/europe-west1" in base_url + # Express Mode native API uses global endpoint without project in URL + assert base_url == "https://aiplatform.googleapis.com/v1/publishers/google" def test_get_vertex_config_api_key_precedence_over_adc(vertex_adapter, monkeypatch): @@ -313,7 +303,8 @@ def test_get_vertex_config_api_key_precedence_over_adc(vertex_adapter, monkeypat token_or_key, base_url, auth_hdr = vertex_adapter.get_vertex_config() assert token_or_key == "AIzaSyKey" # API key, not OAuth token assert auth_hdr == "x-goog-api-key" - assert "projects/key-project" in base_url + # Express Mode uses global endpoint without project in URL + assert base_url == "https://aiplatform.googleapis.com/v1/publishers/google" def test_get_vertex_config_api_key_missing_project(vertex_adapter, monkeypatch): From 2aa288e5232b020d13d87d1a4e62c4c40861182f Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 15:59:48 +0800 Subject: [PATCH 10/12] fix: restore missing base_url variable in create_openai_client --- agent/agent_runtime_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 703a9754cc940..9dc48369376ec 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1876,6 +1876,7 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url if is_native_gemini_base_url(client_kwargs.get("base_url", "")): + base_url = str(client_kwargs.get("base_url", "") or "") safe_kwargs = { k: v for k, v in client_kwargs.items() if k in {"api_key", "base_url", "default_headers", "timeout", "http_client"} From 1bc7368c86df6da6b5be364139d6610aa1fff69c Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 16:21:00 +0800 Subject: [PATCH 11/12] fix(vertex): auxiliary client and default aux model for native API The auxiliary client (title generation, vision, etc.) was creating a raw OpenAI client for Vertex API key mode, which appends /chat/completions to the native base URL and fails. Now routes through _create_openai_client which auto-detects native URLs and creates GeminiNativeClient instead. Also updates Vertex default_aux_model from google/gemini-3-flash-preview (not available via Express Mode) to gemini-3.5-flash (verified working). --- agent/auxiliary_client.py | 25 ++-- agent/vertex_roadmap.md | 138 +++++++++++++++++++++ plugins/model-providers/vertex/__init__.py | 2 +- 3 files changed, 149 insertions(+), 16 deletions(-) create mode 100644 agent/vertex_roadmap.md diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 7ea1917b940ca..daa6882ae1740 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -5333,23 +5333,18 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", "could not mint token / resolve project") return None, None - default_model = "google/gemini-3-flash-preview" + default_model = _get_aux_model_for_provider(provider) or "gemini-3.5-flash" final_model = _normalize_resolved_model(model or default_model, provider) try: - from openai import OpenAI - - if auth_header == "x-goog-api-key": - # Express Mode — use the API key as x-goog-api-key header - # instead of Authorization: Bearer (which only works for - # OAuth2 tokens). - client = OpenAI( - api_key="", - base_url=base_url, - default_headers={"x-goog-api-key": token_or_key}, - ) - else: - # Standard OAuth2 — pass token as Authorization: Bearer - client = OpenAI(api_key=token_or_key, base_url=base_url) + # Route through _create_openai_client which auto-detects native + # Vertex URLs and creates a GeminiNativeClient (which handles + # x-goog-api-key auth and the native generateContent format). + client = _create_openai_client( + api_key=token_or_key, + base_url=base_url, + **({"default_headers": {"x-goog-api-key": token_or_key}} + if auth_header == "x-goog-api-key" else {}), + ) except Exception as exc: logger.warning("resolve_provider_client: cannot create Vertex " "client: %s", exc) diff --git a/agent/vertex_roadmap.md b/agent/vertex_roadmap.md new file mode 100644 index 0000000000000..adc1be2f927fd --- /dev/null +++ b/agent/vertex_roadmap.md @@ -0,0 +1,138 @@ +# Vertex AI Gemini — Feature Roadmap + +Status of Gemini model features through the Vertex OpenAI-compatible endpoint +after API key (Express Mode) auth is established. + +## Legend + +- ✅ **Works** — tested and confirmed +- 🟡 **Partial** — works in theory but needs verification/tuning +- 🔧 **Needs work** — requires implementation + +--- + +## Core Features + +| Feature | Status | Notes | +|---------|--------|-------| +| **Chat completions** | ✅ | Tested: `"Okay, sure."` with full token accounting | +| **Streaming (SSE)** | 🟡 | Chat_completions transport supports SSE; needs live test with Vertex | +| **Authentication** | ✅ | `x-goog-api-key` header, confirmed 200 OK | +| **Reasoning / Thinking** | ✅ | Wired via `extra_body.google.thinking_config` in the plugin | +| **Token tracking** | ✅ | `prompt_tokens`, `completion_tokens`, `reasoning_tokens` all reported | +| **Model selection** | ✅ | Curated list (13 models) + `/model` slash command | + +--- + +## Tool Calling + +| Feature | Status | Notes | +|---------|--------|-------| +| **Function calling** | 🟡 | OpenAI-compat transport handles `tool_calls` generically. Works if Vertex endpoint accepts OpenAI-format `tools` array. Needs live test. | +| **Parallel tool calls** | 🟡 | Depends on tool calling working first | +| **Structured output (JSON mode)** | 🟡 | OpenAI-compat transport supports `response_format: {type: 'json_object'}`. Gemini supports it natively. Needs verification. | + +**Implementation effort:** Low. The chat_completions transport already handles +tool calls for any OpenAI-compatible endpoint. If Vertex returns 400 on the +OpenAI tool format, we may need a thin adapter. + +--- + +## Multimodal / Image Upload + +| Feature | Status | Notes | +|---------|--------|-------| +| **Image input (base64)** | 🔧 | Hermes sends images as `data:` URIs or base64 in messages. Vertex OpenAI-compat endpoint expects `inlineData` format (Gemini native), NOT OpenAI `image_url`. A format adapter is needed. | +| **Image input (URL)** | 🔧 | Same as base64 — format mismatch | +| **Audio input** | 🔧 | Gemini supports audio natively; needs transport adapter | +| **Video input** | 🔧 | Gemini supports video (GCS URIs); needs transport adapter | +| **PDF input** | 🔧 | Gemini supports PDF natively | + +**The problem:** The Vertex OpenAI-compatible endpoint accepts OpenAI-format +messages (`role`, `content` with text), but for multimodal input it uses +Gemini's native format, not OpenAI's `image_url` / `input_audio` format. + +**Fix:** Add a `vertex` message adapter in `agent/transports/` that converts +OpenAI-format multimodal messages to Gemini's `inlineData` parts before +sending. This is the same adapter pattern used for Anthropic's transport. + +--- + +## Prompt Caching + +| Feature | Status | Notes | +|---------|--------|-------| +| **Automatic server-side caching** | 🟡 | Google may cache transparently. Not configurable. | +| **Cached Content API** | 🔧 | Google's `/v1/cachedContents` endpoint for explicit cache management | +| **Cache breakpoints** | ❌ | Not supported by Gemini. `cache_control` markers are Anthropic-only. | + +**Vertex approach:** Google's caching is managed through a separate +`cachedContents` API — create a cache entry, get a `cachedContent` name, +then reference it in `generateContent` requests via `cachedContent` field +in the request body. + +**Implementation effort:** Medium. Requires: +1. New module: `agent/vertex_cache.py` — create/update/delete cached contents +2. Transport mod: pass `cachedContent` field in request body when cache is active +3. Integration with Hermes' context compression / session management + +--- + +## Advanced Features + +| Feature | Status | Notes | +|---------|--------|-------| +| **System instructions** | ✅ | OpenAI-compat transport sends `system` role → Vertex maps to `systemInstruction` | +| **Safety settings** | 🔧 | Gemini supports `safetySettings` array; needs `extra_body` passthrough | +| **Frequency / presence penalty** | 🟡 | OpenAI-compat transport sends these; Vertex may ignore or accept | +| **Max tokens / stop sequences** | ✅ | Standard OpenAI params, handled by chat_completions transport | +| **Temperature / top_p / top_k** | 🟡 | OpenAI `temperature`/`top_p` map automatically. Gemini `top_k` needs `extra_body` | +| **Seed** | 🟡 | OpenAI-compat transport passes `seed`; Gemini accepts it | +| **Logprobs** | ❓ | Not tested. Gemini supports but may need `extra_body` | + +--- + +## Provider-Specific Gaps + +| Area | What's missing | +|------|----------------| +| **OpenAI-compatible messages → Gemini format** | Multimodal parts (images, audio, video) need inlineData conversion | +| **Vertex cachedContents API** | Full lifecycle: create cache, reference, update TTL, delete | +| **`extra_body.google` passthrough** | The plugin already emits `thinking_config`. Need to also pass `safety_settings`, `top_k` | +| **Safety settings** | `safetySettings: [{category, threshold}]` — Gemini-native, needs `extra_body` | +| **Model listing (API key)** | Google doesn't expose `models.list` for Express Mode. Curated list is the only option. | + +--- + +## Implementation Plan + +### Phase 1 — Core (done) +- [x] API key auth (`GOOGLE_VERTEX_API_KEY`, `x-goog-api-key` header) +- [x] Reasoning / thinking config +- [x] Model picker with curated list +- [x] `hermes doctor` detection +- [x] Desktop settings card +- [x] Web dashboard env vars + +### Phase 2 — Tool calling & structured output +- [ ] Verify tool calling works with Vertex OpenAI-compat endpoint +- [ ] Add adapter if needed (Vertex tool format ≠ OpenAI format) +- [ ] Test JSON mode (`response_format: {type: 'json_object'}`) + +### Phase 3 — Multimodal support +- [ ] Create `agent/transports/vertex.py` — message adapter +- [ ] Convert OpenAI `image_url` parts to Gemini `inlineData` parts +- [ ] Test with image upload in chat +- [ ] Add support for audio, video, PDF via GCS URIs + +### Phase 4 — Prompt caching +- [ ] Implement `agent/vertex_cache.py` +- [ ] Create/query cached contents via Vertex API +- [ ] Wire cache references into request body +- [ ] Integrate with session lifecycle + +### Phase 5 — Advanced features +- [ ] Safety settings passthrough via `extra_body` +- [ ] `top_k` support in generation config +- [ ] Logprobs support +- [ ] Context caching for long sessions diff --git a/plugins/model-providers/vertex/__init__.py b/plugins/model-providers/vertex/__init__.py index a3d83202bd635..797a3d80adb26 100644 --- a/plugins/model-providers/vertex/__init__.py +++ b/plugins/model-providers/vertex/__init__.py @@ -100,7 +100,7 @@ def fetch_models( ), base_url="https://aiplatform.googleapis.com", # real base_url computed at runtime auth_type="vertex", - default_aux_model="google/gemini-3-flash-preview", + default_aux_model="gemini-3.5-flash", ) register_provider(vertex) From 399920c23e71abd0c5e7aa6ce3c5be0c4b5bec19 Mon Sep 17 00:00:00 2001 From: sjneoh93 Date: Fri, 24 Jul 2026 16:46:58 +0800 Subject: [PATCH 12/12] chore: remove unwanted roadmap file --- agent/vertex_roadmap.md | 138 ---------------------------------------- 1 file changed, 138 deletions(-) delete mode 100644 agent/vertex_roadmap.md diff --git a/agent/vertex_roadmap.md b/agent/vertex_roadmap.md deleted file mode 100644 index adc1be2f927fd..0000000000000 --- a/agent/vertex_roadmap.md +++ /dev/null @@ -1,138 +0,0 @@ -# Vertex AI Gemini — Feature Roadmap - -Status of Gemini model features through the Vertex OpenAI-compatible endpoint -after API key (Express Mode) auth is established. - -## Legend - -- ✅ **Works** — tested and confirmed -- 🟡 **Partial** — works in theory but needs verification/tuning -- 🔧 **Needs work** — requires implementation - ---- - -## Core Features - -| Feature | Status | Notes | -|---------|--------|-------| -| **Chat completions** | ✅ | Tested: `"Okay, sure."` with full token accounting | -| **Streaming (SSE)** | 🟡 | Chat_completions transport supports SSE; needs live test with Vertex | -| **Authentication** | ✅ | `x-goog-api-key` header, confirmed 200 OK | -| **Reasoning / Thinking** | ✅ | Wired via `extra_body.google.thinking_config` in the plugin | -| **Token tracking** | ✅ | `prompt_tokens`, `completion_tokens`, `reasoning_tokens` all reported | -| **Model selection** | ✅ | Curated list (13 models) + `/model` slash command | - ---- - -## Tool Calling - -| Feature | Status | Notes | -|---------|--------|-------| -| **Function calling** | 🟡 | OpenAI-compat transport handles `tool_calls` generically. Works if Vertex endpoint accepts OpenAI-format `tools` array. Needs live test. | -| **Parallel tool calls** | 🟡 | Depends on tool calling working first | -| **Structured output (JSON mode)** | 🟡 | OpenAI-compat transport supports `response_format: {type: 'json_object'}`. Gemini supports it natively. Needs verification. | - -**Implementation effort:** Low. The chat_completions transport already handles -tool calls for any OpenAI-compatible endpoint. If Vertex returns 400 on the -OpenAI tool format, we may need a thin adapter. - ---- - -## Multimodal / Image Upload - -| Feature | Status | Notes | -|---------|--------|-------| -| **Image input (base64)** | 🔧 | Hermes sends images as `data:` URIs or base64 in messages. Vertex OpenAI-compat endpoint expects `inlineData` format (Gemini native), NOT OpenAI `image_url`. A format adapter is needed. | -| **Image input (URL)** | 🔧 | Same as base64 — format mismatch | -| **Audio input** | 🔧 | Gemini supports audio natively; needs transport adapter | -| **Video input** | 🔧 | Gemini supports video (GCS URIs); needs transport adapter | -| **PDF input** | 🔧 | Gemini supports PDF natively | - -**The problem:** The Vertex OpenAI-compatible endpoint accepts OpenAI-format -messages (`role`, `content` with text), but for multimodal input it uses -Gemini's native format, not OpenAI's `image_url` / `input_audio` format. - -**Fix:** Add a `vertex` message adapter in `agent/transports/` that converts -OpenAI-format multimodal messages to Gemini's `inlineData` parts before -sending. This is the same adapter pattern used for Anthropic's transport. - ---- - -## Prompt Caching - -| Feature | Status | Notes | -|---------|--------|-------| -| **Automatic server-side caching** | 🟡 | Google may cache transparently. Not configurable. | -| **Cached Content API** | 🔧 | Google's `/v1/cachedContents` endpoint for explicit cache management | -| **Cache breakpoints** | ❌ | Not supported by Gemini. `cache_control` markers are Anthropic-only. | - -**Vertex approach:** Google's caching is managed through a separate -`cachedContents` API — create a cache entry, get a `cachedContent` name, -then reference it in `generateContent` requests via `cachedContent` field -in the request body. - -**Implementation effort:** Medium. Requires: -1. New module: `agent/vertex_cache.py` — create/update/delete cached contents -2. Transport mod: pass `cachedContent` field in request body when cache is active -3. Integration with Hermes' context compression / session management - ---- - -## Advanced Features - -| Feature | Status | Notes | -|---------|--------|-------| -| **System instructions** | ✅ | OpenAI-compat transport sends `system` role → Vertex maps to `systemInstruction` | -| **Safety settings** | 🔧 | Gemini supports `safetySettings` array; needs `extra_body` passthrough | -| **Frequency / presence penalty** | 🟡 | OpenAI-compat transport sends these; Vertex may ignore or accept | -| **Max tokens / stop sequences** | ✅ | Standard OpenAI params, handled by chat_completions transport | -| **Temperature / top_p / top_k** | 🟡 | OpenAI `temperature`/`top_p` map automatically. Gemini `top_k` needs `extra_body` | -| **Seed** | 🟡 | OpenAI-compat transport passes `seed`; Gemini accepts it | -| **Logprobs** | ❓ | Not tested. Gemini supports but may need `extra_body` | - ---- - -## Provider-Specific Gaps - -| Area | What's missing | -|------|----------------| -| **OpenAI-compatible messages → Gemini format** | Multimodal parts (images, audio, video) need inlineData conversion | -| **Vertex cachedContents API** | Full lifecycle: create cache, reference, update TTL, delete | -| **`extra_body.google` passthrough** | The plugin already emits `thinking_config`. Need to also pass `safety_settings`, `top_k` | -| **Safety settings** | `safetySettings: [{category, threshold}]` — Gemini-native, needs `extra_body` | -| **Model listing (API key)** | Google doesn't expose `models.list` for Express Mode. Curated list is the only option. | - ---- - -## Implementation Plan - -### Phase 1 — Core (done) -- [x] API key auth (`GOOGLE_VERTEX_API_KEY`, `x-goog-api-key` header) -- [x] Reasoning / thinking config -- [x] Model picker with curated list -- [x] `hermes doctor` detection -- [x] Desktop settings card -- [x] Web dashboard env vars - -### Phase 2 — Tool calling & structured output -- [ ] Verify tool calling works with Vertex OpenAI-compat endpoint -- [ ] Add adapter if needed (Vertex tool format ≠ OpenAI format) -- [ ] Test JSON mode (`response_format: {type: 'json_object'}`) - -### Phase 3 — Multimodal support -- [ ] Create `agent/transports/vertex.py` — message adapter -- [ ] Convert OpenAI `image_url` parts to Gemini `inlineData` parts -- [ ] Test with image upload in chat -- [ ] Add support for audio, video, PDF via GCS URIs - -### Phase 4 — Prompt caching -- [ ] Implement `agent/vertex_cache.py` -- [ ] Create/query cached contents via Vertex API -- [ ] Wire cache references into request body -- [ ] Integrate with session lifecycle - -### Phase 5 — Advanced features -- [ ] Safety settings passthrough via `extra_body` -- [ ] `top_k` support in generation config -- [ ] Logprobs support -- [ ] Context caching for long sessions