fix(gateway): honor max_tokens from custom_providers / providers entries (#20004)

gateway/run.py

@@ -706,6 +706,7 @@ def _resolve_runtime_agent_kwargs() -> dict:
     from hermes_cli.runtime_provider import (
         resolve_runtime_provider,
         format_runtime_provider_error,
+        _get_model_config,
     )
     from hermes_cli.auth import AuthError
 
@@ -724,6 +725,21 @@ def _resolve_runtime_agent_kwargs() -> dict:
     except Exception as exc:
         raise RuntimeError(format_runtime_provider_error(exc)) from exc
 
+    # Resolution order for max_tokens:
+    #   1. Per-provider override from custom_providers / providers entry
+    #      (already lifted onto runtime by _resolve_named_custom_runtime).
+    #   2. Top-level model.max_tokens from config.yaml.
+    #   3. None → AIAgent / transport picks a provider-appropriate default.
+    # See issue #20004.
+    max_tokens = _coerce_max_tokens(runtime.get("max_tokens"))
+    if max_tokens is None:
+        try:
+            model_cfg = _get_model_config()
+        except Exception:
+            model_cfg = {}
+        if isinstance(model_cfg, dict):
+            max_tokens = _coerce_max_tokens(model_cfg.get("max_tokens"))
+
     return {
         "api_key": runtime.get("api_key"),
         "base_url": runtime.get("base_url"),
@@ -732,12 +748,28 @@ def _resolve_runtime_agent_kwargs() -> dict:
         "command": runtime.get("command"),
         "args": list(runtime.get("args") or []),
         "credential_pool": runtime.get("credential_pool"),
+        "max_tokens": max_tokens,
     }
 
 
+def _coerce_max_tokens(raw: object) -> int | None:
+    """Return a positive int or None.  Strings, zero, negatives → None.
+
+    Used by the gateway's runtime-kwargs resolvers so a misconfigured
+    \"max_tokens: 64K\" entry doesn't crash AIAgent construction — it
+    just falls through to the next layer (model.max_tokens → provider
+    default), matching how _normalize_custom_provider_entry validates.
+    """
+    if isinstance(raw, bool):
+        return None
+    if isinstance(raw, int) and raw > 0:
+        return raw
+    return None
+
+
 def _try_resolve_fallback_provider() -> dict | None:
     """Attempt to resolve credentials from the fallback_model/fallback_providers config."""
-    from hermes_cli.runtime_provider import resolve_runtime_provider
+    from hermes_cli.runtime_provider import resolve_runtime_provider, _get_model_config
     try:
         import yaml as _y
         cfg_path = _hermes_home / "config.yaml"
@@ -760,6 +792,17 @@ def _try_resolve_fallback_provider() -> dict | None:
                     explicit_api_key=entry.get("api_key"),
                 )
                 logger.info("Fallback provider resolved: %s", runtime.get("provider"))
+                # Honour the same max_tokens resolution order as the primary
+                # path (custom_providers > model.max_tokens > None) so a
+                # fallback kick-in doesn't silently change the output cap.
+                max_tokens = _coerce_max_tokens(runtime.get("max_tokens"))
+                if max_tokens is None:
+                    try:
+                        model_cfg = _get_model_config()
+                    except Exception:
+                        model_cfg = {}
+                    if isinstance(model_cfg, dict):
+                        max_tokens = _coerce_max_tokens(model_cfg.get("max_tokens"))
                 return {
                     "api_key": runtime.get("api_key"),
                     "base_url": runtime.get("base_url"),
@@ -768,6 +811,7 @@ def _try_resolve_fallback_provider() -> dict | None:
                     "command": runtime.get("command"),
                     "args": list(runtime.get("args") or []),
                     "credential_pool": runtime.get("credential_pool"),
+                    "max_tokens": max_tokens,
                 }
             except Exception as fb_exc:
                 logger.debug("Fallback entry %s failed: %s", entry.get("provider"), fb_exc)
@@ -1775,6 +1819,7 @@ def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwar
             "command": runtime_kwargs.get("command"),
             "args": list(runtime_kwargs.get("args") or []),
             "credential_pool": runtime_kwargs.get("credential_pool"),
+            "max_tokens": runtime_kwargs.get("max_tokens"),
         }
         route = {
             "model": model,

hermes_cli/config.py

@@ -2553,7 +2553,7 @@ def _normalize_custom_provider_entry(
     _KNOWN_KEYS = {
         "name", "api", "url", "base_url", "api_key", "key_env", "api_key_env",
         "api_mode", "transport", "model", "default_model", "models",
-        "context_length", "rate_limit_delay",
+        "context_length", "max_tokens", "rate_limit_delay",
         "request_timeout_seconds", "stale_timeout_seconds",
     }
     for camel, snake in _CAMEL_ALIASES.items():
@@ -2641,6 +2641,13 @@ def _normalize_custom_provider_entry(
     if isinstance(context_length, int) and context_length > 0:
         normalized["context_length"] = context_length
 
+    # Per-provider max output tokens — overrides model.max_tokens at runtime
+    # so a single global config can pin different output caps per endpoint
+    # (e.g. local vLLM with a tight budget vs. an upstream that allows 64k).
+    max_tokens = entry.get("max_tokens")
+    if isinstance(max_tokens, int) and max_tokens > 0:
+        normalized["max_tokens"] = max_tokens
+
     rate_limit_delay = entry.get("rate_limit_delay")
     if isinstance(rate_limit_delay, (int, float)) and rate_limit_delay >= 0:
         normalized["rate_limit_delay"] = rate_limit_delay

hermes_cli/runtime_provider.py

@@ -346,6 +346,21 @@ def _try_resolve_from_custom_pool(
         return None
 
 
+def _attach_custom_provider_max_tokens(
+    result: Dict[str, Any], entry: Dict[str, Any]
+) -> None:
+    """Copy a positive int ``max_tokens`` from a custom-provider config entry.
+
+    Centralises the validation so all three provider lookup paths
+    (providers-dict-by-key, providers-dict-by-display-name, legacy
+    custom_providers list) honour the same per-entry override and reject
+    bogus values (strings, zero, negative) the same way.  See issue #20004.
+    """
+    raw = entry.get("max_tokens")
+    if isinstance(raw, int) and raw > 0:
+        result["max_tokens"] = raw
+
+
 def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, Any]]:
     requested_norm = _normalize_custom_provider_name(requested_provider or "")
     if not requested_norm or requested_norm == "custom":
@@ -410,6 +425,7 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An
                     api_mode = _parse_api_mode(entry.get("api_mode") or entry.get("transport"))
                     if api_mode:
                         result["api_mode"] = api_mode
+                    _attach_custom_provider_max_tokens(result, entry)
                     return result
             # Also check the 'name' field if present
             display_name = entry.get("name", "")
@@ -428,6 +444,7 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An
                         api_mode = _parse_api_mode(entry.get("api_mode") or entry.get("transport"))
                         if api_mode:
                             result["api_mode"] = api_mode
+                        _attach_custom_provider_max_tokens(result, entry)
                         return result
 
     # Fall back to custom_providers: list (legacy format)
@@ -474,11 +491,15 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An
         model_name = str(entry.get("model", "") or "").strip()
         if model_name:
             result["model"] = model_name
+        _attach_custom_provider_max_tokens(result, entry)
         return result
 
     return None
 
 
+
+
+
 def _resolve_named_custom_runtime(
     *,
     requested_provider: str,
@@ -528,6 +549,12 @@ def _resolve_named_custom_runtime(
         model_name = custom_provider.get("model")
         if model_name:
             pool_result["model"] = model_name
+        # Same story for max_tokens: the credential pool can't read the
+        # config-side per-provider override, so lift it onto the pool
+        # result here.  Without this, a pool-backed custom provider would
+        # silently fall back to the global model.max_tokens / provider
+        # default and ignore the user's per-endpoint cap (#20004).
+        _attach_custom_provider_max_tokens(pool_result, custom_provider)
         return pool_result
 
     api_key_candidates = [
@@ -552,6 +579,7 @@ def _resolve_named_custom_runtime(
     # provider name differs from the actual model string the API expects.
     if custom_provider.get("model"):
         result["model"] = custom_provider["model"]
+    _attach_custom_provider_max_tokens(result, custom_provider)
     return result