From b5fc01be084eb0f3dcee30bfcf82c8bbb7731c9b Mon Sep 17 00:00:00 2001 From: zccyman <16263913+zccyman@users.noreply.github.com> Date: Wed, 20 May 2026 07:59:29 +0800 Subject: [PATCH] fix: read max_tokens from custom_providers per-model config Previously, only context_length was read from custom_providers per-model config. max_tokens was silently ignored, always falling back to 4096. Changes: - Extract get_custom_provider_model_field() as generic lookup helper - Add get_custom_provider_max_tokens() symmetric to context_length - Read custom_providers max_tokens in agent_init.py after context_length - Add 10 regression tests for max_tokens lookup Closes #28046 --- .dev-workflow/code-graph.db | Bin 0 -> 65536 bytes agent/agent_init.py | 14 ++ hermes_cli/config.py | 91 +++++++--- .../test_custom_provider_max_tokens.py | 158 ++++++++++++++++++ 4 files changed, 238 insertions(+), 25 deletions(-) create mode 100644 .dev-workflow/code-graph.db create mode 100644 tests/hermes_cli/test_custom_provider_max_tokens.py diff --git a/.dev-workflow/code-graph.db b/.dev-workflow/code-graph.db new file mode 100644 index 0000000000000000000000000000000000000000..15c9d7d1fc69e5bf2eeb6adea8b7af8bacba849b GIT binary patch literal 65536 zcmeI*&u=1S7{Kuv=<=&jYGN`seMzG&No~rqweeuIZgE3IR$Df^YqFWlmUm%N3J8PN zY&7X+6BGX@6aN9@-HQiLUOjnq=AFX808)>gd?)GfzSD6+ZwjzaH^TVe^NPWR7B(GlUhYgeM9VIL}*jr?ORS;G%8OT;)|2&=SL?`#b=eL zyK}K;y{;p>j(R~qy#BBu>W^x*`PfmvX*sfKx*vYrlc|YuyG=8$+SjJPxZ~IGdD|9oz1IsHfaC3%ws|2PD`@3r41(he2+rkqHyHdk zR%awt-0Tp_xrFhcuxtpy3!9~&y-{6H7)3+3)Hv&IW*6`zSqiLpMAA{TzMj9 z1H{9+P(eFhc=05YFy1qCpDc)UtU*h<6W}IWUy{rZ3DWs}({^*F7v>!HMuRh{a%Ps~ z^h1K2e&5Ua>Z4@RFbw^*KW8e7+NEY@N*!7-Ns~mY3U4xsRe_$~s+e!3-D+P~@NwnS zqer!dDDCafmwSKE>&t;-%b}lg9K8MV`|bJYsjE!vOh&dVCg;8@)ZETiS}~O{J~Wn9 znu0S+>LLAAB4Lya{pF;-y1k}bUwx}32Uho7^^$W_I2+_mqB^oOewsM0oZc^1?}TdC ze_GeNkgAnflGh6ea(caDeNClOWq;|_uAj=g)K%-wl2l$mkV@SdyjxhY8ee#7UVfEt zZAm^=3X|a~`qvpd?)dIFAEVS( z#7qEIm2=1LO4A+Cz!BAYqw=6~;!dew)eCA`MN9W0xk0QwH05Q#H*oJbHwpiLm&^S7 z&iJgi7hGJEYE|z$;JF_ zeFP9d009ILKmY**5I_I{1fnOv`agPe%7_p^009ILKmY**5I_I{1P}-du>KE=&_@6P z1Q0*~0R#|0009ILKp=VotpB4or;G>z1Q0*~0R#|0009ILKmdWT0Du2KEJ7av1Q0*~ z0R#|0009ILKmdX03Gn;>=*=l3LI42-5I_I{1Q0*~0R#|0AS}T8KP*BY0R#|0009IL zKmY**5I_Kd=n1g?kKUXzA_NdX009ILKmY**5I_I{1i}KW|HC5m5kLR|1Q0*~0R#|0 z009ILh@OD=_y1z~-?aQ6?v4!s1Q0*~0R#|0009ILKmY**ZiB!^EMCZWdrfJc^+sK1 z|7|^9D0Qr%V?H0*oo4Cq@PoVM^1af*Uit9uz5VwOyx;%F@_%aizug@h0tg_000Iag zfB*srAb None: return compatible - -def get_custom_provider_context_length( - model: str, - base_url: str, - custom_providers: Optional[List[Dict[str, Any]]] = None, - config: Optional[Dict[str, Any]] = None, -) -> Optional[int]: - """Look up a per-model ``context_length`` override from ``custom_providers``. +def get_custom_provider_model_field( + field: str, + model: str | None = None, + base_url: str | None = None, + *, + custom_providers: list | None = None, + config: dict | None = None, + value_type: type = int, + positive_only: bool = True, +) -> int | None: + """Look up a per-model field override from ``custom_providers``. Matches any entry whose ``base_url`` equals ``base_url`` (trailing-slash - insensitive) and returns ``custom_providers[i].models..context_length`` + insensitive) and returns ``custom_providers[i].models..`` if present and valid. Returns ``None`` when no override applies. - This is the single source of truth for custom-provider context overrides, - used by: - * ``AIAgent.__init__`` (startup resolution) - * ``AIAgent.switch_model`` (mid-session ``/model`` switch) - * ``hermes_cli.model_switch.resolve_display_context_length`` (``/model`` confirmation display) - * ``gateway.run._format_session_info`` (``/info`` display) - * ``agent.model_metadata.get_model_context_length`` (when custom_providers is threaded through) - - Before this helper existed, the lookup was duplicated in ``run_agent.py``'s - startup path only; every other path (notably ``/model`` switch) fell back - to the 128K default. See #15779. + Generic helper used for both ``context_length`` and ``max_tokens`` lookups. """ if not model or not base_url: return None @@ -3223,18 +3216,66 @@ def get_custom_provider_context_length( model_cfg = models.get(model) if not isinstance(model_cfg, dict): continue - raw_ctx = model_cfg.get("context_length") - if raw_ctx is None: + raw_val = model_cfg.get(field) + if raw_val is None: continue try: - ctx = int(raw_ctx) + val = value_type(raw_val) except (TypeError, ValueError): continue - if ctx > 0: - return ctx + if positive_only and val <= 0: + continue + return val return None +def get_custom_provider_context_length( + model: str | None = None, + base_url: str | None = None, + custom_providers: list | None = None, + config: dict | None = None, +) -> int | None: + """Look up a per-model ``context_length`` override from ``custom_providers``. + + Matches any entry whose ``base_url`` equals ``base_url`` (trailing-slash + insensitive) and returns ``custom_providers[i].models..context_length`` + if present and valid. Returns ``None`` when no override applies. + + This is the single source of truth for custom-provider context overrides, + used by: + * ``AIAgent.__init__`` (startup resolution) + * ``AIAgent.switch_model`` (mid-session ``/model`` switch) + * ``hermes_cli.model_switch.resolve_display_context_length`` (``/model`` confirmation display) + * ``gateway.run._format_session_info`` (``/info`` display) + * ``agent.model_metadata.get_model_context_length`` (when custom_providers is threaded through) + + Before this helper existed, the lookup was duplicated in ``run_agent.py``'s + startup path only; every other path (notably ``/model`` switch) fell back + to the 128K default. See #15779. + """ + return get_custom_provider_model_field( + "context_length", model, base_url, + custom_providers=custom_providers, config=config, + ) + + +def get_custom_provider_max_tokens( + model: str | None = None, + base_url: str | None = None, + custom_providers: list | None = None, + config: dict | None = None, +) -> int | None: + """Look up a per-model ``max_tokens`` override from ``custom_providers``. + + Symmetric to ``get_custom_provider_context_length`` but for the output + token limit. Returns ``None`` when no override applies. + """ + return get_custom_provider_model_field( + "max_tokens", model, base_url, + custom_providers=custom_providers, config=config, + ) + + def check_config_version() -> Tuple[int, int]: """ Check config version. diff --git a/tests/hermes_cli/test_custom_provider_max_tokens.py b/tests/hermes_cli/test_custom_provider_max_tokens.py new file mode 100644 index 0000000000000..07f99468a5e24 --- /dev/null +++ b/tests/hermes_cli/test_custom_provider_max_tokens.py @@ -0,0 +1,158 @@ +"""Regression tests for custom_providers per-model max_tokens resolution. + +Covers the fix for #28046 — custom_providers per-model max_tokens was silently +ignored, always defaulting to 4096. The fix adds a symmetric lookup to the +existing context_length mechanism via get_custom_provider_model_field. +""" +from __future__ import annotations + +from hermes_cli.config import get_custom_provider_max_tokens + + +class TestGetCustomProviderMaxTokens: + def test_returns_override_for_matching_entry(self): + custom = [ + { + "name": "my-endpoint", + "base_url": "https://example.invalid/v1", + "models": {"gpt-5.5": {"max_tokens": 16384}}, + } + ] + assert ( + get_custom_provider_max_tokens( + "gpt-5.5", "https://example.invalid/v1", custom + ) + == 16384 + ) + + def test_trailing_slash_insensitive(self): + custom = [ + { + "base_url": "https://example.invalid/v1/", + "models": {"m": {"max_tokens": 8192}}, + } + ] + assert ( + get_custom_provider_max_tokens( + "m", "https://example.invalid/v1", custom + ) + == 8192 + ) + + def test_returns_none_when_no_override(self): + custom = [ + { + "base_url": "https://other.invalid/v1", + "models": {"m": {"max_tokens": 4096}}, + } + ] + assert ( + get_custom_provider_max_tokens( + "m", "https://example.invalid/v1", custom + ) + is None + ) + + def test_returns_none_for_zero_value(self): + custom = [ + { + "base_url": "https://example.invalid/v1", + "models": {"m": {"max_tokens": 0}}, + } + ] + assert ( + get_custom_provider_max_tokens( + "m", "https://example.invalid/v1", custom + ) + is None + ) + + def test_returns_none_for_negative_value(self): + custom = [ + { + "base_url": "https://example.invalid/v1", + "models": {"m": {"max_tokens": -100}}, + } + ] + assert ( + get_custom_provider_max_tokens( + "m", "https://example.invalid/v1", custom + ) + is None + ) + + def test_returns_none_for_string_value(self): + custom = [ + { + "base_url": "https://example.invalid/v1", + "models": {"m": {"max_tokens": "16384"}}, + } + ] + # int("16384") succeeds so this should work + assert ( + get_custom_provider_max_tokens( + "m", "https://example.invalid/v1", custom + ) + == 16384 + ) + + def test_returns_none_for_non_int_string(self): + custom = [ + { + "base_url": "https://example.invalid/v1", + "models": {"m": {"max_tokens": "16K"}}, + } + ] + assert ( + get_custom_provider_max_tokens( + "m", "https://example.invalid/v1", custom + ) + is None + ) + + def test_coexists_with_context_length(self): + """Both fields can be present in the same model config.""" + custom = [ + { + "base_url": "https://example.invalid/v1", + "models": { + "m": { + "context_length": 256_000, + "max_tokens": 8192, + } + }, + } + ] + from hermes_cli.config import get_custom_provider_context_length + + assert ( + get_custom_provider_context_length("m", "https://example.invalid/v1", custom) + == 256_000 + ) + assert ( + get_custom_provider_max_tokens("m", "https://example.invalid/v1", custom) + == 8192 + ) + + def test_first_matching_entry_wins(self): + custom = [ + { + "base_url": "https://example.invalid/v1", + "models": {"m": {"max_tokens": 4096}}, + }, + { + "base_url": "https://example.invalid/v1", + "models": {"m": {"max_tokens": 8192}}, + }, + ] + assert ( + get_custom_provider_max_tokens( + "m", "https://example.invalid/v1", custom + ) + == 4096 + ) + + def test_none_inputs(self): + assert get_custom_provider_max_tokens(None, "url", []) is None + assert get_custom_provider_max_tokens("m", None, []) is None + assert get_custom_provider_max_tokens(None, None, []) is None