From f148b2d2bb5e9d99b390888007eb52fd1f7f2fe3 Mon Sep 17 00:00:00 2001 From: dirtyfancy Date: Sun, 12 Apr 2026 08:03:36 +0800 Subject: [PATCH 1/2] fix(model_metadata): respect context_length from custom_providers config with scoping get_model_context_length() now checks custom_providers[].models[] for a config-driven context_length override before falling back to probing or defaults. The lookup is scoped to the active provider (by name or base_url) to avoid collisions between identically-named models on different endpoints. Adds 5 unit tests covering provider-name matching, base-url matching, unmatched-provider exclusion, single-provider fallback, and ambiguous-multi-provider skipping. --- agent/model_metadata.py | 36 +++++++++++ tests/agent/test_model_metadata.py | 97 ++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 03f70b3fe41a..0e65ada48e7b 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -939,6 +939,42 @@ def get_model_context_length( if config_context_length is not None and isinstance(config_context_length, int) and config_context_length > 0: return config_context_length + # 0a. Config-driven overrides from custom_providers[].models[].context_length. + # When provider/base_url is known, scope the lookup to the matching custom + # provider entry to avoid collisions between identically-named models on + # different endpoints. When unknown, allow matching any custom provider. + try: + from hermes_cli.config import load_config + _cfg = load_config() + _custom_providers = _cfg.get("custom_providers", []) if isinstance(_cfg, dict) else [] + if isinstance(_custom_providers, list): + _normalized_base_url = base_url.rstrip("/") if isinstance(base_url, str) and base_url else None + _active_provider = str(provider).strip().lower() if isinstance(provider, str) and provider.strip() else None + _valid_entries = [cp for cp in _custom_providers if isinstance(cp, dict)] + for cp in _valid_entries: + _cp_base = str(cp.get("base_url", "")).strip().rstrip("/") + _cp_name = str(cp.get("name", "")).strip().lower() + + if _normalized_base_url is not None and _cp_base: + if _cp_base != _normalized_base_url: + continue + elif _active_provider is not None and _cp_name: + if _cp_name != _active_provider: + continue + elif len(_valid_entries) > 1: + # Ambiguous: skip unscoped entries when multiple providers exist + continue + + _models = cp.get("models", {}) + if isinstance(_models, dict): + _model_entry = _models.get(model, {}) + if isinstance(_model_entry, dict): + _ctx = _model_entry.get("context_length") + if isinstance(_ctx, int) and _ctx > 0: + return _ctx + except Exception: + pass + # Normalise provider-prefixed model names (e.g. "local:model-name" → # "model-name") so cache lookups and server queries use the bare ID that # local servers actually know about. Ollama "model:tag" colons are preserved. diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index df680fb241c5..c51dc75b3e35 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -709,3 +709,100 @@ def test_special_chars_in_model_name(self, tmp_path): with patch("agent.model_metadata._get_context_cache_path", return_value=cache_file): save_context_length(model, url, 200000) assert get_cached_context_length(model, url) == 200000 + + +class TestCustomProvidersContextLength: + """Tests for config-driven context_length overrides in custom_providers.""" + + @patch("hermes_cli.config.load_config") + def test_match_by_provider_name(self, mock_load_config): + mock_load_config.return_value = { + "custom_providers": [ + { + "name": "zai-customize", + "base_url": "https://open.bigmodel.cn/api/coding/paas/v4", + "models": { + "glm-5.1": {"context_length": 200000}, + }, + } + ] + } + result = get_model_context_length("glm-5.1", provider="zai-customize") + assert result == 200000 + + @patch("hermes_cli.config.load_config") + def test_match_by_base_url(self, mock_load_config): + mock_load_config.return_value = { + "custom_providers": [ + { + "name": "zai-customize", + "base_url": "https://open.bigmodel.cn/api/coding/paas/v4", + "models": { + "glm-5.1": {"context_length": 200000}, + }, + } + ] + } + result = get_model_context_length( + "glm-5.1", base_url="https://open.bigmodel.cn/api/coding/paas/v4" + ) + assert result == 200000 + + @patch("hermes_cli.config.load_config") + def test_unmatched_provider_is_ignored(self, mock_load_config): + """A model name in a different custom provider must not leak across.""" + mock_load_config.return_value = { + "custom_providers": [ + { + "name": "zai-customize", + "base_url": "https://open.bigmodel.cn/api/coding/paas/v4", + "models": { + "glm-5.1": {"context_length": 200000}, + }, + }, + { + "name": "other-provider", + "base_url": "https://other.example.com/v1", + "models": { + "glm-5.1": {"context_length": 128000}, + }, + }, + ] + } + result = get_model_context_length("glm-5.1", provider="zai-customize") + assert result == 200000 + + @patch("hermes_cli.config.load_config") + def test_single_provider_fallback_when_no_scope(self, mock_load_config): + """With only one custom provider, allow matching even without provider/base_url.""" + mock_load_config.return_value = { + "custom_providers": [ + { + "name": "zai-customize", + "models": { + "glm-5.1": {"context_length": 200000}, + }, + } + ] + } + result = get_model_context_length("glm-5.1") + assert result == 200000 + + @patch("hermes_cli.config.load_config") + def test_ambiguous_multiple_providers_skipped_without_scope(self, mock_load_config): + """When multiple providers exist and no scope is given, skip to avoid collisions.""" + mock_load_config.return_value = { + "custom_providers": [ + { + "name": "zai-customize", + "models": {"glm-5.1": {"context_length": 200000}}, + }, + { + "name": "other-provider", + "models": {"glm-5.1": {"context_length": 128000}}, + }, + ] + } + # Must NOT pick either custom provider value because scope is ambiguous + result = get_model_context_length("glm-5.1") + assert result not in (200000, 128000) From b666fb0eb3d785baef5e7c7fc2b13b718def2fde Mon Sep 17 00:00:00 2001 From: dirtyfancy Date: Sun, 12 Apr 2026 08:33:12 +0800 Subject: [PATCH 2/2] fix(model_metadata): address Copilot review on custom_providers lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use read_raw_config() instead of load_config() to avoid deep-merge overhead on every get_model_context_length() call. - Normalize provider names consistently: strip custom: prefix and apply the same space→dash normalization used by runtime_provider.py. - Consistent base_url stripping: apply .strip() before .rstrip('/') to avoid mismatches on leading/trailing whitespace. - Update misleading comment to match actual behavior (unscoped matching is only allowed when there is a single custom provider). - Patch fetch_model_metadata in ambiguous-provider test so it stays fully offline and deterministic. - Add test for custom: provider keys. --- agent/model_metadata.py | 24 ++++++++++---- tests/agent/test_model_metadata.py | 51 +++++++++++++++++++++--------- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 0e65ada48e7b..6fc79e48b0a3 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -942,18 +942,30 @@ def get_model_context_length( # 0a. Config-driven overrides from custom_providers[].models[].context_length. # When provider/base_url is known, scope the lookup to the matching custom # provider entry to avoid collisions between identically-named models on - # different endpoints. When unknown, allow matching any custom provider. + # different endpoints. When neither provider nor base_url is known and there + # is only one custom provider defined, allow matching that single entry. try: - from hermes_cli.config import load_config - _cfg = load_config() + from hermes_cli.config import read_raw_config + from hermes_cli.runtime_provider import _normalize_custom_provider_name + + _cfg = read_raw_config() _custom_providers = _cfg.get("custom_providers", []) if isinstance(_cfg, dict) else [] if isinstance(_custom_providers, list): - _normalized_base_url = base_url.rstrip("/") if isinstance(base_url, str) and base_url else None - _active_provider = str(provider).strip().lower() if isinstance(provider, str) and provider.strip() else None + _normalized_base_url = ( + base_url.strip().rstrip("/") + if isinstance(base_url, str) and base_url.strip() + else None + ) + _active_provider = None + if isinstance(provider, str) and provider.strip(): + _ap = provider.strip() + if _ap.lower().startswith("custom:"): + _ap = _ap.split(":", 1)[1] + _active_provider = _normalize_custom_provider_name(_ap) _valid_entries = [cp for cp in _custom_providers if isinstance(cp, dict)] for cp in _valid_entries: _cp_base = str(cp.get("base_url", "")).strip().rstrip("/") - _cp_name = str(cp.get("name", "")).strip().lower() + _cp_name = _normalize_custom_provider_name(str(cp.get("name", ""))) if _normalized_base_url is not None and _cp_base: if _cp_base != _normalized_base_url: diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index c51dc75b3e35..3505235c3182 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -714,9 +714,9 @@ def test_special_chars_in_model_name(self, tmp_path): class TestCustomProvidersContextLength: """Tests for config-driven context_length overrides in custom_providers.""" - @patch("hermes_cli.config.load_config") - def test_match_by_provider_name(self, mock_load_config): - mock_load_config.return_value = { + @patch("hermes_cli.config.read_raw_config") + def test_match_by_provider_name(self, mock_raw_config): + mock_raw_config.return_value = { "custom_providers": [ { "name": "zai-customize", @@ -730,9 +730,9 @@ def test_match_by_provider_name(self, mock_load_config): result = get_model_context_length("glm-5.1", provider="zai-customize") assert result == 200000 - @patch("hermes_cli.config.load_config") - def test_match_by_base_url(self, mock_load_config): - mock_load_config.return_value = { + @patch("hermes_cli.config.read_raw_config") + def test_match_by_base_url(self, mock_raw_config): + mock_raw_config.return_value = { "custom_providers": [ { "name": "zai-customize", @@ -748,10 +748,10 @@ def test_match_by_base_url(self, mock_load_config): ) assert result == 200000 - @patch("hermes_cli.config.load_config") - def test_unmatched_provider_is_ignored(self, mock_load_config): + @patch("hermes_cli.config.read_raw_config") + def test_unmatched_provider_is_ignored(self, mock_raw_config): """A model name in a different custom provider must not leak across.""" - mock_load_config.return_value = { + mock_raw_config.return_value = { "custom_providers": [ { "name": "zai-customize", @@ -772,10 +772,10 @@ def test_unmatched_provider_is_ignored(self, mock_load_config): result = get_model_context_length("glm-5.1", provider="zai-customize") assert result == 200000 - @patch("hermes_cli.config.load_config") - def test_single_provider_fallback_when_no_scope(self, mock_load_config): + @patch("hermes_cli.config.read_raw_config") + def test_single_provider_fallback_when_no_scope(self, mock_raw_config): """With only one custom provider, allow matching even without provider/base_url.""" - mock_load_config.return_value = { + mock_raw_config.return_value = { "custom_providers": [ { "name": "zai-customize", @@ -788,10 +788,11 @@ def test_single_provider_fallback_when_no_scope(self, mock_load_config): result = get_model_context_length("glm-5.1") assert result == 200000 - @patch("hermes_cli.config.load_config") - def test_ambiguous_multiple_providers_skipped_without_scope(self, mock_load_config): + @patch("agent.model_metadata.fetch_model_metadata") + @patch("hermes_cli.config.read_raw_config") + def test_ambiguous_multiple_providers_skipped_without_scope(self, mock_raw_config, mock_fetch_meta): """When multiple providers exist and no scope is given, skip to avoid collisions.""" - mock_load_config.return_value = { + mock_raw_config.return_value = { "custom_providers": [ { "name": "zai-customize", @@ -803,6 +804,26 @@ def test_ambiguous_multiple_providers_skipped_without_scope(self, mock_load_conf }, ] } + mock_fetch_meta.return_value = {} # Must NOT pick either custom provider value because scope is ambiguous result = get_model_context_length("glm-5.1") assert result not in (200000, 128000) + + @patch("hermes_cli.config.read_raw_config") + def test_match_by_runtime_custom_provider_key(self, mock_raw_config): + """Provider keys like custom: must resolve correctly.""" + mock_raw_config.return_value = { + "custom_providers": [ + { + "name": "local-(127.0.0.1:4141)", + "base_url": "http://127.0.0.1:4141/v1", + "models": { + "llama3.2": {"context_length": 8192}, + }, + } + ] + } + result = get_model_context_length( + "llama3.2", provider="custom:local-(127.0.0.1:4141)" + ) + assert result == 8192