Repository navigation
fix(model_metadata): respect context_length from custom_providers config with scoping #8005
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -939,6 +939,42 @@ def get_model_context_length( | |
| if config_context_length is not None and isinstance(config_context_length, int) and config_context_length > 0: | ||
| return config_context_length | ||
|
|
||
| # 0a. Config-driven overrides from custom_providers[].models[].context_length. | ||
| # When provider/base_url is known, scope the lookup to the matching custom | ||
| # provider entry to avoid collisions between identically-named models on | ||
| # different endpoints. When unknown, allow matching any custom provider. | ||
| try: | ||
| from hermes_cli.config import load_config | ||
| _cfg = load_config() | ||
| _custom_providers = _cfg.get("custom_providers", []) if isinstance(_cfg, dict) else [] | ||
| if isinstance(_custom_providers, list): | ||
| _normalized_base_url = base_url.rstrip("/") if isinstance(base_url, str) and base_url else None | ||
| _active_provider = str(provider).strip().lower() if isinstance(provider, str) and provider.strip() else None | ||
| _valid_entries = [cp for cp in _custom_providers if isinstance(cp, dict)] | ||
| for cp in _valid_entries: | ||
| _cp_base = str(cp.get("base_url", "")).strip().rstrip("/") | ||
| _cp_name = str(cp.get("name", "")).strip().lower() | ||
|
||
|
|
||
| if _normalized_base_url is not None and _cp_base: | ||
| if _cp_base != _normalized_base_url: | ||
| continue | ||
| elif _active_provider is not None and _cp_name: | ||
| if _cp_name != _active_provider: | ||
| continue | ||
|
||
| elif len(_valid_entries) > 1: | ||
| # Ambiguous: skip unscoped entries when multiple providers exist | ||
| continue | ||
|
Comment on lines
+942
to
+978
|
||
|
|
||
| _models = cp.get("models", {}) | ||
| if isinstance(_models, dict): | ||
| _model_entry = _models.get(model, {}) | ||
| if isinstance(_model_entry, dict): | ||
| _ctx = _model_entry.get("context_length") | ||
| if isinstance(_ctx, int) and _ctx > 0: | ||
| return _ctx | ||
| except Exception: | ||
| pass | ||
|
|
||
| # Normalise provider-prefixed model names (e.g. "local:model-name" → | ||
| # "model-name") so cache lookups and server queries use the bare ID that | ||
| # local servers actually know about. Ollama "model:tag" colons are preserved. | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -709,3 +709,100 @@ def test_special_chars_in_model_name(self, tmp_path): | |||||||||||||||||||||||||||||||||||||||||
| with patch("agent.model_metadata._get_context_cache_path", return_value=cache_file): | ||||||||||||||||||||||||||||||||||||||||||
| save_context_length(model, url, 200000) | ||||||||||||||||||||||||||||||||||||||||||
| assert get_cached_context_length(model, url) == 200000 | ||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||
| class TestCustomProvidersContextLength: | ||||||||||||||||||||||||||||||||||||||||||
| """Tests for config-driven context_length overrides in custom_providers.""" | ||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||
| @patch("hermes_cli.config.load_config") | ||||||||||||||||||||||||||||||||||||||||||
| def test_match_by_provider_name(self, mock_load_config): | ||||||||||||||||||||||||||||||||||||||||||
| mock_load_config.return_value = { | ||||||||||||||||||||||||||||||||||||||||||
| "custom_providers": [ | ||||||||||||||||||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||||||||||||||||||
| "name": "zai-customize", | ||||||||||||||||||||||||||||||||||||||||||
| "base_url": "https://open.bigmodel.cn/api/coding/paas/v4", | ||||||||||||||||||||||||||||||||||||||||||
| "models": { | ||||||||||||||||||||||||||||||||||||||||||
| "glm-5.1": {"context_length": 200000}, | ||||||||||||||||||||||||||||||||||||||||||
| }, | ||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||
| ] | ||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||
| result = get_model_context_length("glm-5.1", provider="zai-customize") | ||||||||||||||||||||||||||||||||||||||||||
| assert result == 200000 | ||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||
| @patch("hermes_cli.config.load_config") | ||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||
| @patch("hermes_cli.config.load_config") | |
| @patch("hermes_cli.config.load_config") | |
| def test_match_by_runtime_custom_provider_key(self, mock_load_config): | |
| mock_load_config.return_value = { | |
| "custom_providers": [ | |
| { | |
| "name": "local-(127.0.0.1:4141)", | |
| "base_url": "http://127.0.0.1:4141/v1", | |
| "models": { | |
| "llama3.2": {"context_length": 8192}, | |
| }, | |
| } | |
| ] | |
| } | |
| result = get_model_context_length( | |
| "llama3.2", provider="custom:local-(127.0.0.1:4141)" | |
| ) | |
| assert result == 8192 | |
| @patch("hermes_cli.config.load_config") |
Copilot
AI
Apr 12, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
test_ambiguous_multiple_providers_skipped_without_scope currently allows get_model_context_length() to fall through into fetch_model_metadata(), which can perform a real network call to OpenRouter (making this unit test slower/flaky depending on environment). Patch agent.model_metadata.fetch_model_metadata to return {} (or patch requests.get) so the test is fully deterministic and offline.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This block calls
hermes_cli.config.load_config()on everyget_model_context_length()invocation, which does a deep-merge with defaults and reads from disk. Since this function can be called in hot paths (e.g., gateway request handling), consider using the lightweightread_raw_config()or caching the custom_providers lookup (e.g., via an internal module-level cache with a short TTL) to avoid repeated disk I/O and merges.