From 9e8808591de62076228706793a8f7981220edf79 Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Sun, 2 Nov 2025 13:56:53 +0200 Subject: [PATCH 1/4] fixed issue --- holmes/config.py | 11 +++++++---- holmes/core/llm.py | 3 +++ holmes/main.py | 14 ++++++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/holmes/config.py b/holmes/config.py index 729c99e2af..b0f8d78c9f 100644 --- a/holmes/config.py +++ b/holmes/config.py @@ -285,12 +285,15 @@ def create_console_toolcalling_llm( dal: Optional["SupabaseDal"] = None, refresh_toolsets: bool = False, tracer=None, + model_name: Optional[str] = None, ) -> "ToolCallingLLM": tool_executor = self.create_console_tool_executor(dal, refresh_toolsets) from holmes.core.tool_calling_llm import ToolCallingLLM return ToolCallingLLM( - tool_executor, self.max_steps, self._get_llm(tracer=tracer) + tool_executor, + self.max_steps, + self._get_llm(tracer=tracer, model_key=model_name), ) def create_agui_toolcalling_llm( @@ -344,7 +347,7 @@ def create_issue_investigator( ) def create_console_issue_investigator( - self, dal: Optional["SupabaseDal"] = None + self, dal: Optional["SupabaseDal"] = None, model_name: Optional[str] = None ) -> "IssueInvestigator": all_runbooks = load_builtin_runbooks() for runbook_path in self.custom_runbooks: @@ -360,7 +363,7 @@ def create_console_issue_investigator( tool_executor=tool_executor, runbook_manager=runbook_manager, max_steps=self.max_steps, - llm=self._get_llm(), + llm=self._get_llm(model_key=model_name), cluster_name=self.cluster_name, ) @@ -478,7 +481,7 @@ def _get_llm(self, model_key: Optional[str] = None, tracer=None) -> "DefaultLLM" model_params = model_entry.model_dump(exclude_none=True) api_base = self.api_base api_version = self.api_version - + print(f"model_entry {model_entry}") is_robusta_model = model_params.pop("is_robusta_model", False) sentry_sdk.set_tag("is_robusta_model", is_robusta_model) if is_robusta_model: diff --git a/holmes/core/llm.py b/holmes/core/llm.py index 8b912a21dc..b982552a43 100644 --- a/holmes/core/llm.py +++ b/holmes/core/llm.py @@ -549,6 +549,9 @@ def _init_models(self): def _should_load_config_model(self) -> bool: if self.config.model is not None: + if self._llms and self._llms[self.config.model]: + # model already loaded from file + return False return True # backward compatibility - in the past config.model was set by default to gpt-4o. diff --git a/holmes/main.py b/holmes/main.py index cdab81ba6e..bc5477760f 100644 --- a/holmes/main.py +++ b/holmes/main.py @@ -261,6 +261,7 @@ def ask( dal=None, # type: ignore refresh_toolsets=refresh_toolsets, # flag to refresh the toolset status tracer=tracer, + model_name=model, ) if prompt_file and prompt: @@ -413,7 +414,7 @@ def alertmanager( custom_runbooks=custom_runbooks, ) - ai = config.create_console_issue_investigator() # type: ignore + ai = config.create_console_issue_investigator(model_name=model) # type: ignore source = config.create_alertmanager_source() @@ -542,7 +543,7 @@ def jira( custom_toolsets_from_cli=custom_toolsets, custom_runbooks=custom_runbooks, ) - ai = config.create_console_issue_investigator() # type: ignore + ai = config.create_console_issue_investigator(model_name=model) # type: ignore source = config.create_jira_source() try: issues = source.fetch_issues() @@ -618,6 +619,7 @@ def ticket( "builtin://generic_ticket.jinja2", help=system_prompt_help ), post_processing_prompt: Optional[str] = opt_post_processing_prompt, + model: Optional[str] = opt_model, ): """ Fetch and print a Jira ticket from the specified source. @@ -658,7 +660,7 @@ def ticket( }, ) - ai = ticket_source.config.create_console_issue_investigator() + ai = ticket_source.config.create_console_issue_investigator(model_name=model) console.print( f"[bold yellow]Analyzing ticket: {issue_to_investigate.name}...[/bold yellow]" ) @@ -733,7 +735,7 @@ def github( custom_toolsets_from_cli=custom_toolsets, custom_runbooks=custom_runbooks, ) - ai = config.create_console_issue_investigator() + ai = config.create_console_issue_investigator(model_name=model) source = config.create_github_source() try: issues = source.fetch_issues() @@ -817,7 +819,7 @@ def pagerduty( custom_toolsets_from_cli=custom_toolsets, custom_runbooks=custom_runbooks, ) - ai = config.create_console_issue_investigator() + ai = config.create_console_issue_investigator(model_name=model) source = config.create_pagerduty_source() try: issues = source.fetch_issues() @@ -903,7 +905,7 @@ def opsgenie( custom_toolsets_from_cli=custom_toolsets, custom_runbooks=custom_runbooks, ) - ai = config.create_console_issue_investigator() + ai = config.create_console_issue_investigator(model_name=model) source = config.create_opsgenie_source() try: issues = source.fetch_issues() From 0d06981d23422ae04c480b91f28bf1709729a2ff Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Sun, 2 Nov 2025 13:59:11 +0200 Subject: [PATCH 2/4] removed test log --- holmes/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/holmes/config.py b/holmes/config.py index b0f8d78c9f..0b225ac758 100644 --- a/holmes/config.py +++ b/holmes/config.py @@ -481,7 +481,6 @@ def _get_llm(self, model_key: Optional[str] = None, tracer=None) -> "DefaultLLM" model_params = model_entry.model_dump(exclude_none=True) api_base = self.api_base api_version = self.api_version - print(f"model_entry {model_entry}") is_robusta_model = model_params.pop("is_robusta_model", False) sentry_sdk.set_tag("is_robusta_model", is_robusta_model) if is_robusta_model: From 8dc6d4d75e7e70a3f21686058006f3956eeefa91 Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Sun, 2 Nov 2025 14:16:43 +0200 Subject: [PATCH 3/4] fixed issue --- holmes/core/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holmes/core/llm.py b/holmes/core/llm.py index b982552a43..9ded2aff98 100644 --- a/holmes/core/llm.py +++ b/holmes/core/llm.py @@ -549,7 +549,7 @@ def _init_models(self): def _should_load_config_model(self) -> bool: if self.config.model is not None: - if self._llms and self._llms[self.config.model]: + if self._llms and self.config.model in self._llms: # model already loaded from file return False return True From 61be8c164053bde395bc28ca99e67d3b5515e44f Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Wed, 19 Nov 2025 13:09:03 +0200 Subject: [PATCH 4/4] add tests and docs --- docs/installation/cli-installation.md | 45 ++++++ .../test_config_load_model_list.py | 134 ++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/docs/installation/cli-installation.md b/docs/installation/cli-installation.md index 0efc134c01..d52096c29d 100644 --- a/docs/installation/cli-installation.md +++ b/docs/installation/cli-installation.md @@ -235,6 +235,51 @@ Choose your AI provider (see [all providers](../ai-providers/index.md) for more > **Warning:** Ollama can be tricky to configure correctly. We recommend trying HolmesGPT with a hosted model first (like Claude or OpenAI) to ensure everything works before switching to Ollama. +## Using Model Lists + +You can define multiple models in a YAML file and reference them by name in the CLI. This is useful when you have multiple model configurations with different API keys, endpoints, or parameters. + +**1. Create a model list file:** + +```yaml +# model_list.yaml +sonnet: + aws_access_key_id: 'your-access-key' + aws_region_name: us-east-1 + aws_secret_access_key: 'your-secret-key' + model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0 + temperature: 1 + thinking: + budget_tokens: 10000 + type: enabled + +azure-5: + api_base: https://your-resource.openai.azure.com + api_key: 'your-api-key' + api_version: 2025-01-01-preview + model: azure/gpt-5 + temperature: 0 +``` + +**2. Set the environment variable:** + +```bash +export MODEL_LIST_FILE_LOCATION="/path/to/model_list.yaml" +``` + +**3. Use models by name:** + +```bash +holmes ask "what pods are failing?" --model=sonnet --no-interactive +holmes ask "analyze deployment" --model=azure-5 --no-interactive +``` + +When using `--model`, specify the model name (key) from your YAML file, not the underlying model identifier. All configuration (API keys, endpoints, temperature, etc.) will be automatically loaded from the model list file. + +**Note:** Environment variable substitution is supported using `{{ env.VARIABLE_NAME }}` syntax in the model list file. + +See [Environment Variables Reference](../reference/environment-variables.md#model_list_file_location) for more details. + ## Next Steps - **[Add Data Sources](../data-sources/index.md)** - Use built-in toolsets to connect with ArgoCD, Confluence, and monitoring tools diff --git a/tests/config_class/test_config_load_model_list.py b/tests/config_class/test_config_load_model_list.py index 6f116a574e..77771ef4cf 100644 --- a/tests/config_class/test_config_load_model_list.py +++ b/tests/config_class/test_config_load_model_list.py @@ -1,8 +1,77 @@ +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest import yaml from holmes.common.env_vars import ROBUSTA_API_ENDPOINT from holmes.config import Config +# Global model configs for reuse across tests +SONNET_MODEL_CONFIG = { + "aws_access_key_id": "test-access-key", + "aws_region_name": "us-east-1", + "aws_secret_access_key": "test-secret-key", + "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "temperature": 1, + "thinking": { + "budget_tokens": 10000, + "type": "enabled", + }, +} + +AZURE_MODEL_CONFIG = { + "api_base": "https://test.openai.azure.com", + "api_key": "test-azure-key", + "api_version": "2025-01-01-preview", + "model": "azure/gpt-5", + "temperature": 0, +} + + +def _get_model_list_data(): + """Shared model list data with sonnet (Bedrock) and azure-5 (Azure) models.""" + return { + "sonnet": SONNET_MODEL_CONFIG, + "azure-5": AZURE_MODEL_CONFIG, + } + + +@contextmanager +def _setup_model_list_file(monkeypatch, tmp_path, data=None): + """Context manager to create and configure model list file, ensuring cleanup.""" + if data is None: + data = _get_model_list_data() + temp_config_file = tmp_path / "model_list.yaml" + temp_config_file.write_text(yaml.dump(data)) + monkeypatch.setattr( + "holmes.core.llm.MODEL_LIST_FILE_LOCATION", str(temp_config_file) + ) + try: + yield temp_config_file + finally: + if temp_config_file.exists(): + temp_config_file.unlink() + + +def _get_mock_llm(): + """Helper to create a mock LLM instance.""" + mock_llm_instance = MagicMock() + mock_llm_instance.get_context_window_size.return_value = 128000 + mock_llm_instance.get_maximum_output_token.return_value = 4096 + return mock_llm_instance + + +def _assert_model_config_matches_call_kwargs(call_kwargs, model_config, model_name): + """Assert that call_kwargs correctly uses all values from model_config.""" + assert call_kwargs["name"] == model_name + + for key, value in model_config.items(): + if key in ["model", "api_base", "api_version", "api_key"]: + assert call_kwargs[key] == value + else: + assert call_kwargs["args"][key] == value + def test_load_custom_toolsets_config_valid(monkeypatch, tmp_path): temp_config_file = tmp_path / "custom_toolset.yaml" @@ -45,3 +114,68 @@ def test_config_load_model_list_valid_with_robusta_ai(server_config: Config): server_config.llm_model_registry.default_robusta_model == "Robusta/gpt-5-mini preview (minimal reasoning)" ) + + +def test_model_list_file_location_env_var(monkeypatch, tmp_path): + """Test that MODEL_LIST_FILE_LOCATION environment variable is used to load models.""" + with _setup_model_list_file(monkeypatch, tmp_path) as temp_config_file: + monkeypatch.setenv("MODEL_LIST_FILE_LOCATION", str(temp_config_file)) + + config = Config() + + assert isinstance(config.llm_model_registry.models, dict) + assert len(config.llm_model_registry.models) == 2 + assert "sonnet" in config.llm_model_registry.models + assert "azure-5" in config.llm_model_registry.models + + +@pytest.mark.parametrize( + "model_name,model_config", + [ + ("sonnet", SONNET_MODEL_CONFIG), + ("azure-5", AZURE_MODEL_CONFIG), + ], +) +def test_model_selection_uses_correct_params( + monkeypatch, tmp_path, model_name, model_config +): + """Test that selecting a model from the list uses the correct params for that model.""" + with _setup_model_list_file(monkeypatch, tmp_path): + config = Config() + + with patch("holmes.config.DefaultLLM") as mock_default_llm: + mock_default_llm.return_value = _get_mock_llm() + config._get_llm(model_name) + + call_kwargs = mock_default_llm.call_args[1] + _assert_model_config_matches_call_kwargs( + call_kwargs, model_config, model_name + ) + + +@pytest.mark.parametrize( + "model_name,model_config", + [ + ("sonnet", SONNET_MODEL_CONFIG), + ("azure-5", AZURE_MODEL_CONFIG), + ], +) +def test_create_console_toolcalling_llm_with_model_from_list( + monkeypatch, tmp_path, model_name, model_config +): + """Test that create_console_toolcalling_llm correctly uses model from model list.""" + with _setup_model_list_file(monkeypatch, tmp_path): + config = Config() + + with patch("holmes.config.DefaultLLM") as mock_default_llm, patch( + "holmes.config.Config.create_console_tool_executor" + ) as mock_tool_executor: + mock_default_llm.return_value = _get_mock_llm() + mock_tool_executor.return_value = MagicMock() + + config.create_console_toolcalling_llm(model_name=model_name) + + call_kwargs = mock_default_llm.call_args[1] + _assert_model_config_matches_call_kwargs( + call_kwargs, model_config, model_name + )