[ROB-2379] fixed issue not using model list from cli

docs/installation/cli-installation.md

@@ -250,6 +250,51 @@ _After running the command, HolmesGPT begins its automated investigation, as sho
 _Once the analysis completes, HolmesGPT provides a clear root-cause summary and fix suggestions._
 ![image](../assets/cli-installation/cli-analysis-result.png)
 
+## Using Model Lists
+
+You can define multiple models in a YAML file and reference them by name in the CLI. This is useful when you have multiple model configurations with different API keys, endpoints, or parameters.
+
+**1. Create a model list file:**
+
+```yaml
+# model_list.yaml
+sonnet:
+  aws_access_key_id: 'your-access-key'
+  aws_region_name: us-east-1
+  aws_secret_access_key: 'your-secret-key'
+  model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
+  temperature: 1
+  thinking:
+    budget_tokens: 10000
+    type: enabled
+
+azure-5:
+  api_base: https://your-resource.openai.azure.com
+  api_key: 'your-api-key'
+  api_version: 2025-01-01-preview
+  model: azure/gpt-5
+  temperature: 0
+```
+
+**2. Set the environment variable:**
+
+```bash
+export MODEL_LIST_FILE_LOCATION="/path/to/model_list.yaml"
+```
+
+**3. Use models by name:**
+
+```bash
+holmes ask "what pods are failing?" --model=sonnet --no-interactive
+holmes ask "analyze deployment" --model=azure-5 --no-interactive
+```
+
+When using `--model`, specify the model name (key) from your YAML file, not the underlying model identifier. All configuration (API keys, endpoints, temperature, etc.) will be automatically loaded from the model list file.
+
+**Note:** Environment variable substitution is supported using `{{ env.VARIABLE_NAME }}` syntax in the model list file.
+
+See [Environment Variables Reference](../reference/environment-variables.md#model_list_file_location) for more details.
+
 ## Next Steps
 
 -   **[Add Data Sources](../data-sources/index.md)** - We encourage you to use built-in toolsets to connect with [AWS](https://holmesgpt.dev/data-sources/builtin-toolsets/aws/), [Prometheus](https://holmesgpt.dev/data-sources/builtin-toolsets/prometheus/), [Loki](https://holmesgpt.dev/data-sources/builtin-toolsets/grafanaloki/), [NewRelic](https://holmesgpt.dev/data-sources/builtin-toolsets/newrelic/), [DataDog](https://holmesgpt.dev/data-sources/builtin-toolsets/datadog/), [ArgoCD](https://holmesgpt.dev/data-sources/builtin-toolsets/argocd/), [Confluence](https://holmesgpt.dev/data-sources/builtin-toolsets/confluence/), and other monitoring tools.

holmes/config.py

@@ -285,12 +285,15 @@ def create_console_toolcalling_llm(
         dal: Optional["SupabaseDal"] = None,
         refresh_toolsets: bool = False,
         tracer=None,
+        model_name: Optional[str] = None,
     ) -> "ToolCallingLLM":
         tool_executor = self.create_console_tool_executor(dal, refresh_toolsets)
         from holmes.core.tool_calling_llm import ToolCallingLLM
 
         return ToolCallingLLM(
-            tool_executor, self.max_steps, self._get_llm(tracer=tracer)
+            tool_executor,
+            self.max_steps,
+            self._get_llm(tracer=tracer, model_key=model_name),
         )
 
     def create_agui_toolcalling_llm(
@@ -344,7 +347,7 @@ def create_issue_investigator(
         )
 
     def create_console_issue_investigator(
-        self, dal: Optional["SupabaseDal"] = None
+        self, dal: Optional["SupabaseDal"] = None, model_name: Optional[str] = None
     ) -> "IssueInvestigator":
         all_runbooks = load_builtin_runbooks()
         for runbook_path in self.custom_runbooks:
@@ -360,7 +363,7 @@ def create_console_issue_investigator(
             tool_executor=tool_executor,
             runbook_manager=runbook_manager,
             max_steps=self.max_steps,
-            llm=self._get_llm(),
+            llm=self._get_llm(model_key=model_name),
             cluster_name=self.cluster_name,
         )
 
@@ -478,7 +481,6 @@ def _get_llm(self, model_key: Optional[str] = None, tracer=None) -> "DefaultLLM"
         model_params = model_entry.model_dump(exclude_none=True)
         api_base = self.api_base
         api_version = self.api_version
-
         is_robusta_model = model_params.pop("is_robusta_model", False)
         sentry_sdk.set_tag("is_robusta_model", is_robusta_model)
         if is_robusta_model:

holmes/core/llm.py

@@ -551,6 +551,9 @@ def _init_models(self):
 
     def _should_load_config_model(self) -> bool:
         if self.config.model is not None:
+            if self._llms and self.config.model in self._llms:
+                # model already loaded from file
+                return False
             return True
 
         # backward compatibility - in the past config.model was set by default to gpt-4o.

holmes/main.py

@@ -261,6 +261,7 @@ def ask(
         dal=None,  # type: ignore
         refresh_toolsets=refresh_toolsets,  # flag to refresh the toolset status
         tracer=tracer,
+        model_name=model,
     )
 
     if prompt_file and prompt:
@@ -413,7 +414,7 @@ def alertmanager(
         custom_runbooks=custom_runbooks,
     )
 
-    ai = config.create_console_issue_investigator()  # type: ignore
+    ai = config.create_console_issue_investigator(model_name=model)  # type: ignore
 
     source = config.create_alertmanager_source()
 
@@ -542,7 +543,7 @@ def jira(
         custom_toolsets_from_cli=custom_toolsets,
         custom_runbooks=custom_runbooks,
     )
-    ai = config.create_console_issue_investigator()  # type: ignore
+    ai = config.create_console_issue_investigator(model_name=model)  # type: ignore
     source = config.create_jira_source()
     try:
         issues = source.fetch_issues()
@@ -618,6 +619,7 @@ def ticket(
         "builtin://generic_ticket.jinja2", help=system_prompt_help
     ),
     post_processing_prompt: Optional[str] = opt_post_processing_prompt,
+    model: Optional[str] = opt_model,
 ):
     """
     Fetch and print a Jira ticket from the specified source.
@@ -658,7 +660,7 @@ def ticket(
         },
     )
 
-    ai = ticket_source.config.create_console_issue_investigator()
+    ai = ticket_source.config.create_console_issue_investigator(model_name=model)
     console.print(
         f"[bold yellow]Analyzing ticket: {issue_to_investigate.name}...[/bold yellow]"
     )
@@ -733,7 +735,7 @@ def github(
         custom_toolsets_from_cli=custom_toolsets,
         custom_runbooks=custom_runbooks,
     )
-    ai = config.create_console_issue_investigator()
+    ai = config.create_console_issue_investigator(model_name=model)
     source = config.create_github_source()
     try:
         issues = source.fetch_issues()
@@ -817,7 +819,7 @@ def pagerduty(
         custom_toolsets_from_cli=custom_toolsets,
         custom_runbooks=custom_runbooks,
     )
-    ai = config.create_console_issue_investigator()
+    ai = config.create_console_issue_investigator(model_name=model)
     source = config.create_pagerduty_source()
     try:
         issues = source.fetch_issues()
@@ -903,7 +905,7 @@ def opsgenie(
         custom_toolsets_from_cli=custom_toolsets,
         custom_runbooks=custom_runbooks,
     )
-    ai = config.create_console_issue_investigator()
+    ai = config.create_console_issue_investigator(model_name=model)
     source = config.create_opsgenie_source()
     try:
         issues = source.fetch_issues()

tests/config_class/test_config_load_model_list.py

@@ -1,8 +1,77 @@
+from contextlib import contextmanager
+from unittest.mock import MagicMock, patch
+
+import pytest
 import yaml
 
 from holmes.common.env_vars import ROBUSTA_API_ENDPOINT
 from holmes.config import Config
 
+# Global model configs for reuse across tests
+SONNET_MODEL_CONFIG = {
+    "aws_access_key_id": "test-access-key",
+    "aws_region_name": "us-east-1",
+    "aws_secret_access_key": "test-secret-key",
+    "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+    "temperature": 1,
+    "thinking": {
+        "budget_tokens": 10000,
+        "type": "enabled",
+    },
+}
+
+AZURE_MODEL_CONFIG = {
+    "api_base": "https://test.openai.azure.com",
+    "api_key": "test-azure-key",
+    "api_version": "2025-01-01-preview",
+    "model": "azure/gpt-5",
+    "temperature": 0,
+}
+
+
+def _get_model_list_data():
+    """Shared model list data with sonnet (Bedrock) and azure-5 (Azure) models."""
+    return {
+        "sonnet": SONNET_MODEL_CONFIG,
+        "azure-5": AZURE_MODEL_CONFIG,
+    }
+
+
+@contextmanager
+def _setup_model_list_file(monkeypatch, tmp_path, data=None):
+    """Context manager to create and configure model list file, ensuring cleanup."""
+    if data is None:
+        data = _get_model_list_data()
+    temp_config_file = tmp_path / "model_list.yaml"
+    temp_config_file.write_text(yaml.dump(data))
+    monkeypatch.setattr(
+        "holmes.core.llm.MODEL_LIST_FILE_LOCATION", str(temp_config_file)
+    )
+    try:
+        yield temp_config_file
+    finally:
+        if temp_config_file.exists():
+            temp_config_file.unlink()
+
+
+def _get_mock_llm():
+    """Helper to create a mock LLM instance."""
+    mock_llm_instance = MagicMock()
+    mock_llm_instance.get_context_window_size.return_value = 128000
+    mock_llm_instance.get_maximum_output_token.return_value = 4096
+    return mock_llm_instance
+
+
+def _assert_model_config_matches_call_kwargs(call_kwargs, model_config, model_name):
+    """Assert that call_kwargs correctly uses all values from model_config."""
+    assert call_kwargs["name"] == model_name
+
+    for key, value in model_config.items():
+        if key in ["model", "api_base", "api_version", "api_key"]:
+            assert call_kwargs[key] == value
+        else:
+            assert call_kwargs["args"][key] == value
+
 
 def test_load_custom_toolsets_config_valid(monkeypatch, tmp_path):
     temp_config_file = tmp_path / "custom_toolset.yaml"
@@ -45,3 +114,68 @@ def test_config_load_model_list_valid_with_robusta_ai(server_config: Config):
         server_config.llm_model_registry.default_robusta_model
         == "Robusta/gpt-5-mini preview (minimal reasoning)"
     )
+
+
+def test_model_list_file_location_env_var(monkeypatch, tmp_path):
+    """Test that MODEL_LIST_FILE_LOCATION environment variable is used to load models."""
+    with _setup_model_list_file(monkeypatch, tmp_path) as temp_config_file:
+        monkeypatch.setenv("MODEL_LIST_FILE_LOCATION", str(temp_config_file))
+
+        config = Config()
+
+        assert isinstance(config.llm_model_registry.models, dict)
+        assert len(config.llm_model_registry.models) == 2
+        assert "sonnet" in config.llm_model_registry.models
+        assert "azure-5" in config.llm_model_registry.models
+
+
+@pytest.mark.parametrize(
+    "model_name,model_config",
+    [
+        ("sonnet", SONNET_MODEL_CONFIG),
+        ("azure-5", AZURE_MODEL_CONFIG),
+    ],
+)
+def test_model_selection_uses_correct_params(
+    monkeypatch, tmp_path, model_name, model_config
+):
+    """Test that selecting a model from the list uses the correct params for that model."""
+    with _setup_model_list_file(monkeypatch, tmp_path):
+        config = Config()
+
+        with patch("holmes.config.DefaultLLM") as mock_default_llm:
+            mock_default_llm.return_value = _get_mock_llm()
+            config._get_llm(model_name)
+
+            call_kwargs = mock_default_llm.call_args[1]
+            _assert_model_config_matches_call_kwargs(
+                call_kwargs, model_config, model_name
+            )
+
+
+@pytest.mark.parametrize(
+    "model_name,model_config",
+    [
+        ("sonnet", SONNET_MODEL_CONFIG),
+        ("azure-5", AZURE_MODEL_CONFIG),
+    ],
+)
+def test_create_console_toolcalling_llm_with_model_from_list(
+    monkeypatch, tmp_path, model_name, model_config
+):
+    """Test that create_console_toolcalling_llm correctly uses model from model list."""
+    with _setup_model_list_file(monkeypatch, tmp_path):
+        config = Config()
+
+        with patch("holmes.config.DefaultLLM") as mock_default_llm, patch(
+            "holmes.config.Config.create_console_tool_executor"
+        ) as mock_tool_executor:
+            mock_default_llm.return_value = _get_mock_llm()
+            mock_tool_executor.return_value = MagicMock()
+
+            config.create_console_toolcalling_llm(model_name=model_name)
+
+            call_kwargs = mock_default_llm.call_args[1]
+            _assert_model_config_matches_call_kwargs(
+                call_kwargs, model_config, model_name
+            )