diff --git a/CLAUDE.md b/CLAUDE.md index 441f7332e5..302eaa9188 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -234,3 +234,12 @@ Check in pyproject.toml and NEVER use a marker/tag that doesn't exist there. Ask - `runbooks: {}` - No runbooks available (empty catalog) - `runbooks: {catalog: [...]}` - Custom runbook catalog with entries pointing to .md files in the same directory - If `runbooks` field is not specified, default system runbooks are used + +## Documentation Lookup + +When asked about content from the HolmesGPT documentation website (https://robusta-dev.github.io/holmesgpt/), look in the local `docs/` directory: +- Python SDK examples: `docs/installation/python-installation.md` +- CLI installation: `docs/installation/cli-installation.md` +- Kubernetes deployment: `docs/installation/kubernetes-installation.md` +- Toolset documentation: `docs/data-sources/builtin-toolsets/` +- API reference: `docs/reference/` diff --git a/docs/installation/python-installation.md b/docs/installation/python-installation.md index 12a3368322..e3ada2e0c1 100644 --- a/docs/installation/python-installation.md +++ b/docs/installation/python-installation.md @@ -18,7 +18,8 @@ pip install "https://github.com/robusta-dev/holmesgpt/archive/refs/heads/master. ```python import os from holmes.config import Config -from holmes.plugins.prompts import load_and_render_prompt +from holmes.core.prompt import build_initial_ask_messages +from rich.console import Console print("šŸš€ Initializing HolmesGPT...") @@ -31,22 +32,28 @@ config = Config( ) print(f"āœ… Configuration created with model: {config.model}") -# Create AI instance +# Create AI instance and console print("Creating AI instance...") ai = config.create_console_toolcalling_llm() +console = Console() print("āœ… AI instance ready") # Ask a question -print("Loading system prompt...") -system_prompt = load_and_render_prompt( - "builtin://generic_ask.jinja2", - {"toolsets": ai.tool_executor.toolsets} +question = "what pods are failing in production?" +print(f"\nšŸ” Asking: '{question}'") + +# Build initial messages with system prompt +messages = build_initial_ask_messages( + console=console, + initial_user_prompt=question, + file_paths=None, + tool_executor=ai.tool_executor, + runbooks=config.get_runbook_catalog(), + system_prompt_additions=None ) -print("āœ… System prompt loaded") -print("\nšŸ” Asking: 'what pods are failing in production?'") print("Holmes is thinking...") -response = ai.prompt_call(system_prompt, "what pods are failing in production?") +response = ai.call(messages) print(f"Holmes: {response.result}") ``` @@ -62,7 +69,8 @@ Complete example of using HolmesGPT Python SDK with progress tracking import os from holmes.config import Config -from holmes.plugins.prompts import load_and_render_prompt +from holmes.core.prompt import build_initial_ask_messages +from rich.console import Console def main(): print("šŸš€ Starting HolmesGPT Python SDK Example") @@ -81,8 +89,9 @@ def main(): print(f"āœ… Configuration created with model: {config.model}") print("\nStep 2: Creating AI instance...") - # Create AI instance + # Create AI instance and console ai = config.create_console_toolcalling_llm() + console = Console() print("āœ… AI instance created successfully") print("\nStep 3: Listing available toolsets...") @@ -99,16 +108,7 @@ def main(): for tool in sorted(available_tools): print(f" • {tool}") - print("\nStep 5: Loading system prompt...") - # Load system prompt - system_prompt = load_and_render_prompt( - "builtin://generic_ask.jinja2", - {"toolsets": ai.tool_executor.toolsets} - ) - print("āœ… System prompt loaded successfully") - print(f"Prompt length: {len(system_prompt)} characters") - - print("\nStep 6: Asking questions...") + print("\nStep 5: Asking questions...") # Ask questions questions = [ "what pods are failing in production?", @@ -122,7 +122,18 @@ def main(): try: print("Holmes is thinking...") - response = ai.prompt_call(system_prompt, question) + + # Build initial messages + messages = build_initial_ask_messages( + console=console, + initial_user_prompt=question, + file_paths=None, + tool_executor=ai.tool_executor, + runbooks=config.get_runbook_catalog(), + system_prompt_additions=None + ) + + response = ai.call(messages) print(f"Holmes: {response.result}") # Show tools that were used @@ -200,20 +211,19 @@ def main(): ai = config.create_console_toolcalling_llm() console = Console() - # Load system prompt - system_prompt = load_and_render_prompt( - "builtin://generic_ask.jinja2", - {"toolsets": ai.tool_executor.toolsets} - ) - # First question print("\nšŸ” First Question:") first_question = "what pods are failing in my cluster?" print(f"User: {first_question}") - # Build initial messages (system + first user message) + # Build initial messages (includes system prompt + first user message) messages = build_initial_ask_messages( - console, system_prompt, first_question, None + console=console, + initial_user_prompt=first_question, + file_paths=None, + tool_executor=ai.tool_executor, + runbooks=config.get_runbook_catalog(), + system_prompt_additions=None ) # Call AI with initial messages diff --git a/holmes/core/prompt.py b/holmes/core/prompt.py index c9bffd0bf4..e2e8ef4fd0 100644 --- a/holmes/core/prompt.py +++ b/holmes/core/prompt.py @@ -1,11 +1,13 @@ from rich.console import Console -from typing import Optional, List, Dict +from typing import Optional, List, Dict, Any, Union from pathlib import Path +from holmes.plugins.prompts import load_and_render_prompt +from holmes.plugins.runbooks import RunbookCatalog def append_file_to_user_prompt(user_prompt: str, file_path: Path) -> str: with file_path.open("r") as f: - user_prompt += f"\n\n" + user_prompt += f"\n\n\n{f.read()}\n" return user_prompt @@ -25,11 +27,34 @@ def append_all_files_to_user_prompt( def build_initial_ask_messages( console: Console, - system_prompt_rendered: str, initial_user_prompt: str, file_paths: Optional[List[Path]], + tool_executor: Any, # ToolExecutor type + runbooks: Union[RunbookCatalog, Dict, None] = None, + system_prompt_additions: Optional[str] = None, ) -> List[Dict]: - """Build the initial messages for the AI call.""" + """Build the initial messages for the AI call. + + Args: + console: Rich console for output + initial_user_prompt: The user's prompt + file_paths: Optional list of files to include + tool_executor: The tool executor with available toolsets + runbooks: Optional runbook catalog + system_prompt_additions: Optional additional system prompt content + """ + # Load and render system prompt internally + system_prompt_template = "builtin://generic_ask.jinja2" + template_context = { + "toolsets": tool_executor.toolsets, + "runbooks": runbooks or {}, + "system_prompt_additions": system_prompt_additions or "", + } + system_prompt_rendered = load_and_render_prompt( + system_prompt_template, template_context + ) + + # Append files to user prompt user_prompt_with_files = append_all_files_to_user_prompt( console, initial_user_prompt, file_paths ) diff --git a/holmes/interactive.py b/holmes/interactive.py index 522f33aee8..4171e67092 100644 --- a/holmes/interactive.py +++ b/holmes/interactive.py @@ -761,12 +761,13 @@ def display_recent_tool_outputs( def run_interactive_loop( ai: ToolCallingLLM, console: Console, - system_prompt_rendered: str, initial_user_input: Optional[str], include_files: Optional[List[Path]], post_processing_prompt: Optional[str], show_tool_output: bool, tracer=None, + runbooks=None, + system_prompt_additions: Optional[str] = None, ) -> None: # Initialize tracer - use DummyTracer if no tracer provided if tracer is None: @@ -965,7 +966,12 @@ def get_bottom_toolbar(): if messages is None: messages = build_initial_ask_messages( - console, system_prompt_rendered, user_input, include_files + console, + user_input, + include_files, + ai.tool_executor, + runbooks, + system_prompt_additions, ) else: messages.append({"role": "user", "content": user_input}) diff --git a/holmes/main.py b/holmes/main.py index d510c7e779..9f8adecf25 100644 --- a/holmes/main.py +++ b/holmes/main.py @@ -180,10 +180,6 @@ def ask( destination: Optional[DestinationType] = opt_destination, slack_token: Optional[str] = opt_slack_token, slack_channel: Optional[str] = opt_slack_channel, - # advanced options for this command - system_prompt: Optional[str] = typer.Option( - "builtin://generic_ask.jinja2", help=system_prompt_help - ), show_tool_output: bool = typer.Option( False, "--show-tool-output", @@ -214,6 +210,11 @@ def ask( "--trace", help="Enable tracing to the specified provider (e.g., 'braintrust')", ), + system_prompt_additions: Optional[str] = typer.Option( + None, + "--system-prompt-additions", + help="Additional content to append to the system prompt", + ), ): """ Ask any question and answer using available tools @@ -252,12 +253,6 @@ def ask( refresh_toolsets=refresh_toolsets, # flag to refresh the toolset status tracer=tracer, ) - template_context = { - "toolsets": ai.tool_executor.toolsets, - "runbooks": config.get_runbook_catalog(), - } - - system_prompt_rendered = load_and_render_prompt(system_prompt, template_context) # type: ignore if prompt_file and prompt: raise typer.BadParameter( @@ -292,20 +287,23 @@ def ask( run_interactive_loop( ai, console, - system_prompt_rendered, prompt, include_file, post_processing_prompt, show_tool_output, tracer, + config.get_runbook_catalog(), + system_prompt_additions, ) return messages = build_initial_ask_messages( console, - system_prompt_rendered, prompt, # type: ignore include_file, + ai.tool_executor, + config.get_runbook_catalog(), + system_prompt_additions, ) with tracer.start_trace( diff --git a/holmes/plugins/prompts/generic_ask.jinja2 b/holmes/plugins/prompts/generic_ask.jinja2 index 07a0fe7670..fb7d9b35c8 100644 --- a/holmes/plugins/prompts/generic_ask.jinja2 +++ b/holmes/plugins/prompts/generic_ask.jinja2 @@ -36,3 +36,7 @@ Relevant logs: ``` Validation error led to unhandled Java exception causing a crash. + +{% if system_prompt_additions %} +{{ system_prompt_additions }} +{% endif %} diff --git a/tests/core/test_prompt.py b/tests/core/test_prompt.py new file mode 100644 index 0000000000..4e22096057 --- /dev/null +++ b/tests/core/test_prompt.py @@ -0,0 +1,185 @@ +import pytest +from unittest.mock import Mock +from rich.console import Console + +from holmes.core.prompt import ( + build_initial_ask_messages, + append_file_to_user_prompt, + append_all_files_to_user_prompt, +) + + +@pytest.fixture +def console(): + return Console(force_terminal=False, force_jupyter=False) + + +@pytest.fixture +def mock_tool_executor(): + tool_executor = Mock() + tool_executor.toolsets = [] + return tool_executor + + +def test_build_initial_ask_messages_basic(console, mock_tool_executor): + """Test basic message building without any optional parameters.""" + messages = build_initial_ask_messages( + console, + "Test prompt", + None, + mock_tool_executor, + None, + None, + ) + + assert len(messages) == 2 + assert messages[0]["role"] == "system" + assert messages[1]["role"] == "user" + assert messages[1]["content"] == "Test prompt" + + +def test_build_initial_ask_messages_with_system_prompt_additions( + console, mock_tool_executor +): + """Test message building with system prompt additions.""" + system_additions = "Additional system instructions here." + messages = build_initial_ask_messages( + console, + "Test prompt", + None, + mock_tool_executor, + None, + system_additions, + ) + + assert len(messages) == 2 + assert messages[0]["role"] == "system" + # Check for unique word from the system additions + assert "Additional" in messages[0]["content"] + assert messages[1]["role"] == "user" + assert messages[1]["content"] == "Test prompt" + + +def test_build_initial_ask_messages_with_file(console, mock_tool_executor, tmp_path): + """Test message building with file attachment.""" + # Create a temporary file + test_file = tmp_path / "test.txt" + test_file.write_text("File content here") + + messages = build_initial_ask_messages( + console, + "Test prompt", + [test_file], + mock_tool_executor, + None, + None, + ) + + assert len(messages) == 2 + assert messages[0]["role"] == "system" + assert messages[1]["role"] == "user" + assert "Test prompt" in messages[1]["content"] + assert "File content here" in messages[1]["content"] + # Check for file attachment markers + assert "" in messages[1]["content"] + + +def test_build_initial_ask_messages_with_runbooks(console, mock_tool_executor): + """Test message building with runbooks.""" + runbooks = {"test_runbook": {"description": "Test runbook"}} + + messages = build_initial_ask_messages( + console, + "Test prompt", + None, + mock_tool_executor, + runbooks, + None, + ) + + assert len(messages) == 2 + assert messages[0]["role"] == "system" + # The runbook should be passed to the template context + assert messages[1]["role"] == "user" + assert messages[1]["content"] == "Test prompt" + + +def test_build_initial_ask_messages_all_parameters( + console, mock_tool_executor, tmp_path +): + """Test message building with all parameters.""" + # Create a temporary file + test_file = tmp_path / "test.txt" + test_file.write_text("File content") + + runbooks = {"test_runbook": {"description": "Test runbook"}} + system_additions = "Extra system instructions" + + messages = build_initial_ask_messages( + console, + "Test prompt", + [test_file], + mock_tool_executor, + runbooks, + system_additions, + ) + + assert len(messages) == 2 + assert messages[0]["role"] == "system" + # Check for unique word from system additions + assert "Extra" in messages[0]["content"] + assert messages[1]["role"] == "user" + assert "Test prompt" in messages[1]["content"] + assert "File content" in messages[1]["content"] + + +def test_append_file_to_user_prompt(tmp_path): + """Test appending a single file to user prompt.""" + test_file = tmp_path / "test.txt" + test_file.write_text("Test file content") + + prompt = "Original prompt" + result = append_file_to_user_prompt(prompt, test_file) + + assert "Original prompt" in result + assert "Test file content" in result + # Check for file attachment markers + assert "" in result + + +def test_append_all_files_to_user_prompt(console, tmp_path): + """Test appending multiple files to user prompt.""" + # Create multiple test files + file1 = tmp_path / "file1.txt" + file1.write_text("Content 1") + + file2 = tmp_path / "file2.txt" + file2.write_text("Content 2") + + prompt = "Original prompt" + result = append_all_files_to_user_prompt(console, prompt, [file1, file2]) + + assert "Original prompt" in result + assert "Content 1" in result + assert "Content 2" in result + # Check for file attachment markers + assert "") == 2 + + +def test_append_all_files_to_user_prompt_no_files(console): + """Test appending files when no files are provided.""" + prompt = "Original prompt" + result = append_all_files_to_user_prompt(console, prompt, None) + + assert result == "Original prompt" + + # Also test with empty list + result = append_all_files_to_user_prompt(console, prompt, []) + assert result == "Original prompt" diff --git a/tests/llm/test_ask_holmes.py b/tests/llm/test_ask_holmes.py index ba8ebc652a..b13bdc41a1 100644 --- a/tests/llm/test_ask_holmes.py +++ b/tests/llm/test_ask_holmes.py @@ -25,8 +25,10 @@ Evaluation, MockHelper, check_and_skip_test, - build_initial_ask_messages2, ) + +from holmes.core.prompt import build_initial_ask_messages + from tests.llm.utils.property_manager import ( set_initial_properties, update_test_results, @@ -163,6 +165,7 @@ def test_ask_holmes( expected=test_case.expected_output, dataset_record_id=test_case.id, scores={}, + # metadata={"tags": test_case.tags}, ) except Exception: pass # Don't fail the test due to logging issues @@ -227,6 +230,7 @@ def test_ask_holmes( dataset_record_id=test_case.id, scores=scores, metadata={"system_prompt": prompt}, + # metadata={"tags": test_case.tags}, ) # Print tool calls summary @@ -311,7 +315,7 @@ def ask_holmes( runbook_catalog = load_runbook_catalog() runbooks = runbook_catalog.model_dump() if runbook_catalog else {} - messages = build_initial_ask_messages2( + messages = build_initial_ask_messages( console, test_case.user_prompt, None, diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 4854ad2eb5..5b714b9ae9 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -10,17 +10,11 @@ import pytest from pydantic import BaseModel, TypeAdapter from holmes.core.models import InvestigateRequest, WorkloadHealthRequest -from holmes.core.prompt import ( - append_all_files_to_user_prompt, - append_file_to_user_prompt, -) +from holmes.core.prompt import append_file_to_user_prompt + from holmes.core.tool_calling_llm import ResourceInstructions -from holmes.plugins.runbooks import RunbookCatalog -from holmes.plugins.prompts import load_and_render_prompt from tests.llm.utils.constants import ALLOWED_EVAL_TAGS -from rich.console import Console - def read_file(file_path: Path): with open(file_path, "r", encoding="utf-8") as file: @@ -285,43 +279,3 @@ def load_include_files( extra_prompt = append_file_to_user_prompt(extra_prompt, file_path) return extra_prompt - - -# temporary until we merge https://github.com/robusta-dev/holmesgpt/pull/727 -def build_initial_ask_messages2( - console: Console, - initial_user_prompt: str, - file_paths: Optional[List[Path]], - tool_executor: Any, # ToolExecutor type - runbooks: Union[RunbookCatalog, Dict, None] = None, -) -> List[Dict]: - """Build the initial messages for the AI call. - - Args: - console: Rich console for output - initial_user_prompt: The user's prompt - file_paths: Optional list of files to include - tool_executor: The tool executor with available toolsets - runbooks: Optional runbook catalog - """ - # Load and render system prompt internally - system_prompt_template = "builtin://generic_ask.jinja2" - template_context = { - "toolsets": tool_executor.toolsets, - "runbooks": runbooks or {}, - } - system_prompt_rendered = load_and_render_prompt( - system_prompt_template, template_context - ) - - # Append files to user prompt - user_prompt_with_files = append_all_files_to_user_prompt( - console, initial_user_prompt, file_paths - ) - - messages = [ - {"role": "system", "content": system_prompt_rendered}, - {"role": "user", "content": user_prompt_with_files}, - ] - - return messages