diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index e776aa76c9..f2e6e73de0 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -91,7 +91,6 @@ jobs: // Collect eval names from fixture directories const askHolmesPath = 'tests/llm/fixtures/test_ask_holmes'; - const investigatePath = 'tests/llm/fixtures/test_investigate'; const getEvalNames = (dir) => { try { @@ -106,12 +105,10 @@ jobs: }; const askHolmesEvals = getEvalNames(askHolmesPath); - const investigateEvals = getEvalNames(investigatePath); const body = `## šŸ“‹ Available Eval Names\n\n` + `Use these with \`filter:\` in your \`/eval\` command.\n\n` + - `**test_ask_holmes:**\n${askHolmesEvals}\n\n` + - `**test_investigate:**\n${investigateEvals}`; + `**test_ask_holmes:**\n${askHolmesEvals}`; await github.rest.issues.createComment({ owner: context.repo.owner, @@ -895,7 +892,16 @@ jobs: # Unset MODEL if empty so get_models() uses DEFAULT_MODEL [[ -z "$MODEL" ]] && unset MODEL - PYTEST_ARGS=(--collect-only -q tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -m "$EVAL_MARKER_EXPR") + # Collect test files dynamically + TEST_FILES=() + for f in tests/llm/test_ask_holmes.py tests/llm/test_holmes_checks.py; do + [[ -f "$f" ]] && TEST_FILES+=("$f") + done + if [[ ${#TEST_FILES[@]} -eq 0 ]]; then + echo "āŒ No test files found" + exit 1 + fi + PYTEST_ARGS=(--collect-only -q "${TEST_FILES[@]}" -m "$EVAL_MARKER_EXPR") [[ -n "$EVAL_FILTER" ]] && PYTEST_ARGS+=(-k "$EVAL_FILTER") # Collect test names @@ -1102,7 +1108,16 @@ jobs: START_TIME=$(date +%s) - PYTEST_ARGS=(--no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -s -n20 -m "$EVAL_MARKER_EXPR") + # Collect test files dynamically + TEST_FILES=() + for f in tests/llm/test_ask_holmes.py tests/llm/test_holmes_checks.py; do + [[ -f "$f" ]] && TEST_FILES+=("$f") + done + if [[ ${#TEST_FILES[@]} -eq 0 ]]; then + echo "āŒ No test files found" + exit 1 + fi + PYTEST_ARGS=(--no-cov "${TEST_FILES[@]}" -s -n10 -m "$EVAL_MARKER_EXPR") [[ -n "$EVAL_FILTER" ]] && PYTEST_ARGS+=(-k "$EVAL_FILTER") poetry run pytest "${PYTEST_ARGS[@]}" || true diff --git a/Makefile b/Makefile index c3d1896f32..5436ed524b 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,6 @@ check: poetry run pre-commit run -a -test-llm-investigate: - poetry run pytest tests/llm/test_investigate.py -n 6 -vv - test-llm-ask-holmes: poetry run pytest tests/llm/test_ask_holmes.py -n 6 -vv diff --git a/docs/development/evaluations/adding-evals.md b/docs/development/evaluations/adding-evals.md index c273f05aac..fd1bea84f1 100644 --- a/docs/development/evaluations/adding-evals.md +++ b/docs/development/evaluations/adding-evals.md @@ -113,7 +113,6 @@ Configure your test by defining these fields in `test_case.yaml`: - `port_forwards`: Configure port forwarding for tests - `test_env_vars`: Environment variables during test execution - `conversation_history`: For multi-turn conversation tests -- `expected_sections`: For investigation tests only ## Advanced Features diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index 3618084aa7..775c6dcce7 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -1,7 +1,7 @@ # HolmesGPT API Reference ## Overview -The HolmesGPT API provides endpoints for automated investigations and conversational troubleshooting. This document describes each endpoint, its purpose, request fields, and example usage. +The HolmesGPT API provides endpoints for conversational troubleshooting. This document describes each endpoint, its purpose, request fields, and example usage. ## Model Parameter Behavior @@ -229,174 +229,8 @@ For the most up-to-date list of vision-enabled models, see the [LiteLLM Vision D --- -### `/api/investigate` (POST) -**Description:** Initiate an automated investigation of an issue or incident. - -#### Request Fields - -| Field | Required | Default | Type | Description | -|-------------------------|----------|--------------------------------------------|-----------|--------------------------------------------------| -| source | Yes | | string | Source of the issue (e.g., "prometheus") | -| title | Yes | | string | Title of the investigation | -| description | Yes | | string | Description of the issue | -| subject | Yes | | object | Subject details (e.g., resource info) | -| context | Yes | | object | Additional context | -| source_instance_id | No | "ApiRequest" | string | Source instance identifier | -| include_tool_calls | No | false | boolean | Include tool calls in response | -| include_tool_call_results| No | false | boolean | Include tool call results in response | -| prompt_template | No | "builtin://generic_investigation.jinja2" | string | Prompt template to use | -| sections | No | | object | Structured output sections | -| model | No | | string | Model name from your `modelList` configuration | - -**Example** -```bash -curl -X POST http:///api/investigate \ - -H "Content-Type: application/json" \ - -d '{ - "source": "prometheus", - "title": "Pod CrashLoopBackOff", - "description": "Pod is crashing repeatedly", - "subject": {"namespace": "default", "pod": "my-pod"}, - "context": {}, - "include_tool_calls": true, - "model": "claude-sonnet" - }' -``` - -> **Note:** The `model` value must be a key name from your `modelList` configuration, not a direct model identifier. - -**Example** Response -```json -{ - "analysis": "The pod 'my-pod' in namespace 'default' is crashing due to an OOMKilled event. Consider increasing memory limits.", - "sections": { - "Alert Explanation": "...", - "Key Findings": "...", - "Conclusions and Possible Root Causes": "...", - "Next Steps": "...", - "App or Infra?": "...", - "External links": "..." - }, - "tool_calls": [ - { - "tool_call_id": "1", - "tool_name": "kubectl_logs", - "description": "Fetch pod logs", - "result": {"logs": "..."} - } - ], - "instructions": [...] -} -``` - ---- - -### `/api/stream/investigate` (POST) -**Description:** Same as `/api/investigate`, but returns results as a stream for real-time updates. - -#### Request Fields -Same as [`/api/investigate`](#apiinvestigate-post). - -**Example** -```bash -curl -N -X POST http:///api/stream/investigate \ - -H "Content-Type: application/json" \ - -d '{ - "source": "prometheus", - "title": "Pod CrashLoopBackOff", - "description": "Pod is crashing repeatedly", - "subject": {"namespace": "default", "pod": "my-pod"}, - "context": {}, - "include_tool_calls": true, - "model": "claude-sonnet" - }' -``` - -**Example** Response (streamed) -```bash -event: start_tool_calling -data: {"tool_name": "kubectl_describe", "id": "call_0"} - -event: start_tool_calling -data: {"tool_name": "kubectl_logs", "id": "call_1"} - -event: start_tool_calling -data: {"tool_name": "kubectl_previous_logs", "id": "call_2"} - -event: start_tool_calling -data: {"tool_name": "kubectl_memory_requests_namespace", "id": "call_3"} - -event: tool_calling_result -data: {"tool_call_id": "call_3", "role": "tool", "description": "kubectl get pods -n default -o ...", "name": "kubectl_memory_requests_namespace", "result": {...}} - -event: tool_calling_result -data: {"tool_call_id": "call_0", "role": "tool", "description": "kubectl describe pod my-pod -n default", "name": "kubectl_describe", "result": {...}} - -event: tool_calling_result -data: {"tool_call_id": "call_2", "role": "tool", "description": "kubectl logs my-pod -n default --previous", "name": "kubectl_previous_logs", "result": {...}} - -event: tool_calling_result -data: {"tool_call_id": "call_1", "role": "tool", "description": "kubectl logs my-pod -n default", "name": "kubectl_logs", "result": {...}} - -event: ai_answer_end -data: {"sections": {"Alert Explanation": ...}} -``` - ---- - -### `/api/issue_chat` (POST) -**Description:** Conversational interface for discussing a specific issue or incident, with context from a previous investigation. - -#### Request Fields - -| Field | Required | Default | Type | Description | -|-------------------------|----------|---------|-----------|--------------------------------------------------| -| ask | Yes | | string | User's question | -| investigation_result | Yes | | object | Previous investigation result (see below) | -| issue_type | Yes | | string | Type of issue | -| conversation_history | No | | list | Conversation history (first message must be system)| -| model | No | | string | Model name from your `modelList` configuration | - -**investigation_result** object: -- `result` (string, optional): Previous analysis -- `tools` (list, optional): Tools used/results - -**Example** - -```bash -curl -X POST http:///api/issue_chat \ - -H "Content-Type: application/json" \ - -d '{ - "ask": "How do I fix this issue?", - "investigation_result": { - "result": "Pod crashed due to OOM.", - "tools": [] - }, - "issue_type": "CrashLoopBackOff", - "conversation_history": [ - {"role": "system", "content": "You are a helpful assistant."} - ] - }' -``` - -**Example** Response -```json -{ - "analysis": "To fix the CrashLoopBackOff, increase the memory limit for the pod and check for memory leaks in the application.", - "conversation_history": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "How do I fix this issue?"}, - {"role": "assistant", "content": "To fix the CrashLoopBackOff, increase the memory limit for the pod and check for memory leaks in the application."} - ], - "tool_calls": [...], - "follow_up_actions": [...] -} -``` - ---- - ### `/api/model` (GET) -**Description:** Returns a list of available AI models that can be used for investigations and chat. +**Description:** Returns a list of available AI models that can be used for chat. **Example** @@ -415,7 +249,7 @@ curl http:///api/model ## Server-Sent Events (SSE) Reference -All streaming endpoints (`/api/stream/investigate`, `/api/stream/chat`, `/api/stream/issue_chat`, etc.) emit Server-Sent Events (SSE) to provide real-time updates during the investigation or chat process. +Streaming endpoints (e.g., `/api/chat` with `stream: true`) emit Server-Sent Events (SSE) to provide real-time updates during the chat process. ### Metadata Object Reference @@ -578,26 +412,9 @@ Emitted when the AI has a text message or reasoning to share (typically before t #### `ai_answer_end` -Emitted when the investigation or chat is complete. This is the final event in the stream. - -**For RCA/Investigation (`/api/stream/investigate`):** -```json -{ - "sections": { - "Alert Explanation": "...", - "Key Findings": "...", - "Conclusions and Possible Root Causes": "...", - "Next Steps": "...", - "App or Infra?": "...", - "External links": "..." - }, - "analysis": "Full analysis text...", - "instructions": ["runbook1", "runbook2"], - "metadata": {...} -} -``` +Emitted when the chat is complete. This is the final event in the stream. -**For Chat (`/api/stream/chat`, `/api/stream/issue_chat`):** +**Payload:** ```json { "analysis": "The issue can be resolved by...", @@ -618,18 +435,9 @@ Emitted when the investigation or chat is complete. This is the final event in t } ``` -**Common Fields:** +**Fields:** - `metadata` (object): See [Metadata Object Reference](#metadata-object-reference) for complete structure including token usage, truncations, and compaction info - -**RCA-Specific Fields:** - -- `sections` (object): Structured investigation output with predefined sections (customizable via request) -- `analysis` (string): Full analysis text (markdown format) -- `instructions` (array): List of runbooks that were used during investigation - -**Chat-Specific Fields:** - - `analysis` (string): The AI's response (markdown format) - `conversation_history` (array): Complete conversation history including the latest response - `follow_up_actions` (array|null): Optional follow-up actions the user can take @@ -688,7 +496,7 @@ To continue after approval, send a new request with `tool_decisions`: #### `token_count` -Emitted periodically to provide token usage updates during the investigation. This event is sent after each LLM iteration to help track resource consumption in real-time. +Emitted periodically to provide token usage updates during the chat. This event is sent after each LLM iteration to help track resource consumption in real-time. **Payload:** ```json @@ -699,7 +507,7 @@ Emitted periodically to provide token usage updates during the investigation. Th **Fields:** -- `metadata` (object): See [Metadata Object Reference](#metadata-object-reference) for complete token usage structure. This event provides the same metadata structure as other events, allowing you to monitor token consumption throughout the investigation +- `metadata` (object): See [Metadata Object Reference](#metadata-object-reference) for complete token usage structure. This event provides the same metadata structure as other events, allowing you to monitor token consumption throughout the chat --- @@ -759,20 +567,6 @@ Emitted when an error occurs during processing. ## Event Flow Examples -### Typical RCA Investigation Flow - -``` -1. start_tool_calling (tool 1) -2. start_tool_calling (tool 2) -3. tool_calling_result (tool 1) -4. tool_calling_result (tool 2) -5. token_count -6. start_tool_calling (tool 3) -7. tool_calling_result (tool 3) -8. token_count -9. ai_answer_end -``` - ### Chat with Approval Flow ``` @@ -784,7 +578,7 @@ Emitted when an error occurs during processing. 6. approval_required [Client sends approval decisions] 1. tool_calling_result (approved tool executed) -[investigation resumes] +[chat resumes] ``` ### Chat with History Compaction diff --git a/docs/reference/python-sdk.md b/docs/reference/python-sdk.md index de2b3f5b04..baa26717e5 100644 --- a/docs/reference/python-sdk.md +++ b/docs/reference/python-sdk.md @@ -273,8 +273,7 @@ Main configuration class (`holmes.config.Config`). |--------|---------|-------------| | `Config.load_from_file(config_file, **kwargs)` | `Config` | Load configuration from a YAML file. | | `Config.load_from_env()` | `Config` | Load configuration from environment variables. | -| `create_console_toolcalling_llm()` | `ToolCallingLLM` | Create an AI instance for asking questions. | -| `create_console_issue_investigator()` | `IssueInvestigator` | Create an AI instance for investigating alerts. | +| `create_console_toolcalling_llm()` | `ToolCallingLLM` | Create an AI instance for asking questions and investigating alerts. | | `get_runbook_catalog()` | `RunbookCatalog` or `None` | Get the loaded runbook catalog. | ### `ToolCallingLLM` diff --git a/holmes/config.py b/holmes/config.py index a79a9183a3..1068bd954c 100644 --- a/holmes/config.py +++ b/holmes/config.py @@ -27,7 +27,7 @@ # Source plugin imports moved to their respective create methods to speed up startup if TYPE_CHECKING: - from holmes.core.tool_calling_llm import IssueInvestigator, ToolCallingLLM + from holmes.core.tool_calling_llm import ToolCallingLLM from holmes.plugins.destinations.slack import SlackDestination from holmes.plugins.sources.github import GitHubSource from holmes.plugins.sources.jira import JiraServiceManagementSource, JiraSource @@ -365,41 +365,6 @@ def create_toolcalling_llm( tool_results_dir=tool_results_dir, ) - def create_issue_investigator( - self, - dal: Optional["SupabaseDal"] = None, - model: Optional[str] = None, - tracer=None, - tool_results_dir: Optional[Path] = None, - ) -> "IssueInvestigator": - tool_executor = self.create_tool_executor(dal) - from holmes.core.tool_calling_llm import IssueInvestigator - - return IssueInvestigator( - tool_executor=tool_executor, - max_steps=self.max_steps, - llm=self._get_llm(model, tracer), - tool_results_dir=tool_results_dir, - cluster_name=self.cluster_name, - ) - - def create_console_issue_investigator( - self, - dal: Optional["SupabaseDal"] = None, - model_name: Optional[str] = None, - tool_results_dir: Optional[Path] = None, - ) -> "IssueInvestigator": - tool_executor = self.create_console_tool_executor(dal=dal) - from holmes.core.tool_calling_llm import IssueInvestigator - - return IssueInvestigator( - tool_executor=tool_executor, - max_steps=self.max_steps, - llm=self._get_llm(model_key=model_name), - tool_results_dir=tool_results_dir, - cluster_name=self.cluster_name, - ) - def validate_jira_config(self): if self.jira_url is None: raise ValueError("--jira-url must be specified") diff --git a/holmes/core/conversations.py b/holmes/core/conversations.py index 8a67263bd1..572968cecf 100644 --- a/holmes/core/conversations.py +++ b/holmes/core/conversations.py @@ -3,21 +3,14 @@ import sentry_sdk from holmes.config import Config -from holmes.core.models import ( - IssueChatRequest, - ToolCallConversationResult, -) from holmes.core.prompt import ( PromptComponent, build_prompts, - generate_user_prompt, ) from holmes.core.tool_calling_llm import ToolCallingLLM -from holmes.plugins.prompts import load_and_render_prompt from holmes.plugins.runbooks import RunbookCatalog from holmes.utils.global_instructions import ( Instructions, - generate_runbooks_args, ) DEFAULT_TOOL_SIZE = 10000 @@ -45,191 +38,12 @@ def calculate_tool_size( return tool_size -def truncate_tool_outputs( - tools: list, tool_size: int -) -> list[ToolCallConversationResult]: - return [ - ToolCallConversationResult( - name=tool.name, - description=tool.description, - output=tool.output[:tool_size], - ) - for tool in tools - ] - - def truncate_tool_messages(conversation_history: list, tool_size: int) -> None: for message in conversation_history: if message.get("role") == "tool": message["content"] = message["content"][:tool_size] -def build_issue_chat_messages( - issue_chat_request: IssueChatRequest, - ai: ToolCallingLLM, - config: Config, - global_instructions: Optional[Instructions] = None, - runbooks: Optional[RunbookCatalog] = None, -): - """Build messages for issue conversation, truncating tool outputs to fit context window. - - Expects conversation_history in OpenAI format (system message first). - For new conversations, creates system prompt from generic_ask_for_issue_conversation.jinja2. - For existing conversations, updates the system prompt and truncates tool outputs as needed. - """ - template_path = "builtin://generic_ask_for_issue_conversation.jinja2" - - conversation_history = issue_chat_request.conversation_history - user_prompt = issue_chat_request.ask - investigation_analysis = issue_chat_request.investigation_result.result - tools_for_investigation = issue_chat_request.investigation_result.tools - - if not conversation_history or len(conversation_history) == 0: - runbooks_ctx = generate_runbooks_args( - runbook_catalog=runbooks, - global_instructions=global_instructions, - ) - user_prompt = generate_user_prompt( - user_prompt, - runbooks_ctx, - ) - - number_of_tools_for_investigation = len(tools_for_investigation) # type: ignore - if number_of_tools_for_investigation == 0: - system_prompt = load_and_render_prompt( - template_path, - { - "investigation": investigation_analysis, - "tools_called_for_investigation": tools_for_investigation, - "issue": issue_chat_request.issue_type, - "toolsets": ai.tool_executor.toolsets, - "cluster_name": config.cluster_name, - "runbooks_enabled": bool(runbooks), - }, - ) - messages = [ - { - "role": "system", - "content": system_prompt, - }, - { - "role": "user", - "content": user_prompt, - }, - ] - return messages - - template_context_without_tools = { - "investigation": investigation_analysis, - "tools_called_for_investigation": None, - "issue": issue_chat_request.issue_type, - "toolsets": ai.tool_executor.toolsets, - "cluster_name": config.cluster_name, - "runbooks_enabled": bool(runbooks), - } - system_prompt_without_tools = load_and_render_prompt( - template_path, template_context_without_tools - ) - messages_without_tools = [ - { - "role": "system", - "content": system_prompt_without_tools, - }, - { - "role": "user", - "content": user_prompt, - }, - ] - tool_size = calculate_tool_size( - ai, messages_without_tools, number_of_tools_for_investigation - ) - - truncated_template_context = { - "investigation": investigation_analysis, - "tools_called_for_investigation": truncate_tool_outputs( - tools_for_investigation, tool_size - ), # type: ignore - "issue": issue_chat_request.issue_type, - "toolsets": ai.tool_executor.toolsets, - "cluster_name": config.cluster_name, - "runbooks_enabled": bool(runbooks), - } - system_prompt_with_truncated_tools = load_and_render_prompt( - template_path, truncated_template_context - ) - return [ - { - "role": "system", - "content": system_prompt_with_truncated_tools, - }, - { - "role": "user", - "content": user_prompt, - }, - ] - - runbooks_ctx = generate_runbooks_args( - runbook_catalog=runbooks, - global_instructions=global_instructions, - ) - user_prompt = generate_user_prompt( - user_prompt, - runbooks_ctx, - ) - - conversation_history.append( - { - "role": "user", - "content": user_prompt, - } - ) - number_of_tools = len(tools_for_investigation) + len( # type: ignore - [message for message in conversation_history if message.get("role") == "tool"] - ) - - if number_of_tools == 0: - return conversation_history - - conversation_history_without_tools = [ - message for message in conversation_history if message.get("role") != "tool" - ] - template_context_without_tools = { - "investigation": investigation_analysis, - "tools_called_for_investigation": None, - "issue": issue_chat_request.issue_type, - "toolsets": ai.tool_executor.toolsets, - "cluster_name": config.cluster_name, - "runbooks_enabled": bool(runbooks), - } - system_prompt_without_tools = load_and_render_prompt( - template_path, template_context_without_tools - ) - conversation_history_without_tools[0]["content"] = system_prompt_without_tools - - tool_size = calculate_tool_size( - ai, conversation_history_without_tools, number_of_tools - ) - - template_context = { - "investigation": investigation_analysis, - "tools_called_for_investigation": truncate_tool_outputs( - tools_for_investigation, tool_size - ), # type: ignore - "issue": issue_chat_request.issue_type, - "toolsets": ai.tool_executor.toolsets, - "cluster_name": config.cluster_name, - "runbooks_enabled": bool(runbooks), - } - system_prompt_with_truncated_tools = load_and_render_prompt( - template_path, template_context - ) - conversation_history[0]["content"] = system_prompt_with_truncated_tools - - truncate_tool_messages(conversation_history, tool_size) - - return conversation_history - - def add_or_update_system_prompt( conversation_history: List[Dict[str, Any]], system_prompt: Optional[str], diff --git a/holmes/core/investigation.py b/holmes/core/investigation.py deleted file mode 100644 index 2be5823c2b..0000000000 --- a/holmes/core/investigation.py +++ /dev/null @@ -1,158 +0,0 @@ -import logging -from pathlib import Path -from typing import Any, Dict, Optional - -from holmes.config import Config -from holmes.core.investigation_structured_output import ( - DEFAULT_SECTIONS, - REQUEST_STRUCTURED_OUTPUT_FROM_LLM, - get_output_format_for_investigation, - process_response_into_sections, -) -from holmes.core.issue import Issue -from holmes.core.models import InvestigateRequest, InvestigationResult -from holmes.core.prompt import generate_user_prompt -from holmes.core.supabase_dal import SupabaseDal -from holmes.core.tracing import DummySpan, SpanType -from holmes.plugins.prompts import load_and_render_prompt -from holmes.plugins.runbooks import RunbookCatalog -from holmes.utils import sentry_helper -from holmes.utils.global_instructions import generate_runbooks_args - - -def investigate_issues( - investigate_request: InvestigateRequest, - dal: SupabaseDal, - config: Config, - model: Optional[str] = None, - trace_span=DummySpan(), - runbooks: Optional[RunbookCatalog] = None, - request_context: Optional[Dict[str, Any]] = None, - tool_results_dir: Optional[Path] = None, -) -> InvestigationResult: - context = dal.get_issue_data(investigate_request.context.get("robusta_issue_id")) - - global_instructions = dal.get_global_instructions_for_account() - - raw_data = investigate_request.model_dump() - if context: - raw_data["extra_context"] = context - - # If config is not preinitilized - create_issue_investigator_span = trace_span.start_span( - "create_issue_investigator", SpanType.FUNCTION.value - ) - ai = config.create_issue_investigator( - dal=dal, model=model, tool_results_dir=tool_results_dir - ) - create_issue_investigator_span.end() - - issue = Issue( - id=context["id"] if context else "", - name=investigate_request.title, - source_type=investigate_request.source, - source_instance_id=investigate_request.source_instance_id, - raw=raw_data, - ) - - investigation = ai.investigate( - issue, - prompt=investigate_request.prompt_template, - global_instructions=global_instructions, - sections=investigate_request.sections, - trace_span=trace_span, - runbooks=runbooks, - request_context=request_context, - ) - - (text_response, sections) = process_response_into_sections(investigation.result) - - if sections is None: - sentry_helper.capture_sections_none(content=investigation.result) - - logging.debug(f"text response: {text_response}") - return InvestigationResult( - analysis=text_response, - sections=sections, - tool_calls=investigation.tool_calls or [], - num_llm_calls=investigation.num_llm_calls, - instructions=investigation.instructions, - metadata=investigation.metadata, - ) - - -def get_investigation_context( - investigate_request: InvestigateRequest, - dal: SupabaseDal, - config: Config, - request_structured_output_from_llm: Optional[bool] = None, - tool_results_dir: Optional[Path] = None, -): - ai = config.create_issue_investigator( - dal=dal, model=investigate_request.model, tool_results_dir=tool_results_dir - ) - - raw_data = investigate_request.model_dump() - context = dal.get_issue_data(investigate_request.context.get("robusta_issue_id")) - if context: - raw_data["extra_context"] = context - - issue = Issue( - id=context["id"] if context else "", - name=investigate_request.title, - source_type=investigate_request.source, - source_instance_id=investigate_request.source_instance_id, - raw=raw_data, - ) - - # This section is about setting vars to request the LLM to return structured output. - # It does not mean that Holmes will not return structured sections for investigation as it is - # capable of splitting the markdown into sections - if request_structured_output_from_llm is None: - request_structured_output_from_llm = REQUEST_STRUCTURED_OUTPUT_FROM_LLM - response_format = None - sections = investigate_request.sections - if not sections: - sections = DEFAULT_SECTIONS - request_structured_output_from_llm = False - logging.info( - "No section received from the client. Default sections will be used." - ) - elif ai.llm.model and ai.llm.model.startswith(("bedrock", "gemini")): - # Structured output does not work well with Bedrock Anthropic Sonnet 3.5, or gemini through litellm - request_structured_output_from_llm = False - - if request_structured_output_from_llm: - response_format = get_output_format_for_investigation(sections) - logging.info("Structured output is enabled for this request") - else: - logging.info("Structured output is disabled for this request") - - runbook_catalog = config.get_runbook_catalog() - system_prompt = load_and_render_prompt( - investigate_request.prompt_template, - { - "issue": issue, - "sections": sections, - "structured_output": request_structured_output_from_llm, - "toolsets": ai.tool_executor.toolsets, - "cluster_name": config.cluster_name, - "runbooks_enabled": True if runbook_catalog else False, - }, - ) - base_user = "" - - global_instructions = dal.get_global_instructions_for_account() - runbooks_ctx = generate_runbooks_args( - runbook_catalog=runbook_catalog, - global_instructions=global_instructions, - ) - - base_user = f"{base_user}\n #This is context from the issue:\n{issue.raw}" - - user_prompt = generate_user_prompt( - base_user, - runbooks_ctx, - ) - - return ai, system_prompt, user_prompt, response_format, sections diff --git a/holmes/core/investigation_structured_output.py b/holmes/core/investigation_structured_output.py deleted file mode 100644 index a5853d420b..0000000000 --- a/holmes/core/investigation_structured_output.py +++ /dev/null @@ -1,283 +0,0 @@ -import json -import logging -import re -from contextlib import suppress -from typing import Any, Dict, Optional, Tuple - -from holmes.common.env_vars import load_bool - -REQUEST_STRUCTURED_OUTPUT_FROM_LLM = load_bool( - "REQUEST_STRUCTURED_OUTPUT_FROM_LLM", True -) -PARSE_INVESTIGATION_MARKDOWN_INTO_STRUCTURED_SECTIONS = load_bool( - "PARSE_INVESTIGATION_MARKDOWN_INTO_STRUCTURED_SECTIONS", True -) - - -InputSectionsDataType = Dict[str, str] - -DEFAULT_SECTIONS: InputSectionsDataType = { - "Alert Explanation": '1-2 sentences explaining the alert itself - note don\'t say "The alert indicates a warning event related to a Kubernetes pod doing blah" rather just say "The pod XYZ did blah" because that is what the user actually cares about', - "Key Findings": "What you checked and found", - "Conclusions and Possible Root causes": "What conclusions can you reach based on the data you found? what are possible root causes (if you have enough conviction to say) or what uncertainty remains. Don't say root cause but 'possible root causes'. Be clear to distinguish between what you know for certain and what is a possible explanation", - "Next Steps": "What you would do next to troubleshoot this issue, any commands that could be run to fix it, or other ways to solve it (prefer giving precise bash commands when possible)", - "Related logs": "Truncate and share the most relevant logs, especially if these explain the root cause. For example: \nLogs from pod robusta-holmes:\n```\n```\n. Always embed the surroundding +/- 5 log lines to any relevant logs. ", - "App or Infra?": "Explain whether the issue is more likely an infrastructure or an application level issue and why you think that.", - "External links": "Provide links to external sources and a short sentence describing each link. For example provide links to relevant runbooks, etc. This section is a markdown formatted string.", -} - - -def get_output_format_for_investigation( - sections: InputSectionsDataType, -) -> Dict[str, Any]: - properties = {} - required_fields = [] - - for title, description in sections.items(): - properties[title] = {"type": ["string", "null"], "description": description} - required_fields.append(title) - - schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": required_fields, - "properties": properties, - "additionalProperties": False, - } - output_format = { - "type": "json_schema", - "json_schema": { - "name": "InvestigationResult", - "schema": schema, - "strict": False, # TODO: remove explicit False - let ensure_strict_response_format handle it - }, - } - - return output_format - - -def combine_sections(sections: Dict) -> str: - content = "" - for section_title, section_content in sections.items(): - if section_content: - content = content + f"\n# {section_title}\n{section_content}\n" - return content - - -def parse_markdown_into_sections_from_equal_sign( - markdown_content: str, -) -> Optional[Dict[str, Optional[str]]]: - """Splits a markdown in different sections where the key is a top level title underlined with `====` and the value is the content - ``` - Header Title - =========== - Content here - ``` - => - { - "Header Title": "Content here" - } - """ - matches = re.split(r"(?:^|\n)([^\n]+)\n=+\n", markdown_content.strip()) - - # Remove any empty first element if the text starts with a header - if matches[0].strip() == "": - matches = matches[1:] - - sections = {} - - for i in range(0, len(matches), 2): - if i + 1 < len(matches): - header = matches[i] - content = matches[i + 1].strip() - sections[header] = content - - if len(sections) > 0: - return sections - else: - return None - - -def parse_markdown_into_sections_from_hash_sign( - markdown_content: str, -) -> Optional[Dict[str, Optional[str]]]: - """Splits a markdown in different sections where the key is a top level title underlined with `====` and the value is the content - ``` - # Header Title - Content here - ``` - => - { - "Header Title": "Content here" - } - """ - # Split the text into sections based on headers (# Section) - matches = re.split(r"\n(?=# )", markdown_content.strip()) - - if not matches[0].startswith("#"): - matches = matches[1:] - - sections = {} - - for match in matches: - match = match.strip() - if match: - parts = match.split("\n", 1) - - if len(parts) > 1: - # Remove the # from the title and use it as key - title = parts[0].replace("#", "").strip() - # Use the rest as content - content = parts[1].strip() - sections[title] = content - else: - # Handle case where section has no content - title = parts[0].replace("#", "").strip() - sections[title] = None - - if len(sections) > 0: - return sections - else: - return None - - -def extract_within(content: str, from_idx: int, to_idx: int) -> str: - with suppress(Exception): - extracted_content = content[from_idx:to_idx] - parsed = json.loads( - extracted_content - ) # if this parses as json, set the response as that. - if isinstance(parsed, dict): - logging.warning( - "The LLM did not return structured data but embedded the data into a markdown code block. This indicates the prompt is not optimised for that AI model." - ) - content = extracted_content - return content - - -def pre_format_sections(response: Any) -> Any: - """Pre-cleaning of the response for some known, specific use cases - prior to it being parsed for sections - """ - if isinstance(response, dict): - # No matter if the result is already structured, we want to go through the code below to validate the JSON - response = json.dumps(response) - - if not isinstance(response, str): - # if it's not a string, we make it so as it'll be parsed later - response = str(response) - - # In some cases, the LLM will not return a structured json but instead embed the JSON into a markdown code block - # This is not ideal and actually should not happen - if response.startswith("```json\n") and response.endswith("\n```"): - response = extract_within(response, 8, -3) - - if response.startswith('"{') and response.endswith('}"'): - # Some Anthropic models embed the actual JSON dict inside a JSON string - # In that case it gets parsed once to get rid of the first level of marshalling - with suppress(Exception): - response = json.loads(response) - - # Try to find any embedded code block with or without "json" label and parse it - # This has been seen a lot in newer bedrock models - # This is a more robust check for patterns like ```json\n{...}\n``` or ```\n{...}\n``` - matches = re.findall(r"```(?:json)?\s*\n(.*?)\n```", response, re.DOTALL) - for block in matches: - with suppress(Exception): - parsed = json.loads(block) - if isinstance(parsed, dict): - logging.info("Extracted and parsed embedded JSON block successfully.") - return json.dumps(parsed) - - return response - - -def parse_json_sections( - response: Any, -) -> Tuple[str, Optional[Dict[str, Optional[str]]]]: - response = pre_format_sections(response) - - with suppress(Exception): - parsed_json = json.loads(response) - - if not isinstance(parsed_json, dict): - return (response, None) - sections = {} - for key, value in parsed_json.items(): - if isinstance(value, list) and len(value) == 0: - value = None # For links, LLM returns '[]' which is unsightly when converted to markdown - - if isinstance(value, list): - sections[key] = "\n\n".join(f"{str(item)}" for item in value) - elif value is not None: - sections[key] = str( - value - ) # force to strings. We only expect markdown and don't want to give anything but a string to the UI - else: - sections[key] = value # type: ignore - if sections: - combined = combine_sections(sections) - return (combined, sections) # type: ignore - - return (response, None) - - -def process_response_into_sections( - response: Any, -) -> Tuple[str, Optional[Dict[str, Optional[str]]]]: - sections = None - - if REQUEST_STRUCTURED_OUTPUT_FROM_LLM: - (response, sections) = parse_json_sections(response) - - if not sections and PARSE_INVESTIGATION_MARKDOWN_INTO_STRUCTURED_SECTIONS: - sections = parse_markdown_into_sections_from_hash_sign(response) - if not sections and PARSE_INVESTIGATION_MARKDOWN_INTO_STRUCTURED_SECTIONS: - sections = parse_markdown_into_sections_from_equal_sign(response) - - return (response, sections) - - -def is_response_an_incorrect_tool_call( - sections: Optional[InputSectionsDataType], choice: dict -) -> bool: - """Cf. https://github.com/BerriAI/litellm/issues/8241 - This code detects when LiteLLM is incapable of handling both tool calls and structured output. This only happens when the LLM is returning a single tool call. - In that case the intention is to retry the LLM calls without structured output. - Post processing may still try to generate a structured output from a monolithic markdown. - """ - with suppress(Exception): - message = choice.get("message", {}) - finish_reason = choice.get("finish_reason") - content = message.get("content") - tool_calls = message.get("tool_calls") - role = message.get("role") - if ( - sections - and content - and ( - # azure - finish_reason == "stop" - or - # bedrock - finish_reason == "tool_calls" - ) - and role == "assistant" - and not tool_calls - ): - if not isinstance(content, dict): - content = json.loads(content) - if not isinstance(content, dict): - return False - for section_title in sections: - if section_title in content: - return False - return True - return False - - -def clear_json_markdown(text: str): - if text and text.startswith("```json") and text.endswith("```"): - return text[8:-3] - - return text diff --git a/holmes/core/models.py b/holmes/core/models.py index f3d159d3fb..f4e7978b4d 100644 --- a/holmes/core/models.py +++ b/holmes/core/models.py @@ -1,10 +1,8 @@ import json -from enum import Enum from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel, Field, model_validator -from holmes.core.investigation_structured_output import InputSectionsDataType from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus @@ -93,89 +91,6 @@ def format_tool_result_data( return tool_response -class InvestigationResult(BaseModel): - analysis: Optional[str] = None - sections: Optional[Dict[str, Union[str, None]]] = None - tool_calls: List[ToolCallResult] = [] - num_llm_calls: Optional[int] = None # Number of LLM API calls (turns) - instructions: List[str] = [] - metadata: Optional[Dict[Any, Any]] = None - - -class InvestigateRequest(BaseModel): - source: str # "prometheus" etc - title: str - description: str - subject: dict - context: Dict[str, Any] - source_instance_id: str = "ApiRequest" - include_tool_calls: bool = False - include_tool_call_results: bool = False - prompt_template: str = "builtin://generic_investigation.jinja2" - sections: Optional[InputSectionsDataType] = None - model: Optional[str] = None - # TODO in the future - # response_handler: ... - - -class ToolCallConversationResult(BaseModel): - name: str - description: str - output: str - - -class ConversationInvestigationResponse(BaseModel): - analysis: Optional[str] = None - tool_calls: List[ToolCallResult] = [] - - -class ConversationInvestigationResult(BaseModel): - analysis: Optional[str] = None - tools: Optional[List[ToolCallConversationResult]] = [] - - -class IssueInvestigationResult(BaseModel): - """ - :var result: A dictionary containing the summary of the issue investigation. - :var tools: A list of dictionaries where each dictionary contains information - about the tool, its name, description and output. - - It is based on the holmes investigation saved to Evidence table. - """ - - result: str - tools: Optional[List[ToolCallConversationResult]] = [] - - -class HolmesConversationHistory(BaseModel): - ask: str - answer: ConversationInvestigationResult - - -# HolmesConversationIssueContext, ConversationType and ConversationRequest classes will be deprecated later -class HolmesConversationIssueContext(BaseModel): - investigation_result: IssueInvestigationResult - conversation_history: Optional[List[HolmesConversationHistory]] = [] - issue_type: str - robusta_issue_id: Optional[str] = None - source: Optional[str] = None - - -class ConversationType(str, Enum): - ISSUE = "issue" - - -class ConversationRequest(BaseModel): - user_prompt: str - source: Optional[str] = None - resource: Optional[dict] = None - # ConversationType.ISSUE is default as we gonna deprecate this class and won't add new conversation types - conversation_type: Optional[ConversationType] = ConversationType.ISSUE - context: HolmesConversationIssueContext - include_tool_calls: bool = False - include_tool_call_results: bool = False - - class PendingToolApproval(BaseModel): """Represents a tool call that requires user approval.""" @@ -225,12 +140,6 @@ def check_first_item_role(cls, values): return values -class IssueChatRequest(ChatRequestBaseModel): - ask: str - investigation_result: IssueInvestigationResult - issue_type: str - - class ChatRequest(ChatRequestBaseModel): ask: str images: Optional[List[Union[str, Dict[str, Any]]]] = Field( diff --git a/holmes/core/tool_calling_llm.py b/holmes/core/tool_calling_llm.py index 47f1613a54..1f8351c79e 100644 --- a/holmes/core/tool_calling_llm.py +++ b/holmes/core/tool_calling_llm.py @@ -2,7 +2,6 @@ import json import logging import re -import textwrap import threading from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Type, Union @@ -20,13 +19,6 @@ RESET_REPEATED_TOOL_CALL_CHECK_AFTER_COMPACTION, TEMPERATURE, ) -from holmes.core.investigation_structured_output import ( - DEFAULT_SECTIONS, - REQUEST_STRUCTURED_OUTPUT_FROM_LLM, - InputSectionsDataType, - get_output_format_for_investigation, - is_response_an_incorrect_tool_call, -) from holmes.core.issue import Issue from holmes.core.llm import LLM from holmes.core.llm_usage import extract_usage_from_response @@ -50,14 +42,7 @@ from holmes.core.truncation.input_context_window_limiter import ( limit_input_context_window, ) -from holmes.plugins.prompts import load_and_render_prompt -from holmes.plugins.runbooks import RunbookCatalog -from holmes.utils import sentry_helper from holmes.utils.colors import AI_COLOR -from holmes.utils.global_instructions import ( - Instructions, - generate_runbooks_args, -) from holmes.utils.stream import ( StreamEvents, StreamMessage, @@ -382,7 +367,6 @@ def prompt_call( system_prompt: str, user_prompt: str, response_format: Optional[Union[dict, Type[BaseModel]]] = None, - sections: Optional[InputSectionsDataType] = None, trace_span=DummySpan(), request_context: Optional[Dict[str, Any]] = None, ) -> LLMResult: @@ -394,7 +378,6 @@ def prompt_call( messages, response_format=response_format, user_prompt=user_prompt, - sections=sections, trace_span=trace_span, request_context=request_context, ) @@ -430,7 +413,6 @@ def call( # type: ignore messages: List[Dict[str, str]], response_format: Optional[Union[dict, Type[BaseModel]]] = None, user_prompt: Optional[str] = None, - sections: Optional[InputSectionsDataType] = None, trace_span=DummySpan(), tool_number_offset: int = 0, request_context: Optional[Dict[str, Any]] = None, @@ -521,22 +503,6 @@ def call( # type: ignore response = full_response.choices[0] # type: ignore response_message = response.message # type: ignore - if response_message and response_format: - # Litellm API is bugged. Stringify and parsing ensures all attrs of the choice are available. - dict_response = json.loads(full_response.to_json()) # type: ignore - incorrect_tool_call = is_response_an_incorrect_tool_call( - sections, dict_response.get("choices", [{}])[0] - ) - - if incorrect_tool_call: - logging.warning( - "Detected incorrect tool call. Structured output will be disabled. This can happen on models that do not support tool calling. For Azure AI, make sure the model name contains 'gpt-4.1' or other structured output compatible models. To disable this holmes behaviour, set REQUEST_STRUCTURED_OUTPUT_FROM_LLM to `false`." - ) - # disable structured output going forward and and retry - sentry_helper.capture_structured_output_incorrect_tool_call() - response_format = None - max_steps = max_steps + 1 - continue new_message = response_message.model_dump( exclude_defaults=True, exclude_unset=True, exclude_none=True @@ -964,7 +930,6 @@ def call_stream( system_prompt: str = "", user_prompt: Optional[str] = None, response_format: Optional[Union[dict, Type[BaseModel]]] = None, - sections: Optional[InputSectionsDataType] = None, msgs: Optional[list[dict]] = None, enable_tool_approval: bool = False, tool_decisions: List[ToolApprovalDecision] | None = None, @@ -1069,22 +1034,6 @@ def call_stream( raise response_message = full_response.choices[0].message # type: ignore - if response_message and response_format: - # Litellm API is bugged. Stringify and parsing ensures all attrs of the choice are available. - dict_response = json.loads(full_response.to_json()) # type: ignore - incorrect_tool_call = is_response_an_incorrect_tool_call( - sections, dict_response.get("choices", [{}])[0] - ) - - if incorrect_tool_call: - logging.warning( - "Detected incorrect tool call. Structured output will be disabled. This can happen on models that do not support tool calling. For Azure AI, make sure the model name contains 'gpt-4.1' or other structured output compatible models. To disable this holmes behaviour, set REQUEST_STRUCTURED_OUTPUT_FROM_LLM to `false`." - ) - # disable structured output going forward and and retry - sentry_helper.capture_structured_output_incorrect_tool_call() - response_format = None - max_steps = max_steps + 1 - continue messages.append( response_message.model_dump( @@ -1266,98 +1215,3 @@ def find_assistant_tool_call_request( raise Exception( f"Failed to find assistant request for a tool_call in conversation history. tool_call_id={tool_call_id}" ) - - -# TODO: consider getting rid of this entirely and moving templating into the cmds in holmes_cli.py -class IssueInvestigator(ToolCallingLLM): - """ - Thin wrapper around ToolCallingLLM which: - 1) Provides a default prompt for RCA - 2) Accepts Issue objects - """ - - def __init__( - self, - tool_executor: ToolExecutor, - max_steps: int, - llm: LLM, - tool_results_dir: Optional[Path], - cluster_name: Optional[str], - ): - super().__init__(tool_executor, max_steps, llm, tool_results_dir) - self.cluster_name = cluster_name - - def investigate( - self, - issue: Issue, - prompt: str, - console: Optional[Console] = None, - global_instructions: Optional[Instructions] = None, - sections: Optional[InputSectionsDataType] = None, - trace_span=DummySpan(), - runbooks: Optional[RunbookCatalog] = None, - request_context: Optional[Dict[str, Any]] = None, - ) -> LLMResult: - request_structured_output_from_llm = True - response_format = None - - # This section is about setting vars to request the LLM to return structured output. - # It does not mean that Holmes will not return structured sections for investigation as it is - # capable of splitting the markdown into sections - if not sections or len(sections) == 0: - # If no sections are passed, we will not ask the LLM for structured output - sections = DEFAULT_SECTIONS - request_structured_output_from_llm = False - logging.info( - "No section received from the client. Default sections will be used." - ) - elif self.llm.model and self.llm.model.startswith("bedrock"): - # Structured output does not work well with Bedrock Anthropic Sonnet 3.5 through litellm - request_structured_output_from_llm = False - - if not REQUEST_STRUCTURED_OUTPUT_FROM_LLM: - request_structured_output_from_llm = False - - if request_structured_output_from_llm: - response_format = get_output_format_for_investigation(sections) - logging.info("Structured output is enabled for this request") - else: - logging.info("Structured output is disabled for this request") - - system_prompt = load_and_render_prompt( - prompt, - { - "issue": issue, - "sections": sections, - "structured_output": request_structured_output_from_llm, - "toolsets": self.tool_executor.toolsets, - "cluster_name": self.cluster_name, - "runbooks_enabled": True if runbooks else False, - }, - ) - - base_user = "" - base_user = f"{base_user}\n #This is context from the issue:\n{issue.raw}" - - runbooks_ctx = generate_runbooks_args( - runbook_catalog=runbooks, - global_instructions=global_instructions, - ) - user_prompt = generate_user_prompt( - base_user, - runbooks_ctx, - ) - logging.debug( - "Rendered system prompt:\n%s", textwrap.indent(system_prompt, " ") - ) - logging.debug("Rendered user prompt:\n%s", textwrap.indent(user_prompt, " ")) - - res = self.prompt_call( - system_prompt, - user_prompt, - response_format=response_format, - sections=sections, - trace_span=trace_span, - request_context=request_context, - ) - return res diff --git a/holmes/main.py b/holmes/main.py index fab3db064e..624ada662c 100644 --- a/holmes/main.py +++ b/holmes/main.py @@ -35,6 +35,7 @@ generate_user_prompt, ) from holmes.core.resource_instruction import ResourceInstructionDocument +from holmes.core.tool_calling_llm import LLMResult, ToolCallingLLM from holmes.core.tools import pretty_print_toolset_status from holmes.core.tools_utils.filesystem_result_storage import tool_result_storage from holmes.core.tracing import SpanType, TracingFactory @@ -43,7 +44,6 @@ from holmes.plugins.interfaces import Issue from holmes.plugins.prompts import load_and_render_prompt from holmes.plugins.sources.opsgenie import OPSGENIE_TEAM_INTEGRATION_KEY_HELP -from holmes.utils.console.consts import system_prompt_help from holmes.utils.console.logging import init_logging from holmes.utils.console.result import handle_result from holmes.utils.file_utils import write_json_file @@ -153,6 +153,28 @@ def parse_documents(documents: Optional[str]) -> List[ResourceInstructionDocumen return resource_documents +def _investigate_issue( + ai: ToolCallingLLM, + issue: Issue, + config: Config, +) -> LLMResult: + """Investigate an issue using the standard ask system prompt with investigation additions.""" + investigation_additions = f"Provide a terse analysis of the following {issue.source_type} alert/issue and why it is firing." + system_prompt = build_system_prompt( + toolsets=ai.tool_executor.toolsets, + runbooks=None, + system_prompt_additions=investigation_additions, + cluster_name=config.cluster_name, + ask_user_enabled=False, + prompt_component_overrides={}, + ) + user_prompt = generate_user_prompt( + f"\n #This is context from the issue:\n{issue.raw}", + context={}, + ) + return ai.prompt_call(system_prompt, user_prompt) + + # TODO: add streaming output @app.command() def ask( @@ -420,9 +442,6 @@ def alertmanager( slack_token: Optional[str] = opt_slack_token, slack_channel: Optional[str] = opt_slack_channel, json_output_file: Optional[str] = opt_json_output_file, - system_prompt: Optional[str] = typer.Option( - "builtin://generic_investigation.jinja2", help=system_prompt_help - ), ): """ Investigate a Prometheus/Alertmanager alert @@ -445,45 +464,42 @@ def alertmanager( custom_toolsets_from_cli=custom_toolsets, ) - ai = config.create_console_issue_investigator(model_name=model) # type: ignore + with tool_result_storage() as tool_results_dir: + ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) - source = config.create_alertmanager_source() + source = config.create_alertmanager_source() - try: - issues = source.fetch_issues() - except Exception as e: - logging.error("Failed to fetch issues from alertmanager", exc_info=e) - return + try: + issues = source.fetch_issues() + except Exception as e: + logging.error("Failed to fetch issues from alertmanager", exc_info=e) + return - if alertmanager_limit is not None: - console.print( - f"[bold yellow]Limiting to {alertmanager_limit}/{len(issues)} issues.[/bold yellow]" - ) - issues = issues[:alertmanager_limit] + if alertmanager_limit is not None: + console.print( + f"[bold yellow]Limiting to {alertmanager_limit}/{len(issues)} issues.[/bold yellow]" + ) + issues = issues[:alertmanager_limit] - if alertmanager_alertname is not None: - console.print( - f"[bold yellow]Analyzing {len(issues)} issues matching filter.[/bold yellow] [red]Press Ctrl+C to stop.[/red]" - ) - else: - console.print( - f"[bold yellow]Analyzing all {len(issues)} issues. (Use --alertmanager-alertname to filter.)[/bold yellow] [red]Press Ctrl+C to stop.[/red]" - ) - results = [] - for i, issue in enumerate(issues): - console.print( - f"[bold yellow]Analyzing issue {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" - ) - result = ai.investigate( - issue=issue, - prompt=system_prompt, # type: ignore - console=console, - ) - results.append({"issue": issue.model_dump(), "result": result.model_dump()}) - handle_result(result, console, destination, config, issue, False, True) # type: ignore + if alertmanager_alertname is not None: + console.print( + f"[bold yellow]Analyzing {len(issues)} issues matching filter.[/bold yellow] [red]Press Ctrl+C to stop.[/red]" + ) + else: + console.print( + f"[bold yellow]Analyzing all {len(issues)} issues. (Use --alertmanager-alertname to filter.)[/bold yellow] [red]Press Ctrl+C to stop.[/red]" + ) + results = [] + for i, issue in enumerate(issues): + console.print( + f"[bold yellow]Analyzing issue {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" + ) + result = _investigate_issue(ai, issue, config) + results.append({"issue": issue.model_dump(), "result": result.model_dump()}) + handle_result(result, console, destination, config, issue, False, True) # type: ignore - if json_output_file: - write_json_file(json_output_file, results) + if json_output_file: + write_json_file(json_output_file, results) @generate_app.command("alertmanager-tests") @@ -550,10 +566,6 @@ def jira( max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, json_output_file: Optional[str] = opt_json_output_file, - # advanced options for this command - system_prompt: Optional[str] = typer.Option( - "builtin://generic_investigation.jinja2", help=system_prompt_help - ), ): """ Investigate a Jira ticket @@ -571,7 +583,6 @@ def jira( jira_query=jira_query, custom_toolsets_from_cli=custom_toolsets, ) - ai = config.create_console_issue_investigator(model_name=model) # type: ignore source = config.create_jira_source() try: issues = source.fetch_issues() @@ -584,29 +595,27 @@ def jira( ) results = [] - for i, issue in enumerate(issues): - console.print( - f"[bold yellow]Analyzing Jira ticket {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" - ) - result = ai.investigate( - issue=issue, - prompt=system_prompt, # type: ignore - console=console, - ) - - console.print(Rule()) - console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore - console.print(Rule()) - if update: - source.write_back_result(issue.id, result) - console.print(f"[bold]Updated ticket {issue.url}.[/bold]") - else: + with tool_result_storage() as tool_results_dir: + ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + for i, issue in enumerate(issues): console.print( - f"[bold]Not updating ticket {issue.url}. Use the --update option to do so.[/bold]" + f"[bold yellow]Analyzing Jira ticket {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" ) + result = _investigate_issue(ai, issue, config) + + console.print(Rule()) + console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") + console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print(Rule()) + if update: + source.write_back_result(issue.id, result) + console.print(f"[bold]Updated ticket {issue.url}.[/bold]") + else: + console.print( + f"[bold]Not updating ticket {issue.url}. Use the --update option to do so.[/bold]" + ) - results.append({"issue": issue.model_dump(), "result": result.model_dump()}) + results.append({"issue": issue.model_dump(), "result": result.model_dump()}) if json_output_file: write_json_file(json_output_file, results) @@ -640,6 +649,7 @@ def ticket( None, help="ticket ID to investigate (e.g., 'KAN-1')", ), + update: Optional[bool] = typer.Option(False, help="Update ticket with AI results"), config_file: Optional[Path] = opt_config_file, # type: ignore model: Optional[str] = opt_model, ): @@ -674,45 +684,51 @@ def ticket( ) return - ai = ticket_source.config.create_console_issue_investigator(model_name=model) - - # Render ticket-specific additions - ticket_additions = load_and_render_prompt( - prompt="builtin://_ticket_additions.jinja2", - context={ - "source": source, - "output_instructions": ticket_source.output_instructions, - }, - ) + with tool_result_storage() as tool_results_dir: + ai = ticket_source.config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + + # Render ticket-specific additions + ticket_additions = load_and_render_prompt( + prompt="builtin://_ticket_additions.jinja2", + context={ + "source": source, + "output_instructions": ticket_source.output_instructions, + }, + ) - system_prompt = build_system_prompt( - toolsets=ai.tool_executor.toolsets, - runbooks=None, - system_prompt_additions=ticket_additions, - cluster_name=ticket_source.config.cluster_name, - ask_user_enabled=False, - prompt_component_overrides={}, - ) - console.print( - f"[bold yellow]Analyzing ticket: {issue_to_investigate.name}...[/bold yellow]" - ) - prompt = ( - prompt - + f" for issue '{issue_to_investigate.name}' with description:'{issue_to_investigate.description}'" - ) + system_prompt = build_system_prompt( + toolsets=ai.tool_executor.toolsets, + runbooks=None, + system_prompt_additions=ticket_additions, + cluster_name=ticket_source.config.cluster_name, + ask_user_enabled=False, + prompt_component_overrides={}, + ) + console.print( + f"[bold yellow]Analyzing ticket: {issue_to_investigate.name}...[/bold yellow]" + ) + prompt = ( + prompt + + f" for issue '{issue_to_investigate.name}' with description:'{issue_to_investigate.description}'" + ) - ticket_user_prompt = generate_user_prompt(prompt, context={}) - result = ai.prompt_call(system_prompt, ticket_user_prompt) + ticket_user_prompt = generate_user_prompt(prompt, context={}) + result = ai.prompt_call(system_prompt, ticket_user_prompt) - console.print(Rule()) - console.print( - f"[bold green]AI analysis of {issue_to_investigate.url} {prompt}[/bold green]" - ) - console.print(result.result.replace("\n", "\n\n"), style="bold green") # type: ignore - console.print(Rule()) + console.print(Rule()) + console.print( + f"[bold green]AI analysis of {issue_to_investigate.url} {prompt}[/bold green]" + ) + console.print(result.result.replace("\n", "\n\n"), style="bold green") # type: ignore + console.print(Rule()) - ticket_source.source.write_back_result(issue_to_investigate.id, result) - console.print(f"[bold]Updated ticket {issue_to_investigate.url}.[/bold]") + if update: + ticket_source.source.write_back_result(issue_to_investigate.id, result) + console.print(f"[bold]Updated ticket {issue_to_investigate.url}.[/bold]") + else: + console.print( + f"[bold]Not updating ticket {issue_to_investigate.url}. Use the --update option to do so.[/bold]" + ) @investigate_app.command() @@ -745,10 +761,6 @@ def github( max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, - # advanced options for this command - system_prompt: Optional[str] = typer.Option( - "builtin://generic_investigation.jinja2", help=system_prompt_help - ), ): """ Investigate a GitHub issue @@ -767,7 +779,6 @@ def github( github_query=github_query, custom_toolsets_from_cli=custom_toolsets, ) - ai = config.create_console_issue_investigator(model_name=model) source = config.create_github_source() try: issues = source.fetch_issues() @@ -778,29 +789,27 @@ def github( console.print( f"[bold yellow]Analyzing {len(issues)} GitHub Issues.[/bold yellow] [red]Press Ctrl+C to stop.[/red]" ) - for i, issue in enumerate(issues): - console.print( - f"[bold yellow]Analyzing GitHub issue {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" - ) - - result = ai.investigate( - issue=issue, - prompt=system_prompt, # type: ignore - console=console, - ) - - console.print(Rule()) - console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore - console.print(Rule()) - if update: - source.write_back_result(issue.id, result) - console.print(f"[bold]Updated ticket {issue.url}.[/bold]") - else: + with tool_result_storage() as tool_results_dir: + ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + for i, issue in enumerate(issues): console.print( - f"[bold]Not updating issue {issue.url}. Use the --update option to do so.[/bold]" + f"[bold yellow]Analyzing GitHub issue {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" ) + result = _investigate_issue(ai, issue, config) + + console.print(Rule()) + console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") + console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print(Rule()) + if update: + source.write_back_result(issue.id, result) + console.print(f"[bold]Updated ticket {issue.url}.[/bold]") + else: + console.print( + f"[bold]Not updating issue {issue.url}. Use the --update option to do so.[/bold]" + ) + @investigate_app.command() def pagerduty( @@ -828,10 +837,6 @@ def pagerduty( max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, json_output_file: Optional[str] = opt_json_output_file, - # advanced options for this command - system_prompt: Optional[str] = typer.Option( - "builtin://generic_investigation.jinja2", help=system_prompt_help - ), ): """ Investigate a PagerDuty incident @@ -848,7 +853,6 @@ def pagerduty( pagerduty_incident_key=pagerduty_incident_key, custom_toolsets_from_cli=custom_toolsets, ) - ai = config.create_console_issue_investigator(model_name=model) source = config.create_pagerduty_source() try: issues = source.fetch_issues() @@ -861,29 +865,27 @@ def pagerduty( ) results = [] - for i, issue in enumerate(issues): - console.print( - f"[bold yellow]Analyzing PagerDuty incident {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" - ) - - result = ai.investigate( - issue=issue, - prompt=system_prompt, # type: ignore - console=console, - ) - - console.print(Rule()) - console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore - console.print(Rule()) - if update: - source.write_back_result(issue.id, result) - console.print(f"[bold]Updated alert {issue.url}.[/bold]") - else: + with tool_result_storage() as tool_results_dir: + ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + for i, issue in enumerate(issues): console.print( - f"[bold]Not updating alert {issue.url}. Use the --update option to do so.[/bold]" + f"[bold yellow]Analyzing PagerDuty incident {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" ) - results.append({"issue": issue.model_dump(), "result": result.model_dump()}) + + result = _investigate_issue(ai, issue, config) + + console.print(Rule()) + console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") + console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print(Rule()) + if update: + source.write_back_result(issue.id, result) + console.print(f"[bold]Updated alert {issue.url}.[/bold]") + else: + console.print( + f"[bold]Not updating alert {issue.url}. Use the --update option to do so.[/bold]" + ) + results.append({"issue": issue.model_dump(), "result": result.model_dump()}) if json_output_file: write_json_file(json_output_file, results) @@ -910,10 +912,6 @@ def opsgenie( max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, - # advanced options for this command - system_prompt: Optional[str] = typer.Option( - "builtin://generic_investigation.jinja2", help=system_prompt_help - ), documents: Optional[str] = opt_documents, ): """ @@ -931,7 +929,6 @@ def opsgenie( opsgenie_query=opsgenie_query, custom_toolsets_from_cli=custom_toolsets, ) - ai = config.create_console_issue_investigator(model_name=model) source = config.create_opsgenie_source() try: issues = source.fetch_issues() @@ -942,27 +939,25 @@ def opsgenie( console.print( f"[bold yellow]Analyzing {len(issues)} OpsGenie alerts.[/bold yellow] [red]Press Ctrl+C to stop.[/red]" ) - for i, issue in enumerate(issues): - console.print( - f"[bold yellow]Analyzing OpsGenie alert {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" - ) - result = ai.investigate( - issue=issue, - prompt=system_prompt, # type: ignore - console=console, - ) - - console.print(Rule()) - console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore - console.print(Rule()) - if update: - source.write_back_result(issue.id, result) - console.print(f"[bold]Updated alert {issue.url}.[/bold]") - else: + with tool_result_storage() as tool_results_dir: + ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + for i, issue in enumerate(issues): console.print( - f"[bold]Not updating alert {issue.url}. Use the --update option to do so.[/bold]" + f"[bold yellow]Analyzing OpsGenie alert {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" ) + result = _investigate_issue(ai, issue, config) + + console.print(Rule()) + console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") + console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print(Rule()) + if update: + source.write_back_result(issue.id, result) + console.print(f"[bold]Updated alert {issue.url}.[/bold]") + else: + console.print( + f"[bold]Not updating alert {issue.url}. Use the --update option to do so.[/bold]" + ) @toolset_app.command("list") diff --git a/holmes/plugins/prompts/_noflag_general_instructions.jinja2 b/holmes/plugins/prompts/_noflag_general_instructions.jinja2 deleted file mode 100644 index 55719cf796..0000000000 --- a/holmes/plugins/prompts/_noflag_general_instructions.jinja2 +++ /dev/null @@ -1,72 +0,0 @@ -{% include 'investigation_procedure.jinja2' %} - -{% include '_ai_safety.jinja2' %} - -# In general - -{% if cluster_name -%} -* You are running on cluster {{ cluster_name }}. -{%- endif %} -* when it can provide extra information, first run as many tools as you need to gather more information, then respond. -* if possible, do so repeatedly with different tool calls each time to gather more information. -* do not stop investigating until you are at the final root cause you are able to find. -* use the "five whys" methodology to find the root cause. -* for example, if you found a problem in microservice A that is due to an error in microservice B, look at microservice B too and find the error in that. -* if you cannot find the resource/application that the user referred to, assume they made a typo or included/excluded characters like - and in this case, try to find substrings or search for the correct spellings -* always provide detailed information like exact resource names, versions, labels, etc -* even if you found the root cause, keep investigating to find other possible root causes and to gather data for the answer like exact names -* if a runbook url is present you MUST fetch the runbook before beginning your investigation -* when the user mentions any operational issue (high CPU, memory issues, database down, application errors, etc.), ALWAYS check if there's a matching runbook in the catalog first -* if you don't know, say that the analysis was inconclusive. -* if there are multiple possible causes list them in a numbered list. -* there will often be errors in the data that are not relevant or that do not have an impact - ignore them in your conclusion if you were not able to tie them to an actual error. -* ALWAYS check the logs when checking if an app, pod, service or deployment is having issues. Something "running" and reporting healthy does not mean it is without issues. - -# If investigating Kubernetes problems - -* run as many kubectl commands as you need to gather more information, then respond. -* if possible, do so repeatedly on different Kubernetes objects. -* for example, for deployments first run kubectl on the deployment then a replicaset inside it, then a pod inside that. -* when investigating a pod that crashed or application errors, always run kubectl_describe and fetch the logs -* Do check both the status of the kubernetes resources and the application runtime as well, by investigating logs -* do not give an answer like "The pod is pending" as that doesn't state why the pod is pending and how to fix it. -* do not give an answer like "Pod's node affinity/selector doesn't match any available nodes" because that doesn't include data on WHICH label doesn't match -* if investigating an issue on many pods, there is no need to check more than 3 individual pods in the same deployment. pick up to a representative 3 from each deployment if relevant -* if the user says something isn't working, ALWAYS: -** use kubectl_describe on the owner workload + individual pods and look for any transient issues they might have been referring to -** look for misconfigured ingresses/services etc -** check the application logs because there may be runtime issues - -{% include '_toolsets_instructions.jinja2' %} - -{% include '_permission_errors.jinja2' %} - -# Special cases and how to reply - -* Make sure you differentiate between "I investigated and found error X caused this problem" and "I tried to investigate but while investigating I got some errors that prevented me from completing the investigation." -* As a special case of that, If a tool generates a permission error when attempting to run it, follow the Handling Permission Errors section for detailed guidance. -* That is different than - for example - fetching a pod's logs and seeing that the pod itself has permission errors. in that case, you explain say that permission errors are the cause of the problem and give details -* Issues are a subset of findings. When asked about an issue or a finding and you have an id, use the tool `fetch_finding_by_id`. -* For any question, try to make the answer specific to the user's cluster. -** For example, if asked to port forward, find out the app or pod port (kubectl describe) and provide a port forward command specific to the user's question - -# MANDATORY Task Management - -* You MUST use the TodoWrite tool for ANY investigation requiring multiple steps -* Your FIRST tool call MUST be TodoWrite to create your investigation plan -* Break down ALL complex problems into smaller, manageable tasks -* You MUST update task status (pending → in_progress → completed) as you work through your investigation -* The TodoWrite tool will show you a formatted task list - reference this throughout your investigation -* Mark tasks as 'in_progress' when you start them, 'completed' when finished -* Follow ALL tasks in your plan - don't skip any tasks -* Use task management to ensure you don't miss important investigation steps -* If you discover additional steps during investigation, add them to your task list using TodoWrite -* When calling TodoWrite, you may ALSO call other tools in parallel to speed things up for your users and make them happy! -* On the first TodoWrite call, mark at least one task as in_progress, and start working on it in parallel. -* When calling TodoWrite for the first time, mark the tasks you started working on with ā€˜in_progress’ status. - -# Tool/function calls - -You are able to make tool calls / function calls. Recognise when a tool has already been called and reuse its result. -If a tool call returns nothing, modify the parameters as required instead of repeating the tool call. -When searching for resources in specific namespaces, test a cluster level tool to find the resource(s) and identify what namespace they are part of. diff --git a/holmes/plugins/prompts/generic_ask_for_issue_conversation.jinja2 b/holmes/plugins/prompts/generic_ask_for_issue_conversation.jinja2 deleted file mode 100644 index 6e9047d8eb..0000000000 --- a/holmes/plugins/prompts/generic_ask_for_issue_conversation.jinja2 +++ /dev/null @@ -1,50 +0,0 @@ -You are a tool-calling AI assist provided with common devops and IT tools that you can use to troubleshoot problems or answer questions. -Whenever possible you MUST first use tools to investigate then answer the question. -Ask for multiple tool calls at the same time as it saves time for the user. -Do not say 'based on the tool output' or explicitly refer to tools at all. -If you output an answer and then realize you need to call more tools or there are possible next steps, you may do so by calling tools at that point in time. - -### Context Awareness: -Be aware that this conversation is follow-up questions to a prior investigation conducted for the {{issue}}. -However, not all questions may be directly related to that investigation. -Use results of the investigation and conversation history to maintain continuity when appropriate, ensuring efficiency in your responses. - -#### Results of issue Investigation: -{{investigation}} - -{% if tools_called_for_investigation %} -Tools used for the investigation: -{% for r in tools_called_for_investigation %} -- {{ r }} -{% endfor %} -{% endif %} - -{% if conversation_history %} -Conversation history: -{{conversation_history}} -{% endif %} - -{% include '_global_instructions.jinja2' %} - -{% include '_noflag_general_instructions.jinja2' %} - -Style guide: -* Reply with terse output. -* Be painfully concise. -* Leave out "the" and filler words when possible. -* Be terse but not at the expense of leaving out important data like the root cause and how to fix. - -Examples: - -User: Why did the webserver-example app crash? -(Call tool kubectl_find_resource kind=pod keyword=webserver`) -(Call tool kubectl_previous_logs namespace=demos pod=webserver-example-1299492-d9g9d # this pod name was found from the previous tool call) - -AI: `webserver-example-1299492-d9g9d` crashed due to email validation error during HTTP request for /api/create_user -Relevant logs: - -``` -2021-01-01T00:00:00.000Z [ERROR] Missing required field 'email' in request body -``` - -Validation error led to unhandled Java exception causing a crash. diff --git a/holmes/plugins/prompts/generic_investigation.jinja2 b/holmes/plugins/prompts/generic_investigation.jinja2 deleted file mode 100644 index 07c04bfb6f..0000000000 --- a/holmes/plugins/prompts/generic_investigation.jinja2 +++ /dev/null @@ -1,42 +0,0 @@ -You are a tool-calling AI assist provided with common devops and IT tools that you can use to troubleshoot problems or answer questions. -Whenever possible you MUST first use tools to investigate then answer the question. -Ask for multiple tool calls at the same time as it saves time for the user. -Do not say 'based on the tool output' - -Provide an terse analysis of the following {{ issue.source_type }} alert/issue and why it is firing. -* If the tool requires string format timestamps, query from 'start_timestamp' until 'end_timestamp' -* If the tool requires timestamps in milliseconds, query from 'start_timestamp' until 'end_timestamp' -* If you need timestamp in string format, query from 'start_timestamp_millis' until 'end_timestamp_millis' -* Always try to search for BOTH relevant logs and traces - -If the user provides you with extra instructions in a triple quotes section, ALWAYS perform their instructions and then perform your investigation. - - -# Global Instructions - -You may receive a set of ā€œGlobal Instructionsā€ that describe how to perform certain tasks, handle certain situations, or apply certain best practices. They are not mandatory for every request, but serve as a reference resource and must be used if the current scenario or user request aligns with one of the described methods or conditions. -Use these rules when deciding how to apply them: - -* If the user prompt includes Global Instructions, treat them as a reference resource. -* Some Global Instructions may describe how to handle specific tasks or scenarios. If the user's current request or the instructions in a triple quotes section reference one of these tasks, follow the Global Instruction for that task. -* Some Global Instructions may define general conditions that always apply if a certain scenario occurs (e.g., "whenever investigating a memory issue, always check resource limits"). If such a condition matches the current situation, apply the Global Instruction accordingly. -* If user's prompt or the instructions in a triple quotes section direct you to perform a task (e.g., ā€œFind ownerā€) and there is a Global Instruction on how to do that task, follow the Global Instructions on how to perform it. -* If multiple Global Instructions are relevant, apply all that fit. -* If no Global Instruction is relevant, or no condition applies, ignore them and proceed as normal. -* Before finalizing your answer double-check if any Global Instructions apply. If so, ensure you have correctly followed those instructions. - -{% include '_noflag_general_instructions.jinja2' %} - -# Style Guide - -* `code block` exact names of IT/cloud resources like specific virtual machines. -* *Surround the title of the root cause like this*. -* Whenever there are precise numbers in the data available, quote them. For example: -* Don't say an app is repeatedly crashing, rather say the app has crashed X times so far -* Don't just say x/y nodes don't match a pod's affinity selector, rather say x/y nodes don't match the selector ABC -* Don't say "The alert indicates a warning event related to a Kubernetes pod failing to start due to a container creation error" rather say "The pod failed to start due to a container creation error." -* And so on -* But only quote relevant numbers or metrics that are available. Do not guess. -* Remove unnecessary words - -{% include 'investigation_output_format.jinja2' %} diff --git a/holmes/plugins/prompts/investigation_output_format.jinja2 b/holmes/plugins/prompts/investigation_output_format.jinja2 deleted file mode 100644 index a9f9cfdc10..0000000000 --- a/holmes/plugins/prompts/investigation_output_format.jinja2 +++ /dev/null @@ -1,32 +0,0 @@ - -{% if structured_output %} -Give your answer in a pure JSON format with the following sections. -The content of each section should be formatted with markdown: - -{ -{% for title, description in sections.items() %} - "{{ title }}": "{{ description | replace("\n", "\\n") }}",{% endfor %} -} - -You MUST set a section as `null` if: -- It's not relevant to the investigation -- It does not contain relevant information -- Its content is not related to a possible root cause - - -{% else %} -Give your answer returning a markdown document structured with the following sections. Use top level markdown titles format `#`. -Ignore any section that is not relevant to the investigation. - -{% for title, description in sections.items() %} -# {{ title }} -{{ description }} -{% endfor %} - -# - -You MUST ignore a section and skip it: -- It's not relevant to the investigation -- It does not contain relevant information -- Its content is not related to a possible root cause -{% endif %} diff --git a/holmes/utils/stream.py b/holmes/utils/stream.py index 2956f781bd..0309447968 100644 --- a/holmes/utils/stream.py +++ b/holmes/utils/stream.py @@ -9,9 +9,7 @@ from litellm.types.utils import ModelResponse, TextCompletionResponse from pydantic import BaseModel, Field -from holmes.core.investigation_structured_output import process_response_into_sections from holmes.core.llm import TokenCountMetadata, get_llm_usage -from holmes.utils import sentry_helper class StreamEvents(str, Enum): @@ -65,39 +63,6 @@ def _is_rate_limit_error(e: Exception) -> bool: return isinstance(e, litellm.exceptions.RateLimitError) or "Model is getting throttled" in str(e) -def stream_investigate_formatter( - call_stream: Generator[StreamMessage, None, None], -): - try: - for message in call_stream: - if message.event == StreamEvents.ANSWER_END: - (text_response, sections) = process_response_into_sections( # type: ignore - message.data.get("content") - ) - - if sections is None: - sentry_helper.capture_sections_none( - content=message.data.get("content"), - ) - - yield create_sse_message( - StreamEvents.ANSWER_END.value, - { - "sections": sections or {}, - "analysis": text_response, - "metadata": message.data.get("metadata") or {}, - }, - ) - else: - yield create_sse_message(message.event.value, message.data) - except Exception as e: - logging.error(f"Error during streaming investigation: {e}", exc_info=True) - if _is_rate_limit_error(e): - yield create_rate_limit_error_message(str(e)) - else: - yield create_sse_error_message(description=str(e), error_code=1, msg=str(e)) - - def stream_chat_formatter( call_stream: Generator[StreamMessage, None, None], followups: Optional[List[dict]] = None, diff --git a/server.py b/server.py index f1f19c8b84..110ca348c6 100644 --- a/server.py +++ b/server.py @@ -39,17 +39,13 @@ TOOLSET_STATUS_REFRESH_INTERVAL_SECONDS, ) from holmes.config import DEFAULT_CONFIG_LOCATION, Config -from holmes.core import investigation from holmes.core.conversations import ( build_chat_messages, - build_issue_chat_messages, ) from holmes.core.models import ( ChatRequest, ChatResponse, FollowUpAction, - InvestigateRequest, - IssueChatRequest, ) from holmes.core.prompt import PromptComponent from holmes.core.tools import ToolsetStatusEnum, ToolsetType @@ -60,7 +56,7 @@ from holmes.utils.log import EndpointFilter from holmes.checks.checks_api import init_checks_app from holmes.core.tools_utils.filesystem_result_storage import tool_result_storage -from holmes.utils.stream import stream_chat_formatter, stream_investigate_formatter +from holmes.utils.stream import stream_chat_formatter # removed: add_runbooks_to_user_prompt @@ -268,111 +264,6 @@ async def log_requests(request: Request, call_next): init_checks_app(app, config) -@app.post("/api/investigate") -def investigate_issues(investigate_request: InvestigateRequest, http_request: Request): - try: - runbooks = config.get_runbook_catalog() - request_context = extract_passthrough_headers(http_request) - with tool_result_storage() as tool_results_dir: - result = investigation.investigate_issues( - investigate_request=investigate_request, - dal=dal, - config=config, - model=investigate_request.model, - runbooks=runbooks, - request_context=request_context, - tool_results_dir=tool_results_dir, - ) - return result - - except AuthenticationError as e: - raise HTTPException(status_code=401, detail=e.message) - except litellm.exceptions.RateLimitError as e: - raise HTTPException(status_code=429, detail=e.message) - except Exception as e: - logging.error(f"Error in /api/investigate: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/stream/investigate") -def stream_investigate_issues(req: InvestigateRequest, http_request: Request): - try: - req_info = f"/api/stream/investigate request: title={req.title}" - logging.info(f"Received {req_info}") - storage = tool_result_storage() - tool_results_dir = storage.__enter__() - ai, system_prompt, user_prompt, response_format, sections = ( - investigation.get_investigation_context( - req, dal, config, tool_results_dir=tool_results_dir - ) - ) - request_context = extract_passthrough_headers(http_request) - - return StreamingResponse( - _stream_with_storage_cleanup( - storage, - stream_investigate_formatter( - ai.call_stream( - system_prompt=system_prompt, - user_prompt=user_prompt, - response_format=response_format, - sections=sections, - request_context=request_context, - ), - ), - req_info - ), - media_type="text/event-stream", - ) - - except AuthenticationError as e: - storage.__exit__(None, None, None) - raise HTTPException(status_code=401, detail=e.message) - except Exception as e: - storage.__exit__(None, None, None) - logging.exception(f"Error in /api/stream/investigate: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/issue_chat") -def issue_conversation(issue_chat_request: IssueChatRequest, http_request: Request): - try: - runbooks = config.get_runbook_catalog() - with tool_result_storage() as tool_results_dir: - ai = config.create_toolcalling_llm( - dal=dal, - model=issue_chat_request.model, - tool_results_dir=tool_results_dir, - ) - global_instructions = dal.get_global_instructions_for_account() - - messages = build_issue_chat_messages( - issue_chat_request=issue_chat_request, - ai=ai, - config=config, - global_instructions=global_instructions, - runbooks=runbooks, - ) - request_context = extract_passthrough_headers(http_request) - llm_call = ai.messages_call( - messages=messages, request_context=request_context - ) - - return ChatResponse( - analysis=llm_call.result, - tool_calls=llm_call.tool_calls, - conversation_history=llm_call.messages, - metadata=llm_call.metadata, - ) - except AuthenticationError as e: - raise HTTPException(status_code=401, detail=e.message) - except litellm.exceptions.RateLimitError as e: - raise HTTPException(status_code=429, detail=e.message) - except Exception as e: - logging.error(f"Error in /api/issue_chat: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) - - def already_answered(conversation_history: Optional[List[dict]]) -> bool: if conversation_history is None: return False diff --git a/tests/core/test_prompt.py b/tests/core/test_prompt.py index 05f8ba5f9b..2db96f3acd 100644 --- a/tests/core/test_prompt.py +++ b/tests/core/test_prompt.py @@ -10,13 +10,6 @@ from holmes.config import Config from holmes.core.conversations import ( build_chat_messages, - build_issue_chat_messages, -) -from holmes.core.investigation import get_investigation_context -from holmes.core.models import ( - InvestigateRequest, - IssueChatRequest, - IssueInvestigationResult, ) from holmes.core.prompt import ( PromptComponent, @@ -75,16 +68,6 @@ def mock_ai(mock_tool_executor): return ai -@pytest.fixture -def mock_dal(): - """Create a mock DAL instance.""" - dal = Mock() - dal.get_global_instructions_for_account = Mock(return_value=None) - dal.get_resource_instructions = Mock(return_value=None) - dal.get_issue_data = Mock(return_value=None) - return dal - - def get_user_message_from_messages(messages: list, get_last: bool = False) -> str: """Extract user message content from messages list. @@ -132,36 +115,11 @@ def create_test_files(file_paths: list, tmp_path: Path) -> Optional[list]: return test_files -def create_mock_investigator(): - """Create a mock investigator instance for testing.""" - mock_investigator = Mock() - mock_investigator.tool_executor = Mock() - mock_investigator.tool_executor.toolsets = [] - mock_investigator.runbook_manager = Mock() - mock_investigator.runbook_manager.get_instructions_for_issue = Mock(return_value=[]) - mock_investigator.llm = Mock() - mock_investigator.llm.model = None - return mock_investigator - - def extract_instructions(instructions_obj): """Extract instruction list from DummyInstructions object or return None.""" return instructions_obj.instructions if instructions_obj else None -def create_issue_chat_request(user_ask: str, issue_type: str = "prometheus"): - """Create an IssueChatRequest for testing.""" - return IssueChatRequest( - ask=user_ask, - conversation_history=None, - investigation_result=IssueInvestigationResult( - result="Investigation analysis", - tools=[], - ), - issue_type=issue_type, - ) - - def assert_user_prompt_contains_timestamp(user_prompt: str): """Assert that user prompt contains the UTC timestamp in seconds.""" timestamp_pattern = r"The current UTC timestamp in seconds is (\d+)\." @@ -356,66 +314,6 @@ def test_chat_api_user_prompt( expected_global_instructions=extract_instructions(global_instructions), ) - @pytest.mark.parametrize( - "user_ask,global_instructions", - [ - ("Tell me more about this alert", None), - ("What should I do?", DummyInstructions(["Check metrics"])), - ], - ) - def test_issue_chat_api_user_prompt( - self, - mock_ai, - mock_config, - user_ask, - global_instructions, - ): - """Test user prompt in /api/issue_chat flow.""" - issue_chat_request = create_issue_chat_request(user_ask) - - messages = build_issue_chat_messages( - issue_chat_request=issue_chat_request, - ai=mock_ai, - config=mock_config, - global_instructions=global_instructions, - runbooks=None, - ) - - user_content = get_user_message_from_messages(messages) - validate_user_prompt( - user_content, - user_ask, - expected_global_instructions=extract_instructions(global_instructions), - ) - - -class TestInvestigationFlow: - """Test user prompt validation for investigation flow.""" - - def test_investigate_api_user_prompt(self, mock_config, mock_dal): - """Test user prompt in /api/investigate flow.""" - mock_investigator = create_mock_investigator() - mock_config.create_issue_investigator = Mock(return_value=mock_investigator) - - investigate_request = InvestigateRequest( - title="Test Alert", - description="Test description", - source="prometheus", - subject={}, - context={"robusta_issue_id": "test-123"}, - prompt_template="builtin://generic_investigation.jinja2", - ) - - _, _, user_prompt, _, _ = get_investigation_context( - investigate_request, mock_dal, mock_config - ) - - assert user_prompt is not None - assert isinstance(user_prompt, str) - assert "context from the issue" in user_prompt.lower() - assert_user_prompt_contains_timestamp(user_prompt) - - class TestUserPromptComponents: """Test that user prompts include all expected components via generate_user_prompt.""" diff --git a/tests/llm/conftest.py b/tests/llm/conftest.py index 691b915f20..986a42654e 100644 --- a/tests/llm/conftest.py +++ b/tests/llm/conftest.py @@ -45,7 +45,7 @@ # Configuration constants DEBUG_SEPARATOR = "=" * 80 -LLM_TEST_TYPES = ["test_ask_holmes", "test_investigate"] +LLM_TEST_TYPES = ["test_ask_holmes"] DEFAULT_SYSTEM_PROMPT_URL = ( "https://platform.robusta.dev/api/additional-system-prompt.json" ) @@ -99,12 +99,7 @@ def _has_frontend_tests(session: pytest.Session) -> bool: def is_llm_test(nodeid: str) -> bool: """Check if a test nodeid is for an LLM test.""" - return any( - [ - "test_ask_holmes" in nodeid, - "test_investigate" in nodeid, - ] - ) + return "test_ask_holmes" in nodeid @pytest.fixture(scope="session") @@ -763,8 +758,6 @@ def _collect_test_results_from_stats(terminalreporter): # Extract test type if "test_ask_holmes" in nodeid: test_type = "ask" - elif "test_investigate" in nodeid: - test_type = "investigate" else: test_type = "unknown" @@ -894,7 +887,7 @@ def _collect_test_results_from_stats(terminalreporter): results_with_ids = [] for result in test_results.values(): # If we have a clean test case ID from the test, use it - # This is set in test_ask_holmes.py and test_investigate.py + # This is set in test_ask_holmes.py # via: request.node.user_properties.append(("clean_test_case_id", test_case.id)) # It provides the clean test case ID without model suffixes that pytest adds when # parameterizing with multiple models (e.g., "01_how_many_pods" instead of diff --git a/tests/llm/fixtures/test_investigate/01_oom_kill/fast_oom_deployment.yaml b/tests/llm/fixtures/test_investigate/01_oom_kill/fast_oom_deployment.yaml deleted file mode 100644 index b47cbd8b7f..0000000000 --- a/tests/llm/fixtures/test_investigate/01_oom_kill/fast_oom_deployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: analytics-exporter-fast -spec: - replicas: 1 - selector: - matchLabels: - app: analytics-exporter-fast - template: - metadata: - labels: - app: analytics-exporter-fast - spec: - containers: - - name: memory-eater - image: us-central1-docker.pkg.dev/genuine-flight-317411/devel/memory-eater:1.0 - imagePullPolicy: Always - args: - - 80Mi - - "0" - - 100Mi - - "10" # +10Mi per second. Will die after 2 seconds if limit is 100Mi - - "1" - resources: - limits: - memory: 100Mi - requests: - memory: 100Mi - restartPolicy: Always - nodeSelector: - kubernetes.io/arch: amd64 diff --git a/tests/llm/fixtures/test_investigate/01_oom_kill/investigate_request.json b/tests/llm/fixtures/test_investigate/01_oom_kill/investigate_request.json deleted file mode 100644 index 36cfc1264e..0000000000 --- a/tests/llm/fixtures/test_investigate/01_oom_kill/investigate_request.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "source": "prometheus", - "source_instance_id": "some-instance", - "title": "Pod is crash looping.", - "description": "Crashing pod analytics-exporter-fast-76897854c-cw5wh in namespace default", - "subject": { - "name": "analytics-exporter-fast-76897854c-cw5wh", - "subject_type": "deployment", - "namespace": "default", - "node": "grafana-cloud-control-plane", - "container": "memory-eater", - "annotations": {} - }, - "context": { - "robusta_issue_id": "916704eb-fc7f-4d6f-825f-5ad94f5a116e" - }, - "include_tool_calls": true, - "include_tool_call_results": true, - "sections": { - "Alert Explanation": "1-2 sentences explaining the alert itself - note don't say \"The alert indicates a warning event related to a Kubernetes pod doing blah\" rather just say \"The pod XYZ did blah\" because that is what the user actually cares about", - "Conclusions and Possible Root causes": "What conclusions can you reach based on the data you found? what are possible root causes (if you have enough conviction to say) or what uncertainty remains. Don't say root cause but 'possible root causes'. Be clear to distinguish between what you know for certain and what is a possible explanation", - "Related logs": "Truncate and share the most relevant logs, especially if these explain the root cause. For example: \nLogs from pod robusta-holmes:\n```\n```\n. Always embed the surroundding +/- 5 log lines to any relevant logs. " - } -} diff --git a/tests/llm/fixtures/test_investigate/01_oom_kill/issue_data.json b/tests/llm/fixtures/test_investigate/01_oom_kill/issue_data.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/01_oom_kill/issue_data.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/01_oom_kill/resource_instructions.json b/tests/llm/fixtures/test_investigate/01_oom_kill/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/01_oom_kill/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/01_oom_kill/test_case.yaml b/tests/llm/fixtures/test_investigate/01_oom_kill/test_case.yaml deleted file mode 100644 index efb034d9a3..0000000000 --- a/tests/llm/fixtures/test_investigate/01_oom_kill/test_case.yaml +++ /dev/null @@ -1,16 +0,0 @@ -expected_output: - - Pod `analytics-exporter-fast` is experiencing a `CrashLoopBackOff` - - Pod is OOM killed -before_test: | - kubectl apply -f ./fast_oom_deployment.yaml - sleep 60 -after_test: | - kubectl delete -f ./fast_oom_deployment.yaml -expected_sections: - Key Findings: False - Next Steps: False - App or Infra?: False - External links: False -evaluation: - correctness: 1 -# Success rate 100% for 100 evals diff --git a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/investigate_request.json b/tests/llm/fixtures/test_investigate/02_crashloop_backoff/investigate_request.json deleted file mode 100644 index 8784447efa..0000000000 --- a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/investigate_request.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "source": "prometheus", - "source_instance_id": "some-instance", - "title": "CrashLoopBackoff.", - "description": "Crashing pod logging-agent in namespace default", - "subject": { - "name": "logging-agent", - "subject_type": "pod", - "namespace": "default", - "node": "ip-172-31-21-139.us-east-2.compute.internal", - "container": "downloader", - "labels": {}, - "annotations": {} - }, - "context": { - "robusta_issue_id": "dd1c3198-183a-46a9-824a-eb849b79abb7" - }, - "include_tool_calls": true, - "include_tool_call_results": true -} diff --git a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/issue_data.json b/tests/llm/fixtures/test_investigate/02_crashloop_backoff/issue_data.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/issue_data.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/resource_instructions.json b/tests/llm/fixtures/test_investigate/02_crashloop_backoff/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/test_case.yaml b/tests/llm/fixtures/test_investigate/02_crashloop_backoff/test_case.yaml deleted file mode 100644 index 3e59e5a414..0000000000 --- a/tests/llm/fixtures/test_investigate/02_crashloop_backoff/test_case.yaml +++ /dev/null @@ -1,12 +0,0 @@ -expected_output: - - Pod logging-agent is experiencing a CrashLoopBackOff - - The pod attempts to execute a command wge which is not found - - Mention `wge` might be a typo and the intended command may be `wget` -before_test: | - kubectl apply -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/init_crashloop_backoff/create_init_crashloop_backoff.yaml - sleep 40 -after_test: | - kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/init_crashloop_backoff/create_init_crashloop_backoff.yaml -evaluation: - correctness: 1 -# Success rate 100% for 100 evals diff --git a/tests/llm/fixtures/test_investigate/03_cpu_throttling/investigate_request.json b/tests/llm/fixtures/test_investigate/03_cpu_throttling/investigate_request.json deleted file mode 100644 index 3604412403..0000000000 --- a/tests/llm/fixtures/test_investigate/03_cpu_throttling/investigate_request.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "source": "prometheus", - "source_instance_id": "some-instance", - "title": "CPUThrottlingHigh.", - "description": "100% throttling of CPU in namespace default for container stress in pod frontend-service.", - "subject": { - "name": "frontend-service", - "subject_type": "pod", - "namespace": "default", - "node": "grafana-cloud-control-plane", - "container": "stress", - "labels": {}, - "annotations": {} - }, - "context": { - "robusta_issue_id": "5ff09887-38a0-400e-8a42-07f66903d78b" - }, - "include_tool_calls": true, - "include_tool_call_results": true -} diff --git a/tests/llm/fixtures/test_investigate/03_cpu_throttling/issue_data.json b/tests/llm/fixtures/test_investigate/03_cpu_throttling/issue_data.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/03_cpu_throttling/issue_data.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/03_cpu_throttling/manifest.yaml b/tests/llm/fixtures/test_investigate/03_cpu_throttling/manifest.yaml deleted file mode 100644 index 0c88ba2bee..0000000000 --- a/tests/llm/fixtures/test_investigate/03_cpu_throttling/manifest.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: frontend-service -spec: - containers: - - name: stress - image: jfusterm/stress - imagePullPolicy: IfNotPresent - command: - - "stress" - - "--cpu" - - "100" - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 10m # you really shouldn't set this - see https://home.robusta.dev/blog/stop-using-cpu-limits/ - memory: 64Mi diff --git a/tests/llm/fixtures/test_investigate/03_cpu_throttling/resource_instructions.json b/tests/llm/fixtures/test_investigate/03_cpu_throttling/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/03_cpu_throttling/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/03_cpu_throttling/test_case.yaml b/tests/llm/fixtures/test_investigate/03_cpu_throttling/test_case.yaml deleted file mode 100644 index 6af4f19a1d..0000000000 --- a/tests/llm/fixtures/test_investigate/03_cpu_throttling/test_case.yaml +++ /dev/null @@ -1,10 +0,0 @@ -tags: [prometheus, kubernetes] -expected_output: - - The frontend-service pod in the default namespace is experiencing 100% CPU throttling for the stress container - - Suggest increasing the CPU limit -before_test: | - kubectl apply -f manifest.yaml - sleep 60 -after_test: kubectl delete -f manifest.yaml -evaluation: - correctness: 1 diff --git a/tests/llm/fixtures/test_investigate/03_cpu_throttling/toolsets.yaml b/tests/llm/fixtures/test_investigate/03_cpu_throttling/toolsets.yaml deleted file mode 100644 index 689cc341b4..0000000000 --- a/tests/llm/fixtures/test_investigate/03_cpu_throttling/toolsets.yaml +++ /dev/null @@ -1,9 +0,0 @@ -toolsets: - prometheus/metrics: - enabled: False - kubernetes/core: - enabled: true - kubernetes/logs: - enabled: true - kubernetes/live-metrics: - enabled: true diff --git a/tests/llm/fixtures/test_investigate/05_crashpod/investigate_request.json b/tests/llm/fixtures/test_investigate/05_crashpod/investigate_request.json deleted file mode 100644 index 6b5ac25964..0000000000 --- a/tests/llm/fixtures/test_investigate/05_crashpod/investigate_request.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "source": "prometheus", - "title": "KubePodCrashLooping", - "description": "", - "subject": { - "name": "db-certs-authenticator-757f89d977-795x7", - "namespace": "default", - "kind": "pod", - "node": null, - "container": null, - "cluster": null - }, - "context": { - "issue_type": "KubePodCrashLooping", - "robusta_issue_id": "460f24bf-0171-4523-8f82-b29846b3519a", - "source": "prometheus" - }, - "source_instance_id": "ApiRequest", - "include_tool_calls": true, - "include_tool_call_results": true, - "prompt_template": "builtin://generic_investigation.jinja2" -} diff --git a/tests/llm/fixtures/test_investigate/05_crashpod/issue_data.json b/tests/llm/fixtures/test_investigate/05_crashpod/issue_data.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/05_crashpod/issue_data.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/05_crashpod/resource_instructions.json b/tests/llm/fixtures/test_investigate/05_crashpod/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/05_crashpod/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/05_crashpod/test_case.yaml b/tests/llm/fixtures/test_investigate/05_crashpod/test_case.yaml deleted file mode 100644 index 41524d15db..0000000000 --- a/tests/llm/fixtures/test_investigate/05_crashpod/test_case.yaml +++ /dev/null @@ -1,13 +0,0 @@ -expected_output: - - The pod db-certs-authenticator is experiencing a `CrashLoopBackOff` - - The SSL certificate expired -expected_sections: - Related logs: - - ERROR - SSL certificate expired -before_test: | - kubectl apply -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/crashpod.v2/crashloop-cert-app.yaml - sleep 120 -after_test: | - kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/crashpod.v2/crashloop-cert-app.yaml -evaluation: - correctness: 1 diff --git a/tests/llm/fixtures/test_investigate/06_job_failure/investigate_request.json b/tests/llm/fixtures/test_investigate/06_job_failure/investigate_request.json deleted file mode 100644 index 6a44aaf1cb..0000000000 --- a/tests/llm/fixtures/test_investigate/06_job_failure/investigate_request.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "source": "kubernetes_api_server", - "title": "JobFailure", - "description": "", - "subject": { - "name": "java-api-checker", - "namespace": "default", - "kind": "job", - "node": null, - "container": null, - "cluster": null - }, - "context": { - "issue_type": "JobFailure", - "robusta_issue_id": "d1e5b8cb-4632-48fc-8add-60b58cb200c6", - "source": "kubernetes_api_server" - }, - "source_instance_id": "ApiRequest", - "include_tool_calls": true, - "include_tool_call_results": true, - "prompt_template": "builtin://generic_investigation.jinja2" -} diff --git a/tests/llm/fixtures/test_investigate/06_job_failure/issue_data.json b/tests/llm/fixtures/test_investigate/06_job_failure/issue_data.json deleted file mode 100644 index 9915fe21fa..0000000000 --- a/tests/llm/fixtures/test_investigate/06_job_failure/issue_data.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "category": null, - "cluster": "test_cluster", - "creation_date": "2025-05-29T10:44:14", - "description": null, - "ends_at": null, - "evidence": [ - { - "account_id": "debe17e5-0fe2-4c9c-9a7b-5dba04b83f89", - "collection_timestamp": null, - "creation_date": "2025-05-29T10:44:14", - "data": "[{\\\"type\\\": \\\"markdown\\\", \\\"data\\\": \\\"**Job events:**\\\"}, {\\\"type\\\": \\\"table\\\", \\\"data\\\": {\\\"headers\\\": [\\\"reason\\\", \\\"type\\\", \\\"time\\\", \\\"message\\\"], \\\"rows\\\": [[\\\"BackoffLimitExceeded\\\", \\\"Warning\\\", 1731921566000.0, \\\"Job has reached the specified backoff limit\\\"], [\\\"SuccessfulCreate\\\", \\\"Normal\\\", 1731921503000.0, \\\"Created pod: java-api-checker-tgmn7\\\"], [\\\"SuccessfulCreate\\\", \\\"Normal\\\", 1731921432000.0, \\\"Created pod: java-api-checker-slp89\\\"]], \\\"column_renderers\\\": {\\\"time\\\": \\\"DATETIME\\\"}}, \\\"metadata\\\": {}}]", - "enrichment_type": "k8s_events", - "file_type": "structured_data", - "id": "22615573-cf16-4bc6-aa7c-b5a888648274", - "issue_id": "d1e5b8cb-4632-48fc-8add-60b58cb200c6", - "title": "Job Events" - } - ], - "failure": true, - "finding_type": "issue", - "fingerprint": "01261151cfe344ff0a66f1df2aa82c90652f0787459b385b5cd7c1c3cdc40ab5", - "group_id": null, - "id": "d1e5b8cb-4632-48fc-8add-60b58cb200c6", - "labels": { - "batch.kubernetes.io/controller-uid": "9872d2b2-9be8-489b-8b90-7c94f2d01e14", - "batch.kubernetes.io/job-name": "java-api-checker", - "controller-uid": "9872d2b2-9be8-489b-8b90-7c94f2d01e14", - "job-name": "java-api-checker" - }, - "priority": "HIGH", - "service_key": "default/Job/java-api-checker", - "service_kind": null, - "source": "kubernetes_api_server", - "starts_at": "2025-05-29T10:44:14+00:00", - "subject_name": "java-api-checker", - "subject_namespace": "default", - "subject_node": null, - "subject_type": "job", - "title": "Job java-api-checker on namespace default failed", - "updated_at": "2024-11-18T09:19:26.994827+00:00", - "video_links": [] -} diff --git a/tests/llm/fixtures/test_investigate/06_job_failure/resource_instructions.json b/tests/llm/fixtures/test_investigate/06_job_failure/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/06_job_failure/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/06_job_failure/test_case.yaml b/tests/llm/fixtures/test_investigate/06_job_failure/test_case.yaml deleted file mode 100644 index f99a27c49c..0000000000 --- a/tests/llm/fixtures/test_investigate/06_job_failure/test_case.yaml +++ /dev/null @@ -1,11 +0,0 @@ -expected_output: - - The job `java-api-checker` in the `default` namespace has failed because of a backoff limit. - - issues connecting to `prod-db:3333` -before_test: | - kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/job_failure/job_crash.yaml || true - kubectl apply -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/job_failure/job_crash.yaml - sleep 40 -after_test: | - kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/job_failure/job_crash.yaml -evaluation: - correctness: 1 diff --git a/tests/llm/fixtures/test_investigate/07_job_syntax_error/investigate_request.json b/tests/llm/fixtures/test_investigate/07_job_syntax_error/investigate_request.json deleted file mode 100644 index 99ec3f4007..0000000000 --- a/tests/llm/fixtures/test_investigate/07_job_syntax_error/investigate_request.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "source": "prometheus", - "title": "KubeJobFailed", - "description": "", - "subject": { - "name": "product-data-scraper", - "namespace": "default", - "kind": "job", - "node": null, - "container": null, - "cluster": null - }, - "context": { - "issue_type": "KubeJobFailed", - "robusta_issue_id": "0c6325dd-f881-41e0-b7df-45bfffce16b5", - "source": "prometheus" - }, - "source_instance_id": "ApiRequest", - "include_tool_calls": true, - "include_tool_call_results": true, - "prompt_template": "builtin://generic_investigation.jinja2" -} diff --git a/tests/llm/fixtures/test_investigate/07_job_syntax_error/issue_data.json b/tests/llm/fixtures/test_investigate/07_job_syntax_error/issue_data.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/07_job_syntax_error/issue_data.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/07_job_syntax_error/resource_instructions.json b/tests/llm/fixtures/test_investigate/07_job_syntax_error/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/07_job_syntax_error/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/07_job_syntax_error/test_case.yaml b/tests/llm/fixtures/test_investigate/07_job_syntax_error/test_case.yaml deleted file mode 100644 index 2d11633a73..0000000000 --- a/tests/llm/fixtures/test_investigate/07_job_syntax_error/test_case.yaml +++ /dev/null @@ -1,10 +0,0 @@ -expected_output: - - The Kubernetes job `product-data-scraper` failed to complete successfully - - "Mention a syntax error: `/bin/sh: syntax error: unexpected 'fi'`" -before_test: | - kubectl apply -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/job_run_forever/job_run_forever.yaml - sleep 60 -after_test: | - kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/job_run_forever/job_run_forever.yaml -evaluation: - correctness: 1 diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/helm/Dockerfile b/tests/llm/fixtures/test_investigate/09_high_latency/helm/Dockerfile deleted file mode 100644 index f932d0180a..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/helm/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM python:3.10-slim - -# Set working directory -WORKDIR /app - -# Copy requirements.txt -COPY requirements.txt . - -# Install dependencies -RUN pip install --no-cache-dir -r requirements.txt - -# Copy the FastAPI app -COPY . . - -# Expose the ports -EXPOSE 8000 8001 - -# Run the FastAPI app -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/helm/app.py b/tests/llm/fixtures/test_investigate/09_high_latency/helm/app.py deleted file mode 100644 index ef742d7906..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/helm/app.py +++ /dev/null @@ -1,70 +0,0 @@ -# ruff: noqa: F821 -import logging -import time -from random import randint -from time import sleep - -from fastapi import FastAPI -from fastapi.responses import HTMLResponse -from prometheus_fastapi_instrumentator import Instrumentator - -app = FastAPI() - -# Configure logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -STORED_PROCEDURE = "sp_CheckUserNotifications" - - -# Add Prometheus middleware -Instrumentator().instrument(app).expose(app) - - -def check_promotional_notifications(): - logger.info( - "Connecting to promotions database to see if we should try to upsell user" - ) - try: - logger.info("Successfully connected to database") - start_time = time.time() - logger.info(f"Fetching data using stored procedure: {STORED_PROCEDURE}") - - sleep(randint(5, 10)) - - result = [(True, {"type": "notification", "discount": f"${randint(6,50)}"})] - end_time = time.time() - logger.info(f"Database call completed in {end_time - start_time:.2f} seconds.") - for row in result: - notifications = row[0] # Access the first element of the tuple - logger.info(f"Promotions result: {notifications}") - return notifications - except Exception as e: - logger.error(f"Error checking for promotions: {e}") - return False - - -@app.get("/", response_class=HTMLResponse) -def read_root(): - logger.info("Received request for checkout page.") - start_time = time.time() - has_promotions = check_promotional_notifications() - end_time = time.time() - logger.info(f"Page rendered in {end_time - start_time:.2f} seconds.") - return f""" - - - Checkout Status - - -

Success!

-

Promotions: {has_promotions}

- - - """ - - -if __name__ == "__main__": - # Start Prometheus metrics server - start_http_server(8001) - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/helm/build.sh b/tests/llm/fixtures/test_investigate/09_high_latency/helm/build.sh deleted file mode 100755 index 1e8f8480d9..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/helm/build.sh +++ /dev/null @@ -1 +0,0 @@ -docker buildx build --platform linux/amd64 . -t us-central1-docker.pkg.dev/genuine-flight-317411/devel/rds-demo:no-db-v2 diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/helm/manifest.yaml b/tests/llm/fixtures/test_investigate/09_high_latency/helm/manifest.yaml deleted file mode 100644 index 525bee56d9..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/helm/manifest.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: customer-orders -spec: - replicas: 1 - selector: - matchLabels: - app: customer-orders - template: - metadata: - labels: - app: customer-orders - spec: - containers: - - name: fastapi-app - image: us-central1-docker.pkg.dev/genuine-flight-317411/devel/rds-demo:no-db-v2 - ports: - - containerPort: 8000 - - containerPort: 8001 - - name: curl-sidecar - image: curlimages/curl - args: - - /bin/sh - - -c - - while true; do curl -s http://localhost:8000; sleep 60; done ---- -apiVersion: v1 -kind: Service -metadata: - name: customer-orders-service - labels: - app: customer-orders -spec: - selector: - app: customer-orders - ports: - - protocol: TCP - port: 80 - targetPort: 8000 - name: http - type: ClusterIP ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: customer-orders-service-monitor - labels: - release: robusta -spec: - selector: - matchLabels: - app: customer-orders - endpoints: - - port: http - path: /metrics - interval: 15s - namespaceSelector: - matchNames: - - default ---- -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: customer-orders-alert-rules - labels: - release: robusta -spec: - groups: - - name: customerorders.rules - rules: - - alert: HighLatencyForCustomerCheckout - expr: rate(http_request_duration_seconds_sum[24h]) / (rate(http_request_duration_seconds_count[24h])) > 3 - for: 1m - labels: - severity: critical - deployment: customer-orders - namespace: default - annotations: - summary: "HTTP Requests to the '/checkout' endpoint in customer-orders-app are taking longer than 3 seconds" - description: "HTTP Requests to the '/checkout' endpoint in customer-orders-app are taking longer than 3 seconds" diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/helm/requirements.txt b/tests/llm/fixtures/test_investigate/09_high_latency/helm/requirements.txt deleted file mode 100644 index a117a62bd9..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/helm/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -fastapi -uvicorn[standard] -prometheus-fastapi-instrumentator diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/investigate_request.json b/tests/llm/fixtures/test_investigate/09_high_latency/investigate_request.json deleted file mode 100644 index 7911b230e3..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/investigate_request.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "source": "prometheus", - "title": "HighLatencyForCustomerCheckout", - "description": "", - "subject": { - "name": "customer-orders", - "namespace": "default", - "kind": "deployment", - "node": null, - "container": null, - "cluster": null - }, - "context": { - "issue_type": "HighLatencyForCustomerCheckout", - "robusta_issue_id": "3baf467c-d6ef-4aac-8e8b-6ddb34a07bb1", - "source": "prometheus" - }, - "source_instance_id": "ApiRequest", - "include_tool_calls": true, - "include_tool_call_results": true, - "prompt_template": "builtin://generic_investigation.jinja2" -} diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/issue_data.json b/tests/llm/fixtures/test_investigate/09_high_latency/issue_data.json deleted file mode 100644 index f64b6b7adb..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/issue_data.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "3baf467c-d6ef-4aac-8e8b-6ddb34a07bb1", - "description": "HTTP Requests to the '/checkout' endpoint in customer-orders-app are taking longer than 3 seconds", - "source": "customer-orders", - "priority": "DEBUG", - "subject_type": "Deployment", - "subject_name": "customer-orders", - "subject_namespace": "default", - "subject_node": null, - "service_key": "", - "cluster": "test_cluster", - "creation_date": "2025-01-27 09:11:23.635001", - "title": "HighLatencyForCustomerCheckout", - "aggregation_key": "HighLatencyForCustomerCheckout", - "finding_type": "issue", - "failure": true, - "labels": {}, - "annotations": {}, - "evidence": [] -} diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/resource_instructions.json b/tests/llm/fixtures/test_investigate/09_high_latency/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/09_high_latency/test_case.yaml b/tests/llm/fixtures/test_investigate/09_high_latency/test_case.yaml deleted file mode 100644 index 32b5f4172a..0000000000 --- a/tests/llm/fixtures/test_investigate/09_high_latency/test_case.yaml +++ /dev/null @@ -1,7 +0,0 @@ -expected_output: - - HTTP requests to the `/checkout` endpoint in the `customer-orders` application which are taking longer than 3 seconds to complete -before_test: kubectl apply -f ./helm/manifest.yaml -after_test: kubectl delete -f ./helm/manifest.yaml -evaluation: - correctness: 1 -# Success rate 100% / 100 evals diff --git a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/investigate_request.json b/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/investigate_request.json deleted file mode 100644 index 327ee88206..0000000000 --- a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/investigate_request.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "source": "prometheus", - "title": "KubeDeploymentReplicasMismatch", - "description": "", - "subject": { - "name": "user-profile-resources", - "namespace": "default", - "kind": "deployment", - "node": null, - "container": null, - "cluster": null - }, - "context": { - "issue_type": "KubeDeploymentReplicasMismatch", - "robusta_issue_id": "0cb3f2c2-c266-482e-a48e-71cd62cbf903", - "source": "prometheus" - }, - "source_instance_id": "ApiRequest", - "include_tool_calls": true, - "include_tool_call_results": true, - "prompt_template": "builtin://generic_investigation.jinja2", - "sections": null -} diff --git a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/issue_data.json b/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/issue_data.json deleted file mode 100644 index 1137294c95..0000000000 --- a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/issue_data.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "id": "0cb3f2c2-c266-482e-a48e-71cd62cbf903", - "description": "Deployment default/user-profile-resources has not matched the expected number of replicas for longer than 15 minutes.\n\n0 pod(s) are available. 1 pod(s) are not ready due to scheduling issue", - "source": "prometheus", - "category": null, - "priority": "LOW", - "account_id": "f5497e27-ab3c-478e-952a-c4c12cf57ae4", - "subject_type": "deployment", - "subject_name": "user-profile-resources", - "service_key": "default/Deployment/user-profile-resources", - "subject_namespace": "default", - "cluster": "local-kind-cluster", - "creation_date": "2025-01-27T06:06:29.045567", - "title": "Deployment has not matched the expected number of replicas.", - "aggregation_key": "KubeDeploymentReplicasMismatch", - "finding_type": "issue", - "failure": true, - "group_id": null, - "subject_node": null, - "starts_at": "2025-01-27T06:06:27.153+00:00", - "ends_at": null, - "updated_at": "2025-01-27T06:06:28.95505+00:00", - "fingerprint": "7de1c30ff0d0812a", - "video_links": [ - { - "url": "http://robusta-kube-prometheus-st-prometheus.default:9090/graph?g0.expr=%28kube_deployment_spec_replicas%7Bjob%3D%22kube-state-metrics%22%2Cnamespace%3D~%22.%2A%22%7D+%3E+kube_deployment_status_replicas_available%7Bjob%3D%22kube-state-metrics%22%2Cnamespace%3D~%22.%2A%22%7D%29+and+%28changes%28kube_deployment_status_replicas_updated%7Bjob%3D%22kube-state-metrics%22%2Cnamespace%3D~%22.%2A%22%7D%5B10m%5D%29+%3D%3D+0%29&g0.tab=1", - "name": "View Graph", - "type": "prometheus_generator_url" - } - ], - "service_kind": null, - "service_name": null, - "labels": { - "job": "kube-state-metrics", - "pod": "robusta-kube-state-metrics-8667fd9775-dmqfj", - "service": "robusta-kube-state-metrics", - "endpoint": "http", - "instance": "10.244.0.11:8080", - "severity": "warning", - "alertname": "KubeDeploymentReplicasMismatch", - "container": "kube-state-metrics", - "namespace": "default", - "deployment": "user-profile-resources", - "prometheus": "default/robusta-kube-prometheus-st-prometheus" - }, - "annotations": { - "summary": "Deployment has not matched the expected number of replicas.", - "description": "Deployment default/user-profile-resources has not matched the expected number of replicas for longer than 15 minutes.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch", - "deployment.kubernetes.io/revision": "1", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"name\":\"user-profile-resources\",\"namespace\":\"default\"},\"spec\":{\"replicas\":1,\"selector\":{\"matchLabels\":{\"app\":\"user-profile-resources\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"user-profile-resources\"}},\"spec\":{\"containers\":[{\"image\":\"nginx\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"profile-getter\",\"resources\":{\"limits\":{\"cpu\":3,\"memory\":\"5Gi\",\"nvidia.com/gpu\":5},\"requests\":{\"cpu\":3,\"memory\":\"5Gi\",\"nvidia.com/gpu\":5}}}]}}}}\n" - }, - "evidence": [ - { - "file_type": "structured_data", - "data": "[{\"type\": \"markdown\", \"data\": \"**Pod events for user-profile-resources-659d4dd659-cq4kq:**\"}, {\"type\": \"table\", \"data\": {\"headers\": [\"reason\", \"type\", \"time\", \"message\"], \"rows\": [[\"FailedScheduling\", \"Warning\", 1737957030379.022, \"0/1 nodes are available: 1 Insufficient nvidia.com/gpu. preemption: 0/1 nodes are available: 1 No preemption victims found for incoming pod.\"], [\"FailedScheduling\", \"Warning\", 1737719247664.4758, \"0/1 nodes are available: 1 Insufficient nvidia.com/gpu. preemption: 0/1 nodes are available: 1 No preemption victims found for incoming pod.\"]], \"column_renderers\": {\"time\": \"DATETIME\"}}, \"metadata\": {}}]", - "creation_date": "2025-01-27T06:06:28.752395", - "issue_id": "0cb3f2c2-c266-482e-a48e-71cd62cbf903", - "id": "72a3d89b-ac8d-44f0-8596-ab4495829a6b", - "account_id": "f5497e27-ab3c-478e-952a-c4c12cf57ae4", - "enrichment_type": "k8s_events", - "collection_timestamp": null, - "title": "Deployment Events" - }, - { - "file_type": "structured_data", - "data": "[{\"type\": \"table\", \"data\": {\"headers\": [\"label\", \"value\"], \"rows\": [[\"Pod\", \"user-profile-resources-659d4dd659-cq4kq\"], [\"Reason\", \"0/1 nodes are available: 1 Insufficient nvidia.com/gpu. preemption: 0/1 nodes are available: 1 No preemption victims found for incoming pod.\"]], \"column_renderers\": {}}, \"metadata\": {\"format\": \"vertical\"}}, {\"type\": \"markdown\", \"data\": \"**Pod requires:** 5.0 nvidia.com/gpu\"}]", - "creation_date": "2025-01-27T06:06:28.446375", - "issue_id": "0cb3f2c2-c266-482e-a48e-71cd62cbf903", - "id": "dbfb24f6-296e-4a28-841e-193c38baf0e5", - "account_id": "f5497e27-ab3c-478e-952a-c4c12cf57ae4", - "enrichment_type": "pending_pod_info", - "collection_timestamp": null, - "title": "Unscheduled Pod Information" - }, - { - "file_type": "structured_data", - "data": "[{\"type\": \"markdown\", \"data\": \"**Replicas: Desired (1) --> Running (0)**\"}]", - "creation_date": "2025-01-27T06:06:28.594406", - "issue_id": "0cb3f2c2-c266-482e-a48e-71cd62cbf903", - "id": "f29843a9-8279-43d6-89af-d44cf4d94803", - "account_id": "f5497e27-ab3c-478e-952a-c4c12cf57ae4", - "enrichment_type": null, - "collection_timestamp": null, - "title": null - }, - { - "file_type": "structured_data", - "data": "[{\"type\": \"markdown\", \"data\": \"**Alert labels**\"}, {\"type\": \"table\", \"data\": {\"headers\": [\"label\", \"value\"], \"rows\": [[\"alertname\", \"KubeDeploymentReplicasMismatch\"], [\"container\", \"kube-state-metrics\"], [\"deployment\", \"user-profile-resources\"], [\"endpoint\", \"http\"], [\"instance\", \"10.244.0.11:8080\"], [\"job\", \"kube-state-metrics\"], [\"namespace\", \"default\"], [\"pod\", \"robusta-kube-state-metrics-8667fd9775-dmqfj\"], [\"prometheus\", \"default/robusta-kube-prometheus-st-prometheus\"], [\"service\", \"robusta-kube-state-metrics\"], [\"severity\", \"warning\"]], \"column_renderers\": {}}, \"metadata\": {\"format\": \"vertical\"}}]", - "creation_date": "2025-01-27T06:06:28.90165", - "issue_id": "0cb3f2c2-c266-482e-a48e-71cd62cbf903", - "id": "83b9ca3f-f9d1-4b75-bd61-a4a39b8e82a0", - "account_id": "f5497e27-ab3c-478e-952a-c4c12cf57ae4", - "enrichment_type": "alert_labels", - "collection_timestamp": null, - "title": "Alert labels" - } - ] -} diff --git a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/resource_instructions.json b/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/resource_instructions.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/resource_instructions.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/test_case.yaml b/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/test_case.yaml deleted file mode 100644 index 3b4d6c95f9..0000000000 --- a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/test_case.yaml +++ /dev/null @@ -1,9 +0,0 @@ -tags: [prometheus, kubernetes] -expected_output: insufficient GPU resources -before_test: | - kubectl apply -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/pending_pods/pending_pod_resources.yaml - sleep 30 -after_test: | - kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/pending_pods/pending_pod_resources.yaml -evaluation: - correctness: 1 diff --git a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/toolsets.yaml b/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/toolsets.yaml deleted file mode 100644 index 0362cff502..0000000000 --- a/tests/llm/fixtures/test_investigate/10_KubeDeploymentReplicasMismatch/toolsets.yaml +++ /dev/null @@ -1,9 +0,0 @@ -toolsets: - prometheus/metrics: - enabled: False - kubernetes/core: - enabled: true - robusta: - enabled: true - internet: - enabled: true diff --git a/tests/llm/fixtures/test_investigate/15_dns_resolution/investigate_request.json b/tests/llm/fixtures/test_investigate/15_dns_resolution/investigate_request.json deleted file mode 100644 index d3a8dd7e09..0000000000 --- a/tests/llm/fixtures/test_investigate/15_dns_resolution/investigate_request.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "source": "prometheus", - "title": "AppDown", - "description": "The application is down", - "subject": {}, - "context": { - "issue_type": "AppDown", - "robusta_issue_id": "16ecba1a-7993-4dd1-a98c-d201462ccba8", - "source": "prometheus" - }, - "source_instance_id": "ApiRequest", - "include_tool_calls": true, - "include_tool_call_results": true, - "prompt_template": "builtin://generic_investigation.jinja2" -} diff --git a/tests/llm/fixtures/test_investigate/15_dns_resolution/issue_data.json b/tests/llm/fixtures/test_investigate/15_dns_resolution/issue_data.json deleted file mode 100644 index 9634a524c0..0000000000 --- a/tests/llm/fixtures/test_investigate/15_dns_resolution/issue_data.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "id": "16ecba1a-7993-4dd1-a98c-d201462ccba8", - "name": "AppDown", - "source_type": "prometheus", - "source_instance_id": "http://localhost:9093", - "url": "http://robusta-kube-prometheus-st-prometheus.default:9090/graph?g0.expr=vector%281%29&g0.tab=1", - "raw": { - "status": "active", - "labels": { - "alertname": "AppDown", - "prometheus": "default/robusta-kube-prometheus-st-prometheus", - "severity": "critical", - "namespace": "investigate-namespace-15", - "pod": "client" - }, - "annotations": { - "description": "The service is down", - "runbook_url": "", - "summary": "The service is down" - }, - "startsAt": "2025-06-18T07:21:00.299+00:00", - "endsAt": null, - "generatorURL": "http://robusta-kube-prometheus-st-prometheus.default:9090/graph?g0.expr=vector%281%29&g0.tab=1", - "unique_id": "16ecba1a-7993-4dd1-a98c-d201462ccba8", - "duration": "0:02:30", - "name": "AppDown" - }, - "presentation_status": null, - "presentation_key_metadata": "*Severity*: none", - "presentation_all_metadata": "*Labels:*\n•" -} diff --git a/tests/llm/fixtures/test_investigate/15_dns_resolution/manifest.yaml b/tests/llm/fixtures/test_investigate/15_dns_resolution/manifest.yaml deleted file mode 100644 index 06a36c8e05..0000000000 --- a/tests/llm/fixtures/test_investigate/15_dns_resolution/manifest.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: investigate-namespace-15 ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-deny-egress - namespace: investigate-namespace-15 -spec: - podSelector: {} - policyTypes: - - Egress - egress: [] ---- -apiVersion: v1 -kind: Pod -metadata: - name: client - namespace: investigate-namespace-15 - labels: - app.kubernetes.io/name: client -spec: - containers: - - image: mainred/client:v2 - imagePullPolicy: Always - name: test-client ---- -apiVersion: v1 -kind: Pod -metadata: - name: nginx - namespace: investigate-namespace-15 - labels: - app.kubernetes.io/name: server -spec: - containers: - - name: nginx - image: nginx:stable - ports: - - containerPort: 80 - name: http-web-svc - ---- -apiVersion: v1 -kind: Service -metadata: - name: nginx-service - namespace: investigate-namespace-15 -spec: - selector: - app.kubernetes.io/name: server - ports: - - name: name-of-service-port - protocol: TCP - port: 80 - targetPort: http-web-svc diff --git a/tests/llm/fixtures/test_investigate/15_dns_resolution/resource_instructions.json b/tests/llm/fixtures/test_investigate/15_dns_resolution/resource_instructions.json deleted file mode 100644 index 4a22cf8ca3..0000000000 --- a/tests/llm/fixtures/test_investigate/15_dns_resolution/resource_instructions.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "instructions": [], - "documents": [{ - "url": "https://github.com/robusta-dev/kubernetes-demos/blob/main/runbooks/dns_resolutions.md" - }] -} diff --git a/tests/llm/fixtures/test_investigate/15_dns_resolution/test_case.yaml b/tests/llm/fixtures/test_investigate/15_dns_resolution/test_case.yaml deleted file mode 100644 index 3b5262c33b..0000000000 --- a/tests/llm/fixtures/test_investigate/15_dns_resolution/test_case.yaml +++ /dev/null @@ -1,11 +0,0 @@ -expected_output: - - point to a network policy named `default-deny-egress` -before_test: | - kubectl apply -f./manifest.yaml - sleep 10 -after_test: kubectl delete -f./manifest.yaml -evaluation: - correctness: 0 -tags: - - network - - runbooks diff --git a/tests/llm/fixtures/test_investigate/15_dns_resolution/toolsets.yaml b/tests/llm/fixtures/test_investigate/15_dns_resolution/toolsets.yaml deleted file mode 100644 index 2700c71019..0000000000 --- a/tests/llm/fixtures/test_investigate/15_dns_resolution/toolsets.yaml +++ /dev/null @@ -1,22 +0,0 @@ -toolsets: - kubernetes/core: - enabled: false - kubernetes/logs: - enabled: true - internet: - enabled: true - bash: - enabled: true - config: - kubectl: - allowed_images: - - image: busybox - allowed_commands: - - "cat .*" - - "nslookup .*" - - "sh .*" - - image: "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3" - allowed_commands: - - "cat .*" - - "nslookup .*" - - "sh .*" diff --git a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/investigate_request.json b/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/investigate_request.json deleted file mode 100644 index d3a8dd7e09..0000000000 --- a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/investigate_request.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "source": "prometheus", - "title": "AppDown", - "description": "The application is down", - "subject": {}, - "context": { - "issue_type": "AppDown", - "robusta_issue_id": "16ecba1a-7993-4dd1-a98c-d201462ccba8", - "source": "prometheus" - }, - "source_instance_id": "ApiRequest", - "include_tool_calls": true, - "include_tool_call_results": true, - "prompt_template": "builtin://generic_investigation.jinja2" -} diff --git a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/issue_data.json b/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/issue_data.json deleted file mode 100644 index 3a239702c3..0000000000 --- a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/issue_data.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "id": "16ecba1a-7993-4dd1-a98c-d201462ccba8", - "name": "AppDown", - "source_type": "prometheus", - "source_instance_id": "http://localhost:9093", - "url": "http://robusta-kube-prometheus-st-prometheus.default:9090/graph?g0.expr=vector%281%29&g0.tab=1", - "raw": { - "status": "active", - "labels": { - "alertname": "AppDown", - "prometheus": "default/robusta-kube-prometheus-st-prometheus", - "severity": "critical", - "namespace": "investigate-namespace-16", - "pod": "client" - }, - "annotations": { - "description": "The service is down", - "runbook_url": "", - "summary": "The service is down" - }, - "startsAt": "2025-06-18T07:21:00.299+00:00", - "endsAt": null, - "generatorURL": "http://robusta-kube-prometheus-st-prometheus.default:9090/graph?g0.expr=vector%281%29&g0.tab=1", - "unique_id": "16ecba1a-7993-4dd1-a98c-d201462ccba8", - "duration": "0:02:30", - "name": "AppDown" - }, - "presentation_status": null, - "presentation_key_metadata": "*Severity*: none", - "presentation_all_metadata": "*Labels:*\n•" -} diff --git a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/manifest.yaml b/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/manifest.yaml deleted file mode 100644 index 2211013f6a..0000000000 --- a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/manifest.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: investigate-namespace-16 ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-deny-egress - namespace: investigate-namespace-16 -spec: - podSelector: {} - policyTypes: - - Egress - egress: [] ---- -apiVersion: v1 -kind: Pod -metadata: - name: client - namespace: investigate-namespace-16 - labels: - app.kubernetes.io/name: client -spec: - containers: - - image: mainred/client:v2 - imagePullPolicy: Always - name: test-client ---- -apiVersion: v1 -kind: Pod -metadata: - name: nginx - namespace: investigate-namespace-16 - labels: - app.kubernetes.io/name: server -spec: - containers: - - name: nginx - image: nginx:stable - ports: - - containerPort: 80 - name: http-web-svc - ---- -apiVersion: v1 -kind: Service -metadata: - name: nginx-service - namespace: investigate-namespace-16 -spec: - selector: - app.kubernetes.io/name: server - ports: - - name: name-of-service-port - protocol: TCP - port: 80 - targetPort: http-web-svc diff --git a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/resource_instructions.json b/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/resource_instructions.json deleted file mode 100644 index 4a22cf8ca3..0000000000 --- a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/resource_instructions.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "instructions": [], - "documents": [{ - "url": "https://github.com/robusta-dev/kubernetes-demos/blob/main/runbooks/dns_resolutions.md" - }] -} diff --git a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/test_case.yaml b/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/test_case.yaml deleted file mode 100644 index a2418e687b..0000000000 --- a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/test_case.yaml +++ /dev/null @@ -1,10 +0,0 @@ -expected_output: - - mention lack of tool and user should enable either kubernetes toolsets or bash toolset or both -before_test: | - kubectl apply -f./manifest.yaml - sleep 10 -after_test: kubectl delete -f./manifest.yaml -tags: - - network - - runbooks - - transparency diff --git a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/toolsets.yaml b/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/toolsets.yaml deleted file mode 100644 index 4bafbd3b7f..0000000000 --- a/tests/llm/fixtures/test_investigate/16_dns_resolution_no_tool/toolsets.yaml +++ /dev/null @@ -1,23 +0,0 @@ -toolsets: - kubernetes/core: - enabled: false - kubernetes/logs: - enabled: false - kubernetes/kube-lineage-extras: - enabled: false - internet: - enabled: false - bash: - enabled: false - helm/core: - enabled: false - kafka/admin: - enabled: false - aks/core: - enabled: false - kubernetes/live-metrics: - enabled: false - kubernetes/kube-prometheus-stack: - enabled: false - runbook: - enabled: false diff --git a/tests/llm/test_investigate.py b/tests/llm/test_investigate.py deleted file mode 100644 index e49e215f55..0000000000 --- a/tests/llm/test_investigate.py +++ /dev/null @@ -1,253 +0,0 @@ -# type: ignore -import os -import time -from contextlib import ExitStack -from os import path -from pathlib import Path -from typing import Optional -from unittest.mock import patch - -import pytest - -from holmes.config import Config -from holmes.core.investigation import investigate_issues -from holmes.core.tools_utils.filesystem_result_storage import tool_result_storage -from holmes.core.investigation_structured_output import DEFAULT_SECTIONS -from holmes.core.supabase_dal import SupabaseDal -from holmes.core.tool_calling_llm import IssueInvestigator -from holmes.core.tools_utils.tool_executor import ToolExecutor -from holmes.core.tracing import SpanType, TracingFactory -from tests.llm.utils.braintrust import log_to_braintrust -from tests.llm.utils.classifiers import ( - evaluate_correctness, - evaluate_sections, -) -from tests.llm.utils.commands import apply_env_config, set_test_env_vars -from tests.llm.utils.env_config import EnvConfig, get_env_configs -from tests.llm.utils.iteration_utils import get_test_cases -from tests.llm.utils.mock_dal import TestSupabaseDal -from tests.llm.utils.test_toolset import TestToolsetManager -from tests.llm.utils.property_manager import ( - handle_test_error, - set_initial_properties, - set_trace_properties, - update_test_results, -) -from tests.llm.utils.retry_handler import retry_on_throttle -from tests.llm.utils.test_case_utils import ( - InvestigateTestCase, - check_and_skip_test, - get_models, -) - -TEST_CASES_FOLDER = Path( - path.abspath(path.join(path.dirname(__file__), "fixtures", "test_investigate")) -) - - -class TestConfig(Config): - def __init__(self, test_case: InvestigateTestCase, tracer): - super().__init__() - self._test_case = test_case - self._tracer = tracer - self._cached_tool_executor: Optional[ToolExecutor] = None - - def create_tool_executor(self, dal: Optional[SupabaseDal]) -> ToolExecutor: - if not self._cached_tool_executor: - manager = TestToolsetManager( - test_case_folder=self._test_case.folder, - toolsets_config_path=getattr( - self._test_case, "toolsets_config_path", None - ), - ) - self._cached_tool_executor = ToolExecutor(manager.toolsets) - return self._cached_tool_executor - - def create_issue_investigator( - self, - dal: Optional[SupabaseDal] = None, - model: Optional[str] = None, - tracer=None, - tool_results_dir=None, - ) -> IssueInvestigator: - # Use our tracer instead of the passed one - return super().create_issue_investigator( - dal=dal, model=model, tracer=self._tracer, tool_results_dir=tool_results_dir - ) - - -def get_investigate_test_cases(): - return get_test_cases(TEST_CASES_FOLDER) - - -def _get_env_config_ids(): - """Generate ids for env_config parameterization.""" - return [ec.name for ec in get_env_configs()] - - -@pytest.mark.llm -@pytest.mark.parametrize("env_config", get_env_configs(), ids=_get_env_config_ids()) -@pytest.mark.parametrize("model", get_models()) -@pytest.mark.parametrize("test_case", get_investigate_test_cases()) -def test_investigate( - env_config: EnvConfig, - model: str, - test_case: InvestigateTestCase, - caplog, - request, - shared_test_infrastructure, # type: ignore -): - # Set initial properties early so they're available even if test fails - set_initial_properties(request, test_case, model, env_config) - - tracer = TracingFactory.create_tracer("braintrust") - config = TestConfig(test_case, tracer) - config.model = model - metadata = {"model": model, "env_config": env_config.name} - tracer.start_experiment(additional_metadata=metadata) - - test_dal = TestSupabaseDal( - test_case_folder=Path(test_case.folder), - issue_data=test_case.issue_data, - issues_metadata=None, - resource_instructions=test_case.resource_instructions, - ) - - input = test_case.investigate_request - expected = test_case.expected_output - result = None - output = None - scores = {} - - investigate_request = test_case.investigate_request - if not investigate_request.sections: - investigate_request.sections = DEFAULT_SECTIONS - - try: - with patch.dict( - os.environ, {"HOLMES_STRUCTURED_OUTPUT_CONVERSION_FEATURE_FLAG": "False"} - ): - with tracer.start_trace( - name=f"{test_case.id}[{model}][{env_config.name}]", - span_type=SpanType.EVAL, - ) as eval_span: - set_trace_properties(request, eval_span) - check_and_skip_test(test_case, request, shared_test_infrastructure) - - with ExitStack() as stack: - stack.enter_context(apply_env_config(env_config)) - stack.enter_context(set_test_env_vars(test_case)) - tool_results_dir = stack.enter_context(tool_result_storage()) - - with eval_span.start_span( - "Caching tools executor for create_issue_investigator", - type=SpanType.TASK.value, - ): - config.create_tool_executor(test_dal) - with eval_span.start_span( - "Holmes Run", type=SpanType.TASK.value - ) as holmes_span: - start_time = time.time() - retry_enabled = request.config.getoption( - "retry-on-throttle", default=True - ) - result = retry_on_throttle( - investigate_issues, - investigate_request=investigate_request, - config=config, - dal=test_dal, - trace_span=holmes_span, - retry_enabled=retry_enabled, - test_id=test_case.id, - model=model, - tool_results_dir=tool_results_dir, - ) - holmes_duration = time.time() - start_time - # Log duration directly to eval_span - eval_span.log(metadata={"holmes_duration": holmes_duration}) - # Store metrics in user_properties for GitHub report - request.node.user_properties.append( - ("holmes_duration", holmes_duration) - ) - if result and result.num_llm_calls is not None: - request.node.user_properties.append( - ("num_llm_calls", result.num_llm_calls) - ) - if result and result.tool_calls is not None: - request.node.user_properties.append( - ("tool_call_count", len(result.tool_calls)) - ) - - # Evaluate and log results inside the span context - assert result, "No result returned by investigate_issues()" - - output = result.analysis - - correctness_eval = evaluate_correctness( - output=output, - expected_elements=expected, - parent_span=eval_span, - caplog=caplog, - evaluation_type="strict", - ) - scores["correctness"] = correctness_eval.score - - if test_case.expected_sections: - sections = { - key: bool(value) - for key, value in test_case.expected_sections.items() - } - sections_eval = evaluate_sections( - sections=sections, output=output, parent_span=eval_span - ) - scores["sections"] = sections_eval.score - - # Log evaluation results to the span - log_to_braintrust( - eval_span=eval_span, - test_case=test_case, - model=model, - result=result, - scores=scores, - ) - except Exception as e: - handle_test_error( - request=request, - error=e, - eval_span=eval_span if "eval_span" in locals() else None, - test_case=test_case, - model=model, - result=result, - ) - raise - - tools_called = [t.tool_name for t in result.tool_calls] if result.tool_calls else [] - update_test_results(request, output, tools_called, scores, result) - - assert result.sections, "Missing sections" - assert ( - len(result.sections) >= len(investigate_request.sections) - ), f"Received {len(result.sections)} sections but expected {len(investigate_request.sections)}. Received: {result.sections.keys()}" - for expected_section_title in investigate_request.sections: - assert ( - expected_section_title in result.sections - ), f"Expected title {expected_section_title} in sections" - - assert ( - int(scores.get("correctness", 0)) == 1 - ), f"Test {test_case.id} failed (score: {scores.get('correctness', 0)})" - - if test_case.expected_sections: - for ( - expected_section_title, - expected_section_array_content, - ) in test_case.expected_sections.items(): - if expected_section_array_content: - assert ( - expected_section_title in result.sections - ), f"Expected to see section [{expected_section_title}] in result but that section is missing" - for expected_content in expected_section_array_content: - assert ( - expected_content - in result.sections.get(expected_section_title, "") - ), f"Expected to see content [{expected_content}] in section [{expected_section_title}] but could not find such content" diff --git a/tests/llm/utils/braintrust.py b/tests/llm/utils/braintrust.py index fba03ab00f..76a4f825e1 100644 --- a/tests/llm/utils/braintrust.py +++ b/tests/llm/utils/braintrust.py @@ -8,6 +8,7 @@ from pydantic import BaseModel from holmes.core.llm import TokenCountMetadata +from holmes.core.tool_calling_llm import LLMResult from holmes.core.tracing import ( BRAINTRUST_API_KEY, BRAINTRUST_ORG, @@ -16,7 +17,7 @@ get_experiment_name, get_machine_state_tags, ) -from tests.llm.utils.test_case_utils import HolmesTestCase # type: ignore +from tests.llm.utils.test_case_utils import AskHolmesTestCase, HolmesTestCase # type: ignore braintrust_enabled = False if BRAINTRUST_API_KEY: @@ -185,21 +186,23 @@ def log_to_braintrust( eval_span, test_case: HolmesTestCase, model: str, - result: Optional[Any] = None, # Can be LLMResult or InvestigationResult + result: Optional[Union[LLMResult, CompactionResult]] = None, scores: Optional[dict] = None, error: Optional[Exception] = None, ) -> None: - """Shared function to log evaluation data to Braintrust. + """Log evaluation data to Braintrust. + + Handles both ask_holmes tests (LLMResult) and compaction tests + (CompactionResult), logging appropriate metrics for each. Args: eval_span: The Braintrust evaluation span - test_case: The test case being evaluated (AskHolmesTestCase or InvestigateTestCase) + test_case: The test case being evaluated model: The model being tested - result: The result object (LLMResult for ask, InvestigationResult for investigate) + result: LLMResult for ask_holmes tests, CompactionResult for compaction tests scores: Dictionary of scores (e.g., correctness) error: Exception if the test failed """ - from tests.llm.utils.test_case_utils import AskHolmesTestCase, InvestigateTestCase # Prepare tags tags = (test_case.tags or []).copy() @@ -207,26 +210,14 @@ def log_to_braintrust( # Determine output based on test type and error state if error: - if hasattr( - result, "result" - ): # AskHolmesTestCase with LLMResult or CompactionResult + if hasattr(result, "result"): output = result.result if result else str(error) - elif hasattr( - result, "analysis" - ): # InvestigateTestCase with InvestigationResult - output = result.analysis if result else str(error) else: output = str(error) scores = scores or {} else: - if hasattr( - result, "result" - ): # AskHolmesTestCase with LLMResult or CompactionResult + if hasattr(result, "result"): output = result.result if result else "" - elif hasattr( - result, "analysis" - ): # InvestigateTestCase with InvestigationResult - output = result.analysis if result else "" else: output = "" @@ -322,9 +313,6 @@ def log_to_braintrust( if isinstance(test_case.expected_output, str) else str(test_case.expected_output) ) - elif isinstance(test_case, InvestigateTestCase): - input_data = str(test_case.investigate_request) - expected = str(test_case.expected_output) elif test_case.conversation_history: # compaction test case from tests.llm.utils.conversation_formatter import ( format_conversation_as_markdown, @@ -359,9 +347,6 @@ def get_braintrust_url( """Generate Braintrust URL for a test. Args: - test_suite: Either "ask_holmes" or "investigate" - test_id: Test ID like "01" - test_name: Test name like "how_many_pods" span_id: Optional span ID for direct linking root_span_id: Optional root span ID for direct linking diff --git a/tests/llm/utils/classifiers.py b/tests/llm/utils/classifiers.py index 6ed408b199..bdfd3670f9 100644 --- a/tests/llm/utils/classifiers.py +++ b/tests/llm/utils/classifiers.py @@ -235,85 +235,3 @@ def evaluate_correctness( ) -def evaluate_sections( - sections: dict[str, bool], output: Optional[str], parent_span: Optional[Span] -): - expected_sections = [section for section, expected in sections.items() if expected] - expected_sections_str = "\n".join([f"- {section}" for section in expected_sections]) - if not expected_sections_str: - expected_sections_str = "" - - unexpected_sections = [ - section for section, expected in sections.items() if not expected - ] - unexpected_sections_str = "\n".join( - [f"- {section}" for section in unexpected_sections] - ) - if not unexpected_sections_str: - unexpected_sections_str = "" - - prompt_prefix = """ -You are evaluating the correctness of an OUTPUT given by a LLM. You must return a score that -represents the correctness of that OUTPUT. - -The LLM output is expected to be split into sections. Typically each section is represented by a markdown title `#
`. -Some sections are expected and should be populated in the output. Some sections are unexpected and should not be present in the outpout -(i.e. there is no such title: `# in EXPECTED SECTIONS assume the OUTPUT has all appropriate EXPECTED SECTIONS. -If there are in UNEXPECTED SECTIONS assume the OUTPUT has no UNEXPECTED SECTIONS. - - -# EXPECTED SECTIONS - -{{expected}} - - -# UNEXPECTED SECTIONS - -{{input}} - - -# OUTPUT - -{{output}} - - -Return a choice based on the number of EXPECTED ELEMENTS present in the OUTPUT. -Possible choices: -A. One or more of the EXPECTED SECTIONS is missing and one or more of the UNEXPECTED SECTIONS is present -B. All EXPECTED SECTIONS are present in the OUTPUT and no UNEXPECTED SECTIONS is present in the output -""" - - classifier = LLMClassifier( - name="sections", - prompt_template=prompt_prefix, - choice_scores={"A": 0, "B": 1}, - use_cot=True, - model=classifier_model, - ) - if parent_span: - with parent_span.start_span( - name="Sections", type=SpanTypeAttribute.SCORE - ) as span: - correctness_eval = classifier( - input=unexpected_sections_str, - output=output, - expected=expected_sections_str, - ) - - span.log( - input=prompt_prefix, - output=correctness_eval.metadata.get("rationale", ""), - expected=expected_sections_str, - scores={ - "sections": correctness_eval.score, - }, - metadata=correctness_eval.metadata, - ) - - return correctness_eval - else: - return classifier( - input=unexpected_sections_str, output=output, expected=expected_sections_str - ) diff --git a/tests/llm/utils/langfuse.py b/tests/llm/utils/langfuse.py index adb17f6f8f..fa9937c5d3 100644 --- a/tests/llm/utils/langfuse.py +++ b/tests/llm/utils/langfuse.py @@ -9,7 +9,6 @@ from tests.llm.utils.test_case_utils import ( AskHolmesTestCase, HolmesTestCase, - InvestigateTestCase, ) # init @@ -51,8 +50,6 @@ def get_input(test_case: HolmesTestCase) -> Dict[str, Any]: input = {} if isinstance(test_case, AskHolmesTestCase): input = {"user_prompt": test_case.user_prompt} - elif isinstance(test_case, InvestigateTestCase): - input = test_case.investigate_request.model_dump() input["id"] = test_case.id return input diff --git a/tests/llm/utils/property_manager.py b/tests/llm/utils/property_manager.py index 63341e327a..d22decdad5 100644 --- a/tests/llm/utils/property_manager.py +++ b/tests/llm/utils/property_manager.py @@ -116,7 +116,7 @@ def update_test_results( output: The test output string tools_called: List of tools called or a string description scores: Dictionary of scores (e.g., correctness). If None and test_case is provided, will calculate - result: Optional result object (LLMResult or InvestigationResult) containing cost info + result: Optional result object (LLMResult) containing cost info test_case: Optional test case for score calculation eval_span: Optional Braintrust span for evaluation caplog: Optional caplog for evaluation @@ -126,7 +126,7 @@ def update_test_results( """ # Calculate scores if not provided but test_case is available if scores is None and test_case is not None: - from tests.llm.utils.classifiers import evaluate_correctness, evaluate_sections + from tests.llm.utils.classifiers import evaluate_correctness scores = {} @@ -203,16 +203,6 @@ def update_test_results( ) scores["correctness"] = correctness_eval.score - # Evaluate sections if applicable (for investigate tests) - if hasattr(test_case, "expected_sections") and test_case.expected_sections: - sections = { - key: bool(value) for key, value in test_case.expected_sections.items() - } - sections_eval = evaluate_sections( - sections=sections, output=output, parent_span=eval_span - ) - scores["sections"] = sections_eval.score - # Default scores if still None if scores is None: scores = {} diff --git a/tests/llm/utils/reporting/github_reporter.py b/tests/llm/utils/reporting/github_reporter.py index 5e221666f8..c6e40f02e7 100644 --- a/tests/llm/utils/reporting/github_reporter.py +++ b/tests/llm/utils/reporting/github_reporter.py @@ -283,13 +283,6 @@ def generate_markdown_report( ask_holmes_skipped = 0 ask_holmes_setup_failures = 0 - investigate_total = 0 - investigate_passed = 0 - investigate_regressions = 0 - investigate_mock_failures = 0 - investigate_skipped = 0 - investigate_setup_failures = 0 - for result in sorted_results: status = TestStatus(result) @@ -305,19 +298,6 @@ def generate_markdown_report( ask_holmes_regressions += 1 elif status.is_mock_failure: ask_holmes_mock_failures += 1 - elif result["test_type"] == "investigate": - investigate_total += 1 - if status.is_skipped: - investigate_skipped += 1 - elif status.is_setup_failure: - investigate_setup_failures += 1 - elif status.passed: - investigate_passed += 1 - elif status.is_regression: - investigate_regressions += 1 - elif status.is_mock_failure: - investigate_mock_failures += 1 - # Generate summary lines if ask_holmes_total > 0: markdown += f"- ask_holmes: {ask_holmes_passed}/{ask_holmes_total} test cases were successful, {ask_holmes_regressions} regressions" @@ -328,16 +308,6 @@ def generate_markdown_report( if ask_holmes_mock_failures > 0: markdown += f", {ask_holmes_mock_failures} mock failures" markdown += "\n" - if investigate_total > 0: - markdown += f"- investigate: {investigate_passed}/{investigate_total} test cases were successful, {investigate_regressions} regressions" - if investigate_skipped > 0: - markdown += f", {investigate_skipped} skipped" - if investigate_setup_failures > 0: - markdown += f", {investigate_setup_failures} setup failures" - if investigate_mock_failures > 0: - markdown += f", {investigate_mock_failures} mock failures" - markdown += "\n" - # Generate detailed table markdown += "\n\n| Status | Test case | Time | Turns | Tools | Cost | Total tokens | Input | Output | Cached | Non-cached | Reasoning | Max output | Compactions |\n" markdown += "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n" @@ -481,5 +451,5 @@ def generate_markdown_report( return ( markdown, sorted_results, - ask_holmes_regressions + investigate_regressions, + ask_holmes_regressions, ) diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 9862144e8a..3e84bd94b9 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -11,7 +11,6 @@ from holmes.config import Config from holmes.core.llm import DefaultLLM -from holmes.core.models import InvestigateRequest from holmes.core.prompt import append_file_to_user_prompt from holmes.core.resource_instruction import ResourceInstructions from tests.llm.utils.constants import ALLOWED_EVAL_TAGS, get_allowed_tags_list @@ -169,14 +168,6 @@ class AskHolmesTestCase(HolmesTestCase, BaseModel): test_type: Optional[str] = None # The type of test to run -class InvestigateTestCase(HolmesTestCase, BaseModel): - investigate_request: InvestigateRequest - issue_data: Optional[Dict] - resource_instructions: Optional[ResourceInstructions] - expected_sections: Optional[Dict[str, Union[List[str], bool]]] = None - request: Any = None - - def check_and_skip_test( test_case: HolmesTestCase, request=None, shared_test_infrastructure=None ) -> None: @@ -242,9 +233,6 @@ def __init__(self, test_cases_folder: Path) -> None: super().__init__() self._test_cases_folder = test_cases_folder - def load_investigate_test_cases(self) -> List[InvestigateTestCase]: - return cast(List[InvestigateTestCase], self.load_test_cases()) - def load_ask_holmes_test_cases(self) -> List[AskHolmesTestCase]: return cast(List[AskHolmesTestCase], self.load_test_cases()) @@ -372,18 +360,6 @@ def load_test_cases(self) -> List[HolmesTestCase]: config_dict ) - elif self._test_cases_folder.name == "test_investigate": - config_dict["investigate_request"] = load_investigate_request( - test_case_folder - ) - config_dict["issue_data"] = load_issue_data(test_case_folder) - config_dict["resource_instructions"] = load_resource_instructions( - test_case_folder - ) - config_dict["request"] = TypeAdapter(InvestigateRequest) - test_case = TypeAdapter(InvestigateTestCase).validate_python( - config_dict - ) elif self._test_cases_folder.name == "compaction": # Compaction tests only need conversation_history and expected_output config_dict["conversation_history"] = load_conversation_history( @@ -486,19 +462,6 @@ def load_resource_instructions( return None -def load_investigate_request(test_case_folder: Path) -> InvestigateRequest: - investigate_request_path = test_case_folder.joinpath( - Path("investigate_request.json") - ) - if investigate_request_path.exists(): - return TypeAdapter(InvestigateRequest).validate_json( - read_file(Path(investigate_request_path)) - ) - raise Exception( - f"Investigate test case declared in folder {str(test_case_folder)} should have an investigate_request.json file but none is present" - ) - - def _parse_conversation_history_md_files( conversation_history_dir, ) -> None | List[Dict[str, str]]: diff --git a/tests/test_ai_safety_prompt.py b/tests/test_ai_safety_prompt.py index c1f669a5e9..cc490b3b03 100644 --- a/tests/test_ai_safety_prompt.py +++ b/tests/test_ai_safety_prompt.py @@ -12,8 +12,6 @@ class TestAISafetyPromptInclusion: "template_path", [ "builtin://generic_ask.jinja2", - "builtin://generic_ask_for_issue_conversation.jinja2", - "builtin://generic_investigation.jinja2", ], ) def test_ai_safety_prompt_included(self, template_path): @@ -22,7 +20,6 @@ def test_ai_safety_prompt_included(self, template_path): context = { "toolsets": [], "cluster_name": "test-cluster", - "issue": {"source_type": "test"}, # for investigation template "investigation": "test investigation", # for issue conversation template "tools_called_for_investigation": [], # for issue conversation template "sections": {}, # for investigation template output format diff --git a/tests/test_investigate_structured_output.py b/tests/test_investigate_structured_output.py deleted file mode 100644 index c8675f6ca9..0000000000 --- a/tests/test_investigate_structured_output.py +++ /dev/null @@ -1,258 +0,0 @@ -import json -from typing import Any, Optional - -import pytest - -from holmes.core.investigation_structured_output import ( - DEFAULT_SECTIONS, - get_output_format_for_investigation, - is_response_an_incorrect_tool_call, - parse_json_sections, - process_response_into_sections, -) -from holmes.plugins.prompts import load_and_render_prompt - - -def test_prompt_sections_formatting_structured_output(): - issue = {"source_type": "prometheus"} - prompt = load_and_render_prompt( - "builtin://generic_investigation.jinja2", - {"issue": issue, "sections": DEFAULT_SECTIONS}, - ) - - print(prompt) - assert len(DEFAULT_SECTIONS) > 0 - for title, description in DEFAULT_SECTIONS.items(): - assert ( - f"# {title}" in prompt - ), f'Expected section "{title}" was not found in formatted prompt' - assert ( - f"{description}" in prompt - ), f'Expected description for "{title}" was not found in formatted prompt' - - -def test_prompt_sections_formatting_unstructured_output(): - issue = {"source_type": "prometheus"} - prompt = load_and_render_prompt( - "builtin://generic_investigation.jinja2", - {"issue": issue, "sections": DEFAULT_SECTIONS, "structured_output": True}, - ) - - print(prompt) - assert len(DEFAULT_SECTIONS) > 0 - for title, description in DEFAULT_SECTIONS.items(): - flattened_description = description.replace("\n", "\\n") - assert ( - f'"{title}": "{flattened_description}"' in prompt - ), f'Expected section "{title}" not found in formatted prompt' - - -def test_get_output_format_for_investigation(): - output_format = get_output_format_for_investigation( - {"Title1": "Description1", "Title2": "Description2"} - ) - - assert output_format - assert output_format["json_schema"] - assert output_format["json_schema"]["schema"] - assert output_format["json_schema"]["schema"]["properties"] - - assert output_format["json_schema"]["schema"]["properties"] == { - "Title1": {"type": ["string", "null"], "description": "Description1"}, - "Title2": {"type": ["string", "null"], "description": "Description2"}, - } - - assert output_format["json_schema"]["schema"]["required"] == ["Title1", "Title2"] - - -@pytest.mark.parametrize( - "input_response,expected_text,expected_sections", - [ - ( - {"section1": "test1", "section2": "test2"}, - "\n# section1\ntest1\n\n# section2\ntest2\n", - {"section1": "test1", "section2": "test2"}, - ), - ( - '{"section1": "test1", "section2": "test2"}', - "\n# section1\ntest1\n\n# section2\ntest2\n", - {"section1": "test1", "section2": "test2"}, - ), - ( - '```json\n{"section1": "test1", "section2": "test2"}\n```', - "\n# section1\ntest1\n\n# section2\ntest2\n", - {"section1": "test1", "section2": "test2"}, - ), - ( - '"{\\n \\"Next Steps\\": [\\n \\"1. Verify Prometheus configuration for etcd target\\",\\n \\"2. Check if the etcd metrics endpoint is accessible\\"],\\n \\"Conclusions and Possible Root causes\\": [\\n \\"1. *Missing ServiceMonitor*: No ServiceMonitor was found for the kube-etcd service, which prevents Prometheus from discovering and scraping metrics\\",\\n \\"2. *Misconfigured Prometheus Scrape Configuration*: The monitoring stack might not be correctly configured to scrape etcd metrics\\"\\n ]\\n}"', - "\n# Next Steps\n1. Verify Prometheus configuration for etcd target\n\n2. Check if the etcd metrics endpoint is accessible\n\n# Conclusions and Possible Root causes\n1. *Missing ServiceMonitor*: No ServiceMonitor was found for the kube-etcd service, which prevents Prometheus from discovering and scraping metrics\n\n2. *Misconfigured Prometheus Scrape Configuration*: The monitoring stack might not be correctly configured to scrape etcd metrics\n", - { - "Next Steps": "1. Verify Prometheus configuration for etcd target\n\n2. Check if the etcd metrics endpoint is accessible", - "Conclusions and Possible Root causes": "1. *Missing ServiceMonitor*: No ServiceMonitor was found for the kube-etcd service, which prevents Prometheus from discovering and scraping metrics\n\n2. *Misconfigured Prometheus Scrape Configuration*: The monitoring stack might not be correctly configured to scrape etcd metrics", - }, - ), - (123, "123", None), - (None, "None", None), - ("plain text", "plain text", None), - ('{"invalid": json}', '{"invalid": json}', None), - ([], "[]", None), - ({}, "{}", None), - ( - 'text here long\n\n```json\n{\n "section 1": "section 1 text",\n "section 2": "section 2 text"\n}\n```\n\nanything else here', - "\n# section 1\nsection 1 text\n\n# section 2\nsection 2 text\n", - {"section 1": "section 1 text", "section 2": "section 2 text"}, - ), - ], -) -def test_parse_json_sections( - input_response: Any, expected_text: str, expected_sections: Optional[dict] -): - (text, sections) = parse_json_sections(input_response) - print(f"* ACTUAL\n{text}") - print(f"* EXPECTED\n{expected_text}") - assert text == expected_text - assert sections == expected_sections - - -@pytest.mark.parametrize( - "invalid_json", ['{"key": value}', '{key: "value"}', "not json at all"] -) -def test_parse_json_sections_invalid_json(invalid_json): - result = parse_json_sections(invalid_json) - assert result == (invalid_json, None) - - -@pytest.mark.parametrize( - "response, expected_output", - [ - ( - # azure AI over litellm - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": '{"timezone":"America/New_York"}', - "role": "assistant", - "tool_calls": None, - "function_call": None, - }, - }, - True, - ), - ( - # Bedrock over litellm - { - "finish_reason": "tool_calls", - "index": 0, - "message": { - "content": '{"kind": "pod", "name": "oomkill-deployment-696dbdbf67-d47z6", "namespace": "default"}', - "role": "assistant", - "tool_calls": None, - "function_call": None, - "refusal": None, - }, - }, - True, - ), - ( - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": '{"Alert Explanation":"foobar"}', - "role": "assistant", - "tool_calls": None, - "function_call": None, - }, - }, - False, - ), - ( - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": {"Alert Explanation": "foobar"}, - "role": "assistant", - "tool_calls": None, - "function_call": None, - }, - }, - False, - ), - ( - { - "finish_reason": "tool_call", - "index": 0, - "message": { - "content": {"Alert Explanation": "foobar"}, - "role": "assistant", - "tool_calls": None, - "function_call": None, - }, - }, - False, - ), - ], -) -def test_is_response_an_incorrect_tool_call(response, expected_output): - assert ( - is_response_an_incorrect_tool_call(DEFAULT_SECTIONS, response) - == expected_output - ), f"Expected the following content to be incorrect={expected_output}. {response}" - - -TEST_SECTIONS = { - "Next Steps": "To resolve the issue, correct the command in the init container. Replace 'wge' with 'wget' in the pod's configuration. Apply the changes using:\n```bash\nkubectl apply -f .yaml\n```", - "Key Findings": '- The pod `logging-agent` in the `default` namespace is in a `CrashLoopBackOff` state.\n- The init container `downloader` is failing due to a command error: `exec: "wge": executable file not found in $PATH`.\n- The pod has restarted 860 times.', - "Related logs": 'Logs from pod `logging-agent`:\n```\nError: failed to create containerd task: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: "wge": executable file not found in $PATH: unknown\n```', - "App or Infra?": "This is an application-level issue because the error is due to a misconfigured command in the init container.", - "Alert Explanation": "The pod `logging-agent` has been in a non-ready state for more than 15 minutes due to a `CrashLoopBackOff` caused by a command error in the init container.", - "Conclusions and Possible Root causes": "The possible root cause is a typo in the init container's command, where 'wge' should be 'wget'. This prevents the container from starting, leading to the `CrashLoopBackOff` state.", -} - - -def test_parse_markdown_into_sections_hash_no_leading_line_feed(): - markdown = "" - for title, content in TEST_SECTIONS.items(): - if not markdown: - markdown = f"# {title}\n{content}\n" - else: - markdown = markdown + f"\n# {title}\n{content}\n" - (text, sections) = process_response_into_sections(markdown) - print(markdown) - print(json.dumps(sections, indent=2)) - assert text == markdown - assert sections, "None sections returned" - for title, content in TEST_SECTIONS.items(): - assert title in sections - assert sections.get(title) == content - - -def test_parse_markdown_into_sections_hash_leading_line_feed(): - markdown = "" - for title, content in TEST_SECTIONS.items(): - markdown = markdown + f"\n# {title}\n{content}\n" - (text, sections) = process_response_into_sections(markdown) - print(markdown) - print(json.dumps(sections, indent=2)) - assert text == markdown - assert sections, "None sections returned" - for title, content in TEST_SECTIONS.items(): - assert title in sections - assert sections.get(title) == content - - -def test_parse_markdown_into_sections_equal(): - markdown = "" - for title, content in TEST_SECTIONS.items(): - markdown = markdown + f"\n{title}\n==========\n{content}\n" - - (text, sections) = process_response_into_sections(markdown) - print(markdown) - print(json.dumps(sections, indent=2)) - assert text == markdown - assert sections, "None sections returned" - for title, content in TEST_SECTIONS.items(): - assert title in sections - assert sections.get(title) == content diff --git a/tests/test_issue_investigator.py b/tests/test_issue_investigator.py deleted file mode 100644 index 628a622ace..0000000000 --- a/tests/test_issue_investigator.py +++ /dev/null @@ -1,89 +0,0 @@ -from rich.console import Console - -from holmes.config import Config -from holmes.core.issue import Issue -from holmes.core.models import InvestigateRequest - - -def _test_investigate_issue_using_fetch_webpage(): - investigate_request = InvestigateRequest( - source="prometheus", - title="starting container process caused", - description='starting container process caused "exec: "mycommand": executable file not found in $PATH"', - subject=dict(), - context=dict(), - source_instance_id="ApiRequest", - include_tool_calls=True, - include_tool_call_results=True, - prompt_template="builtin://generic_investigation.jinja2", - ) - raw_data = investigate_request.model_dump() - - runbook_url = "https://containersolutions.github.io/runbooks/posts/kubernetes/create-container-error/" - console = Console() - config = Config.load_from_env() - ai = config.create_issue_investigator(console) - - issue = Issue( - id="", - name=investigate_request.title, - source_type=investigate_request.source, - source_instance_id=investigate_request.source_instance_id, - raw=raw_data, - ) - investigation = ai.investigate( - issue=issue, - prompt=investigate_request.prompt_template, - console=console, - ) - - webpage_tool_calls = list( - filter( - lambda tool_call: tool_call.tool_name == "fetch_webpage", - investigation.tool_calls, - ) - ) - - assert len(webpage_tool_calls) == 1 - assert runbook_url in webpage_tool_calls[0].description - - -def _test_investigate_issue_without_fetch_webpage(): - investigate_request = InvestigateRequest( - source="prometheus", - title="starting container process caused", - description='starting container process caused "exec: "mycommand": executable file not found in $PATH"', - subject=dict(), - context=dict(), - source_instance_id="ApiRequest", - include_tool_calls=True, - include_tool_call_results=True, - prompt_template="builtin://generic_investigation.jinja2", - ) - raw_data = investigate_request.model_dump() - console = Console() - config = Config.load_from_env() - ai = config.create_issue_investigator(console) - - issue = Issue( - id="", - name=investigate_request.title, - source_type=investigate_request.source, - source_instance_id=investigate_request.source_instance_id, - raw=raw_data, - ) - - investigation = ai.investigate( - issue=issue, - prompt=investigate_request.prompt_template, - console=console, - ) - - webpage_tool_calls = list( - filter( - lambda tool_call: tool_call.tool_name == "fetch_webpage", - investigation.tool_calls, - ) - ) - - assert len(webpage_tool_calls) == 0 diff --git a/tests/test_server_endpoints.py b/tests/test_server_endpoints.py index e34a4093f8..30a4fba5ee 100644 --- a/tests/test_server_endpoints.py +++ b/tests/test_server_endpoints.py @@ -283,65 +283,6 @@ def test_api_chat_with_images_missing_url_key( assert "Image dict must contain a 'url' key" in data["detail"] -@patch("holmes.config.Config.create_toolcalling_llm") -@patch("holmes.core.supabase_dal.SupabaseDal.get_global_instructions_for_account") -def test_api_issue_chat_all_fields( - mock_get_global_instructions, - mock_create_toolcalling_llm, - client, -): - mock_ai = MagicMock() - mock_ai.messages_call.return_value = MagicMock( - result="This is a mock analysis for issue chat.", - tool_calls=[ - { - "tool_call_id": "1", - "tool_name": "issue_resolver", - "description": "Resolves issues", - "result": {"status": "success", "data": "Issue resolved"}, - } - ], - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "I have an issue with my deployment."}, - ], - metadata={}, - ) - mock_create_toolcalling_llm.return_value = mock_ai - - mock_get_global_instructions.return_value = [] - - payload = { - "ask": "What can you do?", - "investigation_result": {"result": "Mock investigation result", "tools": []}, - "issue_type": "deployment", - "conversation_history": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "I have an issue with my deployment."}, - ], - } - response = client.post("/api/issue_chat", json=payload) - assert response.status_code == 200 - data = response.json() - - assert "analysis" in data - assert "conversation_history" in data - assert "tool_calls" in data - - assert isinstance(data["analysis"], str) - assert isinstance(data["conversation_history"], list) - assert isinstance(data["tool_calls"], list) - - assert any(msg.get("role") == "user" for msg in data["conversation_history"]) - - if data["tool_calls"]: - tool_call = data["tool_calls"][0] - assert "tool_call_id" in tool_call - assert "tool_name" in tool_call - assert "description" in tool_call - assert "result" in tool_call - - class TestExtractPassthroughHeaders: def test_extract_normal_headers(self): scope = {