diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 2b369fc9ff..e46ca15179 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -192,7 +192,7 @@ jobs: sudo apt-get install -y binutils pyinstaller holmes_cli.py \ --name holmes \ - --add-data 'holmes/plugins/runbooks/*:holmes/plugins/runbooks' \ + --add-data 'holmes/plugins/skills/*:holmes/plugins/skills' \ --add-data 'holmes/plugins/prompts/*:holmes/plugins/prompts' \ --add-data 'holmes/plugins/toolsets/*:holmes/plugins/toolsets' \ --add-data 'holmes/plugins/toolsets/coralogix*:holmes/plugins/toolsets/coralogix' \ diff --git a/.github/workflows/build-binaries-and-brew.yaml b/.github/workflows/build-binaries-and-brew.yaml index 71fbda0865..a31e45c5a3 100644 --- a/.github/workflows/build-binaries-and-brew.yaml +++ b/.github/workflows/build-binaries-and-brew.yaml @@ -83,7 +83,7 @@ jobs: run: | pyinstaller holmes_cli.py \ --name holmes \ - --add-data 'holmes/plugins/runbooks/*:holmes/plugins/runbooks' \ + --add-data 'holmes/plugins/skills/*:holmes/plugins/skills' \ --add-data 'holmes/plugins/prompts/*:holmes/plugins/prompts' \ --add-data 'holmes/plugins/toolsets/*:holmes/plugins/toolsets' \ --add-data 'holmes/plugins/toolsets/coralogix*:holmes/plugins/toolsets/coralogix' \ diff --git a/.gitignore b/.gitignore index f4eb696e13..7ede1044b9 100644 --- a/.gitignore +++ b/.gitignore @@ -170,5 +170,7 @@ evals_report.md config_toolsets.yaml **/.claude/settings.local.json +.claude/settings.json +.claude/skills/dev-skills/ .claude-trace/ eval_results.json diff --git a/docs/reference/.nav.yml b/docs/reference/.nav.yml index 3ea1dd440d..74759becc9 100644 --- a/docs/reference/.nav.yml +++ b/docs/reference/.nav.yml @@ -6,6 +6,6 @@ nav: - Microsoft Teams Bot Permissions: teams-permissions.md - HTTP API: http-api.md - OpenTelemetry: opentelemetry.md - - Runbooks: runbooks.md + - Skills: skills.md - Slash Commands: slash-commands.md - Troubleshooting: troubleshooting.md diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index f8eff0a1af..834986c60a 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -121,7 +121,7 @@ The `ENABLED_PROMPTS` env var accepts a comma-separated list of section keys (e. | `system_prompt_additions` | System | Custom additions from configuration | | `files` | User | Attached file contents | | `todowrite_reminder` | User | Reminder to use TodoWrite for task tracking | -| `time_runbooks` | User | Runbook content and custom instructions | +| `time_skills` | User | Skill content and custom instructions | #### Structured Output with `response_format` diff --git a/docs/reference/runbooks.md b/docs/reference/runbooks.md deleted file mode 100644 index b7a112180c..0000000000 --- a/docs/reference/runbooks.md +++ /dev/null @@ -1,176 +0,0 @@ -# Runbooks - -Runbooks are step-by-step troubleshooting guides that Holmes follows when investigating issues. When a user asks a question or an alert fires, Holmes automatically matches relevant runbooks from its catalog and fetches them using the `fetch_runbook` tool. It then follows the runbook instructions step-by-step, calling tools to gather data and reporting results for each step. - -Runbooks work with all Holmes interfaces — the CLI (`ask` and `investigate` commands), the HTTP server, and the Python SDK. - -## How It Works - -1. Holmes receives a question or alert -2. Holmes compares the issue against runbook descriptions in the catalog -3. If a runbook matches, Holmes fetches it with the `fetch_runbook` tool -4. Holmes follows the runbook steps, calling tools to gather data at each step -5. Holmes reports findings with a checklist showing completed and skipped steps - -## Built-in Runbooks - -Holmes ships with a built-in runbook catalog at `holmes/plugins/runbooks/`. These are available automatically — no configuration needed. - -## Custom Runbook Catalogs - -You can add your own runbooks by creating a catalog and pointing Holmes to it. - -### Creating a Catalog - -A catalog consists of a `catalog.json` index file and one or more markdown runbook files: - -``` -my-runbooks/ -├── catalog.json -├── database/ -│ ├── postgres_troubleshooting.md -│ └── redis_connection_issues.md -└── networking/ - └── dns_resolution.md -``` - -**`catalog.json`:** - -```json -{ - "catalog": [ - { - "id": "postgres-troubleshooting.md", - "update_date": "2026-01-15", - "description": "Troubleshooting PostgreSQL connection and performance issues", - "link": "database/postgres_troubleshooting.md" - }, - { - "id": "redis-connection-issues.md", - "update_date": "2026-01-15", - "description": "Diagnosing Redis connection failures and timeout issues", - "link": "database/redis_connection_issues.md" - }, - { - "id": "dns-resolution.md", - "update_date": "2026-01-15", - "description": "Troubleshooting DNS resolution failures in Kubernetes clusters", - "link": "networking/dns_resolution.md" - } - ] -} -``` - -Each entry has: - -- **`id`**: Unique identifier (typically the filename) -- **`update_date`**: Last updated date (`YYYY-MM-DD`) -- **`description`**: Used by the LLM to match the runbook to user questions — make this descriptive -- **`link`**: Relative path from `catalog.json` to the markdown file - -### Writing a Runbook - -Runbooks are markdown files with a structured format that guides Holmes through troubleshooting steps: - -```markdown -# PostgreSQL Connection Troubleshooting - -## Goal -Diagnose and resolve PostgreSQL database connection issues. -Follow the workflow steps sequentially. - -## Workflow - -1. **Check database pod status** - * Verify pods are running and not restarting - * Check resource usage (CPU, memory) - -2. **Test database connectivity** - * Verify the connection string and credentials - * Check network policies and service endpoints - -3. **Examine database logs** - * Look for authentication failures - * Check for max connection limit errors - -4. **Review client configuration** - * Validate connection pool settings - * Check timeout configurations - -## Synthesize Findings -Correlate the outputs from each step to identify the root cause. - -## Recommended Remediation Steps -* **Authentication failures**: Verify credentials in the application Secret -* **Connection limit**: Increase `max_connections` or add connection pooling -* **Network issues**: Check NetworkPolicies and DNS resolution -``` - -The key sections are: - -- **Goal**: What the runbook addresses -- **Workflow**: Sequential diagnostic steps Holmes will execute using its tools -- **Synthesize Findings**: How to interpret combined results -- **Recommended Remediation Steps**: Solutions based on findings - -### Configuring Custom Catalogs - -=== "Config File" - - Add catalog paths to `~/.holmes/config.yaml`: - - ```yaml - custom_runbook_catalogs: - - /path/to/my-runbooks/catalog.json - - /path/to/team-runbooks/catalog.json - ``` - -=== "Helm Chart" - - Mount your catalog files and reference them in values: - - ```yaml - customRunbookCatalogs: - - /etc/holmes/runbooks/catalog.json - ``` - -=== "Python SDK" - - ```python - from pathlib import Path - - from holmes.config import Config - - config = Config.load_from_file( - config_file=Path("~/.holmes/config.yaml").expanduser(), - ) - # custom_runbook_catalogs is read from the config file - catalog = config.get_runbook_catalog() - ``` - -Multiple catalogs are merged — entries from all catalogs are combined with the built-in catalog. - -## Common Use Cases - -``` -Why is my PostgreSQL database connection timing out? -``` - -``` -Investigate the OOMKilled alert on the payments service -``` - -``` -Help me troubleshoot DNS resolution failures in the staging cluster -``` - -## Troubleshooting - -```bash -# Verify your catalog.json is valid JSON -python3 -c "import json; json.load(open('/path/to/catalog.json'))" - -# Check Holmes logs for runbook loading errors -# Look for "Error decoding JSON" or "Custom catalog file not found" -holmes ask "test question" -v -``` diff --git a/docs/reference/skills.md b/docs/reference/skills.md new file mode 100644 index 0000000000..25b2d0ebaf --- /dev/null +++ b/docs/reference/skills.md @@ -0,0 +1,192 @@ +# Skills + +!!! warning "Breaking Change — Holmes 0.25.0+" + + Skills replace the previous runbook system. If you are upgrading from Holmes 0.24.x or older, you must migrate your runbooks to the new SKILL.md format. See [Migrating from Runbooks](#migrating-from-runbooks) below. + +Skills are step-by-step troubleshooting guides that Holmes follows when investigating issues. When a user asks a question or an alert fires, Holmes automatically matches relevant skills from its catalog and fetches them using the `fetch_skill` tool. It then follows the skill instructions step-by-step, calling tools to gather data and reporting results for each step. + +Skills work with all Holmes interfaces — the CLI (`ask` and `investigate` commands), the HTTP server, and the Python SDK. + +## How It Works + +1. Holmes receives a question or alert +2. Holmes compares the issue against skill descriptions in the catalog +3. If a skill matches, Holmes fetches it with the `fetch_skill` tool +4. Holmes follows the skill steps, calling tools to gather data at each step +5. Holmes reports findings with a checklist showing completed and skipped steps + +## Built-in Skills + +Holmes ships with built-in skills at `holmes/plugins/skills/builtin/`. These are available automatically — no configuration needed. + +## Custom Skills + +You can add your own skills by creating SKILL.md files and pointing Holmes to them. + +### Skill Format + +Each skill is a directory containing a `SKILL.md` file with YAML frontmatter and a markdown body: + +``` +my-skills/ +├── dns-troubleshooting/ +│ └── SKILL.md +├── postgres-performance/ +│ └── SKILL.md +└── redis-connection-issues/ + └── SKILL.md +``` + +**`dns-troubleshooting/SKILL.md`:** + +```markdown +--- +name: dns-troubleshooting +description: Troubleshooting DNS resolution failures in Kubernetes clusters +--- + +# DNS Troubleshooting + +## Goal +Diagnose and resolve DNS resolution issues in the cluster. +Follow the workflow steps sequentially. + +## Workflow + +1. **Check CoreDNS pods** + * Verify pods in kube-system with label k8s-app=kube-dns are running + * Check for restarts or resource pressure + +2. **Test DNS resolution** + * Resolve kubernetes.default.svc.cluster.local from an affected pod + * Resolve an external domain like google.com + +3. **Check for NetworkPolicies blocking DNS** + * List NetworkPolicies in the affected namespace + * Verify UDP port 53 egress to kube-system is allowed + +## Synthesize Findings +Correlate the outputs from each step to identify the root cause. + +## Recommended Remediation Steps +* **CoreDNS down**: Check resource limits and node capacity +* **NetworkPolicy blocking**: Add an egress rule allowing DNS traffic +* **ConfigMap wrong**: Fix the Corefile and restart CoreDNS +``` + +### Frontmatter Fields + +- **`name`** (optional): Lowercase with hyphens. Defaults to the parent directory name. +- **`description`** (required): Used by the LLM to match the skill to user questions — make this descriptive. + +### Writing a Skill + +The key sections in a skill's markdown body are: + +- **Goal**: What the skill addresses +- **Workflow**: Sequential diagnostic steps Holmes will execute using its tools +- **Synthesize Findings**: How to interpret combined results +- **Recommended Remediation Steps**: Solutions based on findings + +### Configuring Custom Skill Paths + +=== "Config File" + + Add skill directory paths to `~/.holmes/config.yaml`: + + ```yaml + custom_skill_paths: + - /path/to/my-skills/ + - /path/to/team-skills/ + ``` + +=== "Helm Chart" + + Mount your skill directories and reference them in values: + + ```yaml + custom_skill_paths: + - /etc/holmes/skills/ + ``` + +=== "Python SDK" + + ```python + from pathlib import Path + + from holmes.config import Config + + config = Config.load_from_file( + config_file=Path("~/.holmes/config.yaml").expanduser(), + ) + # custom_skill_paths is read from the config file + catalog = config.get_skill_catalog() + ``` + +Holmes scans each directory (up to 2 levels deep) for `SKILL.md` files. Multiple paths are merged — skills from all paths are combined with built-in skills. + +## Common Use Cases + +``` +Why is my PostgreSQL database connection timing out? +``` + +``` +Investigate the OOMKilled alert on the payments service +``` + +``` +Help me troubleshoot DNS resolution failures in the staging cluster +``` + +## Migrating from Runbooks + +If you are upgrading from Holmes 0.24.x or older, your existing runbooks need to be converted to the SKILL.md format. + +**For each runbook in your catalog:** + +1. Create a directory named after the runbook (lowercase, hyphens): + ``` + my-skills/postgres-troubleshooting/ + ``` + +2. Create a `SKILL.md` file inside it with the description from your old `catalog.json` entry as frontmatter, and the original markdown content as the body: + ```markdown + --- + name: postgres-troubleshooting + description: Troubleshooting PostgreSQL connection and performance issues + --- + + (paste your original .md runbook content here) + ``` + +3. Replace `custom_runbook_catalogs` in your config with `custom_skill_paths`: + ```yaml + # Old (no longer supported): + # custom_runbook_catalogs: + # - /path/to/catalog.json + + # New: + custom_skill_paths: + - /path/to/my-skills/ + ``` + +The `catalog.json` file is no longer needed — Holmes discovers skills automatically by scanning for `SKILL.md` files. + +## Troubleshooting + +```bash +# Check Holmes logs for skill loading errors +# Look for "Failed to parse" or "missing required 'description' field" +holmes ask "test question" -v + +# Verify your SKILL.md has valid YAML frontmatter +python3 -c " +import yaml +with open('my-skill/SKILL.md') as f: + content = f.read() + parts = content.split('---', 2) + print(yaml.safe_load(parts[1])) +" +``` diff --git a/holmes/config.py b/holmes/config.py index fc9f4db841..b7a13ac39b 100644 --- a/holmes/config.py +++ b/holmes/config.py @@ -24,9 +24,9 @@ from holmes.core.tools_utils.tool_executor import ToolExecutor from holmes.core.toolset_manager import ToolsetManager from holmes.core.transformers.llm_summarize import LLMSummarizeTransformer -from holmes.plugins.runbooks import ( - RunbookCatalog, - load_runbook_catalog, +from holmes.plugins.skills.skill_loader import ( + SkillCatalog, + load_skill_catalog, ) # Source plugin imports moved to their respective create methods to speed up startup @@ -96,7 +96,7 @@ class Config(RobustaBaseConfig): opsgenie_team_integration_key: Optional[SecretStr] = None opsgenie_query: Optional[str] = None - custom_runbook_catalogs: List[Union[str, FilePath]] = [] + custom_skill_paths: List[Union[str, FilePath]] = [] # custom_toolsets is passed from config file, and be used to override built-in toolsets, provides 'stable' customized toolset. # The status of custom toolsets can be cached. @@ -147,7 +147,7 @@ def toolset_manager(self) -> ToolsetManager: mcp_servers=self.mcp_servers, custom_toolsets=self.custom_toolsets, custom_toolsets_from_cli=self.custom_toolsets_from_cli, - custom_runbook_catalogs=self.custom_runbook_catalogs, + custom_skill_paths=self.custom_skill_paths, config_file_path=self._config_file_path, additional_toolsets=self.additional_toolsets, ) @@ -281,11 +281,10 @@ def __get_cluster_name() -> Optional[str]: return env_cluster_name return Config.get_robusta_global_config_value("cluster_name") - def get_runbook_catalog(self) -> Optional[RunbookCatalog]: - runbook_catalog = load_runbook_catalog( - dal=self.dal, custom_catalog_paths=self.custom_runbook_catalogs + def get_skill_catalog(self) -> Optional[SkillCatalog]: + return load_skill_catalog( + dal=self.dal, custom_skill_paths=self.custom_skill_paths, ) - return runbook_catalog # ── Unified factory methods ── diff --git a/holmes/core/conversations.py b/holmes/core/conversations.py index 66d8833a30..698163161e 100644 --- a/holmes/core/conversations.py +++ b/holmes/core/conversations.py @@ -6,7 +6,7 @@ build_prompts, ) from holmes.core.tool_calling_llm import ToolCallingLLM -from holmes.plugins.runbooks import RunbookCatalog +from holmes.plugins.skills.skill_loader import SkillCatalog from holmes.utils.global_instructions import ( Instructions, ) @@ -50,7 +50,7 @@ def build_chat_messages( config: Config, global_instructions: Optional[Instructions] = None, additional_system_prompt: Optional[str] = None, - runbooks: Optional[RunbookCatalog] = None, + skills: Optional[SkillCatalog] = None, images: Optional[List[Union[str, Dict[str, Any]]]] = None, prompt_component_overrides: Optional[Dict[PromptComponent, bool]] = None, ) -> List[dict]: @@ -68,7 +68,7 @@ def build_chat_messages( system_prompt, user_content = build_prompts( toolsets=ai.tool_executor.toolsets, user_prompt=ask, - runbooks=runbooks, + skills=skills, global_instructions=global_instructions, system_prompt_additions=additional_system_prompt, cluster_name=config.cluster_name, diff --git a/holmes/core/prompt.py b/holmes/core/prompt.py index e43252656a..5f7168a5c5 100644 --- a/holmes/core/prompt.py +++ b/holmes/core/prompt.py @@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Union from holmes.plugins.prompts import load_and_render_prompt -from holmes.plugins.runbooks import RunbookCatalog -from holmes.utils.global_instructions import Instructions, generate_runbooks_args +from holmes.plugins.skills.skill_loader import SkillCatalog +from holmes.utils.global_instructions import Instructions, generate_skills_args from holmes.version import get_version @@ -13,7 +13,7 @@ class PromptComponent(str, Enum): # User prompt components FILES = "files" TODOWRITE_REMINDER = "todowrite_reminder" - TIME_RUNBOOKS = "time_runbooks" + TIME_SKILLS = "time_skills" # System prompt components INTRO = "intro" ASK_USER = "ask_user" @@ -82,7 +82,7 @@ def is_prompt_allowed_by_env(component: PromptComponent) -> bool: Environment variable: ENABLED_PROMPTS - If not set: all prompts are ENABLED (production default) - If set to "none": all prompts are disabled - - Comma-separated names (e.g., "files,ai_safety,time_runbooks") + - Comma-separated names (e.g., "files,ai_safety,time_skills") """ enabled_prompts = os.environ.get("ENABLED_PROMPTS", "") @@ -149,10 +149,10 @@ def _has_content(value: Optional[str]) -> bool: return bool(value and isinstance(value, str) and value.strip()) -def _should_enable_runbooks(context: Dict[str, str]) -> bool: +def _should_enable_skills(context: Dict[str, str]) -> bool: return any( ( - _has_content(context.get("runbook_catalog")), + _has_content(context.get("skill_catalog")), _has_content(context.get("custom_instructions")), _has_content(context.get("global_instructions")), ) @@ -163,13 +163,13 @@ def generate_user_prompt( user_prompt: str, context: Dict[str, str], ) -> str: - runbooks_enabled = _should_enable_runbooks(context) + skills_enabled = _should_enable_skills(context) return load_and_render_prompt( "builtin://base_user_prompt.jinja2", context={ "user_prompt": user_prompt, - "runbooks_enabled": runbooks_enabled, + "skills_enabled": skills_enabled, **context, }, ) @@ -177,7 +177,7 @@ def generate_user_prompt( def build_system_prompt( toolsets: List[Any], - runbooks: Optional[RunbookCatalog], + skills: Optional[SkillCatalog], system_prompt_additions: Optional[str], cluster_name: Optional[str], ask_user_enabled: bool, @@ -205,8 +205,8 @@ def is_enabled(component: PromptComponent) -> bool: PromptComponent.GENERAL_INSTRUCTIONS ), "style_guide_enabled": is_enabled(PromptComponent.STYLE_GUIDE), - "runbooks_enabled": bool(runbooks and getattr(runbooks, "catalog", True)) - and is_enabled(PromptComponent.TIME_RUNBOOKS), + "skills_enabled": bool(skills and getattr(skills, "skills", True)) + and is_enabled(PromptComponent.TIME_SKILLS), "cluster_name": cluster_name if is_enabled(PromptComponent.CLUSTER_NAME) else None, @@ -225,7 +225,7 @@ def is_enabled(component: PromptComponent) -> bool: def build_user_prompt( user_prompt: str, - runbooks: Optional[RunbookCatalog], + skills: Optional[SkillCatalog], global_instructions: Optional[Instructions], file_paths: Optional[List[Path]], include_todowrite_reminder: bool, @@ -247,12 +247,12 @@ def is_enabled(component: PromptComponent) -> bool: if include_todowrite_reminder and is_enabled(PromptComponent.TODOWRITE_REMINDER): user_prompt += get_tasks_management_system_reminder() - if is_enabled(PromptComponent.TIME_RUNBOOKS): - runbooks_ctx = generate_runbooks_args( - runbook_catalog=runbooks, + if is_enabled(PromptComponent.TIME_SKILLS): + skills_ctx = generate_skills_args( + skill_catalog=skills, global_instructions=global_instructions, ) - user_prompt = generate_user_prompt(user_prompt, runbooks_ctx) + user_prompt = generate_user_prompt(user_prompt, skills_ctx) if images: return build_vision_content(user_prompt, images) @@ -262,7 +262,7 @@ def is_enabled(component: PromptComponent) -> bool: def build_prompts( toolsets: List[Any], user_prompt: str, - runbooks: Optional[RunbookCatalog], + skills: Optional[SkillCatalog], global_instructions: Optional[Instructions], system_prompt_additions: Optional[str], cluster_name: Optional[str], @@ -278,7 +278,7 @@ def build_prompts( system_prompt = build_system_prompt( toolsets=toolsets, - runbooks=runbooks, + skills=skills, system_prompt_additions=system_prompt_additions, cluster_name=cluster_name, ask_user_enabled=ask_user_enabled, @@ -286,7 +286,7 @@ def build_prompts( ) user_content = build_user_prompt( user_prompt=user_prompt, - runbooks=runbooks, + skills=skills, global_instructions=global_instructions, file_paths=file_paths, include_todowrite_reminder=include_todowrite_reminder, @@ -300,7 +300,7 @@ def build_initial_ask_messages( initial_user_prompt: str, file_paths: Optional[List[Path]], tool_executor: Any, # ToolExecutor type - runbooks: Optional[RunbookCatalog] = None, + skills: Optional[SkillCatalog] = None, system_prompt_additions: Optional[str] = None, global_instructions: Optional[Instructions] = None, cluster_name: Optional[str] = None, @@ -310,7 +310,7 @@ def build_initial_ask_messages( system_prompt, user_prompt = build_prompts( toolsets=tool_executor.toolsets, user_prompt=initial_user_prompt, - runbooks=runbooks, + skills=skills, global_instructions=global_instructions, system_prompt_additions=system_prompt_additions, cluster_name=cluster_name, diff --git a/holmes/core/supabase_dal.py b/holmes/core/supabase_dal.py index ac980b6a1c..c2e109fda7 100644 --- a/holmes/core/supabase_dal.py +++ b/holmes/core/supabase_dal.py @@ -37,7 +37,7 @@ from holmes.core.truncation.dal_truncation_utils import ( truncate_evidences_entities_if_necessary, ) -from holmes.plugins.runbooks import RobustaRunbookInstruction +from holmes.plugins.skills import RobustaSkillInstruction from holmes.utils.definitions import RobustaConfig from holmes.utils.env import get_env_replacement from holmes.utils.global_instructions import Instructions @@ -594,7 +594,7 @@ def get_issue_data(self, issue_id: Optional[str]) -> Optional[Dict]: return issue_data - def get_runbook_catalog(self) -> Optional[List[RobustaRunbookInstruction]]: + def get_skill_catalog(self) -> Optional[List[RobustaSkillInstruction]]: if not self.enabled: return None @@ -604,6 +604,7 @@ def get_runbook_catalog(self) -> Optional[List[RobustaRunbookInstruction]]: .select("*") .eq("account_id", self.account_id) .eq("subject_type", "RunbookCatalog") + .eq("enabled", True) .execute() ) if not res.data: @@ -614,20 +615,24 @@ def get_runbook_catalog(self) -> Optional[List[RobustaRunbookInstruction]]: id = row.get("runbook_id") symptom = row.get("symptoms") title = row.get("subject_name") + clusters = row.get("clusters") if not symptom: - logging.warning("Skipping runbook with empty symptom: %s", id) + logging.warning("Skipping skill with empty symptom: %s", id) + continue + # Filter by cluster: null means all clusters, otherwise check membership + if clusters is not None and self.cluster not in clusters: continue instructions.append( - RobustaRunbookInstruction(id=id, symptom=symptom, title=title) + RobustaSkillInstruction(id=id, symptom=symptom, title=title) ) return instructions except Exception: - logging.exception("Failed to fetch RunbookCatalog", exc_info=True) + logging.exception("Failed to fetch skill catalog", exc_info=True) return None - def get_runbook_content( - self, runbook_id: str - ) -> Optional[RobustaRunbookInstruction]: + def get_skill_content( + self, skill_id: str + ) -> Optional[RobustaSkillInstruction]: if not self.enabled: return None @@ -636,7 +641,7 @@ def get_runbook_content( .select("*") .eq("account_id", self.account_id) .eq("subject_type", "RunbookCatalog") - .eq("runbook_id", runbook_id) + .eq("runbook_id", skill_id) .execute() ) if not res.data or len(res.data) != 1: @@ -659,11 +664,11 @@ def get_runbook_content( else: # in case the format is unexpected, convert to string logging.error( - f"Unexpected runbook instruction format for runbook_id={runbook_id}: {raw_instruction}" + f"Unexpected skill instruction format for skill_id={skill_id}: {raw_instruction}" ) instruction = str(raw_instruction) - return RobustaRunbookInstruction( + return RobustaSkillInstruction( id=id, symptom=symptom, instruction=instruction, title=title ) diff --git a/holmes/core/tool_calling_llm.py b/holmes/core/tool_calling_llm.py index 495dcbd254..ae7a98399b 100644 --- a/holmes/core/tool_calling_llm.py +++ b/holmes/core/tool_calling_llm.py @@ -203,7 +203,7 @@ def __init__( self.llm = llm self.tool_results_dir = tool_results_dir - self._runbook_in_use: bool = False + self._skill_in_use: bool = False def with_executor(self, tool_executor: ToolExecutor) -> "ToolCallingLLM": """Return a shallow copy with a different ToolExecutor. @@ -219,15 +219,15 @@ def with_executor(self, tool_executor: ToolExecutor) -> "ToolCallingLLM": tracer=self.tracer, ) # Preserve transient state so resumed turns keep access to - # runbook-unlocked (restricted) tools. - clone._runbook_in_use = self._runbook_in_use + # skill-unlocked (restricted) tools. + clone._skill_in_use = self._skill_in_use return clone def reset_interaction_state(self) -> None: """ - For interactive loop, reset runbooks in use + For interactive loop, reset skills in use """ - self._runbook_in_use = False + self._skill_in_use = False def _supports_vision(self) -> bool: """Check if vision/multimodal input is enabled. @@ -467,7 +467,7 @@ def _process_frontend_tool_results( def _should_include_restricted_tools(self) -> bool: """Check if restricted tools should be included in the tools list.""" - return self._runbook_in_use + return self._skill_in_use def _get_tools(self) -> list: """Get tools list, filtering restricted tools based on authorization. @@ -699,14 +699,14 @@ def _directly_invoke_tool_call( if toolset_name: self.tool_executor.oauth_connector.store_user_tools(effective_user, toolset_name, tool_response.oauth_tools) - # Track runbook usage - if fetch_runbook is called successfully, + # Track skill usage - if fetch_skill is called successfully, # restricted tools become available for the rest of the current request if ( - tool_name == "fetch_runbook" + tool_name == "fetch_skill" and tool_response.status == StructuredToolResultStatus.SUCCESS ): - self._runbook_in_use = True - logging.debug("Runbook fetched - restricted tools now available") + self._skill_in_use = True + logging.debug("Skill fetched - restricted tools now available") except Exception as e: logging.error( @@ -1334,7 +1334,7 @@ def call_stream( # Update the tool number offset for the next iteration tool_number_offset += len(tools_to_call) - # Re-fetch tools if the tool list changed (runbook activation, OAuth tool discovery, etc.) + # Re-fetch tools if the tool list changed (skill activation, OAuth tool discovery, etc.) if tools is not None: new_tools = self._get_tools() old_names = {t["function"]["name"] for t in tools} diff --git a/holmes/core/tools.py b/holmes/core/tools.py index b2b5bb8134..e7c80d2804 100644 --- a/holmes/core/tools.py +++ b/holmes/core/tools.py @@ -295,7 +295,7 @@ class Tool(ABC, BaseModel): transformers: Optional[List[Transformer]] = None restricted: bool = Field( default=False, - description="If True, tool requires runbook authorization or restricted_tools=true to use", + description="If True, tool requires skill authorization or restricted_tools=true to use", ) # Private attribute to store initialized transformer instances for performance @@ -760,7 +760,7 @@ class Toolset(BaseModel): restricted_tools: List[str] = Field( default_factory=list, - description="Tool names/patterns that require runbook authorization (use '*' for all tools)", + description="Tool names/patterns that require skill authorization (use '*' for all tools)", ) approval_required_tools: List[str] = Field( default_factory=list, diff --git a/holmes/core/tools_utils/tool_executor.py b/holmes/core/tools_utils/tool_executor.py index 63684c4243..d01cbce0a9 100644 --- a/holmes/core/tools_utils/tool_executor.py +++ b/holmes/core/tools_utils/tool_executor.py @@ -137,7 +137,7 @@ def get_all_tools_openai_format( Args: include_restricted: If False, filter out tools marked as restricted. - Set to True when runbook is in use or restricted + Set to True when skill is in use or restricted tools are explicitly enabled. user_id: If provided, replace OAuth _connect placeholders with the user's real tools (loaded after authentication). diff --git a/holmes/core/toolset_manager.py b/holmes/core/toolset_manager.py index 1830cc6068..016480293c 100644 --- a/holmes/core/toolset_manager.py +++ b/holmes/core/toolset_manager.py @@ -26,6 +26,7 @@ # Mapping of deprecated toolset names to their new names DEPRECATED_TOOLSET_NAMES: dict[str, str] = { "coralogix/logs": "coralogix", + "runbook": "skills", } @@ -58,14 +59,14 @@ def __init__( custom_toolsets: Optional[List[FilePath]] = None, custom_toolsets_from_cli: Optional[List[FilePath]] = None, toolset_status_location: Optional[FilePath] = None, - custom_runbook_catalogs: Optional[List[Union[str, FilePath]]] = None, + custom_skill_paths: Optional[List[Union[str, FilePath]]] = None, config_file_path: Optional[Path] = None, additional_toolsets: Optional[List[Toolset]] = None, ): self.toolsets = toolsets self.toolsets = toolsets or {} self.additional_toolsets = additional_toolsets or [] - self.custom_runbook_catalogs = custom_runbook_catalogs + self.custom_skill_paths = custom_skill_paths if mcp_servers is not None: for _, mcp_server in mcp_servers.items(): mcp_server["type"] = ToolsetType.MCP.value @@ -117,12 +118,12 @@ def _list_all_toolsets( 3. custom toolset from config can override both built-in and add new custom toolsets # for backward compatibility """ # Load built-in toolsets - # Extract search paths from custom catalog files + # Extract search paths from custom skill paths additional_search_paths = None - if self.custom_runbook_catalogs: + if self.custom_skill_paths: additional_search_paths = [ - os.path.dirname(os.path.abspath(str(catalog_path))) - for catalog_path in self.custom_runbook_catalogs + str(Path(p).resolve()) if Path(p).is_dir() else os.path.dirname(os.path.abspath(str(p))) + for p in self.custom_skill_paths ] builtin_toolsets = load_builtin_toolsets(dal, additional_search_paths) diff --git a/holmes/interactive.py b/holmes/interactive.py index 0e0a05b66e..543bd9ad0a 100644 --- a/holmes/interactive.py +++ b/holmes/interactive.py @@ -2258,7 +2258,7 @@ def run_interactive_loop( include_files: Optional[List[Path]], show_tool_output: bool, tracer=None, - runbooks=None, + skills=None, system_prompt_additions: Optional[str] = None, check_version: bool = True, feedback_callback: Optional[FeedbackCallback] = None, @@ -2554,7 +2554,7 @@ def get_bottom_toolbar(): user_input, include_files, ai.tool_executor, - runbooks, + skills, system_prompt_additions, prompt_component_overrides=prompt_component_overrides, ) diff --git a/holmes/main.py b/holmes/main.py index 9a56c896f9..f9091821b5 100644 --- a/holmes/main.py +++ b/holmes/main.py @@ -163,7 +163,7 @@ def _investigate_issue( investigation_additions = f"Provide a terse analysis of the following {issue.source_type} alert/issue and why it is firing." system_prompt = build_system_prompt( toolsets=ai.tool_executor.toolsets, - runbooks=None, + skills=None, system_prompt_additions=investigation_additions, cluster_name=config.cluster_name, ask_user_enabled=False, @@ -370,7 +370,7 @@ def ask( include_file, show_tool_output, tracer, - config.get_runbook_catalog(), + config.get_skill_catalog(), system_prompt_additions, json_output_file=json_output_file, bash_always_deny=bash_always_deny, @@ -391,7 +391,7 @@ def ask( prompt, # type: ignore include_file, ai.tool_executor, - config.get_runbook_catalog(), + config.get_skill_catalog(), system_prompt_additions, prompt_component_overrides=prompt_component_overrides, ) @@ -741,7 +741,7 @@ def ticket( system_prompt = build_system_prompt( toolsets=ai.tool_executor.toolsets, - runbooks=None, + skills=None, system_prompt_additions=ticket_additions, cluster_name=ticket_source.config.cluster_name, ask_user_enabled=False, diff --git a/holmes/plugins/prompts/_general_instructions.jinja2 b/holmes/plugins/prompts/_general_instructions.jinja2 index dc81ecfa00..024a980e92 100644 --- a/holmes/plugins/prompts/_general_instructions.jinja2 +++ b/holmes/plugins/prompts/_general_instructions.jinja2 @@ -1,7 +1,7 @@ {% if todowrite_enabled %} {% include 'investigation_procedure.jinja2' %} {% else %} -{% include '_runbooks_instructions.jinja2' %} +{% include '_skills_instructions.jinja2' %} {% endif %} {% if ai_safety_enabled %} @@ -22,8 +22,8 @@ * if you cannot find the resource/application that the user referred to, assume they made a typo or included/excluded characters like - and in this case, try to find substrings or search for the correct spellings * always provide detailed information like exact resource names, versions, labels, etc * even if you found the root cause, keep investigating to find other possible root causes and to gather data for the answer like exact names -* if a runbook url is present, fetch the runbook before beginning your investigation -* if a runbook in the catalog clearly matches the issue, fetch it — but do not fetch runbooks speculatively +* if a skill url is present, fetch the skill before beginning your investigation +* if a skill in the catalog clearly matches the issue, fetch it — but do not fetch skills speculatively * if you don't know, say that the analysis was inconclusive. * if there are multiple possible causes list them in a numbered list. * there will often be errors in the data that are not relevant or that do not have an impact - ignore them in your conclusion if you were not able to tie them to an actual error. diff --git a/holmes/plugins/prompts/_runbook_instructions.jinja2 b/holmes/plugins/prompts/_runbook_instructions.jinja2 deleted file mode 100644 index ebafb9f068..0000000000 --- a/holmes/plugins/prompts/_runbook_instructions.jinja2 +++ /dev/null @@ -1,33 +0,0 @@ -{%- set sections = [ - {'title': 'Runbook Catalog', 'content': runbook_catalog}, - {'title': 'Subject/Issue Runbooks', 'content': custom_instructions}, - {'title': 'Global Instructions', 'content': global_instructions} -] -%} -{%- set available = sections | selectattr('content') | list -%} -{%- if available -%} -# Runbook Selection - -You (HolmesGPT) have access to runbooks with step-by-step troubleshooting instructions. -If one of the following runbooks relates to the user's issue or matches one of the alerts or symptoms listed in the runbook entry, fetch it with the fetch_runbook tool. Only fetch runbooks that clearly match — do not fetch speculatively. -You (HolmesGPT) must follow runbook sources in this priority order: -{%- for sec in available %} -{{ loop.index }}) {{ sec.title }} (priority #{{ loop.index }}) -{%- endfor %} - -{%- for sec in available %} -## {{ sec.title }} (priority #{{ loop.index }}) - -{%- set content = (sec.content|string) -%} -{{ content.replace('\n', '\n ') }} - -{%- endfor %} - - -If a runbook clearly matches the user's issue: -1. Fetch the runbook with the `fetch_runbook` tool. -2. Decide based on the runbook's contents if it is relevant or not. -3. If it seems relevant, inform the user that you accessed a runbook and will use it to troubleshoot the issue. -4. To the maximum extent possible, follow the runbook instructions step-by-step. -5. Provide a detailed report of the steps you performed, including any findings or errors encountered. -6. If a runbook step requires tools or integrations you don't have access to, tell the user that you cannot perform that step due to missing tools. -{%- endif -%} diff --git a/holmes/plugins/prompts/_runbooks_instructions.jinja2 b/holmes/plugins/prompts/_runbooks_instructions.jinja2 deleted file mode 100644 index eb05dbcacc..0000000000 --- a/holmes/plugins/prompts/_runbooks_instructions.jinja2 +++ /dev/null @@ -1,9 +0,0 @@ -{% if runbooks_enabled -%} -# Runbook Usage: -If a runbook in the catalog clearly matches the issue being investigated, fetch it using the `fetch_runbook` tool before diving into other tools. -Only fetch runbooks that are relevant to the specific issue — do not fetch runbooks speculatively or "just in case". -If no runbook matches, skip this step and investigate directly with available tools. - -After fetching a runbook, read the content returned in the tool's data field and follow its steps. -Runbook content takes priority over general investigation steps. -{%- endif %} diff --git a/holmes/plugins/prompts/_skill_instructions.jinja2 b/holmes/plugins/prompts/_skill_instructions.jinja2 new file mode 100644 index 0000000000..2f4cde053f --- /dev/null +++ b/holmes/plugins/prompts/_skill_instructions.jinja2 @@ -0,0 +1,33 @@ +{%- set sections = [ + {'title': 'Skill Catalog', 'content': skill_catalog}, + {'title': 'Subject/Issue Skills', 'content': custom_instructions}, + {'title': 'Global Instructions', 'content': global_instructions} +] -%} +{%- set available = sections | selectattr('content') | list -%} +{%- if available -%} +# Skill Selection + +You (HolmesGPT) have access to skills with step-by-step troubleshooting instructions. +If one of the following skills relates to the user's issue or matches one of the alerts or symptoms listed in the skill entry, fetch it with the fetch_skill tool. Only fetch skills that clearly match — do not fetch speculatively. +You (HolmesGPT) must follow skill sources in this priority order: +{%- for sec in available %} +{{ loop.index }}) {{ sec.title }} (priority #{{ loop.index }}) +{%- endfor %} + +{%- for sec in available %} +## {{ sec.title }} (priority #{{ loop.index }}) + +{%- set content = (sec.content|string) -%} +{{ content.replace('\n', '\n ') }} + +{%- endfor %} + + +If a skill clearly matches the user's issue: +1. Fetch the skill with the `fetch_skill` tool. +2. Decide based on the skill's contents if it is relevant or not. +3. If it seems relevant, inform the user that you accessed a skill and will use it to troubleshoot the issue. +4. To the maximum extent possible, follow the skill instructions step-by-step. +5. Provide a detailed report of the steps you performed, including any findings or errors encountered. +6. If a skill step requires tools or integrations you don't have access to, tell the user that you cannot perform that step due to missing tools. +{%- endif -%} diff --git a/holmes/plugins/prompts/_skills_instructions.jinja2 b/holmes/plugins/prompts/_skills_instructions.jinja2 new file mode 100644 index 0000000000..ef4f3f72fc --- /dev/null +++ b/holmes/plugins/prompts/_skills_instructions.jinja2 @@ -0,0 +1,9 @@ +{% if skills_enabled -%} +# Skill Usage: +If a skill in the catalog clearly matches the issue being investigated, fetch it using the `fetch_skill` tool before diving into other tools. +Only fetch skills that are relevant to the specific issue — do not fetch skills speculatively or "just in case". +If no skill matches, skip this step and investigate directly with available tools. + +After fetching a skill, read the content returned in the tool's data field and follow its steps. +Skill content takes priority over general investigation steps. +{%- endif %} diff --git a/holmes/plugins/prompts/base_user_prompt.jinja2 b/holmes/plugins/prompts/base_user_prompt.jinja2 index bd98b538b2..111fa8362b 100644 --- a/holmes/plugins/prompts/base_user_prompt.jinja2 +++ b/holmes/plugins/prompts/base_user_prompt.jinja2 @@ -1,7 +1,7 @@ {{ user_prompt }} -{% if runbooks_enabled %} -{% include '_runbook_instructions.jinja2' %} +{% if skills_enabled %} +{% include '_skill_instructions.jinja2' %} {% endif %} {% include '_current_date_time.jinja2' %} diff --git a/holmes/plugins/prompts/investigation_procedure.jinja2 b/holmes/plugins/prompts/investigation_procedure.jinja2 index 87127a196b..3d54cdefd2 100644 --- a/holmes/plugins/prompts/investigation_procedure.jinja2 +++ b/holmes/plugins/prompts/investigation_procedure.jinja2 @@ -6,7 +6,7 @@ CRITICAL: For multi-step questions, you MUST start by calling the TodoWrite tool - `content`: specific task description (string) - `status`: "pending" for new tasks (string) -{% include '_runbooks_instructions.jinja2' %} +{% include '_skills_instructions.jinja2' %} MANDATORY Task Status Updates: - When starting a task: Call TodoWrite changing that task's status to "in_progress" @@ -87,8 +87,8 @@ If you see ANY `[ ] pending` or `[~] in_progress` tasks, DO NOT provide final an For ANY question requiring investigation, you MUST follow this structured approach: ## Phase 1: Initial Investigation -{% if runbooks_enabled -%} -1. **Check for matching runbooks**: If a runbook in the catalog clearly matches the issue, fetch it first. Otherwise, skip this step. +{% if skills_enabled -%} +1. **Check for matching skills**: If a skill in the catalog clearly matches the issue, fetch it first. Otherwise, skip this step. 2. **Start with TodoWrite**: Create initial investigation task list 3. **Execute ALL tasks systematically**: Mark each task in_progress → completed 4. **Complete EVERY task** in the current list before proceeding diff --git a/holmes/plugins/runbooks/README.md b/holmes/plugins/runbooks/README.md deleted file mode 100644 index 7cc9b176bb..0000000000 --- a/holmes/plugins/runbooks/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Runbooks - -Runbooks folder contains operational runbooks for the HolmesGPT project. Runbooks provide step-by-step instructions for common tasks, troubleshooting, and maintenance procedures related to the plugins in this directory. - -## Purpose - -- Standardize operational processes -- Enable quick onboarding for new team members -- Reduce downtime by providing clear troubleshooting steps - -## Structure - -### Structured Runbook - -Structured runbooks are designed for specific issues when conditions like issue name, id or source match, the corresponding instructions will be returned for investigation. -For example, the investigation in [kube-prometheus-stack.yaml](kube-prometheus-stack.yaml) will be returned when the issue to be investigated match either KubeSchedulerDown or KubeControllerManagerDown. -This runbook is mainly used for `holmes investigate` - -### Catalog - -Catalog specified in [catalog.json](catalog.json) contains a collection of runbooks written in markdown. -During runtime, LLM will compare the runbook description with the user question and return the most matched runbook for investigation. It's possible no runbook is returned for no match. - -## Generating Runbooks - -To ensure all runbooks follow a consistent format and improve troubleshooting accuracy, contributors should use the standardized [runbook format prompt](runbook-format.prompt.md) when creating new runbooks. - -### Using the Runbook Format Prompt - -1. **Start with the Template**: Use `prompt.md` as your guide when creating new runbooks -2. **Follow the Structure**: Ensure your runbook includes all required sections: - - **Goal**: Clear definition of issues addressed and agent mandate - - **Workflow**: Sequential diagnostic steps with detailed function descriptions - - **Synthesize Findings**: Logic for combining outputs and identifying root causes - - **Recommended Remediation Steps**: Both immediate and permanent solutions - -### Benefits of Using the Standard Format - -- **Consistency**: All runbooks follow the same structure and terminology -- **AI Agent Compatibility**: Ensures runbooks are machine-readable and executable by AI agents -- **Improved Accuracy**: Standardized format reduces ambiguity and improves diagnostic success rates -- **Maintainability**: Easier to update and maintain runbooks across the project - -### Example Usage - -When creating a runbook for a new issue category (e.g., storage problems, authentication failures), provide the issue description to an LLM along with the prompt template to generate a properly formatted runbook that follows the established patterns. diff --git a/holmes/plugins/runbooks/__init__.py b/holmes/plugins/runbooks/__init__.py deleted file mode 100644 index ef0af0ff61..0000000000 --- a/holmes/plugins/runbooks/__init__.py +++ /dev/null @@ -1,202 +0,0 @@ -import json -import logging -import os -import os.path -from datetime import date -from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Tuple, Union - -import yaml -from pydantic import BaseModel - -if TYPE_CHECKING: - from holmes.core.supabase_dal import SupabaseDal - -THIS_DIR = os.path.abspath(os.path.dirname(__file__)) -DEFAULT_RUNBOOK_SEARCH_PATH = THIS_DIR - -CATALOG_FILE = "catalog.json" - - -class RobustaRunbookInstruction(BaseModel): - id: str - symptom: str - title: str - instruction: Optional[str] = None - alerts: List[str] = [] - - """ - Custom YAML dumper to represent multi-line strings in literal block style due to instructions often being multi-line. - for example: - instructions: | - Step 1: Do this - Step 2: Do that - - instead of: - instructions: "Step 1: Do this - Step 2: Do that" - - """ - - class _LiteralDumper(yaml.SafeDumper): - pass - - @staticmethod - def _repr_str(dumper, s: str): - s = s.replace("\\n", "\n") - return dumper.represent_scalar( - "tag:yaml.org,2002:str", s, style="|" if "\n" in s else None - ) - - _LiteralDumper.add_representer(str, _repr_str) # type: ignore - - def to_list_string(self) -> str: - return f"{self.id}" - - def to_prompt_string(self) -> str: - return f"id='{self.id}' | title='{self.title}' | symptom='{self.symptom}' | relevant alerts={', '.join(self.alerts)}" - - def pretty(self) -> str: - try: - data = self.model_dump(exclude_none=True) # pydantic v2 - except AttributeError: - data = self.dict(exclude_none=True) # pydantic v1 - return yaml.dump( - data, Dumper=self._LiteralDumper, sort_keys=False, allow_unicode=True - ) - - -class RunbookCatalogEntry(BaseModel): - """ - RunbookCatalogEntry contains metadata about a runbook. - This entry points to a markdown file containing the runbook content. - """ - - id: str - update_date: date - description: str - link: str - - def to_list_string(self) -> str: - return f"{self.link}" - - def to_prompt_string(self) -> str: - return f"{self.link} | description: {self.description}" - - -class RunbookCatalog(BaseModel): - catalog: List[Union[RunbookCatalogEntry, "RobustaRunbookInstruction"]] # type: ignore - - def list_available_runbooks(self) -> list[str]: - return [entry.to_list_string() for entry in self.catalog] - - def split_by_type( - self, - ) -> Tuple[List[RunbookCatalogEntry], List[RobustaRunbookInstruction]]: - md: List[RunbookCatalogEntry] = [] - robusta: List[RobustaRunbookInstruction] = [] # - for catalog_entry in self.catalog: - if isinstance(catalog_entry, RunbookCatalogEntry): - md.append(catalog_entry) - elif isinstance(catalog_entry, RobustaRunbookInstruction): - robusta.append(catalog_entry) - return md, robusta - - def to_prompt_string(self) -> str: - md, robusta = self.split_by_type() - parts: List[str] = [""] - if md: - parts.append("Here are MD runbooks:") - parts.extend(f"* {e.to_prompt_string()}" for e in md) - if robusta: - parts.append("\nHere are Robusta runbooks:") - parts.extend(f"* {e.to_prompt_string()}" for e in robusta) - return "\n".join(parts) - - -def load_runbook_catalog( - dal: Optional["SupabaseDal"] = None, - custom_catalog_paths: Optional[List[Union[str, Path]]] = None, -) -> Optional[RunbookCatalog]: # type: ignore - dir_path = os.path.dirname(os.path.realpath(__file__)) - catalog = None - catalogPath = os.path.join(dir_path, CATALOG_FILE) - try: - if os.path.isfile(catalogPath): - with open(catalogPath) as file: - catalog_dict = json.load(file) - catalog = RunbookCatalog(**catalog_dict) - except json.JSONDecodeError as e: - logging.error(f"Error decoding JSON from {catalogPath}: {e}") - except Exception as e: - logging.error( - f"Unexpected error while loading runbook catalog from {catalogPath}: {e}" - ) - - # Append custom catalog files if provided - if custom_catalog_paths: - for custom_catalog_path in custom_catalog_paths: - try: - custom_catalog_path_str = str(custom_catalog_path) - if not os.path.isfile(custom_catalog_path_str): - logging.warning( - f"Custom catalog file not found: {custom_catalog_path_str}" - ) - continue - - with open(custom_catalog_path_str) as file: - custom_catalog_dict = json.load(file) - custom_catalog = RunbookCatalog(**custom_catalog_dict) - - if catalog: - catalog.catalog.extend(custom_catalog.catalog) - else: - catalog = custom_catalog - except json.JSONDecodeError as e: - logging.error(f"Error decoding JSON from {custom_catalog_path}: {e}") - except Exception as e: - logging.error( - f"Unexpected error while loading custom catalog from {custom_catalog_path}: {e}" - ) - - # Append additional runbooks from SupabaseDal if provided - if dal: - try: - supabase_entries = dal.get_runbook_catalog() - if not supabase_entries: - return catalog - if catalog: - catalog.catalog.extend(supabase_entries) - else: - # if failed to load from file, create new catalog from supabase - catalog = RunbookCatalog(catalog=supabase_entries) # type: ignore - except Exception as e: - logging.error(f"Error loading runbooks from Supabase: {e}") - - return catalog - - -def get_runbook_by_path( - runbook_relative_path: str, search_paths: List[str] -) -> Optional[str]: - """ - Find a runbook by searching through provided paths. - - Args: - runbook_relative_path: The relative path to the runbook - search_paths: Optional list of directories to search. If None, uses default runbook folder. - - Returns: - Full path to the runbook if found, None otherwise - """ - # Validate runbook_relative_path is not empty - if not runbook_relative_path or not runbook_relative_path.strip(): - return None - - for search_path in search_paths: - runbook_path = os.path.join(search_path, runbook_relative_path) - # Ensure it's a file, not a directory - if os.path.isfile(runbook_path): - return runbook_path - - return None diff --git a/holmes/plugins/runbooks/catalog.json b/holmes/plugins/runbooks/catalog.json deleted file mode 100644 index 1a49decc89..0000000000 --- a/holmes/plugins/runbooks/catalog.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "catalog": [] -} diff --git a/holmes/plugins/runbooks/CLAUDE.md b/holmes/plugins/skills/CLAUDE.md similarity index 53% rename from holmes/plugins/runbooks/CLAUDE.md rename to holmes/plugins/skills/CLAUDE.md index 842b5f29b9..697765dede 100644 --- a/holmes/plugins/runbooks/CLAUDE.md +++ b/holmes/plugins/skills/CLAUDE.md @@ -1,14 +1,14 @@ -You are an expert in automated diagnostics and runbook creation for an AI-driven troubleshooting agents. I will provide you with one or more issue descriptions or test scenarios. +You are an expert in automated diagnostics and skill creation for an AI-driven troubleshooting agents. I will provide you with one or more issue descriptions or test scenarios. -Your task is to generate a strictly executable runbook for AI Agent to follow. The runbook should be machine-readable but human-understandable, and must include the following sections: +Your task is to generate a strictly executable skill for AI Agent to follow. The skill should be machine-readable but human-understandable, and must include the following sections: -# Runbook Content Structure +# Skill Content Structure ## 1. Goal -- **Primary Objective:** Clearly define the specific category of issues this runbook addresses (e.g., "diagnose network connectivity problems", "troubleshoot pod startup failures", "investigate performance degradation"). -- **Scope:** Specify the environment, technology stack, or system components covered by this runbook. +- **Primary Objective:** Clearly define the specific category of issues this skill addresses (e.g., "diagnose network connectivity problems", "troubleshoot pod startup failures", "investigate performance degradation"). +- **Scope:** Specify the environment, technology stack, or system components covered by this skill. - **Agent Mandate:** Explicitly state that the AI agent must follow the workflow steps sequentially and systematically without deviation to ensure consistent, thorough troubleshooting. -- **Expected Outcome:** Define what successful completion of this runbook should achieve (root cause identification, issue resolution, or escalation criteria). +- **Expected Outcome:** Define what successful completion of this skill should achieve (root cause identification, issue resolution, or escalation criteria). ## 2. Workflow for [Issue Category] Diagnosis - Provide numbered, sequential steps the AI agent must execute in order. @@ -40,46 +40,41 @@ Your task is to generate a strictly executable runbook for AI Agent to follow. T # File Organization Guidelines ## Folder Structure -*Category folders are used to distinguish and categorize different runbooks based on their focus area or technology domain. Each runbook must be placed into a specific category folder under `holmes/plugins/runbooks/` for better organization and discoverability. Create a new category folder if your runbook doesn't fit into existing categories.* +*Each skill is a directory containing a `SKILL.md` file. Skills are placed under `holmes/plugins/skills/builtin/` for builtin skills. Create a new skill directory if your skill doesn't fit into existing ones.* -## File Naming -*Use consistent naming conventions for runbook files:* +## Directory Naming +*Use consistent naming conventions for skill directories:* -- Use descriptive, lowercase names with hyphens: `dns-resolution-troubleshooting.md` -- Include the issue type or technology: `redis-connection-issues.md` -- Avoid generic names like `troubleshooting.md` or `debug.md` +- Use descriptive, lowercase names with hyphens: `dns-resolution-troubleshooting/` +- Include the issue type or technology: `redis-connection-issues/` +- Avoid generic names like `troubleshooting/` or `debug/` -### Catalog Registration -After creating your runbook, you must add an entry to `catalog.json` in the runbooks directory to make it discoverable by AI agents. +### Creating a New Skill +Each skill is a directory with a `SKILL.md` file containing YAML frontmatter and markdown body: -**Steps to add a new catalog entry:** +1. **Create a directory** under `holmes/plugins/skills/builtin/` with a descriptive name +2. **Create a `SKILL.md` file** inside it with this structure: + ```markdown + --- + name: my-skill-name + description: Clear description of what issues this skill addresses + --- -1. **Open** `holmes/plugins/runbooks/catalog.json` -2. **Add your entry** to the JSON array following this structure: - ```json - { - "name": "Brief, descriptive name of the runbook", - "path": "category-folder/your-runbook-filename.md", - "description": "Clear description of what issues this runbook addresses", - "tags": ["relevant", "tags", "for", "search"] - } + ## Goal + ... + + ## Workflow + ... ``` -3. **Ensure proper JSON formatting** - add a comma after the previous entry if needed -4. **Validate the JSON** is properly formatted before committing - -**Field Guidelines:** -- `name`: Keep concise but descriptive (e.g., "Redis Connection Issues") -- `path`: Always include the category folder (e.g., "database/redis-connection-issues.md") -- `description`: Explain what specific problems this runbook solves -- `tags`: Include technology names, issue types, and relevant keywords - -Example catalog entry: -```json -{ - "name": "DNS Resolution Troubleshooting", - "path": "networking/dns-resolution-troubleshooting.md", - "description": "Comprehensive guide for diagnosing and resolving DNS resolution issues in Kubernetes clusters", - "tags": ["dns", "networking", "kubernetes", "troubleshooting"] -} +3. The skill is automatically discovered — no catalog registration needed + +**Frontmatter Fields:** +- `name` (optional): Lowercase with hyphens. Defaults to directory name. +- `description` (required): Used by the LLM to match the skill to user issues. Be specific. +Example: +``` +holmes/plugins/skills/builtin/ + dns-resolution-troubleshooting/ + SKILL.md ``` diff --git a/holmes/plugins/skills/README.md b/holmes/plugins/skills/README.md new file mode 100644 index 0000000000..6b2c61ece4 --- /dev/null +++ b/holmes/plugins/skills/README.md @@ -0,0 +1,47 @@ +# Skills + +Skills folder contains operational skills for the HolmesGPT project. Skills provide step-by-step instructions for common tasks, troubleshooting, and maintenance procedures related to the plugins in this directory. + +## Purpose + +- Standardize operational processes +- Enable quick onboarding for new team members +- Reduce downtime by providing clear troubleshooting steps + +## Structure + +### Structured Skill + +Structured skills are designed for specific issues when conditions like issue name, id or source match, the corresponding instructions will be returned for investigation. +For example, the investigation in [kube-prometheus-stack.yaml](kube-prometheus-stack.yaml) will be returned when the issue to be investigated match either KubeSchedulerDown or KubeControllerManagerDown. +This skill is mainly used for `holmes investigate` + +### Skills Directory + +Each skill is a directory containing a `SKILL.md` file with YAML frontmatter (name, description) and markdown body (the instructions). +Skills are placed under `holmes/plugins/skills/builtin/` for builtin skills, or in any directory configured via `custom_skill_paths`. +During runtime, the LLM will compare the skill description with the user question and fetch the most matched skill for investigation. It's possible no skill is fetched for no match. + +## Generating Skills + +To ensure all skills follow a consistent format and improve troubleshooting accuracy, contributors should use the standardized [skill format prompt](skill-format.prompt.md) when creating new skills. + +### Using the Skill Format Prompt + +1. **Start with the Template**: Use `prompt.md` as your guide when creating new skills +2. **Follow the Structure**: Ensure your skill includes all required sections: + - **Goal**: Clear definition of issues addressed and agent mandate + - **Workflow**: Sequential diagnostic steps with detailed function descriptions + - **Synthesize Findings**: Logic for combining outputs and identifying root causes + - **Recommended Remediation Steps**: Both immediate and permanent solutions + +### Benefits of Using the Standard Format + +- **Consistency**: All skills follow the same structure and terminology +- **AI Agent Compatibility**: Ensures skills are machine-readable and executable by AI agents +- **Improved Accuracy**: Standardized format reduces ambiguity and improves diagnostic success rates +- **Maintainability**: Easier to update and maintain skills across the project + +### Example Usage + +When creating a skill for a new issue category (e.g., storage problems, authentication failures), provide the issue description to an LLM along with the prompt template to generate a properly formatted skill that follows the established patterns. diff --git a/holmes/plugins/skills/__init__.py b/holmes/plugins/skills/__init__.py new file mode 100644 index 0000000000..ea928753ee --- /dev/null +++ b/holmes/plugins/skills/__init__.py @@ -0,0 +1,35 @@ +from typing import List, Optional + +import yaml +from pydantic import BaseModel + + +class RobustaSkillInstruction(BaseModel): + """Supabase-hosted skill instruction from the HolmesRunbooks table.""" + + id: str + symptom: str + title: str + instruction: Optional[str] = None + alerts: List[str] = [] + + class _LiteralDumper(yaml.SafeDumper): + pass + + @staticmethod + def _repr_str(dumper, s: str): + s = s.replace("\\n", "\n") + return dumper.represent_scalar( + "tag:yaml.org,2002:str", s, style="|" if "\n" in s else None + ) + + _LiteralDumper.add_representer(str, _repr_str) # type: ignore + + def pretty(self) -> str: + try: + data = self.model_dump(exclude_none=True) + except AttributeError: + data = self.dict(exclude_none=True) + return yaml.dump( + data, Dumper=self._LiteralDumper, sort_keys=False, allow_unicode=True + ) diff --git a/holmes/plugins/toolsets/runbook/__init__.py b/holmes/plugins/skills/builtin/__init__.py similarity index 100% rename from holmes/plugins/toolsets/runbook/__init__.py rename to holmes/plugins/skills/builtin/__init__.py diff --git a/holmes/plugins/skills/skill_loader.py b/holmes/plugins/skills/skill_loader.py new file mode 100644 index 0000000000..63bcf13219 --- /dev/null +++ b/holmes/plugins/skills/skill_loader.py @@ -0,0 +1,214 @@ +import logging +import os +import re +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional, Union + +import yaml +from pydantic import BaseModel + +from holmes.plugins.skills import RobustaSkillInstruction + +if TYPE_CHECKING: + from holmes.core.supabase_dal import SupabaseDal + +THIS_DIR = os.path.abspath(os.path.dirname(__file__)) +BUILTIN_SKILLS_DIR = os.path.join(THIS_DIR, "builtin") + +SKILL_FILENAME = "SKILL.md" + + +class SkillSource(str, Enum): + BUILTIN = "builtin" + USER = "user" + REMOTE = "remote" + + +class Skill(BaseModel): + name: str + description: str + content: str + source: SkillSource + source_path: Optional[str] = None + + def to_prompt_string(self) -> str: + return f"{self.name} | description: {self.description}" + + +class SkillCatalog(BaseModel): + skills: List[Skill] + + def list_available_skills(self) -> List[str]: + return [s.name for s in self.skills] + + def to_prompt_string(self) -> str: + priority = {SkillSource.REMOTE: 0, SkillSource.USER: 1, SkillSource.BUILTIN: 2} + sorted_skills = sorted(self.skills, key=lambda s: priority[s.source]) + + local = [s for s in sorted_skills if s.source != SkillSource.REMOTE] + remote = [s for s in sorted_skills if s.source == SkillSource.REMOTE] + + parts: List[str] = [""] + if local: + parts.append("Here are local skills:") + parts.extend(f"* {s.to_prompt_string()}" for s in local) + if remote: + parts.append("\nHere are Robusta skills:") + parts.extend(f"* {s.to_prompt_string()}" for s in remote) + return "\n".join(parts) + + +def normalize_skill_name(name: str) -> str: + """Normalize a skill name: lowercase, replace underscores/spaces with hyphens.""" + return re.sub(r"[\s_]+", "-", name.strip().lower()) + + +def parse_skill_file(path: Path, source: SkillSource = SkillSource.USER) -> Skill: + """Parse a SKILL.md file with YAML frontmatter + markdown body. + + Expected format: + --- + name: my-skill (optional, defaults to parent directory name) + description: What this skill does (required) + --- + Markdown content here... + """ + text = path.read_text(encoding="utf-8") + + # Split frontmatter from content + if text.startswith("---"): + parts = text.split("---", 2) + if len(parts) >= 3: + frontmatter_str = parts[1] + content = parts[2].strip() + else: + raise ValueError(f"Invalid SKILL.md format in {path}: missing closing '---'") + else: + raise ValueError(f"SKILL.md file {path} must start with '---' (YAML frontmatter)") + + frontmatter = yaml.safe_load(frontmatter_str) or {} + + name = frontmatter.get("name") or path.parent.name + name = normalize_skill_name(name) + + description = frontmatter.get("description") + if not description: + raise ValueError(f"SKILL.md file {path} is missing required 'description' field in frontmatter") + + return Skill( + name=name, + description=description, + content=content, + source=source, + source_path=str(path), + ) + + +def scan_skill_directory( + directory: Path, source: SkillSource = SkillSource.USER, max_depth: int = 2 +) -> List[Skill]: + """Scan a directory for SKILL.md files up to max_depth levels deep.""" + skills: List[Skill] = [] + directory = directory.resolve() + + if not directory.is_dir(): + logging.warning(f"Skill directory does not exist: {directory}") + return skills + + for root, dirs, files in os.walk(directory): + depth = len(Path(root).resolve().relative_to(directory).parts) + if depth >= max_depth: + dirs.clear() + continue + + if SKILL_FILENAME in files: + skill_path = Path(root) / SKILL_FILENAME + try: + skill = parse_skill_file(skill_path, source=source) + skills.append(skill) + except Exception as e: + logging.error(f"Failed to parse {skill_path}: {e}") + + return skills + + +def map_robusta_instruction_to_skill( + instr: RobustaSkillInstruction, +) -> Skill: + """Convert a Supabase RobustaSkillInstruction into a Skill.""" + description = instr.title + if instr.symptom: + description = f"{instr.title} — {instr.symptom}" + + return Skill( + name=instr.id, + description=description, + content=instr.instruction or "", + source=SkillSource.REMOTE, + source_path=instr.id, + ) + + +def load_skill_catalog( + dal: Optional["SupabaseDal"] = None, + custom_skill_paths: Optional[List[Union[str, Path]]] = None, +) -> Optional[SkillCatalog]: + """Load skills from all sources and merge into a single catalog. + + Priority (highest wins on name collision): remote > user > builtin. + """ + skills_by_name: dict[str, Skill] = {} + + # 1. Load builtin skills + builtin_dir = Path(BUILTIN_SKILLS_DIR) + if builtin_dir.is_dir(): + for skill in scan_skill_directory(builtin_dir, source=SkillSource.BUILTIN): + skills_by_name[skill.name] = skill + + # 2. Load user skills from custom_skill_paths (overrides builtins) + if custom_skill_paths: + for skill_path in custom_skill_paths: + path = Path(str(skill_path)) + if path.is_dir(): + for skill in scan_skill_directory(path, source=SkillSource.USER): + if skill.name in skills_by_name: + logging.warning( + f"Skill '{skill.name}' from {skill.source_path} " + f"overrides {skills_by_name[skill.name].source_path}" + ) + skills_by_name[skill.name] = skill + elif path.is_file() and path.name == SKILL_FILENAME: + try: + skill = parse_skill_file(path, source=SkillSource.USER) + if skill.name in skills_by_name: + logging.warning( + f"Skill '{skill.name}' from {skill.source_path} " + f"overrides {skills_by_name[skill.name].source_path}" + ) + skills_by_name[skill.name] = skill + except Exception as e: + logging.error(f"Failed to parse skill file {path}: {e}") + else: + logging.warning(f"Skill path is not a directory or SKILL.md file: {path}") + + # 3. Load remote skills from Supabase (overrides all) + if dal: + try: + supabase_entries = dal.get_skill_catalog() + if supabase_entries: + for entry in supabase_entries: + skill = map_robusta_instruction_to_skill(entry) + if skill.name in skills_by_name: + logging.warning( + f"Remote skill '{skill.name}' overrides " + f"{skills_by_name[skill.name].source_path}" + ) + skills_by_name[skill.name] = skill + except Exception as e: + logging.error(f"Error loading skills from Supabase: {e}") + + if not skills_by_name: + return None + + return SkillCatalog(skills=list(skills_by_name.values())) diff --git a/holmes/plugins/toolsets/__init__.py b/holmes/plugins/toolsets/__init__.py index cab1ba2a0e..db2dabfbbf 100644 --- a/holmes/plugins/toolsets/__init__.py +++ b/holmes/plugins/toolsets/__init__.py @@ -54,7 +54,7 @@ from holmes.plugins.toolsets.newrelic.newrelic import NewRelicToolset from holmes.plugins.toolsets.rabbitmq.toolset_rabbitmq import RabbitMQToolset from holmes.plugins.toolsets.robusta.robusta import RobustaToolset -from holmes.plugins.toolsets.runbook.runbook_fetcher import RunbookToolset +from holmes.plugins.toolsets.skills.skills_fetcher import SkillsToolset from holmes.plugins.toolsets.servicenow_tables.servicenow_tables import ( ServiceNowTablesToolset, ) @@ -113,7 +113,7 @@ def load_python_toolsets( KubectlRunToolset(), ConfluenceToolset(), MongoDBAtlasToolset(), - RunbookToolset(dal=dal, additional_search_paths=additional_search_paths), + SkillsToolset(dal=dal, additional_search_paths=additional_search_paths), AzureSQLToolset(), ServiceNowTablesToolset(), DatabaseToolset(), diff --git a/holmes/plugins/toolsets/internet/internet.py b/holmes/plugins/toolsets/internet/internet.py index b6b6f76169..d54f7d5261 100644 --- a/holmes/plugins/toolsets/internet/internet.py +++ b/holmes/plugins/toolsets/internet/internet.py @@ -178,7 +178,7 @@ class FetchWebpage(Tool): def __init__(self, toolset: "InternetToolset"): super().__init__( name="fetch_webpage", - description="Fetch a webpage. Use this to fetch runbooks if they are present before starting your investigation (if no other tool like confluence is more appropriate)", + description="Fetch a webpage. Use this to fetch skills if they are present before starting your investigation (if no other tool like confluence is more appropriate)", parameters={ "url": ToolParameter( description="The URL to fetch", diff --git a/holmes/plugins/toolsets/investigator/investigator_instructions.jinja2 b/holmes/plugins/toolsets/investigator/investigator_instructions.jinja2 index dee4622450..4ac05461ff 100644 --- a/holmes/plugins/toolsets/investigator/investigator_instructions.jinja2 +++ b/holmes/plugins/toolsets/investigator/investigator_instructions.jinja2 @@ -45,7 +45,7 @@ I found the pods are crashing due to OOMKilled events. Let me mark this task as # Doing tasks -The user will primarily request you perform reliability troubleshooting and incident investigation tasks. This includes analyzing observability data (logs, traces, metrics), identifying misconfigurations, finding root causes of outages, correlating incidents with recent changes, following investigation runbooks, and determining remediation steps. For these tasks the following steps are recommended: +The user will primarily request you perform reliability troubleshooting and incident investigation tasks. This includes analyzing observability data (logs, traces, metrics), identifying misconfigurations, finding root causes of outages, correlating incidents with recent changes, following investigation skills, and determining remediation steps. For these tasks the following steps are recommended: - Use the TodoWrite tool to plan the investigation if required - Use the available observability tools to gather data from multiple sources (metrics, logs, traces, configuration). You are encouraged to use the tools extensively both in parallel and sequentially. - Analyze the collected data to identify patterns, correlations, and root causes diff --git a/holmes/plugins/toolsets/runbook/runbook_fetcher.py b/holmes/plugins/toolsets/runbook/runbook_fetcher.py deleted file mode 100644 index 98d50c18a6..0000000000 --- a/holmes/plugins/toolsets/runbook/runbook_fetcher.py +++ /dev/null @@ -1,266 +0,0 @@ -import logging -import os -import textwrap -from pathlib import Path -from typing import List, Optional, Union, cast - -from holmes.core.supabase_dal import SupabaseDal -from holmes.core.tools import ( - StructuredToolResult, - StructuredToolResultStatus, - Tool, - ToolInvokeContext, - ToolParameter, - Toolset, - ToolsetTag, -) -from holmes.plugins.runbooks import ( - DEFAULT_RUNBOOK_SEARCH_PATH, - get_runbook_by_path, - load_runbook_catalog, -) -from holmes.plugins.toolsets.utils import toolset_name_for_one_liner - - -class RunbookFetcher(Tool): - toolset: "RunbookToolset" - available_runbooks: List[str] = [] - additional_search_paths: Optional[List[str]] = None - _dal: Optional[SupabaseDal] = None - - def __init__( - self, - toolset: "RunbookToolset", - additional_search_paths: Optional[List[str]] = None, - dal: Optional[SupabaseDal] = None, - custom_catalog_paths: Optional[List[Union[str, Path]]] = None, - ): - catalog = load_runbook_catalog( - dal=dal, custom_catalog_paths=custom_catalog_paths - ) - available_runbooks = [] - if catalog: - available_runbooks = catalog.list_available_runbooks() - - if additional_search_paths: - for search_path in additional_search_paths: - if not os.path.isdir(search_path): - continue - - for file in os.listdir(search_path): - if file.endswith(".md") and file not in available_runbooks: - available_runbooks.append(f"{file}") - - runbook_list = ", ".join([f'"{rb}"' for rb in available_runbooks]) - - super().__init__( - name="fetch_runbook", - description="Get runbook content by runbook link. Use this to get troubleshooting steps for incidents", - parameters={ - "runbook_id": ToolParameter( - description=f"The runbook_id: either a UUID or a .md filename. Must be one of: {runbook_list}", - type="string", - required=True, - ), - }, - toolset=toolset, # type: ignore[call-arg] - available_runbooks=available_runbooks, # type: ignore[call-arg] - additional_search_paths=additional_search_paths, # type: ignore[call-arg] - ) - self._dal = dal - - def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: - runbook_id: str = params.get("runbook_id", "") - is_md_file: bool = True if runbook_id.endswith(".md") else False - - # Validate link is not empty - if not runbook_id or not runbook_id.strip(): - err_msg = ( - "Runbook link cannot be empty. Please provide a valid runbook path." - ) - logging.error(err_msg) - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=err_msg, - params=params, - ) - - if is_md_file: - return self._get_md_runbook(runbook_id, params) - else: - return self._get_robusta_runbook(runbook_id, params) - - def _get_robusta_runbook(self, link: str, params: dict) -> StructuredToolResult: - if self._dal and self._dal.enabled: - try: - runbook_content = self._dal.get_runbook_content(link) - if runbook_content: - return StructuredToolResult( - status=StructuredToolResultStatus.SUCCESS, - data=runbook_content.pretty(), - params=params, - ) - else: - err_msg = f"Runbook with UUID '{link}' not found in remote storage." - logging.error(err_msg) - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=err_msg, - params=params, - ) - except Exception as e: - err_msg = f"Failed to fetch runbook with UUID '{link}': {str(e)}" - logging.error(err_msg) - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=err_msg, - params=params, - ) - else: - err_msg = "Runbook link appears to be a UUID, but no remote data access layer (dal) is enabled." - logging.error(err_msg) - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=err_msg, - params=params, - ) - - def _get_md_runbook(self, link: str, params: dict) -> StructuredToolResult: - search_paths = [DEFAULT_RUNBOOK_SEARCH_PATH] - if self.additional_search_paths: - search_paths.extend(self.additional_search_paths) - # Validate link is in the available runbooks list OR is a valid path within allowed directories - if link not in self.available_runbooks: - # Check if the link would resolve to a valid path within allowed directories - # This prevents path traversal attacks like ../../secret.md - is_valid_path = False - for search_path in search_paths: - candidate_path = os.path.join(search_path, link) - # Canonicalize both paths to resolve any .. or . components - real_search_path = os.path.realpath(search_path) - real_candidate_path = os.path.realpath(candidate_path) - - # Check if the resolved path is within the allowed directory - if ( - real_candidate_path.startswith(real_search_path + os.sep) - or real_candidate_path == real_search_path - ): - if os.path.isfile(real_candidate_path): - is_valid_path = True - break - - if not is_valid_path: - err_msg = f"Invalid runbook link '{link}'. Must be one of: {', '.join(self.available_runbooks) if self.available_runbooks else 'No runbooks available'}" - logging.error(err_msg) - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=err_msg, - params=params, - ) - - runbook_path = get_runbook_by_path(link, search_paths) - if runbook_path is None: - err_msg = ( - f"Runbook '{link}' not found in any of the search paths: {search_paths}" - ) - logging.error(err_msg) - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=err_msg, - params=params, - ) - try: - with open(runbook_path, "r") as file: - content = file.read() - wrapped_content = textwrap.dedent(f"""\ - -{textwrap.indent(content, " " * 20)} - - Note: the above are DIRECTIONS not ACTUAL RESULTS. You now need to follow the steps outlined in the runbook yourself USING TOOLS. - Anything that looks like an actual result in the above is just an EXAMPLE. - Now follow those steps and report back what you find. - You must follow them by CALLING TOOLS YOURSELF. - If you are missing tools, follow your general instructions on how to enable them as present in your system prompt. - - Assuming the above runbook is relevant, you MUST start your response (after calling tools to investigate) with: - "I found a runbook named [runbook name/description] and used it to troubleshoot:" - - Then list each step with ✅ for completed steps and ❌ for steps you couldn't complete. - - - I found a runbook named **Troubleshooting Erlang Issues** and used it to troubleshoot: - - 1. ✅ *Check BEAM VM memory usage* - 87% allocated (3.2GB used of 4GB limit) - 2. ✅ *Review GC logs* - 15 full GC cycles in last 30 minutes, avg pause time 2.3s - 3. ✅ *Verify Erlang application logs* - `** exception error: out of memory in process <0.139.0> called by gen_server:handle_msg/6` - 4. ❌ *Could not analyze process mailbox sizes* - Observer tool not enabled in container. Enable remote shell or observer_cli for process introspection. - 5. ✅ *Check pod memory limits* - container limit 4Gi, requests 2Gi - 6. ✅ *Verify BEAM startup arguments* - `+S 4:4 +P 1048576`, no memory instrumentation flags enabled - 7. ❌ *Could not retrieve APM traces* - Datadog traces toolset is disabled. You can enable it by following https://holmesgpt.dev/data-sources/builtin-toolsets/datadog/ - 8. ❌ *Could not query Erlang metrics* - Prometheus integration is not connected. Enable it via https://holmesgpt.dev/data-sources/builtin-toolsets/prometheus/ - 9. ✅ *Examine recent deployments* - app version 2.1.3 deployed 4 hours ago, coincides with memory spike - 10. ❌ *Could not check Stripe API status* - No toolset for Stripe integration exists. To monitor Stripe or similar third-party APIs, add a [custom toolset](https://holmesgpt.dev/data-sources/custom-toolsets/) or use a [remote MCP server](https://holmesgpt.dev/data-sources/remote-mcp-servers/) - - **Root cause:** Memory leak in `gen_server` logic introduced in v2.1.3. BEAM VM hitting memory limit, causing out-of-memory crashes. - - **Fix:** Roll back to v2.1.2 or increase memory limit to 6GB as a temporary workaround. - - """) - return StructuredToolResult( - status=StructuredToolResultStatus.SUCCESS, - data=wrapped_content, - params=params, - ) - except Exception as e: - err_msg = f"Failed to read runbook {runbook_path}: {str(e)}" - logging.error(err_msg) - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=err_msg, - params=params, - ) - - def get_parameterized_one_liner(self, params) -> str: - path: str = params.get("runbook_id", "") - return f"{toolset_name_for_one_liner(self.toolset.name)}: Fetch Runbook {path}" - - -class RunbookToolset(Toolset): - def __init__( - self, - dal: Optional[SupabaseDal], - additional_search_paths: Optional[List[str]] = None, - ): - # Store additional search paths in config for RunbookFetcher to access - config = {} - if additional_search_paths: - config["additional_search_paths"] = additional_search_paths - - # Compute custom catalog paths from additional search paths - custom_catalog_paths = None - if additional_search_paths: - custom_catalog_paths = [ - os.path.join(search_path, "catalog.json") - for search_path in additional_search_paths - if os.path.isfile(os.path.join(search_path, "catalog.json")) - ] - - super().__init__( - name="runbook", - description="Fetch runbooks", - icon_url="https://platform.robusta.dev/demos/runbook.svg", - tools=[ - RunbookFetcher( - self, - additional_search_paths, - dal, - cast(Optional[List[Union[str, Path]]], custom_catalog_paths), - ), - ], - docs_url="https://holmesgpt.dev/data-sources/", - tags=[ - ToolsetTag.CORE, - ], - config=config, - enabled=True, - ) diff --git a/holmes/plugins/toolsets/skills/__init__.py b/holmes/plugins/toolsets/skills/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/holmes/plugins/toolsets/skills/skills_fetcher.py b/holmes/plugins/toolsets/skills/skills_fetcher.py new file mode 100644 index 0000000000..49bc52901c --- /dev/null +++ b/holmes/plugins/toolsets/skills/skills_fetcher.py @@ -0,0 +1,211 @@ +import logging +import textwrap +from typing import List, Optional + +from holmes.core.supabase_dal import SupabaseDal +from holmes.core.tools import ( + StructuredToolResult, + StructuredToolResultStatus, + Tool, + ToolInvokeContext, + ToolParameter, + Toolset, + ToolsetTag, +) +from holmes.plugins.skills.skill_loader import ( + Skill, + SkillCatalog, + SkillSource, + load_skill_catalog, +) +from holmes.plugins.toolsets.utils import toolset_name_for_one_liner + + +class SkillsFetcher(Tool): + toolset: "SkillsToolset" + available_skills: List[str] = [] + _skill_catalog: Optional[SkillCatalog] = None + _dal: Optional[SupabaseDal] = None + + def __init__( + self, + toolset: "SkillsToolset", + skill_catalog: Optional[SkillCatalog] = None, + dal: Optional[SupabaseDal] = None, + ): + available_skills: List[str] = [] + if skill_catalog: + available_skills = skill_catalog.list_available_skills() + + skill_list = ", ".join([f'"{s}"' for s in available_skills]) + + super().__init__( + name="fetch_skill", + description="Get skill content by skill link. Use this to get troubleshooting steps for incidents", + parameters={ + "skill_id": ToolParameter( + description=f"The skill_id: either a UUID or a skill name. Must be one of: {skill_list}", + type="string", + required=True, + ), + }, + toolset=toolset, # type: ignore[call-arg] + available_skills=available_skills, # type: ignore[call-arg] + ) + self._skill_catalog = skill_catalog + self._dal = dal + + def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: + skill_id: str = params.get("skill_id", "") + + if not skill_id or not skill_id.strip(): + err_msg = "Skill link cannot be empty. Please provide a valid skill path." + logging.error(err_msg) + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error=err_msg, + params=params, + ) + + # Look up in skill catalog by name — remote skills have empty content + # (catalog only stores metadata), so fetch full content from Supabase + skill = self._find_skill(skill_id) + if skill and skill.source == SkillSource.REMOTE: + return self._get_robusta_skill(skill_id, params) + elif skill: + return self._format_skill_result(skill, params) + + # Fallback: try Supabase for UUID-style IDs not in catalog + if self._dal and self._dal.enabled: + return self._get_robusta_skill(skill_id, params) + + err_msg = ( + f"Skill '{skill_id}' not found. " + f"Available: {', '.join(self.available_skills) if self.available_skills else 'none'}" + ) + logging.error(err_msg) + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error=err_msg, + params=params, + ) + + def _find_skill(self, name: str) -> Optional[Skill]: + if not self._skill_catalog: + return None + for skill in self._skill_catalog.skills: + if skill.name == name: + return skill + return None + + def _format_skill_result(self, skill: Skill, params: dict) -> StructuredToolResult: + wrapped_content = textwrap.dedent(f"""\ + + {skill.content} + + Note: the above are DIRECTIONS not ACTUAL RESULTS. You now need to follow the steps outlined in the skill yourself USING TOOLS. + Anything that looks like an actual result in the above is just an EXAMPLE. + Now follow those steps and report back what you find. + You must follow them by CALLING TOOLS YOURSELF. + If you are missing tools, follow your general instructions on how to enable them as present in your system prompt. + + Assuming the above skill is relevant, you MUST start your response (after calling tools to investigate) with: + "I found a skill named [skill name/description] and used it to troubleshoot:" + + Then list each step with ✅ for completed steps and ❌ for steps you couldn't complete. + + + I found a skill named **Troubleshooting Erlang Issues** and used it to troubleshoot: + + 1. ✅ *Check BEAM VM memory usage* - 87% allocated (3.2GB used of 4GB limit) + 2. ✅ *Review GC logs* - 15 full GC cycles in last 30 minutes, avg pause time 2.3s + 3. ✅ *Verify Erlang application logs* - `** exception error: out of memory in process <0.139.0> called by gen_server:handle_msg/6` + 4. ❌ *Could not analyze process mailbox sizes* - Observer tool not enabled in container. Enable remote shell or observer_cli for process introspection. + 5. ✅ *Check pod memory limits* - container limit 4Gi, requests 2Gi + 6. ✅ *Verify BEAM startup arguments* - `+S 4:4 +P 1048576`, no memory instrumentation flags enabled + 7. ❌ *Could not retrieve APM traces* - Datadog traces toolset is disabled. You can enable it by following https://holmesgpt.dev/data-sources/builtin-toolsets/datadog/ + 8. ❌ *Could not query Erlang metrics* - Prometheus integration is not connected. Enable it via https://holmesgpt.dev/data-sources/builtin-toolsets/prometheus/ + 9. ✅ *Examine recent deployments* - app version 2.1.3 deployed 4 hours ago, coincides with memory spike + 10. ❌ *Could not check Stripe API status* - No toolset for Stripe integration exists. To monitor Stripe or similar third-party APIs, add a [custom toolset](https://holmesgpt.dev/data-sources/custom-toolsets/) or use a [remote MCP server](https://holmesgpt.dev/data-sources/remote-mcp-servers/) + + **Root cause:** Memory leak in `gen_server` logic introduced in v2.1.3. BEAM VM hitting memory limit, causing out-of-memory crashes. + + **Fix:** Roll back to v2.1.2 or increase memory limit to 6GB as a temporary workaround. + + """) + return StructuredToolResult( + status=StructuredToolResultStatus.SUCCESS, + data=wrapped_content, + params=params, + ) + + def _get_robusta_skill(self, link: str, params: dict) -> StructuredToolResult: + if self._dal and self._dal.enabled: + try: + skill_content = self._dal.get_skill_content(link) + if skill_content: + # Wrap remote skill content with same format as local skills + description = skill_content.title + if skill_content.symptom: + description = f"{skill_content.title} — {skill_content.symptom}" + skill = Skill( + name=skill_content.id, + description=description, + content=skill_content.instruction or skill_content.pretty(), + source=SkillSource.REMOTE, + ) + return self._format_skill_result(skill, params) + else: + err_msg = f"Skill with UUID '{link}' not found in remote storage." + logging.error(err_msg) + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error=err_msg, + params=params, + ) + except Exception as e: + err_msg = f"Failed to fetch skill with UUID '{link}': {str(e)}" + logging.error(err_msg) + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error=err_msg, + params=params, + ) + else: + err_msg = "Skill link appears to be a UUID, but no remote data access layer (dal) is enabled." + logging.error(err_msg) + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error=err_msg, + params=params, + ) + + def get_parameterized_one_liner(self, params) -> str: + path: str = params.get("skill_id", "") + return f"{toolset_name_for_one_liner(self.toolset.name)}: Fetch Skill {path}" + + +class SkillsToolset(Toolset): + def __init__( + self, + dal: Optional[SupabaseDal] = None, + additional_search_paths: Optional[List[str]] = None, + ): + skill_catalog = load_skill_catalog( + dal=dal, + custom_skill_paths=additional_search_paths, + ) + + super().__init__( + name="skills", + description="Fetch skills", + icon_url="https://platform.robusta.dev/demos/runbook.svg", + tools=[ + SkillsFetcher(self, skill_catalog=skill_catalog, dal=dal), + ], + docs_url="https://holmesgpt.dev/data-sources/", + tags=[ + ToolsetTag.CORE, + ], + enabled=True, + ) diff --git a/holmes/utils/global_instructions.py b/holmes/utils/global_instructions.py index cc4ae2cc5c..acd24f6941 100644 --- a/holmes/utils/global_instructions.py +++ b/holmes/utils/global_instructions.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from holmes.plugins.runbooks import RunbookCatalog +from holmes.plugins.skills.skill_loader import SkillCatalog if TYPE_CHECKING: from holmes.core.resource_instruction import ResourceInstructions @@ -35,13 +35,13 @@ def _format_resource_instructions( return items -def generate_runbooks_args( - runbook_catalog: Optional[RunbookCatalog], +def generate_skills_args( + skill_catalog: Optional[SkillCatalog], global_instructions: Optional[Instructions] = None, issue_instructions: Optional[List[str]] = None, resource_instructions: Optional["ResourceInstructions"] = None, # type: ignore ) -> Dict[str, str]: - catalog_str = runbook_catalog.to_prompt_string() if runbook_catalog else "" + catalog_str = skill_catalog.to_prompt_string() if skill_catalog else "" combined_instructions = [] if issue_instructions: @@ -63,7 +63,7 @@ def generate_runbooks_args( ) return { - "runbook_catalog": catalog_str, + "skill_catalog": catalog_str, "custom_instructions": issue_block, "global_instructions": global_block, } diff --git a/pyproject.toml b/pyproject.toml index 9ca93407aa..7a72ddcbfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,7 +139,7 @@ markers = [ "metrics: Tests involving metrics processing", "context_window: Tests involving context window handling", "network: Tests requiring network connectivity", - "runbooks: Tests involving runbook functionality", + "skills: Tests involving skill functionality", "chain-of-causation: Tests involving chain-of-causation analysis", "slackbot: Tests involving Slack bot functionality", "numerical: Tests involving numerical calculations or data", diff --git a/server.py b/server.py index f7d359cfb2..a17a6d9c06 100644 --- a/server.py +++ b/server.py @@ -375,7 +375,7 @@ def chat(chat_request: ChatRequest, http_request: Request): f"streaming={chat_request.stream}" ) - runbooks = config.get_runbook_catalog() + skills = config.get_skill_catalog() prompt_component_overrides = None if chat_request.behavior_controls: @@ -439,7 +439,7 @@ def chat(chat_request: ChatRequest, http_request: Request): config=config, global_instructions=global_instructions, additional_system_prompt=chat_request.additional_system_prompt, - runbooks=runbooks, + skills=skills, images=chat_request.images, prompt_component_overrides=prompt_component_overrides, ) diff --git a/tests/config_class/test_custom_runbook_catalogs.py b/tests/config_class/test_custom_runbook_catalogs.py deleted file mode 100644 index 61a1c1016a..0000000000 --- a/tests/config_class/test_custom_runbook_catalogs.py +++ /dev/null @@ -1,181 +0,0 @@ -import json - -from holmes.config import Config - - -def test_config_custom_runbook_catalogs_from_file(tmp_path): - """Test that custom_runbook_catalogs is loaded from config file.""" - # Create a custom catalog file - custom_catalog_file = tmp_path / "catalog.json" - catalog_data = { - "catalog": [ - { - "id": "test-runbook", - "update_date": "2023-01-01", - "description": "Test runbook", - "link": "test.md", - } - ] - } - custom_catalog_file.write_text(json.dumps(catalog_data)) - - # Create a config file - config_file = tmp_path / "config.yaml" - config_content = f""" -model: "gpt-4" -custom_runbook_catalogs: - - {custom_catalog_file} -""" - config_file.write_text(config_content) - - # Load config - config = Config.load_from_file(config_file) - - assert config.custom_runbook_catalogs is not None - assert len(config.custom_runbook_catalogs) == 1 - assert str(config.custom_runbook_catalogs[0]) == str(custom_catalog_file) - - -def test_config_custom_runbook_catalogs_multiple(tmp_path): - """Test that multiple custom_runbook_catalogs are loaded from config file.""" - # Create multiple custom catalog files - catalog_files = [] - for i in range(2): - custom_catalog_file = tmp_path / f"catalog_{i}.json" - catalog_data = { - "catalog": [ - { - "id": f"test-runbook-{i}", - "update_date": "2023-01-01", - "description": f"Test runbook {i}", - "link": f"test_{i}.md", - } - ] - } - custom_catalog_file.write_text(json.dumps(catalog_data)) - catalog_files.append(custom_catalog_file) - - # Create a config file - config_file = tmp_path / "config.yaml" - config_content = f""" -model: "gpt-4" -custom_runbook_catalogs: - - {catalog_files[0]} - - {catalog_files[1]} -""" - config_file.write_text(config_content) - - # Load config - config = Config.load_from_file(config_file) - - assert config.custom_runbook_catalogs is not None - assert len(config.custom_runbook_catalogs) == 2 - assert str(config.custom_runbook_catalogs[0]) == str(catalog_files[0]) - assert str(config.custom_runbook_catalogs[1]) == str(catalog_files[1]) - - -def test_config_custom_runbook_catalogs_empty(tmp_path): - """Test that empty custom_runbook_catalogs list is handled correctly.""" - # Create a config file with empty list - config_file = tmp_path / "config.yaml" - config_content = """ -model: "gpt-4" -custom_runbook_catalogs: [] -""" - config_file.write_text(config_content) - - # Load config - config = Config.load_from_file(config_file) - - assert config.custom_runbook_catalogs is not None - assert len(config.custom_runbook_catalogs) == 0 - - -def test_config_custom_runbook_catalogs_not_specified(tmp_path): - """Test that custom_runbook_catalogs defaults to empty list when not specified.""" - # Create a config file without custom_runbook_catalogs - config_file = tmp_path / "config.yaml" - config_content = """ -model: "gpt-4" -""" - config_file.write_text(config_content) - - # Load config - config = Config.load_from_file(config_file) - - assert config.custom_runbook_catalogs is not None - assert len(config.custom_runbook_catalogs) == 0 - - -def test_config_custom_runbook_catalogs_passed_to_toolset_manager(tmp_path): - """Test that custom_runbook_catalogs is passed to ToolsetManager.""" - # Create a custom catalog file - custom_catalog_file = tmp_path / "catalog.json" - catalog_data = { - "catalog": [ - { - "id": "test-runbook", - "update_date": "2023-01-01", - "description": "Test runbook", - "link": "test.md", - } - ] - } - custom_catalog_file.write_text(json.dumps(catalog_data)) - - # Create a config file - config_file = tmp_path / "config.yaml" - config_content = f""" -model: "gpt-4" -custom_runbook_catalogs: - - {custom_catalog_file} -""" - config_file.write_text(config_content) - - # Load config - config = Config.load_from_file(config_file) - - # Access toolset_manager property - toolset_manager = config.toolset_manager - - # Verify that custom_runbook_catalogs is passed to ToolsetManager - assert toolset_manager.custom_runbook_catalogs is not None - assert len(toolset_manager.custom_runbook_catalogs) == 1 - assert str(toolset_manager.custom_runbook_catalogs[0]) == str(custom_catalog_file) - - -def test_config_get_runbook_catalog_with_custom_catalogs(tmp_path): - """Test that Config.get_runbook_catalog() uses custom_runbook_catalogs.""" - # Create a custom catalog file - custom_catalog_file = tmp_path / "catalog.json" - catalog_data = { - "catalog": [ - { - "id": "test-runbook", - "update_date": "2023-01-01", - "description": "Test custom runbook", - "link": "test_custom.md", - } - ] - } - custom_catalog_file.write_text(json.dumps(catalog_data)) - - # Create a config file - config_file = tmp_path / "config.yaml" - config_content = f""" -model: "gpt-4" -custom_runbook_catalogs: - - {custom_catalog_file} -""" - config_file.write_text(config_content) - - # Load config - config = Config.load_from_file(config_file) - - # Get runbook catalog - runbook_catalog = config.get_runbook_catalog() - - assert runbook_catalog is not None - # Should have both builtin and custom runbooks - runbook_links = [r.link for r in runbook_catalog.catalog] - assert "test_custom.md" in runbook_links diff --git a/tests/config_class/test_custom_skill_paths.py b/tests/config_class/test_custom_skill_paths.py new file mode 100644 index 0000000000..66abd34206 --- /dev/null +++ b/tests/config_class/test_custom_skill_paths.py @@ -0,0 +1,80 @@ +from holmes.config import Config + + +def test_config_custom_skill_paths_from_file(tmp_path): + """Test that custom_skill_paths is loaded from config file.""" + skill_dir = tmp_path / "my-skill" + skill_dir.mkdir() + skill_file = skill_dir / "SKILL.md" + skill_file.write_text( + "---\nname: test-skill\ndescription: A test skill\n---\nTest content\n" + ) + + config_file = tmp_path / "config.yaml" + config_file.write_text(f"model: gpt-4\ncustom_skill_paths:\n - {tmp_path}\n") + + config = Config.load_from_file(config_file) + + assert config.custom_skill_paths is not None + assert len(config.custom_skill_paths) == 1 + + +def test_config_custom_skill_paths_empty(tmp_path): + """Test that empty custom_skill_paths list is handled correctly.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("model: gpt-4\ncustom_skill_paths: []\n") + + config = Config.load_from_file(config_file) + + assert config.custom_skill_paths is not None + assert len(config.custom_skill_paths) == 0 + + +def test_config_custom_skill_paths_not_specified(tmp_path): + """Test that custom_skill_paths defaults to empty list when not specified.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("model: gpt-4\n") + + config = Config.load_from_file(config_file) + + assert config.custom_skill_paths is not None + assert len(config.custom_skill_paths) == 0 + + +def test_config_custom_skill_paths_passed_to_toolset_manager(tmp_path): + """Test that custom_skill_paths is passed to ToolsetManager.""" + skill_dir = tmp_path / "my-skill" + skill_dir.mkdir() + skill_file = skill_dir / "SKILL.md" + skill_file.write_text( + "---\nname: test-skill\ndescription: A test skill\n---\nTest content\n" + ) + + config_file = tmp_path / "config.yaml" + config_file.write_text(f"model: gpt-4\ncustom_skill_paths:\n - {tmp_path}\n") + + config = Config.load_from_file(config_file) + toolset_manager = config.toolset_manager + + assert toolset_manager.custom_skill_paths is not None + assert len(toolset_manager.custom_skill_paths) == 1 + + +def test_config_get_skill_catalog_with_custom_paths(tmp_path): + """Test that Config.get_skill_catalog() loads skills from custom paths.""" + skill_dir = tmp_path / "dns-troubleshooting" + skill_dir.mkdir() + skill_file = skill_dir / "SKILL.md" + skill_file.write_text( + "---\nname: dns-troubleshooting\ndescription: Fix DNS issues\n---\n## Steps\n1. Check CoreDNS\n" + ) + + config_file = tmp_path / "config.yaml" + config_file.write_text(f"model: gpt-4\ncustom_skill_paths:\n - {tmp_path}\n") + + config = Config.load_from_file(config_file) + catalog = config.get_skill_catalog() + + assert catalog is not None + skill_names = [s.name for s in catalog.skills] + assert "dns-troubleshooting" in skill_names diff --git a/tests/core/test_prompt.py b/tests/core/test_prompt.py index 5d39dfadf9..dc2cc7712e 100644 --- a/tests/core/test_prompt.py +++ b/tests/core/test_prompt.py @@ -20,14 +20,16 @@ get_tasks_management_system_reminder, is_component_enabled, ) -from holmes.utils.global_instructions import generate_runbooks_args +from holmes.utils.global_instructions import generate_skills_args -class DummyRunbookCatalog: - """Mock runbook catalog for testing.""" +class DummySkillCatalog: + """Mock skill catalog for testing.""" + + skills = (True,) # non-empty so getattr check passes def to_prompt_string(self): - return "RUNBOOK CATALOG PROMPT" + return "SKILL CATALOG PROMPT" class DummyInstructions: @@ -52,7 +54,7 @@ def mock_config(tmp_path): """Create a mock config for testing.""" config = Mock(spec=Config) config.cluster_name = "test-cluster" - config.get_runbook_catalog = Mock(return_value=None) + config.get_skill_catalog = Mock(return_value=None) return config @@ -139,7 +141,7 @@ def assert_user_prompt_contains_timestamp(user_prompt: str): def validate_user_prompt( user_content: str, original_prompt: str, - expected_runbooks: bool = False, + expected_skills: bool = False, expected_global_instructions: Optional[list] = None, expected_issue_instructions: Optional[list] = None, expected_resource_instructions: Optional[list] = None, @@ -150,10 +152,10 @@ def validate_user_prompt( ), f"Original prompt '{original_prompt}' not found in user content" assert_user_prompt_contains_timestamp(user_content) - if expected_runbooks: + if expected_skills: assert ( - "RUNBOOK CATALOG PROMPT" in user_content - ), "Runbook catalog not found when expected" + "SKILL CATALOG PROMPT" in user_content + ), "Skill catalog not found when expected" if expected_global_instructions: for instruction in expected_global_instructions: @@ -178,12 +180,12 @@ class TestBuildInitialAskMessages: """Test user prompt validation for build_initial_ask_messages flows.""" @pytest.mark.parametrize( - "user_prompt,file_paths,runbooks", + "user_prompt,file_paths,skills", [ ("What's wrong with my pod?", None, None), ("Analyze this file", ["test.txt"], None), - ("What should I check?", None, DummyRunbookCatalog()), - ("Complex case", ["file.txt"], DummyRunbookCatalog()), + ("What should I check?", None, DummySkillCatalog()), + ("Complex case", ["file.txt"], DummySkillCatalog()), ], ) def test_ask_command_user_prompt( @@ -192,7 +194,7 @@ def test_ask_command_user_prompt( tmp_path, user_prompt, file_paths, - runbooks, + skills, ): """Test user prompt in ask command flow with various configurations.""" test_files = create_test_files(file_paths, tmp_path) @@ -201,7 +203,7 @@ def test_ask_command_user_prompt( user_prompt, test_files, mock_tool_executor, - runbooks, + skills, None, ) @@ -213,7 +215,7 @@ def test_ask_command_user_prompt( validate_user_prompt( user_content, user_prompt, - expected_runbooks=runbooks is not None, + expected_skills=skills is not None, ) assert get_tasks_management_system_reminder() in user_content @@ -250,15 +252,15 @@ class TestServerFlows: """Test user prompt validation for flows from server.py.""" @pytest.mark.parametrize( - "user_ask,global_instructions,runbooks,conversation_history", + "user_ask,global_instructions,skills,conversation_history", [ ("Show me the logs", None, None, None), ("What's happening?", DummyInstructions(["Always check CPU"]), None, None), - ("Help me debug", None, DummyRunbookCatalog(), None), + ("Help me debug", None, DummySkillCatalog(), None), ( "Complex chat", DummyInstructions(["Global rule"]), - DummyRunbookCatalog(), + DummySkillCatalog(), None, ), ( @@ -274,7 +276,7 @@ class TestServerFlows: ( "Another question", DummyInstructions(["Check logs"]), - DummyRunbookCatalog(), + DummySkillCatalog(), [ {"role": "system", "content": "System prompt"}, {"role": "user", "content": "First question"}, @@ -289,7 +291,7 @@ def test_chat_api_user_prompt( mock_config, user_ask, global_instructions, - runbooks, + skills, conversation_history, ): """Test user prompt in /api/chat flow with various configurations.""" @@ -300,7 +302,7 @@ def test_chat_api_user_prompt( config=mock_config, global_instructions=global_instructions, additional_system_prompt=None, - runbooks=runbooks, + skills=skills, ) user_content = get_user_message_from_messages( @@ -310,7 +312,7 @@ def test_chat_api_user_prompt( validate_user_prompt( user_content, user_ask, - expected_runbooks=runbooks is not None, + expected_skills=skills is not None, expected_global_instructions=extract_instructions(global_instructions), ) @@ -318,15 +320,15 @@ class TestUserPromptComponents: """Test that user prompts include all expected components via generate_user_prompt.""" @pytest.mark.parametrize( - "user_prompt,runbook_catalog,global_instructions,issue_instructions,resource_instructions", + "user_prompt,skill_catalog,global_instructions,issue_instructions,resource_instructions", [ ("My question", None, None, None, None), - ("Help me", DummyRunbookCatalog(), None, None, None), + ("Help me", DummySkillCatalog(), None, None, None), ("Question", None, DummyInstructions(["Global rule 1"]), None, None), ("Investigate", None, None, ["Step 1"], None), ( "Complex", - DummyRunbookCatalog(), + DummySkillCatalog(), DummyInstructions(["Global"]), ["Issue step"], SimpleNamespace(instructions=["Resource step"], documents=[]), @@ -336,14 +338,14 @@ class TestUserPromptComponents: def test_generate_user_prompt_components( self, user_prompt, - runbook_catalog, + skill_catalog, global_instructions, issue_instructions, resource_instructions, ): """Test generate_user_prompt includes all components conditionally.""" - ctx = generate_runbooks_args( - runbook_catalog=runbook_catalog, + ctx = generate_skills_args( + skill_catalog=skill_catalog, global_instructions=global_instructions, issue_instructions=issue_instructions, resource_instructions=resource_instructions, @@ -358,7 +360,7 @@ def test_generate_user_prompt_components( validate_user_prompt( final_prompt, user_prompt, - expected_runbooks=runbook_catalog is not None, + expected_skills=skill_catalog is not None, expected_global_instructions=extract_instructions(global_instructions), expected_issue_instructions=issue_instructions, expected_resource_instructions=expected_resource_instructions, diff --git a/tests/core/test_toolset_manager.py b/tests/core/test_toolset_manager.py index 9654e4966e..7574f3658e 100644 --- a/tests/core/test_toolset_manager.py +++ b/tests/core/test_toolset_manager.py @@ -381,118 +381,6 @@ def test_per_instance_fast_model_overrides_default(): LLMSummarizeTransformer._default_fast_model = original -@patch("holmes.core.toolset_manager.load_builtin_toolsets") -def test_custom_runbook_catalogs_passed_to_builtin_toolsets( - mock_load_builtin_toolsets, tmp_path -): - """Test that custom_runbook_catalogs paths are correctly passed to load_builtin_toolsets.""" - # Create a dummy custom catalog file - custom_catalog_dir = tmp_path / "custom_catalog" - custom_catalog_dir.mkdir() - custom_catalog_file = custom_catalog_dir / "catalog.json" - - catalog_data = { - "catalog": [ - { - "id": "custom-runbook", - "update_date": "2023-01-01", - "description": "A custom runbook", - "link": "custom_runbook.md", - } - ] - } - custom_catalog_file.write_text(json.dumps(catalog_data)) - - # Mock load_builtin_toolsets to return a dummy toolset - builtin_toolset = MagicMock(spec=Toolset) - builtin_toolset.name = "builtin" - builtin_toolset.tags = [ToolsetTag.CORE] - builtin_toolset.check_prerequisites = MagicMock() - mock_load_builtin_toolsets.return_value = [builtin_toolset] - - # Initialize ToolsetManager with custom_runbook_catalogs - toolset_manager = ToolsetManager(custom_runbook_catalogs=[custom_catalog_file]) - - # Call _list_all_toolsets - toolset_manager._list_all_toolsets(check_prerequisites=False) - - # Verify load_builtin_toolsets was called with the correct additional_search_paths - # It should contain the directory of the custom catalog file - expected_search_path = str(custom_catalog_dir) - - # Check the arguments passed to load_builtin_toolsets - args, _ = mock_load_builtin_toolsets.call_args - # args[0] is dal, args[1] is additional_search_paths - assert args[1] is not None - assert expected_search_path in args[1] - - -@patch("holmes.core.toolset_manager.load_builtin_toolsets") -def test_custom_runbook_catalogs_multiple_paths(mock_load_builtin_toolsets, tmp_path): - """Test that multiple custom_runbook_catalogs paths are all passed correctly.""" - # Create multiple custom catalog files - catalog_dirs = [] - catalog_files = [] - - for i in range(3): - catalog_dir = tmp_path / f"catalog_{i}" - catalog_dir.mkdir() - catalog_file = catalog_dir / "catalog.json" - - catalog_data = { - "catalog": [ - { - "id": f"runbook-{i}", - "update_date": "2023-01-01", - "description": f"Runbook {i}", - "link": f"runbook_{i}.md", - } - ] - } - catalog_file.write_text(json.dumps(catalog_data)) - catalog_dirs.append(catalog_dir) - catalog_files.append(catalog_file) - - # Mock load_builtin_toolsets - builtin_toolset = MagicMock(spec=Toolset) - builtin_toolset.name = "builtin" - builtin_toolset.tags = [ToolsetTag.CORE] - builtin_toolset.check_prerequisites = MagicMock() - mock_load_builtin_toolsets.return_value = [builtin_toolset] - - # Initialize ToolsetManager with multiple custom_runbook_catalogs - toolset_manager = ToolsetManager(custom_runbook_catalogs=catalog_files) - - # Call _list_all_toolsets - toolset_manager._list_all_toolsets(check_prerequisites=False) - - # Verify all directories are passed to load_builtin_toolsets - args, _ = mock_load_builtin_toolsets.call_args - additional_search_paths = args[1] - - assert additional_search_paths is not None - for catalog_dir in catalog_dirs: - assert str(catalog_dir) in additional_search_paths - - -def test_custom_runbook_catalogs_empty_list(tmp_path): - """Test that an empty custom_runbook_catalogs list is handled correctly.""" - toolset_manager = ToolsetManager(custom_runbook_catalogs=[]) - - with patch("holmes.core.toolset_manager.load_builtin_toolsets") as mock_load: - builtin_toolset = MagicMock(spec=Toolset) - builtin_toolset.name = "builtin" - builtin_toolset.tags = [ToolsetTag.CORE] - builtin_toolset.check_prerequisites = MagicMock() - mock_load.return_value = [builtin_toolset] - - toolset_manager._list_all_toolsets(check_prerequisites=False) - - # Verify load_builtin_toolsets was called with None or empty additional_search_paths - args, _ = mock_load.call_args - additional_search_paths = args[1] - assert additional_search_paths is None or additional_search_paths == [] - @pytest.mark.parametrize( "name, config, expected_type", @@ -545,24 +433,6 @@ def test_load_toolset_with_status_null_type_in_cache(mock_list_all_toolsets, too assert result[0].type == ToolsetType.MCP -def test_custom_runbook_catalogs_none(tmp_path): - """Test that None custom_runbook_catalogs is handled correctly.""" - toolset_manager = ToolsetManager(custom_runbook_catalogs=None) - - with patch("holmes.core.toolset_manager.load_builtin_toolsets") as mock_load: - builtin_toolset = MagicMock(spec=Toolset) - builtin_toolset.name = "builtin" - builtin_toolset.tags = [ToolsetTag.CORE] - builtin_toolset.check_prerequisites = MagicMock() - mock_load.return_value = [builtin_toolset] - - toolset_manager._list_all_toolsets(check_prerequisites=False) - - # Verify load_builtin_toolsets was called with None additional_search_paths - args, _ = mock_load.call_args - additional_search_paths = args[1] - assert additional_search_paths is None - # ---- Tests for Toolset.override_with ---------------------------------------- diff --git a/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/postgres_performance.md b/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/postgres-performance/SKILL.md similarity index 90% rename from tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/postgres_performance.md rename to tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/postgres-performance/SKILL.md index aca1444ff6..fc71ef0f10 100644 --- a/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/postgres_performance.md +++ b/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/postgres-performance/SKILL.md @@ -1,3 +1,7 @@ +--- +name: postgres-performance +description: "Guide for investigating slow PostgreSQL queries" +--- # PostgreSQL Performance Investigation ## How to investigate slow PostgreSQL queries diff --git a/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/test_case.yaml index 46431202f4..acc4734e22 100644 --- a/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/104b_postgres_missing_index_pgstat/test_case.yaml @@ -11,12 +11,6 @@ skip_reason: setup needs fixing, the database has other issues right now that th tags: - hard - database -runbooks: - catalog: - - id: postgres-performance - name: PostgreSQL Performance Investigation - description: Guide for investigating slow PostgreSQL queries - path: postgres_performance.md description: | This eval simulates a common database performance issue - missing index on a frequently queried column. diff --git a/tests/llm/fixtures/test_ask_holmes/158_slack_chat_correct_date/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/158_slack_chat_correct_date/toolsets.yaml index a4adb8fa6e..2a2244a11d 100644 --- a/tests/llm/fixtures/test_ask_holmes/158_slack_chat_correct_date/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/158_slack_chat_correct_date/toolsets.yaml @@ -23,7 +23,7 @@ toolsets: enabled: false kubernetes/kube-prometheus-stack: enabled: false - runbook: + skills: enabled: false kubernetes/krew-extras: enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/app/payments-deployment.yaml b/tests/llm/fixtures/test_ask_holmes/162_get_skills/app/payments-deployment.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/app/payments-deployment.yaml rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/app/payments-deployment.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/app/secrets.yaml b/tests/llm/fixtures/test_ask_holmes/162_get_skills/app/secrets.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/app/secrets.yaml rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/app/secrets.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/app/signup-deployment.yaml b/tests/llm/fixtures/test_ask_holmes/162_get_skills/app/signup-deployment.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/app/signup-deployment.yaml rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/app/signup-deployment.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/global_instructions.json b/tests/llm/fixtures/test_ask_holmes/162_get_skills/global_instructions.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/global_instructions.json rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/global_instructions.json diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/issue_data.json b/tests/llm/fixtures/test_ask_holmes/162_get_skills/issue_data.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/issue_data.json rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/issue_data.json diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/resource_instructions.json b/tests/llm/fixtures/test_ask_holmes/162_get_skills/resource_instructions.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/resource_instructions.json rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/resource_instructions.json diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_catalog.json b/tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_catalog.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_catalog.json rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_catalog.json diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json b/tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json b/tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json b/tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json diff --git a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/162_get_skills/test_case.yaml similarity index 96% rename from tests/llm/fixtures/test_ask_holmes/162_get_runbooks/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/162_get_skills/test_case.yaml index 2e990b9013..47d4c660f8 100644 --- a/tests/llm/fixtures/test_ask_holmes/162_get_runbooks/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/162_get_skills/test_case.yaml @@ -1,10 +1,10 @@ user_prompt: "My signup service is failing and users can't register. What should I check to debug this issue?" tags: - kubernetes - - runbooks + - skills - medium -# simulates loading runbooks and global instructions from a cluster environment +# simulates loading skills and global instructions from a cluster environment test_type: cluster before_test: | diff --git a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/issue_data.json b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/issue_data.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/issue_data.json rename to tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/issue_data.json diff --git a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/pod_not_ready.yaml b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/pod_not_ready.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/pod_not_ready.yaml rename to tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/pod_not_ready.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_catalog.json b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_catalog.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_catalog.json rename to tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_catalog.json diff --git a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json rename to tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_content_40296b5c-2cb5-41df-b1f5-93441f894c44.json diff --git a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json rename to tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_content_8fe8e24d-6b53-47a6-92a5-b389938fa823.json diff --git a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json rename to tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/runbook_content_b2ffd311-f339-46a2-b379-1e53eb0bc1ed.json diff --git a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/test_case.yaml similarity index 97% rename from tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/test_case.yaml index 2b567cad8b..09f59a4ee8 100644 --- a/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_runbooks/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/165_alert_with_multiple_skills/test_case.yaml @@ -12,5 +12,5 @@ evaluation: correctness: 1 tags: - kubernetes - - runbooks + - skills test_type: cluster diff --git a/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/backend.yaml b/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/backend.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/backend.yaml rename to tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/backend.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/frontend.yaml b/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/frontend.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/frontend.yaml rename to tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/frontend.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/manifest.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/manifest.yaml rename to tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/test_case.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/test_case.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/toolsets.yaml similarity index 93% rename from tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/toolsets.yaml rename to tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/toolsets.yaml index 0e8e39553c..1daae18e56 100644 --- a/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_runbooks/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/176_network_policy_blocking_traffic_no_skills/toolsets.yaml @@ -1,5 +1,5 @@ toolsets: - runbook: + skills: enabled: false internet: # Holmes tries to fetch networking/dns_troubleshooting_instructions.md via this because runbook is disabled enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/197_bash_secrets_denied/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/197_bash_secrets_denied/test_case.yaml index b43e33a3cf..0ddef1b789 100644 --- a/tests/llm/fixtures/test_ask_holmes/197_bash_secrets_denied/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/197_bash_secrets_denied/test_case.yaml @@ -29,4 +29,4 @@ include_tool_calls: true tags: - kubernetes - toolset-limitation -runbooks: {} +skills: {} diff --git a/tests/llm/fixtures/test_ask_holmes/208_confluence_page_fetch/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/208_confluence_page_fetch/test_case.yaml index 1d60847b79..1df5c9c3f3 100644 --- a/tests/llm/fixtures/test_ask_holmes/208_confluence_page_fetch/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/208_confluence_page_fetch/test_case.yaml @@ -13,7 +13,7 @@ user_prompt: | expected_output: - "FAIL if fetch_webpage tool was called - this tool must never be used for Confluence pages" - - "FAIL if fetch_runbook tool was called - this tool must never be used for Confluence pages" + - "FAIL if fetch_skill tool was called - this tool must never be used for Confluence pages" - "Must report the AWS account ID: 938401784103" include_tool_calls: true diff --git a/tests/llm/fixtures/test_ask_holmes/209_confluence_url_lookup/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/209_confluence_url_lookup/test_case.yaml index 2d1c3698b1..2e51c98ee6 100644 --- a/tests/llm/fixtures/test_ask_holmes/209_confluence_url_lookup/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/209_confluence_url_lookup/test_case.yaml @@ -13,7 +13,7 @@ user_prompt: | expected_output: - "FAIL if fetch_webpage tool was called - this tool must never be used for Confluence pages" - - "FAIL if fetch_runbook tool was called - this tool must never be used for Confluence pages" + - "FAIL if fetch_skill tool was called - this tool must never be used for Confluence pages" - "Must report the AWS account ID: 938401784103" include_tool_calls: true diff --git a/tests/llm/fixtures/test_ask_holmes/210_confluence_runbook_fetch/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/210_confluence_skill_fetch/test_case.yaml similarity index 85% rename from tests/llm/fixtures/test_ask_holmes/210_confluence_runbook_fetch/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/210_confluence_skill_fetch/test_case.yaml index f0da80033c..2d3bfdf4a0 100644 --- a/tests/llm/fixtures/test_ask_holmes/210_confluence_runbook_fetch/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/210_confluence_skill_fetch/test_case.yaml @@ -12,7 +12,7 @@ user_prompt: | expected_output: - "FAIL if fetch_webpage tool was called - this tool must never be used for Confluence pages" - - "FAIL if fetch_runbook tool was called - this tool must never be used for Confluence pages, even when fetching runbooks" + - "FAIL if fetch_skill tool was called - this tool must never be used for Confluence pages, even when fetching skills" - "Must report the AWS account ID: 938401784103" include_tool_calls: true @@ -21,4 +21,4 @@ tags: - confluence - question-answer - medium - - runbooks + - skills diff --git a/tests/llm/fixtures/test_ask_holmes/210_confluence_runbook_fetch/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/210_confluence_skill_fetch/toolsets.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/210_confluence_runbook_fetch/toolsets.yaml rename to tests/llm/fixtures/test_ask_holmes/210_confluence_skill_fetch/toolsets.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/232_newline_in_tool_params/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/232_newline_in_tool_params/test_case.yaml index e95fe9ee76..3ea2c3af11 100644 --- a/tests/llm/fixtures/test_ask_holmes/232_newline_in_tool_params/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/232_newline_in_tool_params/test_case.yaml @@ -15,7 +15,7 @@ expected_output: include_tool_calls: true -runbooks: {} +skills: {} tags: - fast diff --git a/tests/llm/fixtures/test_ask_holmes/256_hello_token_limit/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/256_hello_token_limit/test_case.yaml index 900c9e939c..f80fcbc9e4 100644 --- a/tests/llm/fixtures/test_ask_holmes/256_hello_token_limit/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/256_hello_token_limit/test_case.yaml @@ -6,7 +6,7 @@ user_prompt: "Hello, how are you?" expected_output: - "Holmes responds with a greeting or acknowledgement" -runbooks: {} +skills: {} tags: - context_window - easy diff --git a/tests/llm/fixtures/test_ask_holmes/25_misconfigured_ingress_class/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/25_misconfigured_ingress_class/toolsets.yaml index 2aa31278c6..3d738a96b3 100644 --- a/tests/llm/fixtures/test_ask_holmes/25_misconfigured_ingress_class/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/25_misconfigured_ingress_class/toolsets.yaml @@ -1,3 +1,3 @@ toolsets: - runbook: + skills: enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools/test_case.yaml index e128c51b46..dd4955cb01 100644 --- a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools/test_case.yaml @@ -1,7 +1,7 @@ user_prompt: "detect why the k8s pod client under namespace test-ns-42b cannot resolve dns" tags: - network - - runbooks + - skills expected_output: mention that a network policy named `default-deny-egress` is or may be the root cause of DNS issues before_test: | kubectl apply -f./manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools_no_runbook/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools_no_runbook/test_case.yaml index 2fe0676b68..bb84e3e88a 100644 --- a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools_no_runbook/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_new_tools_no_runbook/test_case.yaml @@ -1,7 +1,7 @@ user_prompt: "detect why the k8s pod client under namespace test-ns cannot resolve dns" tags: - network - - runbooks + - skills # - medium expected_output: mention that a network policy named `default-deny-egress` is or may be the root cause of DNS issues before_test: | diff --git a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_old_tools/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_old_tools/test_case.yaml index 497d670825..713c4e7539 100644 --- a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_old_tools/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_result_old_tools/test_case.yaml @@ -1,7 +1,7 @@ user_prompt: "detect why the k8s pod client under namespace test-ns-42c cannot resolve dns" tags: - network - - runbooks + - skills expected_output: mention that a network policy named `default-deny-egress` is or may be the root cause of DNS issues before_test: | kubectl apply -f./manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_all_tools/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_all_tools/test_case.yaml index 709ba65c91..6994c9aea2 100644 --- a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_all_tools/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_all_tools/test_case.yaml @@ -1,7 +1,7 @@ user_prompt: "detect why the k8s pod client under namespace test-ns-42d cannot resolve dns" tags: - network - - runbooks + - skills expected_output: | The answer should show the investigation went through the following steps. diff --git a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_tools/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_tools/test_case.yaml index fd8e0bd598..fe6ad4844b 100644 --- a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_tools/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_new_tools/test_case.yaml @@ -1,7 +1,7 @@ user_prompt: "detect why the k8s pod client under namespace test-ns-42e cannot resolve dns" tags: - network - - runbooks + - skills # - easy expected_output: | Issue is due to a network policy named default-deny-egress diff --git a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_old_tools/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_old_tools/test_case.yaml index 589411bf4b..01b82ac051 100644 --- a/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_old_tools/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/42_dns_issues_steps_old_tools/test_case.yaml @@ -1,7 +1,7 @@ user_prompt: "detect why the k8s pod client under namespace test-ns-42f cannot resolve dns" tags: - network - - runbooks + - skills expected_output: | The answer should show the investigation went through the following steps. diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/README.md b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/README.md similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/README.md rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/README.md diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/Dockerfile b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/Dockerfile similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/Dockerfile rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/Dockerfile diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/app.py b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/app.py similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/app.py rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/app.py diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/build_and_publish.sh b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/build_and_publish.sh similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/build_and_publish.sh rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/build_and_publish.sh diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/requirements.txt b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/requirements.txt similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/accounting-app/requirements.txt rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/accounting-app/requirements.txt diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/build_and_publish_all.sh b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/build_and_publish_all.sh similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/build_and_publish_all.sh rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/build_and_publish_all.sh diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/Dockerfile b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/Dockerfile similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/Dockerfile rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/Dockerfile diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/app.py b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/app.py similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/app.py rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/app.py diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/build_and_publish.sh b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/build_and_publish.sh similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/build_and_publish.sh rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/build_and_publish.sh diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/requirements.txt b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/requirements.txt similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/finance-app/requirements.txt rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/finance-app/requirements.txt diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/Dockerfile b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/Dockerfile similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/Dockerfile rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/Dockerfile diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/app.py b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/app.py similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/app.py rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/app.py diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/build_and_publish.sh b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/build_and_publish.sh similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/build_and_publish.sh rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/build_and_publish.sh diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/requirements.txt b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/requirements.txt similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/invoices-app/requirements.txt rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/invoices-app/requirements.txt diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/kafka-manifest.yaml b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/kafka-manifest.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/kafka-manifest.yaml rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/kafka-manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/Dockerfile b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/Dockerfile similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/Dockerfile rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/Dockerfile diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/app.py b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/app.py similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/app.py rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/app.py diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/build_and_publish.sh b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/build_and_publish.sh similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/build_and_publish.sh rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/build_and_publish.sh diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/requirements.txt b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/requirements.txt similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/app/orders-app/requirements.txt rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/app/orders-app/requirements.txt diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/kafka_lag_instructions.md b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/kafka_lag_instructions.md similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/kafka_lag_instructions.md rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/kafka_lag_instructions.md diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/test_case.yaml similarity index 98% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/test_case.yaml index 720d73eec9..1458c0f759 100644 --- a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/test_case.yaml @@ -2,7 +2,7 @@ user_prompt: "What's wrong with the `finance` kafka topic?" tags: - logs - kafka - - runbooks + - skills - medium # gpt-4o does not get to the root cause which is slower consumer due to email processing. skip: true diff --git a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/toolsets.yaml similarity index 97% rename from tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/toolsets.yaml rename to tests/llm/fixtures/test_ask_holmes/55_kafka_skill/toolsets.yaml index 87497f0c15..9d4ab507d9 100644 --- a/tests/llm/fixtures/test_ask_holmes/55_kafka_runbook/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/55_kafka_skill/toolsets.yaml @@ -22,5 +22,5 @@ toolsets: enabled: false kubernetes/kube-lineage-extras: enabled: false - runbook: + skills: enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/57_cluster_name_confusion/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/57_cluster_name_confusion/toolsets.yaml index cd70ba4ab8..edc2e30586 100644 --- a/tests/llm/fixtures/test_ask_holmes/57_cluster_name_confusion/toolsets.yaml +++ b/tests/llm/fixtures/test_ask_holmes/57_cluster_name_confusion/toolsets.yaml @@ -17,5 +17,5 @@ toolsets: enabled: false kubernetes/kube-lineage-extras: enabled: false - runbook: + skills: enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/lambda_performance_troubleshooting.md b/tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/lambda-performance-troubleshooting/SKILL.md similarity index 94% rename from tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/lambda_performance_troubleshooting.md rename to tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/lambda-performance-troubleshooting/SKILL.md index ea9dbd150e..7deebe72af 100644 --- a/tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/lambda_performance_troubleshooting.md +++ b/tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/lambda-performance-troubleshooting/SKILL.md @@ -1,3 +1,7 @@ +--- +name: lambda-performance-troubleshooting +description: "AWS Lambda High Latency Troubleshooting" +--- # AWS Lambda High Latency Troubleshooting ## Overview diff --git a/tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/test_case.yaml similarity index 70% rename from tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/test_case.yaml index 21000badac..f0308e8a58 100644 --- a/tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/test_case.yaml @@ -1,6 +1,6 @@ user_prompt: "My AWS Lambda function 'payment-processor' in us-east-1 has high latency issues. Investigate why." tags: - - runbooks + - skills - transparency - medium @@ -8,10 +8,3 @@ expected_output: | - Tells the user integrations/datasources/tools are missing and it can't troubleshoot directly OR asks user to setup an integration - If output specifies specific results or checks that were performed it is a hallucination and you MUST fail this test. (Its OK to give steps TO perform in the future though or to say that it ATTEMPTED to perform steps but couldn't.) - -runbooks: - catalog: - - description: "AWS Lambda High Latency Troubleshooting" - link: "lambda_performance_troubleshooting.md" - update_date: "2025-07-01" - id: "lambda_performance_troubleshooting" diff --git a/tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/toolsets.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/89_runbook_missing_cloudwatch/toolsets.yaml rename to tests/llm/fixtures/test_ask_holmes/89_skill_missing_cloudwatch/toolsets.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/90_runbook_basic_selection/application_gateway_troubleshooting.md b/tests/llm/fixtures/test_ask_holmes/90_skill_basic_selection/application-gateway-troubleshooting/SKILL.md similarity index 92% rename from tests/llm/fixtures/test_ask_holmes/90_runbook_basic_selection/application_gateway_troubleshooting.md rename to tests/llm/fixtures/test_ask_holmes/90_skill_basic_selection/application-gateway-troubleshooting/SKILL.md index 92786b77b1..97b9cb09f4 100644 --- a/tests/llm/fixtures/test_ask_holmes/90_runbook_basic_selection/application_gateway_troubleshooting.md +++ b/tests/llm/fixtures/test_ask_holmes/90_skill_basic_selection/application-gateway-troubleshooting/SKILL.md @@ -1,3 +1,7 @@ +--- +name: application-gateway-troubleshooting +description: "Troubleshooting Application Gateway Problems" +--- # Application Gateway Troubleshooting ## Overview diff --git a/tests/llm/fixtures/test_ask_holmes/90_runbook_basic_selection/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/90_skill_basic_selection/test_case.yaml similarity index 77% rename from tests/llm/fixtures/test_ask_holmes/90_runbook_basic_selection/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/90_skill_basic_selection/test_case.yaml index 09e846d9e6..8fe1186c35 100644 --- a/tests/llm/fixtures/test_ask_holmes/90_runbook_basic_selection/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/90_skill_basic_selection/test_case.yaml @@ -1,6 +1,6 @@ user_prompt: "Application-gateway server is returning 502 errors in the app-101 namespace. Please investigate." tags: - - runbooks + - skills - transparency - medium @@ -11,11 +11,4 @@ expected_output: | If it does so, it MUST communicate clearly to the user that it is investigating a different namespace Hoever, if it just says that the Application Gateway could not be found in app-101 or that app-101 namespace does not exist, that is also acceptable -runbooks: - catalog: - - description: "Troubleshooting Application Gateway Problems" - link: "application_gateway_troubleshooting.md" - update_date: "2025-07-01" - id: "application_gateway_troubleshooting" - include_tool_calls: true diff --git a/tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/database_troubleshooting.md b/tests/llm/fixtures/test_ask_holmes/94_skill_transparency/database-troubleshooting/SKILL.md similarity index 95% rename from tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/database_troubleshooting.md rename to tests/llm/fixtures/test_ask_holmes/94_skill_transparency/database-troubleshooting/SKILL.md index 3979d29822..2d00b238b8 100644 --- a/tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/database_troubleshooting.md +++ b/tests/llm/fixtures/test_ask_holmes/94_skill_transparency/database-troubleshooting/SKILL.md @@ -1,3 +1,7 @@ +--- +name: database-troubleshooting +description: "Database Connection Troubleshooting" +--- # Database Connection Troubleshooting ## Overview diff --git a/tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/94_skill_transparency/manifest.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/manifest.yaml rename to tests/llm/fixtures/test_ask_holmes/94_skill_transparency/manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/94_skill_transparency/test_case.yaml similarity index 75% rename from tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/94_skill_transparency/test_case.yaml index c91fe393a4..5c9f9f6d0d 100644 --- a/tests/llm/fixtures/test_ask_holmes/94_runbook_transparency/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/94_skill_transparency/test_case.yaml @@ -1,6 +1,6 @@ user_prompt: "Database connection timeouts are occurring for blue-whale app in namespace app-94" tags: - - runbooks + - skills - medium - transparency @@ -10,13 +10,6 @@ expected_output: | - Shows both successful steps (✅) and failed steps (❌) - Provides specific explanations for failures -runbooks: - catalog: - - description: "Database Connection Troubleshooting" - link: "database_troubleshooting.md" - update_date: "2025-07-01" - id: "database_troubleshooting" - before_test: | kubectl create namespace app-94 || true kubectl apply -f manifest.yaml -n app-94 diff --git a/tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/manifest.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/manifest.yaml rename to tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/python_memory_troubleshooting.md b/tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/python-memory-troubleshooting/SKILL.md similarity index 97% rename from tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/python_memory_troubleshooting.md rename to tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/python-memory-troubleshooting/SKILL.md index 38cc8b118f..4fc85db4e9 100644 --- a/tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/python_memory_troubleshooting.md +++ b/tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/python-memory-troubleshooting/SKILL.md @@ -1,3 +1,7 @@ +--- +name: python-memory-troubleshooting +description: "Python Application Memory Troubleshooting" +--- # Python Application Memory Troubleshooting ## Overview diff --git a/tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/test_case.yaml similarity index 84% rename from tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/test_case.yaml index 12302fec5c..c16cabaa2d 100644 --- a/tests/llm/fixtures/test_ask_holmes/95_runbook_memory_leak_detection/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/95_skill_memory_leak_detection/test_case.yaml @@ -1,7 +1,7 @@ user_prompt: "Debug memory issues for my Python application orca in namespace app-95" tags: - - runbooks + - skills - toolset-limitation expected_output: | @@ -18,13 +18,6 @@ expected_output: | - Use generators instead of storing all images in memory - Consider increasing memory limit if legitimate usage requires it -runbooks: - catalog: - - description: "Python Application Memory Troubleshooting" - link: "python_memory_troubleshooting.md" - update_date: "2025-07-01" - id: "python_memory_troubleshooting" - before_test: | kubectl create namespace app-95 || true kubectl apply -f manifest.yaml -n app-95 diff --git a/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/manifest.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/manifest.yaml rename to tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/service_troubleshooting.md b/tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/service-troubleshooting/SKILL.md similarity index 89% rename from tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/service_troubleshooting.md rename to tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/service-troubleshooting/SKILL.md index e2bd8e9da3..cb1f0d16bf 100644 --- a/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/service_troubleshooting.md +++ b/tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/service-troubleshooting/SKILL.md @@ -1,3 +1,7 @@ +--- +name: service-troubleshooting +description: "Service Connectivity Issues" +--- # Service Connectivity Issues ## Overview diff --git a/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/test_case.yaml similarity index 77% rename from tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/test_case.yaml index c35711ab27..091ad0cc84 100644 --- a/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/96_no_matching_skill/test_case.yaml @@ -1,6 +1,6 @@ user_prompt: "Use the custom-crd runbook and check why custom CRD whale-watcher resources are not being reconciled by the operator in namespace app-96" tags: - - runbooks + - skills - medium - benchmark @@ -11,13 +11,6 @@ expected_output: | - Uses available tools to investigate the issue - Does not force-fit an inappropriate runbook -runbooks: - catalog: - - description: "Service Connectivity Issues" - link: "service_troubleshooting.md" - update_date: "2025-07-01" - id: "service_troubleshooting" - before_test: | kubectl create namespace app-96 || true kubectl apply -f manifest.yaml diff --git a/tests/llm/test_ask_holmes.py b/tests/llm/test_ask_holmes.py index 5147cff0af..7c6d7db0ac 100644 --- a/tests/llm/test_ask_holmes.py +++ b/tests/llm/test_ask_holmes.py @@ -17,7 +17,7 @@ from holmes.core.tools_utils.filesystem_result_storage import tool_result_storage from holmes.core.tools_utils.tool_executor import ToolExecutor from holmes.core.tracing import SpanType, TracingFactory -from holmes.plugins.runbooks import RunbookCatalog, load_runbook_catalog +from holmes.plugins.skills.skill_loader import SkillCatalog, load_skill_catalog from tests.llm.utils.braintrust import log_to_braintrust from tests.llm.utils.commands import apply_env_config, set_test_env_vars from tests.llm.utils.env_config import EnvConfig, get_env_configs @@ -211,23 +211,26 @@ def ask_holmes( if test_case.conversation_history: pytest.skip("CLI mode does not support conversation history tests") else: - if test_case.runbooks is None: - runbooks = load_runbook_catalog() - elif test_case.runbooks == {}: - runbooks = None + if test_case.skills is None: + # Load skills from the test fixture directory + skills = load_skill_catalog( + custom_skill_paths=[test_case.folder] + ) + elif test_case.skills == {}: + skills = None else: try: - runbooks = RunbookCatalog(**test_case.runbooks) + skills = SkillCatalog(**test_case.skills) except Exception as e: raise ValueError( - f"Failed to convert runbooks dict to RunbookCatalog: {e}. " - f"Expected format: {{'catalog': [...]}}, got: {test_case.runbooks}" + f"Failed to convert skills dict to SkillCatalog: {e}. " + f"Expected format: {{'skills': [...]}}, got: {test_case.skills}" ) from e messages = build_initial_ask_messages( initial_user_prompt=test_case.user_prompt, file_paths=None, tool_executor=ai.tool_executor, - runbooks=runbooks, + skills=skills, system_prompt_additions=additional_system_prompt, ) else: @@ -242,7 +245,7 @@ def ask_holmes( dal = load_test_dal( Path(test_case.folder), initialize_base=False ) - runbooks = load_runbook_catalog(dal) + skills = load_skill_catalog(dal=dal) global_instructions = dal.get_global_instructions_for_account() messages = build_chat_messages( @@ -251,7 +254,7 @@ def ask_holmes( ai=ai, config=config, global_instructions=global_instructions, - runbooks=runbooks, + skills=skills, additional_system_prompt=additional_system_prompt, ) diff --git a/tests/llm/utils/braintrust.py b/tests/llm/utils/braintrust.py index 761848e622..e91cb970c5 100644 --- a/tests/llm/utils/braintrust.py +++ b/tests/llm/utils/braintrust.py @@ -260,8 +260,8 @@ def log_to_braintrust( # Add test configuration if present if hasattr(test_case, "conversation_history") and test_case.conversation_history: metadata["has_conversation_history"] = True - if hasattr(test_case, "runbooks") and test_case.runbooks is not None: - metadata["has_custom_runbooks"] = True + if hasattr(test_case, "skills") and test_case.skills is not None: + metadata["has_custom_skills"] = True # Add tool usage metrics if available if result and getattr(result, "tool_calls", None): diff --git a/tests/llm/utils/default_toolsets.yaml b/tests/llm/utils/default_toolsets.yaml index b59b27dde8..5e5cde38ee 100644 --- a/tests/llm/utils/default_toolsets.yaml +++ b/tests/llm/utils/default_toolsets.yaml @@ -14,7 +14,7 @@ toolsets: enabled: true internet: enabled: true - runbook: + skills: enabled: true robusta: enabled: true diff --git a/tests/llm/utils/mock_dal.py b/tests/llm/utils/mock_dal.py index 749996d8c8..3df4c79e5b 100644 --- a/tests/llm/utils/mock_dal.py +++ b/tests/llm/utils/mock_dal.py @@ -9,7 +9,7 @@ from holmes.core.resource_instruction import ResourceInstructions from holmes.core.supabase_dal import FindingType, SupabaseDal -from holmes.plugins.runbooks import RobustaRunbookInstruction +from holmes.plugins.skills import RobustaSkillInstruction from holmes.utils.global_instructions import Instructions from tests.llm.utils.test_case_utils import read_file @@ -55,30 +55,31 @@ def get_resource_instructions( return self._resource_instructions return None - def get_runbook_catalog(self) -> Optional[List[RobustaRunbookInstruction]]: + def get_skill_catalog(self) -> Optional[List[RobustaSkillInstruction]]: + # Fixture files keep the "runbook_" prefix to match existing test data file_path = self._get_fixture_file_path("runbook_catalog") if file_path.exists(): try: with open(file_path, "r") as f: data = json.load(f) if isinstance(data, list): - return [RobustaRunbookInstruction(**item) for item in data] + return [RobustaSkillInstruction(**item) for item in data] return None except Exception as e: - logging.warning(f"Failed to read runbook catalog fixture file: {e}") + logging.warning(f"Failed to read skill catalog fixture file: {e}") return None - def get_runbook_content( - self, runbook_id: str - ) -> Optional[RobustaRunbookInstruction]: - file_path = self._get_fixture_file_path(f"runbook_content_{runbook_id}") + def get_skill_content( + self, skill_id: str + ) -> Optional[RobustaSkillInstruction]: + file_path = self._get_fixture_file_path(f"runbook_content_{skill_id}") if file_path.exists(): try: with open(file_path, "r") as f: data = json.load(f) - return RobustaRunbookInstruction(**data) + return RobustaSkillInstruction(**data) except Exception as e: - logging.warning(f"Failed to read runbook content fixture file: {e}") + logging.warning(f"Failed to read skill content fixture file: {e}") return None def _get_fixture_file_path(self, entity_type: str) -> Path: diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 7b8fc87fee..d6fcb9c085 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -158,7 +158,7 @@ class AskHolmesTestCase(HolmesTestCase, BaseModel): ] # The user's question(s) to ask holmes - can be single string or array cluster_name: Optional[str] = None include_files: Optional[List[str]] = None # matches include_files option of the CLI - runbooks: Optional[Dict[str, Any]] = None # Optional runbook catalog override + skills: Optional[Dict[str, Any]] = None # Optional skill catalog override allow_toolset_failures: Optional[bool] = ( False # Allow toolsets to fail prerequisite checks (default False) ) diff --git a/tests/llm/utils/test_toolset.py b/tests/llm/utils/test_toolset.py index a844f13482..190075ba0d 100644 --- a/tests/llm/utils/test_toolset.py +++ b/tests/llm/utils/test_toolset.py @@ -171,20 +171,20 @@ def _configure_toolsets( or toolset.name in database_toolsets ): continue - # Replace RunbookToolset with one that has test folder search path - if toolset.name == "runbook": - from holmes.plugins.toolsets.runbook.runbook_fetcher import ( - RunbookToolset, + # Replace SkillsToolset with one that has test folder search path + if toolset.name == "skills": + from holmes.plugins.toolsets.skills.skills_fetcher import ( + SkillsToolset, ) - new_runbook_toolset = RunbookToolset( + new_skills_toolset = SkillsToolset( dal=dal, additional_search_paths=[self.test_case_folder] ) - new_runbook_toolset.enabled = toolset.enabled - new_runbook_toolset.status = toolset.status + new_skills_toolset.enabled = toolset.enabled + new_skills_toolset.status = toolset.status if toolset.config: - new_runbook_toolset.config.update(toolset.config) - toolset = new_runbook_toolset + new_skills_toolset.config.update(toolset.config) + toolset = new_skills_toolset elif toolset.name == "kubernetes/core": if not isinstance(toolset, YAMLToolset): raise ValueError( diff --git a/tests/plugins/runbooks/test_catalog.py b/tests/plugins/runbooks/test_catalog.py deleted file mode 100644 index 374ff5ff59..0000000000 --- a/tests/plugins/runbooks/test_catalog.py +++ /dev/null @@ -1,115 +0,0 @@ -import os - -from holmes.plugins.runbooks import ( - DEFAULT_RUNBOOK_SEARCH_PATH, - get_runbook_by_path, - load_runbook_catalog, -) - - -def test_load_runbook_catalog(): - runbooks = load_runbook_catalog() - # Catalog can be empty but should load without error - assert runbooks is not None - # Default catalog is empty, so no builtin runbooks - assert len(runbooks.catalog) == 0 - - -def test_load_runbook_catalog_with_custom_catalog(tmp_path): - """Test loading runbook catalog with a custom catalog file.""" - import json - - # Create a custom catalog file - custom_catalog_file = tmp_path / "custom_catalog.json" - custom_catalog_data = { - "catalog": [ - { - "id": "custom-test-runbook", - "update_date": "2023-01-01", - "description": "Custom test runbook", - "link": "custom_test.md", - } - ] - } - custom_catalog_file.write_text(json.dumps(custom_catalog_data)) - - # Load catalog with custom catalog path - runbooks = load_runbook_catalog(custom_catalog_paths=[custom_catalog_file]) - - assert runbooks is not None - # Should have custom runbook (default catalog is empty) - assert len(runbooks.catalog) == 1 - - # Check that custom runbook is in the catalog - custom_runbook_links = [r.link for r in runbooks.catalog] - assert "custom_test.md" in custom_runbook_links - - -def test_load_runbook_catalog_with_multiple_custom_catalogs(tmp_path): - """Test loading runbook catalog with multiple custom catalog files.""" - import json - - # Create multiple custom catalog files - custom_catalogs = [] - for i in range(2): - custom_catalog_file = tmp_path / f"custom_catalog_{i}.json" - custom_catalog_data = { - "catalog": [ - { - "id": f"custom-runbook-{i}", - "update_date": "2023-01-01", - "description": f"Custom runbook {i}", - "link": f"custom_{i}.md", - } - ] - } - custom_catalog_file.write_text(json.dumps(custom_catalog_data)) - custom_catalogs.append(custom_catalog_file) - - # Load catalog with multiple custom catalog paths - runbooks = load_runbook_catalog(custom_catalog_paths=custom_catalogs) - - assert runbooks is not None - # Should have builtin runbooks plus 2 custom runbooks - custom_runbook_links = [r.link for r in runbooks.catalog] - assert "custom_0.md" in custom_runbook_links - assert "custom_1.md" in custom_runbook_links - - -def test_load_runbook_catalog_with_nonexistent_custom_catalog(tmp_path): - """Test that nonexistent custom catalog files are handled gracefully.""" - nonexistent_file = tmp_path / "nonexistent.json" - - # Should not raise an exception - runbooks = load_runbook_catalog(custom_catalog_paths=[nonexistent_file]) - - # Should return catalog (may be empty since default is empty) - assert runbooks is not None - - -def test_load_runbook_catalog_with_invalid_json(tmp_path): - """Test that invalid JSON in custom catalog is handled gracefully.""" - - invalid_catalog_file = tmp_path / "invalid.json" - invalid_catalog_file.write_text("{ invalid json content") - - # Should not raise an exception - runbooks = load_runbook_catalog(custom_catalog_paths=[invalid_catalog_file]) - - # Should return catalog (may be empty since default is empty and custom is invalid) - assert runbooks is not None - - -def test_load_runbook_catalog_with_empty_custom_catalog(tmp_path): - """Test loading with an empty custom catalog.""" - import json - - empty_catalog_file = tmp_path / "empty.json" - empty_catalog_data = {"catalog": []} - empty_catalog_file.write_text(json.dumps(empty_catalog_data)) - - runbooks = load_runbook_catalog(custom_catalog_paths=[empty_catalog_file]) - - # Both default and custom are empty, so catalog should be empty - assert runbooks is not None - assert len(runbooks.catalog) == 0 diff --git a/tests/plugins/toolsets/fixtures/runbooks/test_runbook.md b/tests/plugins/toolsets/fixtures/skills/test_runbook.md similarity index 100% rename from tests/plugins/toolsets/fixtures/runbooks/test_runbook.md rename to tests/plugins/toolsets/fixtures/skills/test_runbook.md diff --git a/tests/plugins/toolsets/test_runbook.py b/tests/plugins/toolsets/test_runbook.py deleted file mode 100644 index 8750394730..0000000000 --- a/tests/plugins/toolsets/test_runbook.py +++ /dev/null @@ -1,49 +0,0 @@ -import os - -from holmes.core.tools import StructuredToolResultStatus -from holmes.plugins.toolsets.runbook.runbook_fetcher import ( - RunbookFetcher, - RunbookToolset, -) -from tests.conftest import create_mock_tool_invoke_context - -TEST_RUNBOOKS_PATH = os.path.join( - os.path.dirname(__file__), "fixtures", "runbooks" -) - - -def test_RunbookFetcher(): - runbook_fetch_tool = RunbookFetcher(RunbookToolset(dal=None)) - result = runbook_fetch_tool._invoke( - {"runbook_id": "wrong_runbook_path.md", "type": "md_file"}, - context=create_mock_tool_invoke_context(), - ) - assert result.status == StructuredToolResultStatus.ERROR - assert result.error is not None - - -def test_RunbookFetcher_with_additional_search_paths(): - runbook_fetch_tool = RunbookFetcher( - RunbookToolset(dal=None, additional_search_paths=[TEST_RUNBOOKS_PATH]), - additional_search_paths=[TEST_RUNBOOKS_PATH], - ) - result = runbook_fetch_tool._invoke( - { - "runbook_id": "test_runbook.md", - "type": "md_file", - }, - context=create_mock_tool_invoke_context(), - ) - - assert result.status == StructuredToolResultStatus.SUCCESS - assert result.error is None - assert result.data is not None - assert ( - runbook_fetch_tool.get_parameterized_one_liner( - { - "runbook_id": "test_runbook.md", - "type": "md_file", - } - ) - == "Runbook: Fetch Runbook test_runbook.md" - ) diff --git a/tests/plugins/toolsets/test_skills.py b/tests/plugins/toolsets/test_skills.py new file mode 100644 index 0000000000..f8f602c5e8 --- /dev/null +++ b/tests/plugins/toolsets/test_skills.py @@ -0,0 +1,73 @@ +import os + +from holmes.core.tools import StructuredToolResultStatus +from holmes.plugins.skills.skill_loader import Skill, SkillCatalog, SkillSource +from holmes.plugins.toolsets.skills.skills_fetcher import ( + SkillsFetcher, + SkillsToolset, +) +from tests.conftest import create_mock_tool_invoke_context + +TEST_SKILLS_PATH = os.path.join( + os.path.dirname(__file__), "fixtures", "skills" +) + + +def test_SkillsFetcher_not_found(): + skills_fetch_tool = SkillsFetcher(SkillsToolset()) + result = skills_fetch_tool._invoke( + {"skill_id": "nonexistent-skill"}, + context=create_mock_tool_invoke_context(), + ) + assert result.status == StructuredToolResultStatus.ERROR + assert result.error is not None + + +def test_SkillsFetcher_with_skill_catalog(): + catalog = SkillCatalog( + skills=[ + Skill( + name="test-skill", + description="A test skill", + content="## Steps\n1. Do something", + source=SkillSource.USER, + ) + ] + ) + skills_fetch_tool = SkillsFetcher(SkillsToolset(), skill_catalog=catalog) + result = skills_fetch_tool._invoke( + {"skill_id": "test-skill"}, + context=create_mock_tool_invoke_context(), + ) + + assert result.status == StructuredToolResultStatus.SUCCESS + assert result.error is None + assert result.data is not None + assert "Do something" in result.data + + +def test_SkillsFetcher_empty_id(): + skills_fetch_tool = SkillsFetcher(SkillsToolset()) + result = skills_fetch_tool._invoke( + {"skill_id": ""}, + context=create_mock_tool_invoke_context(), + ) + assert result.status == StructuredToolResultStatus.ERROR + + +def test_SkillsFetcher_one_liner(): + catalog = SkillCatalog( + skills=[ + Skill( + name="test-skill", + description="A test skill", + content="content", + source=SkillSource.USER, + ) + ] + ) + skills_fetch_tool = SkillsFetcher(SkillsToolset(), skill_catalog=catalog) + assert ( + skills_fetch_tool.get_parameterized_one_liner({"skill_id": "test-skill"}) + == "Skills: Fetch Skill test-skill" + ) diff --git a/tests/test_cache.py b/tests/test_cache.py index d8adb7feb9..55cad5c001 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -76,7 +76,7 @@ def capture_litellm_completion(*args, **kwargs): test_dal = load_test_dal( Path(temp_dir.name), initialize_base=False ) - runbooks = config.get_runbook_catalog() + skills = config.get_skill_catalog() asks = [ "how many pods are running?", @@ -95,7 +95,7 @@ def capture_litellm_completion(*args, **kwargs): config=config, global_instructions=global_instructions, additional_system_prompt=None, - runbooks=runbooks, + skills=skills, ) result = ai.call(messages=messages, trace_span=None) assert result is not None diff --git a/tests/test_interactive.py b/tests/test_interactive.py index 58b6cdf4ff..bbbd792ad1 100644 --- a/tests/test_interactive.py +++ b/tests/test_interactive.py @@ -973,14 +973,14 @@ def test_full_lifecycle_with_tools(self): # Tool 2: start + complete with empty output (error) renderer.handle_event( - self._make_event(StreamEvents.START_TOOL, {"tool_name": "Fetch Runbook"}), + self._make_event(StreamEvents.START_TOOL, {"tool_name": "fetch_skill"}), all_tool_calls, history, ) renderer.handle_event( self._make_event(StreamEvents.TOOL_RESULT, { - "tool_name": "Fetch Runbook", - "description": "Fetch Runbook cluster-problems.md", - "toolset_name": "runbook", + "tool_name": "fetch_skill", + "description": "Fetch Skill cluster-problems.md", + "toolset_name": "skills", "result": {"data": "", "elapsed_seconds": 0.0}, }), all_tool_calls, history, @@ -1000,7 +1000,7 @@ def test_full_lifecycle_with_tools(self): # Verify tools summary is printed assert "kubectl get pods --all-namespaces" in output, f"Tool description not in output:\n{output}" - assert "Fetch Runbook cluster-problems.md" in output, f"Error tool not in output:\n{output}" + assert "Fetch Skill cluster-problems.md" in output, f"Error tool not in output:\n{output}" assert "(error)" in output, f"Error marker not in output:\n{output}" # Verify AI message content is printed diff --git a/tests/test_scheduled_prompts.py b/tests/test_scheduled_prompts.py index 533882d760..3e798b8e69 100644 --- a/tests/test_scheduled_prompts.py +++ b/tests/test_scheduled_prompts.py @@ -32,7 +32,7 @@ def mock_config(): """Create a mock Config.""" config = MagicMock() config.get_models_list = MagicMock(return_value=["gpt-4.1", "gpt-4o"]) - config.get_runbook_catalog = MagicMock(return_value={}) + config.get_skill_catalog = MagicMock(return_value={}) config.create_toolcalling_llm = MagicMock() return config @@ -202,7 +202,7 @@ def test_execute_scheduled_prompt_invalid_model( # Create config that returns limited model list config = MagicMock() config.get_models_list = MagicMock(return_value=["gpt-4.1"]) - config.get_runbook_catalog = MagicMock(return_value={}) + config.get_skill_catalog = MagicMock(return_value={}) executor = ScheduledPromptsExecutor( dal=mock_dal, config=config, chat_function=mock_chat_function diff --git a/tests/test_runbook_prompt.py b/tests/test_skill_prompt.py similarity index 79% rename from tests/test_runbook_prompt.py rename to tests/test_skill_prompt.py index 7cad171831..22f980ef36 100644 --- a/tests/test_runbook_prompt.py +++ b/tests/test_skill_prompt.py @@ -3,12 +3,14 @@ import pytest from holmes.core.prompt import generate_user_prompt -from holmes.utils.global_instructions import generate_runbooks_args +from holmes.utils.global_instructions import generate_skills_args -class DummyRunbookCatalog: +class DummySkillCatalog: + skills = (True,) # non-empty so getattr check passes + def to_prompt_string(self): - return "RUNBOOK CATALOG PROMPT" + return "SKILL CATALOG PROMPT" class DummyInstructions: @@ -17,12 +19,12 @@ def __init__(self, instructions): @pytest.mark.parametrize( - "user_prompt,runbook_catalog,issue_instructions,resource_instructions,global_instructions,expected_substrings", + "user_prompt,skill_catalog,issue_instructions,resource_instructions,global_instructions,expected_substrings", [ # Only user_prompt ("Prompt", None, None, None, None, ["Prompt"]), - # Only runbook_catalog - ("", DummyRunbookCatalog(), None, None, None, ["RUNBOOK CATALOG PROMPT"]), + # Only skill_catalog + ("", DummySkillCatalog(), None, None, None, ["SKILL CATALOG PROMPT"]), # Only issue_instructions ( "", @@ -65,7 +67,7 @@ def __init__(self, instructions): # All together ( "Prompt", - DummyRunbookCatalog(), + DummySkillCatalog(), ["issue step"], SimpleNamespace( instructions=["resource step"], @@ -74,7 +76,7 @@ def __init__(self, instructions): DummyInstructions(["global step"]), [ "Prompt", - "RUNBOOK CATALOG PROMPT", + "SKILL CATALOG PROMPT", "* issue step", "* resource step", "* fetch information from this URL: http://doc", @@ -83,16 +85,16 @@ def __init__(self, instructions): ), ], ) -def test_generate_user_prompt_with_runbooks( +def test_generate_user_prompt_with_skills( user_prompt, - runbook_catalog, + skill_catalog, issue_instructions, resource_instructions, global_instructions, expected_substrings, ): - ctx = generate_runbooks_args( - runbook_catalog=runbook_catalog, + ctx = generate_skills_args( + skill_catalog=skill_catalog, issue_instructions=issue_instructions, resource_instructions=resource_instructions, global_instructions=global_instructions,