diff --git a/docs/data-sources/builtin-toolsets/.nav.yml b/docs/data-sources/builtin-toolsets/.nav.yml index d109428e31..4f60e3341e 100644 --- a/docs/data-sources/builtin-toolsets/.nav.yml +++ b/docs/data-sources/builtin-toolsets/.nav.yml @@ -13,6 +13,7 @@ nav: - Datetime: datetime.md - Docker: docker.md - GitHub: github.md + - Grafana Dashboards: grafanadashboards.md - Loki: grafanaloki.md - Tempo: grafanatempo.md - Helm: helm.md diff --git a/docs/data-sources/builtin-toolsets/grafanadashboards.md b/docs/data-sources/builtin-toolsets/grafanadashboards.md new file mode 100644 index 0000000000..8749325caa --- /dev/null +++ b/docs/data-sources/builtin-toolsets/grafanadashboards.md @@ -0,0 +1,88 @@ +# Grafana Dashboards + +Connect HolmesGPT to Grafana for dashboard analysis, query extraction, and understanding your monitoring setup. This integration enables investigation of dashboard configurations and extraction of Prometheus queries for deeper analysis. + +## Prerequisites + +A [Grafana service account token](https://grafana.com/docs/grafana/latest/administration/service-accounts/) with the following permissions: + +- Basic role → Viewer + +## Configuration + +=== "Holmes CLI" + + Add the following to **~/.holmes/config.yaml**. Create the file if it doesn't exist: + + ```yaml + toolsets: + grafana/dashboards: + enabled: true + config: + api_key: + url: # e.g. https://acme-corp.grafana.net or http://localhost:3000 + # Optional: Custom health check endpoint (defaults to api/health) + # healthcheck: api/health + # Optional: Additional headers for all requests + # headers: + # X-Custom-Header: "custom-value" + ``` + + --8<-- "snippets/toolset_refresh_warning.md" + + To test, run: + + ```bash + holmes ask "Show me all dashboards tagged with 'kubernetes'" + ``` + +=== "Robusta Helm Chart" + + ```yaml + holmes: + toolsets: + grafana/dashboards: + enabled: true + config: + api_key: + url: # e.g. https://acme-corp.grafana.net + # Optional: Additional headers for all requests + # headers: + # X-Custom-Header: "custom-value" + ``` + +## Capabilities + +| Tool Name | Description | +|-----------|-------------| +| grafana_search_dashboards | Search for dashboards and folders by query, tags, UIDs, or folder locations | +| grafana_get_dashboard_by_uid | Retrieve complete dashboard JSON including all panels and queries | +| grafana_get_home_dashboard | Get the home dashboard configuration | +| grafana_get_dashboard_tags | List all tags used across dashboards for categorization | + +## How it Works + +### Dashboard Query Extraction + +When HolmesGPT retrieves a dashboard, it can extract and analyze Prometheus queries from dashboard panels. This is particularly useful for: + +- Understanding what metrics a dashboard monitors +- Extracting queries for further investigation with the Prometheus toolset +- Analyzing dashboard time ranges and variable usage + +### Example Usage + +**Finding dashboards by tag:** +```bash +holmes ask "Find all dashboards tagged with 'production' or 'kubernetes'" +``` + +**Analyzing a specific dashboard:** +```bash +holmes ask "Show me what metrics the 'Node Exporter' dashboard monitors" +``` + +**Extracting queries for investigation:** +```bash +holmes ask "Get the CPU usage queries from the Kubernetes cluster dashboard and check if any nodes are throttling" +``` diff --git a/docs/data-sources/builtin-toolsets/index.md b/docs/data-sources/builtin-toolsets/index.md index e0ef49b22b..4be1bb68bd 100644 --- a/docs/data-sources/builtin-toolsets/index.md +++ b/docs/data-sources/builtin-toolsets/index.md @@ -19,6 +19,7 @@ HolmesGPT includes pre-built integrations for popular monitoring and observabili - [:material-clock:{ .lg .middle } **Datetime**](datetime.md) - [:simple-docker:{ .lg .middle } **Docker**](docker.md) - [:material-github:{ .lg .middle } **GitHub**](github.md) +- [:simple-grafana:{ .lg .middle } **Grafana Dashboards**](grafanadashboards.md) - [:material-package:{ .lg .middle } **Helm**](helm.md) - [:material-web:{ .lg .middle } **Internet**](internet.md) - [:simple-apachekafka:{ .lg .middle } **Kafka**](kafka.md) diff --git a/holmes/core/tools.py b/holmes/core/tools.py index 47d4652f65..be19c8fd11 100644 --- a/holmes/core/tools.py +++ b/holmes/core/tools.py @@ -767,6 +767,16 @@ def _load_llm_instructions(self, jinja_template: str): context={"tool_names": tool_names, "config": self.config}, ) + def _load_llm_instructions_from_file(self, file_dir: str, filename: str) -> None: + """Helper method to load LLM instructions from a jinja2 template file. + + Args: + file_dir: Directory where the template file is located (typically os.path.dirname(__file__)) + filename: Name of the jinja2 template file (e.g., "toolset_grafana_dashboard.jinja2") + """ + template_file_path = os.path.abspath(os.path.join(file_dir, filename)) + self._load_llm_instructions(jinja_template=f"file://{template_file_path}") + class YAMLToolset(Toolset): tools: List[YAMLTool] # type: ignore diff --git a/holmes/plugins/toolsets/grafana/common.py b/holmes/plugins/toolsets/grafana/common.py index 483712e415..185381970e 100644 --- a/holmes/plugins/toolsets/grafana/common.py +++ b/holmes/plugins/toolsets/grafana/common.py @@ -8,7 +8,7 @@ class GrafanaConfig(BaseModel): """A config that represents one of the Grafana related tools like Loki or Tempo - If `grafana_datasource_uid` is set, then it is assume that Holmes will proxy all + If `grafana_datasource_uid` is set, then it is assumed that Holmes will proxy all requests through grafana. In this case `url` should be the grafana URL. If `grafana_datasource_uid` is not set, it is assumed that the `url` is the systems' URL diff --git a/holmes/plugins/toolsets/grafana/toolset_grafana.py b/holmes/plugins/toolsets/grafana/toolset_grafana.py index 2de1336ca7..c6ee64f3fe 100644 --- a/holmes/plugins/toolsets/grafana/toolset_grafana.py +++ b/holmes/plugins/toolsets/grafana/toolset_grafana.py @@ -1,5 +1,7 @@ -from typing import Dict, List -from urllib.parse import urlencode, urljoin +import os +from typing import ClassVar, Dict, Optional, Type, cast +from urllib.parse import urljoin +from abc import ABC from holmes.core.tools import ( StructuredToolResult, Tool, @@ -9,118 +11,237 @@ ) from holmes.plugins.toolsets.grafana.base_grafana_toolset import BaseGrafanaToolset import requests # type: ignore -import logging +from holmes.plugins.toolsets.grafana.common import ( + get_base_url, + GrafanaConfig, + build_headers, +) from holmes.plugins.toolsets.utils import toolset_name_for_one_liner -class ListAndBuildGrafanaDashboardURLs(Tool): - def __init__(self, toolset: BaseGrafanaToolset): +class GrafanaDashboardConfig(GrafanaConfig): + """Configuration specific to Grafana Dashboard toolset with api/health as default healthcheck""" + + healthcheck: Optional[str] = "api/health" + + +class GrafanaToolset(BaseGrafanaToolset): + config_class: ClassVar[Type[GrafanaDashboardConfig]] = GrafanaDashboardConfig + + def __init__(self): + super().__init__( + name="grafana/dashboards", + description="Provides tools for interacting with Grafana dashboards", + icon_url="https://w7.pngwing.com/pngs/434/923/png-transparent-grafana-hd-logo-thumbnail.png", + docs_url="https://holmesgpt.dev/data-sources/builtin-toolsets/grafanadashboards/", + tools=[ + SearchDashboards(self), + GetDashboardByUID(self), + GetHomeDashboard(self), + GetDashboardTags(self), + ], + ) + + self._load_llm_instructions_from_file( + os.path.dirname(__file__), "toolset_grafana_dashboard.jinja2" + ) + + @property + def grafana_config(self) -> GrafanaDashboardConfig: + return cast(GrafanaDashboardConfig, self._grafana_config) + + +class BaseGrafanaTool(Tool, ABC): + """Base class for Grafana tools with common HTTP request functionality.""" + + def __init__(self, toolset: GrafanaToolset, *args, **kwargs): + super().__init__(*args, **kwargs) + self._toolset = toolset + + def _make_grafana_request( + self, + endpoint: str, + params: dict, + query_params: Optional[Dict] = None, + timeout: int = 30, + ) -> StructuredToolResult: + """Make a GET request to Grafana API and return structured result. + + Args: + endpoint: API endpoint path (e.g., "/api/search") + params: Original parameters passed to the tool + query_params: Optional query parameters for the request + + Returns: + StructuredToolResult with the API response data + """ + url = urljoin(get_base_url(self._toolset.grafana_config), endpoint) + headers = build_headers( + api_key=self._toolset.grafana_config.api_key, + additional_headers=self._toolset.grafana_config.headers, + ) + + response = requests.get( + url, headers=headers, params=query_params, timeout=timeout + ) + response.raise_for_status() + data = response.json() + + return StructuredToolResult( + status=StructuredToolResultStatus.SUCCESS, + data=data, + url=url, + params=params, + ) + + +class SearchDashboards(BaseGrafanaTool): + def __init__(self, toolset: GrafanaToolset): super().__init__( - name="list_and_build_grafana_dashboard_urls", - description="Lists all available Grafana dashboard urls", + toolset=toolset, + name="grafana_search_dashboards", + description="Search for Grafana dashboards and folders using the /api/search endpoint", parameters={ - "cluster_name": ToolParameter( - description="The cluster name. Defaults to None.", + "query": ToolParameter( + description="Search text to filter dashboards", + type="string", + required=False, + ), + "tag": ToolParameter( + description="Search dashboards by tag", + type="string", + required=False, + ), + "type": ToolParameter( + description="Filter by type: 'dash-folder' or 'dash-db'", type="string", required=False, ), - "namespace": ToolParameter( - description="The namespace for filtering dashboards.", + "dashboardIds": ToolParameter( + description="List of dashboard IDs to filter (comma-separated)", type="string", required=False, ), - "node_name": ToolParameter( - description="The node name to filter for node-related dashboards.", + "dashboardUIDs": ToolParameter( + description="List of dashboard UIDs to search for (comma-separated)", type="string", required=False, ), - "pod_name": ToolParameter( - description="The pod name to filter dashboards.", + "folderUIDs": ToolParameter( + description="List of folder UIDs to search within (comma-separated)", type="string", required=False, ), + "starred": ToolParameter( + description="Return only starred dashboards", + type="boolean", + required=False, + ), + "limit": ToolParameter( + description="Maximum results (default 1000, max 5000)", + type="integer", + required=False, + ), + "page": ToolParameter( + description="Page number for pagination", + type="integer", + required=False, + ), }, ) - self._toolset = toolset def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: - url = urljoin( - self._toolset._grafana_config.url, "/api/search?query=&type=dash-db" + query_params = {} + if params.get("query"): + query_params["query"] = params["query"] + if params.get("tag"): + query_params["tag"] = params["tag"] + if params.get("type"): + query_params["type"] = params["type"] + if params.get("dashboardIds"): + # Check if dashboardIds also needs to be passed as multiple params + dashboard_ids = params["dashboardIds"].split(",") + query_params["dashboardIds"] = [ + dashboard_id.strip() + for dashboard_id in dashboard_ids + if dashboard_id.strip() + ] + if params.get("dashboardUIDs"): + # Handle dashboardUIDs as a list - split comma-separated values + dashboard_uids = params["dashboardUIDs"].split(",") + query_params["dashboardUIDs"] = [ + uid.strip() for uid in dashboard_uids if uid.strip() + ] + if params.get("folderUIDs"): + # Check if folderUIDs also needs to be passed as multiple params + folder_uids = params["folderUIDs"].split(",") + query_params["folderUIDs"] = [ + uid.strip() for uid in folder_uids if uid.strip() + ] + if params.get("starred") is not None: + query_params["starred"] = str(params["starred"]).lower() + if params.get("limit"): + query_params["limit"] = params["limit"] + if params.get("page"): + query_params["page"] = params["page"] + + return self._make_grafana_request("/api/search", params, query_params) + + def get_parameterized_one_liner(self, params: Dict) -> str: + return f"{toolset_name_for_one_liner(self._toolset.name)}: Search Dashboards" + + +class GetDashboardByUID(BaseGrafanaTool): + def __init__(self, toolset: GrafanaToolset): + super().__init__( + toolset=toolset, + name="grafana_get_dashboard_by_uid", + description="Get a dashboard by its UID using the /api/dashboards/uid/:uid endpoint", + parameters={ + "uid": ToolParameter( + description="The unique identifier of the dashboard", + type="string", + required=True, + ), + }, ) - headers = {"Authorization": f"Bearer {self._toolset._grafana_config.api_key}"} - - try: - response = requests.get(url, headers=headers) - response.raise_for_status() - dashboards = response.json() - formatted_dashboards: List[str] = [] - base_url = ( - self._toolset._grafana_config.external_url - or self._toolset._grafana_config.url - ) - for dash in dashboards: - dashboard_url = urljoin( - base_url, - f"/d/{dash['uid']}/{dash['uri'].split('/')[-1]}", - ) - - params_dict = { - "var-cluster": params.get("cluster_name", ""), - "var-namespace": params.get("namespace", ""), - "var-pod": params.get("pod_name", ""), - "var-node": params.get("node_name", ""), - "var-datasource": self._toolset._grafana_config.grafana_datasource_uid, - "refresh": "5s", - } - - # If filtering for nodes, ensure only node-related dashboards are included - if params.get("node_name") and "node" not in dash["title"].lower(): - continue - - # we add all params since if the dashboard isnt configured for a param it will ignore it if it is added - query_string = urlencode({k: v for k, v in params_dict.items() if v}) - dashboard_url = ( - f"{dashboard_url}?{query_string}" if query_string else dashboard_url - ) - - formatted_dashboards.append( - f"Title: {dash['title']}\nURL: {dashboard_url}\n" - ) - - return StructuredToolResult( - status=StructuredToolResultStatus.SUCCESS - if formatted_dashboards - else StructuredToolResultStatus.NO_DATA, - data="\n".join(formatted_dashboards) - if formatted_dashboards - else "No dashboards found.", - url=url, - params=params, - ) - except requests.RequestException as e: - logging.error(f"Error fetching dashboards: {str(e)}") - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=f"Error fetching dashboards: {str(e)}", - url=url, - params=params, - ) + + def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: + uid = params["uid"] + return self._make_grafana_request(f"/api/dashboards/uid/{uid}", params) def get_parameterized_one_liner(self, params: Dict) -> str: - return ( - f"{toolset_name_for_one_liner(self._toolset.name)}: List Grafana Dashboards" + return f"{toolset_name_for_one_liner(self._toolset.name)}: Get Dashboard {params.get('uid', '')}" + + +class GetHomeDashboard(BaseGrafanaTool): + def __init__(self, toolset: GrafanaToolset): + super().__init__( + toolset=toolset, + name="grafana_get_home_dashboard", + description="Get the home dashboard using the /api/dashboards/home endpoint", + parameters={}, ) + def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: + return self._make_grafana_request("/api/dashboards/home", params) -class GrafanaToolset(BaseGrafanaToolset): - def __init__(self): + def get_parameterized_one_liner(self, params: Dict) -> str: + return f"{toolset_name_for_one_liner(self._toolset.name)}: Get Home Dashboard" + + +class GetDashboardTags(BaseGrafanaTool): + def __init__(self, toolset: GrafanaToolset): super().__init__( - name="grafana/grafana", - description="Provides tools for interacting with Grafana dashboards", - icon_url="https://w7.pngwing.com/pngs/434/923/png-transparent-grafana-hd-logo-thumbnail.png", - docs_url="", - tools=[ - ListAndBuildGrafanaDashboardURLs(self), - ], + toolset=toolset, + name="grafana_get_dashboard_tags", + description="Get all tags used across dashboards using the /api/dashboards/tags endpoint", + parameters={}, ) + + def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: + return self._make_grafana_request("/api/dashboards/tags", params) + + def get_parameterized_one_liner(self, params: Dict) -> str: + return f"{toolset_name_for_one_liner(self._toolset.name)}: Get Dashboard Tags" diff --git a/holmes/plugins/toolsets/grafana/toolset_grafana_dashboard.jinja2 b/holmes/plugins/toolsets/grafana/toolset_grafana_dashboard.jinja2 new file mode 100644 index 0000000000..d69159751c --- /dev/null +++ b/holmes/plugins/toolsets/grafana/toolset_grafana_dashboard.jinja2 @@ -0,0 +1,27 @@ +--- + +## **Instruction: Prometheus Dashboard Query Execution** + +**When running Prometheus queries from Grafana dashboards:** + +### **Key Rules:** +1. **Always run as range queries** (not instant queries) to match dashboard behavior +2. **Use dashboard's time range** from JSON: `"time": {"from": "now-1h", "to": "now"}` +3. **Substitute variables:** + - `$__rate_interval` → `1m` (1-hour range) or `5m` (3+ hour range) + - `$cluster` → omit if empty +4. **Copy query exactly** including `ceil()`, `sort_desc()`, etc. + +### **Example:** +``` +Dashboard: "now-1h" + query with [$__rate_interval] +↓ +Execute: start="-3600" + query with [1m] +``` + +### **Ignore:** +- `"refresh": "10s"` (UI setting, not query-related) + +**This ensures you see the same time series data and historical spikes that users see on their dashboards.** + +---