From b24cdd99af82e534a79cbcc36d4021a3a797a326 Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 30 Dec 2025 17:03:06 +0200 Subject: [PATCH 1/4] Apply JSON filter mixin to OpenSearch status tools Signed-off-by: Codex --- .../builtin-toolsets/grafanadashboards.md | 10 ++ evals_per_branch.json | 5 + .../toolsets/grafana/toolset_grafana.py | 42 +++--- holmes/plugins/toolsets/json_filter_mixin.py | 122 ++++++++++++++++++ .../plugins/toolsets/opensearch/opensearch.py | 81 +++++++----- poetry.lock | 118 ++++++++++++++++- pyproject.toml | 2 + .../generate_dashboard.py | 82 ++++++++++++ .../grafana.yaml | 83 ++++++++++++ .../test_case.yaml | 67 ++++++++++ .../toolsets.yaml | 5 + .../grafana/test_dashboard_filters.py | 70 ++++++++++ 12 files changed, 627 insertions(+), 60 deletions(-) create mode 100644 evals_per_branch.json create mode 100644 holmes/plugins/toolsets/json_filter_mixin.py create mode 100644 tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/generate_dashboard.py create mode 100644 tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/grafana.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/test_case.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/toolsets.yaml create mode 100644 tests/plugins/toolsets/grafana/test_dashboard_filters.py diff --git a/docs/data-sources/builtin-toolsets/grafanadashboards.md b/docs/data-sources/builtin-toolsets/grafanadashboards.md index af4cfc2560..eda3897b0f 100644 --- a/docs/data-sources/builtin-toolsets/grafanadashboards.md +++ b/docs/data-sources/builtin-toolsets/grafanadashboards.md @@ -58,6 +58,16 @@ A [Grafana service account token](https://grafana.com/docs/grafana/latest/admini | grafana_get_home_dashboard | Get the home dashboard configuration | | grafana_get_dashboard_tags | List all tags used across dashboards for categorization | +### Controlling response size + +Large dashboards can produce very big JSON payloads. To keep tool outputs readable you can: + +- **Limit depth:** `max_depth: 1` returns only the first level of keys, replacing deeper structures with a truncated marker. +- **Filter with jsonpath:** `jsonpath: $.dashboard.panels[*].title` extracts just the panel titles. +- **Filter with jq:** `jq: .dashboard.panels[].id` extracts panel IDs using familiar jq syntax. + +Filters run before depth truncation, so you can combine them to precisely scope the response. + ## How it Works ### Dashboard Query Extraction diff --git a/evals_per_branch.json b/evals_per_branch.json new file mode 100644 index 0000000000..a9784653ba --- /dev/null +++ b/evals_per_branch.json @@ -0,0 +1,5 @@ +{ + "work": [ + "tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters" + ] +} diff --git a/holmes/plugins/toolsets/grafana/toolset_grafana.py b/holmes/plugins/toolsets/grafana/toolset_grafana.py index ba3813e8f3..06f1cc86d2 100644 --- a/holmes/plugins/toolsets/grafana/toolset_grafana.py +++ b/holmes/plugins/toolsets/grafana/toolset_grafana.py @@ -18,6 +18,7 @@ build_headers, ) from holmes.plugins.toolsets.utils import toolset_name_for_one_liner +from holmes.plugins.toolsets.json_filter_mixin import JsonFilterMixin class GrafanaDashboardConfig(GrafanaConfig): @@ -237,19 +238,21 @@ def get_parameterized_one_liner(self, params: Dict) -> str: return f"{toolset_name_for_one_liner(self._toolset.name)}: Search Dashboards" -class GetDashboardByUID(BaseGrafanaTool): +class GetDashboardByUID(JsonFilterMixin, BaseGrafanaTool): def __init__(self, toolset: GrafanaToolset): super().__init__( toolset=toolset, name="grafana_get_dashboard_by_uid", description="Get a dashboard by its UID using the /api/dashboards/uid/:uid endpoint", - parameters={ - "uid": ToolParameter( - description="The unique identifier of the dashboard", - type="string", - required=True, - ), - }, + parameters=self.extend_parameters( + { + "uid": ToolParameter( + description="The unique identifier of the dashboard", + type="string", + required=True, + ) + } + ), ) def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: @@ -260,29 +263,25 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes self._toolset.grafana_config, uid=uid ) - return StructuredToolResult( - status=result.status, - data=result.data, - params=result.params, - url=dashboard_url if dashboard_url else result.url, - ) + filtered_result = self.filter_result(result, params) + filtered_result.url = dashboard_url if dashboard_url else result.url + return filtered_result def get_parameterized_one_liner(self, params: Dict) -> str: return f"{toolset_name_for_one_liner(self._toolset.name)}: Get Dashboard {params.get('uid', '')}" -class GetHomeDashboard(BaseGrafanaTool): +class GetHomeDashboard(JsonFilterMixin, BaseGrafanaTool): def __init__(self, toolset: GrafanaToolset): super().__init__( toolset=toolset, name="grafana_get_home_dashboard", description="Get the home dashboard using the /api/dashboards/home endpoint", - parameters={}, + parameters=self.extend_parameters({}), ) def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: result = self._make_grafana_request("/api/dashboards/home", params) - config = self._toolset.grafana_config dashboard_url = None if isinstance(result.data, dict): @@ -290,12 +289,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes if uid: dashboard_url = _build_grafana_dashboard_url(config, uid=uid) - return StructuredToolResult( - status=result.status, - data=result.data, - params=result.params, - url=dashboard_url if dashboard_url else None, - ) + filtered_result = self.filter_result(result, params) + filtered_result.url = dashboard_url if dashboard_url else None + return filtered_result def get_parameterized_one_liner(self, params: Dict) -> str: return f"{toolset_name_for_one_liner(self._toolset.name)}: Get Home Dashboard" diff --git a/holmes/plugins/toolsets/json_filter_mixin.py b/holmes/plugins/toolsets/json_filter_mixin.py new file mode 100644 index 0000000000..b7f3c23af9 --- /dev/null +++ b/holmes/plugins/toolsets/json_filter_mixin.py @@ -0,0 +1,122 @@ +import json +import logging +from typing import Any, Dict, Optional, Tuple + +import jq # type: ignore +from jsonpath_ng import parse as jsonpath_parse # type: ignore + +from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus, ToolParameter + +logger = logging.getLogger(__name__) + + +def _truncate_to_depth(value: Any, max_depth: Optional[int], current_depth: int = 0): + """Recursively truncate dictionaries/lists beyond the requested depth.""" + if max_depth is None or max_depth < 0: + return value + + if current_depth >= max_depth: + if isinstance(value, (dict, list)): + return f"...truncated at depth {max_depth}" + return value + + if isinstance(value, dict): + return { + key: _truncate_to_depth(sub_value, max_depth, current_depth + 1) + for key, sub_value in value.items() + } + if isinstance(value, list): + return [ + _truncate_to_depth(item, max_depth, current_depth + 1) for item in value + ] + + return value + + +def _apply_jsonpath_filter(data: Any, expression: str) -> Tuple[Optional[Any], Optional[str]]: + try: + jsonpath_expr = jsonpath_parse(expression) + matches = [match.value for match in jsonpath_expr.find(data)] + if len(matches) == 1: + return matches[0], None + return matches, None + except Exception as exc: # pragma: no cover - defensive + logger.debug("Failed to apply jsonpath filter", exc_info=exc) + return None, f"Invalid jsonpath expression: {exc}" + + +def _apply_jq_filter(data: Any, expression: str) -> Tuple[Optional[Any], Optional[str]]: + try: + compiled = jq.compile(expression) + results = compiled.input(data).all() + if len(results) == 1: + return results[0], None + return results, None + except Exception as exc: # pragma: no cover - defensive + logger.debug("Failed to apply jq filter", exc_info=exc) + return None, f"Invalid jq filter: {exc}" + + +class JsonFilterMixin: + """Opt-in mixin for tools that return JSON and want filtering controls.""" + + filter_parameters: Dict[str, ToolParameter] = { + "max_depth": ToolParameter( + description="Maximum nesting depth to return from the JSON (0 returns only top-level keys). Leave empty for full response.", + type="integer", + required=False, + ), + "jsonpath": ToolParameter( + description="Optional jsonpath expression to extract specific parts of the JSON.", + type="string", + required=False, + ), + "jq": ToolParameter( + description="Optional jq filter to apply to the JSON.", + type="string", + required=False, + ), + } + + @classmethod + def extend_parameters(cls, existing: Dict[str, ToolParameter]) -> Dict[str, ToolParameter]: + merged = dict(cls.filter_parameters) + merged.update(existing) + return merged + + def _filter_result_data(self, data: Any, params: Dict) -> Tuple[Any, Optional[str]]: + parsed_data = data + if isinstance(data, str): + try: + parsed_data = json.loads(data) + except Exception: + # Not JSON, leave as-is + return data, None + + if params.get("jsonpath"): + parsed_data, error = _apply_jsonpath_filter(parsed_data, params["jsonpath"]) + if error: + return None, error + + if params.get("jq"): + parsed_data, error = _apply_jq_filter(parsed_data, params["jq"]) + if error: + return None, error + + parsed_data = _truncate_to_depth(parsed_data, params.get("max_depth")) + return parsed_data, None + + def filter_result(self, result: StructuredToolResult, params: Dict) -> StructuredToolResult: + filtered_data, error = self._filter_result_data(result.data, params) + if error: + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error=error, + params=params, + url=result.url, + invocation=result.invocation, + icon_url=result.icon_url, + ) + + result.data = filtered_data + return result diff --git a/holmes/plugins/toolsets/opensearch/opensearch.py b/holmes/plugins/toolsets/opensearch/opensearch.py index 7a43596025..1a180066f0 100644 --- a/holmes/plugins/toolsets/opensearch/opensearch.py +++ b/holmes/plugins/toolsets/opensearch/opensearch.py @@ -14,6 +14,7 @@ Toolset, ToolsetTag, ) +from holmes.plugins.toolsets.json_filter_mixin import JsonFilterMixin from holmes.plugins.toolsets.consts import TOOLSET_CONFIG_MISSING_ERROR from holmes.plugins.toolsets.utils import toolset_name_for_one_liner @@ -79,47 +80,52 @@ class BaseOpenSearchTool(Tool): toolset: "OpenSearchToolset" -class ListShards(BaseOpenSearchTool): +class ListShards(JsonFilterMixin, BaseOpenSearchTool): def __init__(self, toolset: "OpenSearchToolset"): super().__init__( name="opensearch_list_shards", description="List the shards within an opensearch cluster", - parameters={ - "host": ToolParameter( - description="The cluster host", - type="string", - required=True, - ) - }, + parameters=self.extend_parameters( + { + "host": ToolParameter( + description="The cluster host", + type="string", + required=True, + ) + } + ), toolset=toolset, ) def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: client = get_client(self.toolset.clients, host=params.get("host", "")) - shards = client.client.cat.shards() - return StructuredToolResult( + shards = client.client.cat.shards(params={"format": "json"}) + result = StructuredToolResult( status=StructuredToolResultStatus.SUCCESS, - data=str(shards), + data=shards, params=params, ) + return self.filter_result(result, params) def get_parameterized_one_liner(self, params: Dict) -> str: host = params.get("host", "") return f"{toolset_name_for_one_liner(self.toolset.name)}: List Shards ({host})" -class GetClusterSettings(BaseOpenSearchTool): +class GetClusterSettings(JsonFilterMixin, BaseOpenSearchTool): def __init__(self, toolset: "OpenSearchToolset"): super().__init__( name="opensearch_get_cluster_settings", description="Retrieve the cluster's settings", - parameters={ - "host": ToolParameter( - description="The cluster host", - type="string", - required=True, - ) - }, + parameters=self.extend_parameters( + { + "host": ToolParameter( + description="The cluster host", + type="string", + required=True, + ) + } + ), toolset=toolset, ) @@ -128,62 +134,67 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes response = client.client.cluster.get_settings( include_defaults=True, flat_settings=True ) - return StructuredToolResult( + result = StructuredToolResult( status=StructuredToolResultStatus.SUCCESS, - data=str(response), + data=response, params=params, ) + return self.filter_result(result, params) def get_parameterized_one_liner(self, params) -> str: host = params.get("host", "") return f"{toolset_name_for_one_liner(self.toolset.name)}: Get Cluster Settings ({host})" -class GetClusterHealth(BaseOpenSearchTool): +class GetClusterHealth(JsonFilterMixin, BaseOpenSearchTool): def __init__(self, toolset: "OpenSearchToolset"): super().__init__( name="opensearch_get_cluster_health", description="Retrieve the cluster's health", - parameters={ - "host": ToolParameter( - description="The cluster host", - type="string", - required=True, - ) - }, + parameters=self.extend_parameters( + { + "host": ToolParameter( + description="The cluster host", + type="string", + required=True, + ) + } + ), toolset=toolset, ) def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: client = get_client(self.toolset.clients, host=params.get("host", "")) health = client.client.cluster.health() - return StructuredToolResult( + result = StructuredToolResult( status=StructuredToolResultStatus.SUCCESS, - data=str(health), + data=health, params=params, ) + return self.filter_result(result, params) def get_parameterized_one_liner(self, params) -> str: host = params.get("host", "") return f"{toolset_name_for_one_liner(self.toolset.name)}: Check Cluster Health ({host})" -class ListOpenSearchHosts(BaseOpenSearchTool): +class ListOpenSearchHosts(JsonFilterMixin, BaseOpenSearchTool): def __init__(self, toolset: "OpenSearchToolset"): super().__init__( name="opensearch_list_hosts", description="List all OpenSearch hosts in the cluster", - parameters={}, + parameters=self.extend_parameters({}), toolset=toolset, ) def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: hosts = [host for client in self.toolset.clients for host in client.hosts] - return StructuredToolResult( + result = StructuredToolResult( status=StructuredToolResultStatus.SUCCESS, - data=str(hosts), + data=hosts, params=params, ) + return self.filter_result(result, params) def get_parameterized_one_liner(self, params: Dict) -> str: return f"{toolset_name_for_one_liner(self.toolset.name)}: List OpenSearch Hosts" diff --git a/poetry.lock b/poetry.lock index 1194df7344..fb6e6f9af7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "ag-ui-protocol" @@ -2513,6 +2513,108 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "jq" +version = "1.10.0" +description = "jq is a lightweight and flexible JSON processor." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "jq-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9bba438d1813e537294e77f6f0ab3f4b23d3a0ae125187edf4827260a31341a0"}, + {file = "jq-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3eb6aed0d9882c43ae4c1757b72afc02063504f69d14eb12352c9b2813137c71"}, + {file = "jq-1.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c2a6a83f8b59dcb0b9a09f1e6b042e667923916767d0bee869e217d067f5f25"}, + {file = "jq-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:731fa11ce06276365c51fe2e23821d33acf6c66e501acfc4dd95507be840dd39"}, + {file = "jq-1.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd313711ad4b6158662a65e1de9a4e34e6f297cacaa2a563b1d7c37fd453770e"}, + {file = "jq-1.10.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:af859195c4a46adc52866cbc08a5d56fea86cbb8d18c9e02b95fea7d0b9c872d"}, + {file = "jq-1.10.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:801d01e2c933fa3da70ce18d73adb29c4fd07ebe0e2da3f39f79719357a60014"}, + {file = "jq-1.10.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f6557f291f0b13db045b35401fa8b67b866fa1488e3a9703a1bcc5d156948c59"}, + {file = "jq-1.10.0-cp310-cp310-win32.whl", hash = "sha256:148a140c16c366c42c63e5a920dc8259ab62034c6f2c6b0f410df579fdf04654"}, + {file = "jq-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:f8aa3f4eb6948be7b9b7c8eb19d4fbdaa2765227d21ea875898e2d20552ad749"}, + {file = "jq-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1363930e8efe63a76e6be93ffc6fea6d9201ba13a21f8a9c7943e9c6a4184cf7"}, + {file = "jq-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:850c99324fdb2e42a2056c27ec45af87b1bc764a14c94cdf011f6e21d885f032"}, + {file = "jq-1.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75aabeae63f36fe421c25cb793f5e166500400e443e7f6ce509261d06d4f8b5d"}, + {file = "jq-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b6e1f04da95c5057346954b24e65cb401cf9c64566e68c4263454717fcf464d"}, + {file = "jq-1.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ff2ac703b1bde9209f124aa7948012b77e93a40f858c24cf0bbd48f150c15e8"}, + {file = "jq-1.10.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:83ae246c191e6c5363cb7987af10c4c3071ec6995424eb749d225fbb985d9e47"}, + {file = "jq-1.10.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:307ed7ac72c03843af46face4ec1b8238f6d0d68f5a37aab3b55d178c329ad34"}, + {file = "jq-1.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ded278e64dad446667656f7144cefe20cea16bb57cf6912ef6d1ddf3bddc9861"}, + {file = "jq-1.10.0-cp311-cp311-win32.whl", hash = "sha256:5d5624d43c8597b06a4a2c5461d1577f29f23991472a5da88b742f7fa529c1d1"}, + {file = "jq-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:08bf55484a20955264358823049ff8deb671bb0025d51707ec591b5eb18a94d7"}, + {file = "jq-1.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe636cfa95b7027e7b43da83ecfd61431c0de80c3e0aa4946534b087149dcb4c"}, + {file = "jq-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:947fc7e1baaa7e95833b950e5a66b3e13a5cff028bff2d009b8c320124d9e69b"}, + {file = "jq-1.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9382f85a347623afa521c43f8f09439e68906fd5b3492016f969a29219796bb9"}, + {file = "jq-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c376aab525d0a1debe403d3bc2f19fda9473696a1eda56bafc88248fc4ae6e7e"}, + {file = "jq-1.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:206f230c67a46776f848858c66b9c377a8e40c2b16195552edd96fd7b45f9a52"}, + {file = "jq-1.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:06986456ebc95ccb9e9c2a1f0e842bc9d441225a554a9f9d4370ad95a19ac000"}, + {file = "jq-1.10.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d02c0be958ddb4d9254ff251b045df2f8ee5995137702eeab4ffa81158bcdbe0"}, + {file = "jq-1.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37cf6fd2ebd2453e75ceef207d5a95a39fcbda371a9b8916db0bd42e8737a621"}, + {file = "jq-1.10.0-cp312-cp312-win32.whl", hash = "sha256:655d75d54a343944a9b011f568156cdc29ae0b35d2fdeefb001f459a4e4fc313"}, + {file = "jq-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:1d67c2653ae41eab48f8888c213c9e1807b43167f26ac623c9f3e00989d3edee"}, + {file = "jq-1.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b11d6e115ebad15d738d49932c3a8b9bb302b928e0fb79acc80987598d147a43"}, + {file = "jq-1.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df278904c5727dfe5bc678131a0636d731cd944879d890adf2fc6de35214b19b"}, + {file = "jq-1.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab4c1ec69fd7719fb1356e2ade7bd2b5a63d6f0eaf5a90fdc5c9f6145f0474ce"}, + {file = "jq-1.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd24dc21c8afcbe5aa812878251cfafa6f1dc6e1126c35d460cc7e67eb331018"}, + {file = "jq-1.10.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c0d3e89cd239c340c3a54e145ddf52fe63de31866cb73368d22a66bfe7e823f"}, + {file = "jq-1.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76710b280e4c464395c3d8e656b849e2704bd06e950a4ebd767860572bbf67df"}, + {file = "jq-1.10.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b11a56f1fb6e2985fd3627dbd8a0637f62b1a704f7b19705733d461dafa26429"}, + {file = "jq-1.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ac05ae44d9aa1e462329e1510e0b5139ac4446de650c7bdfdab226aafdc978ec"}, + {file = "jq-1.10.0-cp313-cp313-win32.whl", hash = "sha256:0bad90f5734e2fc9d09c4116ae9102c357a4d75efa60a85758b0ba633774eddb"}, + {file = "jq-1.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:4ec3fbca80a9dfb5349cdc2531faf14dd832e1847499513cf1fc477bcf46a479"}, + {file = "jq-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:09e6ca3095a3be59353a262c75f680a0934ac03e81d10b78d7eabcb9fb746543"}, + {file = "jq-1.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57e7b81b69700ad6003f6d068ae5432fa54169e2c5b15a1f9073400d83c0115a"}, + {file = "jq-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36a959b8cff3796b42f51a0be5fa37126ee66fc822e29620485a229f6c9baec6"}, + {file = "jq-1.10.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f44bd6b9d03367a9e8a3f8f5c8343b572fbec9d148242f226e2a6f2eb459ba2b"}, + {file = "jq-1.10.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:48fb2cfe083942e370876e865fad3836aedc1b06ef30a2376e53ab35d6a7f728"}, + {file = "jq-1.10.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:d47fe013dc22c82b425ae5358729a3d38de4097dda28d63f591c8bdd97bae6cb"}, + {file = "jq-1.10.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2886606f40f2127ed4bea2aa2d30653d485ed26075dd5b52fb933aa7ec7b23d3"}, + {file = "jq-1.10.0-cp38-cp38-win32.whl", hash = "sha256:f1f277fd820246f0d80da2ddd39b6d5ea99b266c067abce34f1ff50bd3358477"}, + {file = "jq-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:ab050dc7a6c204dde3a3d28e340f937e00cf69c8d3f7dd17e8c3ffef916784df"}, + {file = "jq-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db746ec4f05a6622bca5785f58fa322f04c493de61c6761cbe5a61218babe3d9"}, + {file = "jq-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bb9a6b5ff1e9d261ffae51aefbc66660bc1f5713339943aa95af7631062be163"}, + {file = "jq-1.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6690eda99a5be16f28c6931918769af0c2d066257d4efedc7c4108cfbf4e242f"}, + {file = "jq-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f4b28f04bb69eadb99c7ba3911246e6a200a886d82ae190a0af95037c427de6"}, + {file = "jq-1.10.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7278f1dfc49387551e670133b363a3368eeac74672dd4b520b7da4f8e803058"}, + {file = "jq-1.10.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:28c16ca90940dfb8a2bd28b2d02cb2a546faa91db5b03f2cb71148b158fc098c"}, + {file = "jq-1.10.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f700d2aa1ef2b58c33df0a06ba58e68196cc9f81d1b6eb6baaa34e81fc0dbe6d"}, + {file = "jq-1.10.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cf18f3031da9727245529e16ad13ab8859d73cfe0bc0e09a79696038658c25d5"}, + {file = "jq-1.10.0-cp39-cp39-win32.whl", hash = "sha256:31753d5b45b1806d1d7241a45cb262b1f3db8c0f1f4c618d5a02cbe227d2c436"}, + {file = "jq-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:c4648684034ba5b975be9b0099ca230ef088c14eeffb3357b522d9cba02a05ea"}, + {file = "jq-1.10.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:591d81677336551fd9cf3b5e23c1929ae3cd039a5c2d5fb8042870ed5372fd8c"}, + {file = "jq-1.10.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f7f68716e8533294d2f5152e8659288091ea705778a00e066ed3b418ed724d81"}, + {file = "jq-1.10.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72658f31d5723e7b87eea929e81d5083fd4132636a9dcdbf56ba9ea0e30ecaa3"}, + {file = "jq-1.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:039e84e19306f94bf0d15291f6d358c4086c702de48e2309c3183fd866bf2785"}, + {file = "jq-1.10.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e985ded6dc2105707cb04e98ca6acbe6c24614824ed0a2fae442d2b2bc78fbc4"}, + {file = "jq-1.10.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:185d450fb45cd44ad5939ec5813f1ed0d057fa0cb12a1ba6a5fe0e49d8354958"}, + {file = "jq-1.10.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5e6649eb4ce390342e07c95c2fa10fed043410408620296122f0ac48a7576f1f"}, + {file = "jq-1.10.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:002d93e50fab9d92035dfd79fd134052692be649e1b3835662a053016d9f2ee7"}, + {file = "jq-1.10.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1602f95ffaef7ee357b65f414b59d6284619bd3a0a588c15c3a1ae534811c1fb"}, + {file = "jq-1.10.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b28cfd1eac69e1dc14518ed9c33e497041e9f9310e5f6259fa9d18fe342fb50"}, + {file = "jq-1.10.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:061225bc6b45b399f4dfbece00f4fae78560c1d4e0f2af77203dde621c5f10be"}, + {file = "jq-1.10.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5dba7abfe55efe3f139247a30e1f13e94f33fddfea71245a18a817b619cb9fe9"}, + {file = "jq-1.10.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c6f45acaad61c1947bf2fa172a2ccb2e882231c3cfbfc1ea4a2c4f032122a546"}, + {file = "jq-1.10.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16c250cd0a708d45b9bb08fdf4cac415156274f7f3f8f026e75b5a330d2162dd"}, + {file = "jq-1.10.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:593551afc5f305c7d0adc840587350cb49c4ecba6464f4fc965cae87758621a7"}, + {file = "jq-1.10.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c550705145480b616b6bc114ab7f421c0d9a3041ad3dcb9424f992954823f7c2"}, + {file = "jq-1.10.0.tar.gz", hash = "sha256:fc38803075dbf1867e1b4ed268fef501feecb0c50f3555985a500faedfa70f08"}, +] + +[[package]] +name = "jsonpath-ng" +version = "1.7.0" +description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c"}, + {file = "jsonpath_ng-1.7.0-py2-none-any.whl", hash = "sha256:898c93fc173f0c336784a3fa63d7434297544b7198124a68f9a3ef9597b0ae6e"}, + {file = "jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6"}, +] + +[package.dependencies] +ply = "*" + [[package]] name = "jsonschema" version = "4.25.1" @@ -3784,6 +3886,18 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] +[[package]] +name = "ply" +version = "3.11" +description = "Python Lex & Yacc" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, + {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, +] + [[package]] name = "polyleven" version = "0.9.0" @@ -6215,4 +6329,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "443b859de7e966f4e184f36679ff5f7e1d1f6a38514bbd421f90f5995cecd7d6" +content-hash = "d175a137cdd723745a7144633f27b850c1d6aa3b1cb5ff4b6b1b420591992d85" diff --git a/pyproject.toml b/pyproject.toml index 24baa680e4..57932a7935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,8 @@ prometrix = "0.2.5" httpx = {version = "<0.28", extras = ["socks"]} ag-ui-protocol = "^0.1.9" google-cloud-aiplatform = ">=1.38" +jq = "^1.8.0" +jsonpath-ng = "^1.6.1" [tool.poetry.group.dev] optional = true diff --git a/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/generate_dashboard.py b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/generate_dashboard.py new file mode 100644 index 0000000000..41aff09af0 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/generate_dashboard.py @@ -0,0 +1,82 @@ +import json + + +def build_panel(panel_id: int, row: int, col: int) -> dict: + return { + "id": panel_id, + "title": f"Panel {panel_id} - Requests {row}/{col}", + "type": "timeseries", + "gridPos": {"h": 4, "w": 6, "x": (col % 4) * 6, "y": row * 4}, + "targets": [ + {"refId": "A", "expr": f"rate(requests_total{{service=\"svc-{panel_id}\"}}[5m])"}, + {"refId": "B", "expr": f"histogram_quantile(0.95, rate(latency_bucket{{service=\"svc-{panel_id}\"}}[5m]))"}, + ], + "options": { + "tooltip": {"mode": "multi", "sort": "desc"}, + "legend": {"displayMode": "list", "placement": "right"}, + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "showPoints": "auto", + }, + } + }, + } + + +def build_dashboard(panel_count: int = 40) -> dict: + panels = [] + for i in range(panel_count): + row = i // 4 + col = i % 4 + panels.append(build_panel(i + 1, row, col)) + + # Add a nested panel structure to force deeper JSON + nested_panel = { + "id": panel_count + 1, + "title": "Nested Summary", + "type": "row", + "gridPos": {"h": 2, "w": 24, "x": 0, "y": (panel_count // 4 + 1) * 4}, + "panels": [ + { + "id": panel_count + 2, + "title": "Error Overview", + "type": "stat", + "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0}, + "targets": [{"refId": "A", "expr": "rate(errors_total[5m])"}], + "fieldConfig": {"defaults": {"unit": "ops"}}, + }, + { + "id": panel_count + 3, + "title": "Cache Hit Rate", + "type": "gauge", + "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0}, + "targets": [{"refId": "A", "expr": "avg_over_time(cache_hit_rate[5m])"}], + "fieldConfig": {"defaults": {"unit": "percent"}}, + }, + ], + } + panels.append(nested_panel) + + return { + "dashboard": { + "id": None, + "uid": "filterdemo", + "title": "Filter Demo Dashboard", + "schemaVersion": 36, + "version": 1, + "editable": False, + "panels": panels, + "tags": ["filters", "depth-control", "jq-jsonpath"], + "templating": {"list": []}, + }, + "overwrite": True, + } + + +if __name__ == "__main__": + print(json.dumps(build_dashboard(), indent=2)) diff --git a/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/grafana.yaml b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/grafana.yaml new file mode 100644 index 0000000000..2eab5e9c9b --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/grafana.yaml @@ -0,0 +1,83 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-providers + labels: + app: grafana + namespace: app-170 +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + labels: + app: grafana + namespace: app-170 +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:10.4.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + env: + - name: GF_SECURITY_ADMIN_USER + value: admin + - name: GF_SECURITY_ADMIN_PASSWORD + value: admin + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ORG_ROLE + value: Admin + - name: GF_AUTH_BASIC_ENABLED + value: "false" + - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH + value: /var/lib/grafana/dashboards/filter-demo.json + volumeMounts: + - name: grafana-dashboard-providers + mountPath: /etc/grafana/provisioning/dashboards + - name: grafana-dashboards + mountPath: /var/lib/grafana/dashboards + volumes: + - name: grafana-dashboard-providers + configMap: + name: grafana-dashboard-providers + - name: grafana-dashboards + configMap: + name: grafana-dashboards +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + labels: + app: grafana + namespace: app-170 +spec: + selector: + app: grafana + ports: + - port: 3000 + targetPort: 3000 + protocol: TCP + name: http diff --git a/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/test_case.yaml new file mode 100644 index 0000000000..e4ec9dd63b --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/test_case.yaml @@ -0,0 +1,67 @@ +user_prompt: "Only show the panel ids and titles from the Grafana dashboard with UID 'filterdemo'. Keep the response compact by filtering or limiting depth rather than returning the full dashboard JSON." + +expected_output: + - Mentions dashboard UID filterdemo + - Lists the panel titles Requests Overview and Latency Breakdown + - Indicates the dashboard data was filtered/truncated rather than returning the full JSON + +tags: + - medium + - network + - toolset-limitation + +setup_timeout: 600 + +port_forwards: + - namespace: app-170 + service: grafana + local_port: 3030 + remote_port: 3000 + +before_test: | + kubectl create namespace app-170 || true + + # Generate a large dashboard JSON to exercise filtering and depth controls + python ./generate_dashboard.py > /tmp/filter-demo.json + + # Create the dashboards ConfigMap from generated JSON + kubectl create configmap grafana-dashboards \ + --from-file=filter-demo.json=/tmp/filter-demo.json \ + -n app-170 --dry-run=client -o yaml | kubectl apply -f - + + # Apply Grafana deployment and provider config + kubectl apply -f grafana.yaml -n app-170 + + # Wait for pod to be created + for i in {1..60}; do + if kubectl get pods -n app-170 -l app=grafana -o name | grep -q grafana; then + echo "Pod scheduled" + break + fi + echo "Waiting for Grafana pod to schedule ($i/60)" + sleep 5 + done + + # Wait for Grafana pod readiness with retry loop to avoid race conditions + READY=false + for i in {1..60}; do + if kubectl wait --for=condition=ready pod -l app=grafana -n app-170 --timeout=5s 2>/dev/null; then + READY=true + break + fi + echo "Grafana not ready yet ($i/60). Retrying in 5s..." + sleep 5 + done + + if [ "$READY" = false ]; then + echo "Grafana pod failed to become ready" + kubectl get pods -n app-170 -l app=grafana + exit 1 + fi + + # Verify dashboard is reachable and provisioned + kubectl run grafana-check-170 --rm -i --restart=Never --image=curlimages/curl -n app-170 -- \ + curl -s http://grafana:3000/api/dashboards/uid/filterdemo | grep -q "filterdemo" + +after_test: | + kubectl delete namespace app-170 || true diff --git a/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/toolsets.yaml new file mode 100644 index 0000000000..d0bba3b27f --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/170_grafana_dashboard_filters/toolsets.yaml @@ -0,0 +1,5 @@ +toolsets: + grafana/dashboards: + enabled: true + config: + url: http://localhost:3030 diff --git a/tests/plugins/toolsets/grafana/test_dashboard_filters.py b/tests/plugins/toolsets/grafana/test_dashboard_filters.py new file mode 100644 index 0000000000..420e591616 --- /dev/null +++ b/tests/plugins/toolsets/grafana/test_dashboard_filters.py @@ -0,0 +1,70 @@ +import pytest + +from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus +from holmes.plugins.toolsets.grafana.toolset_grafana import ( + GrafanaDashboardConfig, + GrafanaToolset, + GetDashboardByUID, +) +from holmes.plugins.toolsets.json_filter_mixin import _truncate_to_depth + + +def _build_tool(data): + toolset = GrafanaToolset() + toolset._grafana_config = GrafanaDashboardConfig(url="http://example.com") + tool = GetDashboardByUID(toolset) + tool._make_grafana_request = lambda endpoint, params: StructuredToolResult( + status=StructuredToolResultStatus.SUCCESS, + data=data, + params=params, + url="http://api", + ) + return tool + + +def test_truncate_to_depth_limits_nested_values(): + data = {"a": {"b": {"c": 1}}, "list": [1, {"nested": 2}]} + truncated = _truncate_to_depth(data, 1) + assert truncated["a"] == "...truncated at depth 1" + assert truncated["list"] == "...truncated at depth 1" + + +def test_jsonpath_filter_applies_before_returning_data(): + data = {"dashboard": {"panels": [{"id": 1, "title": "CPU"}]}} + tool = _build_tool(data) + + result = tool._invoke( + {"uid": "abc", "jsonpath": "$.dashboard.panels[*].title"}, context=None + ) + + assert result.status is StructuredToolResultStatus.SUCCESS + assert result.data == "CPU" + + +def test_jq_filter_applies_before_returning_data(): + data = {"dashboard": {"panels": [{"id": 1, "title": "CPU"}]}} + tool = _build_tool(data) + + result = tool._invoke({"uid": "abc", "jq": ".dashboard.panels[].id"}, context=None) + + assert result.status is StructuredToolResultStatus.SUCCESS + assert result.data == 1 + + +def test_invalid_jsonpath_returns_error(): + data = {"dashboard": {"panels": [{"id": 1, "title": "CPU"}]}} + tool = _build_tool(data) + + result = tool._invoke({"uid": "abc", "jsonpath": "$.["}, context=None) + + assert result.status is StructuredToolResultStatus.ERROR + assert "Invalid jsonpath expression" in result.error + + +def test_depth_applies_after_filters(): + data = {"dashboard": {"panels": [{"id": 1, "title": "CPU"}]}} + tool = _build_tool(data) + + result = tool._invoke({"uid": "abc", "max_depth": 1}, context=None) + + assert result.data["dashboard"] == "...truncated at depth 1" From a8e93ffa32e5efa45dd39a6634c6ce3bb69d4595 Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 30 Dec 2025 17:34:33 +0200 Subject: [PATCH 2/4] Harden JSON filter mixin for mocked results Signed-off-by: Codex --- holmes/plugins/toolsets/json_filter_mixin.py | 25 +++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/holmes/plugins/toolsets/json_filter_mixin.py b/holmes/plugins/toolsets/json_filter_mixin.py index b7f3c23af9..76180abdb7 100644 --- a/holmes/plugins/toolsets/json_filter_mixin.py +++ b/holmes/plugins/toolsets/json_filter_mixin.py @@ -107,16 +107,29 @@ def _filter_result_data(self, data: Any, params: Dict) -> Tuple[Any, Optional[st return parsed_data, None def filter_result(self, result: StructuredToolResult, params: Dict) -> StructuredToolResult: - filtered_data, error = self._filter_result_data(result.data, params) + base_result = ( + result + if isinstance(result, StructuredToolResult) + else StructuredToolResult( + status=getattr(result, "status", StructuredToolResultStatus.SUCCESS), + data=getattr(result, "data", None), + params=getattr(result, "params", params), + url=getattr(result, "url", None), + invocation=getattr(result, "invocation", None), + icon_url=getattr(result, "icon_url", None), + ) + ) + + filtered_data, error = self._filter_result_data(base_result.data, params) if error: return StructuredToolResult( status=StructuredToolResultStatus.ERROR, error=error, params=params, - url=result.url, - invocation=result.invocation, - icon_url=result.icon_url, + url=base_result.url, + invocation=base_result.invocation, + icon_url=base_result.icon_url, ) - result.data = filtered_data - return result + base_result.data = filtered_data + return base_result From bd8a435ada6cdea0ba3af99cb8d642e1beffb940 Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 30 Dec 2025 20:44:57 +0200 Subject: [PATCH 3/4] Normalize mock fields in JSON filter mixin Signed-off-by: Codex --- holmes/plugins/toolsets/json_filter_mixin.py | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/holmes/plugins/toolsets/json_filter_mixin.py b/holmes/plugins/toolsets/json_filter_mixin.py index 76180abdb7..0823a29526 100644 --- a/holmes/plugins/toolsets/json_filter_mixin.py +++ b/holmes/plugins/toolsets/json_filter_mixin.py @@ -106,19 +106,33 @@ def _filter_result_data(self, data: Any, params: Dict) -> Tuple[Any, Optional[st parsed_data = _truncate_to_depth(parsed_data, params.get("max_depth")) return parsed_data, None + @staticmethod + def _safe_string(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, str): + return value + try: + return str(value) + except Exception: + return None + def filter_result(self, result: StructuredToolResult, params: Dict) -> StructuredToolResult: - base_result = ( - result - if isinstance(result, StructuredToolResult) - else StructuredToolResult( + base_result = result if isinstance(result, StructuredToolResult) else None + if base_result is None: + base_result = StructuredToolResult( status=getattr(result, "status", StructuredToolResultStatus.SUCCESS), data=getattr(result, "data", None), params=getattr(result, "params", params), - url=getattr(result, "url", None), - invocation=getattr(result, "invocation", None), - icon_url=getattr(result, "icon_url", None), + url=self._safe_string(getattr(result, "url", None)), + invocation=self._safe_string(getattr(result, "invocation", None)), + icon_url=self._safe_string(getattr(result, "icon_url", None)), ) - ) + else: + # Normalize string fields to avoid MagicMock validation failures + base_result.url = self._safe_string(base_result.url) + base_result.invocation = self._safe_string(base_result.invocation) + base_result.icon_url = self._safe_string(base_result.icon_url) filtered_data, error = self._filter_result_data(base_result.data, params) if error: From e2ce0c60fa11791e8f9bc6ba89d00b79c8373bac Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 30 Dec 2025 21:42:51 +0200 Subject: [PATCH 4/4] Apply JSON filter mixin to Coralogix DataPrime query Signed-off-by: Codex --- holmes/plugins/toolsets/coralogix/toolset_coralogix.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/holmes/plugins/toolsets/coralogix/toolset_coralogix.py b/holmes/plugins/toolsets/coralogix/toolset_coralogix.py index 0b857db9b4..4eb821f02e 100644 --- a/holmes/plugins/toolsets/coralogix/toolset_coralogix.py +++ b/holmes/plugins/toolsets/coralogix/toolset_coralogix.py @@ -13,6 +13,7 @@ ToolParameter, Toolset, ) +from holmes.plugins.toolsets.json_filter_mixin import JsonFilterMixin from holmes.plugins.toolsets.consts import TOOLSET_CONFIG_MISSING_ERROR from holmes.plugins.toolsets.coralogix.api import ( health_check, @@ -58,7 +59,7 @@ def _build_coralogix_query_url( return None -class ExecuteDataPrimeQuery(Tool): +class ExecuteDataPrimeQuery(JsonFilterMixin, Tool): def __init__(self, toolset: "CoralogixToolset"): super().__init__( name="coralogix_execute_dataprime_query", @@ -166,14 +167,13 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes tier=tier, ) - # Return a pretty-printed JSON string for readability by the model/user. - final_result = json.dumps(result_dict, indent=2, sort_keys=False) - return StructuredToolResult( + wrapped = StructuredToolResult( status=status, - data=final_result, + data=result_dict, params=params, url=explore_url, ) + return self.filter_result(wrapped, params) def get_parameterized_one_liner(self, params) -> str: description = params.get("description", "")