Skip to content
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
2c1fa16
Add toolsets_matrix support for comparing toolset configs on same eva…
claude Feb 21, 2026
f685f14
Add toolsets_matrix to Datadog evals: builtin vs HTTP toolset comparison
claude Feb 21, 2026
f2f0f26
Make jq errors non-fatal: return raw data preview so LLM can self-cor…
claude Feb 21, 2026
f039811
Cap jq error preview to 2000 chars to avoid flooding context
claude Feb 21, 2026
80cdf91
Simplify eval 110 to a single user_prompt
claude Feb 21, 2026
ab75f0d
Merge branch 'master' into claude/evals-toolset-matrix-mwdXc
aantn Feb 21, 2026
3d7dfa1
Remove toolsets_matrix from embed tests that require query_datadog_me…
claude Feb 21, 2026
d312f85
Merge remote-tracking branch 'origin/claude/evals-toolset-matrix-mwdX…
claude Feb 21, 2026
72973c8
Remove toolsets_matrix from 164, keep only on 91f
claude Feb 22, 2026
69825bb
Merge branch 'master' into claude/evals-toolset-matrix-mwdXc
aantn Feb 22, 2026
8d52f26
Remove toolsets_matrix from 91h, keep only on 91f
claude Feb 22, 2026
f107fa9
Merge remote-tracking branch 'origin/claude/evals-toolset-matrix-mwdX…
claude Feb 22, 2026
a67a8f9
Revert all non-jq changes to split PR
claude Feb 22, 2026
914ac5a
Add jq-filter tag to evals exercising JsonFilterMixin toolsets
claude Feb 22, 2026
e16755b
Remove null-safe jq guidance from prompts and error messages
claude Feb 22, 2026
1414b95
Merge branch 'master' into claude/revert-except-jq-errors-A7N9q
aantn Feb 22, 2026
2e95a9e
Merge origin/master into branch, resolving pyproject.toml conflict
claude Mar 7, 2026
5b0c907
Merge remote-tracking branch 'origin/claude/revert-except-jq-errors-A…
claude Mar 7, 2026
3289fb9
Add defensive try/except around json.dumps in jq error preview
claude Mar 8, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions holmes/plugins/toolsets/http/instructions.jinja2
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ The tool supports:
- JSON response filtering with `jq` expressions and `max_depth`
- Requests are restricted to whitelisted hosts, paths, and HTTP methods

When using jq, always use null-safe patterns to handle missing or null fields:
- Use `(.key // [])[]` instead of `.key[]`
- Use `(.key // {}).subkey` instead of `.key.subkey`
- Example: `[(.metrics // [])[] | select(contains("cpu"))][:20]`

{% if config and config.endpoints %}
**Whitelisted endpoints** (requests to other URLs will be rejected):
{% for endpoint in config.endpoints %}
Expand Down
16 changes: 15 additions & 1 deletion holmes/plugins/toolsets/json_filter_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,23 @@ def _filter_result_data(self, data: Any, params: Dict) -> Tuple[Any, Optional[st
return data, None

if params.get("jq"):
original_data = parsed_data
parsed_data, error = _apply_jq_filter(parsed_data, params["jq"])
if error:
return None, error
# Return original data with error hint so the LLM can see the
# actual response shape and self-correct its jq expression.
# Depth-truncate and cap the serialized size to avoid flooding
# the context with a huge raw payload.
max_preview_chars = 2000
truncated = _truncate_to_depth(original_data, max_depth=2)
preview_str = json.dumps(truncated, separators=(",", ":"), ensure_ascii=False)
if len(preview_str) > max_preview_chars:
preview_str = preview_str[:max_preview_chars] + "…(truncated)"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return {
"jq_error": f"{error}. Use null-safe patterns like (.key // [])[] instead of .key[] to handle missing/null fields.",
"jq_expression": params["jq"],
"raw_response_preview": preview_str,
}, None

parsed_data = _truncate_to_depth(parsed_data, params.get("max_depth"))
return parsed_data, None
Expand Down
13 changes: 10 additions & 3 deletions tests/plugins/toolsets/test_json_filter_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,21 @@ def test_jq_filter_applies_before_returning_data():
assert result.data == "CPU"


def test_invalid_jq_returns_error():
def test_invalid_jq_returns_data_with_error_hint():
data = {"dashboard": {"panels": [{"id": 1, "title": "CPU"}]}}
tool = _build_tool(data)

result = tool._invoke({"uid": "abc", "jq": ".["}, context=None)

assert result.status is StructuredToolResultStatus.ERROR
assert "Invalid jq expression" in result.error
# Invalid jq should return SUCCESS with the raw data preview and error hint
# so the LLM can see the response shape and self-correct its expression
assert result.status is StructuredToolResultStatus.SUCCESS
assert "jq_error" in result.data
assert "Invalid jq expression" in result.data["jq_error"]
assert "raw_response_preview" in result.data
# Preview is a size-capped string, not a dict
assert isinstance(result.data["raw_response_preview"], str)
assert "dashboard" in result.data["raw_response_preview"]


def test_depth_applies_after_filters():
Expand Down