Skip to content

Tool limits are defaults, not ceilings (Datadog logs, Prometheus metadata) #2169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aantn wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Tool limits are defaults, not ceilings (Datadog logs, Prometheus metadata) #2169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5536cb1
Tool limits are defaults, not ceilings (Datadog logs, Prometheus meta…
claude Jun 10, 2026
91bc7c4
Treat only None as missing for limit params (explicit 0 clamps to 1)
claude Jun 10, 2026
06e828f
Retrigger CI benchmarks after OpenRouter weekly limit was raised
claude Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions holmes/plugins/toolsets/datadog/toolset_datadog_logs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@
toolset_name_for_one_liner,
)

# Maximum page size accepted by Datadog's POST /api/v2/logs/events/search.
# Results beyond one page are reachable via the cursor parameter.
DATADOG_LOGS_API_PAGE_MAX = 1000


def format_logs(raw_logs: list[dict]) -> str:
# Use similar structure to Datadog Log Explorer
Expand Down Expand Up @@ -179,7 +183,7 @@ class GetLogs(Tool):
required=False,
),
"limit": ToolParameter(
description=f"Maximum number of log records to return. Defaults to {DEFAULT_LOG_LIMIT}. This value is user-configured and represents the maximum allowed limit.",
description=f"Maximum number of log records to return. Defaults to the configured default_limit ({DEFAULT_LOG_LIMIT} unless overridden). Values above {DATADOG_LOGS_API_PAGE_MAX} are capped at {DATADOG_LOGS_API_PAGE_MAX} (the Datadog API page maximum); use the cursor to fetch further pages.",
type="integer",
required=False,
),
Expand Down Expand Up @@ -216,8 +220,14 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
from_time_ms = from_time_int * 1000
to_time_ms = to_time_int * 1000

config_limit = self.toolset.dd_config.default_limit
limit = min(params.get("limit", config_limit), config_limit)
# The configured default_limit applies when the LLM doesn't ask for
# a specific limit. An explicit request is honored up to the Datadog
# API page maximum (previously it was silently clamped down to the
# config value, making larger fetches inexpressible).
limit = params.get("limit")
if limit is None:
limit = self.toolset.dd_config.default_limit
limit = max(1, min(limit, DATADOG_LOGS_API_PAGE_MAX))
params["limit"] = limit
sort = "timestamp" if params.get("sort_desc", False) else "-timestamp"

Expand Down
134 changes: 73 additions & 61 deletions holmes/plugins/toolsets/prometheus/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1021,6 +1021,43 @@ def get_parameterized_one_liner(self, params) -> str:
)


def metadata_limit_param() -> ToolParameter:
"""The model-overridable `limit` parameter shared by the metadata APIs."""
return ToolParameter(
description=(
f"Maximum number of results to return. Default: {PROMETHEUS_METADATA_API_LIMIT}. "
"Increase when a previous call reported _truncated=true and you need the complete list."
),
type="integer",
required=False,
)


def get_metadata_limit(params: dict) -> int:
limit = params.get("limit")
if limit is None:
limit = PROMETHEUS_METADATA_API_LIMIT
return max(1, int(limit))


def mark_metadata_truncation(data: Any, limit: int, refine_hint: str) -> None:
"""Flag a metadata API response whose result filled the limit.

Tells the model both recovery paths: raise the limit, or refine the query.
The result is a list for most endpoints; /api/v1/metadata returns a dict.
"""
if (
"data" in data
and isinstance(data["data"], (list, dict))
and len(data["data"]) == limit
):
data["_truncated"] = True
data["_message"] = (
f"Results truncated at limit={limit}. Re-run with a higher 'limit' "
f"parameter to see more results, or {refine_hint}"
)


class GetMetricNames(BasePrometheusTool):
"""Thin wrapper around /api/v1/label/__name__/values - the fastest way to discover metric names"""

Expand All @@ -1030,8 +1067,8 @@ def __init__(self, toolset: "PrometheusToolset"):
description=(
"Get list of metric names using /api/v1/label/__name__/values. "
"FASTEST method for metric discovery when you need to explore available metrics. "
f"Returns up to {PROMETHEUS_METADATA_API_LIMIT} unique metric names (limit={PROMETHEUS_METADATA_API_LIMIT}). If {PROMETHEUS_METADATA_API_LIMIT} results returned, more may exist - use a more specific filter. "
f"ALWAYS use match[] parameter to filter metrics - without it you'll get random {PROMETHEUS_METADATA_API_LIMIT} metrics which is rarely useful. "
f"Returns up to 'limit' unique metric names (default {PROMETHEUS_METADATA_API_LIMIT}). If the response reports _truncated=true, raise 'limit' or use a more specific filter. "
f"ALWAYS use match[] parameter to filter metrics - without it you'll get arbitrary metrics which is rarely useful. "
"Note: Does not return metric metadata (type, description, labels). "
"By default returns metrics active in the last 1 hour (configurable via default_metadata_time_window_hrs)."
),
Expand Down Expand Up @@ -1059,6 +1096,7 @@ def __init__(self, toolset: "PrometheusToolset"):
type="string",
required=False,
),
"limit": metadata_limit_param(),
},
toolset=toolset,
)
Expand All @@ -1082,8 +1120,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
url = urljoin(
self.toolset.config.prometheus_url, "api/v1/label/__name__/values"
)
limit = get_metadata_limit(params)
query_params = {
"limit": str(PROMETHEUS_METADATA_API_LIMIT),
"limit": str(limit),
"match[]": match_param,
}

Expand Down Expand Up @@ -1114,16 +1153,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
response.raise_for_status()
data = response.json()

# Check if results were truncated
if (
"data" in data
and isinstance(data["data"], list)
and len(data["data"]) == PROMETHEUS_METADATA_API_LIMIT
):
data["_truncated"] = True
data["_message"] = (
f"Results truncated at limit={PROMETHEUS_METADATA_API_LIMIT}. Use a more specific match filter to see additional metrics."
)
mark_metadata_truncation(
data, limit, "use a more specific match filter."
)

return StructuredToolResult(
status=StructuredToolResultStatus.SUCCESS,
Expand All @@ -1150,7 +1182,7 @@ def __init__(self, toolset: "PrometheusToolset"):
description=(
"Get all values for a specific label using /api/v1/label/{label}/values. "
"Use this to discover pods, namespaces, jobs, instances, etc. "
f"Returns up to {PROMETHEUS_METADATA_API_LIMIT} unique values (limit={PROMETHEUS_METADATA_API_LIMIT}). If {PROMETHEUS_METADATA_API_LIMIT} results returned, more may exist - use match[] to filter. "
f"Returns up to 'limit' unique values (default {PROMETHEUS_METADATA_API_LIMIT}). If the response reports _truncated=true, raise 'limit' or use match[] to filter. "
"Supports optional match[] parameter to filter. "
"By default returns values from metrics active in the last 1 hour (configurable via default_metadata_time_window_hrs)."
),
Expand Down Expand Up @@ -1178,6 +1210,7 @@ def __init__(self, toolset: "PrometheusToolset"):
type="string",
required=False,
),
"limit": metadata_limit_param(),
},
toolset=toolset,
)
Expand All @@ -1201,7 +1234,8 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
url = urljoin(
self.toolset.config.prometheus_url, f"api/v1/label/{label}/values"
)
query_params = {"limit": str(PROMETHEUS_METADATA_API_LIMIT)}
limit = get_metadata_limit(params)
query_params = {"limit": str(limit)}
if params.get("match"):
query_params["match[]"] = params["match"]

Expand Down Expand Up @@ -1232,16 +1266,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
response.raise_for_status()
data = response.json()

# Check if results were truncated
if (
"data" in data
and isinstance(data["data"], list)
and len(data["data"]) == PROMETHEUS_METADATA_API_LIMIT
):
data["_truncated"] = True
data["_message"] = (
f"Results truncated at limit={PROMETHEUS_METADATA_API_LIMIT}. Use match[] parameter to filter label '{label}' values."
)
mark_metadata_truncation(
data, limit, f"use match[] parameter to filter label '{label}' values."
)

return StructuredToolResult(
status=StructuredToolResultStatus.SUCCESS,
Expand Down Expand Up @@ -1269,7 +1296,7 @@ def __init__(self, toolset: "PrometheusToolset"):
description=(
"Get list of all label names using /api/v1/labels. "
"Use this to discover what labels are available across all metrics. "
f"Returns up to {PROMETHEUS_METADATA_API_LIMIT} label names (limit={PROMETHEUS_METADATA_API_LIMIT}). If {PROMETHEUS_METADATA_API_LIMIT} results returned, more may exist - use match[] to filter. "
f"Returns up to 'limit' label names (default {PROMETHEUS_METADATA_API_LIMIT}). If the response reports _truncated=true, raise 'limit' or use match[] to filter. "
"Supports optional match[] parameter to filter. "
"By default returns labels from metrics active in the last 1 hour (configurable via default_metadata_time_window_hrs)."
),
Expand All @@ -1292,6 +1319,7 @@ def __init__(self, toolset: "PrometheusToolset"):
type="string",
required=False,
),
"limit": metadata_limit_param(),
},
toolset=toolset,
)
Expand All @@ -1305,7 +1333,8 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
)
try:
url = urljoin(self.toolset.config.prometheus_url, "api/v1/labels")
query_params = {"limit": str(PROMETHEUS_METADATA_API_LIMIT)}
limit = get_metadata_limit(params)
query_params = {"limit": str(limit)}
if params.get("match"):
query_params["match[]"] = params["match"]

Expand Down Expand Up @@ -1336,16 +1365,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
response.raise_for_status()
data = response.json()

# Check if results were truncated
if (
"data" in data
and isinstance(data["data"], list)
and len(data["data"]) == PROMETHEUS_METADATA_API_LIMIT
):
data["_truncated"] = True
data["_message"] = (
f"Results truncated at limit={PROMETHEUS_METADATA_API_LIMIT}. Use match[] parameter to filter labels."
)
mark_metadata_truncation(
data, limit, "use match[] parameter to filter labels."
)

return StructuredToolResult(
status=StructuredToolResultStatus.SUCCESS,
Expand Down Expand Up @@ -1373,7 +1395,7 @@ def __init__(self, toolset: "PrometheusToolset"):
"Get time series using /api/v1/series. "
"Returns label sets for all time series matching the selector. "
"SLOWER than other discovery methods - use only when you need full label sets. "
f"Returns up to {PROMETHEUS_METADATA_API_LIMIT} series (limit={PROMETHEUS_METADATA_API_LIMIT}). If {PROMETHEUS_METADATA_API_LIMIT} results returned, more series exist - use more specific selector. "
f"Returns up to 'limit' series (default {PROMETHEUS_METADATA_API_LIMIT}). If the response reports _truncated=true, raise 'limit' or use a more specific selector. "
"Requires match[] parameter with PromQL selector. "
"By default returns series active in the last 1 hour (configurable via default_metadata_time_window_hrs)."
),
Expand All @@ -1397,6 +1419,7 @@ def __init__(self, toolset: "PrometheusToolset"):
type="string",
required=False,
),
"limit": metadata_limit_param(),
},
toolset=toolset,
)
Expand All @@ -1418,9 +1441,10 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
)

url = urljoin(self.toolset.config.prometheus_url, "api/v1/series")
limit = get_metadata_limit(params)
query_params = {
"match[]": match,
"limit": str(PROMETHEUS_METADATA_API_LIMIT),
"limit": str(limit),
}

# Add time parameters - use provided values or defaults
Expand Down Expand Up @@ -1450,16 +1474,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
response.raise_for_status()
data = response.json()

# Check if results were truncated
if (
"data" in data
and isinstance(data["data"], list)
and len(data["data"]) == PROMETHEUS_METADATA_API_LIMIT
):
data["_truncated"] = True
data["_message"] = (
f"Results truncated at limit={PROMETHEUS_METADATA_API_LIMIT}. Use a more specific match selector to see additional series."
)
mark_metadata_truncation(
data, limit, "use a more specific match selector to see additional series."
)

return StructuredToolResult(
status=StructuredToolResultStatus.SUCCESS,
Expand Down Expand Up @@ -1487,7 +1504,7 @@ def __init__(self, toolset: "PrometheusToolset"):
"Get metric metadata using /api/v1/metadata. "
"Returns type, help text, and unit for metrics. "
"Use after discovering metric names to get their descriptions. "
f"Returns up to {PROMETHEUS_METADATA_API_LIMIT} metrics (limit={PROMETHEUS_METADATA_API_LIMIT}). If {PROMETHEUS_METADATA_API_LIMIT} results returned, more may exist - filter by specific metric name. "
f"Returns up to 'limit' metrics (default {PROMETHEUS_METADATA_API_LIMIT}). If the response reports _truncated=true, raise 'limit' or filter by specific metric name. "
"Supports optional metric name filter."
),
parameters={
Expand All @@ -1499,6 +1516,7 @@ def __init__(self, toolset: "PrometheusToolset"):
type="string",
required=False,
),
"limit": metadata_limit_param(),
},
toolset=toolset,
)
Expand All @@ -1512,7 +1530,8 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
)
try:
url = urljoin(self.toolset.config.prometheus_url, "api/v1/metadata")
query_params = {"limit": str(PROMETHEUS_METADATA_API_LIMIT)}
limit = get_metadata_limit(params)
query_params = {"limit": str(limit)}

if params.get("metric"):
query_params["metric"] = params["metric"]
Expand All @@ -1529,16 +1548,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes
response.raise_for_status()
data = response.json()

# Check if results were truncated (metadata endpoint returns a dict, not a list)
if (
"data" in data
and isinstance(data["data"], dict)
and len(data["data"]) == PROMETHEUS_METADATA_API_LIMIT
):
data["_truncated"] = True
data["_message"] = (
f"Results truncated at limit={PROMETHEUS_METADATA_API_LIMIT}. Use metric parameter to filter by specific metric name."
)
mark_metadata_truncation(
data, limit, "use metric parameter to filter by specific metric name."
)

return StructuredToolResult(
status=StructuredToolResultStatus.SUCCESS,
Expand Down
Loading
Loading