diff --git a/.claude/skills/create-eval/SKILL.md b/.claude/skills/create-eval/SKILL.md index fbd1b00c68..c8020377fe 100644 --- a/.claude/skills/create-eval/SKILL.md +++ b/.claude/skills/create-eval/SKILL.md @@ -178,4 +178,4 @@ For detailed documentation, consult: - **`references/test-case-format.md`** — Complete test_case.yaml field reference with all options - **`references/anti-hallucination.md`** — Anti-cheat testing patterns and prompt design - **`references/infrastructure-patterns.md`** — Setup scripts, retry loops, port forwards, shared infra -- **`references/running-evals.md`** — CLI flags, environment variables, model comparison, debugging \ No newline at end of file +- **`references/running-evals.md`** — CLI flags, environment variables, model comparison, debugging diff --git a/.claude/skills/create-eval/references/anti-hallucination.md b/.claude/skills/create-eval/references/anti-hallucination.md index 462e17937a..62b29aafb1 100644 --- a/.claude/skills/create-eval/references/anti-hallucination.md +++ b/.claude/skills/create-eval/references/anti-hallucination.md @@ -143,4 +143,4 @@ Before finalizing any eval, verify: - [ ] User prompt uses business language, not implementation details - [ ] Expected output checks specific discoverable values - [ ] Test data looks realistic, not synthetic -- [ ] `expected_output` is invisible to the LLM (only the evaluator sees it) \ No newline at end of file +- [ ] `expected_output` is invisible to the LLM (only the evaluator sees it) diff --git a/.claude/skills/create-eval/references/test-case-format.md b/.claude/skills/create-eval/references/test-case-format.md index 646a02f90f..09966a2115 100644 --- a/.claude/skills/create-eval/references/test-case-format.md +++ b/.claude/skills/create-eval/references/test-case-format.md @@ -189,4 +189,4 @@ before_test: | after_test: | kubectl delete namespace app-212 --ignore-not-found -``` \ No newline at end of file +``` diff --git a/conftest.py b/conftest.py index 5ac1ba3e06..ea8d737bcd 100644 --- a/conftest.py +++ b/conftest.py @@ -121,14 +121,20 @@ def _patched_openai_init(self, *args, **kwargs): if confluence_base and not os.environ.get("CONFLUENCE_SA_BASE_URL"): parsed = urllib.parse.urlparse(confluence_base) if parsed.scheme not in ("http", "https"): - logging.warning(f"CONFLUENCE_BASE_URL has unsupported scheme '{parsed.scheme}', skipping SA URL derivation") + logging.warning( + f"CONFLUENCE_BASE_URL has unsupported scheme '{parsed.scheme}', skipping SA URL derivation" + ) else: try: tenant_url = f"{confluence_base.rstrip('/')}/_edge/tenant_info" with urllib.request.urlopen(tenant_url, timeout=10) as resp: cloud_id = json.loads(resp.read())["cloudId"] - os.environ["CONFLUENCE_SA_BASE_URL"] = f"https://api.atlassian.com/ex/confluence/{cloud_id}" - logging.info(f"Auto-derived CONFLUENCE_SA_BASE_URL from cloud ID {cloud_id}") + os.environ["CONFLUENCE_SA_BASE_URL"] = ( + f"https://api.atlassian.com/ex/confluence/{cloud_id}" + ) + logging.info( + f"Auto-derived CONFLUENCE_SA_BASE_URL from cloud ID {cloud_id}" + ) except Exception as e: logging.warning(f"Could not auto-derive CONFLUENCE_SA_BASE_URL: {e}") diff --git a/docs/data-sources/builtin-toolsets/splunk-mcp.md b/docs/data-sources/builtin-toolsets/splunk-mcp.md index 9d3cd6243a..0cb0ff47ae 100644 --- a/docs/data-sources/builtin-toolsets/splunk-mcp.md +++ b/docs/data-sources/builtin-toolsets/splunk-mcp.md @@ -161,4 +161,3 @@ holmes ask "Search Splunk for the most recent 10 error events" - [Splunk MCP Server on Splunkbase](https://splunkbase.splunk.com/app/7931) - [Splunk MCP Server Tools Reference](https://help.splunk.com/en/splunk-cloud-platform/mcp-server-for-splunk-platform/mcp-server-tools) - diff --git a/docs/development/evaluations/history/frontier_5_models_20260314_204516.md b/docs/development/evaluations/history/frontier_5_models_20260314_204516.md index 9e868c163b..64557a76b1 100644 --- a/docs/development/evaluations/history/frontier_5_models_20260314_204516.md +++ b/docs/development/evaluations/history/frontier_5_models_20260314_204516.md @@ -133,4 +133,4 @@ Status of all evaluations across models. Color coding: | [96_no_matching_runbook](https://github.com/HolmesGPT/holmesgpt/blob/master/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml) [🔗](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22metadata.eval_id%2520%253D%2520%252296_no_matching_runbook%2522%22%2C%20%22label%22%3A%20%22metadata.eval_id%2520equals%252096_no_matching_runbook%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) | [🟡 80% (4/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek-v3.2-chat%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek-v3.2-chat%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 196.5s / 💰 $0.02 | [🟡 60% (3/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini-3.1-pro-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini-3.1-pro-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 37.4s / 💰 $0.16 | [🔴 0% (0/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgpt-5.4%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgpt-5.4%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 51.1s / 💰 $0.19 | [🟢 100% (5/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bopus-4.6%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bopus-4.6%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 63.6s / 💰 $0.45 | [🟢 100% (5/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bsonnet-4.6%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bsonnet-4.6%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 60.7s / 💰 $0.33 | --- -*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-23093326433](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433).* \ No newline at end of file +*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-23093326433](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23093326433).* diff --git a/docs/development/evaluations/history/results_20260122_074259.md b/docs/development/evaluations/history/results_20260122_074259.md index b62f9b2232..bd9df31898 100644 --- a/docs/development/evaluations/history/results_20260122_074259.md +++ b/docs/development/evaluations/history/results_20260122_074259.md @@ -123,4 +123,4 @@ Status of all evaluations across models. Color coding: | [96_no_matching_runbook](https://github.com/HolmesGPT/holmesgpt/blob/master/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml) [🔗](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22metadata.eval_id%2520%253D%2520%252296_no_matching_runbook%2522%22%2C%20%22label%22%3A%20%22metadata.eval_id%2520equals%252096_no_matching_runbook%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) | [🟢 100% (5/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-haiku-4-5-20251001-v1%253A0%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-haiku-4-5-20251001-v1%253A0%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 47.8s / 💰 $0.10 | [🟡 40% (2/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-opus-4-5-20251101-v1%253A0%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-opus-4-5-20251101-v1%253A0%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 61.0s / 💰 $0.35 | [🟢 100% (5/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-sonnet-4-5-20250929-v1%253A0%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-sonnet-4-5-20250929-v1%253A0%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 54.1s / 💰 $0.26 | [🟡 20% (1/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini%252Fgemini-3-flash-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini%252Fgemini-3-flash-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 198.1s / 💰 $0.10 | [🟡 40% (2/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini%252Fgemini-3-pro-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini%252Fgemini-3-pro-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 112.8s / 💰 $0.28 | [🟡 40% (2/5)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bopenai%252Fgpt-5.2%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bopenai%252Fgpt-5.2%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 41.9s / 💰 $0.12 | --- -*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-21238150400](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400).* \ No newline at end of file +*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-21238150400](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21238150400).* diff --git a/docs/development/evaluations/history/results_20260127_161120.md b/docs/development/evaluations/history/results_20260127_161120.md index 0a238579c2..52ece3bdd3 100644 --- a/docs/development/evaluations/history/results_20260127_161120.md +++ b/docs/development/evaluations/history/results_20260127_161120.md @@ -129,4 +129,4 @@ Status of all evaluations across models. Color coding: | [96_no_matching_runbook](https://github.com/HolmesGPT/holmesgpt/blob/master/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml) [🔗](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22metadata.eval_id%2520%253D%2520%252296_no_matching_runbook%2522%22%2C%20%22label%22%3A%20%22metadata.eval_id%2520equals%252096_no_matching_runbook%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek%252Fdeepseek-chat%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek%252Fdeepseek-chat%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 160.5s / 💰 $0.03 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek%252Fdeepseek-reasoner%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek%252Fdeepseek-reasoner%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 366.0s / 💰 $0.03 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-haiku-4-5-20251001-v1%253A0%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-haiku-4-5-20251001-v1%253A0%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 52.2s / 💰 $0.11 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-opus-4-5-20251101-v1%253A0%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-opus-4-5-20251101-v1%253A0%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 83.8s / 💰 $0.48 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-sonnet-4-5-20250929-v1%253A0%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bbedrock%252Feu.anthropic.claude-sonnet-4-5-20250929-v1%253A0%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 57.7s / 💰 $0.29 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini%252Fgemini-3-flash-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini%252Fgemini-3-flash-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 105.7s / 💰 $0.12 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini%252Fgemini-3-pro-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini%252Fgemini-3-pro-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 883.4s / 💰 $0.29 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bopenai%252Fgpt-5.2%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bopenai%252Fgpt-5.2%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 73.8s / 💰 $0.24 | --- -*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-21401358384](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384).* \ No newline at end of file +*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-21401358384](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21401358384).* diff --git a/docs/development/evaluations/history/results_20260129_094857.md b/docs/development/evaluations/history/results_20260129_094857.md index cf37409382..94be8b13b5 100644 --- a/docs/development/evaluations/history/results_20260129_094857.md +++ b/docs/development/evaluations/history/results_20260129_094857.md @@ -131,4 +131,4 @@ Status of all evaluations across models. Color coding: | [96_no_matching_runbook](https://github.com/HolmesGPT/holmesgpt/blob/master/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml) [🔗](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22metadata.eval_id%2520%253D%2520%252296_no_matching_runbook%2522%22%2C%20%22label%22%3A%20%22metadata.eval_id%2520equals%252096_no_matching_runbook%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek-chat%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek-chat%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 205.2s / 💰 $0.03 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek-reasoner%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek-reasoner%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 422.5s / 💰 $0.03 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini-3-flash-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini-3-flash-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 34.6s / 💰 $0.10 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini-3-pro-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini-3-pro-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 117.1s / 💰 $0.25 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgpt-5.2-high-reasoning%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgpt-5.2-high-reasoning%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 836.0s / 💰 $0.75 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bhaiku-4.5%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bhaiku-4.5%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 56.3s / 💰 $0.10 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bkimi-2.5-openrouter%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bkimi-2.5-openrouter%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 218.4s | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bopus-4.5%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bopus-4.5%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 62.5s / 💰 $0.37 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bsonnet-4.5%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bsonnet-4.5%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 66.8s / 💰 $0.31 | --- -*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-21471579810](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810).* \ No newline at end of file +*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-21471579810](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-21471579810).* diff --git a/docs/development/evaluations/history/results_20260311_210836.md b/docs/development/evaluations/history/results_20260311_210836.md index 091598dd1b..5820ff4671 100644 --- a/docs/development/evaluations/history/results_20260311_210836.md +++ b/docs/development/evaluations/history/results_20260311_210836.md @@ -139,4 +139,4 @@ Status of all evaluations across models. Color coding: | [96_no_matching_runbook](https://github.com/HolmesGPT/holmesgpt/blob/master/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml) [🔗](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22metadata.eval_id%2520%253D%2520%252296_no_matching_runbook%2522%22%2C%20%22label%22%3A%20%22metadata.eval_id%2520equals%252096_no_matching_runbook%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek-r1-reasoner%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek-r1-reasoner%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 195.6s / 💰 $0.02 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek-v3.2-chat%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek-v3.2-chat%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 108.2s / 💰 $0.02 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini-3.1-pro-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini-3.1-pro-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 39.5s / 💰 $0.17 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgpt-5.3-codex%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgpt-5.3-codex%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 12.7s / 💰 $0.03 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgpt-5.4%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgpt-5.4%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 51.3s / 💰 $0.20 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bhaiku-4.5%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bhaiku-4.5%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 60.6s / 💰 $0.11 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bopus-4.6%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bopus-4.6%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 80.1s / 💰 $0.47 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bqwen-next-80B-instruct%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bqwen-next-80B-instruct%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 43.9s / 💰 $0.06 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bqwen-next-80B-thinking%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bqwen-next-80B-thinking%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 139.0s / 💰 $0.09 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bsonnet-4.6%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bsonnet-4.6%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 60.2s / 💰 $0.27 | --- -*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-22972733375](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375).* \ No newline at end of file +*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-22972733375](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-22972733375).* diff --git a/docs/development/evaluations/history/results_20260315_041151.md b/docs/development/evaluations/history/results_20260315_041151.md index 0129d95ee3..9206ca8927 100644 --- a/docs/development/evaluations/history/results_20260315_041151.md +++ b/docs/development/evaluations/history/results_20260315_041151.md @@ -139,4 +139,4 @@ Status of all evaluations across models. Color coding: | [96_no_matching_runbook](https://github.com/HolmesGPT/holmesgpt/blob/master/tests/llm/fixtures/test_ask_holmes/96_no_matching_runbook/test_case.yaml) [🔗](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22metadata.eval_id%2520%253D%2520%252296_no_matching_runbook%2522%22%2C%20%22label%22%3A%20%22metadata.eval_id%2520equals%252096_no_matching_runbook%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek-r1-reasoner%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek-r1-reasoner%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 489.4s / 💰 $0.02 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bdeepseek-v3.2-chat%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bdeepseek-v3.2-chat%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 216.6s / 💰 $0.02 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgemini-3.1-pro-preview%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgemini-3.1-pro-preview%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 117.4s / 💰 $0.28 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgpt-5.3-codex%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgpt-5.3-codex%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 13.1s / 💰 $0.03 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bgpt-5.4%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bgpt-5.4%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 81.7s / 💰 $0.14 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bhaiku-4.5%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bhaiku-4.5%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 51.3s / 💰 $0.09 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bopus-4.6%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bopus-4.6%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 56.0s / 💰 $0.46 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bqwen-next-80B-instruct%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bqwen-next-80B-instruct%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 55.0s / 💰 $0.06 | [🔴 0% (0/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bqwen-next-80B-thinking%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bqwen-next-80B-thinking%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 100.4s / 💰 $0.07 | [🟢 100% (1/1)](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491?c=&search=%7B%22filter%22%3A%20%5B%7B%22text%22%3A%20%22span_attributes.name%2520%253D%2520%252296_no_matching_runbook%255Bsonnet-4.6%255D%2522%22%2C%20%22label%22%3A%20%22Name%2520equals%252096_no_matching_runbook%255Bsonnet-4.6%255D%22%2C%20%22originType%22%3A%20%22form%22%7D%5D%7D) / ⏱️ 53.8s / 💰 $0.30 | --- -*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-23102181491](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491).* \ No newline at end of file +*Results are automatically generated and updated weekly. View full traces and detailed analysis in [Braintrust experiment: ci-benchmark-23102181491](https://www.braintrust.dev/app/robustadev/p/HolmesGPT/experiments/ci-benchmark-23102181491).* diff --git a/docs/development/evaluations/model-comparison-summary-20260315.md b/docs/development/evaluations/model-comparison-summary-20260315.md index 2fbf5b6bde..028863de22 100644 --- a/docs/development/evaluations/model-comparison-summary-20260315.md +++ b/docs/development/evaluations/model-comparison-summary-20260315.md @@ -65,4 +65,4 @@ sonnet-4.6 sometimes followed instructions too literally. For example, when told | **Cheapest** | Model(s) with the lowest average cost across the benchmark. | | **Most Expensive** | Model with the highest average cost across the benchmark. | | **Fastest** | Model with the lowest average runtime. | -| **Slowest** | Model with the highest average runtime. | \ No newline at end of file +| **Slowest** | Model with the highest average runtime. | diff --git a/examples/custom_llm.py b/examples/custom_llm.py index 6a6f867906..2bdf4b1240 100644 --- a/examples/custom_llm.py +++ b/examples/custom_llm.py @@ -68,7 +68,9 @@ def ask_holmes(): ) tool_executor = ToolExecutor(load_builtin_toolsets()) - ai = ToolCallingLLM(tool_executor, max_steps=100, llm=MyCustomLLM(), tool_results_dir=None) + ai = ToolCallingLLM( + tool_executor, max_steps=100, llm=MyCustomLLM(), tool_results_dir=None + ) user_prompt = generate_user_prompt(prompt, context={}) messages = [ diff --git a/experimental/ag-ui/server-agui.py b/experimental/ag-ui/server-agui.py index f5bcb9387a..0ae769fb9d 100644 --- a/experimental/ag-ui/server-agui.py +++ b/experimental/ag-ui/server-agui.py @@ -1,4 +1,34 @@ # ruff: noqa: E402 +from holmes.utils.stream import StreamEvents, StreamMessage +from holmes.core.models import ChatRequest +from holmes.core.conversations import build_chat_messages +from holmes.config import Config +from holmes.common.env_vars import HOLMES_HOST, HOLMES_PORT +from starlette.responses import PlainTextResponse +from fastapi.responses import JSONResponse, StreamingResponse +from fastapi.middleware.cors import CORSMiddleware +from fastapi import FastAPI, Request +from ag_ui.encoder import EventEncoder +from ag_ui.core import ( + AssistantMessage, + EventType, + RunAgentInput, + RunErrorEvent, + RunFinishedEvent, + RunStartedEvent, + TextMessageContentEvent, + TextMessageEndEvent, + TextMessageStartEvent, + ToolCallArgsEvent, + ToolCallEndEvent, + ToolCallStartEvent, +) +import uvicorn +import colorlog +import uuid +import time +import logging +import json import os from holmes.utils.cert_utils import add_custom_certificate @@ -11,46 +41,6 @@ # IMPORTING ABOVE MIGHT INITIALIZE AN HTTPS CLIENT THAT DOESN'T TRUST THE CUSTOM CERTIFICATE # Safe to import networked libs below -import json -import logging -import time -import uuid -import uvicorn -import colorlog - -from fastapi import FastAPI, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse, JSONResponse -from starlette.responses import PlainTextResponse - -from holmes.utils.stream import StreamMessage, StreamEvents -from holmes.common.env_vars import ( - HOLMES_HOST, - HOLMES_PORT, -) -from holmes.config import Config -from holmes.core.conversations import ( - build_chat_messages, -) -from holmes.core.models import ( - ChatRequest, -) - -from ag_ui.core import ( - AssistantMessage, - RunAgentInput, - EventType, - RunStartedEvent, - RunFinishedEvent, - TextMessageStartEvent, - TextMessageContentEvent, - TextMessageEndEvent, - ToolCallStartEvent, - ToolCallArgsEvent, - ToolCallEndEvent, - RunErrorEvent, -) -from ag_ui.encoder import EventEncoder def init_logging(): @@ -135,7 +125,7 @@ async def event_generator(message_history): msgs=message_history, enable_tool_approval=chat_request.enable_tool_approval or False, ) - for chunk in hgpt_chat_stream_response: + async for chunk in hgpt_chat_stream_response: if hasattr(chunk, "event"): event_type = ( chunk.event.value diff --git a/holmes/checks/__init__.py b/holmes/checks/__init__.py index 92cf90e6b7..e791b34696 100644 --- a/holmes/checks/__init__.py +++ b/holmes/checks/__init__.py @@ -1,5 +1,10 @@ """Holmes health checks module.""" +from holmes.checks.checks import ( + CheckRunner, + execute_check, + load_checks_config, +) from holmes.checks.models import ( Check, CheckMode, @@ -9,11 +14,6 @@ CheckStatus, DestinationConfig, ) -from holmes.checks.checks import ( - CheckRunner, - execute_check, - load_checks_config, -) __all__ = [ "Check", diff --git a/holmes/checks/checks_api.py b/holmes/checks/checks_api.py index 817ea43d9c..f09f26521d 100644 --- a/holmes/checks/checks_api.py +++ b/holmes/checks/checks_api.py @@ -114,7 +114,11 @@ def execute_health_check( notifications = [] # Send alerts if check failed and has destinations configured - if result.status == CheckStatus.FAIL and request.mode == CheckMode.ALERT and request.destinations: + if ( + result.status == CheckStatus.FAIL + and request.mode == CheckMode.ALERT + and request.destinations + ): try: # Create an Issue object for the failed check check_name = result.check_name diff --git a/holmes/common/cli_commons.py b/holmes/common/cli_commons.py index 061adbf723..a9355f6d7d 100644 --- a/holmes/common/cli_commons.py +++ b/holmes/common/cli_commons.py @@ -5,7 +5,6 @@ from holmes.config import DEFAULT_CONFIG_LOCATION - # Common cli options # The defaults for options that are also in the config file MUST be None or else the cli defaults will override settings in the config file opt_api_key: Optional[str] = typer.Option( diff --git a/holmes/config.py b/holmes/config.py index 4d37c35ca2..9edcec3e65 100644 --- a/holmes/config.py +++ b/holmes/config.py @@ -139,8 +139,6 @@ def llm_model_registry(self) -> LLMModelRegistry: self._llm_model_registry = LLMModelRegistry(self, dal=self.dal) return self._llm_model_registry - - def log_useful_info(self): if self.llm_model_registry.models: logging.info( @@ -545,9 +543,6 @@ def create_source( ticket_id: Optional[str], model: Optional[str] = None, ) -> TicketSource: - from holmes.plugins.sources.jira import JiraServiceManagementSource - from holmes.plugins.sources.pagerduty import PagerDutySource - TicketSource.model_rebuild() supported_sources = [s.value for s in SupportedTicketSources] if source not in supported_sources: diff --git a/holmes/core/azure_token.py b/holmes/core/azure_token.py index faf035bbb2..9a52e0add3 100644 --- a/holmes/core/azure_token.py +++ b/holmes/core/azure_token.py @@ -26,12 +26,17 @@ def get_azure_ad_token() -> str: with _lock: now = time.monotonic() - if _cached_token is not None and (now - _token_timestamp) < TOKEN_EXPIRY_SECONDS: + if ( + _cached_token is not None + and (now - _token_timestamp) < TOKEN_EXPIRY_SECONDS + ): return _cached_token logger.info("Fetching new Azure AD token for Azure OpenAI authentication") credential = DefaultAzureCredential() - token_provider = get_bearer_token_provider(credential, AZURE_COGNITIVE_SERVICES_SCOPE) + token_provider = get_bearer_token_provider( + credential, AZURE_COGNITIVE_SERVICES_SCOPE + ) _cached_token = token_provider() _token_timestamp = now return _cached_token diff --git a/holmes/core/json_schema_coerce.py b/holmes/core/json_schema_coerce.py index 49103c39da..24b37fe838 100644 --- a/holmes/core/json_schema_coerce.py +++ b/holmes/core/json_schema_coerce.py @@ -117,16 +117,26 @@ def _coerce_single_value( parsed = None if expected == "array" and isinstance(parsed, list): - logger.debug("Coerced param '%s' from string to array for tool '%s'", name, tool_name) + logger.debug( + "Coerced param '%s' from string to array for tool '%s'", name, tool_name + ) return parsed if expected == "object" and isinstance(parsed, dict): - logger.debug("Coerced param '%s' from string to object for tool '%s'", name, tool_name) + logger.debug( + "Coerced param '%s' from string to object for tool '%s'", + name, + tool_name, + ) return parsed # --- Single value → array wrap --- # LLM sends "cpu" instead of ["cpu"]. Safe: wrapping never loses data. if expected == "array" and not isinstance(value, list): - logger.debug("Coerced param '%s' by wrapping single value in array for tool '%s'", name, tool_name) + logger.debug( + "Coerced param '%s' by wrapping single value in array for tool '%s'", + name, + tool_name, + ) return [value] # --- Below this point: scalar coercions that strict mode skips. --- @@ -141,7 +151,11 @@ def _coerce_single_value( # "3.7" should NOT silently become 3. if as_float == int(as_float): coerced = int(as_float) - logger.debug("Coerced param '%s' from string to integer for tool '%s'", name, tool_name) + logger.debug( + "Coerced param '%s' from string to integer for tool '%s'", + name, + tool_name, + ) return coerced except (ValueError, OverflowError): pass @@ -151,7 +165,11 @@ def _coerce_single_value( if expected == "number" and isinstance(value, str): try: coerced = float(value) - logger.debug("Coerced param '%s' from string to number for tool '%s'", name, tool_name) + logger.debug( + "Coerced param '%s' from string to number for tool '%s'", + name, + tool_name, + ) return coerced except (ValueError, OverflowError): pass @@ -163,10 +181,18 @@ def _coerce_single_value( if expected == "boolean" and isinstance(value, str): lower = value.lower() if lower == "true": - logger.debug("Coerced param '%s' from string to boolean for tool '%s'", name, tool_name) + logger.debug( + "Coerced param '%s' from string to boolean for tool '%s'", + name, + tool_name, + ) return True if lower == "false": - logger.debug("Coerced param '%s' from string to boolean for tool '%s'", name, tool_name) + logger.debug( + "Coerced param '%s' from string to boolean for tool '%s'", + name, + tool_name, + ) return False return value diff --git a/holmes/core/llm.py b/holmes/core/llm.py index 28cf71b54b..5b55944e02 100644 --- a/holmes/core/llm.py +++ b/holmes/core/llm.py @@ -234,6 +234,28 @@ def completion( ) -> Union[ModelResponse, CustomStreamWrapper]: pass + async def acompletion( + self, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]] = [], + tool_choice: Optional[Union[str, dict]] = None, + response_format: Optional[Union[dict, Type[BaseModel]]] = None, + temperature: Optional[float] = None, + drop_params: Optional[bool] = None, + stream: Optional[bool] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + """Async version of completion. Subclasses may override for true async. + Default implementation falls back to sync completion.""" + return self.completion( + messages=messages, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + temperature=temperature, + drop_params=drop_params, + stream=stream, + ) + class DefaultLLM(LLM): model: str @@ -324,7 +346,10 @@ def check_llm( if ( os.environ.get("AWS_PROFILE") or os.environ.get("AWS_BEARER_TOKEN_BEDROCK") - or (os.environ.get("AWS_ROLE_ARN") and os.environ.get("AWS_WEB_IDENTITY_TOKEN_FILE")) + or ( + os.environ.get("AWS_ROLE_ARN") + and os.environ.get("AWS_WEB_IDENTITY_TOKEN_FILE") + ) ): model_requirements = {"keys_in_environment": True, "missing_keys": []} elif args.get("aws_access_key_id") and args.get("aws_secret_access_key"): @@ -336,7 +361,10 @@ def check_llm( session = boto3.Session() credentials = session.get_credentials() if credentials is not None: - model_requirements = {"keys_in_environment": True, "missing_keys": []} + model_requirements = { + "keys_in_environment": True, + "missing_keys": [], + } else: model_requirements = litellm.validate_environment( model=model, api_key=api_key, api_base=api_base @@ -359,7 +387,10 @@ def check_llm( if key in os.environ and key in model_requirements["missing_keys"]: model_requirements["missing_keys"].remove(key) # type: ignore # When using Azure AD token auth, AZURE_API_KEY is not required - if AZURE_AD_TOKEN_AUTH and "AZURE_API_KEY" in model_requirements["missing_keys"]: + if ( + AZURE_AD_TOKEN_AUTH + and "AZURE_API_KEY" in model_requirements["missing_keys"] + ): model_requirements["missing_keys"].remove("AZURE_API_KEY") # type: ignore if not model_requirements["missing_keys"]: @@ -490,7 +521,9 @@ def count_tokens( # wrong 85-per-image estimate. We add back the correct image tokens # (already computed in the per-message loop) after. if is_anthropic: - bulk_messages = [_strip_images(m) if _has_images(m) else m for m in messages] + bulk_messages = [ + _strip_images(m) if _has_images(m) else m for m in messages + ] else: bulk_messages = messages @@ -537,22 +570,23 @@ def get_litellm_corrected_name_for_robusta_ai(self) -> str: else: return self.model - def completion( + def _prepare_completion_kwargs( self, messages: List[Dict[str, Any]], - tools: Optional[List[Dict[str, Any]]] = None, - tool_choice: Optional[Union[str, dict]] = None, - response_format: Optional[Union[dict, Type[BaseModel]]] = None, - temperature: Optional[float] = None, - drop_params: Optional[bool] = None, - stream: Optional[bool] = None, - ) -> Union[ModelResponse, CustomStreamWrapper]: - tools_args = {} + tools: Optional[List[Dict[str, Any]]], + tool_choice: Optional[Union[str, dict]], + response_format: Optional[Union[dict, Type[BaseModel]]], + temperature: Optional[float], + drop_params: Optional[bool], + stream: Optional[bool], + ) -> Dict[str, Any]: + """Build the kwargs dict shared by completion() and acompletion().""" + tools_args: Dict[str, Any] = {} allowed_openai_params = None if tools and len(tools) > 0 and tool_choice == "auto": tools_args["tools"] = tools - tools_args["tool_choice"] = tool_choice # type: ignore + tools_args["tool_choice"] = tool_choice if THINKING: self.args.setdefault("thinking", json.loads(THINKING)) @@ -576,9 +610,6 @@ def completion( self.args.setdefault("temperature", temperature) - # Get the litellm module to use (wrapped or unwrapped) - litellm_to_use = self.tracer.wrap_llm(litellm) if self.tracer else litellm - # Strip internal fields (e.g. token_count cache) so provider APIs only # receive valid message schema fields. Shallow-copy only when needed to # avoid mutating the caller's dicts (which would invalidate the cache). @@ -596,14 +627,10 @@ def completion( # and pass it to litellm instead of an API key. azure_ad_kwargs: Dict[str, Any] = {} if AZURE_AD_TOKEN_AUTH and litellm_model_name.startswith("azure/"): - # For LiteLLM Azure provider, pass the bearer token via azure_ad_token - # LiteLLM will send it as Authorization: Bearer azure_ad_kwargs["azure_ad_token"] = get_azure_ad_token() - # Also, ensure we do not leak stale API keys when using Entra ID - # Leave api_key as None in completion call when AZURE_AD_TOKEN_AUTH is enabled self.api_key = None - result = litellm_to_use.completion( + return dict( model=litellm_model_name, api_key=self.api_key, base_url=self.api_base, @@ -625,6 +652,10 @@ def completion( ], ) + @staticmethod + def _validate_completion_result( + result: Any, + ) -> Union[ModelResponse, CustomStreamWrapper]: if isinstance(result, ModelResponse): return result elif isinstance(result, CustomStreamWrapper): @@ -632,6 +663,51 @@ def completion( else: raise Exception(f"Unexpected type returned by the LLM {type(result)}") + def completion( + self, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]] = None, + tool_choice: Optional[Union[str, dict]] = None, + response_format: Optional[Union[dict, Type[BaseModel]]] = None, + temperature: Optional[float] = None, + drop_params: Optional[bool] = None, + stream: Optional[bool] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + kwargs = self._prepare_completion_kwargs( + messages, + tools, + tool_choice, + response_format, + temperature, + drop_params, + stream, + ) + litellm_to_use = self.tracer.wrap_llm(litellm) if self.tracer else litellm + result = litellm_to_use.completion(**kwargs) + return self._validate_completion_result(result) + + async def acompletion( + self, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]] = None, + tool_choice: Optional[Union[str, dict]] = None, + response_format: Optional[Union[dict, Type[BaseModel]]] = None, + temperature: Optional[float] = None, + drop_params: Optional[bool] = None, + stream: Optional[bool] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + kwargs = self._prepare_completion_kwargs( + messages, + tools, + tool_choice, + response_format, + temperature, + drop_params, + stream, + ) + result = await litellm.acompletion(**kwargs) + return self._validate_completion_result(result) + def get_maximum_output_token(self) -> int: max_output_tokens = floor(min(64000, self.get_context_window_size() / 5)) diff --git a/holmes/core/llm_usage.py b/holmes/core/llm_usage.py index 033af1470c..1e113ba477 100644 --- a/holmes/core/llm_usage.py +++ b/holmes/core/llm_usage.py @@ -64,7 +64,9 @@ def extract_usage_from_response(response: ModelResponse) -> dict: cached_tokens = _extract_detail_field(prompt_details, "cached_tokens") completion_details = usage.get("completion_tokens_details", None) if completion_details: - reasoning_tokens = _extract_detail_field(completion_details, "reasoning_tokens") or 0 + reasoning_tokens = ( + _extract_detail_field(completion_details, "reasoning_tokens") or 0 + ) except (AttributeError, TypeError, KeyError): logging.debug("Could not extract token usage from LLM response") diff --git a/holmes/core/openai_formatting.py b/holmes/core/openai_formatting.py index 042fe895fa..d265606f58 100644 --- a/holmes/core/openai_formatting.py +++ b/holmes/core/openai_formatting.py @@ -26,7 +26,9 @@ def type_to_open_ai_schema(param_attributes: Any, strict_mode: bool) -> dict[str is_nullable_from_schema = False if isinstance(raw_type, list): - non_null_types = [t.strip() if isinstance(t, str) else t for t in raw_type if t != "null"] + non_null_types = [ + t.strip() if isinstance(t, str) else t for t in raw_type if t != "null" + ] is_nullable_from_schema = "null" in raw_type param_type = non_null_types[0] if non_null_types else "string" else: @@ -48,7 +50,9 @@ def type_to_open_ai_schema(param_attributes: Any, strict_mode: bool) -> dict[str type_obj["additionalProperties"] = False # Preserve additionalProperties schema for dynamic-key objects - elif hasattr(param_attributes, "additional_properties") and param_attributes.additional_properties not in (None, False): + elif hasattr( + param_attributes, "additional_properties" + ) and param_attributes.additional_properties not in (None, False): type_obj["additionalProperties"] = param_attributes.additional_properties elif strict_mode: type_obj["additionalProperties"] = False @@ -82,13 +86,19 @@ def type_to_open_ai_schema(param_attributes: Any, strict_mode: bool) -> dict[str # Merge passthrough JSON Schema keywords (minItems, maxItems, minimum, etc.) # so the LLM sees validation constraints from the source schema. - if type_obj and hasattr(param_attributes, "json_schema_extra") and param_attributes.json_schema_extra: + if ( + type_obj + and hasattr(param_attributes, "json_schema_extra") + and param_attributes.json_schema_extra + ): type_obj.update(param_attributes.json_schema_extra) # Add nullability using anyOf per the OpenAI Structured Outputs spec when strict mode # requires optional params to accept null, or when the source schema explicitly marks # the field as nullable (e.g., MCP ["string", "null"]). - if type_obj and (is_nullable_from_schema or (strict_mode and not param_attributes.required)): + if type_obj and ( + is_nullable_from_schema or (strict_mode and not param_attributes.required) + ): type_obj = {"anyOf": [type_obj, {"type": "null"}]} return type_obj @@ -101,7 +111,9 @@ def format_tool_to_open_ai_standard( # However, tools with dynamic-key objects (additionalProperties with a schema) are # automatically excluded from strict mode since both OpenAI and Anthropic require # additionalProperties: false on all objects in strict mode. - strict_mode = STRICT_TOOL_CALLS_ENABLED and _is_tool_strict_compatible(tool_parameters) + strict_mode = STRICT_TOOL_CALLS_ENABLED and _is_tool_strict_compatible( + tool_parameters + ) tool_properties = {} diff --git a/holmes/core/scheduled_prompts/executor.py b/holmes/core/scheduled_prompts/executor.py index b7f02b2ff0..39d4daca46 100644 --- a/holmes/core/scheduled_prompts/executor.py +++ b/holmes/core/scheduled_prompts/executor.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import os @@ -17,9 +18,7 @@ SCHEDULED_PROMPTS_INACTIVE_POLL_INTERVAL_SECONDS, ) from holmes.core.models import ChatRequest, ChatResponse -from holmes.core.scheduled_prompts.heartbeat_tracer import ( - ScheduledPromptsHeartbeatSpan, -) +from holmes.core.scheduled_prompts.heartbeat_tracer import ScheduledPromptsHeartbeatSpan from holmes.core.scheduled_prompts.models import ScheduledPrompt from holmes.core.supabase_dal import RunStatus @@ -30,10 +29,13 @@ from holmes.config import Config from holmes.core.supabase_dal import SupabaseDal -ChatFunction = Callable[[ChatRequest, Request], Union["ChatResponse", "StreamingResponse"]] +ChatFunction = Callable[ + [ChatRequest, Request], Union["ChatResponse", "StreamingResponse"] +] ADDITIONAL_SYSTEM_PROMPT_URL = f"{ROBUSTA_UI_DOMAIN}/api/additional-system-prompt.json" + class ScheduledPromptsExecutor: def __init__( self, @@ -201,7 +203,12 @@ def _execute_prompt( ) empty_request = Request(scope={"type": "http", "headers": []}) - response = self.chat_function(chat_request, empty_request) + result = self.chat_function(chat_request, empty_request) + # chat_function may be async (returns a coroutine) + if asyncio.iscoroutine(result): + response = asyncio.run(result) + else: + response = result duration_seconds = time.perf_counter() - start if isinstance(response, ChatResponse): diff --git a/holmes/core/tool_calling_llm.py b/holmes/core/tool_calling_llm.py index d48e3455fd..6df6eddbc2 100644 --- a/holmes/core/tool_calling_llm.py +++ b/holmes/core/tool_calling_llm.py @@ -1,10 +1,11 @@ +import asyncio import concurrent.futures import json import logging import re import threading from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Type, Union +from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Type, Union import sentry_sdk from openai import BadRequestError @@ -20,12 +21,7 @@ ) from holmes.core.llm import LLM from holmes.core.llm_usage import RequestStats - -from holmes.core.models import ( - PendingToolApproval, - ToolApprovalDecision, - ToolCallResult, -) +from holmes.core.models import PendingToolApproval, ToolApprovalDecision, ToolCallResult from holmes.core.safeguards import prevent_overly_repeated_tool_call from holmes.core.tools import ( StructuredToolResult, @@ -51,6 +47,7 @@ ) from holmes.utils.tags import parse_messages_tags + class LLMInterruptedError(Exception): """Raised when the user interrupts an in-progress LLM call (e.g. via Escape key).""" @@ -189,7 +186,7 @@ def _has_bash_for_file_access(self) -> bool: return False return False - def _execute_tool_decisions( + async def _execute_tool_decisions( self, messages: List[Dict[str, Any]], tool_decisions: List[ToolApprovalDecision], @@ -249,7 +246,7 @@ def _execute_tool_decisions( decision = tool_call_with_decision.decision tool_result: Optional[ToolCallResult] = None if decision and decision.approved: - tool_result = self._invoke_llm_tool_call( + tool_result = await self._invoke_llm_tool_call( tool_to_call=tool_call, previous_tool_calls=[], trace_span=trace_span, @@ -261,7 +258,11 @@ def _execute_tool_decisions( ) else: # Tool was rejected or no decision found, add rejection message - feedback_text = f" User feedback: {decision.feedback}" if decision and decision.feedback else "" + feedback_text = ( + f" User feedback: {decision.feedback}" + if decision and decision.feedback + else "" + ) tool_result = ToolCallResult( tool_call_id=tool_call.id, tool_name=tool_call.function.name, @@ -323,94 +324,115 @@ def call( # type: ignore cancel_event: Optional[threading.Event] = None, approval_callback: Optional[ApprovalCallback] = None, ) -> LLMResult: - """Synchronous wrapper around call_stream(). Drains the generator + """Synchronous wrapper around call_stream(). Drains the async generator and reconstructs an LLMResult.""" - all_tool_calls: list[dict] = [] - tool_decisions: Optional[List[ToolApprovalDecision]] = None - total_num_llm_calls = 0 - accumulated_stats = RequestStats() - - while True: - stream = self.call_stream( - msgs=messages, - response_format=response_format, - enable_tool_approval=approval_callback is not None, - tool_decisions=tool_decisions, - trace_span=trace_span, - cancel_event=cancel_event, - tool_number_offset=tool_number_offset, - request_context=request_context, - iteration_offset=total_num_llm_calls, - ) + async def _drain() -> LLMResult: + all_tool_calls: list[dict] = [] + tool_decisions: Optional[List[ToolApprovalDecision]] = None + total_num_llm_calls = 0 + accumulated_stats = RequestStats() - tool_decisions = None - terminal_data = None - terminal_event = None - start_tool_count = 0 - saw_tool_results = False - - for event in stream: - # Log blank line when a tool batch ends (transition away from TOOL_RESULT) - if saw_tool_results and event.event != StreamEvents.TOOL_RESULT: - logging.info("") - saw_tool_results = False - - if event.event == StreamEvents.START_TOOL: - start_tool_count += 1 - elif event.event == StreamEvents.TOOL_RESULT: - tool_number_offset += 1 - saw_tool_results = True - all_tool_calls.append(event.data) - if start_tool_count > 0: - logging.info( - f"The AI requested [bold]{start_tool_count}[/bold] tool call(s)." - ) - start_tool_count = 0 - elif event.event == StreamEvents.AI_MESSAGE: - reasoning = event.data.get("reasoning") - content = event.data.get("content") - if reasoning: - logging.info( - f"[italic dim]AI reasoning:\n\n{reasoning}[/italic dim]\n" - ) - if content and content.strip(): - logging.info( - f"[bold {AI_COLOR}]AI:[/bold {AI_COLOR}] {content}" - ) - elif event.event in (StreamEvents.ANSWER_END, StreamEvents.APPROVAL_REQUIRED): - terminal_data = event.data - terminal_event = event.event - break - - if not terminal_data: - raise Exception("Stream ended without ANSWER_END or APPROVAL_REQUIRED") - - # call_stream returns the absolute iteration count (including offset), - # so we assign rather than accumulate to avoid double-counting. - total_num_llm_calls = terminal_data.get("num_llm_calls", 0) - accumulated_stats += RequestStats(**terminal_data.get("costs", {})) - - if terminal_event == StreamEvents.APPROVAL_REQUIRED: - messages = terminal_data["messages"] - tool_decisions = self._prompt_for_approval_decisions( - terminal_data["pending_approvals"], - approval_callback, + while True: + stream = self.call_stream( + msgs=messages, + response_format=response_format, + enable_tool_approval=approval_callback is not None, + tool_decisions=tool_decisions, + trace_span=trace_span, + cancel_event=cancel_event, + tool_number_offset=tool_number_offset, + request_context=request_context, + iteration_offset=total_num_llm_calls, ) - continue - # ANSWER_END — deduplicate tool calls keeping last per ID - deduped: dict[str, dict] = {} - for tc in all_tool_calls: - deduped[tc.get("tool_call_id", id(tc))] = tc - return LLMResult( - result=terminal_data["content"], - tool_calls=list(deduped.values()), - num_llm_calls=total_num_llm_calls, - messages=terminal_data["messages"], - metadata=terminal_data.get("metadata"), - **accumulated_stats.model_dump(), - ) + tool_decisions = None + terminal_data = None + terminal_event = None + start_tool_count = 0 + saw_tool_results = False + + async for event in stream: + # Log blank line when a tool batch ends (transition away from TOOL_RESULT) + if saw_tool_results and event.event != StreamEvents.TOOL_RESULT: + logging.info("") + saw_tool_results = False + + if event.event == StreamEvents.START_TOOL: + start_tool_count += 1 + elif event.event == StreamEvents.TOOL_RESULT: + tool_number_offset += 1 + saw_tool_results = True + all_tool_calls.append(event.data) + if start_tool_count > 0: + logging.info( + f"The AI requested [bold]{start_tool_count}[/bold] tool call(s)." + ) + start_tool_count = 0 + elif event.event == StreamEvents.AI_MESSAGE: + reasoning = event.data.get("reasoning") + content = event.data.get("content") + if reasoning: + logging.info( + f"[italic dim]AI reasoning:\n\n{reasoning}[/italic dim]\n" + ) + if content and content.strip(): + logging.info( + f"[bold {AI_COLOR}]AI:[/bold {AI_COLOR}] {content}" + ) + elif event.event in ( + StreamEvents.ANSWER_END, + StreamEvents.APPROVAL_REQUIRED, + ): + terminal_data = event.data + terminal_event = event.event + break + + if not terminal_data: + raise Exception( + "Stream ended without ANSWER_END or APPROVAL_REQUIRED" + ) + + # call_stream returns the absolute iteration count (including offset), + # so we assign rather than accumulate to avoid double-counting. + total_num_llm_calls = terminal_data.get("num_llm_calls", 0) + accumulated_stats += RequestStats(**terminal_data.get("costs", {})) + + if terminal_event == StreamEvents.APPROVAL_REQUIRED: + messages = terminal_data["messages"] + tool_decisions = self._prompt_for_approval_decisions( + terminal_data["pending_approvals"], + approval_callback, + ) + continue + + # ANSWER_END — deduplicate tool calls keeping last per ID + deduped: dict[str, dict] = {} + for tc in all_tool_calls: + deduped[tc.get("tool_call_id", id(tc))] = tc + return LLMResult( + result=terminal_data["content"], + tool_calls=list(deduped.values()), + num_llm_calls=total_num_llm_calls, + messages=terminal_data["messages"], + metadata=terminal_data.get("metadata"), + **accumulated_stats.model_dump(), + ) + + # If there's already a running event loop (e.g. inside FastAPI), create a + # new thread to run the coroutine. Otherwise use asyncio.run() directly. + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop and loop.is_running(): + # We're inside an existing event loop — run in a new thread. + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(asyncio.run, _drain()) + return future.result() + else: + return asyncio.run(_drain()) def _prompt_for_approval_decisions( self, @@ -431,25 +453,31 @@ def _prompt_for_approval_decisions( # the prefix to disk, making this tool no longer need approval. if self._is_tool_call_already_approved(approval.tool_name, approval.params): logging.info(f"Approval no longer needed for {approval.tool_name}") - decisions.append(ToolApprovalDecision( - tool_call_id=approval.tool_call_id, - approved=True, - )) + decisions.append( + ToolApprovalDecision( + tool_call_id=approval.tool_call_id, + approved=True, + ) + ) continue if not approval_callback: - decisions.append(ToolApprovalDecision( - tool_call_id=approval.tool_call_id, - approved=False, - )) + decisions.append( + ToolApprovalDecision( + tool_call_id=approval.tool_call_id, + approved=False, + ) + ) continue approved, feedback = approval_callback(approval) - decisions.append(ToolApprovalDecision( - tool_call_id=approval.tool_call_id, - approved=approved, - feedback=feedback if not approved else None, - )) + decisions.append( + ToolApprovalDecision( + tool_call_id=approval.tool_call_id, + approved=approved, + feedback=feedback if not approved else None, + ) + ) return decisions @@ -553,7 +581,7 @@ def _log_tool_call_result( }, ) - def _invoke_llm_tool_call( + async def _invoke_llm_tool_call( self, tool_to_call: ChatCompletionMessageToolCall, previous_tool_calls: list[dict], @@ -581,7 +609,9 @@ def _invoke_llm_tool_call( params=None, ), ) - ToolCallingLLM._log_tool_call_result(tool_span, tool_call_result, enable_tool_approval) + ToolCallingLLM._log_tool_call_result( + tool_span, tool_call_result, enable_tool_approval + ) return tool_call_result tool_name = tool_to_call.function.name @@ -605,14 +635,19 @@ def _invoke_llm_tool_call( ) if not tool_response: - tool_response = self._directly_invoke_tool_call( - tool_name=tool_name, - tool_params=tool_params, - user_approved=user_approved, - tool_number=tool_number, - tool_call_id=tool_id, - session_approved_prefixes=session_approved_prefixes, - request_context=request_context, + # Run sync tool execution in a thread pool to avoid blocking the event loop + loop = asyncio.get_event_loop() + tool_response = await loop.run_in_executor( + None, + lambda: self._directly_invoke_tool_call( + tool_name=tool_name, + tool_params=tool_params, + user_approved=user_approved, + tool_number=tool_number, + tool_call_id=tool_id, + session_approved_prefixes=session_approved_prefixes, + request_context=request_context, + ), ) tool = self.tool_executor.get_tool_by_name(tool_name) @@ -686,7 +721,7 @@ def _emit_token_count( metadata["costs"] = stats.model_dump() return build_stream_event_token_count(metadata=metadata) - def call_stream( + async def call_stream( self, response_format: Optional[Union[dict, Type[BaseModel]]] = None, msgs: Optional[list[dict]] = None, @@ -697,10 +732,10 @@ def call_stream( cancel_event: Optional[threading.Event] = None, tool_number_offset: int = 0, iteration_offset: int = 0, - ): + ) -> AsyncGenerator[StreamMessage, None]: """ - This function DOES NOT call llm.completion(stream=true). - This function streams holmes one iteration at a time instead of waiting for all iterations to complete. + Async generator that streams Holmes one iteration at a time. + Uses litellm acompletion for async LLM calls and asyncio.gather for parallel tool execution. """ if trace_span is None: trace_span = DummySpan() @@ -710,7 +745,7 @@ def call_stream( # Process tool decisions if provided if msgs and tool_decisions: logging.info(f"Processing {len(tool_decisions)} tool decisions") - msgs, events = self._execute_tool_decisions( + msgs, events = await self._execute_tool_decisions( msgs, tool_decisions, request_context, trace_span=trace_span ) for ev in events: @@ -748,12 +783,14 @@ def call_stream( llm=self.llm, messages=messages, tools=tools ) except CompactionInsufficientError as e: - yield from e.events + for ev in e.events: + yield ev if e.compaction_usage and e.compaction_usage.total_tokens > 0: stats += e.compaction_usage raise - yield from limit_result.events + for ev in limit_result.events: + yield ev messages = limit_result.messages metadata = metadata | limit_result.metadata @@ -786,7 +823,7 @@ def call_stream( logging.debug(f"sending messages={messages}\n\ntools={tools}") try: - full_response = self.llm.completion( + full_response = await self.llm.acompletion( messages=parse_messages_tags(messages), # type: ignore tools=tools, tool_choice=tool_choice, @@ -804,7 +841,9 @@ def call_stream( f"Tokens: {response_stats.prompt_tokens} prompt + {response_stats.completion_tokens} completion = {response_stats.total_tokens} total" ) elif response_stats.total_cost > 0: - cost_logger.debug(f"LLM iteration cost: ${response_stats.total_cost:.6f} | Token usage not available") + cost_logger.debug( + f"LLM iteration cost: ${response_stats.total_cost:.6f} | Token usage not available" + ) if LOG_LLM_USAGE_RESPONSE: usage = getattr(full_response, "usage", None) if usage: @@ -844,7 +883,9 @@ def call_stream( ) ) - yield self._emit_token_count(messages, tools, full_response, limit_result, metadata, stats) + yield self._emit_token_count( + messages, tools, full_response, limit_result, metadata, stats + ) tools_to_call = getattr(response_message, "tool_calls", None) if not tools_to_call: @@ -880,13 +921,13 @@ def call_stream( # Extract session approved prefixes from conversation history session_prefixes = extract_bash_session_prefixes(messages) - with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: - futures = [] - for tool_index, t in enumerate(tools_to_call, 1): # type: ignore - tool_number = tool_number_offset + tool_index + # Execute tool calls concurrently using asyncio.gather + tool_tasks = [] + for tool_index, t in enumerate(tools_to_call, 1): # type: ignore + tool_number = tool_number_offset + tool_index - future = executor.submit( - self._invoke_llm_tool_call, + tool_tasks.append( + self._invoke_llm_tool_call( tool_to_call=t, # type: ignore previous_tool_calls=tool_calls, trace_span=trace_span, @@ -895,58 +936,46 @@ def call_stream( request_context=request_context, enable_tool_approval=enable_tool_approval, ) - futures.append(future) - yield StreamMessage( - event=StreamEvents.START_TOOL, - data={"tool_name": t.function.name, "id": t.id}, - ) - - for future in concurrent.futures.as_completed(futures): - if cancel_event and cancel_event.is_set(): - for f in futures: - f.cancel() - raise LLMInterruptedError() - - tool_call_result: ToolCallResult = future.result() + ) + yield StreamMessage( + event=StreamEvents.START_TOOL, + data={"tool_name": t.function.name, "id": t.id}, + ) - tool_result_dict = tool_call_result.to_client_dict() + tool_results: list[ToolCallResult] = await asyncio.gather(*tool_tasks) - if ( - tool_call_result.result.status - == StructuredToolResultStatus.APPROVAL_REQUIRED - ): - if enable_tool_approval: - pending_approvals.append( - PendingToolApproval( - tool_call_id=tool_call_result.tool_call_id, - tool_name=tool_call_result.tool_name, - description=tool_call_result.description, - params=tool_call_result.result.params or {}, - ) - ) - - all_tool_calls.append(tool_result_dict) - yield StreamMessage( - event=StreamEvents.TOOL_RESULT, - data=tool_result_dict, - ) - else: - tool_call_result.result.status = ( - StructuredToolResultStatus.ERROR - ) - tool_call_result.result.error = f"Tool call rejected for security reasons: {tool_call_result.result.error}" - tool_result_dict = tool_call_result.to_client_dict() + if cancel_event and cancel_event.is_set(): + raise LLMInterruptedError() - tool_calls.append(tool_result_dict) - all_tool_calls.append(tool_result_dict) - messages.append(tool_call_result.to_llm_message()) + for tool_call_result in tool_results: + tool_result_dict = tool_call_result.to_client_dict() - yield StreamMessage( - event=StreamEvents.TOOL_RESULT, - data=tool_result_dict, + if ( + tool_call_result.result.status + == StructuredToolResultStatus.APPROVAL_REQUIRED + ): + if enable_tool_approval: + pending_approvals.append( + PendingToolApproval( + tool_call_id=tool_call_result.tool_call_id, + tool_name=tool_call_result.tool_name, + description=tool_call_result.description, + params=tool_call_result.result.params or {}, ) + ) + all_tool_calls.append(tool_result_dict) + yield StreamMessage( + event=StreamEvents.TOOL_RESULT, + data=tool_result_dict, + ) else: + tool_call_result.result.status = ( + StructuredToolResultStatus.ERROR + ) + tool_call_result.result.error = f"Tool call rejected for security reasons: {tool_call_result.result.error}" + tool_result_dict = tool_call_result.to_client_dict() + tool_calls.append(tool_result_dict) all_tool_calls.append(tool_result_dict) messages.append(tool_call_result.to_llm_message()) @@ -956,45 +985,57 @@ def call_stream( data=tool_result_dict, ) - # Emit updated token counts after tool results - yield self._emit_token_count(messages, tools, full_response, limit_result, metadata, stats) - - # If we have approval required tools, end the stream with pending approvals - if pending_approvals: - # Mark pending tool calls in assistant messages - for approval in pending_approvals: - tool_call = self.find_assistant_tool_call_request( - tool_call_id=approval.tool_call_id, messages=messages - ) - tool_call["pending_approval"] = True + else: + tool_calls.append(tool_result_dict) + all_tool_calls.append(tool_result_dict) + messages.append(tool_call_result.to_llm_message()) - # End stream with approvals required yield StreamMessage( - event=StreamEvents.APPROVAL_REQUIRED, - data={ - "content": None, - "messages": messages, - "pending_approvals": [ - approval.model_dump() for approval in pending_approvals - ], - "requires_approval": True, - "num_llm_calls": i, - "costs": stats.model_dump(), - }, + event=StreamEvents.TOOL_RESULT, + data=tool_result_dict, ) - return - # Update the tool number offset for the next iteration - tool_number_offset += len(tools_to_call) + # Emit updated token counts after tool results + yield self._emit_token_count( + messages, tools, full_response, limit_result, metadata, stats + ) - # Re-fetch tools if runbook was just activated (enables restricted tools) - if self._runbook_in_use and tools is not None: - new_tools = self._get_tools() - if len(new_tools) != len(tools): - logging.info( - f"Runbook activated - refreshing tools list ({len(tools)} -> {len(new_tools)} tools)" - ) - tools = new_tools + # If we have approval required tools, end the stream with pending approvals + if pending_approvals: + # Mark pending tool calls in assistant messages + for approval in pending_approvals: + tool_call = self.find_assistant_tool_call_request( + tool_call_id=approval.tool_call_id, messages=messages + ) + tool_call["pending_approval"] = True + + # End stream with approvals required + yield StreamMessage( + event=StreamEvents.APPROVAL_REQUIRED, + data={ + "content": None, + "messages": messages, + "pending_approvals": [ + approval.model_dump() for approval in pending_approvals + ], + "requires_approval": True, + "num_llm_calls": i, + "costs": stats.model_dump(), + }, + ) + return + + # Update the tool number offset for the next iteration + tool_number_offset += len(tools_to_call) + + # Re-fetch tools if runbook was just activated (enables restricted tools) + if self._runbook_in_use and tools is not None: + new_tools = self._get_tools() + if len(new_tools) != len(tools): + logging.info( + f"Runbook activated - refreshing tools list ({len(tools)} -> {len(new_tools)} tools)" + ) + tools = new_tools raise Exception( f"Too many LLM calls - exceeded max_steps: {i}/{self.max_steps}" diff --git a/holmes/core/tools.py b/holmes/core/tools.py index f0d694b456..ea86151c1b 100644 --- a/holmes/core/tools.py +++ b/holmes/core/tools.py @@ -26,9 +26,6 @@ ) from jinja2 import Template - -from holmes.core.json_schema_coerce import coerce_params -from requests.structures import CaseInsensitiveDict from pydantic import ( BaseModel, ConfigDict, @@ -37,9 +34,11 @@ PrivateAttr, model_validator, ) +from requests.structures import CaseInsensitiveDict from rich.console import Console from rich.table import Table +from holmes.core.json_schema_coerce import coerce_params from holmes.core.llm import LLM from holmes.core.openai_formatting import format_tool_to_open_ai_standard from holmes.core.transformers import ( @@ -117,7 +116,9 @@ def stringify_data(self, compact: bool = True) -> Tuple[str, bool]: return self.data.model_dump_json(indent=None if compact else 2), True else: if compact: - return json.dumps(self.data, separators=(",", ":"), ensure_ascii=False), True + return json.dumps( + self.data, separators=(",", ":"), ensure_ascii=False + ), True else: return json.dumps(self.data, indent=2, ensure_ascii=False), True except Exception: @@ -197,7 +198,10 @@ def is_strict_compatible(self) -> bool: are incompatible with strict mode. """ # If this parameter has additionalProperties with a schema or True, it's not strict-compatible - if self.additional_properties is not None and self.additional_properties is not False: + if ( + self.additional_properties is not None + and self.additional_properties is not False + ): return False # Recursively check nested properties if self.properties: @@ -678,7 +682,14 @@ class ToolsetEnvironmentPrerequisite(BaseModel): env: List[str] = [] # optional -def _prereq_priority(prereq: Union[StaticPrerequisite, ToolsetCommandPrerequisite, ToolsetEnvironmentPrerequisite, CallablePrerequisite]) -> int: +def _prereq_priority( + prereq: Union[ + StaticPrerequisite, + ToolsetCommandPrerequisite, + ToolsetEnvironmentPrerequisite, + CallablePrerequisite, + ], +) -> int: """Priority ordering for prerequisite checks. Lower number = higher priority. Static checks and env vars are fast config-validity checks (0-1). diff --git a/holmes/core/tools_utils/tool_context_window_limiter.py b/holmes/core/tools_utils/tool_context_window_limiter.py index ed2eebdeb7..18113f47dc 100644 --- a/holmes/core/tools_utils/tool_context_window_limiter.py +++ b/holmes/core/tools_utils/tool_context_window_limiter.py @@ -38,7 +38,9 @@ def prevent_overly_big_tool_response( message = tool_call_result.to_llm_message() messages_token = llm.count_tokens(messages=[message]).total_tokens max_tokens_allowed = llm.get_max_token_count_for_single_tool() - logging.debug(f"prevent_overly_big_tool_response: count_tokens took {(time.monotonic() - t0) * 1000:.1f}ms for {tool_call_result.tool_name} ({messages_token} tokens)") + logging.debug( + f"prevent_overly_big_tool_response: count_tokens took {(time.monotonic() - t0) * 1000:.1f}ms for {tool_call_result.tool_name} ({messages_token} tokens)" + ) if tool_call_result.result.status != StructuredToolResultStatus.SUCCESS: return messages_token diff --git a/holmes/core/tools_utils/tool_executor.py b/holmes/core/tools_utils/tool_executor.py index 27755f1821..edae6362aa 100644 --- a/holmes/core/tools_utils/tool_executor.py +++ b/holmes/core/tools_utils/tool_executor.py @@ -67,7 +67,9 @@ def ensure_toolset_initialized(self, tool_name: str) -> Optional[str]: if toolset.needs_initialization: if not toolset.lazy_initialize(): - error_msg = f"Toolset '{toolset.name}' failed to initialize: {toolset.error}" + error_msg = ( + f"Toolset '{toolset.name}' failed to initialize: {toolset.error}" + ) logging.error(error_msg) return error_msg elif toolset.status == ToolsetStatusEnum.FAILED: diff --git a/holmes/core/truncation/compaction.py b/holmes/core/truncation/compaction.py index bd0fd35c6e..8e0dc2b43b 100644 --- a/holmes/core/truncation/compaction.py +++ b/holmes/core/truncation/compaction.py @@ -72,7 +72,10 @@ def compact_conversation_history( logging.error( "Failed to compact conversation history. Unexpected LLM's response for compaction" ) - return CompactionResult(messages_after_compaction=original_conversation_history, usage=compaction_usage) + return CompactionResult( + messages_after_compaction=original_conversation_history, + usage=compaction_usage, + ) compacted_conversation_history: list[dict] = [] if system_prompt_message: diff --git a/holmes/core/truncation/input_context_window_limiter.py b/holmes/core/truncation/input_context_window_limiter.py index 730c0a5517..457027de23 100644 --- a/holmes/core/truncation/input_context_window_limiter.py +++ b/holmes/core/truncation/input_context_window_limiter.py @@ -53,7 +53,12 @@ def check_compaction_needed( class CompactionInsufficientError(Exception): """Raised when conversation compaction was not sufficient to fit the context window.""" - def __init__(self, message: str, events: list[StreamMessage], compaction_usage: Optional[RequestStats] = None): + def __init__( + self, + message: str, + events: list[StreamMessage], + compaction_usage: Optional[RequestStats] = None, + ): super().__init__(message) self.events = events self.compaction_usage = compaction_usage @@ -90,13 +95,17 @@ def limit_input_context_window( original_conversation_history=messages, llm=llm ) compaction_usage = compaction_result.usage - compacted_tokens = llm.count_tokens(compaction_result.messages_after_compaction, tools=tools) + compacted_tokens = llm.count_tokens( + compaction_result.messages_after_compaction, tools=tools + ) compacted_total_tokens = compacted_tokens.total_tokens if compacted_total_tokens < initial_tokens.total_tokens: messages = compaction_result.messages_after_compaction num_messages_after = len(messages) - compression_ratio = round((1 - compacted_total_tokens / initial_tokens.total_tokens) * 100, 1) + compression_ratio = round( + (1 - compacted_total_tokens / initial_tokens.total_tokens) * 100, 1 + ) compaction_message = f"The conversation history has been compacted from {initial_tokens.total_tokens} to {compacted_total_tokens} tokens" logging.info(compaction_message) conversation_history_compacted = True @@ -171,10 +180,14 @@ def limit_input_context_window( data={"content": failure_msg}, ) ) - raise CompactionInsufficientError(failure_msg, events=events, compaction_usage=compaction_usage) + raise CompactionInsufficientError( + failure_msg, events=events, compaction_usage=compaction_usage + ) elapsed_ms = (time.monotonic() - t0) * 1000 - logging.debug(f"limit_input_context_window: {elapsed_ms:.1f}ms total | {tokens.total_tokens} tokens") + logging.debug( + f"limit_input_context_window: {elapsed_ms:.1f}ms total | {tokens.total_tokens} tokens" + ) return ContextWindowLimiterOutput( events=events, diff --git a/holmes/interactive.py b/holmes/interactive.py index b04b185c3a..3476bb3326 100644 --- a/holmes/interactive.py +++ b/holmes/interactive.py @@ -48,8 +48,8 @@ FeedbackCallback, UserFeedback, ) -from holmes.core.prompt import PromptComponent, build_initial_ask_messages from holmes.core.models import PendingToolApproval +from holmes.core.prompt import PromptComponent, build_initial_ask_messages from holmes.core.tool_calling_llm import ( ApprovalCallback, LLMInterruptedError, @@ -65,6 +65,7 @@ from holmes.plugins.toolsets.bash.common.cli_prefixes import ( save_cli_bash_tools_approved_prefixes as _save_approved_prefixes, ) +from holmes.toolset_config_tui import run_toolset_config_tui from holmes.utils.colors import ( AI_COLOR, ERROR_COLOR, @@ -73,7 +74,6 @@ TOOLS_COLOR, USER_COLOR, ) -from holmes.toolset_config_tui import run_toolset_config_tui from holmes.utils.console.consts import agent_name from holmes.utils.file_utils import write_json_file from holmes.version import check_version_async @@ -1138,9 +1138,7 @@ def _wait_for_completion_or_escape( ch = sys.stdin.read(1) if ch == "\x1b": # Disambiguate standalone Escape from escape sequences (arrow keys etc.) - ready2, _, _ = select_module.select( - [sys.stdin], [], [], 0.05 - ) + ready2, _, _ = select_module.select([sys.stdin], [], [], 0.05) if ready2: # Part of an escape sequence — consume and discard sys.stdin.read(1) @@ -1196,6 +1194,7 @@ def run_interactive_loop( if bash_always_allow: approval_callback = lambda _: (True, None) elif not bash_always_deny: + def approval_handler( pending_approval: PendingToolApproval, ) -> tuple[bool, Optional[str]]: @@ -1530,7 +1529,9 @@ def _run_ai_call( ai_thread.start() interrupted = _wait_for_completion_or_escape( - ai_thread, cancel_event, approval_active, + ai_thread, + cancel_event, + approval_active, terminal_restored, ) diff --git a/holmes/main.py b/holmes/main.py index cd8498b386..52a628585a 100644 --- a/holmes/main.py +++ b/holmes/main.py @@ -59,7 +59,6 @@ app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False) - investigate_app = typer.Typer( add_completion=False, name="investigate", @@ -438,7 +437,6 @@ def alertmanager( model: Optional[str] = opt_model, config_file: Optional[Path] = opt_config_file, # type: ignore custom_toolsets: Optional[List[Path]] = opt_custom_toolsets, - max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, # advanced options for this command @@ -469,7 +467,9 @@ def alertmanager( ) with tool_result_storage() as tool_results_dir: - ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + ai = config.create_console_toolcalling_llm( + model_name=model, tool_results_dir=tool_results_dir + ) source = config.create_alertmanager_source() @@ -566,7 +566,6 @@ def jira( model: Optional[str] = opt_model, config_file: Optional[Path] = opt_config_file, # type: ignore custom_toolsets: Optional[List[Path]] = opt_custom_toolsets, - max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, json_output_file: Optional[str] = opt_json_output_file, @@ -600,7 +599,9 @@ def jira( results = [] with tool_result_storage() as tool_results_dir: - ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + ai = config.create_console_toolcalling_llm( + model_name=model, tool_results_dir=tool_results_dir + ) for i, issue in enumerate(issues): console.print( f"[bold yellow]Analyzing Jira ticket {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" @@ -609,7 +610,9 @@ def jira( console.print(Rule()) console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print( + Markdown(result.result.replace("\n", "\n\n")), style="bold green" + ) # type: ignore console.print(Rule()) if update: source.write_back_result(issue.id, result) @@ -690,7 +693,9 @@ def ticket( return with tool_result_storage() as tool_results_dir: - ai = ticket_source.config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + ai = ticket_source.config.create_console_toolcalling_llm( + model_name=model, tool_results_dir=tool_results_dir + ) # Render ticket-specific additions ticket_additions = load_and_render_prompt( @@ -767,7 +772,6 @@ def github( model: Optional[str] = opt_model, config_file: Optional[Path] = opt_config_file, # type: ignore custom_toolsets: Optional[List[Path]] = opt_custom_toolsets, - max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, ): @@ -799,7 +803,9 @@ def github( f"[bold yellow]Analyzing {len(issues)} GitHub Issues.[/bold yellow] [red]Press Ctrl+C to stop.[/red]" ) with tool_result_storage() as tool_results_dir: - ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + ai = config.create_console_toolcalling_llm( + model_name=model, tool_results_dir=tool_results_dir + ) for i, issue in enumerate(issues): console.print( f"[bold yellow]Analyzing GitHub issue {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" @@ -809,7 +815,9 @@ def github( console.print(Rule()) console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print( + Markdown(result.result.replace("\n", "\n\n")), style="bold green" + ) # type: ignore console.print(Rule()) if update: source.write_back_result(issue.id, result) @@ -842,7 +850,6 @@ def pagerduty( model: Optional[str] = opt_model, config_file: Optional[Path] = opt_config_file, # type: ignore custom_toolsets: Optional[List[Path]] = opt_custom_toolsets, - max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, json_output_file: Optional[str] = opt_json_output_file, @@ -875,7 +882,9 @@ def pagerduty( results = [] with tool_result_storage() as tool_results_dir: - ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + ai = config.create_console_toolcalling_llm( + model_name=model, tool_results_dir=tool_results_dir + ) for i, issue in enumerate(issues): console.print( f"[bold yellow]Analyzing PagerDuty incident {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" @@ -885,7 +894,9 @@ def pagerduty( console.print(Rule()) console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print( + Markdown(result.result.replace("\n", "\n\n")), style="bold green" + ) # type: ignore console.print(Rule()) if update: source.write_back_result(issue.id, result) @@ -918,7 +929,6 @@ def opsgenie( model: Optional[str] = opt_model, config_file: Optional[Path] = opt_config_file, # type: ignore custom_toolsets: Optional[List[Path]] = opt_custom_toolsets, - max_steps: Optional[int] = opt_max_steps, verbose: Optional[List[bool]] = opt_verbose, documents: Optional[str] = opt_documents, @@ -949,7 +959,9 @@ def opsgenie( f"[bold yellow]Analyzing {len(issues)} OpsGenie alerts.[/bold yellow] [red]Press Ctrl+C to stop.[/red]" ) with tool_result_storage() as tool_results_dir: - ai = config.create_console_toolcalling_llm(model_name=model, tool_results_dir=tool_results_dir) + ai = config.create_console_toolcalling_llm( + model_name=model, tool_results_dir=tool_results_dir + ) for i, issue in enumerate(issues): console.print( f"[bold yellow]Analyzing OpsGenie alert {i+1}/{len(issues)}: {issue.name}...[/bold yellow]" @@ -958,7 +970,9 @@ def opsgenie( console.print(Rule()) console.print(f"[bold green]AI analysis of {issue.url}[/bold green]") - console.print(Markdown(result.result.replace("\n", "\n\n")), style="bold green") # type: ignore + console.print( + Markdown(result.result.replace("\n", "\n\n")), style="bold green" + ) # type: ignore console.print(Rule()) if update: source.write_back_result(issue.id, result) diff --git a/holmes/plugins/toolsets/__init__.py b/holmes/plugins/toolsets/__init__.py index cab1ba2a0e..1016b10530 100644 --- a/holmes/plugins/toolsets/__init__.py +++ b/holmes/plugins/toolsets/__init__.py @@ -20,7 +20,6 @@ from holmes.plugins.toolsets.connectivity_check import ConnectivityCheckToolset from holmes.plugins.toolsets.coralogix.toolset_coralogix import CoralogixToolset from holmes.plugins.toolsets.database.database import DatabaseToolset -from holmes.plugins.toolsets.mongodb.mongodb import MongoDBToolset from holmes.plugins.toolsets.datadog.toolset_datadog_general import ( DatadogGeneralToolset, ) @@ -51,6 +50,7 @@ from holmes.plugins.toolsets.kubectl_run.kubectl_run_toolset import KubectlRunToolset from holmes.plugins.toolsets.kubernetes_logs import KubernetesLogsToolset from holmes.plugins.toolsets.mcp.toolset_mcp import RemoteMCPToolset +from holmes.plugins.toolsets.mongodb.mongodb import MongoDBToolset from holmes.plugins.toolsets.newrelic.newrelic import NewRelicToolset from holmes.plugins.toolsets.rabbitmq.toolset_rabbitmq import RabbitMQToolset from holmes.plugins.toolsets.robusta.robusta import RobustaToolset diff --git a/holmes/plugins/toolsets/bash/validation.py b/holmes/plugins/toolsets/bash/validation.py index 5041f0ee9a..a5c025bf56 100644 --- a/holmes/plugins/toolsets/bash/validation.py +++ b/holmes/plugins/toolsets/bash/validation.py @@ -15,7 +15,6 @@ from bashlex import ast from holmes.common.env_vars import HOLMES_TOOL_RESULT_STORAGE_PATH, load_bool - from holmes.plugins.toolsets.bash.common.config import ( HARDCODED_BLOCKS, BashExecutorConfig, @@ -160,7 +159,9 @@ def check_hardcoded_blocks(segment: str) -> Optional[str]: return None -def check_blocked_in_raw_command(command: str, blocked_list: List[str]) -> Optional[str]: +def check_blocked_in_raw_command( + command: str, blocked_list: List[str] +) -> Optional[str]: """ Check for blocked patterns anywhere in a raw command string using word boundaries. diff --git a/holmes/plugins/toolsets/confluence/confluence.py b/holmes/plugins/toolsets/confluence/confluence.py index 0450d50f8f..95b91f7715 100644 --- a/holmes/plugins/toolsets/confluence/confluence.py +++ b/holmes/plugins/toolsets/confluence/confluence.py @@ -50,7 +50,9 @@ class ConfluenceConfig(ToolsetConfig): @model_validator(mode="after") def validate_auth(self) -> "ConfluenceConfig": if self.auth_type == "basic" and not self.user: - raise ValueError("'user' is required when auth_type is 'basic'. For PATs, set auth_type to 'bearer'.") + raise ValueError( + "'user' is required when auth_type is 'basic'. For PATs, set auth_type to 'bearer'." + ) return self @@ -98,7 +100,9 @@ def _resolve_cloud_id(self) -> Optional[str]: if self._conf.cloud_id: return self._conf.cloud_id try: - resp = requests.get(f"{self._conf.api_url.rstrip('/')}/_edge/tenant_info", timeout=10) + resp = requests.get( + f"{self._conf.api_url.rstrip('/')}/_edge/tenant_info", timeout=10 + ) resp.raise_for_status() cloud_id = resp.json().get("cloudId") if cloud_id: @@ -114,7 +118,9 @@ def _activate_gateway(self, cloud_id: str) -> None: # ── Health check ── - def _probe_request(self, path: str, query_params: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + def _probe_request( + self, path: str, query_params: Optional[Dict[str, str]] = None + ) -> Dict[str, Any]: """Direct HTTP request for health-check probing.""" base = (self._gateway_base_url or self._conf.api_url).rstrip("/") prefix = self._conf.api_path_prefix.rstrip("/") @@ -127,7 +133,9 @@ def _probe_request(self, path: str, query_params: Optional[Dict[str, str]] = Non else: auth = (self._conf.user or "", self._conf.api_key) - response = requests.get(url, params=query_params, auth=auth, headers=headers, timeout=30) + response = requests.get( + url, params=query_params, auth=auth, headers=headers, timeout=30 + ) response.raise_for_status() return response.json() @@ -140,13 +148,20 @@ def _perform_health_check(self) -> Tuple[bool, str]: return True, "Confluence API is accessible." except requests.exceptions.HTTPError as e: status = e.response.status_code - if status in (401, 403) and self._is_cloud_url() and not self._gateway_base_url: + if ( + status in (401, 403) + and self._is_cloud_url() + and not self._gateway_base_url + ): ok, msg = self._try_gateway_fallback() if ok: return True, msg return False, f"Confluence API error: HTTP {status}: {e.response.text}" except requests.exceptions.ConnectionError as e: - return False, f"Failed to connect to Confluence at {self._conf.api_url}: {e}" + return ( + False, + f"Failed to connect to Confluence at {self._conf.api_url}: {e}", + ) except requests.exceptions.Timeout: return False, "Confluence health check timed out" except Exception as e: @@ -160,7 +175,10 @@ def _try_gateway_fallback(self) -> Tuple[bool, str]: self._activate_gateway(cloud_id) try: self._probe_request("/rest/api/space", query_params={"limit": "1"}) - return True, "Confluence API is accessible via Atlassian API gateway (scoped token)." + return ( + True, + "Confluence API is accessible via Atlassian API gateway (scoped token).", + ) except Exception as e: self._gateway_base_url = None return False, f"Confluence API gateway fallback failed: {e}" @@ -180,7 +198,11 @@ def _build_endpoint_config(self) -> EndpointConfig: if self._conf.auth_type == "bearer" or self._gateway_base_url: auth = AuthConfig(type="bearer", token=self._conf.api_key) else: - auth = AuthConfig(type="basic", username=self._conf.user or "", password=self._conf.api_key) + auth = AuthConfig( + type="basic", + username=self._conf.user or "", + password=self._conf.api_key, + ) return EndpointConfig( hosts=[host], @@ -226,7 +248,9 @@ def _setup_http_tools(self) -> None: ) ok, msg = http_toolset.prerequisites_callable(http_config.model_dump()) if not ok: - raise RuntimeError(f"Failed to initialize HTTP toolset for Confluence: {msg}") + raise RuntimeError( + f"Failed to initialize HTTP toolset for Confluence: {msg}" + ) self.tools = http_toolset.tools self.llm_instructions = http_toolset.llm_instructions diff --git a/holmes/plugins/toolsets/coralogix/utils.py b/holmes/plugins/toolsets/coralogix/utils.py index 996a488d11..5bcd6f5143 100644 --- a/holmes/plugins/toolsets/coralogix/utils.py +++ b/holmes/plugins/toolsets/coralogix/utils.py @@ -89,7 +89,9 @@ def handle_deprecated_fields(self): deprecated.append("team_hostname -> team_slug") if deprecated: - logging.warning(f"Coralogix: deprecated config field names: {', '.join(deprecated)}") + logging.warning( + f"Coralogix: deprecated config field names: {', '.join(deprecated)}" + ) return self diff --git a/holmes/plugins/toolsets/database/database.py b/holmes/plugins/toolsets/database/database.py index 83afab9fb6..dc548a299e 100644 --- a/holmes/plugins/toolsets/database/database.py +++ b/holmes/plugins/toolsets/database/database.py @@ -5,6 +5,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type from urllib.parse import urlparse +import sqlalchemy from pydantic import ConfigDict, Field from holmes.core.tools import ( @@ -20,8 +21,6 @@ from holmes.plugins.toolsets.utils import toolset_name_for_one_liner from holmes.utils.pydantic_utils import ToolsetConfig -import sqlalchemy - logger = logging.getLogger(__name__) # SQL statements that are safe for read-only access @@ -187,7 +186,6 @@ def __init__(self, name: str = "database/sql", **kwargs: Any): + self._user_llm_instructions ) - def prerequisites_callable(self, config: Dict[str, Any]) -> Tuple[bool, str]: try: self.config = DatabaseConfig(**config) @@ -238,9 +236,7 @@ def _create_engine(self, url: str): connect_args["TrustServerCertificate"] = "yes" return sqlalchemy.create_engine( - url, - pool_pre_ping=True, - connect_args=connect_args + url, pool_pre_ping=True, connect_args=connect_args ) @property @@ -274,7 +270,9 @@ def execute_query(self, sql: str, limit: Optional[int] = None) -> Dict[str, Any] f"Received: {sql[:80]}" ) - effective_limit = min(limit or self.database_config.max_rows, self.database_config.max_rows) + effective_limit = min( + limit or self.database_config.max_rows, self.database_config.max_rows + ) url = _normalise_url(self.database_config.connection_url) engine = self._create_engine(url) try: @@ -310,7 +308,9 @@ def execute_query(self, sql: str, limit: Optional[int] = None) -> Dict[str, Any] "rows": [], "row_count": 0, "truncated": False, - "rows_affected": result.rowcount if result.rowcount >= 0 else None, + "rows_affected": result.rowcount + if result.rowcount >= 0 + else None, } finally: engine.dispose() diff --git a/holmes/plugins/toolsets/datadog/toolset_datadog_general.py b/holmes/plugins/toolsets/datadog/toolset_datadog_general.py index 8de031b8af..4d9a0e0c2d 100644 --- a/holmes/plugins/toolsets/datadog/toolset_datadog_general.py +++ b/holmes/plugins/toolsets/datadog/toolset_datadog_general.py @@ -43,7 +43,10 @@ # Monitors (r"^/api/v\d+/monitor(/search)?$", ""), (r"^/api/v\d+/monitor/\d+$", "Get a specific monitor by ID"), - (r"^/api/v1/monitor/groups/search$", "Search monitor groups (v1 only, no v2 equivalent)"), + ( + r"^/api/v1/monitor/groups/search$", + "Search monitor groups (v1 only, no v2 equivalent)", + ), # Dashboards (r"^/api/v\d+/dashboard(/lists)?$", ""), (r"^/api/v\d+/dashboard/[^/]+$", ""), @@ -100,7 +103,10 @@ # Downtimes (r"^/api/v\d+/downtime$", "List scheduled downtimes"), (r"^/api/v\d+/downtime/[^/]+$", "Get specific downtime by ID"), - (r"^/api/v2/monitor/\d+/downtime_matches$", "Get active downtimes for a specific monitor"), + ( + r"^/api/v2/monitor/\d+/downtime_matches$", + "Get active downtimes for a specific monitor", + ), # Tags (r"^/api/v\d+/tags/hosts(/[^/]+)?$", ""), # Notebooks diff --git a/holmes/plugins/toolsets/datadog/toolset_datadog_logs.py b/holmes/plugins/toolsets/datadog/toolset_datadog_logs.py index 50f7b6e9c5..07d5eb34ca 100644 --- a/holmes/plugins/toolsets/datadog/toolset_datadog_logs.py +++ b/holmes/plugins/toolsets/datadog/toolset_datadog_logs.py @@ -3,7 +3,6 @@ import os from typing import Any, ClassVar, Dict, Optional, Tuple, Type - from holmes.core.tools import ( CallablePrerequisite, StructuredToolResult, diff --git a/holmes/plugins/toolsets/datadog/toolset_datadog_metrics.py b/holmes/plugins/toolsets/datadog/toolset_datadog_metrics.py index cc5996f57a..bd35793177 100644 --- a/holmes/plugins/toolsets/datadog/toolset_datadog_metrics.py +++ b/holmes/plugins/toolsets/datadog/toolset_datadog_metrics.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import Any, Optional, Tuple - from holmes.core.tools import ( CallablePrerequisite, ClassVar, @@ -528,7 +527,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes for metric_name in metric_names: try: - api_url = f"{self.toolset.dd_config.api_url}/api/v1/metrics/{metric_name}" + api_url = ( + f"{self.toolset.dd_config.api_url}/api/v1/metrics/{metric_name}" + ) data = execute_datadog_http_request( url=api_url, diff --git a/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py b/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py index ce4fec9f59..ae80870fd1 100644 --- a/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py +++ b/holmes/plugins/toolsets/datadog/toolset_datadog_traces.py @@ -7,7 +7,6 @@ import re from typing import Any, Dict, Optional, Tuple - from holmes.core.tools import ( CallablePrerequisite, ClassVar, diff --git a/holmes/plugins/toolsets/elasticsearch/elasticsearch.py b/holmes/plugins/toolsets/elasticsearch/elasticsearch.py index 8e4b93fab2..63983514fd 100644 --- a/holmes/plugins/toolsets/elasticsearch/elasticsearch.py +++ b/holmes/plugins/toolsets/elasticsearch/elasticsearch.py @@ -125,7 +125,9 @@ def __init__(self, name: str, description: str, tools: list, **kwargs): def prerequisites_callable(self, config: Dict[str, Any]) -> Tuple[bool, str]: """Check if the Elasticsearch configuration is valid and the cluster is reachable.""" try: - config_class = self.config_classes[0] if self.config_classes else ElasticsearchConfig + config_class = ( + self.config_classes[0] if self.config_classes else ElasticsearchConfig + ) self.config = config_class(**config) return self._perform_health_check() except Exception as e: @@ -159,7 +161,10 @@ def _perform_health_check(self) -> Tuple[bool, str]: ) except requests.exceptions.SSLError as e: error_msg = str(e) - if "certificate required" in error_msg.lower() or "sslcertverificationerror" in error_msg.lower(): + if ( + "certificate required" in error_msg.lower() + or "sslcertverificationerror" in error_msg.lower() + ): return ( False, f"Elasticsearch SSL/TLS error: {error_msg}. " @@ -202,7 +207,10 @@ def _get_auth(self) -> Optional[Tuple[str, str]]: def _get_client_cert(self) -> Optional[Tuple[str, str]]: """Return client certificate tuple for mTLS if configured.""" - if self.elasticsearch_config.client_cert and self.elasticsearch_config.client_key: + if ( + self.elasticsearch_config.client_cert + and self.elasticsearch_config.client_key + ): return ( self.elasticsearch_config.client_cert, self.elasticsearch_config.client_key, @@ -471,13 +479,13 @@ def __init__(self, toolset: ElasticsearchBaseToolset): "Fields to include/exclude in response. Supported formats:\n" "• Array: ['field1', 'field2'] - Include only these fields\n" "• String: 'field1' - Include single field\n" - "• Object: {\"includes\": [\"trace.*\", \"span.*\"], \"excludes\": [\"*.body\", \"*.stack_trace\"]}\n" + '• Object: {"includes": ["trace.*", "span.*"], "excludes": ["*.body", "*.stack_trace"]}\n' " - Use wildcards (*) for pattern matching\n" " - Excludes are useful for filtering large fields (http.request.body, error.stack_trace, http.response.*)\n" "• Boolean: false - Exclude all source (metadata only)\n\n" "Examples:\n" - "- Trace query: {\"includes\": [\"trace.*\", \"span.*\", \"service.*\"], \"excludes\": [\"*.request.*\", \"*.response.*\"]}\n" - "- Logs: [\"@timestamp\", \"message\", \"level\", \"service.name\"]" + '- Trace query: {"includes": ["trace.*", "span.*", "service.*"], "excludes": ["*.request.*", "*.response.*"]}\n' + '- Logs: ["@timestamp", "message", "level", "service.name"]' ), type="object", required=False, diff --git a/holmes/plugins/toolsets/grafana/base_grafana_toolset.py b/holmes/plugins/toolsets/grafana/base_grafana_toolset.py index 672516f1a7..92b5944a39 100644 --- a/holmes/plugins/toolsets/grafana/base_grafana_toolset.py +++ b/holmes/plugins/toolsets/grafana/base_grafana_toolset.py @@ -37,7 +37,9 @@ def prerequisites_callable(self, config: dict[str, Any]) -> Tuple[bool, str]: return False, TOOLSET_CONFIG_MISSING_ERROR try: - config_class = self.config_classes[0] if self.config_classes else GrafanaConfig + config_class = ( + self.config_classes[0] if self.config_classes else GrafanaConfig + ) self._grafana_config = config_class(**config) return self.health_check() diff --git a/holmes/plugins/toolsets/grafana/common.py b/holmes/plugins/toolsets/grafana/common.py index 3c7735a85e..5bf4693e04 100644 --- a/holmes/plugins/toolsets/grafana/common.py +++ b/holmes/plugins/toolsets/grafana/common.py @@ -75,11 +75,27 @@ def get_base_url(config: GrafanaConfig) -> str: class GrafanaTempoLabelsConfig(ToolsetConfig): - pod: str = Field(default="k8s.pod.name", title="Pod Label", description="Label for pod name") - namespace: str = Field(default="k8s.namespace.name", title="Namespace Label", description="Label for namespace") - deployment: str = Field(default="k8s.deployment.name", title="Deployment Label", description="Label for deployment") - node: str = Field(default="k8s.node.name", title="Node Label", description="Label for node name") - service: str = Field(default="service.name", title="Service Label", description="Label for service name") + pod: str = Field( + default="k8s.pod.name", title="Pod Label", description="Label for pod name" + ) + namespace: str = Field( + default="k8s.namespace.name", + title="Namespace Label", + description="Label for namespace", + ) + deployment: str = Field( + default="k8s.deployment.name", + title="Deployment Label", + description="Label for deployment", + ) + node: str = Field( + default="k8s.node.name", title="Node Label", description="Label for node name" + ) + service: str = Field( + default="service.name", + title="Service Label", + description="Label for service name", + ) class GrafanaTempoConfig(GrafanaConfig): diff --git a/holmes/plugins/toolsets/grafana/toolset_grafana.py b/holmes/plugins/toolsets/grafana/toolset_grafana.py index 677c91221a..c96e764aca 100644 --- a/holmes/plugins/toolsets/grafana/toolset_grafana.py +++ b/holmes/plugins/toolsets/grafana/toolset_grafana.py @@ -48,7 +48,9 @@ def _build_grafana_dashboard_url( class GrafanaToolset(BaseGrafanaToolset): - config_classes: ClassVar[list[Type[GrafanaDashboardConfig]]] = [GrafanaDashboardConfig] + config_classes: ClassVar[list[Type[GrafanaDashboardConfig]]] = [ + GrafanaDashboardConfig + ] def __init__(self): super().__init__( diff --git a/holmes/plugins/toolsets/http/http_toolset.py b/holmes/plugins/toolsets/http/http_toolset.py index 3c1f29e3bc..1888a8ee44 100644 --- a/holmes/plugins/toolsets/http/http_toolset.py +++ b/holmes/plugins/toolsets/http/http_toolset.py @@ -2,7 +2,6 @@ import json import logging import os -import re from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple, Type from urllib.parse import urlparse @@ -157,10 +156,20 @@ def prerequisites_callable(self, config: Dict[str, Any]) -> Tuple[bool, str]: "No endpoints configured. Add at least one endpoint with hosts and auth.", ) - if self._http_config.client_cert_path and not os.path.isfile(self._http_config.client_cert_path): - return False, f"Client certificate file not found: {self._http_config.client_cert_path}" - if self._http_config.client_key_path and not os.path.isfile(self._http_config.client_key_path): - return False, f"Client key file not found: {self._http_config.client_key_path}" + if self._http_config.client_cert_path and not os.path.isfile( + self._http_config.client_cert_path + ): + return ( + False, + f"Client certificate file not found: {self._http_config.client_cert_path}", + ) + if self._http_config.client_key_path and not os.path.isfile( + self._http_config.client_key_path + ): + return ( + False, + f"Client key file not found: {self._http_config.client_key_path}", + ) for i, endpoint in enumerate(self._http_config.endpoints): if not endpoint.hosts: @@ -168,7 +177,10 @@ def prerequisites_callable(self, config: Dict[str, Any]) -> Tuple[bool, str]: for method in endpoint.get_methods(): if method not in ALL_METHODS: - return False, f"Endpoint {i} has invalid method: {method}. Allowed: {ALL_METHODS}" + return ( + False, + f"Endpoint {i} has invalid method: {method}. Allowed: {ALL_METHODS}", + ) # Perform health checks for i, endpoint in enumerate(self._http_config.endpoints): @@ -189,9 +201,15 @@ def prerequisites_callable(self, config: Dict[str, Any]) -> Tuple[bool, str]: if self.name == "http": tool_description = f"Make HTTP requests to whitelisted API endpoints ({endpoints_summary})" else: - tool_description = f"Make HTTP requests to {self.name} API ({endpoints_summary})" + tool_description = ( + f"Make HTTP requests to {self.name} API ({endpoints_summary})" + ) - self.tools = [HttpRequest(self, tool_name=tool_name, tool_description=tool_description)] + self.tools = [ + HttpRequest( + self, tool_name=tool_name, tool_description=tool_description + ) + ] self._load_llm_instructions_from_file( os.path.dirname(__file__), "instructions.jinja2" @@ -378,12 +396,20 @@ def get_client_cert(self) -> Optional[Any]: if not self._http_config or not self._http_config.client_cert_path: return None if self._http_config.client_key_path: - return (self._http_config.client_cert_path, self._http_config.client_key_path) + return ( + self._http_config.client_cert_path, + self._http_config.client_key_path, + ) return self._http_config.client_cert_path class HttpRequest(Tool, JsonFilterMixin): - def __init__(self, toolset: HttpToolset, tool_name: str = "http_request", tool_description: Optional[str] = None): + def __init__( + self, + toolset: HttpToolset, + tool_name: str = "http_request", + tool_description: Optional[str] = None, + ): if not tool_description: if toolset.name == "http": tool_description = "Make HTTP requests to whitelisted API endpoints" diff --git a/holmes/plugins/toolsets/inspektor_gadget.yaml b/holmes/plugins/toolsets/inspektor_gadget.yaml index 688b9342b0..a05e14ab4a 100644 --- a/holmes/plugins/toolsets/inspektor_gadget.yaml +++ b/holmes/plugins/toolsets/inspektor_gadget.yaml @@ -245,4 +245,3 @@ toolsets: description: "Pod name to filter" default: "" command: "kubectl debug --profile=sysadmin node/{{ node_name }} --attach --quiet --image=ghcr.io/inspektor-gadget/ig:v0.49.0 -- ig run tcpdump:v0.49.0 -o pcap-ng --timeout {{ timeout }}{% if filter %} --pf {{ filter }}{% endif %}{% if namespace %} --k8s-namespace {{ namespace }}{% endif %}{% if podname %} --k8s-podname {{ podname }}{% endif %} | tcpdump -nvr -" - diff --git a/holmes/plugins/toolsets/internet/internet.py b/holmes/plugins/toolsets/internet/internet.py index 090328b8cc..2d4bc519b3 100644 --- a/holmes/plugins/toolsets/internet/internet.py +++ b/holmes/plugins/toolsets/internet/internet.py @@ -3,10 +3,10 @@ import re from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type -from pydantic import Field import requests # type: ignore from bs4 import BeautifulSoup from markdownify import markdownify +from pydantic import Field from requests import RequestException, Timeout # type: ignore from holmes.core.tools import ( @@ -193,7 +193,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes url: str = params["url"] additional_headers = ( - self.toolset.internet_config.additional_headers if self.toolset.internet_config.additional_headers else {} + self.toolset.internet_config.additional_headers + if self.toolset.internet_config.additional_headers + else {} ) content, mime_type = scrape(url, additional_headers) @@ -233,11 +235,13 @@ class InternetBaseToolsetConfig(ToolsetConfig): {"Authorization": "Bearer "}, ], ) + + class InternetBaseToolset(Toolset): config_classes: ClassVar[list[Type[InternetBaseToolsetConfig]]] = [ InternetBaseToolsetConfig ] - + internet_config: Optional[InternetBaseToolsetConfig] = None def __init__( diff --git a/holmes/plugins/toolsets/mcp/toolset_mcp.py b/holmes/plugins/toolsets/mcp/toolset_mcp.py index bbd34247cb..dbd92b06c8 100644 --- a/holmes/plugins/toolsets/mcp/toolset_mcp.py +++ b/holmes/plugins/toolsets/mcp/toolset_mcp.py @@ -348,7 +348,10 @@ def _resolve_schema( if req not in reqs: reqs.append(req) elif k == "type": - if "type" not in merged or merged["type"] == "null": + if ( + "type" not in merged + or merged["type"] == "null" + ): merged["type"] = v else: merged[k] = v @@ -418,13 +421,20 @@ def _parse_tool_parameter( # OpenAI-formatted schema so the LLM sees constraints like array # length limits, numeric ranges, and string patterns. _PASSTHROUGH_KEYWORDS = { - "minItems", "maxItems", - "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", - "minLength", "maxLength", + "minItems", + "maxItems", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "minLength", + "maxLength", "pattern", "default", } - json_schema_extra = {k: v for k, v in schema.items() if k in _PASSTHROUGH_KEYWORDS} + json_schema_extra = { + k: v for k, v in schema.items() if k in _PASSTHROUGH_KEYWORDS + } return ToolParameter( description=schema.get("description"), diff --git a/holmes/plugins/toolsets/mongodb/mongodb.py b/holmes/plugins/toolsets/mongodb/mongodb.py index 65b194cf4d..80757b3264 100644 --- a/holmes/plugins/toolsets/mongodb/mongodb.py +++ b/holmes/plugins/toolsets/mongodb/mongodb.py @@ -5,6 +5,7 @@ from abc import ABC from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +import pymongo from pydantic import ConfigDict, Field from holmes.core.tools import ( @@ -20,8 +21,6 @@ from holmes.plugins.toolsets.utils import toolset_name_for_one_liner from holmes.utils.pydantic_utils import ToolsetConfig -import pymongo - logger = logging.getLogger(__name__) # Aggregation stages that write data — blocked in read-only mode @@ -33,7 +32,9 @@ def _parse_json_param(value: str, param_name: str) -> Any: try: return json.loads(value) except json.JSONDecodeError as e: - raise ValueError(f"Invalid JSON for '{param_name}': {e}. Received: {value[:200]}") + raise ValueError( + f"Invalid JSON for '{param_name}': {e}. Received: {value[:200]}" + ) def _serialize_value(val: Any) -> Any: @@ -70,7 +71,6 @@ class MongoDBConfig(ToolsetConfig): ``` """ - connection_url: str = Field( title="Connection URL", description=( @@ -148,7 +148,9 @@ def __init__(self, name: str = "mongodb", **kwargs: Any): enabled = kwargs.pop("enabled", False) kwargs.pop("type", None) - description = kwargs.pop("description", None) or f"Query {name} MongoDB database" + description = ( + kwargs.pop("description", None) or f"Query {name} MongoDB database" + ) super().__init__( name=name, @@ -195,7 +197,9 @@ def _perform_health_check(self) -> Tuple[bool, str]: self._client.admin.command("ping") # Resolve default database name if not self.mongodb_config.default_database: - db_name = pymongo.uri_parser.parse_uri(self.mongodb_config.connection_url).get("database") + db_name = pymongo.uri_parser.parse_uri( + self.mongodb_config.connection_url + ).get("database") if db_name: self.mongodb_config.default_database = db_name return True, "Connected to MongoDB" @@ -221,7 +225,9 @@ def _create_client(self) -> pymongo.MongoClient: def mongodb_config(self) -> MongoDBConfig: return self.config # type: ignore - def _get_database(self, database: Optional[str] = None) -> pymongo.database.Database: + def _get_database( + self, database: Optional[str] = None + ) -> pymongo.database.Database: db_name = database or self.mongodb_config.default_database if not db_name: raise ValueError( @@ -327,11 +333,13 @@ def get_collection_schema( # Get indexes indexes = [] for idx_name, idx_info in coll.index_information().items(): - indexes.append({ - "name": idx_name, - "keys": idx_info.get("key", []), - "unique": idx_info.get("unique", False), - }) + indexes.append( + { + "name": idx_name, + "keys": idx_info.get("key", []), + "unique": idx_info.get("unique", False), + } + ) # Get estimated document count estimated_count = coll.estimated_document_count() @@ -370,8 +378,13 @@ def get_server_status(self, sections: Optional[List[str]] = None) -> Dict[str, A # Default sections that are most useful for performance diagnostics default_sections = [ - "connections", "opcounters", "mem", "locks", - "globalLock", "network", "wiredTiger", + "connections", + "opcounters", + "mem", + "locks", + "globalLock", + "network", + "wiredTiger", ] requested = sections or default_sections for section in requested: @@ -416,12 +429,16 @@ def get_list_databases(self) -> Dict[str, Any]: result = self._client.admin.command("listDatabases") # type: ignore databases = [] for db_info in result.get("databases", []): - databases.append({ - "name": db_info.get("name"), - "sizeOnDisk": db_info.get("sizeOnDisk"), - "sizeOnDisk_mb": round(db_info.get("sizeOnDisk", 0) / (1024 * 1024), 2), - "empty": db_info.get("empty", False), - }) + databases.append( + { + "name": db_info.get("name"), + "sizeOnDisk": db_info.get("sizeOnDisk"), + "sizeOnDisk_mb": round( + db_info.get("sizeOnDisk", 0) / (1024 * 1024), 2 + ), + "empty": db_info.get("empty", False), + } + ) return { "databases": databases, "totalSize_mb": round(result.get("totalSize", 0) / (1024 * 1024), 2), @@ -517,7 +534,11 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes limit=params.get("limit"), database=params.get("database"), ) - status = StructuredToolResultStatus.SUCCESS if data.get("documents") else StructuredToolResultStatus.NO_DATA + status = ( + StructuredToolResultStatus.SUCCESS + if data.get("documents") + else StructuredToolResultStatus.NO_DATA + ) return StructuredToolResult( status=status, data=data, @@ -588,7 +609,11 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes pipeline=pipeline, database=params.get("database"), ) - status = StructuredToolResultStatus.SUCCESS if data.get("documents") else StructuredToolResultStatus.NO_DATA + status = ( + StructuredToolResultStatus.SUCCESS + if data.get("documents") + else StructuredToolResultStatus.NO_DATA + ) return StructuredToolResult( status=status, data=data, @@ -635,7 +660,11 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes data = self._toolset.list_collections( database=params.get("database"), ) - status = StructuredToolResultStatus.SUCCESS if data.get("collections") else StructuredToolResultStatus.NO_DATA + status = ( + StructuredToolResultStatus.SUCCESS + if data.get("collections") + else StructuredToolResultStatus.NO_DATA + ) return StructuredToolResult( status=status, data=data, @@ -699,7 +728,9 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes def get_parameterized_one_liner(self, params: Dict) -> str: collection = params.get("collection", "unknown") - return f"{toolset_name_for_one_liner(self._toolset.name)}: Schema of {collection}" + return ( + f"{toolset_name_for_one_liner(self._toolset.name)}: Schema of {collection}" + ) class MongoDBListDatabases(BaseMongoDBTool): @@ -716,7 +747,11 @@ def __init__(self, toolset: MongoDBToolset, tool_prefix: str): def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: try: data = self._toolset.get_list_databases() - status = StructuredToolResultStatus.SUCCESS if data.get("databases") else StructuredToolResultStatus.NO_DATA + status = ( + StructuredToolResultStatus.SUCCESS + if data.get("databases") + else StructuredToolResultStatus.NO_DATA + ) return StructuredToolResult( status=status, data=data, @@ -819,7 +854,11 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes min_duration_ms=params.get("min_duration_ms"), active_only=params.get("active_only", True), ) - status = StructuredToolResultStatus.SUCCESS if data.get("operations") else StructuredToolResultStatus.NO_DATA + status = ( + StructuredToolResultStatus.SUCCESS + if data.get("operations") + else StructuredToolResultStatus.NO_DATA + ) return StructuredToolResult( status=status, data=data, diff --git a/holmes/plugins/toolsets/prometheus/prometheus.py b/holmes/plugins/toolsets/prometheus/prometheus.py index f99b2bd93b..6a199c466d 100644 --- a/holmes/plugins/toolsets/prometheus/prometheus.py +++ b/holmes/plugins/toolsets/prometheus/prometheus.py @@ -17,7 +17,11 @@ from requests import RequestException from requests.exceptions import SSLError # type: ignore -from holmes.common.env_vars import IS_OPENSHIFT, MAX_GRAPH_POINTS, MAX_GRAPH_POINTS_HARD_LIMIT +from holmes.common.env_vars import ( + IS_OPENSHIFT, + MAX_GRAPH_POINTS, + MAX_GRAPH_POINTS_HARD_LIMIT, +) from holmes.common.openshift import load_openshift_token from holmes.core.tools import ( CallablePrerequisite, diff --git a/holmes/plugins/toolsets/servicenow_tables/servicenow_tables.py b/holmes/plugins/toolsets/servicenow_tables/servicenow_tables.py index 422601ea77..ffa12e5dae 100644 --- a/holmes/plugins/toolsets/servicenow_tables/servicenow_tables.py +++ b/holmes/plugins/toolsets/servicenow_tables/servicenow_tables.py @@ -66,7 +66,9 @@ class ServiceNowTablesConfig(ToolsetConfig): class ServiceNowTablesToolset(Toolset): - config_classes: ClassVar[list[Type[ServiceNowTablesConfig]]] = [ServiceNowTablesConfig] + config_classes: ClassVar[list[Type[ServiceNowTablesConfig]]] = [ + ServiceNowTablesConfig + ] def __init__(self): super().__init__( @@ -105,7 +107,10 @@ def _perform_health_check(self, table_name) -> Tuple[bool, str]: query_params={"sysparm_limit": 1}, timeout=10, ) - return True, f"ServiceNow configuration is valid and API is accessible. (checked table: {table_name})" + return ( + True, + f"ServiceNow configuration is valid and API is accessible. (checked table: {table_name})", + ) except requests.exceptions.HTTPError as e: if e.response.status_code == 401: diff --git a/holmes/toolset_config_tui.py b/holmes/toolset_config_tui.py index 9797429601..601517cc5e 100644 --- a/holmes/toolset_config_tui.py +++ b/holmes/toolset_config_tui.py @@ -18,11 +18,11 @@ from prompt_toolkit.application import Application from prompt_toolkit.buffer import Buffer from prompt_toolkit.document import Document +from prompt_toolkit.filters import Condition from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.layout import Layout from prompt_toolkit.layout.containers import Window from prompt_toolkit.layout.controls import FormattedTextControl -from prompt_toolkit.filters import Condition from prompt_toolkit.styles import Style as PTStyle from pydantic import BaseModel from rich.console import Console @@ -151,7 +151,9 @@ class ConfigFieldNode: """One row in the config tree.""" key: str - field_type: str # "str" | "int" | "float" | "bool" | "enum" | "dict" | "list" | "model" + field_type: ( + str # "str" | "int" | "float" | "bool" | "enum" | "dict" | "list" | "model" + ) value: Any = None title: str = "" description: str = "" @@ -274,7 +276,8 @@ def tree_to_dict(nodes: List[ConfigFieldNode]) -> Dict[str, Any]: result[node.key] = { (c.dict_key if c.dict_key is not None else c.key): c.value for c in node.children - if c.dict_key is None or c.dict_key # skip entries with empty dict_key + if c.dict_key is None + or c.dict_key # skip entries with empty dict_key } elif node.field_type == "list": result[node.key] = [c.value for c in node.children] @@ -342,7 +345,9 @@ def _select_config_class( best_count = 0 for cls in config_classes: count = sum( - 1 for k in config_values if k in cls.model_fields and k not in discriminator_fields + 1 + for k in config_values + if k in cls.model_fields and k not in discriminator_fields ) if count > best_count: best_count = count @@ -372,7 +377,9 @@ def set_mcp_config( config_dict: Dict[str, Any], ) -> None: """Set ``mcp_servers[toolset_name]["config"]``, preserving other keys.""" - if toolset_name not in mcp_servers or not isinstance(mcp_servers.get(toolset_name), dict): + if toolset_name not in mcp_servers or not isinstance( + mcp_servers.get(toolset_name), dict + ): mcp_servers[toolset_name] = {} mcp_servers[toolset_name]["config"] = config_dict @@ -622,7 +629,9 @@ def run_tree_editor( _class_config_cache[config_class] = dict(initial_config) # State - cursor = [len(flat_rows) if cursor_on_test_button else 0] # index into (flat_rows + buttons) + cursor = [ + len(flat_rows) if cursor_on_test_button else 0 + ] # index into (flat_rows + buttons) editing = [False] editing_dict_key = [False] # True when editing the key portion of a dict entry edit_buf = [Buffer()] @@ -676,14 +685,18 @@ def _compute_value_columns() -> Dict[Optional[int], int]: groups[parent_id] = name_len return groups - def _value_pad(node: ConfigFieldNode, value_columns: Dict[Optional[int], int]) -> str: + def _value_pad( + node: ConfigFieldNode, value_columns: Dict[Optional[int], int] + ) -> str: """Return the padding between the colon and the value for *node*.""" display_name = node.title if node.title else node.key parent_id = id(node.parent) if node.parent else None max_name_len = value_columns.get(parent_id, len(display_name)) return " " * (max_name_len - len(display_name)) - def _row_content_width(node: ConfigFieldNode, value_columns: Dict[Optional[int], int]) -> int: + def _row_content_width( + node: ConfigFieldNode, value_columns: Dict[Optional[int], int] + ) -> int: """Compute the visible width of a row's content (before comment/hints).""" indent = " " * (node.depth + 1) prefix = " " # use non-selected width for alignment @@ -696,7 +709,9 @@ def _row_content_width(node: ConfigFieldNode, value_columns: Dict[Optional[int], if node.is_header: count = len(node.children) type_bracket = "{}" if node.field_type == "dict" else "[]" - return len(f"{indent}{prefix}{display_name}:{pad} {type_bracket[0]}{count} items{type_bracket[1]}") + return len( + f"{indent}{prefix}{display_name}:{pad} {type_bracket[0]}{count} items{type_bracket[1]}" + ) if node.field_type == "bool": val_display = str(node.value).lower() if node.value is not None else "null" @@ -826,7 +841,11 @@ def _render_leaf_row( row_parts.append(("", "\n")) return row_parts - is_placeholder = (is_list_entry and not node.value) or (node.value is None and not node.required) or node.value == "" + is_placeholder = ( + (is_list_entry and not node.value) + or (node.value is None and not node.required) + or node.value == "" + ) val_style = "class:dim" if is_placeholder else style row_parts.append((val_style, val_display)) @@ -852,7 +871,9 @@ def _render_row( pad = _value_pad(node, value_columns) if node.is_header: - return _render_header_row(node, style, indent, prefix, display_name, pad, comment_col) + return _render_header_row( + node, style, indent, prefix, display_name, pad, comment_col + ) row_idx = flat_rows.index(node) if node in flat_rows else -1 is_editing_this = editing[0] and cursor[0] == row_idx @@ -860,7 +881,9 @@ def _render_row( if node.dict_key is not None: return _render_dict_child_row(node, style, indent, prefix, is_editing_this) - return _render_leaf_row(node, style, indent, prefix, display_name, pad, comment_col, is_editing_this) + return _render_leaf_row( + node, style, indent, prefix, display_name, pad, comment_col, is_editing_this + ) def _get_display_text() -> List[Tuple[str, str]]: parts: List[Tuple[str, str]] = [] @@ -870,7 +893,14 @@ def _get_display_text() -> List[Tuple[str, str]]: value_columns = _compute_value_columns() comment_col = _compute_comment_column(value_columns) for i, node in enumerate(flat_rows): - parts.extend(_render_row(node, selected=(cursor[0] == i), comment_col=comment_col, value_columns=value_columns)) + parts.extend( + _render_row( + node, + selected=(cursor[0] == i), + comment_col=comment_col, + value_columns=value_columns, + ) + ) # Separator parts.append(("", "\n")) @@ -894,7 +924,12 @@ def _get_display_text() -> List[Tuple[str, str]]: parts.extend(status_lines) # Hint line - parts.append(("class:hint", "\n Up/Down: navigate | Enter: edit/select | Backspace/Del: delete entry or set null | Esc: cancel edit\n")) + parts.append( + ( + "class:hint", + "\n Up/Down: navigate | Enter: edit/select | Backspace/Del: delete entry or set null | Esc: cancel edit\n", + ) + ) return parts # ── key bindings ── @@ -959,8 +994,17 @@ def _delete_entry(event: Any) -> None: if editing[0]: buf = edit_buf[0] - is_collection_child = node.parent and node.parent.is_header and node.parent.field_type in ("dict", "list") - if len(buf.text) == 0 and not node.required and not editing_dict_key[0] and not is_collection_child: + is_collection_child = ( + node.parent + and node.parent.is_header + and node.parent.field_type in ("dict", "list") + ) + if ( + len(buf.text) == 0 + and not node.required + and not editing_dict_key[0] + and not is_collection_child + ): # Empty buffer + deletion key → set to node.value = None node.explicitly_set = True @@ -971,7 +1015,11 @@ def _delete_entry(event: Any) -> None: buf.delete() return - if node.parent and node.parent.is_header and node.parent.field_type in ("dict", "list"): + if ( + node.parent + and node.parent.is_header + and node.parent.field_type in ("dict", "list") + ): node.parent.children.remove(node) for i, child in enumerate(node.parent.children): child.key = str(i) @@ -1003,11 +1051,19 @@ def _enter(event: Any) -> None: top_nodes.extend(build_tree_from_schema(config_class, {})) _refresh_flat() cursor[0] = 0 - status_lines = [("class:status-ok", " Configuration reset to defaults.\n")] + status_lines = [ + ("class:status-ok", " Configuration reset to defaults.\n") + ] return elif btn_idx == 2: # Save - config_path = Path(config_file_path) if config_file_path else Path(DEFAULT_CONFIG_LOCATION) - ok, msg = save_config_to_file(config_path, toolset.name, config_dict, is_mcp=is_mcp) + config_path = ( + Path(config_file_path) + if config_file_path + else Path(DEFAULT_CONFIG_LOCATION) + ) + ok, msg = save_config_to_file( + config_path, toolset.name, config_dict, is_mcp=is_mcp + ) style_cls = "class:status-ok" if ok else "class:status-fail" status_lines = [(style_cls, f" {line}\n") for line in msg.splitlines()] if ok: @@ -1038,7 +1094,9 @@ def _make_edit_buffer(text: str) -> Buffer: try: node.value = int(raw) except ValueError: - status_lines = [("class:status-fail", f" Invalid integer: '{raw}'\n")] + status_lines = [ + ("class:status-fail", f" Invalid integer: '{raw}'\n") + ] editing[0] = False editing_dict_key[0] = False return @@ -1046,7 +1104,9 @@ def _make_edit_buffer(text: str) -> Buffer: try: node.value = float(raw) except ValueError: - status_lines = [("class:status-fail", f" Invalid number: '{raw}'\n")] + status_lines = [ + ("class:status-fail", f" Invalid number: '{raw}'\n") + ] editing[0] = False editing_dict_key[0] = False return @@ -1077,7 +1137,9 @@ def _make_edit_buffer(text: str) -> Buffer: # If multiple config classes, check if we need to switch if len(toolset.config_classes) > 1: new_config_dict = tree_to_dict(top_nodes) - new_class = _select_config_class(toolset.config_classes, new_config_dict) + new_class = _select_config_class( + toolset.config_classes, new_config_dict + ) if new_class is not config_class: _rebuild_for_class(new_class, node.key, new_value) return @@ -1156,8 +1218,16 @@ def _backspace(event: Any) -> None: idx = cursor[0] if idx < len(flat_rows): node = flat_rows[idx] - is_collection_child = node.parent and node.parent.is_header and node.parent.field_type in ("dict", "list") - if not node.required and not editing_dict_key[0] and not is_collection_child: + is_collection_child = ( + node.parent + and node.parent.is_header + and node.parent.field_type in ("dict", "list") + ) + if ( + not node.required + and not editing_dict_key[0] + and not is_collection_child + ): node.value = None node.explicitly_set = True editing[0] = False @@ -1170,7 +1240,9 @@ def _backspace(event: Any) -> None: # ── run ── layout = Layout( - Window(FormattedTextControl(_get_display_text, show_cursor=False), wrap_lines=True) + Window( + FormattedTextControl(_get_display_text, show_cursor=False), wrap_lines=True + ) ) app: Application[Any] = Application( layout=layout, @@ -1215,9 +1287,13 @@ def _refresh_toolset_from_file( with open(config_path, "r") as f: file_data = yaml.safe_load(f) or {} if toolset.type == ToolsetType.MCP: - saved_cfg = file_data.get("mcp_servers", {}).get(toolset.name, {}).get("config", {}) + saved_cfg = ( + file_data.get("mcp_servers", {}).get(toolset.name, {}).get("config", {}) + ) else: - saved_cfg = file_data.get("toolsets", {}).get(toolset.name, {}).get("config", {}) + saved_cfg = ( + file_data.get("toolsets", {}).get(toolset.name, {}).get("config", {}) + ) except Exception as e: logger.warning("Could not re-read config file for refresh: %s", e) return @@ -1273,7 +1349,9 @@ def run_toolset_config_tui( return config_values = _get_existing_config(selected, config) - config_path = Path(config_file) if config_file else Path(DEFAULT_CONFIG_LOCATION) + config_path = ( + Path(config_file) if config_file else Path(DEFAULT_CONFIG_LOCATION) + ) test_status: Optional[List[Tuple[str, str]]] = None ever_saved = False cursor_on_test = False diff --git a/holmes/utils/holmes_status.py b/holmes/utils/holmes_status.py index ab226c81f1..d166b50359 100644 --- a/holmes/utils/holmes_status.py +++ b/holmes/utils/holmes_status.py @@ -1,11 +1,12 @@ -from dataclasses import asdict, dataclass import json import logging +from dataclasses import asdict, dataclass from holmes import get_version # type: ignore from holmes.config import Config from holmes.core.supabase_dal import SupabaseDal + @dataclass class HolmesMetadata: is_robusta_ai_enabled: bool @@ -20,7 +21,7 @@ def update_holmes_status_in_db(dal: SupabaseDal, config: Config): "Cluster name is missing in the configuration. Please ensure 'CLUSTER_NAME' is defined in the environment variables, " "or verify that a cluster name is provided in the Robusta configuration file." ) - + metadata = HolmesMetadata(is_robusta_ai_enabled=config.should_try_robusta_ai) dal.upsert_holmes_status( diff --git a/holmes/utils/holmes_sync_toolsets.py b/holmes/utils/holmes_sync_toolsets.py index 15f99af288..8c078883b1 100644 --- a/holmes/utils/holmes_sync_toolsets.py +++ b/holmes/utils/holmes_sync_toolsets.py @@ -1,9 +1,10 @@ import json -import yaml import logging from datetime import datetime from typing import Any, List +import yaml + from holmes.config import Config from holmes.core.supabase_dal import SupabaseDal from holmes.core.tools import Toolset, ToolsetDBModel @@ -74,6 +75,7 @@ def get_config_schema_for_toolset(toolset: Toolset) -> str: } return json.dumps(res) + def render_default_installation_instructions_for_toolset(toolset: Toolset) -> str: env_vars = toolset.get_environment_variables() context: dict[str, Any] = { diff --git a/holmes/utils/pydantic_utils.py b/holmes/utils/pydantic_utils.py index e814c2e6ad..73287166e5 100644 --- a/holmes/utils/pydantic_utils.py +++ b/holmes/utils/pydantic_utils.py @@ -17,7 +17,13 @@ import typer from benedict import benedict # type: ignore -from pydantic import BaseModel, BeforeValidator, ConfigDict, ValidationError, model_validator +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + ValidationError, + model_validator, +) from holmes.plugins.prompts import load_prompt @@ -96,6 +102,7 @@ def handle_deprecated_fields(cls, data: Any) -> Any: return data + class RobustaBaseConfig(BaseModel): model_config = ConfigDict(extra="forbid", validate_default=True) @@ -138,6 +145,7 @@ def load_model_from_file( ) sys.exit() + def build_config_example(model: Type[BaseModel] | BaseModel) -> Dict[str, Any]: """ Build a JSON-serializable example object for a Pydantic model. @@ -193,6 +201,7 @@ def build_config_example(model: Type[BaseModel] | BaseModel) -> Dict[str, Any]: return out + def _extract_base_model_subclass(annotation: Any) -> Optional[Type[BaseModel]]: """ Best-effort extraction of a BaseModel subclass from an annotation. diff --git a/holmes/utils/stream.py b/holmes/utils/stream.py index eb2a967894..c9eb958c31 100644 --- a/holmes/utils/stream.py +++ b/holmes/utils/stream.py @@ -2,7 +2,7 @@ import logging from enum import Enum from functools import partial -from typing import Generator, List, Optional, Union +from typing import AsyncGenerator, List, Optional, Union import litellm from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper @@ -61,15 +61,17 @@ def _is_rate_limit_error(e: Exception) -> bool: instead of litellm.exceptions.RateLimitError, so we need a string check as a fallback. """ - return isinstance(e, litellm.exceptions.RateLimitError) or "Model is getting throttled" in str(e) + return isinstance( + e, litellm.exceptions.RateLimitError + ) or "Model is getting throttled" in str(e) -def stream_chat_formatter( - call_stream: Generator[StreamMessage, None, None], +async def stream_chat_formatter( + call_stream: AsyncGenerator[StreamMessage, None], followups: Optional[List[dict]] = None, ): try: - for message in call_stream: + async for message in call_stream: if message.event == StreamEvents.ANSWER_END: response_data = { "analysis": message.data.get("content"), diff --git a/holmes_operator/scheduler/job_executor.py b/holmes_operator/scheduler/job_executor.py index 50851d5d63..0aa13e189f 100644 --- a/holmes_operator/scheduler/job_executor.py +++ b/holmes_operator/scheduler/job_executor.py @@ -436,7 +436,9 @@ def modify_status(resource): scheduled_namespace=scheduled_namespace, check_name=check_name, ) - logger.info(f"Successfully removed {check_name} from active list during cleanup") + logger.info( + f"Successfully removed {check_name} from active list during cleanup" + ) except Exception as cleanup_error: # Log cleanup failure but don't let it suppress the original exception logger.error( diff --git a/holmes_operator/scheduler/manager.py b/holmes_operator/scheduler/manager.py index 41ed6c3ef6..4ad0da5bc0 100644 --- a/holmes_operator/scheduler/manager.py +++ b/holmes_operator/scheduler/manager.py @@ -1,6 +1,5 @@ import asyncio import logging -from datetime import datetime, timezone from typing import Dict from apscheduler.jobstores.memory import MemoryJobStore diff --git a/scripts/cli_performance_benchmark.py b/scripts/cli_performance_benchmark.py index 03b1d6909e..f104856aeb 100755 --- a/scripts/cli_performance_benchmark.py +++ b/scripts/cli_performance_benchmark.py @@ -85,7 +85,9 @@ def get_git_info() -> tuple[str, str]: return sha, branch -def run_command(cmd: list[str], benchmark_type: str, model: str = "", prompt: str = "") -> BenchmarkResult: +def run_command( + cmd: list[str], benchmark_type: str, model: str = "", prompt: str = "" +) -> BenchmarkResult: """Run a command and measure wall time.""" start = time.perf_counter() result = subprocess.run( @@ -121,12 +123,18 @@ def run_benchmark( for i in range(iterations): run_type = "cold" if i == 0 else "warm" - print(f"{benchmark_type.upper()} iteration {i + 1}/{iterations} ({run_type})...", file=sys.stderr) + print( + f"{benchmark_type.upper()} iteration {i + 1}/{iterations} ({run_type})...", + file=sys.stderr, + ) result = run_command(cmd, benchmark_type, model, prompt) if result.exit_code != 0: - print(f"Warning: iteration {i + 1} failed with exit code {result.exit_code}", file=sys.stderr) + print( + f"Warning: iteration {i + 1} failed with exit code {result.exit_code}", + file=sys.stderr, + ) if result.stderr: print(f"STDERR:\n{result.stderr}", file=sys.stderr) if result.stdout: @@ -177,7 +185,9 @@ def run_e2e_benchmark( cmd = ["poetry", "run", "holmes", "ask", prompt, "--no-interactive", "--no-echo"] if model: cmd.extend(["--model", model]) - return run_benchmark(cmd, "e2e", iterations, model=model or "default", prompt=prompt) + return run_benchmark( + cmd, "e2e", iterations, model=model or "default", prompt=prompt + ) def main(): @@ -231,11 +241,16 @@ def main(): if args.startup_only: iterations = args.iterations or 5 - print(f"Running startup-only benchmark ({iterations} iterations)...", file=sys.stderr) + print( + f"Running startup-only benchmark ({iterations} iterations)...", + file=sys.stderr, + ) summary = run_startup_benchmark(iterations=iterations) elif args.e2e_only: iterations = args.iterations or 3 - print(f"Running e2e-only benchmark ({iterations} iterations)...", file=sys.stderr) + print( + f"Running e2e-only benchmark ({iterations} iterations)...", file=sys.stderr + ) summary = run_e2e_benchmark( prompt=args.prompt, iterations=iterations, @@ -256,7 +271,10 @@ def main(): print(f"{'=' * 50}", file=sys.stderr) print(f"🥶 Cold Start: {summary.cold_start_seconds:.2f}s", file=sys.stderr) print(f"🔥 Warm Start: {summary.warm_mean_seconds:.2f}s (mean)", file=sys.stderr) - print(f" Min: {summary.warm_min_seconds:.2f}s | Max: {summary.warm_max_seconds:.2f}s", file=sys.stderr) + print( + f" Min: {summary.warm_min_seconds:.2f}s | Max: {summary.warm_max_seconds:.2f}s", + file=sys.stderr, + ) # Print JSON to stdout for piping print(json.dumps(result_dict, indent=2)) diff --git a/server.py b/server.py index c8a37974df..5346f61544 100644 --- a/server.py +++ b/server.py @@ -1,30 +1,16 @@ # ruff: noqa: E402 -import os - -from holmes.utils.cert_utils import add_custom_certificate - -ADDITIONAL_CERTIFICATE: str = os.environ.get("CERTIFICATE", "") -if add_custom_certificate(ADDITIONAL_CERTIFICATE): - print("added custom certificate") - -# DO NOT ADD ANY IMPORTS OR CODE ABOVE THIS LINE -# IMPORTING ABOVE MIGHT INITIALIZE AN HTTPS CLIENT THAT DOESN'T TRUST THE CUSTOM CERTIFICATE -import json -import logging -import threading -import time -from pathlib import Path -from typing import List, Optional - -import colorlog -import litellm -import sentry_sdk -import uvicorn -from fastapi import FastAPI, HTTPException, Request -from fastapi.responses import StreamingResponse -from litellm.exceptions import AuthenticationError - -from holmes import get_version, is_official_release +from holmes.utils.stream import stream_chat_formatter +from holmes.utils.log import EndpointFilter +from holmes.utils.holmes_sync_toolsets import holmes_sync_toolsets_status +from holmes.utils.holmes_status import update_holmes_status_in_db +from holmes.utils.connection_utils import patch_socket_create_connection +from holmes.core.tools_utils.filesystem_result_storage import tool_result_storage +from holmes.core.tools import ToolsetStatusEnum, ToolsetType +from holmes.core.scheduled_prompts import ScheduledPromptsExecutor +from holmes.core.prompt import PromptComponent +from holmes.core.models import ChatRequest, ChatResponse, FollowUpAction +from holmes.core.conversations import build_chat_messages +from holmes.config import DEFAULT_CONFIG_LOCATION, Config from holmes.common.env_vars import ( DEVELOPMENT_MODE, ENABLE_CONNECTION_KEEPALIVE, @@ -38,25 +24,32 @@ SENTRY_TRACES_SAMPLE_RATE, TOOLSET_STATUS_REFRESH_INTERVAL_SECONDS, ) -from holmes.config import DEFAULT_CONFIG_LOCATION, Config -from holmes.core.conversations import ( - build_chat_messages, -) -from holmes.core.models import ( - ChatRequest, - ChatResponse, - FollowUpAction, -) -from holmes.core.prompt import PromptComponent -from holmes.core.tools import ToolsetStatusEnum, ToolsetType -from holmes.core.scheduled_prompts import ScheduledPromptsExecutor -from holmes.utils.connection_utils import patch_socket_create_connection -from holmes.utils.holmes_status import update_holmes_status_in_db -from holmes.utils.holmes_sync_toolsets import holmes_sync_toolsets_status -from holmes.utils.log import EndpointFilter from holmes.checks.checks_api import init_checks_app -from holmes.core.tools_utils.filesystem_result_storage import tool_result_storage -from holmes.utils.stream import stream_chat_formatter +from holmes import get_version, is_official_release +from litellm.exceptions import AuthenticationError +from fastapi.responses import StreamingResponse +from fastapi import FastAPI, HTTPException, Request +import uvicorn +import sentry_sdk +import litellm +import colorlog +from typing import List, Optional +from pathlib import Path +import time +import threading +import logging +import json +import os + +from holmes.utils.cert_utils import add_custom_certificate + +ADDITIONAL_CERTIFICATE: str = os.environ.get("CERTIFICATE", "") +if add_custom_certificate(ADDITIONAL_CERTIFICATE): + print("added custom certificate") + +# DO NOT ADD ANY IMPORTS OR CODE ABOVE THIS LINE +# IMPORTING ABOVE MIGHT INITIALIZE AN HTTPS CLIENT THAT DOESN'T TRUST THE CUSTOM CERTIFICATE + # removed: add_runbooks_to_user_prompt @@ -302,17 +295,18 @@ def extract_passthrough_headers(request: Request) -> dict: return {"headers": passthrough_headers} if passthrough_headers else {} -def _stream_with_storage_cleanup(storage, stream_generator, req_info): - """Wrap a stream generator to clean up tool result files after streaming completes.""" +async def _stream_with_storage_cleanup(storage, stream_generator, req_info): + """Wrap an async stream generator to clean up tool result files after streaming completes.""" try: - yield from stream_generator + async for item in stream_generator: + yield item finally: logging.info(f"Stream request end: {req_info}") storage.__exit__(None, None, None) @app.post("/api/chat") -def chat(chat_request: ChatRequest, http_request: Request): +async def chat(chat_request: ChatRequest, http_request: Request): try: # Log incoming request details has_images = bool(chat_request.images) diff --git a/tests/checks/test_checks_api.py b/tests/checks/test_checks_api.py index 3db9a478c1..77a167449c 100644 --- a/tests/checks/test_checks_api.py +++ b/tests/checks/test_checks_api.py @@ -1,10 +1,11 @@ import json +from unittest.mock import MagicMock, patch + import pytest -from unittest.mock import patch, MagicMock from fastapi.testclient import TestClient -from server import app from holmes.core.tool_calling_llm import LLMResult +from server import app @pytest.fixture diff --git a/tests/checks/test_checks_cli.py b/tests/checks/test_checks_cli.py index 0f2277646d..e654708036 100644 --- a/tests/checks/test_checks_cli.py +++ b/tests/checks/test_checks_cli.py @@ -1,14 +1,13 @@ import json import tempfile from pathlib import Path -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch import yaml from typer.testing import CliRunner -from holmes.main import app from holmes.core.tool_calling_llm import LLMResult - +from holmes.main import app runner = CliRunner() diff --git a/tests/config_class/conftest.py b/tests/config_class/conftest.py index 3ec717b8f2..e9b146963f 100644 --- a/tests/config_class/conftest.py +++ b/tests/config_class/conftest.py @@ -20,4 +20,4 @@ def clean_llm_env(monkeypatch): from a YAML file would see 3 models (2 from YAML + 1 from env var). """ monkeypatch.delenv("OPENAI_API_KEY", raising=False) - monkeypatch.delenv("MODEL", raising=False) \ No newline at end of file + monkeypatch.delenv("MODEL", raising=False) diff --git a/tests/config_class/test_config_load_cloud_mcp.py b/tests/config_class/test_config_load_cloud_mcp.py index e2af5aaa1f..7826a5ce05 100644 --- a/tests/config_class/test_config_load_cloud_mcp.py +++ b/tests/config_class/test_config_load_cloud_mcp.py @@ -12,10 +12,16 @@ from holmes.core.tools import ToolsetType from holmes.plugins.toolsets import load_toolsets_from_config -from holmes.plugins.toolsets.mcp.toolset_mcp import MCPMode, RemoteMCPToolset, StdioMCPConfig +from holmes.plugins.toolsets.mcp.toolset_mcp import ( + MCPMode, + RemoteMCPToolset, + StdioMCPConfig, +) -def _prepare_mcp_servers(mcp_servers: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: +def _prepare_mcp_servers( + mcp_servers: Dict[str, Dict[str, Any]], +) -> Dict[str, Dict[str, Any]]: """Simulate ToolsetManager preprocessing: set type=mcp and enabled=true. In the real codebase, ToolsetManager and load_toolsets_from_file inject @@ -57,7 +63,10 @@ def test_load_aws_mcp_stdio_config(): toolset = definitions[0] assert isinstance(toolset, RemoteMCPToolset) assert toolset.name == "aws_api" - assert toolset.description == "AWS API - execute AWS CLI commands for investigating infrastructure issues" + assert ( + toolset.description + == "AWS API - execute AWS CLI commands for investigating infrastructure issues" + ) assert "CloudTrail" in toolset.llm_instructions @@ -299,4 +308,3 @@ def test_mcp_config_preserves_multiline_llm_instructions(): assert "CloudTrail" in toolset.llm_instructions assert "CloudWatch" in toolset.llm_instructions assert "1. Gather current resource state" in toolset.llm_instructions - diff --git a/tests/config_class/test_source_factory.py b/tests/config_class/test_source_factory.py index 96e6f7bfab..84df20248e 100644 --- a/tests/config_class/test_source_factory.py +++ b/tests/config_class/test_source_factory.py @@ -11,7 +11,9 @@ def test_ticket_source_resolves_forward_references(self) -> None: """TicketSource uses Union['JiraServiceManagementSource', 'PagerDutySource'] as a forward reference. Without model_rebuild(), Pydantic v2 raises PydanticUserError at instantiation time.""" - from holmes.plugins.sources.jira import JiraServiceManagementSource # noqa: F401 + from holmes.plugins.sources.jira import ( # noqa: F401 + JiraServiceManagementSource, + ) from holmes.plugins.sources.pagerduty import PagerDutySource # noqa: F401 TicketSource.model_rebuild() diff --git a/tests/core/test_image_token_counting.py b/tests/core/test_image_token_counting.py index b214a27f32..e29582ba4f 100644 --- a/tests/core/test_image_token_counting.py +++ b/tests/core/test_image_token_counting.py @@ -10,7 +10,6 @@ is_anthropic_model, ) - IMG_WIDTH = 100 IMG_HEIGHT = 200 @@ -25,7 +24,11 @@ def _make_png_data_uri(width: int, height: int) -> str: def _chunk(chunk_type: bytes, data: bytes) -> bytes: raw = chunk_type + data - return struct.pack(">I", len(data)) + raw + struct.pack(">I", zlib.crc32(raw) & 0xFFFFFFFF) + return ( + struct.pack(">I", len(data)) + + raw + + struct.pack(">I", zlib.crc32(raw) & 0xFFFFFFFF) + ) sig = b"\x89PNG\r\n\x1a\n" # color_type=2 (RGB), bit_depth=8 @@ -71,6 +74,7 @@ def _make_llm(model: str) -> DefaultLLM: # ---------- is_anthropic_model ---------- + @pytest.mark.parametrize( "model_name, expected", [ @@ -91,6 +95,7 @@ def test_is_anthropic_model(model_name: str, expected: bool): # ---------- _get_image_dimensions ---------- + @pytest.mark.parametrize( "url, expected_dims", [ @@ -107,16 +112,37 @@ def test_get_image_dimensions(url: str, expected_dims: tuple[int, int]): # ---------- count_tokens: single image ---------- + @pytest.mark.parametrize( "model, has_image, image_url, expected_image_tokens", [ - ("anthropic/claude-sonnet-4-5-20250929", True, DATA_URI, _ANTHROPIC_DATA_URI_IMAGE_TOKENS), - ("anthropic/claude-sonnet-4-5-20250929", True, EXTERNAL_URL, _ANTHROPIC_EXTERNAL_IMAGE_TOKENS), + ( + "anthropic/claude-sonnet-4-5-20250929", + True, + DATA_URI, + _ANTHROPIC_DATA_URI_IMAGE_TOKENS, + ), + ( + "anthropic/claude-sonnet-4-5-20250929", + True, + EXTERNAL_URL, + _ANTHROPIC_EXTERNAL_IMAGE_TOKENS, + ), ("anthropic/claude-sonnet-4-5-20250929", False, None, 0), ("gpt-4.1", True, DATA_URI, 0), ("gpt-4.1", False, None, 0), - ("vertex_ai/claude-3-5-sonnet", True, DATA_URI, _ANTHROPIC_DATA_URI_IMAGE_TOKENS), - ("robusta/anthropic/claude-sonnet-4-5-20250929", True, DATA_URI, _ANTHROPIC_DATA_URI_IMAGE_TOKENS), + ( + "vertex_ai/claude-3-5-sonnet", + True, + DATA_URI, + _ANTHROPIC_DATA_URI_IMAGE_TOKENS, + ), + ( + "robusta/anthropic/claude-sonnet-4-5-20250929", + True, + DATA_URI, + _ANTHROPIC_DATA_URI_IMAGE_TOKENS, + ), ("robusta/openai/gpt-4.1", True, DATA_URI, 0), ], ids=[ @@ -156,7 +182,11 @@ def test_count_tokens_image_handling( assert message["token_count"] == text_tokens + expected_image_tokens # total_tokens includes the image correction delta for Anthropic - expected_total = text_tokens + expected_image_tokens if is_anthropic_model(model) else text_tokens + expected_total = ( + text_tokens + expected_image_tokens + if is_anthropic_model(model) + else text_tokens + ) assert result.total_tokens == expected_total @@ -164,11 +194,13 @@ def test_count_tokens_image_handling( _MULTI_IMAGE_SIZES = [ (IMG_WIDTH, IMG_HEIGHT), # 100x200 → 26 tokens - (300, 400), # → 160 tokens - (800, 600), # → 640 tokens + (300, 400), # → 160 tokens + (800, 600), # → 640 tokens ] _MULTI_IMAGE_URIS = [DATA_URI, DATA_URI_300x400, DATA_URI_800x600] -_MULTI_IMAGE_TOKENS = [_anthropic_image_token_count(w, h) for w, h in _MULTI_IMAGE_SIZES] +_MULTI_IMAGE_TOKENS = [ + _anthropic_image_token_count(w, h) for w, h in _MULTI_IMAGE_SIZES +] # Each size must produce a unique count, none matching litellm's default assert len(set(_MULTI_IMAGE_TOKENS)) == len(_MULTI_IMAGE_TOKENS) @@ -201,5 +233,9 @@ def test_count_tokens_multi_image_conversation( assert img_msg["token_count"] == text_tokens + expected_total_image_tokens # total_tokens = litellm bulk (on stripped msgs) + image tokens - expected_total = text_tokens + expected_total_image_tokens if is_anthropic_model(model) else text_tokens + expected_total = ( + text_tokens + expected_total_image_tokens + if is_anthropic_model(model) + else text_tokens + ) assert result.total_tokens == expected_total diff --git a/tests/core/test_llm_api_base_version.py b/tests/core/test_llm_api_base_version.py index 2b0357f009..cd0beb8a4e 100644 --- a/tests/core/test_llm_api_base_version.py +++ b/tests/core/test_llm_api_base_version.py @@ -88,9 +88,7 @@ def test_check_llm_with_api_base_version(self, mock_validate, mock_get_provider) ) @patch("litellm.get_llm_provider") - def test_check_llm_azure_api_version_handling( - self, mock_get_provider, monkeypatch - ): + def test_check_llm_azure_api_version_handling(self, mock_get_provider, monkeypatch): """Test Azure-specific api_version handling in check_llm.""" mock_get_provider.return_value = ("azure/gpt-4o", "azure") monkeypatch.setenv("AZURE_API_KEY", "test-key") @@ -182,9 +180,7 @@ def test_check_llm_azure_all_env_vars_set_passes( ) @patch("litellm.get_llm_provider") - def test_check_llm_azure_all_config_passes( - self, mock_get_provider, monkeypatch - ): + def test_check_llm_azure_all_config_passes(self, mock_get_provider, monkeypatch): """Test Azure provider passes when all variables from configuration.""" mock_get_provider.return_value = ("gpt-4o", "azure") monkeypatch.delenv("AZURE_API_KEY", raising=False) diff --git a/tests/core/test_prompt.py b/tests/core/test_prompt.py index 5d39dfadf9..6ee69c5542 100644 --- a/tests/core/test_prompt.py +++ b/tests/core/test_prompt.py @@ -314,6 +314,7 @@ def test_chat_api_user_prompt( expected_global_instructions=extract_instructions(global_instructions), ) + class TestUserPromptComponents: """Test that user prompts include all expected components via generate_user_prompt.""" diff --git a/tests/core/test_tool_memory_limit.py b/tests/core/test_tool_memory_limit.py index b7a1896a59..a6f136b7e5 100644 --- a/tests/core/test_tool_memory_limit.py +++ b/tests/core/test_tool_memory_limit.py @@ -104,9 +104,15 @@ def test_large_go_stack_trace_is_truncated(self): ] # Add many goroutine stack lines to simulate a real crash for i in range(200): - goroutine_lines.append(f"goroutine {i+2} gp=0x{i:08x} m=nil [GC worker (idle)]:") - goroutine_lines.append(f"runtime.gopark(0x{i:08x}?, 0x0?, 0x0?, 0x0?, 0x0?)") - goroutine_lines.append(f"\truntime/proc.go:435 +0xce fp=0x{i:08x} sp=0x{i:08x}") + goroutine_lines.append( + f"goroutine {i+2} gp=0x{i:08x} m=nil [GC worker (idle)]:" + ) + goroutine_lines.append( + f"runtime.gopark(0x{i:08x}?, 0x0?, 0x0?, 0x0?, 0x0?)" + ) + goroutine_lines.append( + f"\truntime/proc.go:435 +0xce fp=0x{i:08x} sp=0x{i:08x}" + ) output = "\n".join(goroutine_lines) result = check_oom_and_append_hint(output, 2) @@ -153,6 +159,8 @@ def test_output_over_limit_truncated(self): result_lines = result.splitlines() assert len(result_lines) == OOM_OUTPUT_MAX_LINES + 1 # +1 for omission marker assert result_lines[0] == "line 0" - assert result_lines[OOM_OUTPUT_MAX_LINES - 1] == f"line {OOM_OUTPUT_MAX_LINES - 1}" + assert ( + result_lines[OOM_OUTPUT_MAX_LINES - 1] == f"line {OOM_OUTPUT_MAX_LINES - 1}" + ) omitted = total_lines - OOM_OUTPUT_MAX_LINES assert f"[... {omitted} lines of stack trace omitted ...]" in result_lines[-1] diff --git a/tests/core/test_tool_output_deduplication.py b/tests/core/test_tool_output_deduplication.py index aaf167bc32..38007f3af1 100644 --- a/tests/core/test_tool_output_deduplication.py +++ b/tests/core/test_tool_output_deduplication.py @@ -5,12 +5,9 @@ error and data causes duplication in the LLM message. """ -import pytest - from holmes.core.models import format_tool_result_data from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus - TOOL_CALL_ID = "call_test123" TOOL_NAME = "kubectl_get" diff --git a/tests/integration/test_kubernetes_transformer_execution.py b/tests/integration/test_kubernetes_transformer_execution.py index 03b1a09a41..4688d6e420 100644 --- a/tests/integration/test_kubernetes_transformer_execution.py +++ b/tests/integration/test_kubernetes_transformer_execution.py @@ -193,8 +193,11 @@ def test_kubernetes_tabular_query_with_transformer(self): # Execute the tool context = create_mock_tool_invoke_context() result = kubernetes_tabular_query.invoke( - {"kind": "pods", "columns": "NAME:.metadata.name,NAMESPACE:.metadata.namespace,STATUS:.status.phase,NODE:.spec.nodeName"}, - context + { + "kind": "pods", + "columns": "NAME:.metadata.name,NAMESPACE:.metadata.namespace,STATUS:.status.phase,NODE:.spec.nodeName", + }, + context, ) # Should have applied transformation (threshold is 10000) @@ -283,7 +286,13 @@ def test_transformer_error_status_handling(self): # Load the actual kubernetes_logs.yaml file current_dir = os.path.dirname(os.path.abspath(__file__)) kubernetes_logs_yaml_path = os.path.join( - current_dir, "..", "..", "holmes", "plugins", "toolsets", "kubernetes_logs.yaml" + current_dir, + "..", + "..", + "holmes", + "plugins", + "toolsets", + "kubernetes_logs.yaml", ) toolsets = load_toolsets_from_file(kubernetes_logs_yaml_path) diff --git a/tests/llm/conftest.py b/tests/llm/conftest.py index 3efa12a757..3a31e7e0e6 100644 --- a/tests/llm/conftest.py +++ b/tests/llm/conftest.py @@ -869,8 +869,12 @@ def _collect_test_results_from_stats(terminalreporter): "completion_tokens": user_props.get("completion_tokens", 0), "cached_tokens": user_props.get("cached_tokens"), "reasoning_tokens": user_props.get("reasoning_tokens", 0), - "max_completion_tokens_per_call": user_props.get("max_completion_tokens_per_call", 0), - "max_prompt_tokens_per_call": user_props.get("max_prompt_tokens_per_call", 0), + "max_completion_tokens_per_call": user_props.get( + "max_completion_tokens_per_call", 0 + ), + "max_prompt_tokens_per_call": user_props.get( + "max_prompt_tokens_per_call", 0 + ), "num_compactions": user_props.get("num_compactions", 0), # Tag tracking for performance analysis "tags": user_props.get("tags", []), diff --git a/tests/llm/fixtures/test_ask_holmes/212_large_configmap_needle/generate_configs.py b/tests/llm/fixtures/test_ask_holmes/212_large_configmap_needle/generate_configs.py index 5e31af695f..c8532122a3 100644 --- a/tests/llm/fixtures/test_ask_holmes/212_large_configmap_needle/generate_configs.py +++ b/tests/llm/fixtures/test_ask_holmes/212_large_configmap_needle/generate_configs.py @@ -8,51 +8,161 @@ import random import sys - SEED = 212 NEEDLE_SERVICE = "payment-gateway" -NEEDLE_CONNECTION_STRING = "postgresql://admin@db-pmt-7k3m9x.internal.svc:5432/transactions_v2" +NEEDLE_CONNECTION_STRING = ( + "postgresql://admin@db-pmt-7k3m9x.internal.svc:5432/transactions_v2" +) SERVICE_NAMES = [ - "auth-provider", "billing-engine", "cart-service", "checkout-api", - "content-delivery", "coupon-manager", "customer-profile", "data-pipeline", - "delivery-tracker", "discount-engine", "email-dispatcher", "event-bus", - "feature-flags", "feedback-collector", "file-storage", "fraud-detector", - "geo-locator", "graph-resolver", "health-monitor", "identity-broker", - "image-processor", "import-service", "index-builder", "insight-engine", - "integration-hub", "inventory-manager", "invoice-generator", "job-scheduler", - "kafka-bridge", "key-manager", "label-service", "lead-tracker", - "license-manager", "link-shortener", "load-balancer", "log-aggregator", - "loyalty-program", "mail-queue", "marketplace-api", "media-encoder", - "membership-service", "message-broker", "metrics-collector", "migration-runner", - "ml-inference", "notification-hub", "oauth-gateway", "onboarding-flow", - "order-processor", "org-manager", "outbox-relay", "package-registry", - NEEDLE_SERVICE, "pdf-generator", "permission-service", "pipeline-orchestrator", - "platform-gateway", "plugin-loader", "policy-engine", "preference-store", - "price-calculator", "product-catalog", "promo-engine", "provisioner", - "pubsub-adapter", "query-optimizer", "queue-manager", "quota-service", - "rate-limiter", "recommendation-engine", "refund-processor", "registry-sync", - "reminder-service", "render-engine", "report-builder", "request-router", - "resource-allocator", "retry-handler", "review-moderator", "risk-analyzer", - "role-manager", "rule-engine", "sandbox-controller", "scheduler-api", - "schema-registry", "search-indexer", "secret-rotator", "session-manager", - "settlement-service", "shard-manager", "shipping-calculator", "sla-monitor", - "snapshot-service", "social-connector", "sse-gateway", "status-page", - "storage-gateway", "stream-processor", "subscription-manager", "support-ticket", - "sync-coordinator", "tag-service", "task-runner", "tax-calculator", - "telemetry-agent", "template-engine", "tenant-manager", "test-harness", - "theme-service", "throttle-controller", "timeline-service", "token-issuer", - "transaction-log", "transform-pipeline", "translation-service", "trial-manager", - "upload-handler", "usage-tracker", "user-directory", "validation-service", - "vault-proxy", "vendor-api", "version-control", "video-transcoder", - "virtual-network", "visitor-tracker", "webhook-relay", "workflow-engine", - "workspace-manager", "zipkin-collector", + "auth-provider", + "billing-engine", + "cart-service", + "checkout-api", + "content-delivery", + "coupon-manager", + "customer-profile", + "data-pipeline", + "delivery-tracker", + "discount-engine", + "email-dispatcher", + "event-bus", + "feature-flags", + "feedback-collector", + "file-storage", + "fraud-detector", + "geo-locator", + "graph-resolver", + "health-monitor", + "identity-broker", + "image-processor", + "import-service", + "index-builder", + "insight-engine", + "integration-hub", + "inventory-manager", + "invoice-generator", + "job-scheduler", + "kafka-bridge", + "key-manager", + "label-service", + "lead-tracker", + "license-manager", + "link-shortener", + "load-balancer", + "log-aggregator", + "loyalty-program", + "mail-queue", + "marketplace-api", + "media-encoder", + "membership-service", + "message-broker", + "metrics-collector", + "migration-runner", + "ml-inference", + "notification-hub", + "oauth-gateway", + "onboarding-flow", + "order-processor", + "org-manager", + "outbox-relay", + "package-registry", + NEEDLE_SERVICE, + "pdf-generator", + "permission-service", + "pipeline-orchestrator", + "platform-gateway", + "plugin-loader", + "policy-engine", + "preference-store", + "price-calculator", + "product-catalog", + "promo-engine", + "provisioner", + "pubsub-adapter", + "query-optimizer", + "queue-manager", + "quota-service", + "rate-limiter", + "recommendation-engine", + "refund-processor", + "registry-sync", + "reminder-service", + "render-engine", + "report-builder", + "request-router", + "resource-allocator", + "retry-handler", + "review-moderator", + "risk-analyzer", + "role-manager", + "rule-engine", + "sandbox-controller", + "scheduler-api", + "schema-registry", + "search-indexer", + "secret-rotator", + "session-manager", + "settlement-service", + "shard-manager", + "shipping-calculator", + "sla-monitor", + "snapshot-service", + "social-connector", + "sse-gateway", + "status-page", + "storage-gateway", + "stream-processor", + "subscription-manager", + "support-ticket", + "sync-coordinator", + "tag-service", + "task-runner", + "tax-calculator", + "telemetry-agent", + "template-engine", + "tenant-manager", + "test-harness", + "theme-service", + "throttle-controller", + "timeline-service", + "token-issuer", + "transaction-log", + "transform-pipeline", + "translation-service", + "trial-manager", + "upload-handler", + "usage-tracker", + "user-directory", + "validation-service", + "vault-proxy", + "vendor-api", + "version-control", + "video-transcoder", + "virtual-network", + "visitor-tracker", + "webhook-relay", + "workflow-engine", + "workspace-manager", + "zipkin-collector", ] TEAMS = [ - "platform", "payments", "identity", "growth", "infrastructure", - "data", "commerce", "security", "observability", "devex", - "mobile", "frontend", "backend", "ml-ops", "sre", + "platform", + "payments", + "identity", + "growth", + "infrastructure", + "data", + "commerce", + "security", + "observability", + "devex", + "mobile", + "frontend", + "backend", + "ml-ops", + "sre", ] DB_ENGINES = ["postgresql", "mysql", "mongodb", "redis", "cassandra"] @@ -131,65 +241,67 @@ def generate_service_config(rng, name, connection_string=None): lines = [] lines.append(f" {name}:") lines.append(f" name: {name}") - lines.append(f" version: \"{version}\"") + lines.append(f' version: "{version}"') lines.append(f" team: {team}") - lines.append(f" deployment:") + lines.append(" deployment:") lines.append(f" region: {region}") lines.append(f" environment: {env}") lines.append(f" strategy: {rng.choice(['rolling', 'blue-green', 'canary'])}") lines.append(f" max_surge: {rng.choice(['25%', '50%', '1', '2'])}") lines.append(f" max_unavailable: {rng.choice(['0', '1', '25%'])}") - lines.append(f" resources:") - lines.append(f" requests:") - lines.append(f" cpu: \"{cpu_req}\"") - lines.append(f" memory: \"{mem_req}\"") - lines.append(f" limits:") - lines.append(f" cpu: \"{cpu_limit}\"") - lines.append(f" memory: \"{mem_limit}\"") - lines.append(f" networking:") + lines.append(" resources:") + lines.append(" requests:") + lines.append(f' cpu: "{cpu_req}"') + lines.append(f' memory: "{mem_req}"') + lines.append(" limits:") + lines.append(f' cpu: "{cpu_limit}"') + lines.append(f' memory: "{mem_limit}"') + lines.append(" networking:") lines.append(f" protocol: {protocol}") lines.append(f" port: {port}") lines.append(f" health_check: {health_path}") lines.append(f" metrics_endpoint: {metrics_path}") lines.append(f" timeout_ms: {timeout_ms}") - lines.append(f" database:") - lines.append(f" connection_string: \"{connection_string}\"") + lines.append(" database:") + lines.append(f' connection_string: "{connection_string}"') lines.append(f" pool_size: {rng.randint(5, 50)}") lines.append(f" max_idle: {rng.randint(2, 20)}") lines.append(f" connection_timeout_ms: {rng.randint(1000, 10000)}") lines.append(f" read_replicas: {rng.randint(0, 3)}") - lines.append(f" cache:") + lines.append(" cache:") lines.append(f" backend: {cache_backend}") - lines.append(f" host: \"{cache_host}\"") + lines.append(f' host: "{cache_host}"') lines.append(f" port: {cache_port}") lines.append(f" ttl_seconds: {rng.choice([60, 300, 600, 1800, 3600])}") - lines.append(f" max_memory: \"{rng.choice(['64mb', '128mb', '256mb', '512mb'])}\"") - lines.append(f" logging:") + lines.append( + f" max_memory: \"{rng.choice(['64mb', '128mb', '256mb', '512mb'])}\"" + ) + lines.append(" logging:") lines.append(f" level: {log_level}") lines.append(f" output: {rng.choice(['stdout', 'file', 'both'])}") lines.append(f" retention_days: {rng.choice([7, 14, 30, 90])}") lines.append(f" structured: {rng.choice(['true', 'false'])}") - lines.append(f" monitoring:") + lines.append(" monitoring:") lines.append(f" alerts_enabled: {rng.choice(['true', 'false'])}") - lines.append(f" slo_target: \"{round(rng.uniform(99.0, 99.99), 2)}%\"") + lines.append(f' slo_target: "{round(rng.uniform(99.0, 99.99), 2)}%"') lines.append(f" error_budget_burn_rate: {round(rng.uniform(1.0, 10.0), 1)}") lines.append(f" pager_severity: {rng.choice(['P1', 'P2', 'P3', 'P4'])}") - lines.append(f" resilience:") + lines.append(" resilience:") lines.append(f" retry_count: {retry_count}") lines.append(f" retry_backoff_ms: {rng.choice([100, 200, 500, 1000])}") lines.append(f" circuit_breaker_threshold: {circuit_breaker_threshold}") lines.append(f" circuit_breaker_timeout_s: {circuit_breaker_timeout}") lines.append(f" bulkhead_max_concurrent: {rng.randint(10, 100)}") - lines.append(f" autoscaling:") + lines.append(" autoscaling:") lines.append(f" min_replicas: {replicas_min}") lines.append(f" max_replicas: {replicas_max}") lines.append(f" cpu_target_percent: {rng.choice([50, 60, 70, 80])}") lines.append(f" memory_target_percent: {rng.choice([60, 70, 80, 85])}") lines.append(f" scale_down_stabilization_s: {rng.choice([60, 120, 300])}") - lines.append(f" environment:") + lines.append(" environment:") for k, v in env_vars.items(): - lines.append(f" {k}: \"{v}\"") - lines.append(f" dependencies:") + lines.append(f' {k}: "{v}"') + lines.append(" dependencies:") for dep in deps: lines.append(f" - {dep}") @@ -209,7 +321,9 @@ def main(): for name in SERVICE_NAMES: if name == NEEDLE_SERVICE: - section = generate_service_config(rng, name, connection_string=NEEDLE_CONNECTION_STRING) + section = generate_service_config( + rng, name, connection_string=NEEDLE_CONNECTION_STRING + ) else: section = generate_service_config(rng, name) sections.append(section) @@ -223,11 +337,17 @@ def main(): size_tokens_approx = size_bytes // 4 if NEEDLE_CONNECTION_STRING not in content: - print("ERROR: Needle connection string not found in generated content", file=sys.stderr) + print( + "ERROR: Needle connection string not found in generated content", + file=sys.stderr, + ) sys.exit(1) if "7k3m9x" not in content: - print("ERROR: Needle identifier 7k3m9x not found in generated content", file=sys.stderr) + print( + "ERROR: Needle identifier 7k3m9x not found in generated content", + file=sys.stderr, + ) sys.exit(1) print(f"Generated {len(SERVICE_NAMES)} service configs") diff --git a/tests/llm/fixtures/test_ask_holmes/231_confluence_large_page_eval/generate_large_page.py b/tests/llm/fixtures/test_ask_holmes/231_confluence_large_page_eval/generate_large_page.py index 9489416fab..7d01c6c139 100644 --- a/tests/llm/fixtures/test_ask_holmes/231_confluence_large_page_eval/generate_large_page.py +++ b/tests/llm/fixtures/test_ask_holmes/231_confluence_large_page_eval/generate_large_page.py @@ -6,12 +6,9 @@ context window). """ -import hashlib import json import random -import sys import textwrap -import time VERIFICATION_CODE = "HOLMES-EVAL-vK7w3nR9pL" @@ -20,24 +17,50 @@ # --------------------------------------------------------------------------- _SERVICES = [ - "checkout-api", "user-service", "inventory-db", "payment-gateway", - "notification-hub", "catalog-sync", "analytics-pipeline", "search-indexer", - "media-processor", "auth-proxy", "rate-limiter", "config-server", - "session-store", "cache-warmer", "logging-collector", "metrics-exporter", - "event-bus", "scheduler-daemon", "batch-processor", "data-migrator", + "checkout-api", + "user-service", + "inventory-db", + "payment-gateway", + "notification-hub", + "catalog-sync", + "analytics-pipeline", + "search-indexer", + "media-processor", + "auth-proxy", + "rate-limiter", + "config-server", + "session-store", + "cache-warmer", + "logging-collector", + "metrics-exporter", + "event-bus", + "scheduler-daemon", + "batch-processor", + "data-migrator", ] _ENVS = ["dev", "staging", "production", "dr-west", "dr-east"] _ROOT_CAUSES = [ - "memory leak in connection pool", "certificate expiry", "DNS resolution timeout", - "disk I/O saturation", "kernel OOM killer", "CPU throttling due to cgroup limits", - "network partition between AZs", "corrupted WAL segment", "lock contention on shared mutex", - "thread pool exhaustion", "GC pause exceeding health-check timeout", - "misconfigured autoscaler floor", "expired service-account token", - "race condition in cache invalidation", "silent data corruption in replication stream", - "upstream rate-limit enforcement change", "TLS handshake failure after library upgrade", - "pod eviction due to ephemeral storage pressure", "zombie process accumulation", + "memory leak in connection pool", + "certificate expiry", + "DNS resolution timeout", + "disk I/O saturation", + "kernel OOM killer", + "CPU throttling due to cgroup limits", + "network partition between AZs", + "corrupted WAL segment", + "lock contention on shared mutex", + "thread pool exhaustion", + "GC pause exceeding health-check timeout", + "misconfigured autoscaler floor", + "expired service-account token", + "race condition in cache invalidation", + "silent data corruption in replication stream", + "upstream rate-limit enforcement change", + "TLS handshake failure after library upgrade", + "pod eviction due to ephemeral storage pressure", + "zombie process accumulation", "incorrect iptables rule after CNI upgrade", ] @@ -75,8 +98,16 @@ def _paragraph(seed: int, min_sentences: int = 8, max_sentences: int = 15) -> st n2=rng.randint(2, 50), ack=rng.randint(1, 15), pct=round(rng.uniform(0.1, 45.0), 1), - metric=rng.choice(["p99_latency", "error_rate", "cpu_utilization", - "memory_pressure", "disk_iops", "connection_count"]), + metric=rng.choice( + [ + "p99_latency", + "error_rate", + "cpu_utilization", + "memory_pressure", + "disk_iops", + "connection_count", + ] + ), ) sentences.append(s) return " ".join(sentences) @@ -99,8 +130,14 @@ def _incident_section(incident_id: int, seed: int) -> str: f"Duration: {rng.randint(5, 180)} minutes | " f"Root cause: {cause}

", ] - for sub in ["Summary", "Timeline", "Root Cause Analysis", "Remediation", - "Lessons Learned", "Action Items"]: + for sub in [ + "Summary", + "Timeline", + "Root Cause Analysis", + "Remediation", + "Lessons Learned", + "Action Items", + ]: parts.append(f"

{sub}

") # 2-3 paragraphs per subsection for j in range(rng.randint(2, 3)): @@ -112,12 +149,14 @@ def _incident_section(incident_id: int, seed: int) -> str: # Main page generation # --------------------------------------------------------------------------- + def generate_page_body() -> str: parts = [] # --- Introduction (~500 words) --- parts.append("

Platform Incident Post-Mortem Catalogue 2023-2025

") - parts.append(textwrap.dedent("""\ + parts.append( + textwrap.dedent("""\

Purpose

This document is the authoritative record of all platform incidents classified as P1 through P4 across all environments. It is maintained by the Site Reliability @@ -125,16 +164,19 @@ def generate_page_body() -> str: incident includes a timeline, root-cause analysis, remediation steps, and follow-up action items. Teams are expected to reference this catalogue when writing new runbooks or proposing architectural changes.

- """)) + """) + ) - parts.append(textwrap.dedent("""\ + parts.append( + textwrap.dedent("""\

Incident Classification

Incidents are classified by customer impact and blast radius. P1 incidents affect more than 50% of production traffic or involve data loss. P2 incidents affect a single service with measurable customer impact. P3 incidents cause degraded performance without breaching SLO budgets. P4 incidents are near-misses or internal-only issues caught before reaching customers.

- """)) + """) + ) # --- Generate ~80 incident sections --- # Each section is ~800-1200 words. 80 sections ≈ 70-96K words ≈ 280-384K chars @@ -146,32 +188,39 @@ def generate_page_body() -> str: # --- The needle: a summary table buried at the very end --- parts.append("

Quarterly Reliability Summary

") - parts.append(textwrap.dedent("""\ + parts.append( + textwrap.dedent("""\

The table below summarises per-quarter SLO attainment and the internal audit reference codes used by the compliance team for external reporting.

- """)) + """) + ) parts.append("") - parts.append("") + parts.append( + "" + ) # Several normal rows audit_codes = [ ("Q1 2023", "99.92%", "12", "ARC-2023Q1-mT4x8b"), ("Q2 2023", "99.87%", "15", "ARC-2023Q2-jN6y2w"), - ("Q3 2023", "99.95%", "9", "ARC-2023Q3-qP3z7c"), + ("Q3 2023", "99.95%", "9", "ARC-2023Q3-qP3z7c"), ("Q4 2023", "99.78%", "18", "ARC-2023Q4-hR9v1d"), ("Q1 2024", "99.91%", "11", "ARC-2024Q1-kW5m8f"), ("Q2 2024", "99.83%", "16", "ARC-2024Q2-bX2n4g"), - ("Q3 2024", "99.96%", "8", "ARC-2024Q3-eY7p9h"), + ("Q3 2024", "99.96%", "8", "ARC-2024Q3-eY7p9h"), # This is the row with the verification code ("Q4 2024", "99.71%", "21", f"ARC-2024Q4-{VERIFICATION_CODE}"), ("Q1 2025", "99.88%", "13", "ARC-2025Q1-dZ6r3j"), ] for quarter, slo, count, code in audit_codes: - parts.append(f"") + parts.append( + f"" + ) parts.append("
QuarterSLO AttainmentIncidentsAudit Reference Code
QuarterSLO AttainmentIncidentsAudit Reference Code
{quarter}{slo}{count}{code}
{quarter}{slo}{count}{code}
") - parts.append(textwrap.dedent("""\ + parts.append( + textwrap.dedent("""\

Document Revision History

2025-03-01: Added Q1 2025 preliminary data. Updated incident INC-1078 remediation status.

@@ -180,7 +229,8 @@ def generate_page_body() -> str: the November platform migration.

2024-09-30: Added Q3 2024 data. Lowest incident quarter on record.

- """)) + """) + ) return "\n".join(parts) diff --git a/tests/llm/fixtures/test_ask_holmes/50a_logs_since_last_specific_month/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/50a_logs_since_last_specific_month/test_case.yaml index b53b566bb4..211ea742ee 100644 --- a/tests/llm/fixtures/test_ask_holmes/50a_logs_since_last_specific_month/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/50a_logs_since_last_specific_month/test_case.yaml @@ -11,4 +11,4 @@ evaluation: correctness: 1 skip: true -skip_reason: "this test fails now as the toolset are not consistent with the mocked data" \ No newline at end of file +skip_reason: "this test fails now as the toolset are not consistent with the mocked data" diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml index 77c1bb582a..d322eed452 100644 --- a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml @@ -7,4 +7,4 @@ tags: - medium - transparency # Says it is missing health/liveness probes -# The deployment declares ports 8080 (http) and 8443 (https), but **no process is actually listening on these ports**. TCP connectivity tests to both ports on both pods timed out. \ No newline at end of file +# The deployment declares ports 8080 (http) and 8443 (https), but **no process is actually listening on these ports**. TCP connectivity tests to both ports on both pods timed out. diff --git a/tests/llm/test_ask_holmes.py b/tests/llm/test_ask_holmes.py index da1eab5072..06bfa8a5a1 100644 --- a/tests/llm/test_ask_holmes.py +++ b/tests/llm/test_ask_holmes.py @@ -9,6 +9,7 @@ from unittest.mock import patch import pytest + from holmes.config import Config from holmes.core.conversations import build_chat_messages from holmes.core.models import ChatRequest @@ -23,7 +24,6 @@ from tests.llm.utils.env_config import EnvConfig, get_env_configs from tests.llm.utils.iteration_utils import get_test_cases from tests.llm.utils.mock_dal import load_test_dal -from tests.llm.utils.test_toolset import TestToolsetManager from tests.llm.utils.property_manager import ( handle_test_error, set_initial_properties, @@ -37,6 +37,7 @@ create_eval_llm, get_models, ) +from tests.llm.utils.test_toolset import TestToolsetManager TEST_CASES_FOLDER = Path( path.abspath(path.join(path.dirname(__file__), "fixtures", "test_ask_holmes")) @@ -196,8 +197,7 @@ def ask_holmes( ) test_type = ( - test_case.test_type - or os.environ.get("ASK_HOLMES_TEST_TYPE", "cli").lower() + test_case.test_type or os.environ.get("ASK_HOLMES_TEST_TYPE", "cli").lower() ) if test_type == "cli": if test_case.conversation_history: @@ -231,9 +231,7 @@ def ask_holmes( if test_case.cluster_name: config.cluster_name = test_case.cluster_name - dal = load_test_dal( - Path(test_case.folder), initialize_base=False - ) + dal = load_test_dal(Path(test_case.folder), initialize_base=False) runbooks = load_runbook_catalog(dal) global_instructions = dal.get_global_instructions_for_account() diff --git a/tests/llm/test_holmes_checks.py b/tests/llm/test_holmes_checks.py index 190ed6a55c..92851ed7c7 100644 --- a/tests/llm/test_holmes_checks.py +++ b/tests/llm/test_holmes_checks.py @@ -170,7 +170,7 @@ def test_holmes_check( print(f"\n🧪 TEST: {test_case.id}") print(" CONFIGURATION:") - print(f" • Mode: 🔥 LIVE") + print(" • Mode: 🔥 LIVE") print(f" • Checks: {len(test_case.checks)} checks") print(f" • Expected Results: {test_case.expected_results}") diff --git a/tests/llm/utils/braintrust.py b/tests/llm/utils/braintrust.py index 158794a9d7..264d56e8a8 100644 --- a/tests/llm/utils/braintrust.py +++ b/tests/llm/utils/braintrust.py @@ -17,7 +17,10 @@ get_experiment_name, get_machine_state_tags, ) -from tests.llm.utils.test_case_utils import AskHolmesTestCase, HolmesTestCase # type: ignore +from tests.llm.utils.test_case_utils import ( # type: ignore + AskHolmesTestCase, + HolmesTestCase, +) braintrust_enabled = False if BRAINTRUST_API_KEY: @@ -232,8 +235,7 @@ def log_to_braintrust( ): # Find the first message with role "system" system_msg = next( - (m for m in result.messages if m.get("role") == "system"), - None + (m for m in result.messages if m.get("role") == "system"), None ) prompt = system_msg["content"] if system_msg else "" diff --git a/tests/llm/utils/braintrust_history.py b/tests/llm/utils/braintrust_history.py index 1edcbd9915..1d27141276 100644 --- a/tests/llm/utils/braintrust_history.py +++ b/tests/llm/utils/braintrust_history.py @@ -57,7 +57,6 @@ class BenchmarkMetrics: cached_tokens: Optional[int] = None - @dataclass class HistoricalComparison: """Comparison data between current and benchmark metrics.""" @@ -298,7 +297,9 @@ def get_benchmark_baseline() -> ( try: api_key = _get_api_key() if not api_key: - details.status = "No Braintrust API key (BRAINTRUST_API_KEY or BRAINTRUST_SERVICE_TOKEN)" + details.status = ( + "No Braintrust API key (BRAINTRUST_API_KEY or BRAINTRUST_SERVICE_TOKEN)" + ) return {}, details project_id = _get_project_id() diff --git a/tests/llm/utils/classifiers.py b/tests/llm/utils/classifiers.py index bdfd3670f9..879066d9a7 100644 --- a/tests/llm/utils/classifiers.py +++ b/tests/llm/utils/classifiers.py @@ -233,5 +233,3 @@ def evaluate_correctness( return classifier( input=prompt_prefix, output=output, expected=expected_elements_str ) - - diff --git a/tests/llm/utils/mock_dal.py b/tests/llm/utils/mock_dal.py index 749996d8c8..c5ea073385 100644 --- a/tests/llm/utils/mock_dal.py +++ b/tests/llm/utils/mock_dal.py @@ -92,9 +92,7 @@ def get_global_instructions_for_account(self) -> Optional[Instructions]: data = json.load(f) return Instructions(**data) except Exception as e: - logging.warning( - f"Failed to read global instructions fixture file: {e}" - ) + logging.warning(f"Failed to read global instructions fixture file: {e}") return None @@ -140,7 +138,9 @@ def get_issues_metadata( if not include_external and item_cluster == "external": continue else: - allowed = target_clusters + (["external"] if include_external else []) + allowed = target_clusters + ( + ["external"] if include_external else [] + ) if item_cluster not in allowed: continue if workload: diff --git a/tests/llm/utils/property_manager.py b/tests/llm/utils/property_manager.py index 5c3584cafd..335b189396 100644 --- a/tests/llm/utils/property_manager.py +++ b/tests/llm/utils/property_manager.py @@ -236,16 +236,17 @@ def update_test_results( ("completion_tokens", result.completion_tokens) ) if hasattr(result, "cached_tokens"): - request.node.user_properties.append( - ("cached_tokens", result.cached_tokens) - ) + request.node.user_properties.append(("cached_tokens", result.cached_tokens)) if hasattr(result, "reasoning_tokens"): request.node.user_properties.append( ("reasoning_tokens", result.reasoning_tokens) ) if hasattr(result, "max_completion_tokens_per_call"): request.node.user_properties.append( - ("max_completion_tokens_per_call", result.max_completion_tokens_per_call) + ( + "max_completion_tokens_per_call", + result.max_completion_tokens_per_call, + ) ) if hasattr(result, "max_prompt_tokens_per_call"): request.node.user_properties.append( diff --git a/tests/llm/utils/reporting/github_reporter.py b/tests/llm/utils/reporting/github_reporter.py index 7dd24df5ce..cbc2f4b7bd 100644 --- a/tests/llm/utils/reporting/github_reporter.py +++ b/tests/llm/utils/reporting/github_reporter.py @@ -36,7 +36,9 @@ def _format_diff_pct(diff: Optional[float]) -> str: return f"**{indicator}**" if bold else indicator -def _calc_diff_pct(current: Optional[float], baseline: Optional[float]) -> Optional[float]: +def _calc_diff_pct( + current: Optional[float], baseline: Optional[float] +) -> Optional[float]: """Calculate percentage difference: positive = current is higher.""" if not current or not baseline or baseline == 0: return None @@ -63,17 +65,19 @@ def _generate_comparison_tables( comparison = comparison_map.get(key) baseline = benchmark.get(key) - rows.append({ - "name": f"{test_name} ({model})" if model else test_name, - "current_time": result.get("holmes_duration"), - "baseline_time": baseline.duration if baseline else None, - "current_cost": result.get("cost"), - "baseline_cost": baseline.cost if baseline else None, - "current_total_tokens": result.get("total_tokens", 0) or 0, - "baseline_total_tokens": baseline.total_tokens if baseline else None, - "current_cached_tokens": result.get("cached_tokens"), - "baseline_cached_tokens": baseline.cached_tokens if baseline else None, - }) + rows.append( + { + "name": f"{test_name} ({model})" if model else test_name, + "current_time": result.get("holmes_duration"), + "baseline_time": baseline.duration if baseline else None, + "current_cost": result.get("cost"), + "baseline_cost": baseline.cost if baseline else None, + "current_total_tokens": result.get("total_tokens", 0) or 0, + "baseline_total_tokens": baseline.total_tokens if baseline else None, + "current_cached_tokens": result.get("cached_tokens"), + "baseline_cached_tokens": baseline.cached_tokens if baseline else None, + } + ) # --- Time comparison table --- has_time_data = any(r["baseline_time"] is not None for r in rows) @@ -84,7 +88,9 @@ def _generate_comparison_tables( for r in rows: cur = f"{r['current_time']:.1f}s" if r["current_time"] else "—" base = f"{r['baseline_time']:.1f}s" if r["baseline_time"] else "—" - diff = _format_diff_pct(_calc_diff_pct(r["current_time"], r["baseline_time"])) + diff = _format_diff_pct( + _calc_diff_pct(r["current_time"], r["baseline_time"]) + ) lines.append(f"| {r['name']} | {cur} | {base} | {diff} |") lines.append("") @@ -97,7 +103,9 @@ def _generate_comparison_tables( for r in rows: cur = f"${r['current_cost']:.4f}" if r["current_cost"] else "—" base = f"${r['baseline_cost']:.4f}" if r["baseline_cost"] else "—" - diff = _format_diff_pct(_calc_diff_pct(r["current_cost"], r["baseline_cost"])) + diff = _format_diff_pct( + _calc_diff_pct(r["current_cost"], r["baseline_cost"]) + ) lines.append(f"| {r['name']} | {cur} | {base} | {diff} |") lines.append("") @@ -194,7 +202,9 @@ def _generate_historical_details_section( # Experiments used if details.experiments: - lines.append(f"\n**Benchmark experiment{'s' if len(details.experiments) > 1 else ''}:**\n") + lines.append( + f"\n**Benchmark experiment{'s' if len(details.experiments) > 1 else ''}:**\n" + ) for exp in details.experiments: exp_url = f"https://www.braintrust.dev/app/{BRAINTRUST_ORG}/p/{BRAINTRUST_PROJECT}/experiments/{exp.id}" created_info = f" (created: {exp.created[:10]})" if exp.created else "" @@ -414,7 +424,9 @@ def generate_markdown_report( input_str = _fmt_tokens(prompt_tokens) output_str = _fmt_tokens(completion_tokens) cached_tokens_str = f"{cached_tokens:,}" if cached_tokens is not None else "—" - non_cached_tokens_str = f"{non_cached_tokens:,}" if non_cached_tokens is not None else "—" + non_cached_tokens_str = ( + f"{non_cached_tokens:,}" if non_cached_tokens is not None else "—" + ) reasoning_str = _fmt_tokens(reasoning_tokens) max_completion_str = _fmt_tokens(max_completion) max_prompt_str = _fmt_tokens(max_prompt) diff --git a/tests/llm/utils/reporting/terminal_reporter.py b/tests/llm/utils/reporting/terminal_reporter.py index da91af9765..102f12c322 100644 --- a/tests/llm/utils/reporting/terminal_reporter.py +++ b/tests/llm/utils/reporting/terminal_reporter.py @@ -266,7 +266,9 @@ def handle_console_output(sorted_results: List[dict], terminalreporter=None) -> # Format total tokens (fall back to prompt + completion if total not provided) total_tokens = result.get("total_tokens", 0) or 0 if total_tokens == 0: - total_tokens = (result.get("prompt_tokens", 0) or 0) + (result.get("completion_tokens", 0) or 0) + total_tokens = (result.get("prompt_tokens", 0) or 0) + ( + result.get("completion_tokens", 0) or 0 + ) if total_tokens > 0: tokens_str = f"{total_tokens:,}" else: diff --git a/tests/llm/utils/test_case_utils.py b/tests/llm/utils/test_case_utils.py index 3e84bd94b9..130535af4e 100644 --- a/tests/llm/utils/test_case_utils.py +++ b/tests/llm/utils/test_case_utils.py @@ -368,7 +368,9 @@ def load_test_cases(self) -> List[HolmesTestCase]: test_case = TypeAdapter(HolmesTestCase).validate_python(config_dict) elif self._test_cases_folder.name == "test_holmes_checks": # Import CheckTestCase here to avoid circular imports - from tests.llm.test_holmes_checks import CheckTestCase # type: ignore + from tests.llm.test_holmes_checks import ( + CheckTestCase, # type: ignore + ) test_case = TypeAdapter(CheckTestCase).validate_python(config_dict) else: diff --git a/tests/llm/utils/test_toolset.py b/tests/llm/utils/test_toolset.py index a9f9f3cedb..c28d9231a5 100644 --- a/tests/llm/utils/test_toolset.py +++ b/tests/llm/utils/test_toolset.py @@ -2,7 +2,7 @@ import logging import os from pathlib import Path -from typing import List, Optional, Type +from typing import List, Optional from holmes.core.tools import ( Toolset, diff --git a/tests/mocks/toolset_mocks.py b/tests/mocks/toolset_mocks.py index 6382d259ee..f8a8a617ed 100644 --- a/tests/mocks/toolset_mocks.py +++ b/tests/mocks/toolset_mocks.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Dict, List from holmes.core.tools import ( StructuredToolResult, diff --git a/tests/plugins/runbooks/test_catalog.py b/tests/plugins/runbooks/test_catalog.py index 374ff5ff59..f811a4fc2d 100644 --- a/tests/plugins/runbooks/test_catalog.py +++ b/tests/plugins/runbooks/test_catalog.py @@ -1,8 +1,4 @@ -import os - from holmes.plugins.runbooks import ( - DEFAULT_RUNBOOK_SEARCH_PATH, - get_runbook_by_path, load_runbook_catalog, ) diff --git a/tests/plugins/sources/test_pagerduty_source.py b/tests/plugins/sources/test_pagerduty_source.py index 971a865780..e6439e1607 100644 --- a/tests/plugins/sources/test_pagerduty_source.py +++ b/tests/plugins/sources/test_pagerduty_source.py @@ -1,7 +1,5 @@ from unittest.mock import MagicMock, patch -import pytest - from holmes.plugins.sources.pagerduty import PagerDutySource SAMPLE_INCIDENT = { @@ -138,9 +136,7 @@ def test_fetch_issues_still_works(self, mock_get): """Backward compatibility: fetch_issues (plural) sets description from incident data.""" response = MagicMock() response.status_code = 200 - response.json.return_value = { - "incidents": [SAMPLE_INCIDENT] - } + response.json.return_value = {"incidents": [SAMPLE_INCIDENT]} mock_get.return_value = response diff --git a/tests/plugins/toolsets/database/test_database.py b/tests/plugins/toolsets/database/test_database.py index 7ed20f93fe..1134e0dfda 100644 --- a/tests/plugins/toolsets/database/test_database.py +++ b/tests/plugins/toolsets/database/test_database.py @@ -13,18 +13,12 @@ def test_read_only_default(self): def test_read_only_explicit_true(self): """Test that read_only can be explicitly set to True.""" - config = DatabaseConfig( - connection_url="sqlite:///:memory:", - read_only=True - ) + config = DatabaseConfig(connection_url="sqlite:///:memory:", read_only=True) assert config.read_only is True def test_read_only_explicit_false(self): """Test that read_only can be explicitly set to False.""" - config = DatabaseConfig( - connection_url="sqlite:///:memory:", - read_only=False - ) + config = DatabaseConfig(connection_url="sqlite:///:memory:", read_only=False) assert config.read_only is False def test_connection_url_required(self): diff --git a/tests/plugins/toolsets/datadog/logs/test_check_prerequisites.py b/tests/plugins/toolsets/datadog/logs/test_check_prerequisites.py index f001962574..5a795d0cc3 100644 --- a/tests/plugins/toolsets/datadog/logs/test_check_prerequisites.py +++ b/tests/plugins/toolsets/datadog/logs/test_check_prerequisites.py @@ -88,10 +88,7 @@ def test_check_prerequisites_successful_healthcheck(self, mock_execute_request): assert toolset.dd_config is not None assert toolset.dd_config.api_key == "test-api-key" assert toolset.dd_config.app_key == "test-app-key" - assert ( - str(toolset.dd_config.api_url).rstrip("/") - == "https://api.datadoghq.com" - ) + assert str(toolset.dd_config.api_url).rstrip("/") == "https://api.datadoghq.com" assert toolset.dd_config.storage_tiers == DEFAULT_STORAGE_TIERS # Verify healthcheck was called with correct params diff --git a/tests/plugins/toolsets/datadog/metrics/test_datadog_metrics_live.py b/tests/plugins/toolsets/datadog/metrics/test_datadog_metrics_live.py index 55450a3e11..5615ee2a55 100644 --- a/tests/plugins/toolsets/datadog/metrics/test_datadog_metrics_live.py +++ b/tests/plugins/toolsets/datadog/metrics/test_datadog_metrics_live.py @@ -12,7 +12,8 @@ @pytest.mark.skipif( - not os.getenv("RUN_SLOW_TESTS") or not all([os.getenv("DD_API_KEY"), os.getenv("DD_APP_KEY")]), + not os.getenv("RUN_SLOW_TESTS") + or not all([os.getenv("DD_API_KEY"), os.getenv("DD_APP_KEY")]), reason="Slow test - set RUN_SLOW_TESTS=1 and Datadog credentials to run", ) class TestDatadogMetricsLiveIntegration: diff --git a/tests/plugins/toolsets/datadog/traces/test_datadog_traces_live.py b/tests/plugins/toolsets/datadog/traces/test_datadog_traces_live.py index 6af9e768d0..bb0b61620e 100644 --- a/tests/plugins/toolsets/datadog/traces/test_datadog_traces_live.py +++ b/tests/plugins/toolsets/datadog/traces/test_datadog_traces_live.py @@ -9,7 +9,8 @@ @pytest.mark.skipif( - not os.getenv("RUN_SLOW_TESTS") or not all([os.getenv("DD_API_KEY"), os.getenv("DD_APP_KEY")]), + not os.getenv("RUN_SLOW_TESTS") + or not all([os.getenv("DD_API_KEY"), os.getenv("DD_APP_KEY")]), reason="Slow test - set RUN_SLOW_TESTS=1 and Datadog credentials to run", ) class TestDatadogTracesLiveIntegration: diff --git a/tests/plugins/toolsets/grafana/test_grafana_tempo_api.py b/tests/plugins/toolsets/grafana/test_grafana_tempo_api.py index db29dde870..6a9ce71489 100644 --- a/tests/plugins/toolsets/grafana/test_grafana_tempo_api.py +++ b/tests/plugins/toolsets/grafana/test_grafana_tempo_api.py @@ -7,7 +7,10 @@ from requests.exceptions import HTTPError, RequestException # type: ignore from holmes.plugins.toolsets.grafana.common import GrafanaTempoConfig -from holmes.plugins.toolsets.grafana.grafana_tempo_api import GrafanaTempoAPI, TempoAPIError +from holmes.plugins.toolsets.grafana.grafana_tempo_api import ( + GrafanaTempoAPI, + TempoAPIError, +) # Test constants TEST_SERVICE_NAME = "checkout-service" diff --git a/tests/plugins/toolsets/grafana/test_grafana_tempo_tools.py b/tests/plugins/toolsets/grafana/test_grafana_tempo_tools.py index 61c7a31dfe..c8f009de9c 100644 --- a/tests/plugins/toolsets/grafana/test_grafana_tempo_tools.py +++ b/tests/plugins/toolsets/grafana/test_grafana_tempo_tools.py @@ -551,7 +551,9 @@ def test_all_tools_handle_negative_start(self, tempo_toolset): class TestQueryMetricsRangeWithStepAdjustment: """Test QueryMetricsRange with automatic step adjustment.""" - def test_metrics_range_with_no_step_auto_calculates(self, tempo_toolset, monkeypatch): + def test_metrics_range_with_no_step_auto_calculates( + self, tempo_toolset, monkeypatch + ): """Test that step is automatically calculated when not provided.""" import holmes.plugins.toolsets.grafana.toolset_grafana_tempo as tempo_module @@ -580,7 +582,9 @@ def test_metrics_range_with_no_step_auto_calculates(self, tempo_toolset, monkeyp # The function should convert this to "36s" assert kwargs["step"] == "36s" - def test_metrics_range_with_small_step_gets_adjusted(self, tempo_toolset, monkeypatch): + def test_metrics_range_with_small_step_gets_adjusted( + self, tempo_toolset, monkeypatch + ): """Test that a too-small step gets adjusted to prevent too many points.""" import holmes.plugins.toolsets.grafana.toolset_grafana_tempo as tempo_module @@ -658,7 +662,9 @@ def test_metrics_range_with_bare_number_step(self, tempo_toolset): # Step should be "30s" since 30 seconds is fine for 300 second range assert kwargs["step"] == "30s" - def test_metrics_range_step_adjustment_various_ranges(self, tempo_toolset, monkeypatch): + def test_metrics_range_step_adjustment_various_ranges( + self, tempo_toolset, monkeypatch + ): """Test step adjustment for various time ranges.""" import holmes.plugins.toolsets.grafana.toolset_grafana_tempo as tempo_module diff --git a/tests/plugins/toolsets/grafana/test_grafana_tempo_unit.py b/tests/plugins/toolsets/grafana/test_grafana_tempo_unit.py index cd6b0361a8..aa154f1acf 100644 --- a/tests/plugins/toolsets/grafana/test_grafana_tempo_unit.py +++ b/tests/plugins/toolsets/grafana/test_grafana_tempo_unit.py @@ -3,7 +3,7 @@ from holmes.core.tools import ( StructuredToolResultStatus, ) -from holmes.plugins.toolsets.grafana.common import GrafanaTempoConfig, GrafanaTempoLabelsConfig +from holmes.plugins.toolsets.grafana.common import GrafanaTempoConfig from holmes.plugins.toolsets.grafana.toolset_grafana_tempo import ( FetchTracesSimpleComparison, GrafanaTempoToolset, @@ -470,9 +470,7 @@ def test_build_k8s_filters_after_prerequisites_callable(): "grafana_datasource_uid": "tempo_uid", } - with patch( - "holmes.plugins.toolsets.grafana.toolset_grafana_tempo.GrafanaTempoAPI" - ): + with patch("holmes.plugins.toolsets.grafana.toolset_grafana_tempo.GrafanaTempoAPI"): toolset.prerequisites_callable(config) # This is the call path that failed with: diff --git a/tests/plugins/toolsets/http/test_http_toolset.py b/tests/plugins/toolsets/http/test_http_toolset.py index 6edc173b54..262c2b75f2 100644 --- a/tests/plugins/toolsets/http/test_http_toolset.py +++ b/tests/plugins/toolsets/http/test_http_toolset.py @@ -83,7 +83,9 @@ def test_multiple_hosts(self): assert endpoint.hosts == ["api.example.com", "api2.example.com"] def test_custom_methods(self): - endpoint = EndpointConfig(hosts=["example.com"], methods=["GET", "POST", "DELETE"]) + endpoint = EndpointConfig( + hosts=["example.com"], methods=["GET", "POST", "DELETE"] + ) assert endpoint.get_methods() == ["GET", "POST", "DELETE"] def test_methods_normalized_to_uppercase(self): @@ -152,7 +154,9 @@ def test_invalid_url(self, toolset): assert endpoint is None def test_host_match_with_port(self, toolset): - endpoint, error = toolset.match_endpoint("https://api.github.com:8443/repos/foo/bar") + endpoint, error = toolset.match_endpoint( + "https://api.github.com:8443/repos/foo/bar" + ) assert error is None assert endpoint is not None assert "api.github.com" in endpoint.hosts @@ -410,11 +414,7 @@ class TestHttpToolsetMultiInstance: def test_two_instances_different_names(self): ts1 = HttpToolset(name="confluence") ts1.prerequisites_callable( - { - "endpoints": [ - {"hosts": ["*.atlassian.net"], "auth": {"type": "none"}} - ] - } + {"endpoints": [{"hosts": ["*.atlassian.net"], "auth": {"type": "none"}}]} ) ts2 = HttpToolset(name="dagster") @@ -435,11 +435,7 @@ def test_instance_with_llm_instructions(self): name="confluence", llm_instructions="Use Confluence REST API v2." ) toolset.prerequisites_callable( - { - "endpoints": [ - {"hosts": ["*.atlassian.net"], "auth": {"type": "none"}} - ] - } + {"endpoints": [{"hosts": ["*.atlassian.net"], "auth": {"type": "none"}}]} ) assert "Use Confluence REST API v2." in toolset.llm_instructions @@ -481,7 +477,9 @@ def test_headers_invalid_json(self, toolset, mock_context): assert "Invalid headers JSON" in result.error @patch("holmes.plugins.toolsets.http.http_toolset.requests.request") - def test_error_response_includes_error_field(self, mock_request, toolset, mock_context): + def test_error_response_includes_error_field( + self, mock_request, toolset, mock_context + ): mock_response = Mock() mock_response.ok = False mock_response.status_code = 404 @@ -788,7 +786,9 @@ def test_long_url_truncated(self): def test_default_method(self): ts = HttpToolset() tool = HttpRequest(ts) - result = tool.get_parameterized_one_liner({"url": "https://api.example.com/test"}) + result = tool.get_parameterized_one_liner( + {"url": "https://api.example.com/test"} + ) assert result.startswith("HTTP GET") @@ -800,7 +800,11 @@ def test_digest_auth_prereq(self): "endpoints": [ { "hosts": ["api.example.com"], - "auth": {"type": "digest", "username": "user", "password": "pass"}, + "auth": { + "type": "digest", + "username": "user", + "password": "pass", + }, } ] } diff --git a/tests/plugins/toolsets/test_confluence_tools.py b/tests/plugins/toolsets/test_confluence_tools.py index 1bc252a978..0516ba55b6 100644 --- a/tests/plugins/toolsets/test_confluence_tools.py +++ b/tests/plugins/toolsets/test_confluence_tools.py @@ -45,7 +45,9 @@ def test_gateway_activated_with_explicit_cloud_id(self): } with ( - patch.object(ConfluenceToolset, "_probe_request", return_value={"results": []}), + patch.object( + ConfluenceToolset, "_probe_request", return_value={"results": []} + ), patch.object(ConfluenceToolset, "_setup_http_tools"), ): ok, msg = ts.prerequisites_callable(config) @@ -84,7 +86,10 @@ def side_effect(*args, **kwargs): with ( patch.object(ConfluenceToolset, "_probe_request", side_effect=side_effect), - patch("holmes.plugins.toolsets.confluence.confluence.requests.get", return_value=tenant_resp), + patch( + "holmes.plugins.toolsets.confluence.confluence.requests.get", + return_value=tenant_resp, + ), patch.object(ConfluenceToolset, "_setup_http_tools"), ): ok, msg = ts.prerequisites_callable(config) @@ -141,7 +146,9 @@ def test_gateway_fallback_fails_gracefully(self): forbidden_resp = MagicMock() forbidden_resp.status_code = 403 - forbidden_resp.text = '{"message":"Current user not permitted to use Confluence"}' + forbidden_resp.text = ( + '{"message":"Current user not permitted to use Confluence"}' + ) # Both direct and gateway calls fail with 403 with ( @@ -150,7 +157,9 @@ def test_gateway_fallback_fails_gracefully(self): "_probe_request", side_effect=requests.exceptions.HTTPError(response=forbidden_resp), ), - patch("holmes.plugins.toolsets.confluence.confluence.requests.get") as mock_get, + patch( + "holmes.plugins.toolsets.confluence.confluence.requests.get" + ) as mock_get, ): tenant_resp = MagicMock() tenant_resp.status_code = 200 @@ -174,7 +183,9 @@ def test_cloud_id_resolution_failure(self): forbidden_resp = MagicMock() forbidden_resp.status_code = 403 - forbidden_resp.text = '{"message":"Current user not permitted to use Confluence"}' + forbidden_resp.text = ( + '{"message":"Current user not permitted to use Confluence"}' + ) with ( patch.object( @@ -255,7 +266,9 @@ def test_setup_http_tools_registers_tool(self): api_key="api-token", ) - with patch("holmes.plugins.toolsets.http.http_toolset.requests.get") as mock_get: + with patch( + "holmes.plugins.toolsets.http.http_toolset.requests.get" + ) as mock_get: mock_resp = MagicMock() mock_resp.ok = True mock_get.return_value = mock_resp diff --git a/tests/plugins/toolsets/test_elasticsearch_mtls.py b/tests/plugins/toolsets/test_elasticsearch_mtls.py index adba078b8d..6a9df86dd6 100644 --- a/tests/plugins/toolsets/test_elasticsearch_mtls.py +++ b/tests/plugins/toolsets/test_elasticsearch_mtls.py @@ -45,7 +45,9 @@ def test_ca_cert_accepted_but_ignored(self): api_url="https://es:9200", ca_cert="/path/to/ca.crt", ) - assert not hasattr(config, "ca_cert") or config.model_fields.get("ca_cert") is None + assert ( + not hasattr(config, "ca_cert") or config.model_fields.get("ca_cert") is None + ) assert config.client_cert is None def test_config_without_mtls(self): diff --git a/tests/plugins/toolsets/test_kubernetes_transformers.py b/tests/plugins/toolsets/test_kubernetes_transformers.py index f6ce08842b..629f7696db 100644 --- a/tests/plugins/toolsets/test_kubernetes_transformers.py +++ b/tests/plugins/toolsets/test_kubernetes_transformers.py @@ -51,9 +51,7 @@ def test_load_kubernetes_yaml_with_transformers(self): kubernetes_jq_query = tool break - assert ( - kubernetes_jq_query is not None - ), "kubernetes_jq_query tool not found" + assert kubernetes_jq_query is not None, "kubernetes_jq_query tool not found" assert kubernetes_jq_query.transformers is not None assert len(kubernetes_jq_query.transformers) == 1 assert kubernetes_jq_query.transformers[0].name == "llm_summarize" @@ -485,9 +483,7 @@ def test_threshold_values_are_appropriate(self): ), None, ) - assert ( - kubernetes_jq_query is not None - ), "kubernetes_jq_query tool not found" + assert kubernetes_jq_query is not None, "kubernetes_jq_query tool not found" assert kubernetes_jq_query.transformers is not None assert kubernetes_jq_query.transformers[0].config["input_threshold"] == 10000 diff --git a/tests/plugins/toolsets/test_logging_api.py b/tests/plugins/toolsets/test_logging_api.py index 585307ddfe..b4704d38b2 100644 --- a/tests/plugins/toolsets/test_logging_api.py +++ b/tests/plugins/toolsets/test_logging_api.py @@ -1,11 +1,7 @@ """Tests for the logging API, specifically the PodLoggingTool behavior.""" -import os from unittest.mock import MagicMock -import pytest - -from holmes.core.llm import DefaultLLM from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus from holmes.core.tools_utils.token_counting import count_tool_response_tokens from holmes.plugins.toolsets.logging_utils.logging_api import ( diff --git a/tests/plugins/toolsets/test_runbook.py b/tests/plugins/toolsets/test_runbook.py index 8750394730..66a1cf56a3 100644 --- a/tests/plugins/toolsets/test_runbook.py +++ b/tests/plugins/toolsets/test_runbook.py @@ -7,9 +7,7 @@ ) from tests.conftest import create_mock_tool_invoke_context -TEST_RUNBOOKS_PATH = os.path.join( - os.path.dirname(__file__), "fixtures", "runbooks" -) +TEST_RUNBOOKS_PATH = os.path.join(os.path.dirname(__file__), "fixtures", "runbooks") def test_RunbookFetcher(): diff --git a/tests/test_approval_workflow.py b/tests/test_approval_workflow.py index 98f805640a..dab886a26f 100644 --- a/tests/test_approval_workflow.py +++ b/tests/test_approval_workflow.py @@ -277,7 +277,10 @@ def test_streaming_chat_approval_workflow_approve_and_execute( # Mock _execute_tool_decisions to simulate approval and execution ai._execute_tool_decisions = MagicMock( - side_effect=lambda messages, tool_decisions, request_context=None, trace_span=None: ( + side_effect=lambda messages, + tool_decisions, + request_context=None, + trace_span=None: ( messages + [ { @@ -429,7 +432,10 @@ def test_streaming_chat_approval_workflow_reject_command( # Mock _execute_tool_decisions to simulate rejection ai._execute_tool_decisions = MagicMock( - side_effect=lambda messages, tool_decisions, request_context=None, trace_span=None: ( + side_effect=lambda messages, + tool_decisions, + request_context=None, + trace_span=None: ( messages + [ { diff --git a/tests/test_bash_toolset_validation.py b/tests/test_bash_toolset_validation.py index 49e69ce952..2bda998d51 100644 --- a/tests/test_bash_toolset_validation.py +++ b/tests/test_bash_toolset_validation.py @@ -8,7 +8,7 @@ import pytest -from holmes.plugins.toolsets.bash.bash_toolset import BashExecutorToolset, RunBashCommand +from holmes.plugins.toolsets.bash.bash_toolset import BashExecutorToolset from holmes.plugins.toolsets.bash.common.config import ( HARDCODED_BLOCKS, BashExecutorConfig, @@ -144,7 +144,9 @@ def test_piped_command(self): def test_multiple_pipes(self): """Test parsing multiple pipes.""" - segments, has_compound = parse_command_segments("kubectl get pods | grep error | head -10") + segments, has_compound = parse_command_segments( + "kubectl get pods | grep error | head -10" + ) assert segments == ["kubectl get pods", "grep error", "head -10"] assert not has_compound @@ -156,7 +158,9 @@ def test_and_operator(self): def test_or_operator(self): """Test parsing || operator.""" - segments, has_compound = parse_command_segments("test -f file.txt || touch file.txt") + segments, has_compound = parse_command_segments( + "test -f file.txt || touch file.txt" + ) assert segments == ["test -f file.txt", "touch file.txt"] assert not has_compound @@ -181,14 +185,18 @@ def test_invalid_pipe_syntax_raises(self): def test_for_loop_extracts_inner_segments(self): """For loop returns inner command segments with compound flag.""" - segments, has_compound = parse_command_segments('for i in 1 2 3; do echo "$i"; done') + segments, has_compound = parse_command_segments( + 'for i in 1 2 3; do echo "$i"; done' + ) assert has_compound assert len(segments) > 0 assert any("echo" in s for s in segments) def test_if_statement_extracts_inner_segments(self): """If statement returns inner command segments with compound flag.""" - segments, has_compound = parse_command_segments("if [ -f file ]; then cat file; fi") + segments, has_compound = parse_command_segments( + "if [ -f file ]; then cat file; fi" + ) assert has_compound assert len(segments) > 0 @@ -238,39 +246,78 @@ class TestCheckBlockedInRawCommand: def test_sudo_in_compound_detected(self): """Test that sudo inside a compound command is detected.""" - assert check_blocked_in_raw_command("for i in 1 2; do sudo echo $i; done", HARDCODED_BLOCKS) == "sudo" + assert ( + check_blocked_in_raw_command( + "for i in 1 2; do sudo echo $i; done", HARDCODED_BLOCKS + ) + == "sudo" + ) def test_su_in_compound_detected(self): """Test that su inside a compound command is detected.""" - assert check_blocked_in_raw_command("if true; then su - root; fi", HARDCODED_BLOCKS) == "su" + assert ( + check_blocked_in_raw_command( + "if true; then su - root; fi", HARDCODED_BLOCKS + ) + == "su" + ) def test_sudo_in_subshell_detected(self): """Test that sudo inside a subshell is detected.""" - assert check_blocked_in_raw_command("echo $(sudo whoami)", HARDCODED_BLOCKS) == "sudo" + assert ( + check_blocked_in_raw_command("echo $(sudo whoami)", HARDCODED_BLOCKS) + == "sudo" + ) def test_normal_compound_not_blocked(self): """Test that normal compound commands are not blocked.""" - assert check_blocked_in_raw_command("for i in 1 2 3; do echo $i; done", HARDCODED_BLOCKS) is None + assert ( + check_blocked_in_raw_command( + "for i in 1 2 3; do echo $i; done", HARDCODED_BLOCKS + ) + is None + ) def test_no_false_positives_from_substring(self): """Test that words containing 'su' as substring are NOT blocked.""" - assert check_blocked_in_raw_command("for f in issue result; do echo $f; done", HARDCODED_BLOCKS) is None + assert ( + check_blocked_in_raw_command( + "for f in issue result; do echo $f; done", HARDCODED_BLOCKS + ) + is None + ) assert check_blocked_in_raw_command("echo sum", HARDCODED_BLOCKS) is None def test_case_insensitive(self): """Test that blocking is case-insensitive.""" - assert check_blocked_in_raw_command("for i in 1; do SUDO echo $i; done", HARDCODED_BLOCKS) == "sudo" + assert ( + check_blocked_in_raw_command( + "for i in 1; do SUDO echo $i; done", HARDCODED_BLOCKS + ) + == "sudo" + ) def test_deny_list_pattern_detected(self): """Test that deny list patterns are detected in raw commands.""" deny_list = ["kubectl get secret", "rm"] - assert check_blocked_in_raw_command("case $x in 1) kubectl get secret;; esac", deny_list) == "kubectl get secret" - assert check_blocked_in_raw_command("case $x in 1) rm -rf /tmp;; esac", deny_list) == "rm" + assert ( + check_blocked_in_raw_command( + "case $x in 1) kubectl get secret;; esac", deny_list + ) + == "kubectl get secret" + ) + assert ( + check_blocked_in_raw_command("case $x in 1) rm -rf /tmp;; esac", deny_list) + == "rm" + ) def test_deny_list_no_false_positives(self): """Test that deny list scanning doesn't have false positives from substrings.""" deny_list = ["rm"] - assert check_blocked_in_raw_command("case $x in 1) echo format;; esac", deny_list) is None + assert ( + check_blocked_in_raw_command("case $x in 1) echo format;; esac", deny_list) + is None + ) class TestGetEffectiveLists: @@ -868,7 +915,9 @@ def test_case_statement_requires_approval(self): deny_list, ) assert result.status == ValidationStatus.APPROVAL_REQUIRED - assert result.message == "Command contains complex syntax which requires approval." + assert ( + result.message == "Command contains complex syntax which requires approval." + ) assert result.prefixes_needing_approval == [] # ==================== Subshells: validated via segment checking ==================== @@ -948,7 +997,9 @@ def test_process_substitution_inner_not_allowed(self): deny_list, ) assert result.status == ValidationStatus.APPROVAL_REQUIRED - assert "Segment(s) not in allow list: 'cat file1', 'cat file2'" in result.message + assert ( + "Segment(s) not in allow list: 'cat file1', 'cat file2'" in result.message + ) # ==================== STILL BLOCKED: Hardcoded blocks inside scripts ==================== @@ -1058,7 +1109,9 @@ def test_unparseable_command_requires_approval(self): deny_list, ) assert result.status == ValidationStatus.APPROVAL_REQUIRED - assert result.message == "Command contains complex syntax which requires approval." + assert ( + result.message == "Command contains complex syntax which requires approval." + ) # ==================== Integration: requires_approval prefixes_to_save ==================== @@ -1116,7 +1169,9 @@ def test_unapproved_segment_saves_prefixes(self, tool, context): assert result.needs_approval is True assert result.prefixes_to_save == ["mycustomtool"] - def test_piped_command_with_unapproved_segment_saves_only_unapproved(self, tool, context): + def test_piped_command_with_unapproved_segment_saves_only_unapproved( + self, tool, context + ): """Piped command where one segment is unapproved saves only the unapproved prefix.""" params = { "command": "echo hello | mycustomtool --process", diff --git a/tests/test_cache.py b/tests/test_cache.py index d8adb7feb9..82b7b7e92e 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -16,8 +16,8 @@ from holmes.core.tool_calling_llm import ToolCallingLLM from holmes.core.tools_utils.tool_executor import ToolExecutor from tests.llm.utils.mock_dal import load_test_dal -from tests.llm.utils.test_toolset import TestToolsetManager from tests.llm.utils.test_case_utils import get_models +from tests.llm.utils.test_toolset import TestToolsetManager logger = logging.getLogger(__name__) @@ -73,9 +73,7 @@ def capture_litellm_completion(*args, **kwargs): ) config = Config() - test_dal = load_test_dal( - Path(temp_dir.name), initialize_base=False - ) + test_dal = load_test_dal(Path(temp_dir.name), initialize_base=False) runbooks = config.get_runbook_catalog() asks = [ diff --git a/tests/test_check_prerequisites.py b/tests/test_check_prerequisites.py index 937b55b8a1..00514d0861 100644 --- a/tests/test_check_prerequisites.py +++ b/tests/test_check_prerequisites.py @@ -1,11 +1,9 @@ # type: ignore import os import subprocess -from typing import Any, Dict, List +from typing import Dict, List from unittest.mock import Mock, call, patch -import pytest - from holmes.core.tools import ( CallablePrerequisite, StaticPrerequisite, diff --git a/tests/test_database_toolset.py b/tests/test_database_toolset.py index e7fe8adac2..f381377531 100644 --- a/tests/test_database_toolset.py +++ b/tests/test_database_toolset.py @@ -9,11 +9,11 @@ sqlalchemy = pytest.importorskip("sqlalchemy") from holmes.plugins.toolsets.database.database import ( # noqa: E402 - DatabaseConfig, - DatabaseToolset, _READONLY_PATTERN, _WRITE_ANYWHERE_PATTERN, _WRITE_PATTERN, + DatabaseConfig, + DatabaseToolset, _normalise_url, _serialize_value, ) diff --git a/tests/test_format_tags.py b/tests/test_format_tags.py index 894070147b..0be40363ef 100644 --- a/tests/test_format_tags.py +++ b/tests/test_format_tags.py @@ -56,9 +56,7 @@ def test_parse_message_tags_multimodal_with_tags(): }, {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc123"}}, ] - result = parse_messages_tags( - [{"role": "user", "content": original_content}] - ) + result = parse_messages_tags([{"role": "user", "content": original_content}]) expected_content = [ { "type": "text", diff --git a/tests/test_header_propagation.py b/tests/test_header_propagation.py index f77a488ac4..53a9b729d4 100644 --- a/tests/test_header_propagation.py +++ b/tests/test_header_propagation.py @@ -9,12 +9,8 @@ 5. ToolInvokeContext (pre-rendered headers) """ -import os -from typing import Any, Dict, Optional, Tuple from unittest.mock import Mock, patch -import pytest - from holmes.core.tools import ( StructuredToolResultStatus, ToolInvokeContext, @@ -22,11 +18,11 @@ ) from holmes.utils.header_rendering import render_header_templates - # --------------------------------------------------------------------------- # Shared utility tests # --------------------------------------------------------------------------- + class TestRenderTemplateHeaders: def test_static_value(self): result = render_header_templates({"X-Static": "hello"}) @@ -34,9 +30,7 @@ def test_static_value(self): def test_env_var(self, monkeypatch): monkeypatch.setenv("TEST_HEADER_VAR", "from-env") - result = render_header_templates( - {"X-Env": "{{ env.TEST_HEADER_VAR }}"} - ) + result = render_header_templates({"X-Env": "{{ env.TEST_HEADER_VAR }}"}) assert result == {"X-Env": "from-env"} def test_request_context_header(self): @@ -92,6 +86,7 @@ def test_mixed_templates(self, monkeypatch): # YAML tool Jinja2 template context (request_context + env) # --------------------------------------------------------------------------- + class TestYAMLToolTemplateContext: def test_command_renders_request_context_header(self): """request_context.headers is available in command Jinja2 templates.""" @@ -101,8 +96,12 @@ def test_command_renders_request_context_header(self): command="echo {{ request_context.headers['X-Tenant-Id'] }}", ) ctx = ToolInvokeContext.model_construct( - tool_number=1, user_approved=False, llm=Mock(), - max_token_count=1000, tool_call_id="c1", tool_name="t", + tool_number=1, + user_approved=False, + llm=Mock(), + max_token_count=1000, + tool_call_id="c1", + tool_name="t", request_context={"headers": {"X-Tenant-Id": "tenant-abc"}}, ) result = tool._invoke({}, ctx) @@ -118,8 +117,12 @@ def test_command_renders_env_var(self, monkeypatch): command="echo Bearer {{ env.MY_TOKEN }}", ) ctx = ToolInvokeContext.model_construct( - tool_number=1, user_approved=False, llm=Mock(), - max_token_count=1000, tool_call_id="c1", tool_name="t", + tool_number=1, + user_approved=False, + llm=Mock(), + max_token_count=1000, + tool_call_id="c1", + tool_name="t", ) result = tool._invoke({}, ctx) assert result.status == StructuredToolResultStatus.SUCCESS @@ -133,8 +136,12 @@ def test_command_works_without_request_context(self): command="echo hello", ) ctx = ToolInvokeContext.model_construct( - tool_number=1, user_approved=False, llm=Mock(), - max_token_count=1000, tool_call_id="c1", tool_name="t", + tool_number=1, + user_approved=False, + llm=Mock(), + max_token_count=1000, + tool_call_id="c1", + tool_name="t", ) result = tool._invoke({}, ctx) assert result.status == StructuredToolResultStatus.SUCCESS @@ -148,8 +155,12 @@ def test_script_renders_request_context_header(self): script="#!/bin/bash\necho {{ request_context.headers['X-Auth'] }}", ) ctx = ToolInvokeContext.model_construct( - tool_number=1, user_approved=False, llm=Mock(), - max_token_count=1000, tool_call_id="c1", tool_name="t", + tool_number=1, + user_approved=False, + llm=Mock(), + max_token_count=1000, + tool_call_id="c1", + tool_name="t", request_context={"headers": {"X-Auth": "Bearer secret"}}, ) result = tool._invoke({}, ctx) @@ -164,8 +175,12 @@ def test_command_renders_case_insensitive_header(self): command="echo {{ request_context.headers['x-tenant-id'] }}", ) ctx = ToolInvokeContext.model_construct( - tool_number=1, user_approved=False, llm=Mock(), - max_token_count=1000, tool_call_id="c1", tool_name="t", + tool_number=1, + user_approved=False, + llm=Mock(), + max_token_count=1000, + tool_call_id="c1", + tool_name="t", request_context={"headers": {"X-Tenant-Id": "tenant-abc"}}, ) result = tool._invoke({}, ctx) @@ -177,6 +192,7 @@ def test_command_renders_case_insensitive_header(self): # ToolInvokeContext tests # --------------------------------------------------------------------------- + class TestToolInvokeContextHeaders: def test_model_dump_redacts_request_context_headers(self): ctx = ToolInvokeContext.model_construct( @@ -193,35 +209,32 @@ def test_model_dump_redacts_request_context_headers(self): assert dumped["request_context"] == {"headers": "***REDACTED***"} - - # --------------------------------------------------------------------------- # HTTP toolset header propagation tests # --------------------------------------------------------------------------- + class TestHttpToolsetHeaderPropagation: @patch("holmes.plugins.toolsets.http.http_toolset.requests.request") def test_extra_headers_merged_into_request(self, mock_request): """Verify that config-level extra_headers are merged into HTTP requests.""" - from holmes.plugins.toolsets.http.http_toolset import HttpRequest, HttpToolset + from holmes.plugins.toolsets.http.http_toolset import HttpToolset # Create an HTTP toolset with extra_headers in config toolset = HttpToolset( name="test_http", enabled=True, config={ - "endpoints": [ - {"hosts": ["api.example.com"], "methods": ["GET"]} - ], + "endpoints": [{"hosts": ["api.example.com"], "methods": ["GET"]}], "extra_headers": {"X-Custom": "static-val"}, }, ) - ok, _ = toolset.prerequisites_callable({ - "endpoints": [ - {"hosts": ["api.example.com"], "methods": ["GET"]} - ], - "extra_headers": {"X-Custom": "static-val"}, - }) + ok, _ = toolset.prerequisites_callable( + { + "endpoints": [{"hosts": ["api.example.com"], "methods": ["GET"]}], + "extra_headers": {"X-Custom": "static-val"}, + } + ) assert ok mock_response = Mock() @@ -248,26 +261,24 @@ def test_extra_headers_merged_into_request(self, mock_request): @patch("holmes.plugins.toolsets.http.http_toolset.requests.request") def test_extra_headers_override_defaults(self, mock_request): """Verify that extra_headers override default headers.""" - from holmes.plugins.toolsets.http.http_toolset import HttpRequest, HttpToolset + from holmes.plugins.toolsets.http.http_toolset import HttpToolset toolset = HttpToolset( name="test_http", enabled=True, config={ - "endpoints": [ - {"hosts": ["api.example.com"], "methods": ["GET"]} - ], + "endpoints": [{"hosts": ["api.example.com"], "methods": ["GET"]}], "default_headers": {"X-Default": "original"}, "extra_headers": {"X-Default": "overridden"}, }, ) - ok, _ = toolset.prerequisites_callable({ - "endpoints": [ - {"hosts": ["api.example.com"], "methods": ["GET"]} - ], - "default_headers": {"X-Default": "original"}, - "extra_headers": {"X-Default": "overridden"}, - }) + ok, _ = toolset.prerequisites_callable( + { + "endpoints": [{"hosts": ["api.example.com"], "methods": ["GET"]}], + "default_headers": {"X-Default": "original"}, + "extra_headers": {"X-Default": "overridden"}, + } + ) assert ok mock_response = Mock() @@ -294,6 +305,7 @@ def test_extra_headers_override_defaults(self, mock_request): # MCP config-level extra_headers tests # --------------------------------------------------------------------------- + class TestMCPConfigExtraHeaders: def test_config_level_extra_headers_rendered(self): """Verify that config-level extra_headers are rendered in MCP headers.""" diff --git a/tests/test_http_docs.py b/tests/test_http_docs.py index ac964c51e5..06301ae54f 100644 --- a/tests/test_http_docs.py +++ b/tests/test_http_docs.py @@ -63,7 +63,9 @@ def create_mock_model_entry() -> ModelEntry: ) -def create_mock_litellm_response(content: str = "Mock analysis response for documentation test.") -> ModelResponse: +def create_mock_litellm_response( + content: str = "Mock analysis response for documentation test.", +) -> ModelResponse: """Create a mock litellm ModelResponse matching the real API structure.""" return ModelResponse( id="chatcmpl-mock-doc-test", @@ -124,12 +126,14 @@ def format_doc_test_failure( if extra_info: lines.append(f"Details: {extra_info}") - lines.extend([ - "", - f"Curl: {curl_preview}", - "", - "To fix: Edit the curl example or its annotation", - ]) + lines.extend( + [ + "", + f"Curl: {curl_preview}", + "", + "To fix: Edit the curl example or its annotation", + ] + ) return "\n".join(lines) @@ -154,7 +158,9 @@ def collect_doc_curl_tests() -> list[tuple[str, DocCurlTest]]: continue relative_path = md_file.relative_to(DOCS_DIR) - test_id = doc_test.curl.test_id or f"{relative_path}:{doc_test.curl.line_number}" + test_id = ( + doc_test.curl.test_id or f"{relative_path}:{doc_test.curl.line_number}" + ) tests.append((test_id, doc_test)) return tests @@ -211,7 +217,9 @@ def execute_curl_test(client: TestClient, doc_test: DocCurlTest) -> dict[str, An return { "status_code": response.status_code, "response": response, - "json": response.json() if response.headers.get("content-type", "").startswith("application/json") else None, + "json": response.json() + if response.headers.get("content-type", "").startswith("application/json") + else None, } @@ -252,7 +260,9 @@ def test_documented_curl( # Log response for debugging (visible in CI artifacts/extended logs) print(f"\n=== Test: {test_id} ===") print(f"Endpoint: {doc_test.curl.method} {doc_test.curl.url}") - print(f"Status: {result['status_code']} (expected: {doc_test.curl.expected_status})") + print( + f"Status: {result['status_code']} (expected: {doc_test.curl.expected_status})" + ) print(f"Response: {result.get('json', result.get('response', 'N/A'))}") # Validate status code @@ -291,9 +301,11 @@ def setup_mocks(self, monkeypatch): """Setup mocks for all tests.""" monkeypatch.setenv("OPENAI_API_KEY", "test-key") - with patch("litellm.completion") as mock_completion, \ - patch("holmes.core.llm.LLMModelRegistry.get_model_params") as mock_model, \ - patch("holmes.core.supabase_dal.SupabaseDal.get_global_instructions_for_account") as mock_instr: + with patch("litellm.completion") as mock_completion, patch( + "holmes.core.llm.LLMModelRegistry.get_model_params" + ) as mock_model, patch( + "holmes.core.supabase_dal.SupabaseDal.get_global_instructions_for_account" + ) as mock_instr: mock_model.return_value = create_mock_model_entry() mock_completion.return_value = create_mock_litellm_response() mock_instr.return_value = [] diff --git a/tests/test_json_schema_coerce.py b/tests/test_json_schema_coerce.py index 43acc89103..252bb7feb5 100644 --- a/tests/test_json_schema_coerce.py +++ b/tests/test_json_schema_coerce.py @@ -5,11 +5,11 @@ from holmes.core.json_schema_coerce import coerce_params from holmes.core.tools import ToolParameter - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _schema(**fields: ToolParameter) -> dict: return fields @@ -18,11 +18,14 @@ def _schema(**fields: ToolParameter) -> dict: # Stringified JSON → array / object (structural coercions) # --------------------------------------------------------------------------- + class TestStringifiedJsonCoercion: def test_stringified_array(self): result = coerce_params( {"metrics": '["cpu", "memory"]'}, - _schema(metrics=ToolParameter(type="array", items=ToolParameter(type="string"))), + _schema( + metrics=ToolParameter(type="array", items=ToolParameter(type="string")) + ), ) assert result["metrics"] == ["cpu", "memory"] @@ -76,18 +79,23 @@ def test_nullable_array(self): # Single value → array wrap # --------------------------------------------------------------------------- + class TestSingleValueArrayWrap: def test_wrap_string_in_array(self): result = coerce_params( {"metrics": "cpu"}, - _schema(metrics=ToolParameter(type="array", items=ToolParameter(type="string"))), + _schema( + metrics=ToolParameter(type="array", items=ToolParameter(type="string")) + ), ) assert result["metrics"] == ["cpu"] def test_wrap_int_in_array(self): result = coerce_params( {"ids": 42}, - _schema(ids=ToolParameter(type="array", items=ToolParameter(type="integer"))), + _schema( + ids=ToolParameter(type="array", items=ToolParameter(type="integer")) + ), ) assert result["ids"] == [42] @@ -110,6 +118,7 @@ def test_nullable_array_wrap(self): # String → integer # --------------------------------------------------------------------------- + class TestStringToInteger: def test_whole_number(self): result = coerce_params( @@ -170,6 +179,7 @@ def test_strict_mode_skips_string_to_int(self): # String → number (float) # --------------------------------------------------------------------------- + class TestStringToNumber: def test_float_string(self): result = coerce_params( @@ -220,6 +230,7 @@ def test_strict_mode_skips_string_to_number(self): # String → boolean # --------------------------------------------------------------------------- + class TestStringToBoolean: def test_true_string(self): result = coerce_params( @@ -271,6 +282,7 @@ def test_strict_mode_skips_string_to_bool(self): # Edge cases and general behavior # --------------------------------------------------------------------------- + class TestGeneralBehavior: def test_empty_params(self): result = coerce_params({}, _schema(metrics=ToolParameter(type="array"))) diff --git a/tests/test_mcp_refresh_backoff.py b/tests/test_mcp_refresh_backoff.py index dbab56625d..e7df655ca5 100644 --- a/tests/test_mcp_refresh_backoff.py +++ b/tests/test_mcp_refresh_backoff.py @@ -5,15 +5,21 @@ class TestMCPRefreshBackoff: def test_backoff_schedule_when_mcp_failed(self): """Walks through the full backoff schedule: 30s, 60s, 120s.""" - sleep, idx = _get_next_refresh_interval(has_failed_mcp=True, backoff_index=0, default_interval=300) + sleep, idx = _get_next_refresh_interval( + has_failed_mcp=True, backoff_index=0, default_interval=300 + ) assert sleep == 30 assert idx == 1 - sleep, idx = _get_next_refresh_interval(has_failed_mcp=True, backoff_index=1, default_interval=300) + sleep, idx = _get_next_refresh_interval( + has_failed_mcp=True, backoff_index=1, default_interval=300 + ) assert sleep == 60 assert idx == 2 - sleep, idx = _get_next_refresh_interval(has_failed_mcp=True, backoff_index=2, default_interval=300) + sleep, idx = _get_next_refresh_interval( + has_failed_mcp=True, backoff_index=2, default_interval=300 + ) assert sleep == 120 assert idx == 3 @@ -28,30 +34,42 @@ def test_falls_back_to_default_after_schedule_exhausted(self): def test_default_interval_when_no_failures(self): """When no MCP servers are failing, always uses the default interval.""" - sleep, idx = _get_next_refresh_interval(has_failed_mcp=False, backoff_index=0, default_interval=300) + sleep, idx = _get_next_refresh_interval( + has_failed_mcp=False, backoff_index=0, default_interval=300 + ) assert sleep == 300 assert idx == 0 def test_resets_backoff_when_mcp_recovers(self): """Simulates MCP failing then recovering mid-backoff.""" # First iteration: failed, start backoff - sleep, idx = _get_next_refresh_interval(has_failed_mcp=True, backoff_index=0, default_interval=300) + sleep, idx = _get_next_refresh_interval( + has_failed_mcp=True, backoff_index=0, default_interval=300 + ) assert sleep == 30 assert idx == 1 # MCP recovers — should reset to default - sleep, idx = _get_next_refresh_interval(has_failed_mcp=False, backoff_index=idx, default_interval=300) + sleep, idx = _get_next_refresh_interval( + has_failed_mcp=False, backoff_index=idx, default_interval=300 + ) assert sleep == 300 assert idx == 0 def test_backoff_restarts_on_new_failure(self): """After recovery and a new failure, backoff restarts from the beginning.""" # Exhaust backoff, recover, then fail again - _, idx = _get_next_refresh_interval(has_failed_mcp=True, backoff_index=0, default_interval=300) - _, idx = _get_next_refresh_interval(has_failed_mcp=False, backoff_index=idx, default_interval=300) + _, idx = _get_next_refresh_interval( + has_failed_mcp=True, backoff_index=0, default_interval=300 + ) + _, idx = _get_next_refresh_interval( + has_failed_mcp=False, backoff_index=idx, default_interval=300 + ) assert idx == 0 # reset # New failure starts from 30s again - sleep, idx = _get_next_refresh_interval(has_failed_mcp=True, backoff_index=idx, default_interval=300) + sleep, idx = _get_next_refresh_interval( + has_failed_mcp=True, backoff_index=idx, default_interval=300 + ) assert sleep == 30 assert idx == 1 diff --git a/tests/test_mcp_toolset.py b/tests/test_mcp_toolset.py index 400602ec85..6e47f07de5 100644 --- a/tests/test_mcp_toolset.py +++ b/tests/test_mcp_toolset.py @@ -338,7 +338,10 @@ def test_schema_with_refs_and_anyof_parsed_correctly(self) -> None: "include": ToolParameter( type="array", description="List of additional information to include in the response. Available options: 'users', 'services', 'assignments', 'acknowledgers', 'custom_fields', 'teams', 'escalation_policies', 'notes', 'urgencies', 'priorities'", - required=False, items=ToolParameter(type="string", required=True, description=None), + required=False, + items=ToolParameter( + type="string", required=True, description=None + ), json_schema_extra={"default": None}, ), }, @@ -365,35 +368,65 @@ def test_schema_with_allof_parsed_correctly(self) -> None: { "type": "object", "properties": { - "id": {"type": "string", "description": "The ID of the user"}, - "name": {"type": "string", "description": "The name of the user"} + "id": { + "type": "string", + "description": "The ID of the user", + }, + "name": { + "type": "string", + "description": "The name of the user", + }, }, - "required": ["id"] + "required": ["id"], }, { "type": "object", "properties": { - "email": {"type": "string", "description": "The email of the user"}, - "age": {"type": "integer", "description": "The age of the user"} + "email": { + "type": "string", + "description": "The email of the user", + }, + "age": { + "type": "integer", + "description": "The age of the user", + }, }, - "required": ["email"] - } + "required": ["email"], + }, ] } }, - "required": ["user_data"] + "required": ["user_data"], }, description="Update user data", annotations=None, ) expected_schema = { - "user_data": ToolParameter(type="object", required=True, properties={ - "id": ToolParameter(type="string", required=True, description="The ID of the user"), - "name": ToolParameter(type="string", required=False, description="The name of the user"), - "email": ToolParameter(type="string", required=True, description="The email of the user"), - "age": ToolParameter(type="integer", required=False, description="The age of the user"), - }), + "user_data": ToolParameter( + type="object", + required=True, + properties={ + "id": ToolParameter( + type="string", required=True, description="The ID of the user" + ), + "name": ToolParameter( + type="string", + required=False, + description="The name of the user", + ), + "email": ToolParameter( + type="string", + required=True, + description="The email of the user", + ), + "age": ToolParameter( + type="integer", + required=False, + description="The age of the user", + ), + }, + ), } mock_toolset = RemoteMCPToolset( @@ -625,7 +658,9 @@ def test_additional_properties_anyof_preserved(self) -> None: # Verify it flows through to OpenAI format openai_format = tool.get_openai_format() - filters_schema = openai_format["function"]["parameters"]["properties"]["filters"] + filters_schema = openai_format["function"]["parameters"]["properties"][ + "filters" + ] assert "additionalProperties" in filters_schema assert "anyOf" in filters_schema["additionalProperties"] assert len(filters_schema["additionalProperties"]["anyOf"]) == 2 @@ -678,10 +713,18 @@ def test_json_schema_validation_keywords_preserved(self) -> None: assert metrics_param.json_schema_extra == {"minItems": 1, "maxItems": 12} limit_param = tool.parameters["limit"] - assert limit_param.json_schema_extra == {"minimum": 1, "maximum": 1000, "default": 100} + assert limit_param.json_schema_extra == { + "minimum": 1, + "maximum": 1000, + "default": 100, + } name_param = tool.parameters["name_pattern"] - assert name_param.json_schema_extra == {"pattern": "^[a-z]+$", "minLength": 1, "maxLength": 255} + assert name_param.json_schema_extra == { + "pattern": "^[a-z]+$", + "minLength": 1, + "maxLength": 255, + } # Verify they flow through to OpenAI format openai_format = tool.get_openai_format() diff --git a/tests/test_openai_formatting.py b/tests/test_openai_formatting.py index 36027ff263..0d72c6b694 100644 --- a/tests/test_openai_formatting.py +++ b/tests/test_openai_formatting.py @@ -1,8 +1,10 @@ import pytest -from holmes.core.openai_formatting import type_to_open_ai_schema, format_tool_to_open_ai_standard +from holmes.core.openai_formatting import ( + format_tool_to_open_ai_standard, + type_to_open_ai_schema, +) from holmes.core.tools import ToolParameter - from tests.mocks.toolset_mocks import DummyTool @@ -118,7 +120,9 @@ def test_format_tool_no_strict_for_dynamic_keys(monkeypatch): def test_format_tool_disabled_via_env(monkeypatch): - monkeypatch.setattr("holmes.core.openai_formatting.STRICT_TOOL_CALLS_ENABLED", False) + monkeypatch.setattr( + "holmes.core.openai_formatting.STRICT_TOOL_CALLS_ENABLED", False + ) params = { "query": ToolParameter(type="string", required=True, description="The query"), } @@ -134,7 +138,9 @@ def _make_tool(self, parameters: dict) -> DummyTool: return DummyTool(parameters=parameters) def test_coerce_stringified_array(self): - tool = self._make_tool({"metrics": ToolParameter(type="array", items=ToolParameter(type="string"))}) + tool = self._make_tool( + {"metrics": ToolParameter(type="array", items=ToolParameter(type="string"))} + ) result = tool._coerce_params({"metrics": '["cpu", "memory"]'}) assert result["metrics"] == ["cpu", "memory"] diff --git a/tests/test_server_endpoints.py b/tests/test_server_endpoints.py index 646f8281a9..bf8cfea558 100644 --- a/tests/test_server_endpoints.py +++ b/tests/test_server_endpoints.py @@ -392,7 +392,9 @@ def test_preserves_header_case(self): def test_custom_blocked_headers_via_env(self, monkeypatch): """Test that HOLMES_PASSTHROUGH_BLOCKED_HEADERS env var works""" # Set custom blocked headers via environment variable - monkeypatch.setenv("HOLMES_PASSTHROUGH_BLOCKED_HEADERS", "x-internal-token,x-secret") + monkeypatch.setenv( + "HOLMES_PASSTHROUGH_BLOCKED_HEADERS", "x-internal-token,x-secret" + ) scope = { "type": "http", diff --git a/tests/test_tool_calling_llm.py b/tests/test_tool_calling_llm.py index bbc56440ce..530376ee82 100644 --- a/tests/test_tool_calling_llm.py +++ b/tests/test_tool_calling_llm.py @@ -15,14 +15,14 @@ import json import threading -from typing import Any, Dict, List, Optional +from typing import List from unittest.mock import MagicMock, patch import pytest from holmes.core.llm import LLM, ContextWindowUsage -from holmes.core.models import PendingToolApproval, ToolApprovalDecision, ToolCallResult from holmes.core.llm_usage import RequestStats +from holmes.core.models import PendingToolApproval, ToolApprovalDecision, ToolCallResult from holmes.core.tool_calling_llm import LLMInterruptedError, ToolCallingLLM from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus from holmes.core.tools_utils.tool_executor import ToolExecutor @@ -81,7 +81,9 @@ def _make_mock_tool_call(tool_call_id="tc_1", tool_name="kubectl_get", arguments return tc -def _make_llm_response(content="done", tool_calls=None, cost=0.001, prompt_tokens=50, completion_tokens=20): +def _make_llm_response( + content="done", tool_calls=None, cost=0.001, prompt_tokens=50, completion_tokens=20 +): """Create a mock LLM response matching litellm ModelResponse shape.""" resp = MagicMock() resp.choices = [MagicMock()] @@ -124,7 +126,9 @@ def _make_llm_response(content="done", tool_calls=None, cost=0.001, prompt_token return resp -def _make_tool_call_result(tool_call_id="tc_1", tool_name="kubectl_get", data="pod1 Running"): +def _make_tool_call_result( + tool_call_id="tc_1", tool_name="kubectl_get", data="pod1 Running" +): return ToolCallResult( tool_call_id=tool_call_id, tool_name=tool_name, @@ -137,7 +141,9 @@ def _make_tool_call_result(tool_call_id="tc_1", tool_name="kubectl_get", data="p ) -def _make_tool_call_result_error(tool_call_id="tc_1", tool_name="kubectl_get", error="command not found"): +def _make_tool_call_result_error( + tool_call_id="tc_1", tool_name="kubectl_get", error="command not found" +): return ToolCallResult( tool_call_id=tool_call_id, tool_name=tool_name, @@ -150,8 +156,9 @@ def _make_tool_call_result_error(tool_call_id="tc_1", tool_name="kubectl_get", e ) -def _make_tool_call_result_approval(tool_call_id="tc_1", tool_name="kubectl_delete", - invocation="kubectl delete pod foo"): +def _make_tool_call_result_approval( + tool_call_id="tc_1", tool_name="kubectl_delete", invocation="kubectl delete pod foo" +): return ToolCallResult( tool_call_id=tool_call_id, tool_name=tool_name, @@ -159,7 +166,10 @@ def _make_tool_call_result_approval(tool_call_id="tc_1", tool_name="kubectl_dele result=StructuredToolResult( status=StructuredToolResultStatus.APPROVAL_REQUIRED, invocation=invocation, - params={"command": "kubectl delete pod foo", "suggested_prefixes": ["kubectl delete"]}, + params={ + "command": "kubectl delete pod foo", + "suggested_prefixes": ["kubectl delete"], + }, ), ) @@ -190,6 +200,7 @@ def mock_tool_executor(): @pytest.fixture def make_ai(mock_llm, mock_tool_executor): """Factory that returns a ToolCallingLLM with default mocks.""" + def _make(max_steps=10): ai = ToolCallingLLM( tool_executor=mock_tool_executor, @@ -198,6 +209,7 @@ def _make(max_steps=10): tool_results_dir=None, ) return ai + return _make @@ -208,7 +220,9 @@ def _collect_stream_events(stream) -> List[StreamMessage]: return list(stream) -def _events_of_type(events: List[StreamMessage], event_type: StreamEvents) -> List[StreamMessage]: +def _events_of_type( + events: List[StreamMessage], event_type: StreamEvents +) -> List[StreamMessage]: return [e for e in events if e.event == event_type] @@ -331,7 +345,9 @@ def test_approval_approved(self, _mock_limit, make_ai, mock_llm): # Final result includes the approved tool, deduplicated assert result.result == "Pod deleted" tool_call_ids = [tc.tool_call_id for tc in result.tool_calls] - assert len(tool_call_ids) == len(set(tool_call_ids)), "Duplicate tool_call_id in result" + assert len(tool_call_ids) == len( + set(tool_call_ids) + ), "Duplicate tool_call_id in result" assert "tc_del" in tool_call_ids assert len(result.tool_calls) == 1 @@ -344,7 +360,9 @@ def test_approval_denied_with_feedback(self, _mock_limit, make_ai, mock_llm): # Round 2: _execute_tool_decisions adds denial error, LLM gives final answer mock_llm.completion.side_effect = [resp_with_tool, resp_final] - callback = MagicMock(return_value=(False, "try using namespace kube-system instead")) + callback = MagicMock( + return_value=(False, "try using namespace kube-system instead") + ) ai = make_ai() approval_result = _make_tool_call_result_approval( @@ -382,9 +400,27 @@ def test_costs_summed_across_iterations(self, _mock_limit, make_ai, mock_llm): tc1 = _make_mock_tool_call(tool_call_id="tc_1") tc2 = _make_mock_tool_call(tool_call_id="tc_2") - resp1 = _make_llm_response(content="step 1", tool_calls=[tc1], cost=0.01, prompt_tokens=100, completion_tokens=50) - resp2 = _make_llm_response(content="step 2", tool_calls=[tc2], cost=0.02, prompt_tokens=200, completion_tokens=80) - resp3 = _make_llm_response(content="final answer", tool_calls=None, cost=0.03, prompt_tokens=300, completion_tokens=100) + resp1 = _make_llm_response( + content="step 1", + tool_calls=[tc1], + cost=0.01, + prompt_tokens=100, + completion_tokens=50, + ) + resp2 = _make_llm_response( + content="step 2", + tool_calls=[tc2], + cost=0.02, + prompt_tokens=200, + completion_tokens=80, + ) + resp3 = _make_llm_response( + content="final answer", + tool_calls=None, + cost=0.03, + prompt_tokens=300, + completion_tokens=100, + ) mock_llm.completion.side_effect = [resp1, resp2, resp3] ai = make_ai() @@ -487,7 +523,9 @@ class TestToolError: def test_call_continues_after_tool_error(self, _mock_limit, make_ai, mock_llm): tc = _make_mock_tool_call() resp_with_tool = _make_llm_response(content="checking", tool_calls=[tc]) - resp_final = _make_llm_response(content="The command failed, here is why...", tool_calls=None) + resp_final = _make_llm_response( + content="The command failed, here is why...", tool_calls=None + ) mock_llm.completion.side_effect = [resp_with_tool, resp_final] ai = make_ai() @@ -591,7 +629,10 @@ def test_call_passes_response_format(self, _mock_limit, make_ai, mock_llm): # Verify response_format was passed through call_kwargs = mock_llm.completion.call_args - assert call_kwargs.kwargs.get("response_format") == fmt or call_kwargs[1].get("response_format") == fmt + assert ( + call_kwargs.kwargs.get("response_format") == fmt + or call_kwargs[1].get("response_format") == fmt + ) @patch(LIMIT_PATCH, side_effect=_make_context_limiter_passthrough) def test_call_stream_passes_response_format(self, _mock_limit, make_ai, mock_llm): @@ -601,11 +642,16 @@ def test_call_stream_passes_response_format(self, _mock_limit, make_ai, mock_llm ai = make_ai() fmt = {"type": "json_object"} events = _collect_stream_events( - ai.call_stream(msgs=[{"role": "user", "content": "give me json"}], response_format=fmt) + ai.call_stream( + msgs=[{"role": "user", "content": "give me json"}], response_format=fmt + ) ) call_kwargs = mock_llm.completion.call_args - assert call_kwargs.kwargs.get("response_format") == fmt or call_kwargs[1].get("response_format") == fmt + assert ( + call_kwargs.kwargs.get("response_format") == fmt + or call_kwargs[1].get("response_format") == fmt + ) # Should still get ANSWER_END answer_ends = _events_of_type(events, StreamEvents.ANSWER_END) @@ -664,19 +710,31 @@ class TestParallelToolExecution: @patch(LIMIT_PATCH, side_effect=_make_context_limiter_passthrough) def test_call_parallel_tools(self, _mock_limit, make_ai, mock_llm): - tc1 = _make_mock_tool_call(tool_call_id="tc_a", tool_name="kubectl_get", - arguments={"command": "kubectl get pods"}) - tc2 = _make_mock_tool_call(tool_call_id="tc_b", tool_name="kubectl_get", - arguments={"command": "kubectl get services"}) + tc1 = _make_mock_tool_call( + tool_call_id="tc_a", + tool_name="kubectl_get", + arguments={"command": "kubectl get pods"}, + ) + tc2 = _make_mock_tool_call( + tool_call_id="tc_b", + tool_name="kubectl_get", + arguments={"command": "kubectl get services"}, + ) - resp_with_tools = _make_llm_response(content="Checking both", tool_calls=[tc1, tc2]) - resp_final = _make_llm_response(content="Found 2 pods and 3 services", tool_calls=None) + resp_with_tools = _make_llm_response( + content="Checking both", tool_calls=[tc1, tc2] + ) + resp_final = _make_llm_response( + content="Found 2 pods and 3 services", tool_calls=None + ) mock_llm.completion.side_effect = [resp_with_tools, resp_final] ai = make_ai() ai._invoke_llm_tool_call = MagicMock( side_effect=[ - _make_tool_call_result(tool_call_id="tc_a", data="pod1 Running\npod2 Running"), + _make_tool_call_result( + tool_call_id="tc_a", data="pod1 Running\npod2 Running" + ), _make_tool_call_result(tool_call_id="tc_b", data="svc1\nsvc2\nsvc3"), ] ) @@ -746,14 +804,18 @@ def test_stream_approval_required_stops(self, _mock_limit, make_ai, mock_llm): assert len(approval_events) == 1 assert approval_events[0].data["requires_approval"] is True assert len(approval_events[0].data["pending_approvals"]) == 1 - assert approval_events[0].data["pending_approvals"][0]["tool_call_id"] == "tc_del" + assert ( + approval_events[0].data["pending_approvals"][0]["tool_call_id"] == "tc_del" + ) # Stream should NOT have ANSWER_END since it stopped for approval answer_ends = _events_of_type(events, StreamEvents.ANSWER_END) assert len(answer_ends) == 0 @patch(LIMIT_PATCH, side_effect=_make_context_limiter_passthrough) - def test_stream_approval_disabled_converts_to_error(self, _mock_limit, make_ai, mock_llm): + def test_stream_approval_disabled_converts_to_error( + self, _mock_limit, make_ai, mock_llm + ): """When enable_tool_approval=False (default), APPROVAL_REQUIRED becomes ERROR.""" tc = _make_mock_tool_call(tool_call_id="tc_del", tool_name="kubectl_delete") resp_with_tool = _make_llm_response(content="Deleting", tool_calls=[tc]) @@ -820,16 +882,29 @@ def test_call_accumulates_compaction_costs(self, make_ai, mock_llm): ) tc = _make_mock_tool_call() - resp1 = _make_llm_response(content="step", tool_calls=[tc], cost=0.01, - prompt_tokens=100, completion_tokens=50) - resp2 = _make_llm_response(content="done", tool_calls=None, cost=0.02, - prompt_tokens=200, completion_tokens=80) + resp1 = _make_llm_response( + content="step", + tool_calls=[tc], + cost=0.01, + prompt_tokens=100, + completion_tokens=50, + ) + resp2 = _make_llm_response( + content="done", + tool_calls=None, + cost=0.02, + prompt_tokens=200, + completion_tokens=80, + ) mock_llm.completion.side_effect = [resp1, resp2] ai = make_ai() ai._invoke_llm_tool_call = MagicMock(return_value=_make_tool_call_result()) - with patch(LIMIT_PATCH, side_effect=[limiter_output_with_compaction, limiter_output_normal]): + with patch( + LIMIT_PATCH, + side_effect=[limiter_output_with_compaction, limiter_output_normal], + ): result = ai.call([{"role": "user", "content": "analyze"}]) # Costs = compaction(0.005) + LLM1(0.01) + LLM2(0.02) = 0.035 @@ -849,8 +924,13 @@ class TestStreamCostsInMetadata: @patch(LIMIT_PATCH, side_effect=_make_context_limiter_passthrough) def test_stream_metadata_has_costs(self, _mock_limit, make_ai, mock_llm): - resp = _make_llm_response(content="answer", tool_calls=None, cost=0.01, - prompt_tokens=100, completion_tokens=50) + resp = _make_llm_response( + content="answer", + tool_calls=None, + cost=0.01, + prompt_tokens=100, + completion_tokens=50, + ) mock_llm.completion.return_value = resp ai = make_ai() @@ -865,7 +945,9 @@ def test_stream_metadata_has_costs(self, _mock_limit, make_ai, mock_llm): tc_data = token_counts[0].data assert "metadata" in tc_data assert "costs" in tc_data["metadata"] - assert tc_data["metadata"]["costs"]["total_cost"] == pytest.approx(0.01, abs=1e-6) + assert tc_data["metadata"]["costs"]["total_cost"] == pytest.approx( + 0.01, abs=1e-6 + ) # --------------------------------------------------------------------------- @@ -971,8 +1053,20 @@ def test_call_vs_stream_equivalence(self, _mock_limit, make_ai, mock_llm): def _make_responses(): """Generate fresh response mocks (each can only be consumed once).""" - resp_tool = _make_llm_response(content="checking", tool_calls=[tc], cost=0.01, prompt_tokens=100, completion_tokens=50) - resp_final = _make_llm_response(content="All good", tool_calls=None, cost=0.02, prompt_tokens=200, completion_tokens=80) + resp_tool = _make_llm_response( + content="checking", + tool_calls=[tc], + cost=0.01, + prompt_tokens=100, + completion_tokens=50, + ) + resp_final = _make_llm_response( + content="All good", + tool_calls=None, + cost=0.02, + prompt_tokens=200, + completion_tokens=80, + ) return [resp_tool, resp_final] tool_result = _make_tool_call_result() @@ -1006,7 +1100,9 @@ def _make_responses(): # Compare costs stream_costs = answer_end["costs"] - assert call_result.total_cost == pytest.approx(stream_costs["total_cost"], abs=1e-9) + assert call_result.total_cost == pytest.approx( + stream_costs["total_cost"], abs=1e-9 + ) assert call_result.prompt_tokens == stream_costs["prompt_tokens"] assert call_result.completion_tokens == stream_costs["completion_tokens"] @@ -1072,7 +1168,9 @@ def test_mixed_batch_approval(self, _mock_limit, make_ai, mock_llm): tc_ok = _make_mock_tool_call(tool_call_id="tc_ok", tool_name="kubectl_get") tc_del = _make_mock_tool_call(tool_call_id="tc_del", tool_name="kubectl_delete") - resp_with_tools = _make_llm_response(content="Running both", tool_calls=[tc_ok, tc_del]) + resp_with_tools = _make_llm_response( + content="Running both", tool_calls=[tc_ok, tc_del] + ) resp_final = _make_llm_response(content="Done", tool_calls=None) mock_llm.completion.side_effect = [resp_with_tools, resp_final] @@ -1103,12 +1201,17 @@ def _route_tool_call(tool_to_call, **kwargs): ai._invoke_llm_tool_call = MagicMock(side_effect=_route_tool_call) ai._is_tool_call_already_approved = MagicMock(return_value=False) - result = ai.call([{"role": "user", "content": "Get pods and delete one"}], approval_callback=callback) + result = ai.call( + [{"role": "user", "content": "Get pods and delete one"}], + approval_callback=callback, + ) assert result.result == "Done" # Both tools should appear exactly once (deduplicated) tool_call_ids = [tc.tool_call_id for tc in result.tool_calls] - assert len(tool_call_ids) == len(set(tool_call_ids)), "Duplicate tool_call_id in result" + assert len(tool_call_ids) == len( + set(tool_call_ids) + ), "Duplicate tool_call_id in result" assert set(tool_call_ids) == {"tc_ok", "tc_del"} tool_names = [tc.tool_name for tc in result.tool_calls] assert "kubectl_get" in tool_names @@ -1120,20 +1223,42 @@ def _route_tool_call(tool_to_call, **kwargs): # --------------------------------------------------------------------------- EXPECTED_COSTS_KEYS = { - "total_cost", "total_tokens", "prompt_tokens", "completion_tokens", - "cached_tokens", "reasoning_tokens", "max_completion_tokens_per_call", - "max_prompt_tokens_per_call", "num_compactions", + "total_cost", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "cached_tokens", + "reasoning_tokens", + "max_completion_tokens_per_call", + "max_prompt_tokens_per_call", + "num_compactions", } -EXPECTED_TOKEN_COUNT_METADATA_KEYS = {"costs", "usage", "tokens", "max_tokens", "max_output_tokens"} +EXPECTED_TOKEN_COUNT_METADATA_KEYS = { + "costs", + "usage", + "tokens", + "max_tokens", + "max_output_tokens", +} EXPECTED_ANSWER_END_KEYS = { - "content", "messages", "metadata", "tool_calls", "num_llm_calls", "prompt", "costs", + "content", + "messages", + "metadata", + "tool_calls", + "num_llm_calls", + "prompt", + "costs", } EXPECTED_APPROVAL_REQUIRED_KEYS = { - "content", "messages", "pending_approvals", - "requires_approval", "num_llm_calls", "costs", + "content", + "messages", + "pending_approvals", + "requires_approval", + "num_llm_calls", + "costs", } @@ -1171,9 +1296,9 @@ def test_token_count_event_shape(self, _mock_limit, make_ai, mock_llm): f"TOKEN_COUNT metadata missing keys: " f"{EXPECTED_TOKEN_COUNT_METADATA_KEYS - set(meta.keys())}" ) - assert set(meta["costs"].keys()) == EXPECTED_COSTS_KEYS, ( - f"costs keys mismatch: got {set(meta['costs'].keys())}" - ) + assert ( + set(meta["costs"].keys()) == EXPECTED_COSTS_KEYS + ), f"costs keys mismatch: got {set(meta['costs'].keys())}" @patch(LIMIT_PATCH, side_effect=_make_context_limiter_passthrough) def test_answer_end_event_shape(self, _mock_limit, make_ai, mock_llm): @@ -1196,9 +1321,9 @@ def test_answer_end_event_shape(self, _mock_limit, make_ai, mock_llm): assert len(answer_ends) == 1 data = answer_ends[0].data - assert set(data.keys()) == EXPECTED_ANSWER_END_KEYS, ( - f"ANSWER_END keys mismatch: got {set(data.keys())}" - ) + assert ( + set(data.keys()) == EXPECTED_ANSWER_END_KEYS + ), f"ANSWER_END keys mismatch: got {set(data.keys())}" assert set(data["costs"].keys()) == EXPECTED_COSTS_KEYS assert isinstance(data["messages"], list) assert isinstance(data["tool_calls"], list) @@ -1227,9 +1352,9 @@ def test_approval_required_event_shape(self, _mock_limit, make_ai, mock_llm): assert len(approval_events) == 1 data = approval_events[0].data - assert set(data.keys()) == EXPECTED_APPROVAL_REQUIRED_KEYS, ( - f"APPROVAL_REQUIRED keys mismatch: got {set(data.keys())}" - ) + assert ( + set(data.keys()) == EXPECTED_APPROVAL_REQUIRED_KEYS + ), f"APPROVAL_REQUIRED keys mismatch: got {set(data.keys())}" assert set(data["costs"].keys()) == EXPECTED_COSTS_KEYS assert data["requires_approval"] is True assert isinstance(data["pending_approvals"], list) diff --git a/tests/test_toolset_auto_enable.py b/tests/test_toolset_auto_enable.py index 395f703ddb..d51b722765 100644 --- a/tests/test_toolset_auto_enable.py +++ b/tests/test_toolset_auto_enable.py @@ -1,25 +1,25 @@ """Tests for the smart auto-enable logic for toolsets.""" -from typing import ClassVar, Dict, List, Optional, Type +from typing import List, Optional, Type -import pytest from pydantic import BaseModel, Field from holmes.core.tools import Toolset, ToolsetTag from holmes.utils.pydantic_utils import ToolsetConfig - # --- Test config classes --- class AllOptionalConfig(ToolsetConfig): """Config where every field has a default.""" + url: Optional[str] = Field(default=None) timeout: int = Field(default=30) class RequiredFieldConfig(ToolsetConfig): """Config with a required field (no default).""" + api_url: str = Field(title="API URL") api_key: Optional[str] = Field(default=None) diff --git a/tests/test_toolset_config_tui.py b/tests/test_toolset_config_tui.py index 4915c30f75..28477a869c 100644 --- a/tests/test_toolset_config_tui.py +++ b/tests/test_toolset_config_tui.py @@ -1,11 +1,9 @@ """Tests for holmes.toolset_config_tui module.""" -import os -import tempfile from enum import Enum from pathlib import Path from typing import Any, ClassVar, Dict, List, Optional, Type -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import pytest import yaml # type: ignore @@ -28,15 +26,13 @@ _resolve_primitive_type, _select_config_class, build_tree_from_schema, - run_toolset_config_tui, + run_config_test, save_config_to_file, select_toolset, - run_config_test, tree_to_dict, ) from holmes.utils.pydantic_utils import ToolsetConfig - # ── Fixtures ────────────────────────────────────────────────────────── @@ -66,9 +62,7 @@ class NestedConfig(ToolsetConfig): labels: NestedLabelsConfig = Field( default_factory=NestedLabelsConfig, title="Labels" ) - additional_headers: Optional[Dict[str, str]] = Field( - default=None, title="Headers" - ) + additional_headers: Optional[Dict[str, str]] = Field(default=None, title="Headers") tags: Optional[List[str]] = Field(default=None, title="Tags") @@ -76,7 +70,9 @@ class DummyTool(Tool): name: str = "dummy_tool" description: str = "A dummy tool" - def _invoke(self, _params: dict, _user_approved: bool = False) -> StructuredToolResult: + def _invoke( + self, _params: dict, _user_approved: bool = False + ) -> StructuredToolResult: return StructuredToolResult(status="success", data="ok") # type: ignore def get_parameterized_one_liner(self, _params: Dict) -> str: @@ -90,7 +86,9 @@ def make_toolset( ) -> Toolset: """Create a test toolset with optional config_classes.""" if config_classes: - cls = type(f"{name}_TestSubclass", (Toolset,), {"config_classes": config_classes}) + cls = type( + f"{name}_TestSubclass", (Toolset,), {"config_classes": config_classes} + ) else: cls = Toolset return cls( @@ -296,7 +294,9 @@ def test_order_preserved(self) -> None: nodes = build_tree_from_schema(NestedConfig, values) flat = _flatten_tree(nodes) # Headers should appear before their children in the flat list - headers_idx = next(i for i, n in enumerate(flat) if n.key == "additional_headers") + headers_idx = next( + i for i, n in enumerate(flat) if n.key == "additional_headers" + ) child_idx = next(i for i, n in enumerate(flat) if n.dict_key == "X-Custom") assert child_idx > headers_idx @@ -361,7 +361,9 @@ def test_empty_dict_preserved(self) -> None: assert headers_node.is_header is True result = tree_to_dict(nodes) # Empty headers should still produce an empty dict - assert result.get("additional_headers") == {} or "additional_headers" not in result + assert ( + result.get("additional_headers") == {} or "additional_headers" not in result + ) def test_empty_list_preserved(self) -> None: nodes = build_tree_from_schema(NestedConfig, {"api_url": "http://x"}) @@ -387,7 +389,9 @@ def test_save_to_new_file(self, tmp_path: Path) -> None: with open(config_file) as f: saved = yaml.safe_load(f) assert saved["toolsets"]["test/toolset"]["enabled"] is True - assert saved["toolsets"]["test/toolset"]["config"]["api_url"] == "http://test:9090" + assert ( + saved["toolsets"]["test/toolset"]["config"]["api_url"] == "http://test:9090" + ) def test_merge_into_existing(self, tmp_path: Path) -> None: config_file = tmp_path / "config.yaml" @@ -414,7 +418,9 @@ def test_merge_into_existing(self, tmp_path: Path) -> None: assert saved["toolsets"]["other/toolset"]["config"]["url"] == "http://other" # New toolset added assert saved["toolsets"]["new/toolset"]["enabled"] is True - assert saved["toolsets"]["new/toolset"]["config"]["api_url"] == "http://new:8080" + assert ( + saved["toolsets"]["new/toolset"]["config"]["api_url"] == "http://new:8080" + ) def test_replace_existing_toolset_config(self, tmp_path: Path) -> None: config_file = tmp_path / "config.yaml" @@ -470,13 +476,17 @@ def test_handles_none_mcp_servers_section(self, tmp_path: Path) -> None: with open(config_file, "w") as f: yaml.dump({"mcp_servers": None}, f) - ok, _ = save_config_to_file(config_file, "jira_server", {"mode": "stdio"}, is_mcp=True) + ok, _ = save_config_to_file( + config_file, "jira_server", {"mode": "stdio"}, is_mcp=True + ) assert ok is True with open(config_file) as f: saved = yaml.safe_load(f) assert saved["mcp_servers"]["jira_server"]["config"]["mode"] == "stdio" - def test_does_not_print_to_stdout(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + def test_does_not_print_to_stdout( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: config_file = tmp_path / "config.yaml" save_config_to_file(config_file, "test/toolset", {"key": "val"}) @@ -516,7 +526,9 @@ def test_does_not_mutate_original(self) -> None: # Original should be unchanged assert ts.status == ToolsetStatusEnum.DISABLED - def test_allows_stdout_outside_tui(self, capsys: pytest.CaptureFixture[str]) -> None: + def test_allows_stdout_outside_tui( + self, capsys: pytest.CaptureFixture[str] + ) -> None: """run_config_test is called outside the TUI so output flows freely.""" def noisy_check(config: dict) -> tuple: @@ -569,7 +581,9 @@ def test_returns_existing_config_when_present(self) -> None: ts = ConfigurableToolset() config = MagicMock() config.toolsets = { - "test/configurable": {"config": {"api_url": "http://saved:9090", "timeout": 60}} + "test/configurable": { + "config": {"api_url": "http://saved:9090", "timeout": 60} + } } result = _get_existing_config(ts, config) @@ -607,9 +621,7 @@ def test_remove_dict_entry(self) -> None: assert node.children[0].key == "B" def test_add_list_entry(self) -> None: - node = ConfigFieldNode( - key="tags", field_type="list", is_header=True, depth=0 - ) + node = ConfigFieldNode(key="tags", field_type="list", is_header=True, depth=0) child = ConfigFieldNode( key="0", field_type="str", value="new_tag", depth=1, parent=node ) @@ -618,9 +630,7 @@ def test_add_list_entry(self) -> None: assert result["tags"] == ["new_tag"] def test_remove_list_entry(self) -> None: - node = ConfigFieldNode( - key="tags", field_type="list", is_header=True, depth=0 - ) + node = ConfigFieldNode(key="tags", field_type="list", is_header=True, depth=0) c0 = ConfigFieldNode(key="0", field_type="str", value="a", depth=1, parent=node) c1 = ConfigFieldNode(key="1", field_type="str", value="b", depth=1, parent=node) node.children.extend([c0, c1]) @@ -654,7 +664,11 @@ def test_optional_field_null_not_explicitly_set_omitted(self) -> None: def test_optional_field_null_explicitly_set_saved(self) -> None: """When an optional field is explicitly set to None, tree_to_dict includes it.""" node = ConfigFieldNode( - key="api_key", field_type="str", value=None, required=False, depth=0, + key="api_key", + field_type="str", + value=None, + required=False, + depth=0, explicitly_set=True, ) result = tree_to_dict([node]) @@ -664,7 +678,11 @@ def test_optional_field_null_explicitly_set_saved(self) -> None: def test_empty_string_preserved_in_tree_to_dict(self) -> None: """Empty string is saved as empty string, not as None.""" node = ConfigFieldNode( - key="api_key", field_type="str", value="", required=False, depth=0, + key="api_key", + field_type="str", + value="", + required=False, + depth=0, explicitly_set=True, ) result = tree_to_dict([node]) @@ -775,7 +793,7 @@ def test_mcp_config_mode_is_first_field(self) -> None: assert stdio_nodes[0].key == "mode" def test_stdio_mcp_config_tree(self) -> None: - from holmes.plugins.toolsets.mcp.toolset_mcp import StdioMCPConfig, MCPMode + from holmes.plugins.toolsets.mcp.toolset_mcp import StdioMCPConfig values = {"mode": "stdio", "command": "uvx", "args": ["mcp-atlassian"]} nodes = build_tree_from_schema(StdioMCPConfig, values) @@ -882,15 +900,11 @@ def test_single_class(self) -> None: assert result is EnumConfig def test_picks_correct_class_by_discriminator(self) -> None: - result = _select_config_class( - [EnumConfig, AltEnumConfig], {"mode": "beta"} - ) + result = _select_config_class([EnumConfig, AltEnumConfig], {"mode": "beta"}) assert result is AltEnumConfig def test_falls_back_to_first_class(self) -> None: - result = _select_config_class( - [EnumConfig, AltEnumConfig], {"mode": "gamma"} - ) + result = _select_config_class([EnumConfig, AltEnumConfig], {"mode": "gamma"}) assert result is EnumConfig def test_no_value_returns_first(self) -> None: @@ -906,17 +920,26 @@ def test_field_matching_when_no_discriminator_value(self) -> None: assert result is AltEnumConfig def test_field_matching_prefers_higher_overlap(self) -> None: - result = _select_config_class( - [EnumConfig, AltEnumConfig], {"name": "test"} - ) + result = _select_config_class([EnumConfig, AltEnumConfig], {"name": "test"}) assert result is EnumConfig def test_mcp_config_classes(self) -> None: - from holmes.plugins.toolsets.mcp.toolset_mcp import MCPConfig, MCPMode, StdioMCPConfig + from holmes.plugins.toolsets.mcp.toolset_mcp import MCPConfig, StdioMCPConfig - assert _select_config_class([MCPConfig, StdioMCPConfig], {"mode": "stdio"}) is StdioMCPConfig - assert _select_config_class([MCPConfig, StdioMCPConfig], {"mode": "sse"}) is MCPConfig - assert _select_config_class([MCPConfig, StdioMCPConfig], {"mode": "streamable-http"}) is MCPConfig + assert ( + _select_config_class([MCPConfig, StdioMCPConfig], {"mode": "stdio"}) + is StdioMCPConfig + ) + assert ( + _select_config_class([MCPConfig, StdioMCPConfig], {"mode": "sse"}) + is MCPConfig + ) + assert ( + _select_config_class( + [MCPConfig, StdioMCPConfig], {"mode": "streamable-http"} + ) + is MCPConfig + ) # ── MCP save/load ──────────────────────────────────────────────────── @@ -927,7 +950,9 @@ def test_save_mcp_to_mcp_servers_section(self, tmp_path: Path) -> None: config_file = tmp_path / "config.yaml" config_dict = {"mode": "stdio", "command": "uvx", "args": ["mcp-atlassian"]} - ok, msg = save_config_to_file(config_file, "jira_server", config_dict, is_mcp=True) + ok, msg = save_config_to_file( + config_file, "jira_server", config_dict, is_mcp=True + ) assert ok is True with open(config_file) as f: @@ -963,7 +988,9 @@ def test_save_mcp_preserves_existing_fields(self, tmp_path: Path) -> None: def test_regular_save_still_uses_toolsets(self, tmp_path: Path) -> None: config_file = tmp_path / "config.yaml" - ok, _ = save_config_to_file(config_file, "grafana/dashboards", {"api_url": "http://x"}) + ok, _ = save_config_to_file( + config_file, "grafana/dashboards", {"api_url": "http://x"} + ) assert ok is True with open(config_file) as f: saved = yaml.safe_load(f) diff --git a/tests/utils/test_pydantic_utils.py b/tests/utils/test_pydantic_utils.py index 433ebb2928..a1e41fb5b7 100644 --- a/tests/utils/test_pydantic_utils.py +++ b/tests/utils/test_pydantic_utils.py @@ -35,4 +35,3 @@ def test_build_config_example_uses_default_default_factory_examples_nested_and_p # Nested recursion assert example["nested"] == {"region": "us-east-1", "enabled": True} -