From bdabdf0aa31bebf429aec0be5b1ae423555488bd Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 23 Dec 2025 09:46:50 +0200 Subject: [PATCH 01/12] Add toolset to trigger intentional OOM kill Signed-off-by: Codex --- holmes/plugins/toolsets/__init__.py | 2 + holmes/plugins/toolsets/oom_kill.py | 111 ++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 holmes/plugins/toolsets/oom_kill.py diff --git a/holmes/plugins/toolsets/__init__.py b/holmes/plugins/toolsets/__init__.py index 2368859c3c..44b9b3fdb9 100644 --- a/holmes/plugins/toolsets/__init__.py +++ b/holmes/plugins/toolsets/__init__.py @@ -35,6 +35,7 @@ from holmes.plugins.toolsets.internet.notion import NotionToolset from holmes.plugins.toolsets.kafka import KafkaToolset from holmes.plugins.toolsets.kubernetes_logs import KubernetesLogsToolset +from holmes.plugins.toolsets.oom_kill import OOMKillToolset from holmes.plugins.toolsets.mcp.toolset_mcp import RemoteMCPToolset from holmes.plugins.toolsets.newrelic.newrelic import NewRelicToolset from holmes.plugins.toolsets.opensearch.opensearch import OpenSearchToolset @@ -96,6 +97,7 @@ def load_python_toolsets( OpenSearchLogsToolset(), OpenSearchTracesToolset(), OpenSearchQueryAssistToolset(), + OOMKillToolset(), CoralogixToolset(), RabbitMQToolset(), GitToolset(), diff --git a/holmes/plugins/toolsets/oom_kill.py b/holmes/plugins/toolsets/oom_kill.py new file mode 100644 index 0000000000..15dea2aa58 --- /dev/null +++ b/holmes/plugins/toolsets/oom_kill.py @@ -0,0 +1,111 @@ +import textwrap +from typing import Any, Dict + +from holmes.core.tools import ( + CallablePrerequisite, + StructuredToolResult, + StructuredToolResultStatus, + Tool, + ToolInvokeContext, + ToolParameter, + Toolset, + ToolsetTag, +) +from holmes.plugins.toolsets.bash.common.bash import execute_bash_command + + +class TriggerOOMKill(Tool): + def __init__(self, toolset: "OOMKillToolset"): + super().__init__( + name="trigger_oom_kill", + description=( + "Allocates approximately 30GB of memory on the Holmes host to provoke the " + "OOM killer. This is intended for stress testing only and will likely crash " + "the running process." + ), + parameters={ + "confirm": ToolParameter( + description=( + "Set to true to acknowledge this will allocate ~30GB of memory and " + "may kill Holmes on the host." + ), + type="boolean", + required=True, + ), + "hold_seconds": ToolParameter( + description=( + "How long to keep the memory allocated before exiting. Defaults to 300 seconds." + ), + type="integer", + required=False, + ), + }, + toolset=toolset, + ) + + def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: + if params.get("confirm") is not True: + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error=( + "Confirmation required: set 'confirm' to true to run the intentional OOM trigger." + ), + params=params, + ) + + hold_seconds = params.get("hold_seconds", 300) + if not isinstance(hold_seconds, int) or hold_seconds <= 0: + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error="hold_seconds must be a positive integer.", + params=params, + ) + + command = textwrap.dedent( + f""" + python - <<'PY' + import time + + size_bytes = 30 * 1024 * 1024 * 1024 + print(f"Allocating {{size_bytes / 1024 / 1024 / 1024:.0f}} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s") + data = bytearray(size_bytes) + time.sleep({hold_seconds}) + PY + """ + ).strip() + + timeout = hold_seconds + 30 + return execute_bash_command(cmd=command, timeout=timeout, params=params) + + def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str: + hold_seconds = params.get("hold_seconds", 300) + return ( + "python - <<'PY' ... # allocates ~30GB and sleeps for " + f"{hold_seconds}s to trigger OOM" + ) + + +class OOMKillToolset(Toolset): + def __init__(self): + super().__init__( + name="oom_kill", + enabled=False, + description=( + "Dangerous toolset that intentionally exhausts memory on the Holmes host to trigger an OOM kill. " + "Use only in controlled stress tests." + ), + docs_url=None, + icon_url=None, + prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)], + tools=[TriggerOOMKill(self)], + experimental=True, + tags=[ToolsetTag.CORE], + is_default=False, + ) + + def prerequisites_callable(self, config: dict[str, Any]) -> tuple[bool, str]: + # No special configuration is required for this toolset. + return True, "" + + def get_example_config(self) -> Dict[str, Any]: + return {} From 199ee2664f467383bb635732f4422b9ee907515b Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 23 Dec 2025 09:56:55 +0200 Subject: [PATCH 02/12] Add bash YAML toolset for intentional OOM kill Signed-off-by: Codex --- holmes/plugins/toolsets/oom_kill.yaml | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 holmes/plugins/toolsets/oom_kill.yaml diff --git a/holmes/plugins/toolsets/oom_kill.yaml b/holmes/plugins/toolsets/oom_kill.yaml new file mode 100644 index 0000000000..4bac386d82 --- /dev/null +++ b/holmes/plugins/toolsets/oom_kill.yaml @@ -0,0 +1,34 @@ +toolsets: + oom_kill/bash: + description: "DANGEROUS: intentionally allocate ~30GB on the Holmes host to trigger OOM kill for stress testing." + docs_url: "" + icon_url: "" + tags: + - cli + additional_instructions: | + ⚠️ This toolset is intentionally destructive. Only use in controlled environments. + tools: + - name: "trigger_oom_kill_bash" + description: "Allocate ~30GB of memory and hold it for a period to provoke the OOM killer." + command: | + python - <<'PY' + import sys + import time + + confirm = "{{ confirm }}" + if str(confirm).lower() not in ("true", "yes", "1"): + sys.exit("Confirmation required: set confirm=true to run the OOM trigger.") + + try: + hold_seconds = int("{{ hold_seconds|default(300) }}") + except Exception: + sys.exit("hold_seconds must be an integer.") + + if hold_seconds <= 0: + sys.exit("hold_seconds must be positive.") + + size_bytes = 30 * 1024 * 1024 * 1024 + print(f"Allocating {size_bytes / 1024 / 1024 / 1024:.0f} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s") + data = bytearray(size_bytes) + time.sleep(hold_seconds) + PY From 0058135e88b5a4f6e18257bf2a6f31a7c6c70897 Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 23 Dec 2025 10:12:25 +0200 Subject: [PATCH 03/12] Remove confirmation requirement for OOM kill toolsets Signed-off-by: Codex --- holmes/plugins/toolsets/oom_kill.py | 20 ++------------------ holmes/plugins/toolsets/oom_kill.yaml | 17 ++++------------- 2 files changed, 6 insertions(+), 31 deletions(-) diff --git a/holmes/plugins/toolsets/oom_kill.py b/holmes/plugins/toolsets/oom_kill.py index 15dea2aa58..1c4ebeb257 100644 --- a/holmes/plugins/toolsets/oom_kill.py +++ b/holmes/plugins/toolsets/oom_kill.py @@ -21,17 +21,10 @@ def __init__(self, toolset: "OOMKillToolset"): description=( "Allocates approximately 30GB of memory on the Holmes host to provoke the " "OOM killer. This is intended for stress testing only and will likely crash " - "the running process." + "the running process. No confirmation is required because this is meant for " + "automated stress scenarios." ), parameters={ - "confirm": ToolParameter( - description=( - "Set to true to acknowledge this will allocate ~30GB of memory and " - "may kill Holmes on the host." - ), - type="boolean", - required=True, - ), "hold_seconds": ToolParameter( description=( "How long to keep the memory allocated before exiting. Defaults to 300 seconds." @@ -44,15 +37,6 @@ def __init__(self, toolset: "OOMKillToolset"): ) def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: - if params.get("confirm") is not True: - return StructuredToolResult( - status=StructuredToolResultStatus.ERROR, - error=( - "Confirmation required: set 'confirm' to true to run the intentional OOM trigger." - ), - params=params, - ) - hold_seconds = params.get("hold_seconds", 300) if not isinstance(hold_seconds, int) or hold_seconds <= 0: return StructuredToolResult( diff --git a/holmes/plugins/toolsets/oom_kill.yaml b/holmes/plugins/toolsets/oom_kill.yaml index 4bac386d82..0a5f05f1d1 100644 --- a/holmes/plugins/toolsets/oom_kill.yaml +++ b/holmes/plugins/toolsets/oom_kill.yaml @@ -1,6 +1,6 @@ toolsets: oom_kill/bash: - description: "DANGEROUS: intentionally allocate ~30GB on the Holmes host to trigger OOM kill for stress testing." + description: "DANGEROUS: intentionally allocate ~30GB on the Holmes host to trigger OOM kill for stress testing (no confirmation required)." docs_url: "" icon_url: "" tags: @@ -9,23 +9,14 @@ toolsets: ⚠️ This toolset is intentionally destructive. Only use in controlled environments. tools: - name: "trigger_oom_kill_bash" - description: "Allocate ~30GB of memory and hold it for a period to provoke the OOM killer." + description: "Allocate ~30GB of memory and hold it for a period to provoke the OOM killer. No confirmation required; intended for automated stress tests." command: | python - <<'PY' - import sys import time - confirm = "{{ confirm }}" - if str(confirm).lower() not in ("true", "yes", "1"): - sys.exit("Confirmation required: set confirm=true to run the OOM trigger.") - - try: - hold_seconds = int("{{ hold_seconds|default(300) }}") - except Exception: - sys.exit("hold_seconds must be an integer.") - + hold_seconds = int("{{ hold_seconds|default(300) }}") if hold_seconds <= 0: - sys.exit("hold_seconds must be positive.") + raise SystemExit("hold_seconds must be positive.") size_bytes = 30 * 1024 * 1024 * 1024 print(f"Allocating {size_bytes / 1024 / 1024 / 1024:.0f} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s") From 15f45c9193f41fbefa3ea9fef789bdce1b5dff4e Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 23 Dec 2025 13:15:24 +0200 Subject: [PATCH 04/12] Gate OOM kill toolsets behind env guard and add stress test docs Signed-off-by: Codex --- docs/development/stress-testing-oom-kill.md | 80 +++++++++++++++++++++ holmes/plugins/toolsets/oom_kill.py | 6 +- holmes/plugins/toolsets/oom_kill.yaml | 3 + 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 docs/development/stress-testing-oom-kill.md diff --git a/docs/development/stress-testing-oom-kill.md b/docs/development/stress-testing-oom-kill.md new file mode 100644 index 0000000000..0d5804e5fc --- /dev/null +++ b/docs/development/stress-testing-oom-kill.md @@ -0,0 +1,80 @@ +# Stress Testing Holmes with Intentional OOM Kills + +> ⚠️ **Never enable this toolset in production.** It allocates ~30 GB of RAM on the Holmes host and is intended only for controlled, non-production stress tests. + +This guide explains how to intentionally trigger OOM kills using the built-in OOM toolsets, how to enable them safely, and how to confirm they are available. + +## Available Toolsets + +Holmes ships with two disabled-by-default toolsets for inducing an OOM kill: + +- **`oom_kill` (Python)** – `trigger_oom_kill` allocates ~30 GB and sleeps for a configurable duration. +- **`oom_kill/bash` (YAML/bash)** – `trigger_oom_kill_bash` does the same via a bash-executed Python snippet. + +Both toolsets require the environment variable `ALLOW_HOLMES_OOMKILL_TOOLSET` to pass prerequisites and must be explicitly enabled in configuration. + +## Enabling via Helm/ArgoCD (cluster install) + +1. **Set the env guard** (required): + ```bash + # Example: add to your values or patch deployment env + argocd app set \ + --helm-set-string additionalEnvVars[0].name=ALLOW_HOLMES_OOMKILL_TOOLSET \ + --helm-set-string additionalEnvVars[0].value=true + ``` + +2. **Enable the toolsets**: + ```bash + # Note the escaped slash for the bash toolset name + argocd app set \ + --helm-set-string toolsets.oom_kill.enabled=true \ + --helm-set-string toolsets.oom_kill\\/bash.enabled=true + ``` + +3. **Sync to apply**: + ```bash + argocd app sync + ``` + +4. **Verify** (optional): + ```bash + kubectl -n exec -it -- \ + cat /app/custom_toolset.yaml + # Expect oom_kill and oom_kill/bash present and enabled + ``` + +## Enabling in Local CLI Mode + +Add to your local config (e.g., `config.yaml`) and set the env guard before running the CLI: + +```yaml +toolsets: + oom_kill: + enabled: true + oom_kill/bash: + enabled: true +``` + +Then run: +```bash +export ALLOW_HOLMES_OOMKILL_TOOLSET=true +holmes --config ./config.yaml ... +``` + +Because both toolsets are disabled by default and gated by `ALLOW_HOLMES_OOMKILL_TOOLSET`, they will **not** be auto-enabled in local mode unless you explicitly enable them and set the env variable. + +## Using the Tools + +- **Python toolset**: `trigger_oom_kill` (param: `hold_seconds`, default 300). +- **Bash toolset**: `trigger_oom_kill_bash` (param: `hold_seconds`, default 300). + +Example invocation (conceptual): +``` +trigger_oom_kill: allocate ~30GB and sleep for 120s +``` + +## Safety Considerations + +- Keep this toolset out of production environments. +- Ensure hosts have proper isolation; the process is expected to be OOM-killed. +- Consider running in a dedicated test cluster or namespace. diff --git a/holmes/plugins/toolsets/oom_kill.py b/holmes/plugins/toolsets/oom_kill.py index 1c4ebeb257..ccaef02c6a 100644 --- a/holmes/plugins/toolsets/oom_kill.py +++ b/holmes/plugins/toolsets/oom_kill.py @@ -10,6 +10,7 @@ ToolParameter, Toolset, ToolsetTag, + ToolsetEnvironmentPrerequisite, ) from holmes.plugins.toolsets.bash.common.bash import execute_bash_command @@ -80,7 +81,10 @@ def __init__(self): ), docs_url=None, icon_url=None, - prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)], + prerequisites=[ + ToolsetEnvironmentPrerequisite(env=["ALLOW_HOLMES_OOMKILL_TOOLSET"]), + CallablePrerequisite(callable=self.prerequisites_callable), + ], tools=[TriggerOOMKill(self)], experimental=True, tags=[ToolsetTag.CORE], diff --git a/holmes/plugins/toolsets/oom_kill.yaml b/holmes/plugins/toolsets/oom_kill.yaml index 0a5f05f1d1..2e66ae5628 100644 --- a/holmes/plugins/toolsets/oom_kill.yaml +++ b/holmes/plugins/toolsets/oom_kill.yaml @@ -5,6 +5,9 @@ toolsets: icon_url: "" tags: - cli + prerequisites: + - env: + - ALLOW_HOLMES_OOMKILL_TOOLSET additional_instructions: | ⚠️ This toolset is intentionally destructive. Only use in controlled environments. tools: From 735d8619123a98ceb4a501c39ba3d9f852b96665 Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Tue, 23 Dec 2025 13:15:31 +0200 Subject: [PATCH 05/12] Rename bash OOM toolset and add ulimit guard Signed-off-by: Codex --- docs/development/stress-testing-oom-kill.md | 11 +++++------ holmes/plugins/toolsets/oom_kill.yaml | 3 ++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/development/stress-testing-oom-kill.md b/docs/development/stress-testing-oom-kill.md index 0d5804e5fc..a542ceabc8 100644 --- a/docs/development/stress-testing-oom-kill.md +++ b/docs/development/stress-testing-oom-kill.md @@ -9,9 +9,9 @@ This guide explains how to intentionally trigger OOM kills using the built-in OO Holmes ships with two disabled-by-default toolsets for inducing an OOM kill: - **`oom_kill` (Python)** – `trigger_oom_kill` allocates ~30 GB and sleeps for a configurable duration. -- **`oom_kill/bash` (YAML/bash)** – `trigger_oom_kill_bash` does the same via a bash-executed Python snippet. +- **`oom_kill_bash` (YAML/bash)** – `trigger_oom_kill_bash` does the same via a bash-executed Python snippet and applies `ulimit -v 2097152` (2 GiB virtual memory cap) before allocation to reduce blast radius. -Both toolsets require the environment variable `ALLOW_HOLMES_OOMKILL_TOOLSET` to pass prerequisites and must be explicitly enabled in configuration. +Both toolsets require the environment variable `ALLOW_HOLMES_OOMKILL_TOOLSET` to pass prerequisites and must be explicitly enabled in configuration. They are **disabled by default** and will not be loaded unless you opt in. ## Enabling via Helm/ArgoCD (cluster install) @@ -23,12 +23,11 @@ Both toolsets require the environment variable `ALLOW_HOLMES_OOMKILL_TOOLSET` to --helm-set-string additionalEnvVars[0].value=true ``` -2. **Enable the toolsets**: +2. **Enable the toolsets** (note the underscore name for the bash variant): ```bash - # Note the escaped slash for the bash toolset name argocd app set \ --helm-set-string toolsets.oom_kill.enabled=true \ - --helm-set-string toolsets.oom_kill\\/bash.enabled=true + --helm-set-string toolsets.oom_kill_bash.enabled=true ``` 3. **Sync to apply**: @@ -51,7 +50,7 @@ Add to your local config (e.g., `config.yaml`) and set the env guard before runn toolsets: oom_kill: enabled: true - oom_kill/bash: + oom_kill_bash: enabled: true ``` diff --git a/holmes/plugins/toolsets/oom_kill.yaml b/holmes/plugins/toolsets/oom_kill.yaml index 2e66ae5628..b3351684c4 100644 --- a/holmes/plugins/toolsets/oom_kill.yaml +++ b/holmes/plugins/toolsets/oom_kill.yaml @@ -1,5 +1,5 @@ toolsets: - oom_kill/bash: + oom_kill_bash: description: "DANGEROUS: intentionally allocate ~30GB on the Holmes host to trigger OOM kill for stress testing (no confirmation required)." docs_url: "" icon_url: "" @@ -14,6 +14,7 @@ toolsets: - name: "trigger_oom_kill_bash" description: "Allocate ~30GB of memory and hold it for a period to provoke the OOM killer. No confirmation required; intended for automated stress tests." command: | + ulimit -v 2097152 || true python - <<'PY' import time From 465745d207941631a75e976aa079ba27d9417ea0 Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Wed, 24 Dec 2025 10:45:48 +0200 Subject: [PATCH 06/12] Clarify Helm values paths for OOM toolsets in docs Signed-off-by: Codex --- docs/development/stress-testing-oom-kill.md | 37 +++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/development/stress-testing-oom-kill.md b/docs/development/stress-testing-oom-kill.md index a542ceabc8..1f86487df6 100644 --- a/docs/development/stress-testing-oom-kill.md +++ b/docs/development/stress-testing-oom-kill.md @@ -15,15 +15,46 @@ Both toolsets require the environment variable `ALLOW_HOLMES_OOMKILL_TOOLSET` to ## Enabling via Helm/ArgoCD (cluster install) +Use the correct values path for your deployment method. + +=== "Robusta Helm Chart (Holmes as subchart)" + +1. **Set the env guard** (required): + ```bash + argocd app set \ + --helm-set-string holmes.additionalEnvVars[0].name=ALLOW_HOLMES_OOMKILL_TOOLSET \ + --helm-set-string holmes.additionalEnvVars[0].value=true + ``` + +2. **Enable the toolsets**: + ```bash + argocd app set \ + --helm-set-string holmes.toolsets.oom_kill.enabled=true \ + --helm-set-string holmes.toolsets.oom_kill_bash.enabled=true + ``` + +3. **Sync to apply**: + ```bash + argocd app sync + ``` + +4. **Verify** (optional): + ```bash + kubectl -n exec -it -- \ + cat /app/custom_toolset.yaml + # Expect oom_kill and oom_kill_bash present and enabled + ``` + +=== "Holmes Helm Chart (direct)" + 1. **Set the env guard** (required): ```bash - # Example: add to your values or patch deployment env argocd app set \ --helm-set-string additionalEnvVars[0].name=ALLOW_HOLMES_OOMKILL_TOOLSET \ --helm-set-string additionalEnvVars[0].value=true ``` -2. **Enable the toolsets** (note the underscore name for the bash variant): +2. **Enable the toolsets**: ```bash argocd app set \ --helm-set-string toolsets.oom_kill.enabled=true \ @@ -39,7 +70,7 @@ Both toolsets require the environment variable `ALLOW_HOLMES_OOMKILL_TOOLSET` to ```bash kubectl -n exec -it -- \ cat /app/custom_toolset.yaml - # Expect oom_kill and oom_kill/bash present and enabled + # Expect oom_kill and oom_kill_bash present and enabled ``` ## Enabling in Local CLI Mode From b96b09230c3ace6167992b5f7b32a5102f6f71e9 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Wed, 24 Dec 2025 23:24:13 +0200 Subject: [PATCH 07/12] Fix oom_kill_bash toolset tag to load in server mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed tag from 'cli' to 'core' so the toolset is available when Holmes runs as a server in Kubernetes, not just in CLI mode. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- holmes/plugins/toolsets/oom_kill.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holmes/plugins/toolsets/oom_kill.yaml b/holmes/plugins/toolsets/oom_kill.yaml index b3351684c4..d7b653d191 100644 --- a/holmes/plugins/toolsets/oom_kill.yaml +++ b/holmes/plugins/toolsets/oom_kill.yaml @@ -4,7 +4,7 @@ toolsets: docs_url: "" icon_url: "" tags: - - cli + - core prerequisites: - env: - ALLOW_HOLMES_OOMKILL_TOOLSET From d63ee32bf4bc3193d525c0735acee74f43b0aa53 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 25 Dec 2025 07:57:02 +0200 Subject: [PATCH 08/12] Add memory limit protection for tool subprocesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add ulimit-based memory protection to prevent OOM from crashing main process - Support configurable limit via HOLMES_TOOL_MEMORY_LIMIT env var (default: 2GB) - Parse human-readable sizes (e.g., "4GB", "512MB", "1.5G") - Detect OOM kills and show helpful hint about increasing the limit - Apply protection to both YAML tools and bash toolset 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- holmes/core/tools.py | 16 +- holmes/plugins/toolsets/bash/common/bash.py | 12 +- holmes/plugins/toolsets/oom_kill.yaml | 1 - holmes/utils/memory_limit.py | 124 ++++++++++++++ tests/core/test_tool_memory_limit.py | 174 ++++++++++++++++++++ 5 files changed, 316 insertions(+), 11 deletions(-) create mode 100644 holmes/utils/memory_limit.py create mode 100644 tests/core/test_tool_memory_limit.py diff --git a/holmes/core/tools.py b/holmes/core/tools.py index 2111cb01dd..49c8cb6668 100644 --- a/holmes/core/tools.py +++ b/holmes/core/tools.py @@ -5,6 +5,7 @@ import shlex import subprocess import tempfile +import time from abc import ABC, abstractmethod from datetime import datetime from enum import Enum @@ -30,21 +31,21 @@ PrivateAttr, ) from rich.console import Console +from rich.table import Table from holmes.core.llm import LLM from holmes.core.openai_formatting import format_tool_to_open_ai_standard -from holmes.plugins.prompts import load_and_render_prompt from holmes.core.transformers import ( registry, TransformerError, Transformer, ) +from holmes.plugins.prompts import load_and_render_prompt +from holmes.utils.config_utils import merge_transformers +from holmes.utils.memory_limit import get_ulimit_prefix, check_oom_and_append_hint if TYPE_CHECKING: from holmes.core.transformers import BaseTransformer -from holmes.utils.config_utils import merge_transformers -import time -from rich.table import Table logger = logging.getLogger(__name__) @@ -497,8 +498,9 @@ def __invoke_script(self, params) -> str: def __execute_subprocess(self, cmd) -> Tuple[str, int]: try: logger.debug(f"Running `{cmd}`") + protected_cmd = get_ulimit_prefix() + cmd result = subprocess.run( - cmd, + protected_cmd, shell=True, text=True, check=False, # do not throw error, we just return the error code @@ -507,7 +509,9 @@ def __execute_subprocess(self, cmd) -> Tuple[str, int]: stderr=subprocess.STDOUT, ) - return result.stdout.strip(), result.returncode + output = result.stdout.strip() + output = check_oom_and_append_hint(output, result.returncode) + return output, result.returncode except Exception as e: logger.error( f"An unexpected error occurred while running '{cmd}': {e}", diff --git a/holmes/plugins/toolsets/bash/common/bash.py b/holmes/plugins/toolsets/bash/common/bash.py index 3849609a3f..2e003dd0ac 100644 --- a/holmes/plugins/toolsets/bash/common/bash.py +++ b/holmes/plugins/toolsets/bash/common/bash.py @@ -1,11 +1,14 @@ import subprocess + from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus +from holmes.utils.memory_limit import get_ulimit_prefix, check_oom_and_append_hint def execute_bash_command(cmd: str, timeout: int, params: dict) -> StructuredToolResult: try: + protected_cmd = get_ulimit_prefix() + cmd process = subprocess.run( - cmd, + protected_cmd, shell=True, executable="/bin/bash", stdout=subprocess.PIPE, @@ -16,7 +19,8 @@ def execute_bash_command(cmd: str, timeout: int, params: dict) -> StructuredTool ) stdout = process.stdout.strip() if process.stdout else "" - result_data = f"{cmd}\n" f"{stdout}" + stdout = check_oom_and_append_hint(stdout, process.returncode) + result_data = f"{cmd}\n{stdout}" if process.returncode == 0: status = ( @@ -44,10 +48,10 @@ def execute_bash_command(cmd: str, timeout: int, params: dict) -> StructuredTool params=params, ) except FileNotFoundError: - # This might occur if /bin/bash is not found, or if shell=False and command is not found + # This might occur if /bin/bash is not found, or command is not found return StructuredToolResult( status=StructuredToolResultStatus.ERROR, - error="Error: Bash executable or command not found. Ensure bash is installed and the command is valid.", + error="Error: Bash executable or command not found.", params=params, ) except Exception as e: diff --git a/holmes/plugins/toolsets/oom_kill.yaml b/holmes/plugins/toolsets/oom_kill.yaml index d7b653d191..0117836431 100644 --- a/holmes/plugins/toolsets/oom_kill.yaml +++ b/holmes/plugins/toolsets/oom_kill.yaml @@ -14,7 +14,6 @@ toolsets: - name: "trigger_oom_kill_bash" description: "Allocate ~30GB of memory and hold it for a period to provoke the OOM killer. No confirmation required; intended for automated stress tests." command: | - ulimit -v 2097152 || true python - <<'PY' import time diff --git a/holmes/utils/memory_limit.py b/holmes/utils/memory_limit.py new file mode 100644 index 0000000000..66df6eec36 --- /dev/null +++ b/holmes/utils/memory_limit.py @@ -0,0 +1,124 @@ +""" +Memory limit utilities for tool subprocess execution. + +Provides functions to parse human-readable memory sizes and apply +ulimit-based memory protection to prevent OOM from crashing the main process. +""" + +import logging +import os +import re + +logger = logging.getLogger(__name__) + +# Environment variable for configuring memory limit for tool subprocesses +TOOL_MEMORY_LIMIT_ENV = "HOLMES_TOOL_MEMORY_LIMIT" +TOOL_MEMORY_LIMIT_DEFAULT = "2GB" + + +def parse_size_to_kb(size_str: str) -> int: + """ + Parse a human-readable size string to kilobytes. + + Supports formats like: "2GB", "2gb", "2 GB", "2g", "1024MB", "2097152KB", "2097152". + If no unit is specified, assumes kilobytes. + + Args: + size_str: Human-readable size string + + Returns: + Size in kilobytes (for use with ulimit -v) + + Raises: + ValueError: If the size string cannot be parsed + """ + size_str = size_str.strip().upper() + + # Match number (with optional decimal) and optional unit + match = re.match(r"^(\d+(?:\.\d+)?)\s*([KMGT]?B?)?$", size_str) + if not match: + raise ValueError(f"Invalid size format: {size_str}") + + value = float(match.group(1)) + unit = match.group(2) or "K" # Default to KB if no unit + + # Normalize unit (handle both "G" and "GB" style) + unit = unit.rstrip("B") or "K" + + multipliers = { + "K": 1, + "M": 1024, + "G": 1024 * 1024, + "T": 1024 * 1024 * 1024, + } + + if unit not in multipliers: + raise ValueError(f"Unknown size unit: {unit}") + + return int(value * multipliers[unit]) + + +def get_memory_limit_kb() -> int: + """ + Get the configured memory limit in KB from environment variable. + + Returns the parsed memory limit, falling back to default if the env var + is not set or has an invalid value. + """ + memory_limit_str = os.environ.get(TOOL_MEMORY_LIMIT_ENV, TOOL_MEMORY_LIMIT_DEFAULT) + try: + return parse_size_to_kb(memory_limit_str) + except ValueError as e: + logger.warning( + f"Invalid {TOOL_MEMORY_LIMIT_ENV}='{memory_limit_str}': {e}. " + f"Using default: {TOOL_MEMORY_LIMIT_DEFAULT}" + ) + return parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + + +def get_ulimit_prefix() -> str: + """ + Get the ulimit command prefix for memory protection. + + Returns a shell command prefix that sets virtual memory limit. + The '|| true' ensures we continue even if ulimit is not supported. + """ + memory_limit_kb = get_memory_limit_kb() + return f"ulimit -v {memory_limit_kb} || true; " + + +def check_oom_and_append_hint(output: str, return_code: int) -> str: + """ + Check if a command was OOM killed and append a helpful hint. + + Args: + output: The command output + return_code: The command's return code + + Returns: + Output with OOM hint appended if OOM was detected + """ + # Common OOM indicators: + # - Return code 137 (128 + 9 = SIGKILL, commonly OOM) + # - Return code -9 (SIGKILL on some systems) + # - "Killed" in output (Linux OOM killer message) + # - "MemoryError" (Python) + # - "Cannot allocate memory" (various tools) + is_oom = ( + return_code in (137, -9) + or "Killed" in output + or "MemoryError" in output + or "Cannot allocate memory" in output + or "bad_alloc" in output + ) + + if is_oom: + current_limit = os.environ.get(TOOL_MEMORY_LIMIT_ENV, TOOL_MEMORY_LIMIT_DEFAULT) + hint = ( + f"\n\n[OOM] Command was likely killed due to memory limits. " + f"Current limit: {current_limit}. " + f"To increase, set {TOOL_MEMORY_LIMIT_ENV} (e.g., '4GB', '8GB')." + ) + return output + hint + + return output diff --git a/tests/core/test_tool_memory_limit.py b/tests/core/test_tool_memory_limit.py new file mode 100644 index 0000000000..7b77c7cb0e --- /dev/null +++ b/tests/core/test_tool_memory_limit.py @@ -0,0 +1,174 @@ +import pytest + +from holmes.utils.memory_limit import ( + parse_size_to_kb, + get_memory_limit_kb, + get_ulimit_prefix, + check_oom_and_append_hint, + TOOL_MEMORY_LIMIT_ENV, + TOOL_MEMORY_LIMIT_DEFAULT, +) + + +class TestParseSizeToKb: + """Tests for the parse_size_to_kb function.""" + + @pytest.mark.parametrize( + "input_value,expected_kb", + [ + # Gigabytes + ("2GB", 2 * 1024 * 1024), + ("2gb", 2 * 1024 * 1024), + ("2Gb", 2 * 1024 * 1024), + ("2G", 2 * 1024 * 1024), + ("2g", 2 * 1024 * 1024), + ("2 GB", 2 * 1024 * 1024), + ("2 G", 2 * 1024 * 1024), + ("1GB", 1024 * 1024), + ("4GB", 4 * 1024 * 1024), + # Megabytes + ("1024MB", 1024 * 1024), + ("1024M", 1024 * 1024), + ("1024mb", 1024 * 1024), + ("1024m", 1024 * 1024), + ("512MB", 512 * 1024), + ("512 MB", 512 * 1024), + # Kilobytes + ("2097152KB", 2097152), + ("2097152K", 2097152), + ("2097152kb", 2097152), + ("2097152k", 2097152), + ("1024KB", 1024), + ("1024 KB", 1024), + # Terabytes + ("1TB", 1024 * 1024 * 1024), + ("1T", 1024 * 1024 * 1024), + ("1tb", 1024 * 1024 * 1024), + ("2TB", 2 * 1024 * 1024 * 1024), + # No unit (defaults to KB) + ("2097152", 2097152), + ("1024", 1024), + ("512", 512), + # Decimal values + ("1.5GB", int(1.5 * 1024 * 1024)), + ("1.5G", int(1.5 * 1024 * 1024)), + ("2.5MB", int(2.5 * 1024)), + ("0.5GB", int(0.5 * 1024 * 1024)), + # Whitespace handling + (" 2GB ", 2 * 1024 * 1024), + ("2 GB", 2 * 1024 * 1024), + ], + ) + def test_valid_size_strings(self, input_value: str, expected_kb: int): + """Test parsing of valid size strings.""" + result = parse_size_to_kb(input_value) + assert result == expected_kb + + @pytest.mark.parametrize( + "invalid_input", + [ + "invalid", + "abc", + "GB", + "2XB", + "2PB", # Petabytes not supported + "-2GB", # Negative values + "", + " ", + ], + ) + def test_invalid_size_strings(self, invalid_input: str): + """Test that invalid size strings raise ValueError.""" + with pytest.raises(ValueError): + parse_size_to_kb(invalid_input) + + def test_default_value_parses_correctly(self): + """Test that the default value '2GB' parses to expected KB.""" + result = parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + assert result == 2097152 # 2GB in KB + + +class TestGetMemoryLimitKb: + """Tests for get_memory_limit_kb function.""" + + def test_returns_default_when_env_not_set(self, monkeypatch): + """Test that default value is used when env var is not set.""" + monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) + result = get_memory_limit_kb() + assert result == parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + + def test_returns_custom_value_from_env(self, monkeypatch): + """Test that custom value is used when env var is set.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "4GB") + result = get_memory_limit_kb() + assert result == 4 * 1024 * 1024 + + def test_falls_back_to_default_on_invalid_env(self, monkeypatch): + """Test fallback to default when env var has invalid value.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "invalid") + result = get_memory_limit_kb() + assert result == parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + + +class TestGetUlimitPrefix: + """Tests for get_ulimit_prefix function.""" + + def test_returns_ulimit_command_with_default(self, monkeypatch): + """Test ulimit prefix format with default value.""" + monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) + result = get_ulimit_prefix() + assert result == "ulimit -v 2097152 || true; " + + def test_returns_ulimit_command_with_custom_value(self, monkeypatch): + """Test ulimit prefix format with custom value.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "4GB") + result = get_ulimit_prefix() + assert result == "ulimit -v 4194304 || true; " + + +class TestCheckOomAndAppendHint: + """Tests for check_oom_and_append_hint function.""" + + def test_no_hint_on_success(self): + """Test that no hint is appended on successful command.""" + output = "command output" + result = check_oom_and_append_hint(output, 0) + assert result == output + assert "[OOM]" not in result + + def test_no_hint_on_regular_error(self): + """Test that no hint is appended on regular (non-OOM) error.""" + output = "some error occurred" + result = check_oom_and_append_hint(output, 1) + assert result == output + assert "[OOM]" not in result + + @pytest.mark.parametrize( + "return_code,output", + [ + (137, ""), # SIGKILL (128 + 9) + (-9, ""), # SIGKILL on some systems + (0, "Killed"), # Linux OOM killer message + (1, "MemoryError: unable to allocate"), # Python OOM + (1, "Cannot allocate memory"), # System allocation failure + (1, "std::bad_alloc"), # C++ allocation failure + ], + ) + def test_hint_appended_on_oom_indicators(self, return_code: int, output: str): + """Test that hint is appended when OOM indicators are detected.""" + result = check_oom_and_append_hint(output, return_code) + assert "[OOM]" in result + assert TOOL_MEMORY_LIMIT_ENV in result + assert "4GB" in result or "8GB" in result # Example values in hint + + def test_hint_includes_current_limit(self, monkeypatch): + """Test that hint shows the current configured limit.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "1GB") + result = check_oom_and_append_hint("Killed", 137) + assert "Current limit: 1GB" in result + + def test_hint_shows_default_when_not_configured(self, monkeypatch): + """Test that hint shows default when env var not set.""" + monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) + result = check_oom_and_append_hint("Killed", 137) + assert f"Current limit: {TOOL_MEMORY_LIMIT_DEFAULT}" in result From 09dedea62fcddc14c25c3cd97f52e0ae1492e775 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 25 Dec 2025 08:14:05 +0200 Subject: [PATCH 09/12] Fix mypy error in oom_kill.py by declaring toolset field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow the established pattern for Tool subclasses that need toolset reference. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- holmes/plugins/toolsets/oom_kill.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/holmes/plugins/toolsets/oom_kill.py b/holmes/plugins/toolsets/oom_kill.py index ccaef02c6a..b6ba1e48ca 100644 --- a/holmes/plugins/toolsets/oom_kill.py +++ b/holmes/plugins/toolsets/oom_kill.py @@ -16,6 +16,8 @@ class TriggerOOMKill(Tool): + toolset: "OOMKillToolset" + def __init__(self, toolset: "OOMKillToolset"): super().__init__( name="trigger_oom_kill", @@ -34,7 +36,7 @@ def __init__(self, toolset: "OOMKillToolset"): required=False, ), }, - toolset=toolset, + toolset=toolset, # type: ignore[call-arg] ) def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: From 50bfb773e3319afd21520e03639a1747a4217e3f Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 25 Dec 2025 12:40:37 +0200 Subject: [PATCH 10/12] tweaks --- holmes/plugins/toolsets/oom_kill.py | 20 +++++--------------- holmes/utils/memory_limit.py | 10 ++++++---- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/holmes/plugins/toolsets/oom_kill.py b/holmes/plugins/toolsets/oom_kill.py index b6ba1e48ca..ad4ae56b0e 100644 --- a/holmes/plugins/toolsets/oom_kill.py +++ b/holmes/plugins/toolsets/oom_kill.py @@ -1,4 +1,5 @@ import textwrap +import time from typing import Any, Dict from holmes.core.tools import ( @@ -48,21 +49,10 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes params=params, ) - command = textwrap.dedent( - f""" - python - <<'PY' - import time - - size_bytes = 30 * 1024 * 1024 * 1024 - print(f"Allocating {{size_bytes / 1024 / 1024 / 1024:.0f}} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s") - data = bytearray(size_bytes) - time.sleep({hold_seconds}) - PY - """ - ).strip() - - timeout = hold_seconds + 30 - return execute_bash_command(cmd=command, timeout=timeout, params=params) + size_bytes = 30 * 1024 * 1024 * 1024 + print(f"Allocating {{size_bytes / 1024 / 1024 / 1024:.0f}} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s") + data = bytearray(size_bytes) # type: ignore + time.sleep({hold_seconds}) def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str: hold_seconds = params.get("hold_seconds", 300) diff --git a/holmes/utils/memory_limit.py b/holmes/utils/memory_limit.py index 66df6eec36..f54d3c5285 100644 --- a/holmes/utils/memory_limit.py +++ b/holmes/utils/memory_limit.py @@ -13,7 +13,7 @@ # Environment variable for configuring memory limit for tool subprocesses TOOL_MEMORY_LIMIT_ENV = "HOLMES_TOOL_MEMORY_LIMIT" -TOOL_MEMORY_LIMIT_DEFAULT = "2GB" +TOOL_MEMORY_LIMIT_DEFAULT = "500MB" def parse_size_to_kb(size_str: str) -> int: @@ -115,9 +115,11 @@ def check_oom_and_append_hint(output: str, return_code: int) -> str: if is_oom: current_limit = os.environ.get(TOOL_MEMORY_LIMIT_ENV, TOOL_MEMORY_LIMIT_DEFAULT) hint = ( - f"\n\n[OOM] Command was likely killed due to memory limits. " - f"Current limit: {current_limit}. " - f"To increase, set {TOOL_MEMORY_LIMIT_ENV} (e.g., '4GB', '8GB')." + f"\n\n[OOM] Command was killed due to memory limits (current limit: {current_limit}). " + f"Try querying the data differently to reduce memory usage - add filters to narrow the results, " + f"use smaller time ranges, or try alternative tools that may be more memory-efficient. " + f"If you cannot succeed with a modified query, you may recommend the user increase the limit " + f"by setting {TOOL_MEMORY_LIMIT_ENV} (e.g., '1GB', '2GB')." ) return output + hint From 7a5b967225807d567311139d3fa1648bf0671285 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 25 Dec 2025 12:41:31 +0200 Subject: [PATCH 11/12] format --- holmes/plugins/toolsets/oom_kill.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/holmes/plugins/toolsets/oom_kill.py b/holmes/plugins/toolsets/oom_kill.py index ad4ae56b0e..2d8f820269 100644 --- a/holmes/plugins/toolsets/oom_kill.py +++ b/holmes/plugins/toolsets/oom_kill.py @@ -1,4 +1,3 @@ -import textwrap import time from typing import Any, Dict @@ -13,7 +12,6 @@ ToolsetTag, ToolsetEnvironmentPrerequisite, ) -from holmes.plugins.toolsets.bash.common.bash import execute_bash_command class TriggerOOMKill(Tool): @@ -50,8 +48,10 @@ def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolRes ) size_bytes = 30 * 1024 * 1024 * 1024 - print(f"Allocating {{size_bytes / 1024 / 1024 / 1024:.0f}} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s") - data = bytearray(size_bytes) # type: ignore + print( + f"Allocating {{size_bytes / 1024 / 1024 / 1024:.0f}} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s" + ) + data = bytearray(size_bytes) # type: ignore time.sleep({hold_seconds}) def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str: From 08418f0d5b59eed5a55b2710a8f3c5d5126b08e1 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 25 Dec 2025 12:59:00 +0200 Subject: [PATCH 12/12] format --- tests/core/test_tool_memory_limit.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/core/test_tool_memory_limit.py b/tests/core/test_tool_memory_limit.py index 7b77c7cb0e..e47b575676 100644 --- a/tests/core/test_tool_memory_limit.py +++ b/tests/core/test_tool_memory_limit.py @@ -83,9 +83,9 @@ def test_invalid_size_strings(self, invalid_input: str): parse_size_to_kb(invalid_input) def test_default_value_parses_correctly(self): - """Test that the default value '2GB' parses to expected KB.""" + """Test that the default value parses without error.""" result = parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) - assert result == 2097152 # 2GB in KB + assert result > 0 # Just verify it parses to a positive value class TestGetMemoryLimitKb: @@ -117,7 +117,8 @@ def test_returns_ulimit_command_with_default(self, monkeypatch): """Test ulimit prefix format with default value.""" monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) result = get_ulimit_prefix() - assert result == "ulimit -v 2097152 || true; " + expected_kb = parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + assert result == f"ulimit -v {expected_kb} || true; " def test_returns_ulimit_command_with_custom_value(self, monkeypatch): """Test ulimit prefix format with custom value.""" @@ -159,16 +160,16 @@ def test_hint_appended_on_oom_indicators(self, return_code: int, output: str): result = check_oom_and_append_hint(output, return_code) assert "[OOM]" in result assert TOOL_MEMORY_LIMIT_ENV in result - assert "4GB" in result or "8GB" in result # Example values in hint + assert TOOL_MEMORY_LIMIT_DEFAULT in result # Shows current limit def test_hint_includes_current_limit(self, monkeypatch): """Test that hint shows the current configured limit.""" monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "1GB") result = check_oom_and_append_hint("Killed", 137) - assert "Current limit: 1GB" in result + assert "current limit: 1GB" in result def test_hint_shows_default_when_not_configured(self, monkeypatch): """Test that hint shows default when env var not set.""" monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) result = check_oom_and_append_hint("Killed", 137) - assert f"Current limit: {TOOL_MEMORY_LIMIT_DEFAULT}" in result + assert f"current limit: {TOOL_MEMORY_LIMIT_DEFAULT}" in result