diff --git a/docs/development/stress-testing-oom-kill.md b/docs/development/stress-testing-oom-kill.md new file mode 100644 index 0000000000..1f86487df6 --- /dev/null +++ b/docs/development/stress-testing-oom-kill.md @@ -0,0 +1,110 @@ +# Stress Testing Holmes with Intentional OOM Kills + +> ⚠️ **Never enable this toolset in production.** It allocates ~30 GB of RAM on the Holmes host and is intended only for controlled, non-production stress tests. + +This guide explains how to intentionally trigger OOM kills using the built-in OOM toolsets, how to enable them safely, and how to confirm they are available. + +## Available Toolsets + +Holmes ships with two disabled-by-default toolsets for inducing an OOM kill: + +- **`oom_kill` (Python)** – `trigger_oom_kill` allocates ~30 GB and sleeps for a configurable duration. +- **`oom_kill_bash` (YAML/bash)** – `trigger_oom_kill_bash` does the same via a bash-executed Python snippet and applies `ulimit -v 2097152` (2 GiB virtual memory cap) before allocation to reduce blast radius. + +Both toolsets require the environment variable `ALLOW_HOLMES_OOMKILL_TOOLSET` to pass prerequisites and must be explicitly enabled in configuration. They are **disabled by default** and will not be loaded unless you opt in. + +## Enabling via Helm/ArgoCD (cluster install) + +Use the correct values path for your deployment method. + +=== "Robusta Helm Chart (Holmes as subchart)" + +1. **Set the env guard** (required): + ```bash + argocd app set \ + --helm-set-string holmes.additionalEnvVars[0].name=ALLOW_HOLMES_OOMKILL_TOOLSET \ + --helm-set-string holmes.additionalEnvVars[0].value=true + ``` + +2. **Enable the toolsets**: + ```bash + argocd app set \ + --helm-set-string holmes.toolsets.oom_kill.enabled=true \ + --helm-set-string holmes.toolsets.oom_kill_bash.enabled=true + ``` + +3. **Sync to apply**: + ```bash + argocd app sync + ``` + +4. **Verify** (optional): + ```bash + kubectl -n exec -it -- \ + cat /app/custom_toolset.yaml + # Expect oom_kill and oom_kill_bash present and enabled + ``` + +=== "Holmes Helm Chart (direct)" + +1. **Set the env guard** (required): + ```bash + argocd app set \ + --helm-set-string additionalEnvVars[0].name=ALLOW_HOLMES_OOMKILL_TOOLSET \ + --helm-set-string additionalEnvVars[0].value=true + ``` + +2. **Enable the toolsets**: + ```bash + argocd app set \ + --helm-set-string toolsets.oom_kill.enabled=true \ + --helm-set-string toolsets.oom_kill_bash.enabled=true + ``` + +3. **Sync to apply**: + ```bash + argocd app sync + ``` + +4. **Verify** (optional): + ```bash + kubectl -n exec -it -- \ + cat /app/custom_toolset.yaml + # Expect oom_kill and oom_kill_bash present and enabled + ``` + +## Enabling in Local CLI Mode + +Add to your local config (e.g., `config.yaml`) and set the env guard before running the CLI: + +```yaml +toolsets: + oom_kill: + enabled: true + oom_kill_bash: + enabled: true +``` + +Then run: +```bash +export ALLOW_HOLMES_OOMKILL_TOOLSET=true +holmes --config ./config.yaml ... +``` + +Because both toolsets are disabled by default and gated by `ALLOW_HOLMES_OOMKILL_TOOLSET`, they will **not** be auto-enabled in local mode unless you explicitly enable them and set the env variable. + +## Using the Tools + +- **Python toolset**: `trigger_oom_kill` (param: `hold_seconds`, default 300). +- **Bash toolset**: `trigger_oom_kill_bash` (param: `hold_seconds`, default 300). + +Example invocation (conceptual): +``` +trigger_oom_kill: allocate ~30GB and sleep for 120s +``` + +## Safety Considerations + +- Keep this toolset out of production environments. +- Ensure hosts have proper isolation; the process is expected to be OOM-killed. +- Consider running in a dedicated test cluster or namespace. diff --git a/holmes/core/tools.py b/holmes/core/tools.py index 2111cb01dd..49c8cb6668 100644 --- a/holmes/core/tools.py +++ b/holmes/core/tools.py @@ -5,6 +5,7 @@ import shlex import subprocess import tempfile +import time from abc import ABC, abstractmethod from datetime import datetime from enum import Enum @@ -30,21 +31,21 @@ PrivateAttr, ) from rich.console import Console +from rich.table import Table from holmes.core.llm import LLM from holmes.core.openai_formatting import format_tool_to_open_ai_standard -from holmes.plugins.prompts import load_and_render_prompt from holmes.core.transformers import ( registry, TransformerError, Transformer, ) +from holmes.plugins.prompts import load_and_render_prompt +from holmes.utils.config_utils import merge_transformers +from holmes.utils.memory_limit import get_ulimit_prefix, check_oom_and_append_hint if TYPE_CHECKING: from holmes.core.transformers import BaseTransformer -from holmes.utils.config_utils import merge_transformers -import time -from rich.table import Table logger = logging.getLogger(__name__) @@ -497,8 +498,9 @@ def __invoke_script(self, params) -> str: def __execute_subprocess(self, cmd) -> Tuple[str, int]: try: logger.debug(f"Running `{cmd}`") + protected_cmd = get_ulimit_prefix() + cmd result = subprocess.run( - cmd, + protected_cmd, shell=True, text=True, check=False, # do not throw error, we just return the error code @@ -507,7 +509,9 @@ def __execute_subprocess(self, cmd) -> Tuple[str, int]: stderr=subprocess.STDOUT, ) - return result.stdout.strip(), result.returncode + output = result.stdout.strip() + output = check_oom_and_append_hint(output, result.returncode) + return output, result.returncode except Exception as e: logger.error( f"An unexpected error occurred while running '{cmd}': {e}", diff --git a/holmes/plugins/toolsets/__init__.py b/holmes/plugins/toolsets/__init__.py index 2368859c3c..44b9b3fdb9 100644 --- a/holmes/plugins/toolsets/__init__.py +++ b/holmes/plugins/toolsets/__init__.py @@ -35,6 +35,7 @@ from holmes.plugins.toolsets.internet.notion import NotionToolset from holmes.plugins.toolsets.kafka import KafkaToolset from holmes.plugins.toolsets.kubernetes_logs import KubernetesLogsToolset +from holmes.plugins.toolsets.oom_kill import OOMKillToolset from holmes.plugins.toolsets.mcp.toolset_mcp import RemoteMCPToolset from holmes.plugins.toolsets.newrelic.newrelic import NewRelicToolset from holmes.plugins.toolsets.opensearch.opensearch import OpenSearchToolset @@ -96,6 +97,7 @@ def load_python_toolsets( OpenSearchLogsToolset(), OpenSearchTracesToolset(), OpenSearchQueryAssistToolset(), + OOMKillToolset(), CoralogixToolset(), RabbitMQToolset(), GitToolset(), diff --git a/holmes/plugins/toolsets/bash/common/bash.py b/holmes/plugins/toolsets/bash/common/bash.py index 3849609a3f..2e003dd0ac 100644 --- a/holmes/plugins/toolsets/bash/common/bash.py +++ b/holmes/plugins/toolsets/bash/common/bash.py @@ -1,11 +1,14 @@ import subprocess + from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus +from holmes.utils.memory_limit import get_ulimit_prefix, check_oom_and_append_hint def execute_bash_command(cmd: str, timeout: int, params: dict) -> StructuredToolResult: try: + protected_cmd = get_ulimit_prefix() + cmd process = subprocess.run( - cmd, + protected_cmd, shell=True, executable="/bin/bash", stdout=subprocess.PIPE, @@ -16,7 +19,8 @@ def execute_bash_command(cmd: str, timeout: int, params: dict) -> StructuredTool ) stdout = process.stdout.strip() if process.stdout else "" - result_data = f"{cmd}\n" f"{stdout}" + stdout = check_oom_and_append_hint(stdout, process.returncode) + result_data = f"{cmd}\n{stdout}" if process.returncode == 0: status = ( @@ -44,10 +48,10 @@ def execute_bash_command(cmd: str, timeout: int, params: dict) -> StructuredTool params=params, ) except FileNotFoundError: - # This might occur if /bin/bash is not found, or if shell=False and command is not found + # This might occur if /bin/bash is not found, or command is not found return StructuredToolResult( status=StructuredToolResultStatus.ERROR, - error="Error: Bash executable or command not found. Ensure bash is installed and the command is valid.", + error="Error: Bash executable or command not found.", params=params, ) except Exception as e: diff --git a/holmes/plugins/toolsets/oom_kill.py b/holmes/plugins/toolsets/oom_kill.py new file mode 100644 index 0000000000..2d8f820269 --- /dev/null +++ b/holmes/plugins/toolsets/oom_kill.py @@ -0,0 +1,91 @@ +import time +from typing import Any, Dict + +from holmes.core.tools import ( + CallablePrerequisite, + StructuredToolResult, + StructuredToolResultStatus, + Tool, + ToolInvokeContext, + ToolParameter, + Toolset, + ToolsetTag, + ToolsetEnvironmentPrerequisite, +) + + +class TriggerOOMKill(Tool): + toolset: "OOMKillToolset" + + def __init__(self, toolset: "OOMKillToolset"): + super().__init__( + name="trigger_oom_kill", + description=( + "Allocates approximately 30GB of memory on the Holmes host to provoke the " + "OOM killer. This is intended for stress testing only and will likely crash " + "the running process. No confirmation is required because this is meant for " + "automated stress scenarios." + ), + parameters={ + "hold_seconds": ToolParameter( + description=( + "How long to keep the memory allocated before exiting. Defaults to 300 seconds." + ), + type="integer", + required=False, + ), + }, + toolset=toolset, # type: ignore[call-arg] + ) + + def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult: + hold_seconds = params.get("hold_seconds", 300) + if not isinstance(hold_seconds, int) or hold_seconds <= 0: + return StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error="hold_seconds must be a positive integer.", + params=params, + ) + + size_bytes = 30 * 1024 * 1024 * 1024 + print( + f"Allocating {{size_bytes / 1024 / 1024 / 1024:.0f}} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s" + ) + data = bytearray(size_bytes) # type: ignore + time.sleep({hold_seconds}) + + def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str: + hold_seconds = params.get("hold_seconds", 300) + return ( + "python - <<'PY' ... # allocates ~30GB and sleeps for " + f"{hold_seconds}s to trigger OOM" + ) + + +class OOMKillToolset(Toolset): + def __init__(self): + super().__init__( + name="oom_kill", + enabled=False, + description=( + "Dangerous toolset that intentionally exhausts memory on the Holmes host to trigger an OOM kill. " + "Use only in controlled stress tests." + ), + docs_url=None, + icon_url=None, + prerequisites=[ + ToolsetEnvironmentPrerequisite(env=["ALLOW_HOLMES_OOMKILL_TOOLSET"]), + CallablePrerequisite(callable=self.prerequisites_callable), + ], + tools=[TriggerOOMKill(self)], + experimental=True, + tags=[ToolsetTag.CORE], + is_default=False, + ) + + def prerequisites_callable(self, config: dict[str, Any]) -> tuple[bool, str]: + # No special configuration is required for this toolset. + return True, "" + + def get_example_config(self) -> Dict[str, Any]: + return {} diff --git a/holmes/plugins/toolsets/oom_kill.yaml b/holmes/plugins/toolsets/oom_kill.yaml new file mode 100644 index 0000000000..0117836431 --- /dev/null +++ b/holmes/plugins/toolsets/oom_kill.yaml @@ -0,0 +1,28 @@ +toolsets: + oom_kill_bash: + description: "DANGEROUS: intentionally allocate ~30GB on the Holmes host to trigger OOM kill for stress testing (no confirmation required)." + docs_url: "" + icon_url: "" + tags: + - core + prerequisites: + - env: + - ALLOW_HOLMES_OOMKILL_TOOLSET + additional_instructions: | + ⚠️ This toolset is intentionally destructive. Only use in controlled environments. + tools: + - name: "trigger_oom_kill_bash" + description: "Allocate ~30GB of memory and hold it for a period to provoke the OOM killer. No confirmation required; intended for automated stress tests." + command: | + python - <<'PY' + import time + + hold_seconds = int("{{ hold_seconds|default(300) }}") + if hold_seconds <= 0: + raise SystemExit("hold_seconds must be positive.") + + size_bytes = 30 * 1024 * 1024 * 1024 + print(f"Allocating {size_bytes / 1024 / 1024 / 1024:.0f} GB of memory to intentionally trigger OOM kill; sleeping for {hold_seconds}s") + data = bytearray(size_bytes) + time.sleep(hold_seconds) + PY diff --git a/holmes/utils/memory_limit.py b/holmes/utils/memory_limit.py new file mode 100644 index 0000000000..f54d3c5285 --- /dev/null +++ b/holmes/utils/memory_limit.py @@ -0,0 +1,126 @@ +""" +Memory limit utilities for tool subprocess execution. + +Provides functions to parse human-readable memory sizes and apply +ulimit-based memory protection to prevent OOM from crashing the main process. +""" + +import logging +import os +import re + +logger = logging.getLogger(__name__) + +# Environment variable for configuring memory limit for tool subprocesses +TOOL_MEMORY_LIMIT_ENV = "HOLMES_TOOL_MEMORY_LIMIT" +TOOL_MEMORY_LIMIT_DEFAULT = "500MB" + + +def parse_size_to_kb(size_str: str) -> int: + """ + Parse a human-readable size string to kilobytes. + + Supports formats like: "2GB", "2gb", "2 GB", "2g", "1024MB", "2097152KB", "2097152". + If no unit is specified, assumes kilobytes. + + Args: + size_str: Human-readable size string + + Returns: + Size in kilobytes (for use with ulimit -v) + + Raises: + ValueError: If the size string cannot be parsed + """ + size_str = size_str.strip().upper() + + # Match number (with optional decimal) and optional unit + match = re.match(r"^(\d+(?:\.\d+)?)\s*([KMGT]?B?)?$", size_str) + if not match: + raise ValueError(f"Invalid size format: {size_str}") + + value = float(match.group(1)) + unit = match.group(2) or "K" # Default to KB if no unit + + # Normalize unit (handle both "G" and "GB" style) + unit = unit.rstrip("B") or "K" + + multipliers = { + "K": 1, + "M": 1024, + "G": 1024 * 1024, + "T": 1024 * 1024 * 1024, + } + + if unit not in multipliers: + raise ValueError(f"Unknown size unit: {unit}") + + return int(value * multipliers[unit]) + + +def get_memory_limit_kb() -> int: + """ + Get the configured memory limit in KB from environment variable. + + Returns the parsed memory limit, falling back to default if the env var + is not set or has an invalid value. + """ + memory_limit_str = os.environ.get(TOOL_MEMORY_LIMIT_ENV, TOOL_MEMORY_LIMIT_DEFAULT) + try: + return parse_size_to_kb(memory_limit_str) + except ValueError as e: + logger.warning( + f"Invalid {TOOL_MEMORY_LIMIT_ENV}='{memory_limit_str}': {e}. " + f"Using default: {TOOL_MEMORY_LIMIT_DEFAULT}" + ) + return parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + + +def get_ulimit_prefix() -> str: + """ + Get the ulimit command prefix for memory protection. + + Returns a shell command prefix that sets virtual memory limit. + The '|| true' ensures we continue even if ulimit is not supported. + """ + memory_limit_kb = get_memory_limit_kb() + return f"ulimit -v {memory_limit_kb} || true; " + + +def check_oom_and_append_hint(output: str, return_code: int) -> str: + """ + Check if a command was OOM killed and append a helpful hint. + + Args: + output: The command output + return_code: The command's return code + + Returns: + Output with OOM hint appended if OOM was detected + """ + # Common OOM indicators: + # - Return code 137 (128 + 9 = SIGKILL, commonly OOM) + # - Return code -9 (SIGKILL on some systems) + # - "Killed" in output (Linux OOM killer message) + # - "MemoryError" (Python) + # - "Cannot allocate memory" (various tools) + is_oom = ( + return_code in (137, -9) + or "Killed" in output + or "MemoryError" in output + or "Cannot allocate memory" in output + or "bad_alloc" in output + ) + + if is_oom: + current_limit = os.environ.get(TOOL_MEMORY_LIMIT_ENV, TOOL_MEMORY_LIMIT_DEFAULT) + hint = ( + f"\n\n[OOM] Command was killed due to memory limits (current limit: {current_limit}). " + f"Try querying the data differently to reduce memory usage - add filters to narrow the results, " + f"use smaller time ranges, or try alternative tools that may be more memory-efficient. " + f"If you cannot succeed with a modified query, you may recommend the user increase the limit " + f"by setting {TOOL_MEMORY_LIMIT_ENV} (e.g., '1GB', '2GB')." + ) + return output + hint + + return output diff --git a/tests/core/test_tool_memory_limit.py b/tests/core/test_tool_memory_limit.py new file mode 100644 index 0000000000..e47b575676 --- /dev/null +++ b/tests/core/test_tool_memory_limit.py @@ -0,0 +1,175 @@ +import pytest + +from holmes.utils.memory_limit import ( + parse_size_to_kb, + get_memory_limit_kb, + get_ulimit_prefix, + check_oom_and_append_hint, + TOOL_MEMORY_LIMIT_ENV, + TOOL_MEMORY_LIMIT_DEFAULT, +) + + +class TestParseSizeToKb: + """Tests for the parse_size_to_kb function.""" + + @pytest.mark.parametrize( + "input_value,expected_kb", + [ + # Gigabytes + ("2GB", 2 * 1024 * 1024), + ("2gb", 2 * 1024 * 1024), + ("2Gb", 2 * 1024 * 1024), + ("2G", 2 * 1024 * 1024), + ("2g", 2 * 1024 * 1024), + ("2 GB", 2 * 1024 * 1024), + ("2 G", 2 * 1024 * 1024), + ("1GB", 1024 * 1024), + ("4GB", 4 * 1024 * 1024), + # Megabytes + ("1024MB", 1024 * 1024), + ("1024M", 1024 * 1024), + ("1024mb", 1024 * 1024), + ("1024m", 1024 * 1024), + ("512MB", 512 * 1024), + ("512 MB", 512 * 1024), + # Kilobytes + ("2097152KB", 2097152), + ("2097152K", 2097152), + ("2097152kb", 2097152), + ("2097152k", 2097152), + ("1024KB", 1024), + ("1024 KB", 1024), + # Terabytes + ("1TB", 1024 * 1024 * 1024), + ("1T", 1024 * 1024 * 1024), + ("1tb", 1024 * 1024 * 1024), + ("2TB", 2 * 1024 * 1024 * 1024), + # No unit (defaults to KB) + ("2097152", 2097152), + ("1024", 1024), + ("512", 512), + # Decimal values + ("1.5GB", int(1.5 * 1024 * 1024)), + ("1.5G", int(1.5 * 1024 * 1024)), + ("2.5MB", int(2.5 * 1024)), + ("0.5GB", int(0.5 * 1024 * 1024)), + # Whitespace handling + (" 2GB ", 2 * 1024 * 1024), + ("2 GB", 2 * 1024 * 1024), + ], + ) + def test_valid_size_strings(self, input_value: str, expected_kb: int): + """Test parsing of valid size strings.""" + result = parse_size_to_kb(input_value) + assert result == expected_kb + + @pytest.mark.parametrize( + "invalid_input", + [ + "invalid", + "abc", + "GB", + "2XB", + "2PB", # Petabytes not supported + "-2GB", # Negative values + "", + " ", + ], + ) + def test_invalid_size_strings(self, invalid_input: str): + """Test that invalid size strings raise ValueError.""" + with pytest.raises(ValueError): + parse_size_to_kb(invalid_input) + + def test_default_value_parses_correctly(self): + """Test that the default value parses without error.""" + result = parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + assert result > 0 # Just verify it parses to a positive value + + +class TestGetMemoryLimitKb: + """Tests for get_memory_limit_kb function.""" + + def test_returns_default_when_env_not_set(self, monkeypatch): + """Test that default value is used when env var is not set.""" + monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) + result = get_memory_limit_kb() + assert result == parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + + def test_returns_custom_value_from_env(self, monkeypatch): + """Test that custom value is used when env var is set.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "4GB") + result = get_memory_limit_kb() + assert result == 4 * 1024 * 1024 + + def test_falls_back_to_default_on_invalid_env(self, monkeypatch): + """Test fallback to default when env var has invalid value.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "invalid") + result = get_memory_limit_kb() + assert result == parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + + +class TestGetUlimitPrefix: + """Tests for get_ulimit_prefix function.""" + + def test_returns_ulimit_command_with_default(self, monkeypatch): + """Test ulimit prefix format with default value.""" + monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) + result = get_ulimit_prefix() + expected_kb = parse_size_to_kb(TOOL_MEMORY_LIMIT_DEFAULT) + assert result == f"ulimit -v {expected_kb} || true; " + + def test_returns_ulimit_command_with_custom_value(self, monkeypatch): + """Test ulimit prefix format with custom value.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "4GB") + result = get_ulimit_prefix() + assert result == "ulimit -v 4194304 || true; " + + +class TestCheckOomAndAppendHint: + """Tests for check_oom_and_append_hint function.""" + + def test_no_hint_on_success(self): + """Test that no hint is appended on successful command.""" + output = "command output" + result = check_oom_and_append_hint(output, 0) + assert result == output + assert "[OOM]" not in result + + def test_no_hint_on_regular_error(self): + """Test that no hint is appended on regular (non-OOM) error.""" + output = "some error occurred" + result = check_oom_and_append_hint(output, 1) + assert result == output + assert "[OOM]" not in result + + @pytest.mark.parametrize( + "return_code,output", + [ + (137, ""), # SIGKILL (128 + 9) + (-9, ""), # SIGKILL on some systems + (0, "Killed"), # Linux OOM killer message + (1, "MemoryError: unable to allocate"), # Python OOM + (1, "Cannot allocate memory"), # System allocation failure + (1, "std::bad_alloc"), # C++ allocation failure + ], + ) + def test_hint_appended_on_oom_indicators(self, return_code: int, output: str): + """Test that hint is appended when OOM indicators are detected.""" + result = check_oom_and_append_hint(output, return_code) + assert "[OOM]" in result + assert TOOL_MEMORY_LIMIT_ENV in result + assert TOOL_MEMORY_LIMIT_DEFAULT in result # Shows current limit + + def test_hint_includes_current_limit(self, monkeypatch): + """Test that hint shows the current configured limit.""" + monkeypatch.setenv(TOOL_MEMORY_LIMIT_ENV, "1GB") + result = check_oom_and_append_hint("Killed", 137) + assert "current limit: 1GB" in result + + def test_hint_shows_default_when_not_configured(self, monkeypatch): + """Test that hint shows default when env var not set.""" + monkeypatch.delenv(TOOL_MEMORY_LIMIT_ENV, raising=False) + result = check_oom_and_append_hint("Killed", 137) + assert f"current limit: {TOOL_MEMORY_LIMIT_DEFAULT}" in result