Add 5 bash resource checker evals for kubectl exec scenarios

pyproject.toml

@@ -138,6 +138,7 @@ markers = [
     "elasticsearch: Tests for Elasticsearch/OpenSearch toolset functionality",
     "integration: Integration tests requiring external services (e.g., running server, real APIs)",
     "confluence: Tests for Confluence page fetching toolset",
+    "bash-resource-checker: Tests for bash toolset with kubectl exec to check resources inside containers",
     # REGRESSION TEST SELECTION CRITERIA:
     # - Must pass 30+ iterations reliably with Sonnet-4.5 model
     # - No external API dependencies (Loki OK, NewRelic/DataDog excluded)

tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/manifest.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: data-cruncher
+  namespace: app-211
+  labels:
+    app: data-cruncher
+spec:
+  containers:
+    - name: worker
+      image: python:3.11-alpine
+      command:
+        - /bin/sh
+        - -c
+        - |
+          # Create 95 normal sleep processes (minimal memory)
+          for i in $(seq 1 95); do
+            sleep 3600 &
+          done
+
+          # Create 5 memory-hungry Python processes (~20MB each)
+          for i in $(seq 1 5); do
+            python3 -c "
+          import time
+          # Allocate ~20MB of memory
+          data = bytearray(20 * 1024 * 1024)
+          # Fill with data to ensure it's allocated
+          for j in range(0, len(data), 4096):
+              data[j] = 0xff
+          time.sleep(3600)
+          " &
+          done
+
+          # Keep container running
+          sleep 3600
+      resources:
+        limits:
+          memory: "256Mi"
+          cpu: "200m"
+        requests:
+          memory: "128Mi"
+          cpu: "100m"

tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/test_case.yaml

@@ -0,0 +1,61 @@
+user_prompt: "The data-cruncher pod in namespace app-211 is consuming too much memory. Check which processes inside the container are using the most memory and identify any that stand out."
+
+include_tool_calls: true
+
+expected_output:
+  - "Must call bash tool with kubectl exec"
+  - "Must identify Python processes as consuming the most memory"
+  - "Must report processes sorted by memory usage or identify top consumers"
+
+description: |
+  Tests that Holmes can use bash to run kubectl exec and inspect processes inside a container.
+  The pod runs 95 sleep processes plus 5 Python processes that allocate ~20MB each.
+  Holmes must use bash with 'kubectl exec' since no existing toolset can inspect in-container processes.
+
+tags:
+  - kubernetes
+  - regression
+  - bash-resource-checker
+
+before_test: |
+  kubectl create namespace app-211 --dry-run=client -o yaml | kubectl apply -f -
+  kubectl apply -f manifest.yaml
+
+  # Wait for pod to be ready
+  POD_READY=false
+  for i in {1..60}; do
+    if kubectl wait --for=condition=ready pod/data-cruncher -n app-211 --timeout=2s 2>/dev/null; then
+      echo "Pod ready!"
+      POD_READY=true
+      break
+    fi
+    sleep 1
+  done
+
+  if [ "$POD_READY" = false ]; then
+    echo "Pod not ready after 60 seconds"
+    kubectl get pods -n app-211
+    kubectl describe pod/data-cruncher -n app-211
+    exit 1
+  fi
+
+  # Verify processes are running inside the container
+  PROC_COUNT=$(kubectl exec -n app-211 data-cruncher -- ps aux 2>/dev/null | wc -l)
+  echo "Process count in container: $PROC_COUNT"
+  if [ "$PROC_COUNT" -lt 50 ]; then
+    echo "Not enough processes running in container"
+    kubectl exec -n app-211 data-cruncher -- ps aux
+    exit 1
+  fi
+
+  # Verify Python processes are running
+  PYTHON_COUNT=$(kubectl exec -n app-211 data-cruncher -- ps aux 2>/dev/null | grep -c "python3" || echo "0")
+  echo "Python process count: $PYTHON_COUNT"
+  if [ "$PYTHON_COUNT" -lt 5 ]; then
+    echo "Not enough Python processes running"
+    exit 1
+  fi
+  echo "Setup complete - container has $PROC_COUNT processes including $PYTHON_COUNT Python processes"
+
+after_test: |
+  kubectl delete namespace app-211 --ignore-not-found

tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/toolsets.yaml

@@ -0,0 +1,21 @@
+# Bash toolset for process memory audit
+# Enable bash with kubectl exec in allow list
+# Disable kubernetes toolsets to force use of bash for in-container inspection
+toolsets:
+  bash:
+    enabled: true
+    config:
+      allow:
+        - "kubectl exec"
+        - "kubectl get"
+      deny: []
+  kubernetes/core:
+    enabled: true  # Keep basic k8s tools for context
+  kubernetes/logs:
+    enabled: false
+  kubernetes/live-metrics:
+    enabled: false
+  kubernetes/kube-prometheus-stack:
+    enabled: false
+  kubernetes/krew-extras:
+    enabled: false

tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/manifest.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: file-processor
+  namespace: app-212
+  labels:
+    app: file-processor
+spec:
+  containers:
+    - name: worker
+      image: alpine:3.19
+      command:
+        - /bin/sh
+        - -c
+        - |
+          mkdir -p /var/data
+
+          # Create 90 small files (1KB each)
+          for i in $(seq 1 90); do
+            dd if=/dev/zero of=/var/data/small_$i.dat bs=1K count=1 2>/dev/null
+          done
+
+          # Create 10 large files (15MB each) - use /dev/zero for speed
+          for i in $(seq 91 100); do
+            dd if=/dev/zero of=/var/data/large_$i.dat bs=1M count=15 2>/dev/null
+          done
+
+          echo "Files created successfully"
+          ls -la /var/data | head -20
+
+          # Keep container running
+          sleep 3600
+      resources:
+        limits:
+          memory: "256Mi"
+          cpu: "200m"
+        requests:
+          memory: "128Mi"
+          cpu: "100m"

tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/test_case.yaml

@@ -0,0 +1,71 @@
+user_prompt: "The file-processor pod in namespace app-212 might have disk space issues. Check which files in /var/data are larger than 10MB and report the total disk usage of that directory."
+
+include_tool_calls: true
+
+expected_output:
+  - "Must call bash tool with kubectl exec"
+  - "Must identify files larger than 10MB (should find files like large_91.dat through large_100.dat)"
+  - "Must report disk usage information"
+
+description: |
+  Tests that Holmes can use bash to run kubectl exec and inspect filesystem inside a container.
+  The pod has 100 files in /var/data: 90 small files (1KB each) and 10 large files (15MB each).
+  Holmes must use bash with 'kubectl exec' since no existing toolset can access container filesystems.
+
+tags:
+  - kubernetes
+  - regression
+  - bash-resource-checker
+
+before_test: |
+  kubectl create namespace app-212 --dry-run=client -o yaml | kubectl apply -f -
+  kubectl apply -f manifest.yaml
+
+  # Wait for pod to be ready
+  POD_READY=false
+  for i in {1..60}; do
+    if kubectl wait --for=condition=ready pod/file-processor -n app-212 --timeout=2s 2>/dev/null; then
+      echo "Pod ready!"
+      POD_READY=true
+      break
+    fi
+    sleep 1
+  done
+
+  if [ "$POD_READY" = false ]; then
+    echo "Pod not ready after 60 seconds"
+    kubectl get pods -n app-212
+    kubectl describe pod/file-processor -n app-212
+    exit 1
+  fi
+
+  # Wait for files to be created
+  FILES_READY=false
+  for i in {1..30}; do
+    FILE_COUNT=$(kubectl exec -n app-212 file-processor -- ls /var/data 2>/dev/null | wc -l)
+    echo "Attempt $i: $FILE_COUNT files in /var/data"
+    if [ "$FILE_COUNT" -ge 100 ]; then
+      echo "All 100 files created!"
+      FILES_READY=true
+      break
+    fi
+    sleep 2
+  done
+
+  if [ "$FILES_READY" = false ]; then
+    echo "Files not created properly"
+    kubectl exec -n app-212 file-processor -- ls -la /var/data
+    exit 1
+  fi
+
+  # Verify large files exist
+  LARGE_COUNT=$(kubectl exec -n app-212 file-processor -- find /var/data -size +10M 2>/dev/null | wc -l)
+  echo "Large files (>10MB): $LARGE_COUNT"
+  if [ "$LARGE_COUNT" -lt 10 ]; then
+    echo "Not enough large files created"
+    exit 1
+  fi
+  echo "Setup complete"
+
+after_test: |
+  kubectl delete namespace app-212 --ignore-not-found

tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/toolsets.yaml

@@ -0,0 +1,20 @@
+# Bash toolset for container disk audit
+# Enable bash with kubectl exec in allow list
+toolsets:
+  bash:
+    enabled: true
+    config:
+      allow:
+        - "kubectl exec"
+        - "kubectl get"
+      deny: []
+  kubernetes/core:
+    enabled: true
+  kubernetes/logs:
+    enabled: false
+  kubernetes/live-metrics:
+    enabled: false
+  kubernetes/kube-prometheus-stack:
+    enabled: false
+  kubernetes/krew-extras:
+    enabled: false

tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/manifest.yaml

@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: connection-pool
+  namespace: app-213
+  labels:
+    app: connection-pool
+spec:
+  containers:
+    - name: worker
+      image: alpine:3.19
+      command:
+        - /bin/sh
+        - -c
+        - |
+          # Install netcat and iproute2 for ss
+          apk add --no-cache netcat-openbsd iproute2 >/dev/null 2>&1
+
+          # Create 50 TCP listeners on ports 8001-8050
+          for port in $(seq 8001 8050); do
+            # Use while loop to keep listener alive after connection
+            while true; do nc -l -p $port; done &
+          done
+
+          echo "Created 50 TCP listeners"
+
+          # Wait a moment for listeners to start
+          sleep 2
+
+          # Create connections to each listener (these will be ESTABLISHED)
+          for port in $(seq 8001 8050); do
+            # Keep connection open by sleeping after connect
+            (sleep 3600 | nc 127.0.0.1 $port) &
+          done
+
+          echo "Created 50 client connections"
+
+          # Show connection summary
+          sleep 2
+          echo "Connection states:"
+          ss -tan | head -20
+
+          # Keep container running
+          sleep 3600
+      resources:
+        limits:
+          memory: "128Mi"
+          cpu: "200m"
+        requests:
+          memory: "64Mi"
+          cpu: "100m"

tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/test_case.yaml

@@ -0,0 +1,64 @@
+user_prompt: "The connection-pool pod in namespace app-213 may have connection leaks. Check how many TCP connections it has and group them by state (ESTABLISHED, TIME_WAIT, LISTEN, etc.)."
+
+include_tool_calls: true
+
+expected_output:
+  - "Must call bash tool with kubectl exec"
+  - "Must report TCP connection states"
+  - "Must identify LISTEN connections on ports 8001-8050"
+  - "Must identify ESTABLISHED connections"
+
+description: |
+  Tests that Holmes can use bash to run kubectl exec and inspect network connections inside a container.
+  The pod creates 50 TCP listeners and 50 established connections.
+  Holmes must use bash with 'kubectl exec' to run ss/netstat since no toolset inspects container network state.
+
+tags:
+  - kubernetes
+  - regression
+  - bash-resource-checker
+
+before_test: |
+  kubectl create namespace app-213 --dry-run=client -o yaml | kubectl apply -f -
+  kubectl apply -f manifest.yaml
+
+  # Wait for pod to be ready
+  POD_READY=false
+  for i in {1..60}; do
+    if kubectl wait --for=condition=ready pod/connection-pool -n app-213 --timeout=2s 2>/dev/null; then
+      echo "Pod ready!"
+      POD_READY=true
+      break
+    fi
+    sleep 1
+  done
+
+  if [ "$POD_READY" = false ]; then
+    echo "Pod not ready after 60 seconds"
+    kubectl get pods -n app-213
+    kubectl describe pod/connection-pool -n app-213
+    exit 1
+  fi
+
+  # Wait for connections to be established
+  CONN_READY=false
+  for i in {1..30}; do
+    LISTEN_COUNT=$(kubectl exec -n app-213 connection-pool -- ss -tln 2>/dev/null | grep -c LISTEN || echo "0")
+    echo "Attempt $i: $LISTEN_COUNT LISTEN connections"
+    if [ "$LISTEN_COUNT" -ge 40 ]; then
+      echo "Connections established!"
+      CONN_READY=true
+      break
+    fi
+    sleep 2
+  done
+
+  if [ "$CONN_READY" = false ]; then
+    echo "Connections not established properly"
+    kubectl exec -n app-213 connection-pool -- ss -tan
+    exit 1
+  fi
+  echo "Setup complete"
+
+after_test: |
+  kubectl delete namespace app-213 --ignore-not-found