diff --git a/pyproject.toml b/pyproject.toml index 96ea9db732..ea5862a449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,7 @@ markers = [ "elasticsearch: Tests for Elasticsearch/OpenSearch toolset functionality", "integration: Integration tests requiring external services (e.g., running server, real APIs)", "confluence: Tests for Confluence page fetching toolset", + "bash-resource-checker: Tests for bash toolset with kubectl exec to check resources inside containers", # REGRESSION TEST SELECTION CRITERIA: # - Must pass 30+ iterations reliably with Sonnet-4.5 model # - No external API dependencies (Loki OK, NewRelic/DataDog excluded) diff --git a/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/manifest.yaml new file mode 100644 index 0000000000..e2f5530def --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/manifest.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: data-cruncher + namespace: app-211 + labels: + app: data-cruncher +spec: + containers: + - name: worker + image: python:3.11-alpine + command: + - /bin/sh + - -c + - | + # Create 95 normal sleep processes (minimal memory) + for i in $(seq 1 95); do + sleep 3600 & + done + + # Create 5 memory-hungry Python processes (~20MB each) + for i in $(seq 1 5); do + python3 -c " + import time + # Allocate ~20MB of memory + data = bytearray(20 * 1024 * 1024) + # Fill with data to ensure it's allocated + for j in range(0, len(data), 4096): + data[j] = 0xff + time.sleep(3600) + " & + done + + # Keep container running + sleep 3600 + resources: + limits: + memory: "256Mi" + cpu: "200m" + requests: + memory: "128Mi" + cpu: "100m" diff --git a/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/test_case.yaml new file mode 100644 index 0000000000..cd2cecb31e --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/test_case.yaml @@ -0,0 +1,61 @@ +user_prompt: "The data-cruncher pod in namespace app-211 is consuming too much memory. Check which processes inside the container are using the most memory and identify any that stand out." + +include_tool_calls: true + +expected_output: + - "Must call bash tool with kubectl exec" + - "Must identify Python processes as consuming the most memory" + - "Must report processes sorted by memory usage or identify top consumers" + +description: | + Tests that Holmes can use bash to run kubectl exec and inspect processes inside a container. + The pod runs 95 sleep processes plus 5 Python processes that allocate ~20MB each. + Holmes must use bash with 'kubectl exec' since no existing toolset can inspect in-container processes. + +tags: + - kubernetes + - regression + - bash-resource-checker + +before_test: | + kubectl create namespace app-211 --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f manifest.yaml + + # Wait for pod to be ready + POD_READY=false + for i in {1..60}; do + if kubectl wait --for=condition=ready pod/data-cruncher -n app-211 --timeout=2s 2>/dev/null; then + echo "Pod ready!" + POD_READY=true + break + fi + sleep 1 + done + + if [ "$POD_READY" = false ]; then + echo "Pod not ready after 60 seconds" + kubectl get pods -n app-211 + kubectl describe pod/data-cruncher -n app-211 + exit 1 + fi + + # Verify processes are running inside the container + PROC_COUNT=$(kubectl exec -n app-211 data-cruncher -- ps aux 2>/dev/null | wc -l) + echo "Process count in container: $PROC_COUNT" + if [ "$PROC_COUNT" -lt 50 ]; then + echo "Not enough processes running in container" + kubectl exec -n app-211 data-cruncher -- ps aux + exit 1 + fi + + # Verify Python processes are running + PYTHON_COUNT=$(kubectl exec -n app-211 data-cruncher -- ps aux 2>/dev/null | grep -c "python3" || echo "0") + echo "Python process count: $PYTHON_COUNT" + if [ "$PYTHON_COUNT" -lt 5 ]; then + echo "Not enough Python processes running" + exit 1 + fi + echo "Setup complete - container has $PROC_COUNT processes including $PYTHON_COUNT Python processes" + +after_test: | + kubectl delete namespace app-211 --ignore-not-found diff --git a/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/toolsets.yaml new file mode 100644 index 0000000000..4b9fcb0b8d --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/211_bash_process_memory_audit/toolsets.yaml @@ -0,0 +1,21 @@ +# Bash toolset for process memory audit +# Enable bash with kubectl exec in allow list +# Disable kubernetes toolsets to force use of bash for in-container inspection +toolsets: + bash: + enabled: true + config: + allow: + - "kubectl exec" + - "kubectl get" + deny: [] + kubernetes/core: + enabled: true # Keep basic k8s tools for context + kubernetes/logs: + enabled: false + kubernetes/live-metrics: + enabled: false + kubernetes/kube-prometheus-stack: + enabled: false + kubernetes/krew-extras: + enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/manifest.yaml new file mode 100644 index 0000000000..ef61630de8 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/manifest.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Pod +metadata: + name: file-processor + namespace: app-212 + labels: + app: file-processor +spec: + containers: + - name: worker + image: alpine:3.19 + command: + - /bin/sh + - -c + - | + mkdir -p /var/data + + # Create 90 small files (1KB each) + for i in $(seq 1 90); do + dd if=/dev/zero of=/var/data/small_$i.dat bs=1K count=1 2>/dev/null + done + + # Create 10 large files (15MB each) - use /dev/zero for speed + for i in $(seq 91 100); do + dd if=/dev/zero of=/var/data/large_$i.dat bs=1M count=15 2>/dev/null + done + + echo "Files created successfully" + ls -la /var/data | head -20 + + # Keep container running + sleep 3600 + resources: + limits: + memory: "256Mi" + cpu: "200m" + requests: + memory: "128Mi" + cpu: "100m" diff --git a/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/test_case.yaml new file mode 100644 index 0000000000..ae2d6209b0 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/test_case.yaml @@ -0,0 +1,71 @@ +user_prompt: "The file-processor pod in namespace app-212 might have disk space issues. Check which files in /var/data are larger than 10MB and report the total disk usage of that directory." + +include_tool_calls: true + +expected_output: + - "Must call bash tool with kubectl exec" + - "Must identify files larger than 10MB (should find files like large_91.dat through large_100.dat)" + - "Must report disk usage information" + +description: | + Tests that Holmes can use bash to run kubectl exec and inspect filesystem inside a container. + The pod has 100 files in /var/data: 90 small files (1KB each) and 10 large files (15MB each). + Holmes must use bash with 'kubectl exec' since no existing toolset can access container filesystems. + +tags: + - kubernetes + - regression + - bash-resource-checker + +before_test: | + kubectl create namespace app-212 --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f manifest.yaml + + # Wait for pod to be ready + POD_READY=false + for i in {1..60}; do + if kubectl wait --for=condition=ready pod/file-processor -n app-212 --timeout=2s 2>/dev/null; then + echo "Pod ready!" + POD_READY=true + break + fi + sleep 1 + done + + if [ "$POD_READY" = false ]; then + echo "Pod not ready after 60 seconds" + kubectl get pods -n app-212 + kubectl describe pod/file-processor -n app-212 + exit 1 + fi + + # Wait for files to be created + FILES_READY=false + for i in {1..30}; do + FILE_COUNT=$(kubectl exec -n app-212 file-processor -- ls /var/data 2>/dev/null | wc -l) + echo "Attempt $i: $FILE_COUNT files in /var/data" + if [ "$FILE_COUNT" -ge 100 ]; then + echo "All 100 files created!" + FILES_READY=true + break + fi + sleep 2 + done + + if [ "$FILES_READY" = false ]; then + echo "Files not created properly" + kubectl exec -n app-212 file-processor -- ls -la /var/data + exit 1 + fi + + # Verify large files exist + LARGE_COUNT=$(kubectl exec -n app-212 file-processor -- find /var/data -size +10M 2>/dev/null | wc -l) + echo "Large files (>10MB): $LARGE_COUNT" + if [ "$LARGE_COUNT" -lt 10 ]; then + echo "Not enough large files created" + exit 1 + fi + echo "Setup complete" + +after_test: | + kubectl delete namespace app-212 --ignore-not-found diff --git a/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/toolsets.yaml new file mode 100644 index 0000000000..154aaa001c --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/212_bash_container_disk_audit/toolsets.yaml @@ -0,0 +1,20 @@ +# Bash toolset for container disk audit +# Enable bash with kubectl exec in allow list +toolsets: + bash: + enabled: true + config: + allow: + - "kubectl exec" + - "kubectl get" + deny: [] + kubernetes/core: + enabled: true + kubernetes/logs: + enabled: false + kubernetes/live-metrics: + enabled: false + kubernetes/kube-prometheus-stack: + enabled: false + kubernetes/krew-extras: + enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/manifest.yaml new file mode 100644 index 0000000000..87afae3493 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/manifest.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Pod +metadata: + name: connection-pool + namespace: app-213 + labels: + app: connection-pool +spec: + containers: + - name: worker + image: alpine:3.19 + command: + - /bin/sh + - -c + - | + # Install netcat and iproute2 for ss + apk add --no-cache netcat-openbsd iproute2 >/dev/null 2>&1 + + # Create 50 TCP listeners on ports 8001-8050 + for port in $(seq 8001 8050); do + # Use while loop to keep listener alive after connection + while true; do nc -l -p $port; done & + done + + echo "Created 50 TCP listeners" + + # Wait a moment for listeners to start + sleep 2 + + # Create connections to each listener (these will be ESTABLISHED) + for port in $(seq 8001 8050); do + # Keep connection open by sleeping after connect + (sleep 3600 | nc 127.0.0.1 $port) & + done + + echo "Created 50 client connections" + + # Show connection summary + sleep 2 + echo "Connection states:" + ss -tan | head -20 + + # Keep container running + sleep 3600 + resources: + limits: + memory: "128Mi" + cpu: "200m" + requests: + memory: "64Mi" + cpu: "100m" diff --git a/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/test_case.yaml new file mode 100644 index 0000000000..7d77949362 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/test_case.yaml @@ -0,0 +1,64 @@ +user_prompt: "The connection-pool pod in namespace app-213 may have connection leaks. Check how many TCP connections it has and group them by state (ESTABLISHED, TIME_WAIT, LISTEN, etc.)." + +include_tool_calls: true + +expected_output: + - "Must call bash tool with kubectl exec" + - "Must report TCP connection states" + - "Must identify LISTEN connections on ports 8001-8050" + - "Must identify ESTABLISHED connections" + +description: | + Tests that Holmes can use bash to run kubectl exec and inspect network connections inside a container. + The pod creates 50 TCP listeners and 50 established connections. + Holmes must use bash with 'kubectl exec' to run ss/netstat since no toolset inspects container network state. + +tags: + - kubernetes + - regression + - bash-resource-checker + +before_test: | + kubectl create namespace app-213 --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f manifest.yaml + + # Wait for pod to be ready + POD_READY=false + for i in {1..60}; do + if kubectl wait --for=condition=ready pod/connection-pool -n app-213 --timeout=2s 2>/dev/null; then + echo "Pod ready!" + POD_READY=true + break + fi + sleep 1 + done + + if [ "$POD_READY" = false ]; then + echo "Pod not ready after 60 seconds" + kubectl get pods -n app-213 + kubectl describe pod/connection-pool -n app-213 + exit 1 + fi + + # Wait for connections to be established + CONN_READY=false + for i in {1..30}; do + LISTEN_COUNT=$(kubectl exec -n app-213 connection-pool -- ss -tln 2>/dev/null | grep -c LISTEN || echo "0") + echo "Attempt $i: $LISTEN_COUNT LISTEN connections" + if [ "$LISTEN_COUNT" -ge 40 ]; then + echo "Connections established!" + CONN_READY=true + break + fi + sleep 2 + done + + if [ "$CONN_READY" = false ]; then + echo "Connections not established properly" + kubectl exec -n app-213 connection-pool -- ss -tan + exit 1 + fi + echo "Setup complete" + +after_test: | + kubectl delete namespace app-213 --ignore-not-found diff --git a/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/toolsets.yaml new file mode 100644 index 0000000000..335291dd5e --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/213_bash_network_connection_audit/toolsets.yaml @@ -0,0 +1,20 @@ +# Bash toolset for network connection audit +# Enable bash with kubectl exec in allow list +toolsets: + bash: + enabled: true + config: + allow: + - "kubectl exec" + - "kubectl get" + deny: [] + kubernetes/core: + enabled: true + kubernetes/logs: + enabled: false + kubernetes/live-metrics: + enabled: false + kubernetes/kube-prometheus-stack: + enabled: false + kubernetes/krew-extras: + enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/manifest.yaml new file mode 100644 index 0000000000..ea37e2c484 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/manifest.yaml @@ -0,0 +1,58 @@ +# StatefulSet with 100 cache-worker pods +# Pods with disk issues are scattered non-sequentially using formula: (ordinal * 7 + 3) % 11 == 0 +# Bad pods: 1, 12, 23, 34, 45, 56, 67, 78, 89 (9 scattered pods) +# Holmes MUST check every pod to find all of them - can't guess or sample +apiVersion: v1 +kind: Service +metadata: + name: cache-worker + namespace: app-214 +spec: + clusterIP: None + selector: + app: cache-worker +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: cache-worker + namespace: app-214 +spec: + serviceName: cache-worker + replicas: 100 + podManagementPolicy: Parallel + selector: + matchLabels: + app: cache-worker + template: + metadata: + labels: + app: cache-worker + spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c"] + args: + - | + POD_ID=$(hostname | sed 's/cache-worker-//') + # Non-sequential pattern: (ordinal * 7 + 3) % 11 == 0 means disk issue + # This creates scattered bad pods: 1, 12, 23, 34, 45, 56, 67, 78, 89 + HASH=$(( (POD_ID * 7 + 3) % 11 )) + if [ "$HASH" -eq 0 ]; then + # This pod has disk issue - create 60MB file + dd if=/dev/zero of=/tmp/cache.dat bs=1M count=60 2>/dev/null + echo "DISK_STATUS=CRITICAL" > /tmp/status + else + # Normal pod - small file + dd if=/dev/zero of=/tmp/cache.dat bs=1K count=100 2>/dev/null + echo "DISK_STATUS=OK" > /tmp/status + fi + sleep 3600 + resources: + limits: + memory: "128Mi" + cpu: "50m" + requests: + memory: "32Mi" + cpu: "10m" diff --git a/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/test_case.yaml new file mode 100644 index 0000000000..c72b522dcf --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/test_case.yaml @@ -0,0 +1,73 @@ +user_prompt: "Check disk usage inside all cache-worker pods in namespace app-214. List ALL pods that have /tmp usage over 50MB." + +include_tool_calls: true + +expected_output: + - "Must identify ALL 9 pods with high disk usage: cache-worker-1, cache-worker-12, cache-worker-23, cache-worker-34, cache-worker-45, cache-worker-56, cache-worker-67, cache-worker-78, cache-worker-89" + - "Must not miss any of the 9 scattered pods" + +description: | + TDD test: 100 pods with disk issues scattered non-sequentially. + Bad pods use formula (ordinal*7+3)%11==0: pods 1, 12, 23, 34, 45, 56, 67, 78, 89 + Holmes CANNOT guess which pods are bad - must check every pod. + Without bash loops, checking 100 pods individually will fail/timeout. + +tags: + - kubernetes + - regression + - bash-resource-checker + +before_test: | + kubectl create namespace app-214 --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f manifest.yaml + + # Wait for all pods to be ready + PODS_READY=false + for i in {1..180}; do + READY_COUNT=$(kubectl get pods -n app-214 -l app=cache-worker --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l | tr -d ' ') + echo "Attempt $i: $READY_COUNT/100 pods ready" + if [ "$READY_COUNT" -ge 100 ]; then + echo "All 100 pods ready!" + PODS_READY=true + break + fi + sleep 1 + done + + if [ "$PODS_READY" = false ]; then + echo "Not all pods ready" + kubectl get pods -n app-214 + exit 1 + fi + + # Wait for files to be created - check pod 1 (one of the bad pods) + FILES_READY=false + for i in {1..30}; do + LARGE_SIZE=$(kubectl exec -n app-214 cache-worker-1 -- du -sm /tmp 2>/dev/null | cut -f1 || echo "0") + echo "Attempt $i: cache-worker-1 /tmp size: ${LARGE_SIZE}MB" + if [ "$LARGE_SIZE" -gt 50 ]; then + echo "Large files created in bad pod!" + FILES_READY=true + break + fi + sleep 2 + done + + if [ "$FILES_READY" = false ]; then + echo "Files not created properly in bad pods" + kubectl exec -n app-214 cache-worker-1 -- du -sh /tmp + exit 1 + fi + + # Verify a good pod has small files + GOOD_SIZE=$(kubectl exec -n app-214 cache-worker-0 -- du -sm /tmp 2>/dev/null | cut -f1 || echo "999") + echo "cache-worker-0 (good pod) /tmp size: ${GOOD_SIZE}MB" + if [ "$GOOD_SIZE" -gt 10 ]; then + echo "ERROR: Good pod has unexpectedly large files" + exit 1 + fi + + echo "Setup complete - bad pods scattered at: 1, 12, 23, 34, 45, 56, 67, 78, 89" + +after_test: | + kubectl delete namespace app-214 --ignore-not-found diff --git a/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/toolsets.yaml new file mode 100644 index 0000000000..4e5f11371b --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/214_bash_multipod_disk_check/toolsets.yaml @@ -0,0 +1,20 @@ +# Bash toolset for multi-pod disk check +# Enable bash with kubectl exec in allow list +toolsets: + bash: + enabled: true + config: + allow: + - "kubectl exec" + - "kubectl get" + deny: [] + kubernetes/core: + enabled: true + kubernetes/logs: + enabled: false + kubernetes/live-metrics: + enabled: false + kubernetes/kube-prometheus-stack: + enabled: false + kubernetes/krew-extras: + enabled: false diff --git a/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/manifest.yaml new file mode 100644 index 0000000000..0c4f13674b --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/manifest.yaml @@ -0,0 +1,273 @@ +# Database pod and service +--- +apiVersion: v1 +kind: Pod +metadata: + name: database + namespace: app-215 + labels: + app: database + role: db +spec: + containers: + - name: db + image: alpine:3.19 + command: + - /bin/sh + - -c + - | + apk add --no-cache netcat-openbsd >/dev/null 2>&1 + while true; do nc -l -p 5432; done + ports: + - containerPort: 5432 + resources: + limits: + memory: "64Mi" + cpu: "100m" +--- +apiVersion: v1 +kind: Service +metadata: + name: database + namespace: app-215 +spec: + selector: + app: database + ports: + - port: 5432 + targetPort: 5432 +--- +# NetworkPolicy: Only allow pods with label "db-access: allowed" to reach database +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: database-access + namespace: app-215 +spec: + podSelector: + matchLabels: + app: database + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + db-access: allowed + ports: + - protocol: TCP + port: 5432 +--- +# StatefulSet: 90 worker pods WITH db-access label (can connect) +apiVersion: v1 +kind: Service +metadata: + name: api-worker + namespace: app-215 +spec: + clusterIP: None + selector: + app: api-worker +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: api-worker + namespace: app-215 +spec: + serviceName: api-worker + replicas: 90 + podManagementPolicy: Parallel + selector: + matchLabels: + app: api-worker + db-access: allowed + template: + metadata: + labels: + app: api-worker + db-access: allowed + spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" + requests: + memory: "32Mi" + cpu: "10m" +--- +# 10 individual pods WITHOUT db-access label (blocked by NetworkPolicy) +# Named api-worker-90 through api-worker-99 to continue the sequence +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-90 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-91 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-92 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-93 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-94 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-95 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-96 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-97 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-98 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: api-worker-99 + namespace: app-215 + labels: + app: api-worker +spec: + containers: + - name: worker + image: alpine:3.19 + command: ["/bin/sh", "-c", "apk add --no-cache netcat-openbsd >/dev/null 2>&1; sleep 3600"] + resources: + limits: + memory: "64Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/test_case.yaml new file mode 100644 index 0000000000..e547567a37 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/test_case.yaml @@ -0,0 +1,73 @@ +user_prompt: "Verify that all api-worker pods in namespace app-215 can reach the database service on port 5432. Report which pods have connectivity issues." + +include_tool_calls: true + +expected_output: + - "Must identify api-worker-90 through api-worker-99 as having connectivity issues" + - "Must check all 100 pods" + +description: | + TDD test: 100 worker pods deployed - too many for individual kubectl exec calls. + Holmes needs to write a bash script with a loop, but bash toolset blocks loops. + Pods 0-89 can reach database, pods 90-99 are blocked by NetworkPolicy. + +tags: + - kubernetes + - regression + - bash-resource-checker + +before_test: | + kubectl create namespace app-215 --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f manifest.yaml + + # Wait for all pods to be ready + PODS_READY=false + for i in {1..180}; do + READY_COUNT=$(kubectl get pods -n app-215 --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l | tr -d ' ') + echo "Attempt $i: $READY_COUNT/101 pods ready (100 workers + 1 db)" + if [ "$READY_COUNT" -ge 101 ]; then + echo "All pods ready!" + PODS_READY=true + break + fi + sleep 1 + done + + if [ "$PODS_READY" = false ]; then + echo "Not all pods ready" + kubectl get pods -n app-215 + exit 1 + fi + + # Verify database service is accessible from allowed pod (pod 0) + DB_CHECK=false + for i in {1..30}; do + if kubectl exec -n app-215 api-worker-0 -- nc -zv -w 2 database 5432 2>&1 | grep -q "open\|succeeded"; then + echo "Database connectivity verified from allowed pod" + DB_CHECK=true + break + fi + sleep 1 + done + + if [ "$DB_CHECK" = false ]; then + echo "Database service not accessible from allowed pod" + kubectl exec -n app-215 api-worker-0 -- nc -zv -w 2 database 5432 + exit 1 + fi + + # Verify blocked pod cannot connect (pod 99) + BLOCK_CHECK=false + if ! kubectl exec -n app-215 api-worker-99 -- nc -zv -w 2 database 5432 2>&1 | grep -q "open\|succeeded"; then + echo "NetworkPolicy blocking verified - api-worker-99 cannot reach database" + BLOCK_CHECK=true + fi + + if [ "$BLOCK_CHECK" = false ]; then + echo "WARNING: NetworkPolicy may not be working - blocked pod can reach database" + fi + + echo "Setup complete" + +after_test: | + kubectl delete namespace app-215 --ignore-not-found diff --git a/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/toolsets.yaml b/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/toolsets.yaml new file mode 100644 index 0000000000..5f5e9da01f --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/215_bash_multipod_connectivity/toolsets.yaml @@ -0,0 +1,20 @@ +# Bash toolset for multi-pod connectivity test +# Enable bash with kubectl exec in allow list +toolsets: + bash: + enabled: true + config: + allow: + - "kubectl exec" + - "kubectl get" + deny: [] + kubernetes/core: + enabled: true + kubernetes/logs: + enabled: false + kubernetes/live-metrics: + enabled: false + kubernetes/kube-prometheus-stack: + enabled: false + kubernetes/krew-extras: + enabled: false