new evals for logging and more

pyproject.toml

@@ -96,8 +96,10 @@ markers = [
     "k8s-misconfig: Tests involving Kubernetes misconfigurations",
     "chain-of-causation: Tests involving chain-of-causation analysis",
     "slackbot: Tests involving Slack bot functionality",
+    "understanding-real-intent: Tests that require understanding the user's real intent behind a question, not just giving a technically correct answer that is not useful",
     "counting: Ask holmes to count kubernetes/cloud resources"
 ]
+
 addopts = [
     "--cov-config=pyproject.toml",
     "--cov=holmes",

tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt

@@ -0,0 +1,2 @@
+{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"video-streamer"}}
+{"schema_version": "robusta:v1.0.0", "status": "error", "error": "Command `kubectl get -A --show-labels -o wide pod | grep video-streamer` failed with return code 1\nOutput:\n", "return_code": 1, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep video-streamer", "params": {"kind": "pod", "keyword": "video-streamer"}}

tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt

@@ -0,0 +1,3 @@
+{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"pod","namespace":"app-57"}}
+{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide pod -n app-57", "params": {"kind": "pod", "namespace": "app-57"}}
+No resources found in app-57 namespace.

tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/manifest.yaml

@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-57
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: video-streamer
+  namespace: test-57
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: video-streamer
+  template:
+    metadata:
+      labels:
+        app: video-streamer
+    spec:
+      containers:
+      - name: app-container
+        image: busybox:1.35
+        command: ["/bin/sh"]
+        args:
+        - -c
+        - |
+          echo "$(date) INFO: Application starting up"
+          echo "$(date) INFO: Connecting to database"
+          echo "$(date) WARN: Database connection slow"
+          echo "$(date) ERROR: Failed to connect to database: connection timeout"
+          echo "$(date) ERROR: Retrying database connection..."
+          echo "$(date) ERROR: Database connection failed after 3 retries"
+          echo "$(date) INFO: Switching to backup database"
+          echo "$(date) ERROR: Backup database also unavailable"
+          echo "$(date) ERROR: Application cannot start without database"
+          echo "$(date) FATAL: Shutting down application"
+          while true; do sleep 3600; done
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "50m"
+          limits:
+            memory: "32Mi"
+            cpu: "50m"

tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml

@@ -0,0 +1,10 @@
+# we ask about app-57 namespace but the deployment is in test-57 namespace
+user_prompt: "get last logs from video-streamer in app-57 namespace"
+expected_output: "video-streamer not found in namespace app-57, did you mean namespace test-57?"
+before_test: 'kubectl apply -f ./manifest.yaml'
+after_test: 'kubectl delete -f ./manifest.yaml'
+evaluation:
+  correctness: 0
+generate_mocks: False
+tags:
+  - logs

tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt

@@ -0,0 +1,7 @@
+{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"payment-service-7cb79f8c87-z96z8","namespace":"production-62","filter":"ERROR","limit":10}}
+{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "production-62", "pod_name": "payment-service-7cb79f8c87-z96z8", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}}
+Fri Jul 11 13:23:24 UTC 2025 ERROR: Failed to connect to database: connection timeout
+Fri Jul 11 13:23:24 UTC 2025 ERROR: Retrying database connection...
+Fri Jul 11 13:23:24 UTC 2025 ERROR: Database connection failed after 3 retries
+Fri Jul 11 13:23:24 UTC 2025 ERROR: Backup database also unavailable
+Fri Jul 11 13:23:24 UTC 2025 ERROR: Application cannot start without database

tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt

@@ -0,0 +1,3 @@
+{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"payment-service"}}
+{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep payment-service", "params": {"kind": "pod", "keyword": "payment-service"}}
+production-62   payment-service-7cb79f8c87-z96z8                             1/1     Running                       0                  14s    10.0.2.187     gke-natan-develop-pool-2-41363a6e-yi0r   <none>           <none>            app=payment-service,pod-template-hash=7cb79f8c87

tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml

@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: production-62
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: payment-service
+  namespace: production-62
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: payment-service
+  template:
+    metadata:
+      labels:
+        app: payment-service
+    spec:
+      containers:
+      - name: app-container
+        image: busybox:1.35
+        command: ["/bin/sh"]
+        args:
+        - -c
+        - |
+          echo "$(date) INFO: Application starting up"
+          echo "$(date) INFO: Connecting to database"
+          echo "$(date) WARN: Database connection slow"
+          echo "$(date) ERROR: Failed to connect to database: connection timeout"
+          echo "$(date) ERROR: Retrying database connection..."
+          echo "$(date) ERROR: Database connection failed after 3 retries"
+          echo "$(date) INFO: Switching to backup database"
+          echo "$(date) ERROR: Backup database also unavailable"
+          echo "$(date) ERROR: Application cannot start without database"
+          echo "$(date) FATAL: Shutting down application"
+          while true; do sleep 3600; done
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "50m"
+          limits:
+            memory: "32Mi"
+            cpu: "50m"

tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml

@@ -0,0 +1,23 @@
+user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-62 namespace"
+expected_output: |
+  The following logs are shown:
+    ERROR: Failed to connect to database: connection timeout
+    ERROR: Retrying database connection...
+    ERROR: Database connection failed after 3 retries
+    ERROR: Backup database also unavailable
+    ERROR: Application cannot start without database
+
+  The following log is NOT shown:
+    INFO: Application starting up
+    WARN: Database connection slow
+
+  The following log MAY be shown (it does not impact the correctness of the test):
+    FATAL: Shutting down application
+
+before_test: 'kubectl apply -f ./manifest.yaml'
+after_test: 'kubectl delete -f ./manifest.yaml'
+evaluation:
+  correctness: 1
+generate_mocks: True
+tags:
+  - logs

tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt

@@ -0,0 +1,2 @@
+{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"user-api-6969cc599-n99kp","namespace":"staging-63","filter":"ERROR","limit":10}}
+{"schema_version": "robusta:v1.0.0", "status": "no_data", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "staging-63", "pod_name": "user-api-6969cc599-n99kp", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}}

tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt

@@ -0,0 +1,3 @@
+{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"user-api"}}
+{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep user-api", "params": {"kind": "pod", "keyword": "user-api"}}
+staging-63      user-api-6969cc599-n99kp                                     1/1     Running                       0                  16s    10.0.2.184     gke-natan-develop-pool-2-41363a6e-yi0r   <none>           <none>            app=user-api,pod-template-hash=6969cc599

tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml

@@ -0,0 +1,50 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: staging-63
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: user-api
+  namespace: staging-63
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: user-api
+  template:
+    metadata:
+      labels:
+        app: user-api
+    spec:
+      containers:
+      - name: app-container
+        image: busybox:1.35
+        command: ["/bin/sh"]
+        args:
+        - -c
+        - |
+          echo "$(date) INFO: Application starting up"
+          echo "$(date) INFO: Loading configuration"
+          echo "$(date) INFO: Connecting to database"
+          echo "$(date) INFO: Database connection established"
+          echo "$(date) INFO: Running health checks"
+          echo "$(date) INFO: All systems operational"
+          echo "$(date) INFO: Ready to serve requests"
+          echo "$(date) DEBUG: Processing request batch 1"
+          echo "$(date) INFO: Request batch 1 completed successfully"
+          echo "$(date) DEBUG: Processing request batch 2"
+          echo "$(date) INFO: Request batch 2 completed successfully"
+          echo "$(date) INFO: Application running smoothly"
+          while true; do
+            echo "$(date) INFO: Heartbeat - application healthy"
+            sleep 60
+          done
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "50m"
+          limits:
+            memory: "32Mi"
+            cpu: "50m"

tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml

@@ -0,0 +1,9 @@
+user_prompt: "post last 10 logs that contain \"ERROR\" from user-api in staging-63 namespace"
+expected_output: "No ERROR logs found"
+before_test: 'kubectl apply -f ./manifest.yaml'
+after_test: 'kubectl delete -f ./manifest.yaml'
+evaluation:
+  correctness: 1
+generate_mocks: True
+tags:
+  - logs

tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt

@@ -0,0 +1,50 @@
+{"toolset_name":"kubernetes/core","tool_name":"kubectl_describe","match_params":{"kind":"deployment","name":"invoice-generator","namespace":"invoices-64"}}
+{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl describe deployment invoice-generator -n invoices-64", "params": {"kind": "deployment", "name": "invoice-generator", "namespace": "invoices-64"}}
+Name:                   invoice-generator
+Namespace:              invoices-64
+CreationTimestamp:      Fri, 11 Jul 2025 16:20:40 +0300
+Labels:                 app=invoice-generator
+                        version=v1.2.0
+Annotations:            deployment.kubernetes.io/revision: 1
+Selector:               app=invoice-generator
+Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
+StrategyType:           RollingUpdate
+MinReadySeconds:        0
+RollingUpdateStrategy:  25% max unavailable, 25% max surge
+Pod Template:
+  Labels:  app=invoice-generator
+  Containers:
+   invoice-generator:
+    Image:      busybox
+    Port:       8080/TCP
+    Host Port:  0/TCP
+    Command:
+      /bin/sh
+    Args:
+      -c
+      echo "$(date) INFO: Invoice generator starting up"
+      while true; do
+        echo "$(date) INFO: Generating invoice..."
+        sleep 5
+      done
+
+    Limits:
+      cpu:     50m
+      memory:  32Mi
+    Requests:
+      cpu:        50m
+      memory:     32Mi
+    Environment:  <none>
+    Mounts:       <none>
+  Volumes:        <none>
+Conditions:
+  Type           Status  Reason
+  ----           ------  ------
+  Available      True    MinimumReplicasAvailable
+  Progressing    True    NewReplicaSetAvailable
+OldReplicaSets:  <none>
+NewReplicaSet:   invoice-generator-c6c6d857 (1/1 replicas created)
+Events:
+  Type    Reason             Age   From                   Message
+  ----    ------             ----  ----                   -------
+  Normal  ScalingReplicaSet  18s   deployment-controller  Scaled up replica set invoice-generator-c6c6d857 from 0 to 1

tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt

@@ -0,0 +1,3 @@
+{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"deployment","keyword":"invoice-generator"}}
+{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide deployment | grep invoice-generator", "params": {"kind": "deployment", "keyword": "invoice-generator"}}
+invoices-64     invoice-generator                           1/1     1            1           14s    invoice-generator                                         busybox                                                                                                                                                                                                                                                                                                                                                                                                                                                                                app=invoice-generator                                                                                                                                      app=invoice-generator,version=v1.2.0

tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt

@@ -0,0 +1,3 @@
+{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"scaledobject","namespace":"invoices-64"}}
+{"schema_version": "robusta:v1.0.0", "status": "error", "error": "Command `kubectl get --show-labels -o wide scaledobject -n invoices-64` failed with return code 1\nOutput:\nerror: the server doesn't have a resource type \"scaledobject\"", "return_code": 1, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide scaledobject -n invoices-64", "params": {"kind": "scaledobject", "namespace": "invoices-64"}}
+error: the server doesn't have a resource type "scaledobject"

tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml

@@ -0,0 +1,64 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: invoices-64
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: invoice-generator
+  namespace: invoices-64
+  labels:
+    app: invoice-generator
+    version: v1.2.0
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: invoice-generator
+  template:
+    metadata:
+      labels:
+        app: invoice-generator
+    spec:
+      containers:
+      - name: invoice-generator
+        image: busybox
+        ports:
+        - containerPort: 8080
+        resources:
+          requests:
+            cpu: 50m
+            memory: 32Mi
+          limits:
+            cpu: 50m
+            memory: 32Mi
+        command: ["/bin/sh"]
+        args:
+        - -c
+        - |
+          echo "$(date) INFO: Invoice generator starting up"
+          while true; do
+            echo "$(date) INFO: Generating invoice..."
+            sleep 5
+          done
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: invoice-generator
+  namespace: invoices-64
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: invoice-generator
+  minReplicas: 1
+  maxReplicas: 5
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 60

tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml

@@ -0,0 +1,13 @@
+user_prompt: 'Why isnt KEDA scaling the invoice-generator deployment?'
+description: Verify Holmes goes beyond 'technically correct' answers and provides a useful answer based on the user's actual intent
+expected_output: |
+  - KEDA is not installed in your cluster OR you have no KEDA scaler defined for this resource (either answer is fine)
+  - But I found an HPA attached to the invoice-generator deployment
+  - Current CPU usage is 9%, which is below the 60% threshold configured in the HPA. That's why it isn't scaling up
+evaluation:
+  correctness: 0
+generate_mocks: False
+tags:
+  - understanding-real-intent
+before_test: 'kubectl apply -f ./manifest.yaml'
+after_test: 'kubectl delete -f ./manifest.yaml'