diff --git a/pyproject.toml b/pyproject.toml index e6e2bf18fb..68e929958a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,8 +96,10 @@ markers = [ "k8s-misconfig: Tests involving Kubernetes misconfigurations", "chain-of-causation: Tests involving chain-of-causation analysis", "slackbot: Tests involving Slack bot functionality", + "understanding-real-intent: Tests that require understanding the user's real intent behind a question, not just giving a technically correct answer that is not useful", "counting: Ask holmes to count kubernetes/cloud resources" ] + addopts = [ "--cov-config=pyproject.toml", "--cov=holmes", diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt new file mode 100644 index 0000000000..2d6ba3d297 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt @@ -0,0 +1,2 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"video-streamer"}} +{"schema_version": "robusta:v1.0.0", "status": "error", "error": "Command `kubectl get -A --show-labels -o wide pod | grep video-streamer` failed with return code 1\nOutput:\n", "return_code": 1, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep video-streamer", "params": {"kind": "pod", "keyword": "video-streamer"}} diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt new file mode 100644 index 0000000000..200220701a --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"pod","namespace":"app-57"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide pod -n app-57", "params": {"kind": "pod", "namespace": "app-57"}} +No resources found in app-57 namespace. diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/manifest.yaml new file mode 100644 index 0000000000..176ac521b7 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/manifest.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-57 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: video-streamer + namespace: test-57 +spec: + replicas: 1 + selector: + matchLabels: + app: video-streamer + template: + metadata: + labels: + app: video-streamer + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Connecting to database" + echo "$(date) WARN: Database connection slow" + echo "$(date) ERROR: Failed to connect to database: connection timeout" + echo "$(date) ERROR: Retrying database connection..." + echo "$(date) ERROR: Database connection failed after 3 retries" + echo "$(date) INFO: Switching to backup database" + echo "$(date) ERROR: Backup database also unavailable" + echo "$(date) ERROR: Application cannot start without database" + echo "$(date) FATAL: Shutting down application" + while true; do sleep 3600; done + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml new file mode 100644 index 0000000000..450bd5af37 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml @@ -0,0 +1,10 @@ +# we ask about app-57 namespace but the deployment is in test-57 namespace +user_prompt: "get last logs from video-streamer in app-57 namespace" +expected_output: "video-streamer not found in namespace app-57, did you mean namespace test-57?" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 0 +generate_mocks: False +tags: + - logs diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt new file mode 100644 index 0000000000..08bb9b743c --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt @@ -0,0 +1,7 @@ +{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"payment-service-7cb79f8c87-z96z8","namespace":"production-62","filter":"ERROR","limit":10}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "production-62", "pod_name": "payment-service-7cb79f8c87-z96z8", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} +Fri Jul 11 13:23:24 UTC 2025 ERROR: Failed to connect to database: connection timeout +Fri Jul 11 13:23:24 UTC 2025 ERROR: Retrying database connection... +Fri Jul 11 13:23:24 UTC 2025 ERROR: Database connection failed after 3 retries +Fri Jul 11 13:23:24 UTC 2025 ERROR: Backup database also unavailable +Fri Jul 11 13:23:24 UTC 2025 ERROR: Application cannot start without database diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt new file mode 100644 index 0000000000..9a45625a72 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"payment-service"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep payment-service", "params": {"kind": "pod", "keyword": "payment-service"}} +production-62 payment-service-7cb79f8c87-z96z8 1/1 Running 0 14s 10.0.2.187 gke-natan-develop-pool-2-41363a6e-yi0r app=payment-service,pod-template-hash=7cb79f8c87 diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml new file mode 100644 index 0000000000..744ef87b24 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: production-62 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: production-62 +spec: + replicas: 1 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Connecting to database" + echo "$(date) WARN: Database connection slow" + echo "$(date) ERROR: Failed to connect to database: connection timeout" + echo "$(date) ERROR: Retrying database connection..." + echo "$(date) ERROR: Database connection failed after 3 retries" + echo "$(date) INFO: Switching to backup database" + echo "$(date) ERROR: Backup database also unavailable" + echo "$(date) ERROR: Application cannot start without database" + echo "$(date) FATAL: Shutting down application" + while true; do sleep 3600; done + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml new file mode 100644 index 0000000000..adb9cf4dac --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml @@ -0,0 +1,23 @@ +user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-62 namespace" +expected_output: | + The following logs are shown: + ERROR: Failed to connect to database: connection timeout + ERROR: Retrying database connection... + ERROR: Database connection failed after 3 retries + ERROR: Backup database also unavailable + ERROR: Application cannot start without database + + The following log is NOT shown: + INFO: Application starting up + WARN: Database connection slow + + The following log MAY be shown (it does not impact the correctness of the test): + FATAL: Shutting down application + +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 1 +generate_mocks: True +tags: + - logs diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt new file mode 100644 index 0000000000..65c18c2d57 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt @@ -0,0 +1,2 @@ +{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"user-api-6969cc599-n99kp","namespace":"staging-63","filter":"ERROR","limit":10}} +{"schema_version": "robusta:v1.0.0", "status": "no_data", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "staging-63", "pod_name": "user-api-6969cc599-n99kp", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt new file mode 100644 index 0000000000..1f24a3b41a --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"user-api"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep user-api", "params": {"kind": "pod", "keyword": "user-api"}} +staging-63 user-api-6969cc599-n99kp 1/1 Running 0 16s 10.0.2.184 gke-natan-develop-pool-2-41363a6e-yi0r app=user-api,pod-template-hash=6969cc599 diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml new file mode 100644 index 0000000000..78aaed4d88 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: staging-63 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: user-api + namespace: staging-63 +spec: + replicas: 1 + selector: + matchLabels: + app: user-api + template: + metadata: + labels: + app: user-api + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Loading configuration" + echo "$(date) INFO: Connecting to database" + echo "$(date) INFO: Database connection established" + echo "$(date) INFO: Running health checks" + echo "$(date) INFO: All systems operational" + echo "$(date) INFO: Ready to serve requests" + echo "$(date) DEBUG: Processing request batch 1" + echo "$(date) INFO: Request batch 1 completed successfully" + echo "$(date) DEBUG: Processing request batch 2" + echo "$(date) INFO: Request batch 2 completed successfully" + echo "$(date) INFO: Application running smoothly" + while true; do + echo "$(date) INFO: Heartbeat - application healthy" + sleep 60 + done + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml new file mode 100644 index 0000000000..85849c5a00 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml @@ -0,0 +1,9 @@ +user_prompt: "post last 10 logs that contain \"ERROR\" from user-api in staging-63 namespace" +expected_output: "No ERROR logs found" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 1 +generate_mocks: True +tags: + - logs diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt new file mode 100644 index 0000000000..c64725db71 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt @@ -0,0 +1,50 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_describe","match_params":{"kind":"deployment","name":"invoice-generator","namespace":"invoices-64"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl describe deployment invoice-generator -n invoices-64", "params": {"kind": "deployment", "name": "invoice-generator", "namespace": "invoices-64"}} +Name: invoice-generator +Namespace: invoices-64 +CreationTimestamp: Fri, 11 Jul 2025 16:20:40 +0300 +Labels: app=invoice-generator + version=v1.2.0 +Annotations: deployment.kubernetes.io/revision: 1 +Selector: app=invoice-generator +Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable +StrategyType: RollingUpdate +MinReadySeconds: 0 +RollingUpdateStrategy: 25% max unavailable, 25% max surge +Pod Template: + Labels: app=invoice-generator + Containers: + invoice-generator: + Image: busybox + Port: 8080/TCP + Host Port: 0/TCP + Command: + /bin/sh + Args: + -c + echo "$(date) INFO: Invoice generator starting up" + while true; do + echo "$(date) INFO: Generating invoice..." + sleep 5 + done + + Limits: + cpu: 50m + memory: 32Mi + Requests: + cpu: 50m + memory: 32Mi + Environment: + Mounts: + Volumes: +Conditions: + Type Status Reason + ---- ------ ------ + Available True MinimumReplicasAvailable + Progressing True NewReplicaSetAvailable +OldReplicaSets: +NewReplicaSet: invoice-generator-c6c6d857 (1/1 replicas created) +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal ScalingReplicaSet 18s deployment-controller Scaled up replica set invoice-generator-c6c6d857 from 0 to 1 diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt new file mode 100644 index 0000000000..b3851a0c48 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"deployment","keyword":"invoice-generator"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide deployment | grep invoice-generator", "params": {"kind": "deployment", "keyword": "invoice-generator"}} +invoices-64 invoice-generator 1/1 1 1 14s invoice-generator busybox app=invoice-generator app=invoice-generator,version=v1.2.0 diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt new file mode 100644 index 0000000000..5046164f09 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"scaledobject","namespace":"invoices-64"}} +{"schema_version": "robusta:v1.0.0", "status": "error", "error": "Command `kubectl get --show-labels -o wide scaledobject -n invoices-64` failed with return code 1\nOutput:\nerror: the server doesn't have a resource type \"scaledobject\"", "return_code": 1, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide scaledobject -n invoices-64", "params": {"kind": "scaledobject", "namespace": "invoices-64"}} +error: the server doesn't have a resource type "scaledobject" diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml new file mode 100644 index 0000000000..1e0961aeb7 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: invoices-64 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: invoice-generator + namespace: invoices-64 + labels: + app: invoice-generator + version: v1.2.0 +spec: + replicas: 1 + selector: + matchLabels: + app: invoice-generator + template: + metadata: + labels: + app: invoice-generator + spec: + containers: + - name: invoice-generator + image: busybox + ports: + - containerPort: 8080 + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 50m + memory: 32Mi + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Invoice generator starting up" + while true; do + echo "$(date) INFO: Generating invoice..." + sleep 5 + done +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: invoice-generator + namespace: invoices-64 +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: invoice-generator + minReplicas: 1 + maxReplicas: 5 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml new file mode 100644 index 0000000000..1948b90ec4 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml @@ -0,0 +1,13 @@ +user_prompt: 'Why isnt KEDA scaling the invoice-generator deployment?' +description: Verify Holmes goes beyond 'technically correct' answers and provides a useful answer based on the user's actual intent +expected_output: | + - KEDA is not installed in your cluster OR you have no KEDA scaler defined for this resource (either answer is fine) + - But I found an HPA attached to the invoice-generator deployment + - Current CPU usage is 9%, which is below the 60% threshold configured in the HPA. That's why it isn't scaling up +evaluation: + correctness: 0 +generate_mocks: False +tags: + - understanding-real-intent +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt new file mode 100644 index 0000000000..a55a4b99ba --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt @@ -0,0 +1,84 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_describe","match_params":{"kind":"pod","name":"payment-processor-598445cb54-xd5s8","namespace":"production-65"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl describe pod payment-processor-598445cb54-xd5s8 -n production-65", "params": {"kind": "pod", "name": "payment-processor-598445cb54-xd5s8", "namespace": "production-65"}} +Name: payment-processor-598445cb54-xd5s8 +Namespace: production-65 +Priority: 0 +Service Account: default +Node: gke-natan-develop-pool-2-41363a6e-xyv9/10.128.0.12 +Start Time: Fri, 11 Jul 2025 16:22:00 +0300 +Labels: app=payment-processor + pod-template-hash=598445cb54 +Annotations: +Status: Running +IP: 10.0.1.72 +IPs: + IP: 10.0.1.72 +Controlled By: ReplicaSet/payment-processor-598445cb54 +Containers: + payment-processor: + Container ID: containerd://9cc731c27ef2a4ed3fae2468cae688abb45d1678160ec7cdebbc4b583cf755e5 + Image: busybox:1.35 + Image ID: docker.io/library/busybox@sha256:98ad9d1a2be345201bb0709b0d38655eb1b370145c7d94ca1fe9c421f76e245a + Ports: 8080/TCP, 8443/TCP + Host Ports: 0/TCP, 0/TCP + Command: + /bin/sh + Args: + -c + echo "$(date) INFO: Payment processor starting up" + echo "$(date) INFO: Loading payment gateway configuration" + echo "$(date) INFO: Connecting to payment APIs" + echo "$(date) INFO: Payment gateway connection established" + echo "$(date) INFO: Initializing fraud detection engine" + echo "$(date) INFO: Fraud detection engine ready" + echo "$(date) INFO: Running payment processor health checks" + echo "$(date) INFO: All payment systems operational" + echo "$(date) INFO: Ready to process payments" + echo "$(date) INFO: Payment processor fully initialized" + while true; do + echo "$(date) INFO: Payment processor heartbeat - all systems healthy" + echo "$(date) INFO: Processed 10 payments successfully" + echo "$(date) INFO: Payment queue size: 0" + echo "$(date) INFO: All payment gateways responding normally" + sleep 60 + done + + State: Running + Started: Fri, 11 Jul 2025 16:22:20 +0300 + Ready: True + Restart Count: 0 + Limits: + cpu: 10m + memory: 32Mi + Requests: + cpu: 10m + memory: 32Mi + Environment: + Mounts: + /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-wxtbf (ro) +Conditions: + Type Status + PodReadyToStartContainers True + Initialized True + Ready True + ContainersReady True + PodScheduled True +Volumes: + kube-api-access-wxtbf: + Type: Projected (a volume that contains injected data from multiple sources) + TokenExpirationSeconds: 3607 + ConfigMapName: kube-root-ca.crt + ConfigMapOptional: + DownwardAPI: true +QoS Class: Guaranteed +Node-Selectors: +Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s + node.kubernetes.io/unreachable:NoExecute op=Exists for 300s +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 24s default-scheduler Successfully assigned production-65/payment-processor-598445cb54-xd5s8 to gke-natan-develop-pool-2-41363a6e-xyv9 + Normal Pulling 11s kubelet Pulling image "busybox:1.35" + Normal Pulled 8s kubelet Successfully pulled image "busybox:1.35" in 2.378s (2.378s including waiting). Image size: 2159953 bytes. + Normal Created 8s kubelet Created container: payment-processor + Normal Started 4s kubelet Started container payment-processor diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt new file mode 100644 index 0000000000..1e38ad88ad --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt @@ -0,0 +1,4 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_name","match_params":{"kind":"deployment","name":"payment-processor","namespace":"production-65"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide deployment payment-processor -n production-65", "params": {"kind": "deployment", "name": "payment-processor", "namespace": "production-65"}} +NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR LABELS +payment-processor 0/2 2 0 14s payment-processor busybox:1.35 app=payment-processor app=payment-processor,version=v2.1.0 diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt new file mode 100644 index 0000000000..36bca731b2 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt @@ -0,0 +1,18 @@ +{"toolset_name":"kubernetes/krew-extras","tool_name":"kubectl_lineage_children","match_params":{"kind":"deployment","name":"payment-processor","namespace":"production-65"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl lineage deployment payment-processor -n production-65", "params": {"kind": "deployment", "name": "payment-processor", "namespace": "production-65"}} +NAME READY STATUS AGE +Deployment/payment-processor 1/2 19s +├── Event/payment-processor.1851351aea934102 - ScalingReplicaSet: Scaled up replica set payment-processor-598445cb54 from 0 to 2 19s +└── ReplicaSet/payment-processor-598445cb54 1/2 19s + ├── Event/payment-processor-598445cb54.1851351aed976c0e - SuccessfulCreate: Created pod: payment-processor-598445cb54-6tmvg 19s + ├── Event/payment-processor-598445cb54.1851351af128a7ed - SuccessfulCreate: Created pod: payment-processor-598445cb54-xd5s8 19s + ├── Pod/payment-processor-598445cb54-6tmvg 1/1 Running 19s + │ ├── Event/payment-processor-598445cb54-6tmvg.1851351aee9ba490 - Scheduled: Successfully assigned production-65/payment-processor-598445cb54-6tmvg to gke-natan-develop-pool-2-41363a6e-yi0r 19s + │ ├── Event/payment-processor-598445cb54-6tmvg.1851351d70e8c809 - Pulled: Container image "busybox:1.35" already present on machine 8s + │ ├── Event/payment-processor-598445cb54-6tmvg.1851351d85022885 - Created: Created container: payment-processor 8s + │ └── Event/payment-processor-598445cb54-6tmvg.1851351e4d6226a4 - Started: Started container payment-processor 4s + └── Pod/payment-processor-598445cb54-xd5s8 0/1 ContainerCreating 19s + ├── Event/payment-processor-598445cb54-xd5s8.1851351af2985f41 - Scheduled: Successfully assigned production-65/payment-processor-598445cb54-xd5s8 to gke-natan-develop-pool-2-41363a6e-xyv9 19s + ├── Event/payment-processor-598445cb54-xd5s8.1851351e28d42867 - Pulling: Pulling image "busybox:1.35" 5s + ├── Event/payment-processor-598445cb54-xd5s8.1851351eb6931fac - Pulled: Successfully pulled image "busybox:1.35" in 2.378s (2.378s including waiting). Image size: 2159953 bytes. 2s + └── Event/payment-processor-598445cb54-xd5s8.1851351eba6d1c32 - Created: Created container: payment-processor 2s diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/manifest.yaml new file mode 100644 index 0000000000..f3b4807e93 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/manifest.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: production-65 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-processor + namespace: production-65 + labels: + app: payment-processor + version: v2.1.0 +spec: + replicas: 2 + selector: + matchLabels: + app: payment-processor + template: + metadata: + labels: + app: payment-processor + spec: + containers: + - name: payment-processor + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Payment processor starting up" + echo "$(date) INFO: Loading payment gateway configuration" + echo "$(date) INFO: Connecting to payment APIs" + echo "$(date) INFO: Payment gateway connection established" + echo "$(date) INFO: Initializing fraud detection engine" + echo "$(date) INFO: Fraud detection engine ready" + echo "$(date) INFO: Running payment processor health checks" + echo "$(date) INFO: All payment systems operational" + echo "$(date) INFO: Ready to process payments" + echo "$(date) INFO: Payment processor fully initialized" + while true; do + echo "$(date) INFO: Payment processor heartbeat - all systems healthy" + echo "$(date) INFO: Processed 10 payments successfully" + echo "$(date) INFO: Payment queue size: 0" + echo "$(date) INFO: All payment gateways responding normally" + sleep 60 + done + ports: + - containerPort: 8080 + name: http + - containerPort: 8443 + name: https + resources: + requests: + memory: "32Mi" + cpu: "10m" + limits: + memory: "32Mi" + cpu: "10m" diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml new file mode 100644 index 0000000000..6bd8e0e881 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml @@ -0,0 +1,12 @@ +user_prompt: "are there health issues with workload payment-processor in production-65 namespace?" +expected_output: + - "No, I can't find anything. Can you share more details?" + - "Everything appears to be running normally" + - "What specific symptoms are you seeing?" +evaluation: + correctness: 0 +generate_mocks: False +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +tags: + - understanding-real-intent diff --git a/tests/llm/utils/constants.py b/tests/llm/utils/constants.py index 8326b71f4b..9015fc6d3c 100644 --- a/tests/llm/utils/constants.py +++ b/tests/llm/utils/constants.py @@ -17,6 +17,7 @@ "misleading-history", # An eval with misleading historical data "k8s-misconfig", # An eval related to Kubernetes misconfigurations "chain-of-causation", # An eval that tests chain-of-causation analysis - "slackbot", # An eval related to Slack bot functionality + "slackbot", # An eval related to Slack bot functionality, + "understanding-real-intent", # understand the user's real intent behind a question, don't just give a technicaly correct answer that is not useful "counting", # An eval that tests HolmesGPT's ability to accurately count Kubernetes resources ]