From a1045ee56e31f8b32659aa1342a0b97e1fe90688 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Mon, 7 Jul 2025 13:42:22 +0300 Subject: [PATCH 01/11] logging evals --- .../manifest.yaml | 45 +++++++++++++++++ .../test_case.yaml | 7 +++ .../test_case_with_timeout.yaml.example | 8 +++ .../manifest.yaml | 50 +++++++++++++++++++ .../test_case.yaml | 7 +++ .../manifest.yaml | 45 +++++++++++++++++ .../test_case.yaml | 8 +++ 7 files changed, 170 insertions(+) create mode 100644 tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example create mode 100644 tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml new file mode 100644 index 0000000000..c725948640 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: production-55 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: production-55 +spec: + replicas: 1 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Connecting to database" + echo "$(date) WARN: Database connection slow" + echo "$(date) ERROR: Failed to connect to database: connection timeout" + echo "$(date) ERROR: Retrying database connection..." + echo "$(date) ERROR: Database connection failed after 3 retries" + echo "$(date) INFO: Switching to backup database" + echo "$(date) ERROR: Backup database also unavailable" + echo "$(date) ERROR: Application cannot start without database" + echo "$(date) FATAL: Shutting down application" + sleep 300 + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml new file mode 100644 index 0000000000..7838abdde9 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml @@ -0,0 +1,7 @@ +user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-55 namespace" +expected_output: "ERROR: Failed to connect to database" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 1 +generate_mocks: False diff --git a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example new file mode 100644 index 0000000000..4d698a4432 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example @@ -0,0 +1,8 @@ +user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-55 namespace" +expected_output: "ERROR: Failed to connect to database" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +timeout: 120 # Override default 70s timeout with 120s for slow commands +evaluation: + correctness: 1 +generate_mocks: False diff --git a/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml new file mode 100644 index 0000000000..148ed2a64a --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: staging-56 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: user-api + namespace: staging-56 +spec: + replicas: 1 + selector: + matchLabels: + app: user-api + template: + metadata: + labels: + app: user-api + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Loading configuration" + echo "$(date) INFO: Connecting to database" + echo "$(date) INFO: Database connection established" + echo "$(date) INFO: Running health checks" + echo "$(date) INFO: All systems operational" + echo "$(date) INFO: Ready to serve requests" + echo "$(date) DEBUG: Processing request batch 1" + echo "$(date) INFO: Request batch 1 completed successfully" + echo "$(date) DEBUG: Processing request batch 2" + echo "$(date) INFO: Request batch 2 completed successfully" + echo "$(date) INFO: Application running smoothly" + while true; do + echo "$(date) INFO: Heartbeat - application healthy" + sleep 60 + done + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml new file mode 100644 index 0000000000..44af493a23 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml @@ -0,0 +1,7 @@ +user_prompt: "post last 10 logs that contain \"ERROR\" from user-api in staging-56 namespace" +expected_output: "No ERROR logs found" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 1 +generate_mocks: False diff --git a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml new file mode 100644 index 0000000000..19011b0190 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-57 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: video-streamer + namespace: test-57 +spec: + replicas: 1 + selector: + matchLabels: + app: video-streamer + template: + metadata: + labels: + app: video-streamer + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Connecting to database" + echo "$(date) WARN: Database connection slow" + echo "$(date) ERROR: Failed to connect to database: connection timeout" + echo "$(date) ERROR: Retrying database connection..." + echo "$(date) ERROR: Database connection failed after 3 retries" + echo "$(date) INFO: Switching to backup database" + echo "$(date) ERROR: Backup database also unavailable" + echo "$(date) ERROR: Application cannot start without database" + echo "$(date) FATAL: Shutting down application" + sleep 300 + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml new file mode 100644 index 0000000000..ab04c97275 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml @@ -0,0 +1,8 @@ +# we ask about app-57 namespace but the deployment is in test-57 namespace +user_prompt: "post last 10 logs that contain \"ERROR\" from video-streamer in app-57 namespace" +expected_output: "video-streamer not found in namespace app-57, did you mean namespace test-57?" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 1 +generate_mocks: False From 9152a0347c823f352c5e8876d0e856f2f9c29c62 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Wed, 9 Jul 2025 11:50:52 +0300 Subject: [PATCH 02/11] fixes --- .../57_fetch_logs_no_namespace_confusion/manifest.yaml | 2 +- .../57_fetch_logs_no_namespace_confusion/test_case.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml index 19011b0190..176ac521b7 100644 --- a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml +++ b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml @@ -35,7 +35,7 @@ spec: echo "$(date) ERROR: Backup database also unavailable" echo "$(date) ERROR: Application cannot start without database" echo "$(date) FATAL: Shutting down application" - sleep 300 + while true; do sleep 3600; done resources: requests: memory: "32Mi" diff --git a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml index ab04c97275..4ee7f24b48 100644 --- a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml @@ -6,3 +6,5 @@ after_test: 'kubectl delete -f ./manifest.yaml' evaluation: correctness: 1 generate_mocks: False +tags: + - logs From e60160a6743cf2793a869e9aa594c507a4c4ce11 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Fri, 11 Jul 2025 10:38:11 +0300 Subject: [PATCH 03/11] update log evals --- .../manifest.yaml | 45 +++++++++++++++++ .../test_case.yaml | 9 ++++ .../manifest.yaml | 50 +++++++++++++++++++ .../test_case.yaml | 9 ++++ 4 files changed, 113 insertions(+) create mode 100644 tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml new file mode 100644 index 0000000000..744ef87b24 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/manifest.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: production-62 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: production-62 +spec: + replicas: 1 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Connecting to database" + echo "$(date) WARN: Database connection slow" + echo "$(date) ERROR: Failed to connect to database: connection timeout" + echo "$(date) ERROR: Retrying database connection..." + echo "$(date) ERROR: Database connection failed after 3 retries" + echo "$(date) INFO: Switching to backup database" + echo "$(date) ERROR: Backup database also unavailable" + echo "$(date) ERROR: Application cannot start without database" + echo "$(date) FATAL: Shutting down application" + while true; do sleep 3600; done + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml new file mode 100644 index 0000000000..d61107f4cc --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml @@ -0,0 +1,9 @@ +user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-62 namespace" +expected_output: "ERROR: Failed to connect to database" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 1 +generate_mocks: False +tags: + - logs diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml new file mode 100644 index 0000000000..78aaed4d88 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/manifest.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: staging-63 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: user-api + namespace: staging-63 +spec: + replicas: 1 + selector: + matchLabels: + app: user-api + template: + metadata: + labels: + app: user-api + spec: + containers: + - name: app-container + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Application starting up" + echo "$(date) INFO: Loading configuration" + echo "$(date) INFO: Connecting to database" + echo "$(date) INFO: Database connection established" + echo "$(date) INFO: Running health checks" + echo "$(date) INFO: All systems operational" + echo "$(date) INFO: Ready to serve requests" + echo "$(date) DEBUG: Processing request batch 1" + echo "$(date) INFO: Request batch 1 completed successfully" + echo "$(date) DEBUG: Processing request batch 2" + echo "$(date) INFO: Request batch 2 completed successfully" + echo "$(date) INFO: Application running smoothly" + while true; do + echo "$(date) INFO: Heartbeat - application healthy" + sleep 60 + done + resources: + requests: + memory: "32Mi" + cpu: "50m" + limits: + memory: "32Mi" + cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml new file mode 100644 index 0000000000..3c5a08924c --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml @@ -0,0 +1,9 @@ +user_prompt: "post last 10 logs that contain \"ERROR\" from user-api in staging-63 namespace" +expected_output: "No ERROR logs found" +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +evaluation: + correctness: 1 +generate_mocks: False +tags: + - logs From bb1d2ef8818f03c54daddc4b94f62f67a390c09a Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Fri, 11 Jul 2025 10:48:24 +0300 Subject: [PATCH 04/11] fixes --- .../manifest.yaml | 45 ----------------- .../test_case.yaml | 7 --- .../test_case_with_timeout.yaml.example | 8 --- .../manifest.yaml | 50 ------------------- .../test_case.yaml | 7 --- .../manifest.yaml | 0 .../test_case.yaml | 2 +- .../test_case.yaml | 16 +++++- 8 files changed, 16 insertions(+), 119 deletions(-) delete mode 100644 tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml delete mode 100644 tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml delete mode 100644 tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example delete mode 100644 tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml delete mode 100644 tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml rename tests/llm/fixtures/test_ask_holmes/{57_fetch_logs_no_namespace_confusion => 57_wrong_namespace}/manifest.yaml (100%) rename tests/llm/fixtures/test_ask_holmes/{57_fetch_logs_no_namespace_confusion => 57_wrong_namespace}/test_case.yaml (77%) diff --git a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml deleted file mode 100644 index c725948640..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/manifest.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: production-55 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: payment-service - namespace: production-55 -spec: - replicas: 1 - selector: - matchLabels: - app: payment-service - template: - metadata: - labels: - app: payment-service - spec: - containers: - - name: app-container - image: busybox:1.35 - command: ["/bin/sh"] - args: - - -c - - | - echo "$(date) INFO: Application starting up" - echo "$(date) INFO: Connecting to database" - echo "$(date) WARN: Database connection slow" - echo "$(date) ERROR: Failed to connect to database: connection timeout" - echo "$(date) ERROR: Retrying database connection..." - echo "$(date) ERROR: Database connection failed after 3 retries" - echo "$(date) INFO: Switching to backup database" - echo "$(date) ERROR: Backup database also unavailable" - echo "$(date) ERROR: Application cannot start without database" - echo "$(date) FATAL: Shutting down application" - sleep 300 - resources: - requests: - memory: "32Mi" - cpu: "50m" - limits: - memory: "32Mi" - cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml deleted file mode 100644 index 7838abdde9..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case.yaml +++ /dev/null @@ -1,7 +0,0 @@ -user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-55 namespace" -expected_output: "ERROR: Failed to connect to database" -before_test: 'kubectl apply -f ./manifest.yaml' -after_test: 'kubectl delete -f ./manifest.yaml' -evaluation: - correctness: 1 -generate_mocks: False diff --git a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example b/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example deleted file mode 100644 index 4d698a4432..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/55_fetch_error_logs_with_errors/test_case_with_timeout.yaml.example +++ /dev/null @@ -1,8 +0,0 @@ -user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-55 namespace" -expected_output: "ERROR: Failed to connect to database" -before_test: 'kubectl apply -f ./manifest.yaml' -after_test: 'kubectl delete -f ./manifest.yaml' -timeout: 120 # Override default 70s timeout with 120s for slow commands -evaluation: - correctness: 1 -generate_mocks: False diff --git a/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml deleted file mode 100644 index 148ed2a64a..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/manifest.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: staging-56 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: user-api - namespace: staging-56 -spec: - replicas: 1 - selector: - matchLabels: - app: user-api - template: - metadata: - labels: - app: user-api - spec: - containers: - - name: app-container - image: busybox:1.35 - command: ["/bin/sh"] - args: - - -c - - | - echo "$(date) INFO: Application starting up" - echo "$(date) INFO: Loading configuration" - echo "$(date) INFO: Connecting to database" - echo "$(date) INFO: Database connection established" - echo "$(date) INFO: Running health checks" - echo "$(date) INFO: All systems operational" - echo "$(date) INFO: Ready to serve requests" - echo "$(date) DEBUG: Processing request batch 1" - echo "$(date) INFO: Request batch 1 completed successfully" - echo "$(date) DEBUG: Processing request batch 2" - echo "$(date) INFO: Request batch 2 completed successfully" - echo "$(date) INFO: Application running smoothly" - while true; do - echo "$(date) INFO: Heartbeat - application healthy" - sleep 60 - done - resources: - requests: - memory: "32Mi" - cpu: "50m" - limits: - memory: "32Mi" - cpu: "50m" diff --git a/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml deleted file mode 100644 index 44af493a23..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/56_fetch_error_logs_no_errors/test_case.yaml +++ /dev/null @@ -1,7 +0,0 @@ -user_prompt: "post last 10 logs that contain \"ERROR\" from user-api in staging-56 namespace" -expected_output: "No ERROR logs found" -before_test: 'kubectl apply -f ./manifest.yaml' -after_test: 'kubectl delete -f ./manifest.yaml' -evaluation: - correctness: 1 -generate_mocks: False diff --git a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/manifest.yaml similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/manifest.yaml rename to tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/manifest.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml similarity index 77% rename from tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml rename to tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml index 4ee7f24b48..8c66a850ab 100644 --- a/tests/llm/fixtures/test_ask_holmes/57_fetch_logs_no_namespace_confusion/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml @@ -1,5 +1,5 @@ # we ask about app-57 namespace but the deployment is in test-57 namespace -user_prompt: "post last 10 logs that contain \"ERROR\" from video-streamer in app-57 namespace" +user_prompt: "get last logs from video-streamer in app-57 namespace" expected_output: "video-streamer not found in namespace app-57, did you mean namespace test-57?" before_test: 'kubectl apply -f ./manifest.yaml' after_test: 'kubectl delete -f ./manifest.yaml' diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml index d61107f4cc..ac7d5c8db5 100644 --- a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml @@ -1,5 +1,19 @@ user_prompt: "post last 10 logs that contain \"ERROR\" from payment-service in production-62 namespace" -expected_output: "ERROR: Failed to connect to database" +expected_output: | + The following logs are shown: + ERROR: Failed to connect to database: connection timeout + ERROR: Retrying database connection... + ERROR: Database connection failed after 3 retries + ERROR: Backup database also unavailable + ERROR: Application cannot start without database + + The following log is NOT shown: + INFO: Application starting up + WARN: Database connection slow + + The following log MAY be shown (it does not impact the correctness of the test): + FATAL: Shutting down application + before_test: 'kubectl apply -f ./manifest.yaml' after_test: 'kubectl delete -f ./manifest.yaml' evaluation: From 2eaa4a9a757f0c7f5e68c01abb6b04b4a234f28c Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Fri, 11 Jul 2025 14:24:43 +0300 Subject: [PATCH 05/11] update test 57 --- .../kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED | 2 ++ ...ubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED | 3 +++ .../fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED new file mode 100644 index 0000000000..2d6ba3d297 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED @@ -0,0 +1,2 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"video-streamer"}} +{"schema_version": "robusta:v1.0.0", "status": "error", "error": "Command `kubectl get -A --show-labels -o wide pod | grep video-streamer` failed with return code 1\nOutput:\n", "return_code": 1, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep video-streamer", "params": {"kind": "pod", "keyword": "video-streamer"}} diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED new file mode 100644 index 0000000000..6ed85c34e1 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"pod","namespace":"app-57"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide pod -n app-57", "params": {"kind": "pod", "namespace": "app-57"}} +No resources found in app-57 namespace. \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml index 8c66a850ab..450bd5af37 100644 --- a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/test_case.yaml @@ -4,7 +4,7 @@ expected_output: "video-streamer not found in namespace app-57, did you mean nam before_test: 'kubectl apply -f ./manifest.yaml' after_test: 'kubectl delete -f ./manifest.yaml' evaluation: - correctness: 1 + correctness: 0 generate_mocks: False tags: - logs From f08ca84ee85eee0a8c9cbeff0bfab1148d24179c Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Fri, 11 Jul 2025 14:37:55 +0300 Subject: [PATCH 06/11] add mock data --- ...n-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED | 7 +++++++ ...ectl_find_resourcepayment-service_pod.txt.AUTOGENERATED | 3 +++ .../62_fetch_error_logs_with_errors/test_case.yaml | 2 +- ...0_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED | 2 ++ .../kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED | 3 +++ .../63_fetch_error_logs_no_errors/test_case.yaml | 2 +- 6 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED new file mode 100644 index 0000000000..ecaad0e878 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED @@ -0,0 +1,7 @@ +{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"payment-service-7cb79f8c87-7khkm","namespace":"production-62","filter":"ERROR","limit":10}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "production-62", "pod_name": "payment-service-7cb79f8c87-7khkm", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} +Fri Jul 11 11:34:19 UTC 2025 ERROR: Failed to connect to database: connection timeout +Fri Jul 11 11:34:19 UTC 2025 ERROR: Retrying database connection... +Fri Jul 11 11:34:19 UTC 2025 ERROR: Database connection failed after 3 retries +Fri Jul 11 11:34:20 UTC 2025 ERROR: Backup database also unavailable +Fri Jul 11 11:34:20 UTC 2025 ERROR: Application cannot start without database \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED new file mode 100644 index 0000000000..912aa2cb50 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"payment-service"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep payment-service", "params": {"kind": "pod", "keyword": "payment-service"}} +production-62 payment-service-7cb79f8c87-7khkm 1/1 Running 0 15s 10.0.2.182 gke-natan-develop-pool-2-41363a6e-yi0r app=payment-service,pod-template-hash=7cb79f8c87 \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml index ac7d5c8db5..adb9cf4dac 100644 --- a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/test_case.yaml @@ -18,6 +18,6 @@ before_test: 'kubectl apply -f ./manifest.yaml' after_test: 'kubectl delete -f ./manifest.yaml' evaluation: correctness: 1 -generate_mocks: False +generate_mocks: True tags: - logs diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED new file mode 100644 index 0000000000..99f711ddff --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED @@ -0,0 +1,2 @@ +{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"user-api-6969cc599-hg6tb","namespace":"staging-63","filter":"ERROR","limit":10}} +{"schema_version": "robusta:v1.0.0", "status": "no_data", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "staging-63", "pod_name": "user-api-6969cc599-hg6tb", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED new file mode 100644 index 0000000000..8d0abb2705 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"user-api"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep user-api", "params": {"kind": "pod", "keyword": "user-api"}} +staging-63 user-api-6969cc599-hg6tb 1/1 Running 0 13s 10.0.2.183 gke-natan-develop-pool-2-41363a6e-yi0r app=user-api,pod-template-hash=6969cc599 \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml index 3c5a08924c..85849c5a00 100644 --- a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/test_case.yaml @@ -4,6 +4,6 @@ before_test: 'kubectl apply -f ./manifest.yaml' after_test: 'kubectl delete -f ./manifest.yaml' evaluation: correctness: 1 -generate_mocks: False +generate_mocks: True tags: - logs From 2dadb5c468c44d3975733cf1bd8df8091b0c6130 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Fri, 11 Jul 2025 16:29:43 +0300 Subject: [PATCH 07/11] add 2 more tests --- ...ourcepayment-service_pod.txt.AUTOGENERATED | 2 +- ...ind_resourceuser-api_pod.txt.AUTOGENERATED | 2 +- ...ce-generator_invoices-64.txt.AUTOGENERATED | 50 +++++++++++ ...ice-generator_deployment.txt.AUTOGENERATED | 3 + ...scaledobject_invoices-64.txt.AUTOGENERATED | 3 + .../64_keda_vs_hpa_confusion/manifest.yaml | 64 ++++++++++++++ .../64_keda_vs_hpa_confusion/test_case.yaml | 13 +++ ...cb54-xd5s8_production-65.txt.AUTOGENERATED | 84 +++++++++++++++++++ ...-processor_production-65.txt.AUTOGENERATED | 4 + ...-processor_production-65.txt.AUTOGENERATED | 18 ++++ .../65_health_check_followup/manifest.yaml | 59 +++++++++++++ .../65_health_check_followup/test_case.yaml | 12 +++ 12 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/65_health_check_followup/manifest.yaml create mode 100644 tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED index 912aa2cb50..faa950af36 100644 --- a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED @@ -1,3 +1,3 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"payment-service"}} {"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep payment-service", "params": {"kind": "pod", "keyword": "payment-service"}} -production-62 payment-service-7cb79f8c87-7khkm 1/1 Running 0 15s 10.0.2.182 gke-natan-develop-pool-2-41363a6e-yi0r app=payment-service,pod-template-hash=7cb79f8c87 \ No newline at end of file +production-62 payment-service-7cb79f8c87-z96z8 1/1 Running 0 14s 10.0.2.187 gke-natan-develop-pool-2-41363a6e-yi0r app=payment-service,pod-template-hash=7cb79f8c87 \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED index 8d0abb2705..08de567e05 100644 --- a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED @@ -1,3 +1,3 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"user-api"}} {"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep user-api", "params": {"kind": "pod", "keyword": "user-api"}} -staging-63 user-api-6969cc599-hg6tb 1/1 Running 0 13s 10.0.2.183 gke-natan-develop-pool-2-41363a6e-yi0r app=user-api,pod-template-hash=6969cc599 \ No newline at end of file +staging-63 user-api-6969cc599-n99kp 1/1 Running 0 16s 10.0.2.184 gke-natan-develop-pool-2-41363a6e-yi0r app=user-api,pod-template-hash=6969cc599 \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED new file mode 100644 index 0000000000..8ec8d8750a --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED @@ -0,0 +1,50 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_describe","match_params":{"kind":"deployment","name":"invoice-generator","namespace":"invoices-64"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl describe deployment invoice-generator -n invoices-64", "params": {"kind": "deployment", "name": "invoice-generator", "namespace": "invoices-64"}} +Name: invoice-generator +Namespace: invoices-64 +CreationTimestamp: Fri, 11 Jul 2025 16:20:40 +0300 +Labels: app=invoice-generator + version=v1.2.0 +Annotations: deployment.kubernetes.io/revision: 1 +Selector: app=invoice-generator +Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable +StrategyType: RollingUpdate +MinReadySeconds: 0 +RollingUpdateStrategy: 25% max unavailable, 25% max surge +Pod Template: + Labels: app=invoice-generator + Containers: + invoice-generator: + Image: busybox + Port: 8080/TCP + Host Port: 0/TCP + Command: + /bin/sh + Args: + -c + echo "$(date) INFO: Invoice generator starting up" + while true; do + echo "$(date) INFO: Generating invoice..." + sleep 5 + done + + Limits: + cpu: 50m + memory: 32Mi + Requests: + cpu: 50m + memory: 32Mi + Environment: + Mounts: + Volumes: +Conditions: + Type Status Reason + ---- ------ ------ + Available True MinimumReplicasAvailable + Progressing True NewReplicaSetAvailable +OldReplicaSets: +NewReplicaSet: invoice-generator-c6c6d857 (1/1 replicas created) +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal ScalingReplicaSet 18s deployment-controller Scaled up replica set invoice-generator-c6c6d857 from 0 to 1 \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED new file mode 100644 index 0000000000..64ad8fa762 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"deployment","keyword":"invoice-generator"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide deployment | grep invoice-generator", "params": {"kind": "deployment", "keyword": "invoice-generator"}} +invoices-64 invoice-generator 1/1 1 1 14s invoice-generator busybox app=invoice-generator app=invoice-generator,version=v1.2.0 \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED new file mode 100644 index 0000000000..c0cac120e8 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED @@ -0,0 +1,3 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"scaledobject","namespace":"invoices-64"}} +{"schema_version": "robusta:v1.0.0", "status": "error", "error": "Command `kubectl get --show-labels -o wide scaledobject -n invoices-64` failed with return code 1\nOutput:\nerror: the server doesn't have a resource type \"scaledobject\"", "return_code": 1, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide scaledobject -n invoices-64", "params": {"kind": "scaledobject", "namespace": "invoices-64"}} +error: the server doesn't have a resource type "scaledobject" \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml new file mode 100644 index 0000000000..1e0961aeb7 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/manifest.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: invoices-64 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: invoice-generator + namespace: invoices-64 + labels: + app: invoice-generator + version: v1.2.0 +spec: + replicas: 1 + selector: + matchLabels: + app: invoice-generator + template: + metadata: + labels: + app: invoice-generator + spec: + containers: + - name: invoice-generator + image: busybox + ports: + - containerPort: 8080 + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 50m + memory: 32Mi + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Invoice generator starting up" + while true; do + echo "$(date) INFO: Generating invoice..." + sleep 5 + done +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: invoice-generator + namespace: invoices-64 +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: invoice-generator + minReplicas: 1 + maxReplicas: 5 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml new file mode 100644 index 0000000000..1948b90ec4 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/test_case.yaml @@ -0,0 +1,13 @@ +user_prompt: 'Why isnt KEDA scaling the invoice-generator deployment?' +description: Verify Holmes goes beyond 'technically correct' answers and provides a useful answer based on the user's actual intent +expected_output: | + - KEDA is not installed in your cluster OR you have no KEDA scaler defined for this resource (either answer is fine) + - But I found an HPA attached to the invoice-generator deployment + - Current CPU usage is 9%, which is below the 60% threshold configured in the HPA. That's why it isn't scaling up +evaluation: + correctness: 0 +generate_mocks: False +tags: + - understanding-real-intent +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED new file mode 100644 index 0000000000..f4338701dd --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED @@ -0,0 +1,84 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_describe","match_params":{"kind":"pod","name":"payment-processor-598445cb54-xd5s8","namespace":"production-65"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl describe pod payment-processor-598445cb54-xd5s8 -n production-65", "params": {"kind": "pod", "name": "payment-processor-598445cb54-xd5s8", "namespace": "production-65"}} +Name: payment-processor-598445cb54-xd5s8 +Namespace: production-65 +Priority: 0 +Service Account: default +Node: gke-natan-develop-pool-2-41363a6e-xyv9/10.128.0.12 +Start Time: Fri, 11 Jul 2025 16:22:00 +0300 +Labels: app=payment-processor + pod-template-hash=598445cb54 +Annotations: +Status: Running +IP: 10.0.1.72 +IPs: + IP: 10.0.1.72 +Controlled By: ReplicaSet/payment-processor-598445cb54 +Containers: + payment-processor: + Container ID: containerd://9cc731c27ef2a4ed3fae2468cae688abb45d1678160ec7cdebbc4b583cf755e5 + Image: busybox:1.35 + Image ID: docker.io/library/busybox@sha256:98ad9d1a2be345201bb0709b0d38655eb1b370145c7d94ca1fe9c421f76e245a + Ports: 8080/TCP, 8443/TCP + Host Ports: 0/TCP, 0/TCP + Command: + /bin/sh + Args: + -c + echo "$(date) INFO: Payment processor starting up" + echo "$(date) INFO: Loading payment gateway configuration" + echo "$(date) INFO: Connecting to payment APIs" + echo "$(date) INFO: Payment gateway connection established" + echo "$(date) INFO: Initializing fraud detection engine" + echo "$(date) INFO: Fraud detection engine ready" + echo "$(date) INFO: Running payment processor health checks" + echo "$(date) INFO: All payment systems operational" + echo "$(date) INFO: Ready to process payments" + echo "$(date) INFO: Payment processor fully initialized" + while true; do + echo "$(date) INFO: Payment processor heartbeat - all systems healthy" + echo "$(date) INFO: Processed 10 payments successfully" + echo "$(date) INFO: Payment queue size: 0" + echo "$(date) INFO: All payment gateways responding normally" + sleep 60 + done + + State: Running + Started: Fri, 11 Jul 2025 16:22:20 +0300 + Ready: True + Restart Count: 0 + Limits: + cpu: 10m + memory: 32Mi + Requests: + cpu: 10m + memory: 32Mi + Environment: + Mounts: + /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-wxtbf (ro) +Conditions: + Type Status + PodReadyToStartContainers True + Initialized True + Ready True + ContainersReady True + PodScheduled True +Volumes: + kube-api-access-wxtbf: + Type: Projected (a volume that contains injected data from multiple sources) + TokenExpirationSeconds: 3607 + ConfigMapName: kube-root-ca.crt + ConfigMapOptional: + DownwardAPI: true +QoS Class: Guaranteed +Node-Selectors: +Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s + node.kubernetes.io/unreachable:NoExecute op=Exists for 300s +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 24s default-scheduler Successfully assigned production-65/payment-processor-598445cb54-xd5s8 to gke-natan-develop-pool-2-41363a6e-xyv9 + Normal Pulling 11s kubelet Pulling image "busybox:1.35" + Normal Pulled 8s kubelet Successfully pulled image "busybox:1.35" in 2.378s (2.378s including waiting). Image size: 2159953 bytes. + Normal Created 8s kubelet Created container: payment-processor + Normal Started 4s kubelet Started container payment-processor \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED new file mode 100644 index 0000000000..ab99106612 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED @@ -0,0 +1,4 @@ +{"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_name","match_params":{"kind":"deployment","name":"payment-processor","namespace":"production-65"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide deployment payment-processor -n production-65", "params": {"kind": "deployment", "name": "payment-processor", "namespace": "production-65"}} +NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR LABELS +payment-processor 0/2 2 0 14s payment-processor busybox:1.35 app=payment-processor app=payment-processor,version=v2.1.0 \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED new file mode 100644 index 0000000000..1631ae7889 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED @@ -0,0 +1,18 @@ +{"toolset_name":"kubernetes/krew-extras","tool_name":"kubectl_lineage_children","match_params":{"kind":"deployment","name":"payment-processor","namespace":"production-65"}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl lineage deployment payment-processor -n production-65", "params": {"kind": "deployment", "name": "payment-processor", "namespace": "production-65"}} +NAME READY STATUS AGE +Deployment/payment-processor 1/2 19s +├── Event/payment-processor.1851351aea934102 - ScalingReplicaSet: Scaled up replica set payment-processor-598445cb54 from 0 to 2 19s +└── ReplicaSet/payment-processor-598445cb54 1/2 19s + ├── Event/payment-processor-598445cb54.1851351aed976c0e - SuccessfulCreate: Created pod: payment-processor-598445cb54-6tmvg 19s + ├── Event/payment-processor-598445cb54.1851351af128a7ed - SuccessfulCreate: Created pod: payment-processor-598445cb54-xd5s8 19s + ├── Pod/payment-processor-598445cb54-6tmvg 1/1 Running 19s + │ ├── Event/payment-processor-598445cb54-6tmvg.1851351aee9ba490 - Scheduled: Successfully assigned production-65/payment-processor-598445cb54-6tmvg to gke-natan-develop-pool-2-41363a6e-yi0r 19s + │ ├── Event/payment-processor-598445cb54-6tmvg.1851351d70e8c809 - Pulled: Container image "busybox:1.35" already present on machine 8s + │ ├── Event/payment-processor-598445cb54-6tmvg.1851351d85022885 - Created: Created container: payment-processor 8s + │ └── Event/payment-processor-598445cb54-6tmvg.1851351e4d6226a4 - Started: Started container payment-processor 4s + └── Pod/payment-processor-598445cb54-xd5s8 0/1 ContainerCreating 19s + ├── Event/payment-processor-598445cb54-xd5s8.1851351af2985f41 - Scheduled: Successfully assigned production-65/payment-processor-598445cb54-xd5s8 to gke-natan-develop-pool-2-41363a6e-xyv9 19s + ├── Event/payment-processor-598445cb54-xd5s8.1851351e28d42867 - Pulling: Pulling image "busybox:1.35" 5s + ├── Event/payment-processor-598445cb54-xd5s8.1851351eb6931fac - Pulled: Successfully pulled image "busybox:1.35" in 2.378s (2.378s including waiting). Image size: 2159953 bytes. 2s + └── Event/payment-processor-598445cb54-xd5s8.1851351eba6d1c32 - Created: Created container: payment-processor 2s \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/manifest.yaml b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/manifest.yaml new file mode 100644 index 0000000000..f3b4807e93 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/manifest.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: production-65 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-processor + namespace: production-65 + labels: + app: payment-processor + version: v2.1.0 +spec: + replicas: 2 + selector: + matchLabels: + app: payment-processor + template: + metadata: + labels: + app: payment-processor + spec: + containers: + - name: payment-processor + image: busybox:1.35 + command: ["/bin/sh"] + args: + - -c + - | + echo "$(date) INFO: Payment processor starting up" + echo "$(date) INFO: Loading payment gateway configuration" + echo "$(date) INFO: Connecting to payment APIs" + echo "$(date) INFO: Payment gateway connection established" + echo "$(date) INFO: Initializing fraud detection engine" + echo "$(date) INFO: Fraud detection engine ready" + echo "$(date) INFO: Running payment processor health checks" + echo "$(date) INFO: All payment systems operational" + echo "$(date) INFO: Ready to process payments" + echo "$(date) INFO: Payment processor fully initialized" + while true; do + echo "$(date) INFO: Payment processor heartbeat - all systems healthy" + echo "$(date) INFO: Processed 10 payments successfully" + echo "$(date) INFO: Payment queue size: 0" + echo "$(date) INFO: All payment gateways responding normally" + sleep 60 + done + ports: + - containerPort: 8080 + name: http + - containerPort: 8443 + name: https + resources: + requests: + memory: "32Mi" + cpu: "10m" + limits: + memory: "32Mi" + cpu: "10m" diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml new file mode 100644 index 0000000000..6bd8e0e881 --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/test_case.yaml @@ -0,0 +1,12 @@ +user_prompt: "are there health issues with workload payment-processor in production-65 namespace?" +expected_output: + - "No, I can't find anything. Can you share more details?" + - "Everything appears to be running normally" + - "What specific symptoms are you seeing?" +evaluation: + correctness: 0 +generate_mocks: False +before_test: 'kubectl apply -f ./manifest.yaml' +after_test: 'kubectl delete -f ./manifest.yaml' +tags: + - understanding-real-intent From b2b7cd0da52d24df8b226b00497bd4fa0fc3d3a5 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Fri, 11 Jul 2025 17:28:36 +0300 Subject: [PATCH 08/11] add missing tags --- pyproject.toml | 1 + tests/llm/utils/constants.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cf6b1bdf32..38711fad8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ markers = [ "k8s-misconfig: Tests involving Kubernetes misconfigurations", "chain-of-causation: Tests involving chain-of-causation analysis", "slackbot: Tests involving Slack bot functionality", + "understanding-real-intent: Tests that require understanding the user's real intent behind a question, not just giving a technically correct answer that is not useful" ] addopts = [ "--cov-config=pyproject.toml", diff --git a/tests/llm/utils/constants.py b/tests/llm/utils/constants.py index 858744ca0e..87de796536 100644 --- a/tests/llm/utils/constants.py +++ b/tests/llm/utils/constants.py @@ -17,5 +17,6 @@ "misleading-history", # An eval with misleading historical data "k8s-misconfig", # An eval related to Kubernetes misconfigurations "chain-of-causation", # An eval that tests chain-of-causation analysis - "slackbot", # An eval related to Slack bot functionality + "slackbot", # An eval related to Slack bot functionality, + "understanding-real-intent", # understand the user's real intent behind a question, don't just give a technicaly correct answer that is not useful ] From 22c51c681c2ee3e618a08fe3565dba1e742d36da Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Sun, 13 Jul 2025 14:06:22 +0300 Subject: [PATCH 09/11] fix mock data for evals --- ...NERATED => kubectl_find_resourcevideo-streamer_pod.txt} | 0 ...ATED => kubectl_get_by_kind_in_namespacepod_app-57.txt} | 2 +- ...n-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED | 7 ------- ...R_10_production-62_payment-service-7cb79f8c87-z96z8.txt | 7 +++++++ ...ERATED => kubectl_find_resourcepayment-service_pod.txt} | 2 +- ...d_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt} | 4 ++-- ...AUTOGENERATED => kubectl_find_resourceuser-api_pod.txt} | 2 +- ...l_describedeployment_invoice-generator_invoices-64.txt} | 2 +- ... kubectl_find_resourceinvoice-generator_deployment.txt} | 2 +- ...l_get_by_kind_in_namespacescaledobject_invoices-64.txt} | 2 +- ...d_payment-processor-598445cb54-xd5s8_production-65.txt} | 2 +- ..._by_namedeployment_payment-processor_production-65.txt} | 2 +- ...childrendeployment_payment-processor_production-65.txt} | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) rename tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/{kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED => kubectl_find_resourcevideo-streamer_pod.txt} (100%) rename tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/{kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED => kubectl_get_by_kind_in_namespacepod_app-57.txt} (90%) delete mode 100644 tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED create mode 100644 tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt rename tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/{kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED => kubectl_find_resourcepayment-service_pod.txt} (96%) rename tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/{fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED => fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt} (72%) rename tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/{kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED => kubectl_find_resourceuser-api_pod.txt} (97%) rename tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/{kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED => kubectl_describedeployment_invoice-generator_invoices-64.txt} (97%) rename tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/{kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED => kubectl_find_resourceinvoice-generator_deployment.txt} (95%) rename tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/{kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED => kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt} (90%) rename tests/llm/fixtures/test_ask_holmes/65_health_check_followup/{kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED => kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt} (99%) rename tests/llm/fixtures/test_ask_holmes/65_health_check_followup/{kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED => kubectl_get_by_namedeployment_payment-processor_production-65.txt} (96%) rename tests/llm/fixtures/test_ask_holmes/65_health_check_followup/{kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED => kubectl_lineage_childrendeployment_payment-processor_production-65.txt} (99%) diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt similarity index 100% rename from tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_find_resourcevideo-streamer_pod.txt diff --git a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt similarity index 90% rename from tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt index 6ed85c34e1..200220701a 100644 --- a/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/57_wrong_namespace/kubectl_get_by_kind_in_namespacepod_app-57.txt @@ -1,3 +1,3 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"pod","namespace":"app-57"}} {"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide pod -n app-57", "params": {"kind": "pod", "namespace": "app-57"}} -No resources found in app-57 namespace. \ No newline at end of file +No resources found in app-57 namespace. diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED deleted file mode 100644 index ecaad0e878..0000000000 --- a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-7khkm.txt.AUTOGENERATED +++ /dev/null @@ -1,7 +0,0 @@ -{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"payment-service-7cb79f8c87-7khkm","namespace":"production-62","filter":"ERROR","limit":10}} -{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "production-62", "pod_name": "payment-service-7cb79f8c87-7khkm", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} -Fri Jul 11 11:34:19 UTC 2025 ERROR: Failed to connect to database: connection timeout -Fri Jul 11 11:34:19 UTC 2025 ERROR: Retrying database connection... -Fri Jul 11 11:34:19 UTC 2025 ERROR: Database connection failed after 3 retries -Fri Jul 11 11:34:20 UTC 2025 ERROR: Backup database also unavailable -Fri Jul 11 11:34:20 UTC 2025 ERROR: Application cannot start without database \ No newline at end of file diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt new file mode 100644 index 0000000000..08bb9b743c --- /dev/null +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/fetch_pod_logsERROR_10_production-62_payment-service-7cb79f8c87-z96z8.txt @@ -0,0 +1,7 @@ +{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"payment-service-7cb79f8c87-z96z8","namespace":"production-62","filter":"ERROR","limit":10}} +{"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "production-62", "pod_name": "payment-service-7cb79f8c87-z96z8", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} +Fri Jul 11 13:23:24 UTC 2025 ERROR: Failed to connect to database: connection timeout +Fri Jul 11 13:23:24 UTC 2025 ERROR: Retrying database connection... +Fri Jul 11 13:23:24 UTC 2025 ERROR: Database connection failed after 3 retries +Fri Jul 11 13:23:24 UTC 2025 ERROR: Backup database also unavailable +Fri Jul 11 13:23:24 UTC 2025 ERROR: Application cannot start without database diff --git a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt similarity index 96% rename from tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt index faa950af36..9a45625a72 100644 --- a/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/62_fetch_error_logs_with_errors/kubectl_find_resourcepayment-service_pod.txt @@ -1,3 +1,3 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"payment-service"}} {"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep payment-service", "params": {"kind": "pod", "keyword": "payment-service"}} -production-62 payment-service-7cb79f8c87-z96z8 1/1 Running 0 14s 10.0.2.187 gke-natan-develop-pool-2-41363a6e-yi0r app=payment-service,pod-template-hash=7cb79f8c87 \ No newline at end of file +production-62 payment-service-7cb79f8c87-z96z8 1/1 Running 0 14s 10.0.2.187 gke-natan-develop-pool-2-41363a6e-yi0r app=payment-service,pod-template-hash=7cb79f8c87 diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt similarity index 72% rename from tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt index 99f711ddff..65c18c2d57 100644 --- a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-hg6tb.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/fetch_pod_logsERROR_10_staging-63_user-api-6969cc599-n99kp.txt @@ -1,2 +1,2 @@ -{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"user-api-6969cc599-hg6tb","namespace":"staging-63","filter":"ERROR","limit":10}} -{"schema_version": "robusta:v1.0.0", "status": "no_data", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "staging-63", "pod_name": "user-api-6969cc599-hg6tb", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} +{"toolset_name":"kubernetes/logs","tool_name":"fetch_pod_logs","match_params":{"pod_name":"user-api-6969cc599-n99kp","namespace":"staging-63","filter":"ERROR","limit":10}} +{"schema_version": "robusta:v1.0.0", "status": "no_data", "error": null, "return_code": 0, "data": null, "url": null, "invocation": null, "params": {"namespace": "staging-63", "pod_name": "user-api-6969cc599-n99kp", "start_time": null, "end_time": null, "filter": "ERROR", "limit": 10}} diff --git a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt similarity index 97% rename from tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt index 08de567e05..1f24a3b41a 100644 --- a/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/63_fetch_error_logs_no_errors/kubectl_find_resourceuser-api_pod.txt @@ -1,3 +1,3 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"pod","keyword":"user-api"}} {"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide pod | grep user-api", "params": {"kind": "pod", "keyword": "user-api"}} -staging-63 user-api-6969cc599-n99kp 1/1 Running 0 16s 10.0.2.184 gke-natan-develop-pool-2-41363a6e-yi0r app=user-api,pod-template-hash=6969cc599 \ No newline at end of file +staging-63 user-api-6969cc599-n99kp 1/1 Running 0 16s 10.0.2.184 gke-natan-develop-pool-2-41363a6e-yi0r app=user-api,pod-template-hash=6969cc599 diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt similarity index 97% rename from tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt index 8ec8d8750a..c64725db71 100644 --- a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_describedeployment_invoice-generator_invoices-64.txt @@ -47,4 +47,4 @@ NewReplicaSet: invoice-generator-c6c6d857 (1/1 replicas created) Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal ScalingReplicaSet 18s deployment-controller Scaled up replica set invoice-generator-c6c6d857 from 0 to 1 \ No newline at end of file + Normal ScalingReplicaSet 18s deployment-controller Scaled up replica set invoice-generator-c6c6d857 from 0 to 1 diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt similarity index 95% rename from tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt index 64ad8fa762..b3851a0c48 100644 --- a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_find_resourceinvoice-generator_deployment.txt @@ -1,3 +1,3 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_find_resource","match_params":{"kind":"deployment","keyword":"invoice-generator"}} {"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get -A --show-labels -o wide deployment | grep invoice-generator", "params": {"kind": "deployment", "keyword": "invoice-generator"}} -invoices-64 invoice-generator 1/1 1 1 14s invoice-generator busybox app=invoice-generator app=invoice-generator,version=v1.2.0 \ No newline at end of file +invoices-64 invoice-generator 1/1 1 1 14s invoice-generator busybox app=invoice-generator app=invoice-generator,version=v1.2.0 diff --git a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt similarity index 90% rename from tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt index c0cac120e8..5046164f09 100644 --- a/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/64_keda_vs_hpa_confusion/kubectl_get_by_kind_in_namespacescaledobject_invoices-64.txt @@ -1,3 +1,3 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_kind_in_namespace","match_params":{"kind":"scaledobject","namespace":"invoices-64"}} {"schema_version": "robusta:v1.0.0", "status": "error", "error": "Command `kubectl get --show-labels -o wide scaledobject -n invoices-64` failed with return code 1\nOutput:\nerror: the server doesn't have a resource type \"scaledobject\"", "return_code": 1, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide scaledobject -n invoices-64", "params": {"kind": "scaledobject", "namespace": "invoices-64"}} -error: the server doesn't have a resource type "scaledobject" \ No newline at end of file +error: the server doesn't have a resource type "scaledobject" diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt similarity index 99% rename from tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt index f4338701dd..a55a4b99ba 100644 --- a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_describepod_payment-processor-598445cb54-xd5s8_production-65.txt @@ -81,4 +81,4 @@ Events: Normal Pulling 11s kubelet Pulling image "busybox:1.35" Normal Pulled 8s kubelet Successfully pulled image "busybox:1.35" in 2.378s (2.378s including waiting). Image size: 2159953 bytes. Normal Created 8s kubelet Created container: payment-processor - Normal Started 4s kubelet Started container payment-processor \ No newline at end of file + Normal Started 4s kubelet Started container payment-processor diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt similarity index 96% rename from tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt index ab99106612..1e38ad88ad 100644 --- a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_get_by_namedeployment_payment-processor_production-65.txt @@ -1,4 +1,4 @@ {"toolset_name":"kubernetes/core","tool_name":"kubectl_get_by_name","match_params":{"kind":"deployment","name":"payment-processor","namespace":"production-65"}} {"schema_version": "robusta:v1.0.0", "status": "success", "error": null, "return_code": 0, "data": null, "url": null, "invocation": "kubectl get --show-labels -o wide deployment payment-processor -n production-65", "params": {"kind": "deployment", "name": "payment-processor", "namespace": "production-65"}} NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR LABELS -payment-processor 0/2 2 0 14s payment-processor busybox:1.35 app=payment-processor app=payment-processor,version=v2.1.0 \ No newline at end of file +payment-processor 0/2 2 0 14s payment-processor busybox:1.35 app=payment-processor app=payment-processor,version=v2.1.0 diff --git a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt similarity index 99% rename from tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED rename to tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt index 1631ae7889..36bca731b2 100644 --- a/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt.AUTOGENERATED +++ b/tests/llm/fixtures/test_ask_holmes/65_health_check_followup/kubectl_lineage_childrendeployment_payment-processor_production-65.txt @@ -15,4 +15,4 @@ Deployment/payment-processor 1/2 ├── Event/payment-processor-598445cb54-xd5s8.1851351af2985f41 - Scheduled: Successfully assigned production-65/payment-processor-598445cb54-xd5s8 to gke-natan-develop-pool-2-41363a6e-xyv9 19s ├── Event/payment-processor-598445cb54-xd5s8.1851351e28d42867 - Pulling: Pulling image "busybox:1.35" 5s ├── Event/payment-processor-598445cb54-xd5s8.1851351eb6931fac - Pulled: Successfully pulled image "busybox:1.35" in 2.378s (2.378s including waiting). Image size: 2159953 bytes. 2s - └── Event/payment-processor-598445cb54-xd5s8.1851351eba6d1c32 - Created: Created container: payment-processor 2s \ No newline at end of file + └── Event/payment-processor-598445cb54-xd5s8.1851351eba6d1c32 - Created: Created container: payment-processor 2s From e5a26e435edf535a08ea106d5e81ef74f6dfb1da Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Mon, 21 Jul 2025 15:09:07 +0300 Subject: [PATCH 10/11] Update pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 50544f1719..7ca930d23c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ markers = [ "slackbot: Tests involving Slack bot functionality", "understanding-real-intent: Tests that require understanding the user's real intent behind a question, not just giving a technically correct answer that is not useful" ] + addopts = [ "--cov-config=pyproject.toml", "--cov=holmes", From c9f9be612c2d1aa1db3f79092d66cb18efb8ad58 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Mon, 21 Jul 2025 15:09:29 +0300 Subject: [PATCH 11/11] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e327dae676..43b2c0b73b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ markers = [ "k8s-misconfig: Tests involving Kubernetes misconfigurations", "chain-of-causation: Tests involving chain-of-causation analysis", "slackbot: Tests involving Slack bot functionality", - "understanding-real-intent: Tests that require understanding the user's real intent behind a question, not just giving a technically correct answer that is not useful" + "understanding-real-intent: Tests that require understanding the user's real intent behind a question, not just giving a technically correct answer that is not useful", "counting: Ask holmes to count kubernetes/cloud resources" ]