From df73ecbe0326e79baa3bd9ad652cf7510ba193ec Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 12:27:47 +0300 Subject: [PATCH 01/10] POC --- .github/workflows/llm-evaluation.yaml | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index 6e45c782f2..6adfb8d1fd 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -31,11 +31,74 @@ jobs: poetry config virtualenvs.create false poetry install --no-root + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + kubectl version --client + + - name: Install KIND + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + kind version + + - name: Create KIND cluster + run: | + cat < kind-config.yaml + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + nodes: + - role: control-plane + extraPortMappings: + - containerPort: 30000 + hostPort: 30000 + protocol: TCP + - containerPort: 30001 + hostPort: 30001 + protocol: TCP + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "200" + - | + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + - | + kind: KubeletConfiguration + maxPods: 200 + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "200" + - | + kind: KubeletConfiguration + maxPods: 200 + EOF + + # Create cluster with resource limits appropriate for GitHub Actions (7GB RAM, 2 CPU) + kind create cluster --config kind-config.yaml --wait 5m + + # Configure kubectl + kubectl cluster-info --context kind-kind + kubectl get nodes + + # Wait for all system pods to be ready + kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=300s + - name: Run tests id: evals continue-on-error: true shell: bash env: + RUN_LIVE: "true" AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} From ea643dc1d8c622befef8b42c4de0e80f5b94b70d Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 13:41:52 +0300 Subject: [PATCH 02/10] improvements --- .github/workflows/llm-evaluation.yaml | 12 ++++++++++++ .../02_what_is_wrong_with_pod/test_case.yaml | 18 +++++++++++++++++- tests/llm/utils/reporting/github_reporter.py | 18 ++++++++++++++++++ tests/llm/utils/test_results.py | 7 ++++++- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index 6adfb8d1fd..47e4da8ebd 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -93,6 +93,18 @@ jobs: # Wait for all system pods to be ready kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=300s + # Verify cluster is working by creating a test pod + kubectl run test-pod --image=busybox:1.35 --restart=Never -- echo "Cluster is ready" + kubectl wait --for=condition=Ready pod/test-pod --timeout=30s + kubectl delete pod test-pod + + # Show cluster info for debugging + echo "=== Cluster nodes ===" + kubectl get nodes -o wide + echo "=== System pods ===" + kubectl get pods -n kube-system + echo "=== Cluster is ready for tests ===" + - name: Run tests id: evals continue-on-error: true diff --git a/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml index 36b498f54e..918dd1c6a9 100644 --- a/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml @@ -29,7 +29,23 @@ before_test: | requests: memory: "50Mi" EOF - sleep 60 + # Wait for pod to be created and start running + kubectl wait --for=condition=Ready pod/giant-narwhal-6958c5bdd8-69gtn -n app-02 --timeout=30s + # Wait for the pod to be OOMKilled (check every 2 seconds for up to 120 seconds) + echo "Waiting for pod to be OOMKilled..." + for i in {1..60}; do + STATUS=$(kubectl get pod giant-narwhal-6958c5bdd8-69gtn -n app-02 -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' 2>/dev/null || echo "") + if [ "$STATUS" = "OOMKilled" ]; then + echo "Pod was OOMKilled successfully" + break + fi + if [ $i -eq 60 ]; then + echo "Timeout waiting for OOMKill after 120 seconds" + kubectl describe pod giant-narwhal-6958c5bdd8-69gtn -n app-02 + exit 1 + fi + sleep 2 + done after_test: | kubectl delete namespace app-02 diff --git a/tests/llm/utils/reporting/github_reporter.py b/tests/llm/utils/reporting/github_reporter.py index 8842263a59..5a86cee881 100644 --- a/tests/llm/utils/reporting/github_reporter.py +++ b/tests/llm/utils/reporting/github_reporter.py @@ -31,18 +31,21 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict ask_holmes_regressions = 0 ask_holmes_mock_failures = 0 ask_holmes_skipped = 0 + ask_holmes_setup_failures = 0 investigate_total = 0 investigate_passed = 0 investigate_regressions = 0 investigate_mock_failures = 0 investigate_skipped = 0 + investigate_setup_failures = 0 workload_health_total = 0 workload_health_passed = 0 workload_health_regressions = 0 workload_health_mock_failures = 0 workload_health_skipped = 0 + workload_health_setup_failures = 0 for result in sorted_results: status = TestStatus(result) @@ -51,6 +54,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict ask_holmes_total += 1 if status.is_skipped: ask_holmes_skipped += 1 + elif status.is_setup_failure: + ask_holmes_setup_failures += 1 elif status.passed: ask_holmes_passed += 1 elif status.is_regression: @@ -61,6 +66,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict investigate_total += 1 if status.is_skipped: investigate_skipped += 1 + elif status.is_setup_failure: + investigate_setup_failures += 1 elif status.passed: investigate_passed += 1 elif status.is_regression: @@ -71,6 +78,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict workload_health_total += 1 if status.is_skipped: workload_health_skipped += 1 + elif status.is_setup_failure: + workload_health_setup_failures += 1 elif status.passed: workload_health_passed += 1 elif status.is_regression: @@ -83,6 +92,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += f"- ask_holmes: {ask_holmes_passed}/{ask_holmes_total} test cases were successful, {ask_holmes_regressions} regressions" if ask_holmes_skipped > 0: markdown += f", {ask_holmes_skipped} skipped" + if ask_holmes_setup_failures > 0: + markdown += f", {ask_holmes_setup_failures} setup failures" if ask_holmes_mock_failures > 0: markdown += f", {ask_holmes_mock_failures} mock failures" markdown += "\n" @@ -90,6 +101,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += f"- investigate: {investigate_passed}/{investigate_total} test cases were successful, {investigate_regressions} regressions" if investigate_skipped > 0: markdown += f", {investigate_skipped} skipped" + if investigate_setup_failures > 0: + markdown += f", {investigate_setup_failures} setup failures" if investigate_mock_failures > 0: markdown += f", {investigate_mock_failures} mock failures" markdown += "\n" @@ -97,6 +110,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += f"- workload_health: {workload_health_passed}/{workload_health_total} test cases were successful, {workload_health_regressions} regressions" if workload_health_skipped > 0: markdown += f", {workload_health_skipped} skipped" + if workload_health_setup_failures > 0: + markdown += f", {workload_health_setup_failures} setup failures" if workload_health_mock_failures > 0: markdown += f", {workload_health_mock_failures} mock failures" markdown += "\n" @@ -125,6 +140,9 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += ( "\n- :warning: the test failed but is known to be flaky or known to fail" ) + markdown += ( + "\n- :construction: the test had a setup failure (not a code regression)" + ) markdown += ( "\n- :wrench: the test failed due to mock data issues (not a code regression)" ) diff --git a/tests/llm/utils/test_results.py b/tests/llm/utils/test_results.py index 225ef72ce8..10747f6ef4 100644 --- a/tests/llm/utils/test_results.py +++ b/tests/llm/utils/test_results.py @@ -73,7 +73,12 @@ def is_skipped(self) -> bool: @property def is_regression(self) -> bool: - if self.is_skipped or self.passed or self.is_mock_failure: + if ( + self.is_skipped + or self.passed + or self.is_mock_failure + or self.is_setup_failure + ): return False # Known failure (expected to fail) if self.actual_score == 0 and self.expected_score == 0: From 5e05d77bb1cfb155dd07c08c88d91ba53ca3f879 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 13:49:18 +0300 Subject: [PATCH 03/10] Update llm-evaluation.yaml --- .github/workflows/llm-evaluation.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index 47e4da8ebd..b19302a3f8 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -88,14 +88,21 @@ jobs: # Configure kubectl kubectl cluster-info --context kind-kind + + # Wait for all nodes to be ready + echo "Waiting for nodes to be ready..." + kubectl wait --for=condition=Ready nodes --all --timeout=300s kubectl get nodes # Wait for all system pods to be ready + echo "Waiting for system pods to be ready..." kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=300s # Verify cluster is working by creating a test pod + echo "Creating test pod to verify cluster..." kubectl run test-pod --image=busybox:1.35 --restart=Never -- echo "Cluster is ready" - kubectl wait --for=condition=Ready pod/test-pod --timeout=30s + kubectl wait --for=condition=Ready pod/test-pod --timeout=60s + kubectl logs test-pod kubectl delete pod test-pod # Show cluster info for debugging From 7e938082291ff1538695cd69c5561765b85a6c4d Mon Sep 17 00:00:00 2001 From: Natan Yellin Date: Thu, 14 Aug 2025 13:59:24 +0300 Subject: [PATCH 04/10] Update tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml index 918dd1c6a9..d4c4a6e06d 100644 --- a/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml @@ -30,7 +30,7 @@ before_test: | memory: "50Mi" EOF # Wait for pod to be created and start running - kubectl wait --for=condition=Ready pod/giant-narwhal-6958c5bdd8-69gtn -n app-02 --timeout=30s + kubectl wait --for=condition=Ready pod/giant-narwhal-6958c5bdd8-69gtn -n app-02 --timeout=30s || true # Wait for the pod to be OOMKilled (check every 2 seconds for up to 120 seconds) echo "Waiting for pod to be OOMKilled..." for i in {1..60}; do From 6d4d9a7f8a3475c378b42816318772a880d46d30 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 14:01:05 +0300 Subject: [PATCH 05/10] Update test_case.yaml --- .../test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml index 918dd1c6a9..6819d55e24 100644 --- a/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/02_what_is_wrong_with_pod/test_case.yaml @@ -1,4 +1,4 @@ -user_prompt: 'What is wrong with << { "type": "pod", "name": "giant-narwhal-6958c5bdd8-69gtn" } >> ?' +user_prompt: 'What is wrong with << { "type": "pod", "name": "giant-narwhal-6958c5bdd8-69gtn", "namespace": "app-02" } >> ?' expected_output: - The pod was killed due to it being out of memory From c7953cce6b6fb0c1825eedec70d65f7c577b4b25 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 21:01:47 +0300 Subject: [PATCH 06/10] increase parallelism --- .github/workflows/llm-evaluation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index b19302a3f8..27a9f5ccaf 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -128,7 +128,7 @@ jobs: EXPERIMENT_ID: github-${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }} GENERATE_REGRESSIONS_FILE: "true" run: | - poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -n6 -m 'llm and easy' + poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -n20 -m 'llm and easy' - uses: actions/github-script@v7 if: always() with: From c8221b5a4066084a1e4c106592e21ebd38d90e3c Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 22:00:41 +0300 Subject: [PATCH 07/10] Update llm-evaluation.yaml --- .github/workflows/llm-evaluation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index 27a9f5ccaf..2c8d267c01 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -101,7 +101,7 @@ jobs: # Verify cluster is working by creating a test pod echo "Creating test pod to verify cluster..." kubectl run test-pod --image=busybox:1.35 --restart=Never -- echo "Cluster is ready" - kubectl wait --for=condition=Ready pod/test-pod --timeout=60s + kubectl wait --for=condition=Ready pod/test-pod --timeout=120s kubectl logs test-pod kubectl delete pod test-pod From d2b286358f4576d529d4a41ffc04d9829d03bbce Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 23:22:27 +0300 Subject: [PATCH 08/10] Update llm-evaluation.yaml --- .github/workflows/llm-evaluation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index 2c8d267c01..b11de9f72d 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -101,7 +101,7 @@ jobs: # Verify cluster is working by creating a test pod echo "Creating test pod to verify cluster..." kubectl run test-pod --image=busybox:1.35 --restart=Never -- echo "Cluster is ready" - kubectl wait --for=condition=Ready pod/test-pod --timeout=120s + kubectl wait --for=condition=Ready pod/test-pod --timeout=60s kubectl logs test-pod kubectl delete pod test-pod @@ -128,7 +128,7 @@ jobs: EXPERIMENT_ID: github-${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }} GENERATE_REGRESSIONS_FILE: "true" run: | - poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -n20 -m 'llm and easy' + poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -s -n10 -m 'llm and easy' - uses: actions/github-script@v7 if: always() with: From c24d9ce62336b6b0ceac4d6e1c11cabd580fd1f6 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 14 Aug 2025 23:22:58 +0300 Subject: [PATCH 09/10] Update llm-evaluation.yaml --- .github/workflows/llm-evaluation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index b11de9f72d..f6ab5bd46b 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -101,7 +101,7 @@ jobs: # Verify cluster is working by creating a test pod echo "Creating test pod to verify cluster..." kubectl run test-pod --image=busybox:1.35 --restart=Never -- echo "Cluster is ready" - kubectl wait --for=condition=Ready pod/test-pod --timeout=60s + kubectl wait --for=condition=Ready pod/test-pod --timeout=120s kubectl logs test-pod kubectl delete pod test-pod From d6d76112e08bba325ca9304eef3dac104b9f6703 Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Sat, 16 Aug 2025 11:38:09 +0300 Subject: [PATCH 10/10] faster test --- .../18_crash_looping_v2/test_case.yaml | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/llm/fixtures/test_ask_holmes/18_crash_looping_v2/test_case.yaml b/tests/llm/fixtures/test_ask_holmes/18_crash_looping_v2/test_case.yaml index 5577fef64a..dbc8ba3ee7 100644 --- a/tests/llm/fixtures/test_ask_holmes/18_crash_looping_v2/test_case.yaml +++ b/tests/llm/fixtures/test_ask_holmes/18_crash_looping_v2/test_case.yaml @@ -2,10 +2,32 @@ user_prompt: "What is the issue with db-certs-authenticator?" expected_output: - The `db-certs-authenticator` pod experienced an expired SSL certificate. before_test: | - kubectl apply -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/crashpod.v2/crashloop-cert-app.yaml - sleep 120 + kubectl create namespace app-18 --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/crashpod.v2/crashloop-cert-app.yaml -n app-18 + # Poll for pod to crash and enter error/crashloop state in app-18 namespace + timeout=360 + elapsed=0 + while [ $elapsed -lt $timeout ]; do + # Check if pod exists and get its status + pod_status=$(kubectl get pods -n app-18 -l app=flask -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "") + restart_count=$(kubectl get pods -n app-18 -l app=flask -o jsonpath='{.items[0].status.containerStatuses[0].restartCount}' 2>/dev/null || echo "0") + waiting_reason=$(kubectl get pods -n app-18 -l app=flask -o jsonpath='{.items[0].status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || echo "") + + # Success if pod is in CrashLoopBackOff or has restarted at least once + if [ "$waiting_reason" = "CrashLoopBackOff" ] || [ "$restart_count" -gt "0" ]; then + echo "Pod has crashed (status: $waiting_reason, restarts: $restart_count)" + exit 0 + fi + + echo "Waiting for pod to crash... Status: $pod_status, Restarts: $restart_count" + sleep 10 + elapsed=$((elapsed + 10)) + done + echo "Timeout waiting for pod to crash" + exit 1 after_test: | - kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/crashpod.v2/crashloop-cert-app.yaml + kubectl delete -f https://raw.githubusercontent.com/robusta-dev/kubernetes-demos/main/crashpod.v2/crashloop-cert-app.yaml -n app-18 + kubectl delete namespace app-18 tags: - easy - kubernetes