diff --git a/.github/workflows/llm-evaluation.yaml b/.github/workflows/llm-evaluation.yaml index 6e45c782f2..f6ab5bd46b 100644 --- a/.github/workflows/llm-evaluation.yaml +++ b/.github/workflows/llm-evaluation.yaml @@ -31,11 +31,93 @@ jobs: poetry config virtualenvs.create false poetry install --no-root + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + kubectl version --client + + - name: Install KIND + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + kind version + + - name: Create KIND cluster + run: | + cat < kind-config.yaml + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + nodes: + - role: control-plane + extraPortMappings: + - containerPort: 30000 + hostPort: 30000 + protocol: TCP + - containerPort: 30001 + hostPort: 30001 + protocol: TCP + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "200" + - | + kind: KubeProxyConfiguration + metricsBindAddress: "0.0.0.0:10249" + - | + kind: KubeletConfiguration + maxPods: 200 + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "200" + - | + kind: KubeletConfiguration + maxPods: 200 + EOF + + # Create cluster with resource limits appropriate for GitHub Actions (7GB RAM, 2 CPU) + kind create cluster --config kind-config.yaml --wait 5m + + # Configure kubectl + kubectl cluster-info --context kind-kind + + # Wait for all nodes to be ready + echo "Waiting for nodes to be ready..." + kubectl wait --for=condition=Ready nodes --all --timeout=300s + kubectl get nodes + + # Wait for all system pods to be ready + echo "Waiting for system pods to be ready..." + kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=300s + + # Verify cluster is working by creating a test pod + echo "Creating test pod to verify cluster..." + kubectl run test-pod --image=busybox:1.35 --restart=Never -- echo "Cluster is ready" + kubectl wait --for=condition=Ready pod/test-pod --timeout=120s + kubectl logs test-pod + kubectl delete pod test-pod + + # Show cluster info for debugging + echo "=== Cluster nodes ===" + kubectl get nodes -o wide + echo "=== System pods ===" + kubectl get pods -n kube-system + echo "=== Cluster is ready for tests ===" + - name: Run tests id: evals continue-on-error: true shell: bash env: + RUN_LIVE: "true" AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} @@ -46,7 +128,7 @@ jobs: EXPERIMENT_ID: github-${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }} GENERATE_REGRESSIONS_FILE: "true" run: | - poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -n6 -m 'llm and easy' + poetry run pytest --no-cov tests/llm/test_ask_holmes.py tests/llm/test_investigate.py -s -n10 -m 'llm and easy' - uses: actions/github-script@v7 if: always() with: diff --git a/tests/llm/utils/reporting/github_reporter.py b/tests/llm/utils/reporting/github_reporter.py index 8842263a59..5a86cee881 100644 --- a/tests/llm/utils/reporting/github_reporter.py +++ b/tests/llm/utils/reporting/github_reporter.py @@ -31,18 +31,21 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict ask_holmes_regressions = 0 ask_holmes_mock_failures = 0 ask_holmes_skipped = 0 + ask_holmes_setup_failures = 0 investigate_total = 0 investigate_passed = 0 investigate_regressions = 0 investigate_mock_failures = 0 investigate_skipped = 0 + investigate_setup_failures = 0 workload_health_total = 0 workload_health_passed = 0 workload_health_regressions = 0 workload_health_mock_failures = 0 workload_health_skipped = 0 + workload_health_setup_failures = 0 for result in sorted_results: status = TestStatus(result) @@ -51,6 +54,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict ask_holmes_total += 1 if status.is_skipped: ask_holmes_skipped += 1 + elif status.is_setup_failure: + ask_holmes_setup_failures += 1 elif status.passed: ask_holmes_passed += 1 elif status.is_regression: @@ -61,6 +66,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict investigate_total += 1 if status.is_skipped: investigate_skipped += 1 + elif status.is_setup_failure: + investigate_setup_failures += 1 elif status.passed: investigate_passed += 1 elif status.is_regression: @@ -71,6 +78,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict workload_health_total += 1 if status.is_skipped: workload_health_skipped += 1 + elif status.is_setup_failure: + workload_health_setup_failures += 1 elif status.passed: workload_health_passed += 1 elif status.is_regression: @@ -83,6 +92,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += f"- ask_holmes: {ask_holmes_passed}/{ask_holmes_total} test cases were successful, {ask_holmes_regressions} regressions" if ask_holmes_skipped > 0: markdown += f", {ask_holmes_skipped} skipped" + if ask_holmes_setup_failures > 0: + markdown += f", {ask_holmes_setup_failures} setup failures" if ask_holmes_mock_failures > 0: markdown += f", {ask_holmes_mock_failures} mock failures" markdown += "\n" @@ -90,6 +101,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += f"- investigate: {investigate_passed}/{investigate_total} test cases were successful, {investigate_regressions} regressions" if investigate_skipped > 0: markdown += f", {investigate_skipped} skipped" + if investigate_setup_failures > 0: + markdown += f", {investigate_setup_failures} setup failures" if investigate_mock_failures > 0: markdown += f", {investigate_mock_failures} mock failures" markdown += "\n" @@ -97,6 +110,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += f"- workload_health: {workload_health_passed}/{workload_health_total} test cases were successful, {workload_health_regressions} regressions" if workload_health_skipped > 0: markdown += f", {workload_health_skipped} skipped" + if workload_health_setup_failures > 0: + markdown += f", {workload_health_setup_failures} setup failures" if workload_health_mock_failures > 0: markdown += f", {workload_health_mock_failures} mock failures" markdown += "\n" @@ -125,6 +140,9 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict markdown += ( "\n- :warning: the test failed but is known to be flaky or known to fail" ) + markdown += ( + "\n- :construction: the test had a setup failure (not a code regression)" + ) markdown += ( "\n- :wrench: the test failed due to mock data issues (not a code regression)" ) diff --git a/tests/llm/utils/test_results.py b/tests/llm/utils/test_results.py index 225ef72ce8..10747f6ef4 100644 --- a/tests/llm/utils/test_results.py +++ b/tests/llm/utils/test_results.py @@ -73,7 +73,12 @@ def is_skipped(self) -> bool: @property def is_regression(self) -> bool: - if self.is_skipped or self.passed or self.is_mock_failure: + if ( + self.is_skipped + or self.passed + or self.is_mock_failure + or self.is_setup_failure + ): return False # Known failure (expected to fail) if self.actual_score == 0 and self.expected_score == 0: