Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions .github/workflows/llm-evaluation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,86 @@ jobs:
poetry config virtualenvs.create false
poetry install --no-root

- name: Install kubectl
run: |
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
chmod +x kubectl
sudo mv kubectl /usr/local/bin/
kubectl version --client

Comment thread
aantn marked this conversation as resolved.
Resolved
- name: Install KIND
run: |
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin/kind
kind version

Comment thread
aantn marked this conversation as resolved.
Resolved
- name: Create KIND cluster
run: |
cat <<EOF > kind-config.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
extraPortMappings:
- containerPort: 30000
hostPort: 30000
protocol: TCP
- containerPort: 30001
hostPort: 30001
protocol: TCP
kubeadmConfigPatches:
- |
kind: InitConfiguration
nodeRegistration:
kubeletExtraArgs:
max-pods: "200"
- |
kind: KubeProxyConfiguration
metricsBindAddress: "0.0.0.0:10249"
- |
kind: KubeletConfiguration
maxPods: 200
- role: worker
kubeadmConfigPatches:
- |
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
max-pods: "200"
- |
kind: KubeletConfiguration
maxPods: 200
EOF

Comment thread
aantn marked this conversation as resolved.
Resolved
# Create cluster with resource limits appropriate for GitHub Actions (7GB RAM, 2 CPU)
kind create cluster --config kind-config.yaml --wait 5m

# Configure kubectl
kubectl cluster-info --context kind-kind
kubectl get nodes

# Wait for all system pods to be ready
kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=300s

Comment thread
aantn marked this conversation as resolved.
Resolved
# Verify cluster is working by creating a test pod
kubectl run test-pod --image=busybox:1.35 --restart=Never -- echo "Cluster is ready"
kubectl wait --for=condition=Ready pod/test-pod --timeout=30s
kubectl delete pod test-pod

Comment thread
aantn marked this conversation as resolved.
Resolved
# Show cluster info for debugging
echo "=== Cluster nodes ==="
kubectl get nodes -o wide
echo "=== System pods ==="
kubectl get pods -n kube-system
echo "=== Cluster is ready for tests ==="

- name: Run tests
id: evals
continue-on-error: true
shell: bash
env:
RUN_LIVE: "true"
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,23 @@ before_test: |
requests:
memory: "50Mi"
EOF
sleep 60
# Wait for pod to be created and start running
kubectl wait --for=condition=Ready pod/giant-narwhal-6958c5bdd8-69gtn -n app-02 --timeout=30s
Comment thread
aantn marked this conversation as resolved.
Resolved
# Wait for the pod to be OOMKilled (check every 2 seconds for up to 120 seconds)
echo "Waiting for pod to be OOMKilled..."
for i in {1..60}; do
STATUS=$(kubectl get pod giant-narwhal-6958c5bdd8-69gtn -n app-02 -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' 2>/dev/null || echo "")
if [ "$STATUS" = "OOMKilled" ]; then
echo "Pod was OOMKilled successfully"
break
fi
if [ $i -eq 60 ]; then
echo "Timeout waiting for OOMKill after 120 seconds"
kubectl describe pod giant-narwhal-6958c5bdd8-69gtn -n app-02
exit 1
fi
sleep 2
done

after_test: |
kubectl delete namespace app-02
18 changes: 18 additions & 0 deletions tests/llm/utils/reporting/github_reporter.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,21 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict
ask_holmes_regressions = 0
ask_holmes_mock_failures = 0
ask_holmes_skipped = 0
ask_holmes_setup_failures = 0

investigate_total = 0
investigate_passed = 0
investigate_regressions = 0
investigate_mock_failures = 0
investigate_skipped = 0
investigate_setup_failures = 0

workload_health_total = 0
workload_health_passed = 0
workload_health_regressions = 0
workload_health_mock_failures = 0
workload_health_skipped = 0
workload_health_setup_failures = 0

for result in sorted_results:
status = TestStatus(result)
Expand All @@ -51,6 +54,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict
ask_holmes_total += 1
if status.is_skipped:
ask_holmes_skipped += 1
elif status.is_setup_failure:
ask_holmes_setup_failures += 1
elif status.passed:
ask_holmes_passed += 1
elif status.is_regression:
Expand All @@ -61,6 +66,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict
investigate_total += 1
if status.is_skipped:
investigate_skipped += 1
elif status.is_setup_failure:
investigate_setup_failures += 1
elif status.passed:
investigate_passed += 1
elif status.is_regression:
Expand All @@ -71,6 +78,8 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict
workload_health_total += 1
if status.is_skipped:
workload_health_skipped += 1
elif status.is_setup_failure:
workload_health_setup_failures += 1
elif status.passed:
workload_health_passed += 1
elif status.is_regression:
Expand All @@ -83,20 +92,26 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict
markdown += f"- ask_holmes: {ask_holmes_passed}/{ask_holmes_total} test cases were successful, {ask_holmes_regressions} regressions"
if ask_holmes_skipped > 0:
markdown += f", {ask_holmes_skipped} skipped"
if ask_holmes_setup_failures > 0:
markdown += f", {ask_holmes_setup_failures} setup failures"
if ask_holmes_mock_failures > 0:
markdown += f", {ask_holmes_mock_failures} mock failures"
markdown += "\n"
if investigate_total > 0:
markdown += f"- investigate: {investigate_passed}/{investigate_total} test cases were successful, {investigate_regressions} regressions"
if investigate_skipped > 0:
markdown += f", {investigate_skipped} skipped"
if investigate_setup_failures > 0:
markdown += f", {investigate_setup_failures} setup failures"
if investigate_mock_failures > 0:
markdown += f", {investigate_mock_failures} mock failures"
markdown += "\n"
if workload_health_total > 0:
markdown += f"- workload_health: {workload_health_passed}/{workload_health_total} test cases were successful, {workload_health_regressions} regressions"
if workload_health_skipped > 0:
markdown += f", {workload_health_skipped} skipped"
if workload_health_setup_failures > 0:
markdown += f", {workload_health_setup_failures} setup failures"
if workload_health_mock_failures > 0:
markdown += f", {workload_health_mock_failures} mock failures"
markdown += "\n"
Expand Down Expand Up @@ -125,6 +140,9 @@ def generate_markdown_report(sorted_results: List[dict]) -> Tuple[str, List[dict
markdown += (
"\n- :warning: the test failed but is known to be flaky or known to fail"
)
markdown += (
"\n- :construction: the test had a setup failure (not a code regression)"
)
markdown += (
"\n- :wrench: the test failed due to mock data issues (not a code regression)"
)
Expand Down
7 changes: 6 additions & 1 deletion tests/llm/utils/test_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,12 @@ def is_skipped(self) -> bool:

@property
def is_regression(self) -> bool:
if self.is_skipped or self.passed or self.is_mock_failure:
if (
self.is_skipped
or self.passed
or self.is_mock_failure
or self.is_setup_failure
):
return False
# Known failure (expected to fail)
if self.actual_score == 0 and self.expected_score == 0:
Expand Down