From 6667c195ad89cd8d84a97250820684acce1778c2 Mon Sep 17 00:00:00 2001 From: openhands Date: Wed, 18 Feb 2026 21:58:25 +0000 Subject: [PATCH] fix: Move litellm install before model loading and use correct API key Fixes two issues blocking all evaluation runs: 1. Import order: Move 'Install dependencies' step before 'Load model IDs' step to ensure litellm is available when resolve_model_config.py imports it. Previously caused ModuleNotFoundError. 2. API key: Change from secrets.LLM_API_KEY to secrets.LLM_API_KEY_EVAL to match the integration-runner workflow which uses the correct secret. Fixes #2117 Co-authored-by: openhands --- .github/workflows/run-eval.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-eval.yml b/.github/workflows/run-eval.yml index 565abec7b67..486d2b2b80b 100644 --- a/.github/workflows/run-eval.yml +++ b/.github/workflows/run-eval.yml @@ -144,6 +144,10 @@ jobs: run: | python3 .github/run-eval/validate_sdk_ref.py + - name: Install dependencies + run: | + pip install 'litellm>=1.81.0' + - name: Load model IDs from Python script id: load-models run: | @@ -244,15 +248,11 @@ jobs: printf 'pr_number=%s\n' "$PR_NUMBER" >> "$GITHUB_OUTPUT" printf 'trigger_desc=%s\n' "$TRIGGER_DESCRIPTION" >> "$GITHUB_OUTPUT" - - name: Install dependencies - run: | - pip install 'litellm>=1.81.0' - - name: Resolve model configurations and verify availability id: resolve-models env: MODEL_IDS: ${{ steps.params.outputs.models }} - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_API_KEY: ${{ secrets.LLM_API_KEY_EVAL }} LLM_BASE_URL: https://llm-proxy.eval.all-hands.dev run: | python3 .github/run-eval/resolve_model_config.py