diff --git a/scripts/test_providers.sh b/scripts/test_providers.sh index e8fdc28fb005..93d25149e8d8 100755 --- a/scripts/test_providers.sh +++ b/scripts/test_providers.sh @@ -28,13 +28,14 @@ fi SCRIPT_DIR=$(pwd) +# Format: "provider -> model1|model2|model3" PROVIDERS=( - "openrouter:google/gemini-2.5-pro:google/gemini-2.5-flash:anthropic/claude-sonnet-4.5:qwen/qwen3-coder:z-ai/glm-4.6" - "xai:grok-3" - "openai:gpt-4o:gpt-4o-mini:gpt-3.5-turbo:gpt-5" - "anthropic:claude-sonnet-4-5-20250929:claude-opus-4-1-20250805" - "google:gemini-2.5-pro:gemini-2.5-flash:gemini-3-pro-preview:gemini-3-flash-preview" - "tetrate:claude-sonnet-4-20250514" + "openrouter -> google/gemini-2.5-pro|anthropic/claude-sonnet-4.5|qwen/qwen3-coder:exacto|z-ai/glm-4.6:exacto|nvidia/nemotron-3-nano-30b-a3b" + "xai -> grok-3" + "openai -> gpt-4o|gpt-4o-mini|gpt-3.5-turbo|gpt-5" + "anthropic -> claude-sonnet-4-5-20250929|claude-opus-4-1-20250805" + "google -> gemini-2.5-pro|gemini-2.5-flash|gemini-3-pro-preview|gemini-3-flash-preview" + "tetrate -> claude-sonnet-4-20250514" ) # In CI, only run Databricks tests if DATABRICKS_HOST and DATABRICKS_TOKEN are set @@ -42,13 +43,13 @@ PROVIDERS=( if [ -n "$CI" ]; then if [ -n "$DATABRICKS_HOST" ] && [ -n "$DATABRICKS_TOKEN" ]; then echo "✓ Including Databricks tests" - PROVIDERS+=("databricks:databricks-claude-sonnet-4:gemini-2-5-flash:gpt-4o") + PROVIDERS+=("databricks -> databricks-claude-sonnet-4|gemini-2-5-flash|gpt-4o") else echo "⚠️ Skipping Databricks tests (DATABRICKS_HOST and DATABRICKS_TOKEN required in CI)" fi else echo "✓ Including Databricks tests" - PROVIDERS+=("databricks:databricks-claude-sonnet-4:gemini-2-5-flash:gpt-4o") + PROVIDERS+=("databricks -> databricks-claude-sonnet-4|gemini-2-5-flash|gpt-4o") fi # Configure mode-specific settings @@ -73,10 +74,12 @@ echo "" RESULTS=() for provider_config in "${PROVIDERS[@]}"; do - IFS=':' read -ra PARTS <<< "$provider_config" - PROVIDER="${PARTS[0]}" - for i in $(seq 1 $((${#PARTS[@]} - 1))); do - MODEL="${PARTS[$i]}" + # Split on " -> " to get provider and models + PROVIDER="${provider_config%% -> *}" + MODELS_STR="${provider_config#* -> }" + # Split models on "|" + IFS='|' read -ra MODELS <<< "$MODELS_STR" + for MODEL in "${MODELS[@]}"; do export GOOSE_PROVIDER="$PROVIDER" export GOOSE_MODEL="$MODEL" TESTDIR=$(mktemp -d)