ci: enable integ tests for anthropic, cohere, mistral, openai, writer

.github/workflows/integration-test.yml

@@ -67,6 +67,7 @@ jobs:
         env:
           AWS_REGION: us-east-1
           AWS_REGION_NAME: us-east-1 # Needed for LiteLLM
+          STRANDS_TEST_API_KEYS_SECRET_NAME: ${{ secrets.STRANDS_TEST_API_KEYS_SECRET_NAME }}
         id: tests
         run: |
           hatch test tests_integ

tests_integ/conftest.py

@@ -1,5 +1,17 @@
+import json
+import logging
+import os
+
+import boto3
 import pytest
 
+logger = logging.getLogger(__name__)
+
+
+def pytest_sessionstart(session):
+    _load_api_keys_from_secrets_manager()
+
+
 ## Data
 
 
@@ -28,3 +40,44 @@ async def alist(items):
         return [item async for item in items]
 
     return alist
+
+
+## Models
+
+
+def _load_api_keys_from_secrets_manager():
+    """Load API keys as environment variables from AWS Secrets Manager."""
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager")
+    try:
+        secret_name = os.getenv("STRANDS_TEST_API_KEYS_SECRET_NAME")
+        response = client.get_secret_value(SecretId=secret_name)
+
+        if "SecretString" in response:
+            secret = json.loads(response["SecretString"])
+            for key, value in secret.items():
+                os.environ[f"{key.upper()}_API_KEY"] = str(value)
+        else:
+            raise Exception("No API keys found in secrets manager")
+
+    except Exception as e:
+        logger.warning("Error retrieving secret", e)
+
+    """
+    Validate that required environment variables are set when running in GitHub Actions.
+    This prevents tests from being unintentionally skipped due to missing credentials.
+    """
+    if os.environ.get("GITHUB_ACTIONS") != "true":
+        logger.warning("Tests running outside GitHub Actions, skipping required provider validation")
+        return
+
+    required_providers = {
+        "ANTHROPIC_API_KEY",
+        "COHERE_API_KEY",
+        "MISTRAL_API_KEY",
+        "OPENAI_API_KEY",
+        "WRITER_API_KEY",
+    }
+    for provider in required_providers:
+        if provider not in os.environ or not os.environ[provider]:
+            raise ValueError(f"Missing required environment variables for {provider}")

tests_integ/models/providers.py

@@ -72,11 +72,11 @@ def __init__(self):
 bedrock = ProviderInfo(id="bedrock", factory=lambda: BedrockModel())
 cohere = ProviderInfo(
     id="cohere",
-    environment_variable="CO_API_KEY",
+    environment_variable="COHERE_API_KEY",
     factory=lambda: OpenAIModel(
         client_args={
             "base_url": "https://api.cohere.com/compatibility/v1",
-            "api_key": os.getenv("CO_API_KEY"),
+            "api_key": os.getenv("COHERE_API_KEY"),
         },
         model_id="command-a-03-2025",
         params={"stream_options": None},

tests_integ/models/conformance.py → tests_integ/models/test_conformance.py

@@ -1,6 +1,6 @@
 import pytest
 
-from strands.types.models import Model
+from strands.models import Model
 from tests_integ.models.providers import ProviderInfo, all_providers
 
 

tests_integ/models/test_model_anthropic.py

@@ -8,8 +8,16 @@
 from strands.models.anthropic import AnthropicModel
 from tests_integ.models import providers
 
-# these tests only run if we have the anthropic api key
-pytestmark = providers.anthropic.mark
+"""
+These tests only run if we have the anthropic api key
+
+Because of infrequent burst usage Anthropic is occasionally failing tests with 529s. We retrying with delay to
+avoid these false negatives.
+
+{'type': 'error', 'error': {'details': None, 'type': 'overloaded_error', 'message': 'Overloaded'}}
+https://docs.anthropic.com/en/api/errors#http-errors
+"""
+pytestmark = [providers.anthropic.mark, pytest.mark.flaky(reruns=2, reruns_delay=5)]
 
 
 @pytest.fixture