diff --git a/pyproject.toml b/pyproject.toml index 48cc0c26..6a9fa21e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ langchain = [ "openinference-instrumentation-langchain>=0.1.0", "opentelemetry-instrumentation-langchain>=0.40.0,<0.62.0", ] +adk = ["google-adk>=2.0.0,<3.0.0"] [tool.ruff] line-length = 120 @@ -73,7 +74,7 @@ include = ["src/**/*.py", "tests/**/*.py", "tests_integ/**/*.py"] [tool.hatch.envs.hatch-test] installer = "uv" extra-args = ["-n", "auto", "-vv"] -features = ["otel", "langfuse", "langchain", "opensearch"] +features = ["otel", "langfuse", "langchain", "opensearch", "adk"] dependencies = [ "pytest>=8.0.0,<10.0.0", "pytest-cov>=7.0.0,<8.0.0", diff --git a/tests_integ/conftest.py b/tests_integ/conftest.py index d0d69f6d..84825963 100644 --- a/tests_integ/conftest.py +++ b/tests_integ/conftest.py @@ -4,8 +4,40 @@ This conftest provides common fixtures that build on those. """ +import json +import logging +import os + +import boto3 import pytest +logger = logging.getLogger(__name__) + + +def _load_api_keys_from_secrets_manager(): + """Load API keys as environment variables from AWS Secrets Manager.""" + session = boto3.session.Session() + client = session.client(service_name="secretsmanager") + if "STRANDS_TEST_API_KEYS_SECRET_NAME" in os.environ: + try: + secret_name = os.environ["STRANDS_TEST_API_KEYS_SECRET_NAME"] + response = client.get_secret_value(SecretId=secret_name) + + if "SecretString" in response: + secret = json.loads(response["SecretString"]) + for key, value in secret.items(): + os.environ[f"{key.upper()}_API_KEY"] = str(value) + + except Exception as e: + logger.warning("Error retrieving secret: %s", e) + + +def pytest_sessionstart(session): + """Load API keys from Secrets Manager at session start.""" + os.environ.setdefault("AWS_DEFAULT_REGION", "us-east-1") + os.environ.setdefault("AWS_REGION", "us-east-1") + _load_api_keys_from_secrets_manager() + @pytest.fixture(scope="module") def evaluation_data(provider, session_id): diff --git a/tests_integ/test_adk_eval.py b/tests_integ/test_adk_eval.py new file mode 100644 index 00000000..6c3da9ed --- /dev/null +++ b/tests_integ/test_adk_eval.py @@ -0,0 +1,352 @@ +"""Integration tests for ADK agent evaluation via the ADKOtelSessionMapper. + +Requirements: + pip install strands-agents-evals[adk] + Set GOOGLE_API_KEY environment variable with a valid Google AI API key. + +Run with: pytest tests_integ/test_adk_eval.py -v +""" + +import asyncio +import os +import uuid + +import pytest +from google.adk.agents import Agent +from google.adk.models.google_llm import Gemini +from google.adk.runners import InMemoryRunner +from google.genai import types + +from strands_evals import Case, Experiment +from strands_evals.evaluators import ( + ConcisenessEvaluator, + CorrectnessEvaluator, + FaithfulnessEvaluator, + GoalSuccessRateEvaluator, + ResponseRelevanceEvaluator, + ToolParameterAccuracyEvaluator, + ToolSelectionAccuracyEvaluator, +) +from strands_evals.mappers import ADKOtelSessionMapper, detect_otel_mapper, readable_spans_to_dicts +from strands_evals.telemetry import StrandsEvalsTelemetry +from strands_evals.types.trace import AgentInvocationSpan, Session, ToolExecutionSpan + +# Uses Gemini 3 Flash via Google AI API (requires GOOGLE_API_KEY env var). +DEFAULT_MODEL_ID = "gemini-3-flash-preview" +DEFAULT_MODEL = Gemini(model=DEFAULT_MODEL_ID, google_api_key=os.environ.get("GOOGLE_API_KEY", "")) + + +# ============================================================================= +# Fixtures +# ============================================================================= + + +@pytest.fixture(scope="module") +def telemetry(): + """ADK auto-instruments via the global TracerProvider.""" + telemetry = StrandsEvalsTelemetry().setup_in_memory_exporter() + yield telemetry + + +@pytest.fixture +def weather_tool(): + def get_weather(city: str) -> str: + """Get the current weather for a city.""" + weather_data = { + "seattle": "Rainy, 55F", + "new york": "Sunny, 72F", + "london": "Cloudy, 60F", + "tokyo": "Clear, 68F", + } + city_lower = city.lower() + for c, w in weather_data.items(): + if c in city_lower: + return f"Weather in {city}: {w}" + return f"Weather in {city}: Partly cloudy, 65F" + + return get_weather + + +@pytest.fixture +def unit_converter_tool(): + def convert_units(value: float, from_unit: str, to_unit: str) -> str: + """Convert a value between units.""" + conversions = { + ("km", "miles"): lambda v: v * 0.621371, + ("miles", "km"): lambda v: v * 1.60934, + ("celsius", "fahrenheit"): lambda v: v * 9 / 5 + 32, + ("fahrenheit", "celsius"): lambda v: (v - 32) * 5 / 9, + } + key = (from_unit.lower(), to_unit.lower()) + if key in conversions: + result = conversions[key](value) + return f"{value} {from_unit} = {result:.2f} {to_unit}" + return f"Cannot convert from {from_unit} to {to_unit}" + + return convert_units + + +@pytest.fixture +def create_runner(weather_tool): + def _create(): + agent = Agent( + name="weather_agent", + model=DEFAULT_MODEL, + description="A helpful weather assistant", + instruction="You are a weather assistant. Use the get_weather tool to look up weather. Be concise.", + tools=[weather_tool], + ) + return InMemoryRunner(agent=agent, app_name="test_app") + + return _create + + +@pytest.fixture +def create_multi_tool_runner(weather_tool, unit_converter_tool): + def _create(): + agent = Agent( + name="assistant_agent", + model=DEFAULT_MODEL, + description="A helpful assistant with weather and unit conversion tools", + instruction=( + "You are a helpful assistant. Use the get_weather tool for weather questions " + "and the convert_units tool for unit conversions. Be concise." + ), + tools=[weather_tool, unit_converter_tool], + ) + return InMemoryRunner(agent=agent, app_name="test_app") + + return _create + + +@pytest.fixture +def create_multi_agent_runner(weather_tool): + def _create(): + weather_agent = Agent( + name="weather_specialist", + model=DEFAULT_MODEL, + description="Specialist agent that looks up weather information for cities", + instruction=( + "You are a weather specialist. Use the get_weather tool to answer weather questions. Be concise." + ), + tools=[weather_tool], + ) + + root_agent = Agent( + name="coordinator", + model=DEFAULT_MODEL, + description="A coordinator agent that delegates to specialists", + instruction=( + "You are a coordinator. For weather questions, delegate to the weather_specialist agent. " + "Summarize the specialist's response in one sentence." + ), + sub_agents=[weather_agent], + ) + + return InMemoryRunner(agent=root_agent, app_name="test_app") + + return _create + + +# ============================================================================= +# Helpers +# ============================================================================= + + +async def _run_adk_agent(runner: InMemoryRunner, query: str, user_id: str | None = None) -> str: + """Run an ADK agent and return the response text. + + Uses a unique user_id per call to prevent ADK from reusing conversation history. + """ + if user_id is None: + user_id = f"test_user_{uuid.uuid4().hex[:8]}" + + session = await runner.session_service.create_session(app_name=runner.app_name, user_id=user_id) + user_message = types.Content(role="user", parts=[types.Part.from_text(text=query)]) + + response_text = "" + async for event in runner.run_async(user_id=user_id, session_id=session.id, new_message=user_message): + if event.content and event.content.parts: + for part in event.content.parts: + if part.text: + response_text += part.text + + return response_text + + +# ============================================================================= +# Tests — Single Agent +# ============================================================================= + + +def test_adk_single_query(telemetry, create_runner): + """Spans are captured, mapper is auto-detected, and mapped into a valid session.""" + telemetry.in_memory_exporter.clear() + + runner = create_runner() + response = asyncio.run(_run_adk_agent(runner, "What's the weather in Seattle?")) + + spans = readable_spans_to_dicts(telemetry.in_memory_exporter.get_finished_spans()) + assert len(spans) > 0 + + mapper = detect_otel_mapper(spans) + assert isinstance(mapper, ADKOtelSessionMapper), f"Expected ADKOtelSessionMapper but got {type(mapper).__name__}" + + session = mapper.map_to_session(spans, session_id="test-single") + + assert session.session_id == "test-single" + assert len(session.traces) > 0 + assert "seattle" in response.lower() or "weather" in response.lower() + + +def test_adk_single_agent_evaluation(telemetry, create_runner): + """Single-agent session evaluates correctly (smoke test).""" + test_cases = [ + Case[str, str]( + name="weather-seattle", + input="What's the weather in Seattle?", + expected_output="The weather in Seattle is rainy and 55F.", + expected_assertion="The agent used the get_weather tool for Seattle and responded with the weather.", + ), + ] + + def task_function(case: Case) -> dict: + telemetry.in_memory_exporter.clear() + runner = create_runner() + response = asyncio.run(_run_adk_agent(runner, case.input)) + + spans = readable_spans_to_dicts(telemetry.in_memory_exporter.get_finished_spans()) + mapper = detect_otel_mapper(spans) + session = mapper.map_to_session(spans, session_id=case.session_id) + return {"output": response, "trajectory": session} + + experiment = Experiment(cases=test_cases, evaluators=[GoalSuccessRateEvaluator()]) + report = experiment.run_evaluations(task_function) + + assert len(report.scores) == 1 + assert all(report.test_passes), f"Some evaluations failed: {report.reasons}" + + +def test_adk_single_agent_multi_tool(telemetry, create_multi_tool_runner): + """Mapper distinguishes between different tool invocations in one session.""" + test_cases = [ + Case[str, str]( + name="multi-tool-weather", + input="What's the weather in Seattle?", + expected_output="The weather in Seattle is rainy and 55F.", + expected_assertion="The agent used the get_weather tool with city='Seattle'.", + ), + Case[str, str]( + name="multi-tool-conversion", + input="Convert 100 km to miles", + expected_output="100 km is approximately 62.14 miles.", + expected_assertion="The agent used the convert_units tool to convert 100 km to miles.", + ), + ] + + sessions: list = [] + + def task_function(case: Case) -> dict: + telemetry.in_memory_exporter.clear() + runner = create_multi_tool_runner() + response = asyncio.run(_run_adk_agent(runner, case.input)) + + spans = readable_spans_to_dicts(telemetry.in_memory_exporter.get_finished_spans()) + mapper = detect_otel_mapper(spans) + session = mapper.map_to_session(spans, session_id=case.session_id) + sessions.append(session) + return {"output": response, "trajectory": session} + + evaluators = [ToolSelectionAccuracyEvaluator(), ToolParameterAccuracyEvaluator()] + experiment = Experiment(cases=test_cases, evaluators=evaluators) + report = experiment.run_evaluations(task_function) + + # Exact count: 2 cases × 2 evaluators = 4 + assert len(report.scores) == 4 + assert all(report.test_passes), f"Some evaluations failed: {report.reasons}" + + # Verify mapped sessions have tool spans with tool_call_id populated + for session in sessions: + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) >= 1, "Expected at least one tool execution span" + for tool_span in tool_spans: + assert tool_span.tool_call.tool_call_id is not None, ( + f"tool_call_id should be populated, got None for tool '{tool_span.tool_call.name}'" + ) + + +# ============================================================================= +# Tests — Multi Agent +# ============================================================================= + + +def test_adk_multi_agent_evaluation(telemetry, create_multi_agent_runner): + """Full evaluator coverage: delegation + tool call spans evaluate correctly at all levels.""" + test_cases = [ + Case[str, str]( + name="multi-agent-seattle", + input="What's the weather in Seattle?", + expected_output="The weather in Seattle is rainy and 55F.", + expected_assertion="The agent obtained weather information for Seattle and responded with the result.", + ), + ] + + def task_function(case: Case) -> dict: + telemetry.in_memory_exporter.clear() + runner = create_multi_agent_runner() + response = asyncio.run(_run_adk_agent(runner, case.input)) + + spans = readable_spans_to_dicts(telemetry.in_memory_exporter.get_finished_spans()) + mapper = detect_otel_mapper(spans) + session = mapper.map_to_session(spans, session_id=case.session_id) + return {"output": response, "trajectory": session} + + evaluators = [ + GoalSuccessRateEvaluator(), + CorrectnessEvaluator(), + ResponseRelevanceEvaluator(), + FaithfulnessEvaluator(), + ConcisenessEvaluator(), + ToolSelectionAccuracyEvaluator(), + ] + + experiment = Experiment(cases=test_cases, evaluators=evaluators) + report = experiment.run_evaluations(task_function) + + # Exact count: 1 case × 6 evaluators = 6 + assert len(report.scores) == 6 + assert all(report.test_passes), f"Some evaluations failed: {report.reasons}" + + # Deserialize the trajectory from the first report case back into a Session object. + session = Session.model_validate(report.cases[0]["actual_trajectory"]) + + assert len(session.traces) == 2, ( + f"Multi-agent should produce 2 traces (coordinator + specialist), got {len(session.traces)}" + ) + + # Identify traces by agent name in their AgentInvocationSpan metadata + agent_spans = [s for t in session.traces for s in t.spans if isinstance(s, AgentInvocationSpan)] + agent_names = {s.metadata.get("agent_name") for s in agent_spans} + assert "coordinator" in agent_names, f"Expected coordinator agent, got {agent_names}" + assert "weather_specialist" in agent_names, f"Expected weather_specialist agent, got {agent_names}" + + # Tool spans should only appear on the specialist trace, not double-counted on coordinator + for trace in session.traces: + trace_agent_spans = [s for s in trace.spans if isinstance(s, AgentInvocationSpan)] + trace_tool_spans = [s for s in trace.spans if isinstance(s, ToolExecutionSpan)] + trace_agent_names = {s.metadata.get("agent_name") for s in trace_agent_spans} + + if "coordinator" in trace_agent_names and "weather_specialist" not in trace_agent_names: + tool_names = {s.tool_call.name for s in trace_tool_spans} + assert "get_weather" not in tool_names, f"Specialist tool leaked onto coordinator trace: {tool_names}" + assert tool_names <= {"transfer_to_agent"}, f"Unexpected tools on coordinator trace: {tool_names}" + elif "weather_specialist" in trace_agent_names: + assert len(trace_tool_spans) >= 1, "Specialist trace should have at least one tool span" + # Verify tool_call_id is populated + for tool_span in trace_tool_spans: + assert tool_span.tool_call.tool_call_id is not None, ( + f"tool_call_id should be populated, got None for tool '{tool_span.tool_call.name}'" + ) + # get_weather should be the tool used + tool_names = {s.tool_call.name for s in trace_tool_spans} + assert "get_weather" in tool_names, f"Expected get_weather tool, got {tool_names}"