diff --git a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/event_emitter.py b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/event_emitter.py index 827a62f815..396b2c855b 100644 --- a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/event_emitter.py +++ b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/event_emitter.py @@ -1,5 +1,6 @@ from dataclasses import asdict from enum import Enum +import json from typing import Optional, Union from opentelemetry._events import Event, EventLogger @@ -127,8 +128,29 @@ def emit_streaming_response_events( event_logger: Optional[EventLogger], complete_response: dict ): for message in complete_response.get("events", []): - emit_event( - ChoiceEvent( + # Parse tool calls + if message.get("type") == "tool_use": + tool_calls = [ + ToolCall( + id=message.get("id"), + function={ + "name": message.get("name"), + "arguments": json.loads(message.get("input", '{}')), + }, + type="function", + ) + ] + event = ChoiceEvent( + index=message.get("index", 0), + message={ + "content": None, + "role": message.get("role", "assistant"), + }, + finish_reason=message.get("finish_reason", "unknown"), + tool_calls=tool_calls, + ) + else: + event = ChoiceEvent( index=message.get("index", 0), message={ "content": { @@ -138,9 +160,8 @@ def emit_streaming_response_events( "role": message.get("role", "assistant"), }, finish_reason=message.get("finish_reason", "unknown"), - ), - event_logger, - ) + ) + emit_event(event, event_logger) def emit_event( diff --git a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/span_utils.py b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/span_utils.py index da7baa2063..9ac7568b1e 100644 --- a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/span_utils.py +++ b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/span_utils.py @@ -113,18 +113,43 @@ async def aset_input_attributes(span, kwargs): ) for i, message in enumerate(kwargs.get("messages")): prompt_index = i + (1 if has_system_message else 0) + content = message.get("content") + tool_use_blocks = [] + other_blocks = [] + if isinstance(content, list): + for block in content: + if dict(block).get("type") == "tool_use": + tool_use_blocks.append(dict(block)) + else: + other_blocks.append(block) + content = other_blocks set_span_attribute( span, f"{SpanAttributes.LLM_PROMPTS}.{prompt_index}.content", - await _dump_content( - message_index=i, span=span, content=message.get("content") - ), + await _dump_content(message_index=i, span=span, content=content), ) set_span_attribute( span, f"{SpanAttributes.LLM_PROMPTS}.{prompt_index}.role", message.get("role"), ) + if tool_use_blocks: + for tool_num, tool_use_block in enumerate(tool_use_blocks): + set_span_attribute( + span, + f"{SpanAttributes.LLM_PROMPTS}.{prompt_index}.tool_calls.{tool_num}.id", + tool_use_block.get("id"), + ) + set_span_attribute( + span, + f"{SpanAttributes.LLM_PROMPTS}.{prompt_index}.tool_calls.{tool_num}.name", + tool_use_block.get("name"), + ) + set_span_attribute( + span, + f"{SpanAttributes.LLM_PROMPTS}.{prompt_index}.tool_calls.{tool_num}.arguments", + json.dumps(tool_use_block.get("input")), + ) if kwargs.get("tools") is not None: for i, tool in enumerate(kwargs.get("tools")): @@ -160,7 +185,7 @@ def _set_span_completions(span, response): content_block_type = content.type # usually, Antrhopic responds with just one text block, # but the API allows for multiple text blocks, so concatenate them - if content_block_type == "text": + if content_block_type == "text" and hasattr(content, "text"): text += content.text elif content_block_type == "thinking": content = dict(content) @@ -242,15 +267,32 @@ def set_streaming_response_attributes(span, complete_response_events): if not span.is_recording() or not complete_response_events: return - try: - for event in complete_response_events: - index = event.get("index") - prefix = f"{SpanAttributes.LLM_COMPLETIONS}.{index}" + index = 0 + for event in complete_response_events: + prefix = f"{SpanAttributes.LLM_COMPLETIONS}.{index}" + set_span_attribute(span, f"{prefix}.finish_reason", event.get("finish_reason")) + role = "thinking" if event.get("type") == "thinking" else "assistant" + # Thinking is added as a separate completion, so we need to increment the index + if event.get("type") == "thinking": + index += 1 + set_span_attribute(span, f"{prefix}.role", role) + if event.get("type") == "tool_use": + set_span_attribute( + span, + f"{prefix}.tool_calls.0.id", + event.get("id"), + ) set_span_attribute( - span, f"{prefix}.finish_reason", event.get("finish_reason") + span, + f"{prefix}.tool_calls.0.name", + event.get("name"), ) - role = "thinking" if event.get("type") == "thinking" else "assistant" - set_span_attribute(span, f"{prefix}.role", role) + tool_arguments = event.get("input") + if tool_arguments is not None: + set_span_attribute( + span, + f"{prefix}.tool_calls.0.arguments", + tool_arguments, + ) + else: set_span_attribute(span, f"{prefix}.content", event.get("text")) - except Exception as e: - logger.warning("Failed to set completion attributes, error: %s", str(e)) diff --git a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/streaming.py b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/streaming.py index bf8f33963f..0744506679 100644 --- a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/streaming.py +++ b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/streaming.py @@ -40,15 +40,18 @@ def _process_response_item(item, complete_response): complete_response["events"].append( {"index": index, "text": "", "type": item.content_block.type} ) - elif item.type == "content_block_delta" and item.delta.type in [ - "thinking_delta", - "text_delta", - ]: + if item.content_block.type == "tool_use": + complete_response["events"][index]["id"] = item.content_block.id + complete_response["events"][index]["name"] = item.content_block.name + complete_response["events"][index]["input"] = """""" + elif item.type == "content_block_delta": index = item.index if item.delta.type == "thinking_delta": complete_response["events"][index]["text"] += item.delta.thinking elif item.delta.type == "text_delta": complete_response["events"][index]["text"] += item.delta.text + elif item.delta.type == "input_json_delta": + complete_response["events"][index]["input"] += item.delta.partial_json elif item.type == "message_delta": for event in complete_response.get("events", []): event["finish_reason"] = item.delta.stop_reason diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_legacy.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_legacy.yaml new file mode 100644 index 0000000000..b1b4517f8e --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_legacy.yaml @@ -0,0 +1,115 @@ +interactions: +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is the weather + and current time in San Francisco?"},{"role":"assistant","content":[{"type":"text","text":"I''ll + help you get the weather and current time in San Francisco."},{"id":"call_1","type":"tool_use","name":"get_weather","input":{"location":"San + Francisco, CA"}}]},{"role":"user","content":[{"type":"tool_result","content":"Sunny + and 65 degrees Fahrenheit","tool_use_id":"call_1"}]}],"model":"claude-3-5-haiku-20241022","tools":[{"name":"get_weather","description":"Get + the current weather in a given location","input_schema":{"type":"object","properties":{"location":{"type":"string","description":"The + city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"],"description":"The + unit of temperature, either ''celsius'' or ''fahrenheit''"}},"required":["location"]}},{"name":"get_time","description":"Get + the current time in a given time zone","input_schema":{"type":"object","properties":{"timezone":{"type":"string","description":"The + IANA time zone name, e.g. America/Los_Angeles"}},"required":["timezone"]}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '1117' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.57.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.57.1 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.5 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA2RQS0vEMBD+L3NOsVu7uuTWxfWwD2HZk4iEkAxtaJrUZCJq6X+XVPawePzme83M + BEYDhyG2olyd908dni++7i7hq6XjuNs/9w0woO8RswpjlC0Cg+BtHsgYTSTpCBgMXqMFDsrKpLG4 + L9ZFJ02fiqqs6lVZVcBAeUfoCPjbdM0k761IMYcum2ScRLk6rA/daafb6qReHnXTb+vXujPAwMkh + +1okQWZYfG5MBHyCjH+8y3QzYDBK3h19FI1r0WKEeX5nEMmPIqCM3t22L0TEj4ROIXCXrGWQlnv5 + 9NchyPfoIvD1w4aBkqpDoQJKMt6JW0V55QNK/Z/ziW7yNgwihk+jUJDBABzyW7UMGub5FwAA//8D + ALuiEKakAQAA + headers: + CF-RAY: + - 9665173afb72342f-LHR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 28 Jul 2025 14:33:18 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 04aa8588-6567-40cb-9042-a54b20ebaf4f + anthropic-ratelimit-input-tokens-limit: + - '400000' + anthropic-ratelimit-input-tokens-remaining: + - '400000' + anthropic-ratelimit-input-tokens-reset: + - '2025-07-28T14:33:17Z' + anthropic-ratelimit-output-tokens-limit: + - '80000' + anthropic-ratelimit-output-tokens-remaining: + - '80000' + anthropic-ratelimit-output-tokens-reset: + - '2025-07-28T14:33:18Z' + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + anthropic-ratelimit-requests-reset: + - '2025-07-28T14:33:17Z' + anthropic-ratelimit-tokens-limit: + - '480000' + anthropic-ratelimit-tokens-remaining: + - '480000' + anthropic-ratelimit-tokens-reset: + - '2025-07-28T14:33:17Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CRZaepSK9Jc89WTRSudjf + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_with_events_with_content.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_with_events_with_content.yaml new file mode 100644 index 0000000000..57ec5ba8e5 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_with_events_with_content.yaml @@ -0,0 +1,120 @@ +interactions: +- request: + body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "What is + the weather and current time in San Francisco?"}, {"role": "assistant", "content": + [{"type": "text", "text": "I''ll help you get the weather and current time in + San Francisco."}, {"id": "call_1", "type": "tool_use", "name": "get_weather", + "input": {"location": "San Francisco, CA"}}]}, {"role": "user", "content": [{"type": + "tool_result", "content": "Sunny and 65 degrees Fahrenheit", "tool_use_id": + "call_1"}]}], "model": "claude-3-5-haiku-20241022", "tools": [{"name": "get_weather", + "description": "Get the current weather in a given location", "input_schema": + {"type": "object", "properties": {"location": {"type": "string", "description": + "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string", "enum": + ["celsius", "fahrenheit"], "description": "The unit of temperature, either ''celsius'' + or ''fahrenheit''"}}, "required": ["location"]}}, {"name": "get_time", "description": + "Get the current time in a given time zone", "input_schema": {"type": "object", + "properties": {"timezone": {"type": "string", "description": "The IANA time + zone name, e.g. America/Los_Angeles"}}, "required": ["timezone"]}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '1189' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.49.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.49.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//ZFDbSgMxEP2Xec7idnslb6UVxdqFKkitSAjZYRubTWomKdVl/12ypQ/F + xzPnNjMt6Ao4NFSLfDDBh8W0XNnt/n25XSwfN+f1uozAIPwcMamQSNYIDLwzaSCJNAVpAzBoXIUG + OCgjY4XZMBtne6kPMSvyYjTIiwIYKGcD2gD8o71mBueMiJRC+00SjiIfDBdv8jRffb2W05eduy92 + 5ei8eQIGVjbJV2MQQTe9zx5jAN5Cwr/OJnreoNdK3j07EnNbo0GCrvtkQMEdhUdJzt629wThd0Sr + ELiNxjCI/b28vXSI4A5oCfh4MmOgpNqjUB5l0M6KW0V+5T3K6j/nYrjJmzEg9CetUASNHjikt1bS + V9B1fwAAAP//AwCvh17XpAEAAA== + headers: + CF-RAY: + - 966d97b4ce4786d0-LHR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 29 Jul 2025 15:19:06 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 04aa8588-6567-40cb-9042-a54b20ebaf4f + anthropic-ratelimit-input-tokens-limit: + - '400000' + anthropic-ratelimit-input-tokens-remaining: + - '400000' + anthropic-ratelimit-input-tokens-reset: + - '2025-07-29T15:19:06Z' + anthropic-ratelimit-output-tokens-limit: + - '80000' + anthropic-ratelimit-output-tokens-remaining: + - '80000' + anthropic-ratelimit-output-tokens-reset: + - '2025-07-29T15:19:06Z' + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + anthropic-ratelimit-requests-reset: + - '2025-07-29T15:19:05Z' + anthropic-ratelimit-tokens-limit: + - '480000' + anthropic-ratelimit-tokens-remaining: + - '480000' + anthropic-ratelimit-tokens-reset: + - '2025-07-29T15:19:06Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CRbXxLLnRT7FYiRdsa3YF + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_with_events_with_no_content.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_with_events_with_no_content.yaml new file mode 100644 index 0000000000..23cef3798c --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_history_with_events_with_no_content.yaml @@ -0,0 +1,120 @@ +interactions: +- request: + body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "What is + the weather and current time in San Francisco?"}, {"role": "assistant", "content": + [{"type": "text", "text": "I''ll help you get the weather and current time in + San Francisco."}, {"id": "call_1", "type": "tool_use", "name": "get_weather", + "input": {"location": "San Francisco, CA"}}]}, {"role": "user", "content": [{"type": + "tool_result", "content": "Sunny and 65 degrees Fahrenheit", "tool_use_id": + "call_1"}]}], "model": "claude-3-5-haiku-20241022", "tools": [{"name": "get_weather", + "description": "Get the current weather in a given location", "input_schema": + {"type": "object", "properties": {"location": {"type": "string", "description": + "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string", "enum": + ["celsius", "fahrenheit"], "description": "The unit of temperature, either ''celsius'' + or ''fahrenheit''"}}, "required": ["location"]}}, {"name": "get_time", "description": + "Get the current time in a given time zone", "input_schema": {"type": "object", + "properties": {"timezone": {"type": "string", "description": "The IANA time + zone name, e.g. America/Los_Angeles"}}, "required": ["timezone"]}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '1189' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.49.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.49.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//ZFBLa8MwDP4vOjssSR8U33oYjDF66SiMMYxna4lJIre23K0N+e/DGT2U + HT99L0kjOAsShtiosnr9rNP39vntcvppwsG3i+prt9yBAL4cMaswRt0gCAi+zwMdo4usiUHA4C32 + IMH0OlksFsWqaLXrUlGX9bIq6xoEGE+MxCDfx1sme9+rFHPovEnGSZXVfnkd7FNLj2nP1HX+crDu + vAYBpIfsa5AVu2H20TExyBEyvnrK9HbA4Ix+ePFRbanBHiNM04eAyP6oAuro6b59JiKeEpJBkJT6 + XkCa75XjX4di3yFFkKv1RoDRpkVlAmp2ntS9orzxAbX9z/nEd3kbARHD2RlU7DCAhPxWq4OFafoF + AAD//wMAHf9as6QBAAA= + headers: + CF-RAY: + - 966daace6fe877b7-LHR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 29 Jul 2025 15:32:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 04aa8588-6567-40cb-9042-a54b20ebaf4f + anthropic-ratelimit-input-tokens-limit: + - '400000' + anthropic-ratelimit-input-tokens-remaining: + - '400000' + anthropic-ratelimit-input-tokens-reset: + - '2025-07-29T15:32:08Z' + anthropic-ratelimit-output-tokens-limit: + - '80000' + anthropic-ratelimit-output-tokens-remaining: + - '80000' + anthropic-ratelimit-output-tokens-reset: + - '2025-07-29T15:32:09Z' + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + anthropic-ratelimit-requests-reset: + - '2025-07-29T15:32:08Z' + anthropic-ratelimit-tokens-limit: + - '480000' + anthropic-ratelimit-tokens-remaining: + - '480000' + anthropic-ratelimit-tokens-reset: + - '2025-07-29T15:32:08Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CRbYx1CxeHxwFBrQp9oGf + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_legacy.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_legacy.yaml new file mode 100644 index 0000000000..f215f311ea --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_legacy.yaml @@ -0,0 +1,299 @@ +interactions: +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is the weather + and current time in San Francisco?"}],"model":"claude-3-5-sonnet-20240620","stream":true,"tools":[{"name":"get_weather","description":"Get + the current weather in a given location","input_schema":{"type":"object","properties":{"location":{"type":"string","description":"The + city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"],"description":"The + unit of temperature, either ''celsius'' or ''fahrenheit''"}},"required":["location"]}},{"name":"get_time","description":"Get + the current time in a given time zone","input_schema":{"type":"object","properties":{"timezone":{"type":"string","description":"The + IANA time zone name, e.g. America/Los_Angeles"}},"required":["timezone"]}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '795' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.57.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.57.1 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.5 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: 'event: message_start + + data: {"type":"message_start","message":{"id":"msg_0138UNF3YbNp49KkqZtUBWqz","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":506,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5,"service_tier":"standard"}} } + + + event: content_block_start + + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} + } + + + event: ping + + data: {"type": "ping"} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Certainly! + I can"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + help you with that information"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":". + To"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + get the weather"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + and current time in San"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + Francisco, I''ll nee"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d + to use two separate"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + functions."} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + Let me fetch that data"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + for you."} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":0 } + + + event: content_block_start + + data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_014x5X91kx3fvdhpLvwXZWE2","name":"get_weather","input":{}} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"locati"} + } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"on\": + \"San F"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ra"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ncisco,"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + CA\""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":", + \"unit\""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":": + \"celsius\"}"} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":1 } + + + event: content_block_start + + data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_0121kXsENLvoDZ72LCuAnCCz","name":"get_time","input":{}} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":""}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"timez"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"one\":"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":" + \"America/L"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"os_Angele"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"s\"}"} + } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":2 } + + + event: message_delta + + data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":153} } + + + event: message_stop + + data: {"type":"message_stop" } + + + ' + headers: + CF-RAY: + - 966bc3a37d39459a-LHR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 29 Jul 2025 09:59:34 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 04aa8588-6567-40cb-9042-a54b20ebaf4f + anthropic-ratelimit-input-tokens-limit: + - '400000' + anthropic-ratelimit-input-tokens-remaining: + - '400000' + anthropic-ratelimit-input-tokens-reset: + - '2025-07-29T09:59:33Z' + anthropic-ratelimit-output-tokens-limit: + - '80000' + anthropic-ratelimit-output-tokens-remaining: + - '80000' + anthropic-ratelimit-output-tokens-reset: + - '2025-07-29T09:59:33Z' + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + anthropic-ratelimit-requests-reset: + - '2025-07-29T09:59:33Z' + anthropic-ratelimit-tokens-limit: + - '480000' + anthropic-ratelimit-tokens-remaining: + - '480000' + anthropic-ratelimit-tokens-reset: + - '2025-07-29T09:59:33Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CRb7b5b9MWofEnfaAHTBu + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_with_events_with_content.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_with_events_with_content.yaml new file mode 100644 index 0000000000..70d5fbf8b9 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_with_events_with_content.yaml @@ -0,0 +1,303 @@ +interactions: +- request: + body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "What is + the weather like right now in New York? Also what time is it there now?"}], + "model": "claude-3-5-sonnet-20240620", "stream": true, "tools": [{"name": "get_weather", + "description": "Get the current weather in a given location", "input_schema": + {"type": "object", "properties": {"location": {"type": "string", "description": + "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string", "enum": + ["celsius", "fahrenheit"], "description": "The unit of temperature, either ''celsius'' + or ''fahrenheit''"}}, "required": ["location"]}}, {"name": "get_time", "description": + "Get the current time in a given time zone", "input_schema": {"type": "object", + "properties": {"timezone": {"type": "string", "description": "The IANA time + zone name, e.g. America/Los_Angeles"}}, "required": ["timezone"]}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '869' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.49.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.49.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: 'event: message_start + + data: {"type":"message_start","message":{"id":"msg_01RvjFrekzod2e3Dj6rZwmf3","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":514,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5,"service_tier":"standard"}} } + + + event: content_block_start + + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + + event: ping + + data: {"type": "ping"} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Certainly! + I''"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d + be happy to help"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + you with both"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + the current weather in New"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + York and the current time"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + there. To"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + get this information, I"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"''ll + need to use two"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + different tools."} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + Let''s"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + start with the weather,"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + and then we''ll check"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + the time."} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":0 } + + + event: content_block_start + + data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01UGYEgvuRFeXbTZKyDyqo9P","name":"get_weather","input":{}} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"location"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\": + \"New Yo"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"rk, + NY\""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":", + \"unit\": "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"f"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ahrenhei"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"t\"}"} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":1} + + + event: content_block_start + + data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01VCGwdaiXbGQJHRCzoWgK2U","name":"get_time","input":{}} + } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"timezone\":"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":" + \"America/Ne"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"w_York\"}"} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":2 } + + + event: message_delta + + data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":168} } + + + event: message_stop + + data: {"type":"message_stop" } + + + ' + headers: + CF-RAY: + - 966db86309af7a09-LHR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 29 Jul 2025 15:41:24 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 04aa8588-6567-40cb-9042-a54b20ebaf4f + anthropic-ratelimit-input-tokens-limit: + - '400000' + anthropic-ratelimit-input-tokens-remaining: + - '400000' + anthropic-ratelimit-input-tokens-reset: + - '2025-07-29T15:41:24Z' + anthropic-ratelimit-output-tokens-limit: + - '80000' + anthropic-ratelimit-output-tokens-remaining: + - '80000' + anthropic-ratelimit-output-tokens-reset: + - '2025-07-29T15:41:24Z' + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + anthropic-ratelimit-requests-reset: + - '2025-07-29T15:41:24Z' + anthropic-ratelimit-tokens-limit: + - '480000' + anthropic-ratelimit-tokens-remaining: + - '480000' + anthropic-ratelimit-tokens-reset: + - '2025-07-29T15:41:24Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CRbZf1WDh1Dbp1ot2EjAr + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_with_events_with_no_content.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_with_events_with_no_content.yaml new file mode 100644 index 0000000000..7c506e2005 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_tools_streaming_with_events_with_no_content.yaml @@ -0,0 +1,334 @@ +interactions: +- request: + body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "What is + the weather like right now in New York? Also what time is it there now?"}], + "model": "claude-3-5-sonnet-20240620", "stream": true, "tools": [{"name": "get_weather", + "description": "Get the current weather in a given location", "input_schema": + {"type": "object", "properties": {"location": {"type": "string", "description": + "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string", "enum": + ["celsius", "fahrenheit"], "description": "The unit of temperature, either ''celsius'' + or ''fahrenheit''"}}, "required": ["location"]}}, {"name": "get_time", "description": + "Get the current time in a given time zone", "input_schema": {"type": "object", + "properties": {"timezone": {"type": "string", "description": "The IANA time + zone name, e.g. America/Los_Angeles"}}, "required": ["timezone"]}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '869' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.49.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.49.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: 'event: message_start + + data: {"type":"message_start","message":{"id":"msg_01JSRVhRSH3H7Fpoimm84exq","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":514,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5,"service_tier":"standard"}} } + + + event: content_block_start + + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + + event: ping + + data: {"type": "ping"} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Certainly! + I''"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d + be happy to help"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + you with information"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + about the current weather an"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d + time in New York."} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + To"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + provide"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + you with accurate information,"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + I''ll need to use"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + two different tools."} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + Let''s start with the"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + weather and then check"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + the time"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"."} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":0 } + + + event: content_block_start + + data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01YUs66wivdF51ENZFX8gX9S","name":"get_weather","input":{}} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"location"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\": + \""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"Ne"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"w + Y"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ork, + NY\""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":", + \"unit\""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":": + \"fahrenhe"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"it\"}"} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":1 } + + + event: content_block_start + + data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01NRtod2L7M7TBDj9GCzsZCx","name":"get_time","input":{}} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"ti"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"mezone\":"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":" + \"Ame"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"rica/"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"New_"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"Yo"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"rk\"}"} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":2 } + + + event: message_delta + + data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":165} } + + + event: message_stop + + data: {"type":"message_stop" } + + + ' + headers: + CF-RAY: + - 966dceb35aa686d0-LHR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 29 Jul 2025 15:56:38 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 04aa8588-6567-40cb-9042-a54b20ebaf4f + anthropic-ratelimit-input-tokens-limit: + - '400000' + anthropic-ratelimit-input-tokens-remaining: + - '400000' + anthropic-ratelimit-input-tokens-reset: + - '2025-07-29T15:56:38Z' + anthropic-ratelimit-output-tokens-limit: + - '80000' + anthropic-ratelimit-output-tokens-remaining: + - '80000' + anthropic-ratelimit-output-tokens-reset: + - '2025-07-29T15:56:38Z' + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + anthropic-ratelimit-requests-reset: + - '2025-07-29T15:56:38Z' + anthropic-ratelimit-tokens-limit: + - '480000' + anthropic-ratelimit-tokens-remaining: + - '480000' + anthropic-ratelimit-tokens-reset: + - '2025-07-29T15:56:38Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CRbapP4d6AdWez2L13z2d + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/test_messages.py b/packages/opentelemetry-instrumentation-anthropic/tests/test_messages.py index 6b87c0d1c2..4a2aac525a 100644 --- a/packages/opentelemetry-instrumentation-anthropic/tests/test_messages.py +++ b/packages/opentelemetry-instrumentation-anthropic/tests/test_messages.py @@ -30,6 +30,43 @@ } +TOOLS = [ + { + "name": "get_weather", + "description": "Get the current weather in a given location", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The unit of temperature, either 'celsius' or 'fahrenheit'", + }, + }, + "required": ["location"], + }, + }, + { + "name": "get_time", + "description": "Get the current time in a given time zone", + "input_schema": { + "type": "object", + "properties": { + "timezone": { + "type": "string", + "description": "The IANA time zone name, e.g. America/Los_Angeles", + } + }, + "required": ["timezone"], + }, + }, +] + + @pytest.mark.vcr def test_anthropic_message_create_legacy( instrument_legacy, anthropic_client, span_exporter, log_exporter, reader @@ -1234,41 +1271,7 @@ def test_anthropic_tools_legacy( response = anthropic_client.messages.create( model="claude-3-5-sonnet-20240620", max_tokens=1024, - tools=[ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The unit of temperature, either 'celsius' or 'fahrenheit'", - }, - }, - "required": ["location"], - }, - }, - { - "name": "get_time", - "description": "Get the current time in a given time zone", - "input_schema": { - "type": "object", - "properties": { - "timezone": { - "type": "string", - "description": "The IANA time zone name, e.g. America/Los_Angeles", - } - }, - "required": ["timezone"], - }, - }, - ], + tools=TOOLS, messages=[ { "role": "user", @@ -1411,45 +1414,10 @@ def test_anthropic_tools_with_events_with_content( "content": "What is the weather like right now in New York? Also what time is it there now?", } - tool_0 = { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The unit of temperature, either 'celsius' or 'fahrenheit'", - }, - }, - "required": ["location"], - }, - } - - tool_1 = { - "name": "get_time", - "description": "Get the current time in a given time zone", - "input_schema": { - "type": "object", - "properties": { - "timezone": { - "type": "string", - "description": "The IANA time zone name, e.g. America/Los_Angeles", - } - }, - "required": ["timezone"], - }, - } - anthropic_client.messages.create( model="claude-3-5-sonnet-20240620", max_tokens=1024, - tools=[tool_0, tool_1], + tools=TOOLS, messages=[user_message], ) try: @@ -1487,7 +1455,7 @@ def test_anthropic_tools_with_events_with_content( # Validate the tool messages input vent assert_message_in_logs( - logs[1], "gen_ai.user.message", {"content": {"tools": [tool_0, tool_1]}} + logs[1], "gen_ai.user.message", {"content": {"tools": TOOLS}} ) # Validate the ai response @@ -1545,41 +1513,7 @@ def test_anthropic_tools_with_events_with_no_content( anthropic_client.messages.create( model="claude-3-5-sonnet-20240620", max_tokens=1024, - tools=[ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The unit of temperature, either 'celsius' or 'fahrenheit'", - }, - }, - "required": ["location"], - }, - }, - { - "name": "get_time", - "description": "Get the current time in a given time zone", - "input_schema": { - "type": "object", - "properties": { - "timezone": { - "type": "string", - "description": "The IANA time zone name, e.g. America/Los_Angeles", - } - }, - "required": ["timezone"], - }, - }, - ], + tools=TOOLS, messages=[ { "role": "user", @@ -1669,37 +1603,746 @@ def test_anthropic_tools_with_events_with_no_content( @pytest.mark.vcr -def test_with_asyncio_run_legacy( - instrument_legacy, async_anthropic_client, span_exporter, log_exporter +def test_anthropic_tools_history_legacy( + instrument_legacy, anthropic_client, span_exporter, log_exporter, reader ): - asyncio.run( - async_anthropic_client.messages.create( - model="claude-3-5-sonnet-20240620", - max_tokens=1024, - system=[ - { - "type": "text", - "text": "You help generate concise summaries of news articles and blog posts that user sends you.", - }, - ], - messages=[ - { - "role": "user", - "content": "What is the weather in San Francisco?", - }, - ], - ) + response = anthropic_client.messages.create( + model="claude-3-5-haiku-20241022", + max_tokens=1024, + tools=TOOLS, + messages=[ + { + "role": "user", + "content": "What is the weather and current time in San Francisco?", + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll help you get the weather and current time in San Francisco.", + }, + { + "id": "call_1", + "type": "tool_use", + "name": "get_weather", + "input": {"location": "San Francisco, CA"}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "content": "Sunny and 65 degrees Fahrenheit", + "tool_use_id": "call_1", + } + ], + }, + ], ) + try: + anthropic_client.messages.create( + unknown_parameter="unknown", + ) + except Exception: + pass + spans = span_exporter.get_finished_spans() - assert [span.name for span in spans] == [ - "anthropic.chat", + # verify overall shape + assert all(span.name == "anthropic.chat" for span in spans) + assert len(spans) == 1 + + anthropic_span = spans[0] + + # verify usage + assert anthropic_span.attributes["gen_ai.usage.prompt_tokens"] == 568 + assert ( + anthropic_span.attributes["gen_ai.usage.completion_tokens"] + + anthropic_span.attributes["gen_ai.usage.prompt_tokens"] + == anthropic_span.attributes["llm.usage.total_tokens"] + ) + + # verify metrics + metrics_data = reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + verify_metrics(resource_metrics, "claude-3-5-haiku-20241022") + + # verify request and inputs + assert ( + anthropic_span.attributes["gen_ai.prompt.0.content"] + == "What is the weather and current time in San Francisco?" + ) + assert anthropic_span.attributes["gen_ai.prompt.0.role"] == "user" + assert ( + anthropic_span.attributes["gen_ai.prompt.1.content"] + == "I'll help you get the weather and current time in San Francisco." + ) + assert anthropic_span.attributes["gen_ai.prompt.1.role"] == "assistant" + assert json.loads(anthropic_span.attributes["gen_ai.prompt.2.content"]) == [ + { + "type": "tool_result", + "content": "Sunny and 65 degrees Fahrenheit", + "tool_use_id": "call_1", + } ] + assert anthropic_span.attributes["gen_ai.prompt.2.role"] == "user" + assert anthropic_span.attributes["llm.request.functions.0.name"] == "get_weather" + assert ( + anthropic_span.attributes["llm.request.functions.0.description"] + == "Get the current weather in a given location" + ) + assert anthropic_span.attributes[ + "llm.request.functions.0.input_schema" + ] == json.dumps( + { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The unit of temperature, either 'celsius' or 'fahrenheit'", + }, + }, + "required": ["location"], + } + ) + assert anthropic_span.attributes["llm.request.functions.1.name"] == "get_time" + assert ( + anthropic_span.attributes["llm.request.functions.1.description"] + == "Get the current time in a given time zone" + ) + assert anthropic_span.attributes[ + "llm.request.functions.1.input_schema" + ] == json.dumps( + { + "type": "object", + "properties": { + "timezone": { + "type": "string", + "description": "The IANA time zone name, e.g. America/Los_Angeles", + } + }, + "required": ["timezone"], + } + ) + + # verify response and output + assert ( + anthropic_span.attributes["gen_ai.completion.0.finish_reason"] + == response.stop_reason + ) + assert "gen_ai.completion.0.content" not in anthropic_span.attributes + assert anthropic_span.attributes["gen_ai.completion.0.role"] == "assistant" + + assert ( + anthropic_span.attributes["gen_ai.completion.0.tool_calls.0.id"] + ) == response.content[0].id + assert ( + anthropic_span.attributes["gen_ai.completion.0.tool_calls.0.name"] + ) == response.content[0].name + response_input = json.dumps(response.content[0].input) + assert ( + anthropic_span.attributes["gen_ai.completion.0.tool_calls.0.arguments"] + == response_input + ) + + assert ( + anthropic_span.attributes.get("gen_ai.response.id") + == "msg_01QJDheQSo4hSrxgtLpEJFkA" + ) logs = log_exporter.get_finished_logs() - assert len(logs) == 0, ( - "Assert that it doesn't emit logs when use_legacy_attributes is True" + assert ( + len(logs) == 0 + ), "Assert that it doesn't emit logs when use_legacy_attributes is True" + + +@pytest.mark.vcr +def test_anthropic_tools_history_with_events_with_content( + instrument_with_content, anthropic_client, span_exporter, log_exporter, reader +): + user_message = { + "role": "user", + "content": "What is the weather and current time in San Francisco?", + } + first_assistant_message = { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll help you get the weather and current time in San Francisco.", + }, + { + "id": "call_1", + "type": "tool_use", + "name": "get_weather", + "input": {"location": "San Francisco, CA"}, + }, + ], + } + tool_result_message = { + "role": "user", + "content": [ + {"type": "tool_result", "content": "Sunny and 65 degrees Fahrenheit", "tool_use_id": "call_1"}, + ], + } + anthropic_client.messages.create( + model="claude-3-5-haiku-20241022", + max_tokens=1024, + tools=TOOLS, + messages=[ + user_message, + first_assistant_message, + tool_result_message, + ], ) + try: + anthropic_client.messages.create( + unknown_parameter="unknown", + ) + except Exception: + pass + + spans = span_exporter.get_finished_spans() + # verify overall shape + assert all(span.name == "anthropic.chat" for span in spans) + assert len(spans) == 1 + + anthropic_span = spans[0] + + # verify usage + assert anthropic_span.attributes["gen_ai.usage.prompt_tokens"] == 568 + assert ( + anthropic_span.attributes["gen_ai.usage.completion_tokens"] + + anthropic_span.attributes["gen_ai.usage.prompt_tokens"] + == anthropic_span.attributes["llm.usage.total_tokens"] + ) + + # verify metrics + metrics_data = reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + verify_metrics(resource_metrics, "claude-3-5-haiku-20241022") + + logs = log_exporter.get_finished_logs() + assert len(logs) == 5 + + # Validate user message + user_message.pop("role", None) + assert_message_in_logs(logs[0], "gen_ai.user.message", user_message) + + first_assistant_message.pop("role", None) + # Validate the tool messages input event + assert_message_in_logs( + logs[1], "gen_ai.assistant.message", first_assistant_message + ) + + tool_result_message.pop("role", None) + assert_message_in_logs(logs[2], "gen_ai.user.message", tool_result_message) + + assert_message_in_logs(logs[3], "gen_ai.user.message", {"content": {"tools": TOOLS}}) + + # Validate the second tool call + tool_call = { + "index": 0, + "finish_reason": "tool_use", + "tool_calls": [ + { + "id": "toolu_013CVavAKjSN7RZoE2ZN4xQJ", + "type": "function", + "function": { + "name": "get_time", + "arguments": {"timezone": "America/Los_Angeles"}, + }, + } + ], + "message": {"content": None}, + } + assert_message_in_logs(logs[4], "gen_ai.choice", tool_call) + + +@pytest.mark.vcr +def test_anthropic_tools_history_with_events_with_no_content( + instrument_with_no_content, anthropic_client, span_exporter, log_exporter, reader +): + user_message = { + "role": "user", + "content": "What is the weather and current time in San Francisco?", + } + first_assistant_message = { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll help you get the weather and current time in San Francisco.", + }, + { + "id": "call_1", + "type": "tool_use", + "name": "get_weather", + "input": {"location": "San Francisco, CA"}, + }, + ], + } + tool_result_message = { + "role": "user", + "content": [ + {"type": "tool_result", "content": "Sunny and 65 degrees Fahrenheit", "tool_use_id": "call_1"}, + ], + } + anthropic_client.messages.create( + model="claude-3-5-haiku-20241022", + max_tokens=1024, + tools=TOOLS, + messages=[ + user_message, + first_assistant_message, + tool_result_message, + ], + ) + try: + anthropic_client.messages.create( + unknown_parameter="unknown", + ) + except Exception: + pass + + spans = span_exporter.get_finished_spans() + # verify overall shape + assert all(span.name == "anthropic.chat" for span in spans) + assert len(spans) == 1 + + anthropic_span = spans[0] + + # verify usage + assert anthropic_span.attributes["gen_ai.usage.prompt_tokens"] == 568 + assert ( + anthropic_span.attributes["gen_ai.usage.completion_tokens"] + + anthropic_span.attributes["gen_ai.usage.prompt_tokens"] + == anthropic_span.attributes["llm.usage.total_tokens"] + ) + + # verify metrics + metrics_data = reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + verify_metrics(resource_metrics, "claude-3-5-haiku-20241022") + + logs = log_exporter.get_finished_logs() + assert len(logs) == 5 + + # Validate user message + assert_message_in_logs(logs[0], "gen_ai.user.message", {}) + + # Validate the tool messages input event + assert_message_in_logs( + logs[1], "gen_ai.assistant.message", {} + ) + + assert_message_in_logs(logs[2], "gen_ai.user.message", {}) + + assert_message_in_logs(logs[3], "gen_ai.user.message", {}) + + # Validate the second tool call + tool_call = { + "index": 0, + "finish_reason": "tool_use", + "tool_calls": [ + { + "id": "toolu_01S4zmdHhnEuStnkkoyVdiv6", + "type": "function", + "function": { + "name": "get_time", + # no arguments + }, + } + ], + # empty message + "message": {}, + } + assert_message_in_logs(logs[4], "gen_ai.choice", tool_call) + + +@pytest.mark.vcr +def test_anthropic_tools_streaming_legacy( + instrument_legacy, anthropic_client, span_exporter, log_exporter, reader +): + response = anthropic_client.messages.create( + model="claude-3-5-sonnet-20240620", + max_tokens=1024, + tools=TOOLS, + messages=[ + { + "role": "user", + "content": "What is the weather and current time in San Francisco?", + } + ], + stream=True, + ) + + try: + anthropic_client.messages.create( + unknown_parameter="unknown", + ) + except Exception: + pass + + # consume the streaming iterator + for _ in response: + pass + + spans = span_exporter.get_finished_spans() + # verify overall shape + assert all(span.name == "anthropic.chat" for span in spans) + assert len(spans) == 1 + + anthropic_span = spans[0] + + # verify usage + assert anthropic_span.attributes["gen_ai.usage.prompt_tokens"] == 506 + assert ( + anthropic_span.attributes["gen_ai.usage.completion_tokens"] + + anthropic_span.attributes["gen_ai.usage.prompt_tokens"] + == anthropic_span.attributes["llm.usage.total_tokens"] + ) + + # verify metrics + metrics_data = reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + verify_metrics(resource_metrics, "claude-3-5-sonnet-20240620") + + # verify request and inputs + assert ( + anthropic_span.attributes["gen_ai.prompt.0.content"] + == "What is the weather and current time in San Francisco?" + ) + assert anthropic_span.attributes["gen_ai.prompt.0.role"] == "user" + assert anthropic_span.attributes["llm.request.functions.0.name"] == "get_weather" + assert ( + anthropic_span.attributes["llm.request.functions.0.description"] + == "Get the current weather in a given location" + ) + assert anthropic_span.attributes[ + "llm.request.functions.0.input_schema" + ] == json.dumps( + { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The unit of temperature, either 'celsius' or 'fahrenheit'", + }, + }, + "required": ["location"], + } + ) + assert anthropic_span.attributes["llm.request.functions.1.name"] == "get_time" + assert ( + anthropic_span.attributes["llm.request.functions.1.description"] + == "Get the current time in a given time zone" + ) + assert anthropic_span.attributes[ + "llm.request.functions.1.input_schema" + ] == json.dumps( + { + "type": "object", + "properties": { + "timezone": { + "type": "string", + "description": "The IANA time zone name, e.g. America/Los_Angeles", + } + }, + "required": ["timezone"], + } + ) + + # verify response and output + assert ( + anthropic_span.attributes["gen_ai.completion.0.content"] + == "Certainly! I can help you with that information. " + "To get the weather and current time in San Francisco, I'll need to use " + "two separate functions. Let me fetch that data for you." + ) + assert anthropic_span.attributes["gen_ai.completion.0.role"] == "assistant" + + assert ( + anthropic_span.attributes["gen_ai.completion.0.tool_calls.0.id"] + ) == "toolu_0121kXsENLvoDZ72LCuAnCCz" + assert ( + anthropic_span.attributes["gen_ai.completion.0.tool_calls.0.name"] + ) == "get_time" + assert json.loads( + anthropic_span.attributes["gen_ai.completion.0.tool_calls.0.arguments"] + ) == {"timezone": "America/Los_Angeles"} + + assert ( + anthropic_span.attributes.get("gen_ai.response.id") + == "msg_0138UNF3YbNp49KkqZtUBWqz" + ) + + logs = log_exporter.get_finished_logs() + assert ( + len(logs) == 0 + ), "Assert that it doesn't emit logs when use_legacy_attributes is True" + + +@pytest.mark.vcr +def test_anthropic_tools_streaming_with_events_with_content( + instrument_with_content, anthropic_client, span_exporter, log_exporter, reader +): + user_message = { + "role": "user", + "content": "What is the weather like right now in New York? Also what time is it there now?", + } + + response = anthropic_client.messages.create( + model="claude-3-5-sonnet-20240620", + max_tokens=1024, + tools=TOOLS, + messages=[user_message], + stream=True, + ) + + # consume the streaming iterator + for _ in response: + pass + + try: + anthropic_client.messages.create( + unknown_parameter="unknown", + ) + except Exception: + pass + spans = span_exporter.get_finished_spans() + # verify overall shape + assert all(span.name == "anthropic.chat" for span in spans) + assert len(spans) == 1 + + anthropic_span = spans[0] + + # verify usage + assert anthropic_span.attributes["gen_ai.usage.prompt_tokens"] == 514 + assert ( + anthropic_span.attributes["gen_ai.usage.completion_tokens"] + + anthropic_span.attributes["gen_ai.usage.prompt_tokens"] + == anthropic_span.attributes["llm.usage.total_tokens"] + ) + + # verify metrics + metrics_data = reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + verify_metrics(resource_metrics, "claude-3-5-sonnet-20240620") + + logs = log_exporter.get_finished_logs() + assert len(logs) == 5 + + # Validate user message + user_message.pop("role", None) + assert_message_in_logs(logs[0], "gen_ai.user.message", user_message) + + # Validate the tool messages input vent + assert_message_in_logs( + logs[1], "gen_ai.user.message", {"content": {"tools": TOOLS}} + ) + + # Validate the ai response + ideal_response = { + "index": 0, + "finish_reason": "tool_use", + "message": { + "content": { + "content": "Certainly! I'd be happy to help you with both the current " + "weather in New York and the current " + "time there. To get this information, I'll need to use two different " + "tools. Let's start with the weather, " + "and then we'll check the time.", + "type": "text", + } + }, + } + assert_message_in_logs(logs[2], "gen_ai.choice", ideal_response) + + # Validate the first tool call + tool_call_0 = { + "index": 1, + "finish_reason": "tool_use", + "tool_calls": [ + { + "id": "toolu_01UGYEgvuRFeXbTZKyDyqo9P", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"location": "New York, NY", "unit": "fahrenheit"}, + }, + } + ], + "message": {"content": None}, + } + assert_message_in_logs(logs[3], "gen_ai.choice", tool_call_0) + + # Validate the second tool call + tool_call_1 = { + "index": 2, + "finish_reason": "tool_use", + "tool_calls": [ + { + "id": "toolu_01VCGwdaiXbGQJHRCzoWgK2U", + "type": "function", + "function": { + "name": "get_time", + "arguments": {"timezone": "America/New_York"}, + }, + } + ], + "message": {"content": None}, + } + assert_message_in_logs(logs[4], "gen_ai.choice", tool_call_1) + + +@pytest.mark.vcr +def test_anthropic_tools_streaming_with_events_with_no_content( + instrument_with_no_content, anthropic_client, span_exporter, log_exporter, reader +): + user_message = { + "role": "user", + "content": "What is the weather like right now in New York? Also what time is it there now?", + } + + response = anthropic_client.messages.create( + model="claude-3-5-sonnet-20240620", + max_tokens=1024, + tools=TOOLS, + messages=[user_message], + stream=True, + ) + + # consume the streaming iterator + for _ in response: + pass + + try: + anthropic_client.messages.create( + unknown_parameter="unknown", + ) + except Exception: + pass + spans = span_exporter.get_finished_spans() + # verify overall shape + assert all(span.name == "anthropic.chat" for span in spans) + assert len(spans) == 1 + + anthropic_span = spans[0] + + # verify usage + assert anthropic_span.attributes["gen_ai.usage.prompt_tokens"] == 514 + assert ( + anthropic_span.attributes["gen_ai.usage.completion_tokens"] + + anthropic_span.attributes["gen_ai.usage.prompt_tokens"] + == anthropic_span.attributes["llm.usage.total_tokens"] + ) + + # verify metrics + metrics_data = reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + verify_metrics(resource_metrics, "claude-3-5-sonnet-20240620") + + logs = log_exporter.get_finished_logs() + assert len(logs) == 5 + + # Validate user message + user_message.pop("role", None) + assert_message_in_logs(logs[0], "gen_ai.user.message", {}) + + # Validate the tool messages input vent + assert_message_in_logs(logs[1], "gen_ai.user.message", {}) + + assert_message_in_logs(logs[2], "gen_ai.choice", { + "finish_reason": "tool_use", + "index": 0, + "message": {}, + }) + + # Validate the first tool call + tool_call_0 = { + "index": 1, + "finish_reason": "tool_use", + "tool_calls": [ + { + "id": "toolu_01YUs66wivdF51ENZFX8gX9S", + "type": "function", + "function": { + "name": "get_weather", + # no arguments + }, + } + ], + # empty message + "message": {}, + } + assert_message_in_logs(logs[3], "gen_ai.choice", tool_call_0) + + # Validate the second tool call + tool_call_1 = { + "index": 2, + "finish_reason": "tool_use", + "tool_calls": [ + { + "id": "toolu_01NRtod2L7M7TBDj9GCzsZCx", + "type": "function", + "function": { + "name": "get_time", + # no arguments + }, + } + ], + # empty message + "message": {}, + } + assert_message_in_logs(logs[4], "gen_ai.choice", tool_call_1) + + +@pytest.mark.vcr +def test_with_asyncio_run_legacy( + instrument_legacy, async_anthropic_client, span_exporter, log_exporter +): + asyncio.run( + async_anthropic_client.messages.create( + model="claude-3-5-sonnet-20240620", + max_tokens=1024, + system=[ + { + "type": "text", + "text": "You help generate concise summaries of news articles and blog posts that user sends you.", + }, + ], + messages=[ + { + "role": "user", + "content": "What is the weather in San Francisco?", + }, + ], + ) + ) + + spans = span_exporter.get_finished_spans() + assert [span.name for span in spans] == [ + "anthropic.chat", + ] + + logs = log_exporter.get_finished_logs() + assert ( + len(logs) == 0 + ), "Assert that it doesn't emit logs when use_legacy_attributes is True" @pytest.mark.vcr