diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_handler.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_handler.py index 9ab46f5ba4..e97a093711 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_handler.py @@ -451,11 +451,8 @@ def _create_llm_span( _extract_class_name_from_serialized(serialized) ) - _set_span_attribute(span, GenAIAttributes.GEN_AI_SYSTEM, vendor) - _set_span_attribute(span, SpanAttributes.LLM_REQUEST_TYPE, request_type.value) - _set_span_attribute( - span, GenAIAttributes.GEN_AI_OPERATION_NAME, GenAICustomOperationName.LLM_REQUEST.value - ) + _set_span_attribute(span, GenAIAttributes.GEN_AI_PROVIDER_NAME, vendor) + _set_span_attribute(span, GenAIAttributes.GEN_AI_OPERATION_NAME, request_type.value) # we already have an LLM span by this point, # so skip any downstream instrumentation from here @@ -732,16 +729,16 @@ def on_llm_end( span, GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens ) _set_span_attribute( - span, SpanAttributes.LLM_USAGE_TOTAL_TOKENS, total_tokens + span, SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, total_tokens ) # Record token usage metrics - vendor = span.attributes.get(GenAIAttributes.GEN_AI_SYSTEM, "Langchain") + vendor = span.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME, "langchain") if prompt_tokens > 0: self.token_histogram.record( prompt_tokens, attributes={ - GenAIAttributes.GEN_AI_SYSTEM: vendor, + GenAIAttributes.GEN_AI_PROVIDER_NAME: vendor, GenAIAttributes.GEN_AI_TOKEN_TYPE: "input", GenAIAttributes.GEN_AI_RESPONSE_MODEL: model_name or "unknown", }, @@ -751,7 +748,7 @@ def on_llm_end( self.token_histogram.record( completion_tokens, attributes={ - GenAIAttributes.GEN_AI_SYSTEM: vendor, + GenAIAttributes.GEN_AI_PROVIDER_NAME: vendor, GenAIAttributes.GEN_AI_TOKEN_TYPE: "output", GenAIAttributes.GEN_AI_RESPONSE_MODEL: model_name or "unknown", }, @@ -768,11 +765,11 @@ def on_llm_end( # Record duration before ending span duration = time.time() - self.spans[run_id].start_time - vendor = span.attributes.get(GenAIAttributes.GEN_AI_SYSTEM, "Langchain") + vendor = span.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME, "langchain") self.duration_histogram.record( duration, attributes={ - GenAIAttributes.GEN_AI_SYSTEM: vendor, + GenAIAttributes.GEN_AI_PROVIDER_NAME: vendor, GenAIAttributes.GEN_AI_RESPONSE_MODEL: model_name or "unknown", }, ) diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/event_emitter.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/event_emitter.py index 76f11b04b3..caf7a5f4cd 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/event_emitter.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/event_emitter.py @@ -28,7 +28,7 @@ class Roles(Enum): VALID_MESSAGE_ROLES = {role.value for role in Roles} """The valid roles for naming the message event.""" -EVENT_ATTRIBUTES = {GenAIAttributes.GEN_AI_SYSTEM: "langchain"} +EVENT_ATTRIBUTES = {GenAIAttributes.GEN_AI_PROVIDER_NAME: "langchain"} """The attributes to be used for the event.""" diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py index f7d71b18ab..af375badfd 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py @@ -97,23 +97,19 @@ def set_request_params(span, kwargs, span_holder: SpanHolder): _set_span_attribute(span, GenAIAttributes.GEN_AI_REQUEST_TOP_P, params.get("top_p")) tools = kwargs.get("invocation_params", {}).get("tools", []) - for i, tool in enumerate(tools): - tool_function = tool.get("function", tool) - _set_span_attribute( - span, - f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{i}.name", - tool_function.get("name"), - ) - _set_span_attribute( - span, - f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{i}.description", - tool_function.get("description"), - ) - _set_span_attribute( - span, - f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{i}.parameters", - json.dumps(tool_function.get("parameters", tool.get("input_schema"))), - ) + if tools: + tool_defs = [] + for tool in tools: + tool_function = tool.get("function", tool) + tool_def = { + "name": tool_function.get("name"), + "description": tool_function.get("description"), + } + params = tool_function.get("parameters") or tool.get("input_schema") + if params is not None: + tool_def["parameters"] = params + tool_defs.append(tool_def) + span.set_attribute(GenAIAttributes.GEN_AI_TOOL_DEFINITIONS, json.dumps(tool_defs)) def set_llm_request( @@ -126,17 +122,8 @@ def set_llm_request( set_request_params(span, kwargs, span_holder) if should_send_prompts(): - for i, msg in enumerate(prompts): - _set_span_attribute( - span, - f"{GenAIAttributes.GEN_AI_PROMPT}.{i}.role", - "user", - ) - _set_span_attribute( - span, - f"{GenAIAttributes.GEN_AI_PROMPT}.{i}.content", - msg, - ) + messages = [{"role": "user", "content": msg} for msg in prompts] + span.set_attribute(GenAIAttributes.GEN_AI_INPUT_MESSAGES, json.dumps(messages)) def set_chat_request( @@ -148,81 +135,62 @@ def set_chat_request( ) -> None: set_request_params(span, serialized.get("kwargs", {}), span_holder) + functions = kwargs.get("invocation_params", {}).get("functions", []) + if functions: + tool_defs = [ + { + "name": f.get("name"), + "description": f.get("description"), + "parameters": f.get("parameters"), + } + for f in functions + ] + span.set_attribute(GenAIAttributes.GEN_AI_TOOL_DEFINITIONS, json.dumps(tool_defs)) + if should_send_prompts(): - for i, function in enumerate( - kwargs.get("invocation_params", {}).get("functions", []) - ): - prefix = f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{i}" - - _set_span_attribute(span, f"{prefix}.name", function.get("name")) - _set_span_attribute( - span, f"{prefix}.description", function.get("description") - ) - _set_span_attribute( - span, f"{prefix}.parameters", json.dumps(function.get("parameters")) - ) - - i = 0 + input_messages = [] for message in messages: for msg in message: - _set_span_attribute( - span, - f"{GenAIAttributes.GEN_AI_PROMPT}.{i}.role", - _message_type_to_role(msg.type), - ) + msg_obj = {"role": _message_type_to_role(msg.type)} + tool_calls = ( msg.tool_calls if hasattr(msg, "tool_calls") else msg.additional_kwargs.get("tool_calls") ) - if tool_calls: - _set_chat_tool_calls( - span, f"{GenAIAttributes.GEN_AI_PROMPT}.{i}", tool_calls - ) + msg_obj["tool_calls"] = _build_tool_calls_list(tool_calls) - # Always set content if it exists, regardless of tool_calls presence content = ( msg.content if isinstance(msg.content, str) else json.dumps(msg.content, cls=CallbackFilteredJSONEncoder) ) - _set_span_attribute( - span, - f"{GenAIAttributes.GEN_AI_PROMPT}.{i}.content", - content, - ) + if content: + msg_obj["content"] = content if msg.type == "tool" and hasattr(msg, "tool_call_id"): - _set_span_attribute( - span, - f"{GenAIAttributes.GEN_AI_PROMPT}.{i}.tool_call_id", - msg.tool_call_id, - ) + msg_obj["tool_call_id"] = msg.tool_call_id - i += 1 + input_messages.append(msg_obj) + + if input_messages: + span.set_attribute(GenAIAttributes.GEN_AI_INPUT_MESSAGES, json.dumps(input_messages)) def set_chat_response(span: Span, response: LLMResult) -> None: if not should_send_prompts(): return - i = 0 + output_messages = [] for generations in response.generations: for generation in generations: - prefix = f"{GenAIAttributes.GEN_AI_COMPLETION}.{i}" - if hasattr(generation, "message") and generation.message and hasattr(generation.message, "type"): role = _message_type_to_role(generation.message.type) else: - # For non-chat completions (Generation objects), default to assistant role = "assistant" - _set_span_attribute( - span, - f"{prefix}.role", - role, - ) + msg_obj = {"role": role} # Try to get content from various sources content = None @@ -235,38 +203,19 @@ def set_chat_response(span: Span, response: LLMResult) -> None: content = json.dumps(generation.message.content, cls=CallbackFilteredJSONEncoder) if content: - _set_span_attribute( - span, - f"{prefix}.content", - content, - ) + msg_obj["content"] = content # Set finish reason if available if generation.generation_info and generation.generation_info.get("finish_reason"): - _set_span_attribute( - span, - f"{prefix}.finish_reason", - generation.generation_info.get("finish_reason"), - ) + msg_obj["finish_reason"] = generation.generation_info.get("finish_reason") # Handle tool calls and function calls if hasattr(generation, "message") and generation.message: # Handle legacy function_call format (single function call) if generation.message.additional_kwargs.get("function_call"): - _set_span_attribute( - span, - f"{prefix}.tool_calls.0.name", - generation.message.additional_kwargs.get("function_call").get( - "name" - ), - ) - _set_span_attribute( - span, - f"{prefix}.tool_calls.0.arguments", - generation.message.additional_kwargs.get("function_call").get( - "arguments" - ), - ) + fc = generation.message.additional_kwargs.get("function_call") + msg_obj["role"] = "assistant" + msg_obj["tool_calls"] = [{"name": fc.get("name"), "arguments": fc.get("arguments")}] # Handle new tool_calls format (multiple tool calls) tool_calls = ( @@ -275,13 +224,13 @@ def set_chat_response(span: Span, response: LLMResult) -> None: else generation.message.additional_kwargs.get("tool_calls") ) if tool_calls and isinstance(tool_calls, list): - _set_span_attribute( - span, - f"{prefix}.role", - "assistant", - ) - _set_chat_tool_calls(span, prefix, tool_calls) - i += 1 + msg_obj["role"] = "assistant" + msg_obj["tool_calls"] = _build_tool_calls_list(tool_calls) + + output_messages.append(msg_obj) + + if output_messages: + span.set_attribute(GenAIAttributes.GEN_AI_OUTPUT_MESSAGES, json.dumps(output_messages)) def set_chat_response_usage( @@ -325,9 +274,8 @@ def set_chat_response_usage( "input_token_details", {} ) cache_read_tokens += input_token_details.get("cache_read", 0) - except Exception as e: + except Exception: # If there's any issue processing usage metadata, continue without it - print(f"DEBUG: Error processing usage metadata: {e}") pass if ( @@ -348,22 +296,22 @@ def set_chat_response_usage( ) _set_span_attribute( span, - SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, total_tokens, ) _set_span_attribute( span, - SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS, + SpanAttributes.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read_tokens, ) if record_token_usage: - vendor = span.attributes.get(GenAIAttributes.GEN_AI_SYSTEM, "Langchain") + vendor = span.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME, "langchain") if input_tokens > 0: token_histogram.record( input_tokens, attributes={ - GenAIAttributes.GEN_AI_SYSTEM: vendor, + GenAIAttributes.GEN_AI_PROVIDER_NAME: vendor, GenAIAttributes.GEN_AI_TOKEN_TYPE: "input", GenAIAttributes.GEN_AI_RESPONSE_MODEL: model_name, }, @@ -373,7 +321,7 @@ def set_chat_response_usage( token_histogram.record( output_tokens, attributes={ - GenAIAttributes.GEN_AI_SYSTEM: vendor, + GenAIAttributes.GEN_AI_PROVIDER_NAME: vendor, GenAIAttributes.GEN_AI_TOKEN_TYPE: "output", GenAIAttributes.GEN_AI_RESPONSE_MODEL: model_name, }, @@ -397,11 +345,9 @@ def _extract_model_name_from_association_metadata(metadata: Optional[dict[str, A return "unknown" -def _set_chat_tool_calls( - span: Span, prefix: str, tool_calls: list[dict[str, Any]] -) -> None: - for idx, tool_call in enumerate(tool_calls): - tool_call_prefix = f"{prefix}.tool_calls.{idx}" +def _build_tool_calls_list(tool_calls: list[dict[str, Any]]) -> list[dict[str, Any]]: + result = [] + for tool_call in tool_calls: tool_call_dict = dict(tool_call) tool_id = tool_call_dict.get("id") tool_name = tool_call_dict.get( @@ -411,14 +357,12 @@ def _set_chat_tool_calls( "args", tool_call_dict.get("function", {}).get("arguments") ) - _set_span_attribute(span, f"{tool_call_prefix}.id", tool_id) - _set_span_attribute( - span, - f"{tool_call_prefix}.name", - tool_name, - ) - _set_span_attribute( - span, - f"{tool_call_prefix}.arguments", - json.dumps(tool_args, cls=CallbackFilteredJSONEncoder), - ) + call_obj = {} + if tool_id: + call_obj["id"] = tool_id + if tool_name: + call_obj["name"] = tool_name + if tool_args is not None: + call_obj["arguments"] = json.dumps(tool_args, cls=CallbackFilteredJSONEncoder) + result.append(call_obj) + return result diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/utils.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/utils.py index 079ade8bb3..c65a58c810 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/utils.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/utils.py @@ -16,7 +16,7 @@ TRACELOOP_TRACE_CONTENT = "TRACELOOP_TRACE_CONTENT" -EVENT_ATTRIBUTES = {GenAIAttributes.GEN_AI_SYSTEM: "langchain"} +EVENT_ATTRIBUTES = {GenAIAttributes.GEN_AI_PROVIDER_NAME: "langchain"} class CallbackFilteredJSONEncoder(json.JSONEncoder): diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/vendor_detection.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/vendor_detection.py index 887e174523..0e003b8f25 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/vendor_detection.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/vendor_detection.py @@ -26,7 +26,7 @@ def _get_vendor_rules() -> List[VendorRule]: VendorRule( exact_matches={"AzureChatOpenAI", "AzureOpenAI", "AzureOpenAIEmbeddings"}, patterns=["azure"], - vendor_name="Azure" + vendor_name="az.ai.openai" ), VendorRule( exact_matches={"ChatOpenAI", "OpenAI", "OpenAIEmbeddings"}, @@ -36,12 +36,12 @@ def _get_vendor_rules() -> List[VendorRule]: VendorRule( exact_matches={"ChatBedrock", "BedrockEmbeddings", "Bedrock", "BedrockChat"}, patterns=["bedrock", "aws"], - vendor_name="AWS" + vendor_name="aws.bedrock" ), VendorRule( exact_matches={"ChatAnthropic", "AnthropicLLM"}, patterns=["anthropic"], - vendor_name="Anthropic" + vendor_name="anthropic" ), VendorRule( exact_matches={ @@ -49,12 +49,12 @@ def _get_vendor_rules() -> List[VendorRule]: "GoogleGenerativeAI", "GooglePaLM", "ChatGooglePaLM" }, patterns=["vertex", "google", "palm", "gemini"], - vendor_name="Google" + vendor_name="gcp.gen_ai" ), VendorRule( exact_matches={"ChatCohere", "CohereEmbeddings", "Cohere"}, patterns=["cohere"], - vendor_name="Cohere" + vendor_name="cohere" ), VendorRule( exact_matches={ @@ -62,37 +62,37 @@ def _get_vendor_rules() -> List[VendorRule]: "HuggingFaceEmbeddings", "ChatHuggingFace" }, patterns=["huggingface"], - vendor_name="HuggingFace" + vendor_name="hugging_face" ), VendorRule( exact_matches={"ChatOllama", "OllamaEmbeddings", "Ollama"}, patterns=["ollama"], - vendor_name="Ollama" + vendor_name="ollama" ), VendorRule( exact_matches={"Together", "ChatTogether"}, patterns=["together"], - vendor_name="Together" + vendor_name="together_ai" ), VendorRule( exact_matches={"Replicate", "ChatReplicate"}, patterns=["replicate"], - vendor_name="Replicate" + vendor_name="replicate" ), VendorRule( exact_matches={"ChatFireworks", "Fireworks"}, patterns=["fireworks"], - vendor_name="Fireworks" + vendor_name="fireworks" ), VendorRule( exact_matches={"ChatGroq"}, patterns=["groq"], - vendor_name="Groq" + vendor_name="groq" ), VendorRule( exact_matches={"ChatMistralAI", "MistralAI"}, patterns=["mistral"], - vendor_name="MistralAI" + vendor_name="mistral_ai" ), ] @@ -109,7 +109,7 @@ def detect_vendor_from_class(class_name: str) -> str: Vendor string, defaults to "Langchain" if no match found """ if not class_name: - return "Langchain" + return "langchain" vendor_rules = _get_vendor_rules() @@ -117,4 +117,4 @@ def detect_vendor_from_class(class_name: str) -> str: if rule.matches(class_name): return rule.vendor_name - return "Langchain" + return "langchain" diff --git a/packages/opentelemetry-instrumentation-langchain/pyproject.toml b/packages/opentelemetry-instrumentation-langchain/pyproject.toml index 1725903e45..03459aea43 100644 --- a/packages/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/packages/opentelemetry-instrumentation-langchain/pyproject.toml @@ -13,7 +13,7 @@ requires-python = ">=3.10,<4" dependencies = [ "opentelemetry-api>=1.38.0,<2", "opentelemetry-instrumentation>=0.59b0", - "opentelemetry-semantic-conventions-ai>=0.4.16,<0.5.0", + "opentelemetry-semantic-conventions-ai>=0.5.0,<0.6.0", "opentelemetry-semantic-conventions>=0.59b0", ] diff --git a/packages/opentelemetry-instrumentation-langchain/tests/metrics/test_langchain_metrics.py b/packages/opentelemetry-instrumentation-langchain/tests/metrics/test_langchain_metrics.py index 7f596cf4c4..88f4c37231 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/metrics/test_langchain_metrics.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/metrics/test_langchain_metrics.py @@ -54,7 +54,7 @@ def test_llm_chain_metrics(instrument_legacy, reader, chain): ] assert data_point.sum > 0 assert ( - data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] + data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" ) @@ -68,7 +68,7 @@ def test_llm_chain_metrics(instrument_legacy, reader, chain): ) for data_point in metric.data.data_points: assert ( - data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] + data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" ) @@ -106,7 +106,7 @@ def test_llm_chain_streaming_metrics(instrument_legacy, reader, llm): ] assert data_point.sum > 0 assert ( - data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] + data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" ) @@ -120,7 +120,7 @@ def test_llm_chain_streaming_metrics(instrument_legacy, reader, llm): ) for data_point in metric.data.data_points: assert ( - data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] + data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" ) @@ -135,14 +135,14 @@ def verify_token_metrics(data_points): "input", ] assert data_point.sum > 0 - assert data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "openai" + assert data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" def verify_duration_metrics(data_points): assert any(data_point.count > 0 for data_point in data_points) assert any(data_point.sum > 0 for data_point in data_points) for data_point in data_points: - assert data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "openai" + assert data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" def verify_langchain_metrics(reader): @@ -237,7 +237,7 @@ def calculate(state: State): token_usage_data_point = token_usage_metric.data.data_points[0] assert token_usage_data_point.sum > 0 assert ( - token_usage_data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "openai" + token_usage_data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" and token_usage_data_point.attributes[GenAIAttributes.GEN_AI_TOKEN_TYPE] in ["input", "output"] ) @@ -252,7 +252,7 @@ def calculate(state: State): assert duration_metric is not None duration_data_point = duration_metric.data.data_points[0] assert duration_data_point.sum > 0 - assert duration_data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "openai" + assert duration_data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" generation_choices_metric = next( ( @@ -266,7 +266,7 @@ def calculate(state: State): generation_choices_data_points = generation_choices_metric.data.data_points for data_point in generation_choices_data_points: assert ( - data_point.attributes[GenAIAttributes.GEN_AI_SYSTEM] + data_point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" ) assert data_point.value > 0 diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py b/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py index 9c8835ef04..077aed9dfc 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py @@ -197,7 +197,7 @@ def test_agents_with_events_with_no_content( logs = log_exporter.get_finished_logs() assert len(logs) == 8 assert all( - log.log_record.attributes.get(GenAIAttributes.GEN_AI_SYSTEM) == "langchain" + log.log_record.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME) == "langchain" for log in logs ) diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py b/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py index bc5631ea47..f649dd3fdc 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py @@ -127,7 +127,8 @@ def test_sequential_chain(instrument_legacy, span_exporter, log_exporter): (openai_span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL]) == "gpt-3.5-turbo-instruct" ) - assert openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"] + input_messages = json.loads(openai_span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + assert input_messages[0]["content"] logs = log_exporter.get_finished_logs() assert ( @@ -818,7 +819,7 @@ async def test_astream_with_events_with_no_content( def assert_message_in_logs(log: ReadableLogRecord, event_name: str, expected_content: dict): assert log.log_record.event_name == event_name - assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_SYSTEM) == "langchain" + assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME) == "langchain" if not expected_content: assert not log.log_record.body diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py b/packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py index 59b272062e..74f02cac1d 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py @@ -155,7 +155,7 @@ def test_sequential_chain_with_events_with_no_content( def assert_message_in_logs(log: ReadableLogRecord, event_name: str, expected_content: dict): assert log.log_record.event_name == event_name - assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_SYSTEM) == "langchain" + assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME) == "langchain" if not expected_content: assert not log.log_record.body diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_generation_role_extraction.py b/packages/opentelemetry-instrumentation-langchain/tests/test_generation_role_extraction.py index 60068843a3..8c133e2e1b 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_generation_role_extraction.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_generation_role_extraction.py @@ -6,6 +6,7 @@ in observability traces. """ +import json import pytest from unittest.mock import Mock from langchain_core.outputs import LLMResult, ChatGeneration, Generation @@ -32,98 +33,78 @@ def set_attribute(key, value): def test_chat_generation_with_ai_message_role(self, mock_span, monkeypatch): """Test that ChatGeneration with AIMessage correctly extracts 'assistant' role.""" - # Mock should_send_prompts to return True monkeypatch.setattr( "opentelemetry.instrumentation.langchain.span_utils.should_send_prompts", lambda: True ) - # Create ChatGeneration with AIMessage generation = ChatGeneration(message=AIMessage(content="Hello!")) llm_result = LLMResult(generations=[[generation]]) - # Call the function set_chat_response(mock_span, llm_result) - # Assert role is 'assistant', not 'unknown' - role_key = f"{GenAIAttributes.GEN_AI_COMPLETION}.0.role" - assert role_key in mock_span.attributes - assert mock_span.attributes[role_key] == "assistant" + output_messages = json.loads(mock_span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES]) + assert output_messages[0]["role"] == "assistant" def test_chat_generation_with_tool_message_role(self, mock_span, monkeypatch): """Test that ChatGeneration with ToolMessage correctly extracts 'tool' role.""" - # Mock should_send_prompts to return True monkeypatch.setattr( "opentelemetry.instrumentation.langchain.span_utils.should_send_prompts", lambda: True ) - # Create ChatGeneration with ToolMessage generation = ChatGeneration( message=ToolMessage(content="Tool result", tool_call_id="123") ) llm_result = LLMResult(generations=[[generation]]) - # Call the function set_chat_response(mock_span, llm_result) - # Assert role is 'tool', not 'unknown' - role_key = f"{GenAIAttributes.GEN_AI_COMPLETION}.0.role" - assert role_key in mock_span.attributes - assert mock_span.attributes[role_key] == "tool" + output_messages = json.loads(mock_span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES]) + assert output_messages[0]["role"] == "tool" def test_generation_without_message_defaults_to_assistant(self, mock_span, monkeypatch): """Test that Generation (non-chat) defaults to 'assistant' role.""" - # Mock should_send_prompts to return True monkeypatch.setattr( "opentelemetry.instrumentation.langchain.span_utils.should_send_prompts", lambda: True ) - # Create Generation without message (legacy completion) generation = Generation(text="This is a completion") llm_result = LLMResult(generations=[[generation]]) - # Call the function set_chat_response(mock_span, llm_result) - # Assert role defaults to 'assistant', not 'unknown' - role_key = f"{GenAIAttributes.GEN_AI_COMPLETION}.0.role" - assert role_key in mock_span.attributes - assert mock_span.attributes[role_key] == "assistant" + output_messages = json.loads(mock_span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES]) + assert output_messages[0]["role"] == "assistant" def test_multiple_generations_with_different_roles(self, mock_span, monkeypatch): """Test that multiple generations with different message types are handled correctly.""" - # Mock should_send_prompts to return True monkeypatch.setattr( "opentelemetry.instrumentation.langchain.span_utils.should_send_prompts", lambda: True ) - # Create multiple generations with different message types gen1 = ChatGeneration(message=AIMessage(content="AI response")) gen2 = ChatGeneration(message=ToolMessage(content="Tool result", tool_call_id="123")) gen3 = Generation(text="Legacy completion") llm_result = LLMResult(generations=[[gen1], [gen2], [gen3]]) - # Call the function set_chat_response(mock_span, llm_result) - # Assert all roles are correctly set - assert mock_span.attributes[f"{GenAIAttributes.GEN_AI_COMPLETION}.0.role"] == "assistant" - assert mock_span.attributes[f"{GenAIAttributes.GEN_AI_COMPLETION}.1.role"] == "tool" - assert mock_span.attributes[f"{GenAIAttributes.GEN_AI_COMPLETION}.2.role"] == "assistant" + output_messages = json.loads(mock_span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES]) + assert output_messages[0]["role"] == "assistant" + assert output_messages[1]["role"] == "tool" + assert output_messages[2]["role"] == "assistant" def test_generation_type_attribute_is_not_used(self, mock_span, monkeypatch): """Test that generation.type (which returns class name) is not used directly.""" - # Mock should_send_prompts to return True monkeypatch.setattr( "opentelemetry.instrumentation.langchain.span_utils.should_send_prompts", lambda: True ) - # Create ChatGeneration - note that generation.type would be "ChatGeneration" generation = ChatGeneration(message=AIMessage(content="Test")) # Verify the bug scenario: generation.type returns class name, not message type @@ -132,11 +113,7 @@ def test_generation_type_attribute_is_not_used(self, mock_span, monkeypatch): llm_result = LLMResult(generations=[[generation]]) - # Call the function set_chat_response(mock_span, llm_result) - # Assert role is 'assistant', not 'unknown' - # If the bug existed, passing generation.type directly to _message_type_to_role - # would return 'unknown' because "ChatGeneration" doesn't match any message type - role_key = f"{GenAIAttributes.GEN_AI_COMPLETION}.0.role" - assert mock_span.attributes[role_key] == "assistant" + output_messages = json.loads(mock_span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES]) + assert output_messages[0]["role"] == "assistant" diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_langgraph.py b/packages/opentelemetry-instrumentation-langchain/tests/test_langgraph.py index c912106200..7c84283714 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_langgraph.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_langgraph.py @@ -56,7 +56,7 @@ def calculate(state: State): # agent_id removed per maintainer feedback - rely on agent name only assert openai_span.parent.span_id == calculate_task_span.context.span_id - assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert openai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert openai_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o" assert ( openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"] @@ -76,7 +76,7 @@ def calculate(state: State): assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 24 assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 11 - assert openai_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 35 + assert openai_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 35 @pytest.mark.vcr diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_lcel.py b/packages/opentelemetry-instrumentation-langchain/tests/test_lcel.py index cf981f9bc1..96c37b3cf1 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_lcel.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_lcel.py @@ -1003,7 +1003,7 @@ class Joke(BaseModel): def assert_message_in_logs(log: ReadableLogRecord, event_name: str, expected_content: dict): assert log.log_record.event_name == event_name - assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_SYSTEM) == "langchain" + assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME) == "langchain" if not expected_content: assert not log.log_record.body diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_llms.py b/packages/opentelemetry-instrumentation-langchain/tests/test_llms.py index fcf8767b75..4bd98a7acb 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_llms.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_llms.py @@ -147,17 +147,13 @@ def test_custom_llm(instrument_legacy, span_exporter, log_exporter): span for span in spans if span.name == "HuggingFaceTextGenInference.completion" ) - assert hugging_face_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "completion" + assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "completion" assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "unknown" - assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "HuggingFace" - assert ( - hugging_face_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"] - == "System: You are a helpful assistant\nHuman: tell me a short joke" - ) - assert ( - hugging_face_span.attributes[f"{GenAIAttributes.GEN_AI_COMPLETION}.0.content"] - == response - ) + assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "hugging_face" + input_messages = json.loads(hugging_face_span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + assert input_messages[0]["content"] == "System: You are a helpful assistant\nHuman: tell me a short joke" + output_messages = json.loads(hugging_face_span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES]) + assert output_messages[0]["content"] == response logs = log_exporter.get_finished_logs() assert len(logs) == 0, ( @@ -192,7 +188,7 @@ def test_custom_llm_with_events_with_content( span for span in spans if span.name == "HuggingFaceTextGenInference.completion" ) - assert hugging_face_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "completion" + assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "completion" assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "unknown" logs = log_exporter.get_finished_logs() @@ -243,7 +239,7 @@ def test_custom_llm_with_events_with_no_content( span for span in spans if span.name == "HuggingFaceTextGenInference.completion" ) - assert hugging_face_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "completion" + assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "completion" assert hugging_face_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "unknown" logs = log_exporter.get_finished_logs() @@ -283,20 +279,18 @@ def test_openai(instrument_legacy, span_exporter, log_exporter): openai_span = next(span for span in spans if span.name == "ChatOpenAI.chat") - assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert openai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert openai_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" - assert openai_span.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "openai" - assert ( - (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"]) - == "You are a helpful assistant" - ) - assert (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.role"]) == "system" - assert (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.content"]) == prompt - assert (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.role"]) == "user" + assert openai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" + input_messages = json.loads(openai_span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + assert input_messages[0]["content"] == "You are a helpful assistant" + assert input_messages[0]["role"] == "system" + assert input_messages[1]["content"] == prompt + assert input_messages[1]["role"] == "user" assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 1497 assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 1037 - assert openai_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 2534 + assert openai_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 2534 workflow_span = next( span for span in spans if span.name == "RunnableSequence.workflow" @@ -338,12 +332,12 @@ def test_openai_with_events_with_content( openai_span = next(span for span in spans if span.name == "ChatOpenAI.chat") - assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert openai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert openai_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 1497 assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 1037 - assert openai_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 2534 + assert openai_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 2534 logs = log_exporter.get_finished_logs() assert len(logs) == 3 @@ -389,12 +383,12 @@ def test_openai_with_events_with_no_content( openai_span = next(span for span in spans if span.name == "ChatOpenAI.chat") - assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert openai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert openai_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 1497 assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 1037 - assert openai_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 2534 + assert openai_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 2534 logs = log_exporter.get_finished_logs() assert len(logs) == 3 @@ -446,33 +440,17 @@ class Joke(BaseModel): openai_span = next(span for span in spans if span.name == "ChatOpenAI.chat") - assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert openai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert openai_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" - assert ( - (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"]) - == "You are helpful assistant" - ) - assert (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.role"]) == "system" - assert ( - (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.content"]) - == "tell me a short joke" - ) - assert (openai_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.role"]) == "user" - assert ( - openai_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name"] - == "Joke" - ) - assert ( - openai_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.description"] - == "Joke to tell user." - ) - assert ( - json.loads( - openai_span.attributes[ - f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.parameters" - ] - ) - ) == { + input_messages = json.loads(openai_span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + assert input_messages[0]["content"] == "You are helpful assistant" + assert input_messages[0]["role"] == "system" + assert input_messages[1]["content"] == "tell me a short joke" + assert input_messages[1]["role"] == "user" + tool_defs = json.loads(openai_span.attributes[GenAIAttributes.GEN_AI_TOOL_DEFINITIONS]) + assert tool_defs[0]["name"] == "Joke" + assert tool_defs[0]["description"] == "Joke to tell user." + assert tool_defs[0]["parameters"] == { "type": "object", "properties": { "setup": {"description": "question to set up a joke", "type": "string"}, @@ -485,7 +463,7 @@ class Joke(BaseModel): } assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 76 assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 35 - assert openai_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 111 + assert openai_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 111 workflow_span = next( span for span in spans if span.name == "RunnableSequence.workflow" @@ -536,12 +514,12 @@ class Joke(BaseModel): openai_span = next(span for span in spans if span.name == "ChatOpenAI.chat") - assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert openai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert openai_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 76 assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 35 - assert openai_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 111 + assert openai_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 111 logs = log_exporter.get_finished_logs() assert len(logs) == 3 @@ -609,12 +587,12 @@ class Joke(BaseModel): openai_span = next(span for span in spans if span.name == "ChatOpenAI.chat") - assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert openai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert openai_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 76 assert openai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 35 - assert openai_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 111 + assert openai_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 111 logs = log_exporter.get_finished_logs() assert len(logs) == 3 @@ -658,25 +636,18 @@ def test_anthropic(instrument_legacy, span_exporter, log_exporter): span for span in spans if span.name == "RunnableSequence.workflow" ) - assert anthropic_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert anthropic_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert anthropic_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "claude-2.1" - assert anthropic_span.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "Anthropic" + assert anthropic_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "anthropic" assert anthropic_span.attributes[GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE] == 0.5 - assert ( - (anthropic_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"]) - == "You are a helpful assistant" - ) - assert ( - (anthropic_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.role"]) == "system" - ) - assert ( - (anthropic_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.content"]) - == "tell me a short joke" - ) - assert (anthropic_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.role"]) == "user" + input_messages = json.loads(anthropic_span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + assert input_messages[0]["content"] == "You are a helpful assistant" + assert input_messages[0]["role"] == "system" + assert input_messages[1]["content"] == "tell me a short joke" + assert input_messages[1]["role"] == "user" assert anthropic_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 19 assert anthropic_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 22 - assert anthropic_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 41 + assert anthropic_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 41 assert ( anthropic_span.attributes["gen_ai.response.id"] == "msg_017fMG9SRDFTBhcD1ibtN1nK" @@ -725,13 +696,13 @@ def test_anthropic_with_events_with_content( anthropic_span = next(span for span in spans if span.name == "ChatAnthropic.chat") - assert anthropic_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert anthropic_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert anthropic_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "claude-2.1" assert anthropic_span.attributes[GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE] == 0.5 assert anthropic_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 19 assert anthropic_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 22 - assert anthropic_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 41 + assert anthropic_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 41 assert ( anthropic_span.attributes["gen_ai.response.id"] == "msg_017fMG9SRDFTBhcD1ibtN1nK" @@ -781,13 +752,13 @@ def test_anthropic_with_events_with_no_content( anthropic_span = next(span for span in spans if span.name == "ChatAnthropic.chat") - assert anthropic_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert anthropic_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert anthropic_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "claude-2.1" assert anthropic_span.attributes[GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE] == 0.5 assert anthropic_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 19 assert anthropic_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 22 - assert anthropic_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 41 + assert anthropic_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 41 assert ( anthropic_span.attributes["gen_ai.response.id"] == "msg_017fMG9SRDFTBhcD1ibtN1nK" @@ -843,25 +814,20 @@ def test_bedrock(instrument_legacy, span_exporter, log_exporter): span for span in spans if span.name == "RunnableSequence.workflow" ) - assert bedrock_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert bedrock_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert ( bedrock_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "anthropic.claude-3-haiku-20240307-v1:0" ) - assert bedrock_span.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "AWS" - assert ( - (bedrock_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"]) - == "You are a helpful assistant" - ) - assert (bedrock_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.role"]) == "system" - assert ( - (bedrock_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.content"]) - == "tell me a short joke" - ) - assert (bedrock_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.role"]) == "user" + assert bedrock_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "aws.bedrock" + input_messages = json.loads(bedrock_span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + assert input_messages[0]["content"] == "You are a helpful assistant" + assert input_messages[0]["role"] == "system" + assert input_messages[1]["content"] == "tell me a short joke" + assert input_messages[1]["role"] == "user" assert bedrock_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 16 assert bedrock_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 27 - assert bedrock_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 43 + assert bedrock_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 43 output = json.loads( workflow_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT] ) @@ -914,7 +880,7 @@ def test_bedrock_with_events_with_content( bedrock_span = next(span for span in spans if span.name == "ChatBedrock.chat") - assert bedrock_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert bedrock_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert ( bedrock_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "anthropic.claude-3-haiku-20240307-v1:0" @@ -922,7 +888,7 @@ def test_bedrock_with_events_with_content( assert bedrock_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 16 assert bedrock_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 27 - assert bedrock_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 43 + assert bedrock_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 43 logs = log_exporter.get_finished_logs() assert len(logs) == 3 @@ -977,14 +943,14 @@ def test_bedrock_with_events_with_no_content( bedrock_span = next(span for span in spans if span.name == "ChatBedrock.chat") - assert bedrock_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat" + assert bedrock_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat" assert ( bedrock_span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == "anthropic.claude-3-haiku-20240307-v1:0" ) assert bedrock_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 16 assert bedrock_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 27 - assert bedrock_span.attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 43 + assert bedrock_span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 43 logs = log_exporter.get_finished_logs() assert len(logs) == 3 @@ -1050,7 +1016,7 @@ def test_trace_propagation(instrument_legacy, span_exporter, log_exporter, LLM): VLLMOpenAI: "openai", ChatOpenAI: "openai" } - assert openai_span.attributes[GenAIAttributes.GEN_AI_SYSTEM] == expected_vendors[LLM] + assert openai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == expected_vendors[LLM] args, kwargs = send_spy.mock.call_args request = args[0] @@ -1658,7 +1624,7 @@ async def test_trace_propagation_stream_async_with_events_with_no_content( def assert_message_in_logs(log: ReadableLogRecord, event_name: str, expected_content: dict): assert log.log_record.event_name == event_name - assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_SYSTEM) == "langchain" + assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME) == "langchain" if not expected_content: assert not log.log_record.body diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_semconv_compliance.py b/packages/opentelemetry-instrumentation-langchain/tests/test_semconv_compliance.py new file mode 100644 index 0000000000..35a01e3380 --- /dev/null +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_semconv_compliance.py @@ -0,0 +1,8 @@ +# ruff: noqa: F401, F403 +""" +Semconv compliance tests re-used from opentelemetry-semantic-conventions-ai. + +Ensures the installed semconv package has the expected constant values. +To add more compliance checks, update _testing.py in that package — not here. +""" +from opentelemetry.semconv_ai._testing import * diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_structured_output.py b/packages/opentelemetry-instrumentation-langchain/tests/test_structured_output.py index 64899bea03..90aeec4032 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_structured_output.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_structured_output.py @@ -1,3 +1,4 @@ +import json from typing import List import pytest @@ -32,7 +33,8 @@ def test_structured_output(instrument_legacy, span_exporter, log_exporter): chat_span = next(span for span in spans if span.name == "ChatOpenAI.chat") - assert chat_span.attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"] == query_text + input_messages = json.loads(chat_span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + assert input_messages[0]["content"] == query_text logs = log_exporter.get_finished_logs() assert ( @@ -104,7 +106,7 @@ def test_structured_output_with_events_with_no_content( def assert_message_in_logs(log: ReadableLogRecord, event_name: str, expected_content: dict): assert log.log_record.event_name == event_name - assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_SYSTEM) == "langchain" + assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME) == "langchain" if not expected_content: assert not log.log_record.body diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_tool_call_content.py b/packages/opentelemetry-instrumentation-langchain/tests/test_tool_call_content.py index c0b447940b..834fc27321 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_tool_call_content.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_tool_call_content.py @@ -2,10 +2,11 @@ Test for the fix of the issue where assistant message content is missing when tool calls are present in LangGraph/LangChain instrumentation. -This test reproduces the issue reported in GitHub where gen_ai.prompt.X.content +This test reproduces the issue reported in GitHub where gen_ai.input.messages attributes were missing for assistant messages that contained tool_calls. """ +import json from unittest.mock import Mock from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from opentelemetry.instrumentation.langchain.span_utils import set_chat_request @@ -51,48 +52,23 @@ def test_assistant_message_with_tool_calls_includes_content(): call_args = [call[0] for call in mock_span.set_attribute.call_args_list] attributes = {args[0]: args[1] for args in call_args} - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.role" in attributes - assert attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.role"] == "user" - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"] - == "what is the current time? First greet me." - ) - assert f"{GenAIAttributes.GEN_AI_PROMPT}.1.role" in attributes - assert attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.role"] == "assistant" - assert f"{GenAIAttributes.GEN_AI_PROMPT}.1.content" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.content"] - == "Hello! Let me check the current time for you." - ) - assert f"{GenAIAttributes.GEN_AI_PROMPT}.1.tool_calls.0.id" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.tool_calls.0.id"] - == "call_qU7pH3EdQvzwkPyKPOdpgaKA" - ) - assert f"{GenAIAttributes.GEN_AI_PROMPT}.1.tool_calls.0.name" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.1.tool_calls.0.name"] - == "get_current_time" - ) - assert f"{GenAIAttributes.GEN_AI_PROMPT}.2.role" in attributes - assert attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.2.role"] == "tool" - assert f"{GenAIAttributes.GEN_AI_PROMPT}.2.content" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.2.content"] == "2025-08-15 08:15:21" - ) - assert f"{GenAIAttributes.GEN_AI_PROMPT}.2.tool_call_id" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.2.tool_call_id"] - == "call_qU7pH3EdQvzwkPyKPOdpgaKA" - ) - assert f"{GenAIAttributes.GEN_AI_PROMPT}.3.role" in attributes - assert attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.3.role"] == "assistant" - assert f"{GenAIAttributes.GEN_AI_PROMPT}.3.content" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.3.content"] - == "The current time is 2025-08-15 08:15:21" - ) + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES in attributes + input_messages = json.loads(attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + + assert input_messages[0]["role"] == "user" + assert input_messages[0]["content"] == "what is the current time? First greet me." + + assert input_messages[1]["role"] == "assistant" + assert input_messages[1]["content"] == "Hello! Let me check the current time for you." + assert input_messages[1]["tool_calls"][0]["id"] == "call_qU7pH3EdQvzwkPyKPOdpgaKA" + assert input_messages[1]["tool_calls"][0]["name"] == "get_current_time" + + assert input_messages[2]["role"] == "tool" + assert input_messages[2]["content"] == "2025-08-15 08:15:21" + assert input_messages[2]["tool_call_id"] == "call_qU7pH3EdQvzwkPyKPOdpgaKA" + + assert input_messages[3]["role"] == "assistant" + assert input_messages[3]["content"] == "The current time is 2025-08-15 08:15:21" def test_assistant_message_with_only_tool_calls_no_content(): @@ -121,16 +97,12 @@ def test_assistant_message_with_only_tool_calls_no_content(): call_args = [call[0] for call in mock_span.set_attribute.call_args_list] attributes = {args[0]: args[1] for args in call_args} - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.role" in attributes - assert attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.role"] == "assistant" - # Content is being set as empty string, so we expect it to be present - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" in attributes - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.tool_calls.0.id" in attributes - assert attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.tool_calls.0.id"] == "call_123" - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.tool_calls.0.name" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.tool_calls.0.name"] == "some_tool" - ) + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES in attributes + input_messages = json.loads(attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) + + assert input_messages[0]["role"] == "assistant" + assert input_messages[0]["tool_calls"][0]["id"] == "call_123" + assert input_messages[0]["tool_calls"][0]["name"] == "some_tool" def test_assistant_message_with_only_content_no_tool_calls(): @@ -148,16 +120,11 @@ def test_assistant_message_with_only_content_no_tool_calls(): set_chat_request(mock_span, {}, messages, {}, mock_span_holder) call_args = [call[0] for call in mock_span.set_attribute.call_args_list] - attributes = {args[0]: args[1] for args in call_args} - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.role" in attributes - assert attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.role"] == "assistant" - assert f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" in attributes - assert ( - attributes[f"{GenAIAttributes.GEN_AI_PROMPT}.0.content"] - == "Just a regular response with no tool calls" - ) + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES in attributes + input_messages = json.loads(attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES]) - tool_call_attributes = [attr for attr in attributes.keys() if "tool_calls" in attr] - assert len(tool_call_attributes) == 0 + assert input_messages[0]["role"] == "assistant" + assert input_messages[0]["content"] == "Just a regular response with no tool calls" + assert "tool_calls" not in input_messages[0] diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_tool_calls.py b/packages/opentelemetry-instrumentation-langchain/tests/test_tool_calls.py index 1908535164..72b4a1c6f1 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_tool_calls.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_tool_calls.py @@ -15,7 +15,6 @@ from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) -from opentelemetry.semconv_ai import SpanAttributes def food_analysis( @@ -41,8 +40,8 @@ def test_tool_calls(instrument_legacy, span_exporter, log_exporter): # span for span in spans if span.name == "ChatOpenAI.chat" # ) - # assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name"] == "food_analysis" - # assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.parameters"]) == { + # assert chat_span.attributes[f"{GenAIAttributes.GEN_AI_TOOL_DEFINITIONS}.0.name"] == "food_analysis" + # assert json.loads(chat_span.attributes[f"{GenAIAttributes.GEN_AI_TOOL_DEFINITIONS}.0.parameters"]) == { # "properties": { # "name": {"type": "string"}, # "healthy": {"type": "boolean"}, @@ -192,17 +191,9 @@ def get_weather(location: str) -> str: chat_span = spans[0] assert chat_span.name == "ChatOpenAI.chat" - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name"] == "get_weather" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.parameters"]) == { - "properties": { - "location": {"type": "string"}, - }, - "required": ["location"], - "type": "object", - } - - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name"] == "get_weather" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.parameters"]) == { + tool_defs = json.loads(chat_span.attributes[GenAIAttributes.GEN_AI_TOOL_DEFINITIONS]) + assert tool_defs[0]["name"] == "get_weather" + assert tool_defs[0]["parameters"] == { "properties": { "location": {"type": "string"}, }, @@ -480,17 +471,17 @@ def get_news(location: str) -> str: chat_span = spans[0] assert chat_span.name == "ChatAnthropic.chat" - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name"] == "get_weather" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.parameters"]) == { + tool_defs = json.loads(chat_span.attributes[GenAIAttributes.GEN_AI_TOOL_DEFINITIONS]) + assert tool_defs[0]["name"] == "get_weather" + assert tool_defs[0]["parameters"] == { "properties": { "location": {"type": "string"}, }, "required": ["location"], "type": "object", } - - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.name"] == "get_news" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.parameters"]) == { + assert tool_defs[1]["name"] == "get_news" + assert tool_defs[1]["parameters"] == { "properties": { "location": {"type": "string"}, }, @@ -697,17 +688,17 @@ def get_news(location: str) -> str: chat_span = spans[0] assert chat_span.name == "ChatAnthropic.chat" - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name"] == "get_weather" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.parameters"]) == { + tool_defs = json.loads(chat_span.attributes[GenAIAttributes.GEN_AI_TOOL_DEFINITIONS]) + assert tool_defs[0]["name"] == "get_weather" + assert tool_defs[0]["parameters"] == { "properties": { "location": {"type": "string"}, }, "required": ["location"], "type": "object", } - - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.name"] == "get_news" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.parameters"]) == { + assert tool_defs[1]["name"] == "get_news" + assert tool_defs[1]["parameters"] == { "properties": { "location": {"type": "string"}, }, @@ -1045,17 +1036,17 @@ def get_news(location: str) -> str: chat_span = spans[0] assert chat_span.name == "ChatOpenAI.chat" - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.name"] == "get_weather" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.0.parameters"]) == { + tool_defs = json.loads(chat_span.attributes[GenAIAttributes.GEN_AI_TOOL_DEFINITIONS]) + assert tool_defs[0]["name"] == "get_weather" + assert tool_defs[0]["parameters"] == { "properties": { "location": {"type": "string"}, }, "required": ["location"], "type": "object", } - - assert chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.name"] == "get_news" - assert json.loads(chat_span.attributes[f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.1.parameters"]) == { + assert tool_defs[1]["name"] == "get_news" + assert tool_defs[1]["parameters"] == { "properties": { "location": {"type": "string"}, }, @@ -1226,7 +1217,7 @@ def get_news(location: str) -> str: def assert_message_in_logs(log: ReadableLogRecord, event_name: str, expected_content: dict): assert log.log_record.event_name == event_name - assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_SYSTEM) == "langchain" + assert log.log_record.attributes.get(GenAIAttributes.GEN_AI_PROVIDER_NAME) == "langchain" if not expected_content: assert not log.log_record.body diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_vendor_detection.py b/packages/opentelemetry-instrumentation-langchain/tests/test_vendor_detection.py new file mode 100644 index 0000000000..af2dac4b9f --- /dev/null +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_vendor_detection.py @@ -0,0 +1,59 @@ +import pytest +from opentelemetry.instrumentation.langchain.vendor_detection import detect_vendor_from_class + + +@pytest.mark.parametrize("class_name,expected", [ + # Exact matches + ("AzureChatOpenAI", "az.ai.openai"), + ("AzureOpenAI", "az.ai.openai"), + ("AzureOpenAIEmbeddings", "az.ai.openai"), + ("ChatOpenAI", "openai"), + ("OpenAI", "openai"), + ("OpenAIEmbeddings", "openai"), + ("ChatBedrock", "aws.bedrock"), + ("BedrockEmbeddings", "aws.bedrock"), + ("Bedrock", "aws.bedrock"), + ("BedrockChat", "aws.bedrock"), + ("ChatAnthropic", "anthropic"), + ("AnthropicLLM", "anthropic"), + ("ChatVertexAI", "gcp.gen_ai"), + ("VertexAI", "gcp.gen_ai"), + ("ChatGoogleGenerativeAI", "gcp.gen_ai"), + ("GoogleGenerativeAI", "gcp.gen_ai"), + ("ChatCohere", "cohere"), + ("Cohere", "cohere"), + ("HuggingFacePipeline", "hugging_face"), + ("HuggingFaceTextGenInference", "hugging_face"), + ("ChatHuggingFace", "hugging_face"), + ("ChatOllama", "ollama"), + ("Ollama", "ollama"), + ("Together", "together_ai"), + ("ChatTogether", "together_ai"), + ("Replicate", "replicate"), + ("ChatReplicate", "replicate"), + ("ChatFireworks", "fireworks"), + ("Fireworks", "fireworks"), + ("ChatGroq", "groq"), + ("ChatMistralAI", "mistral_ai"), + ("MistralAI", "mistral_ai"), + # Pattern matches + ("SomeAzureModel", "az.ai.openai"), + ("CustomOpenAIModel", "openai"), + ("AwsBedrockModel", "aws.bedrock"), + ("AnthropicCustom", "anthropic"), + ("VertexCustom", "gcp.gen_ai"), + ("GeminiModel", "gcp.gen_ai"), + ("CohereCustom", "cohere"), + ("OllamaCustom", "ollama"), + ("TogetherCustom", "together_ai"), + ("ReplicateCustom", "replicate"), + ("FireworksCustom", "fireworks"), + ("GroqCustom", "groq"), + ("MistralCustom", "mistral_ai"), + # Default fallback + ("UnknownModel", "langchain"), + ("", "langchain"), + (None, "langchain"), +]) +def test_detect_vendor_from_class(class_name, expected): + assert detect_vendor_from_class(class_name) == expected