diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4fb3ba005..69dae8b7a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,15 @@ jobs: runs-on: ubuntu-latest steps: + - name: Free up disk space + run: | + sudo apt-get remove -y '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri || true + sudo apt-get autoremove -y + sudo apt-get clean + # Remove Docker images + docker rmi $(docker images -aq) || true + # Show available space + df -h - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -55,6 +64,15 @@ jobs: python-version: ["3.11"] steps: + - name: Free up disk space + run: | + sudo apt-get remove -y '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri || true + sudo apt-get autoremove -y + sudo apt-get clean + # Remove Docker images + docker rmi $(docker images -aq) || true + # Show available space + df -h - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -91,6 +109,15 @@ jobs: python-version: ["3.10", "3.11", "3.12"] steps: + - name: Free up disk space + run: | + sudo apt-get remove -y '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri || true + sudo apt-get autoremove -y + sudo apt-get clean + # Remove Docker images + docker rmi $(docker images -aq) || true + # Show available space + df -h - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py index 68de21895e..5469aad680 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py @@ -176,17 +176,17 @@ def set_chat_request( span, f"{SpanAttributes.LLM_PROMPTS}.{i}", tool_calls ) - else: - content = ( - msg.content - if isinstance(msg.content, str) - else json.dumps(msg.content, cls=CallbackFilteredJSONEncoder) - ) - _set_span_attribute( - span, - f"{SpanAttributes.LLM_PROMPTS}.{i}.content", - content, - ) + # Always set content if it exists, regardless of tool_calls presence + content = ( + msg.content + if isinstance(msg.content, str) + else json.dumps(msg.content, cls=CallbackFilteredJSONEncoder) + ) + _set_span_attribute( + span, + f"{SpanAttributes.LLM_PROMPTS}.{i}.content", + content, + ) if msg.type == "tool" and hasattr(msg, "tool_call_id"): _set_span_attribute( diff --git a/packages/opentelemetry-instrumentation-langchain/tests/conftest.py b/packages/opentelemetry-instrumentation-langchain/tests/conftest.py index 4278f13dae..01930dca3d 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/conftest.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -144,8 +144,6 @@ def environment(): os.environ["COHERE_API_KEY"] = "test" if not os.environ.get("TAVILY_API_KEY"): os.environ["TAVILY_API_KEY"] = "test" - if not os.environ.get("LANGSMITH_API_KEY"): - os.environ["LANGSMITH_API_KEY"] = "test" @pytest.fixture(scope="module") diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py b/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py index 837557b603..bac44f561a 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_agents.py @@ -1,10 +1,9 @@ -import os from typing import Tuple import pytest -from langchain import hub from langchain.agents import AgentExecutor, create_tool_calling_agent from langchain_community.tools.tavily_search import TavilySearchResults +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain_openai import ChatOpenAI from opentelemetry.sdk._logs import LogData from opentelemetry.semconv._incubating.attributes import ( @@ -14,6 +13,16 @@ gen_ai_attributes as GenAIAttributes, ) +# Constant prompt template to replace hub.pull("hwchase17/openai-functions-agent") +OPENAI_FUNCTIONS_AGENT_PROMPT = ChatPromptTemplate.from_messages( + [ + ("system", "You are a helpful assistant"), + MessagesPlaceholder("chat_history", optional=True), + ("human", "{input}"), + MessagesPlaceholder("agent_scratchpad"), + ] +) + @pytest.mark.vcr def test_agents(instrument_legacy, span_exporter, log_exporter): @@ -22,10 +31,7 @@ def test_agents(instrument_legacy, span_exporter, log_exporter): model = ChatOpenAI(model="gpt-3.5-turbo") - prompt = hub.pull( - "hwchase17/openai-functions-agent", - api_key=os.environ["LANGSMITH_API_KEY"], - ) + prompt = OPENAI_FUNCTIONS_AGENT_PROMPT agent = create_tool_calling_agent(model, tools, prompt) agent_executor = AgentExecutor(agent=agent, tools=tools) @@ -68,10 +74,7 @@ def test_agents_with_events_with_content( model = ChatOpenAI(model="gpt-3.5-turbo") - prompt = hub.pull( - "hwchase17/openai-functions-agent", - api_key=os.environ["LANGSMITH_API_KEY"], - ) + prompt = OPENAI_FUNCTIONS_AGENT_PROMPT agent = create_tool_calling_agent(model, tools, prompt) agent_executor = AgentExecutor(agent=agent, tools=tools) @@ -166,10 +169,7 @@ def test_agents_with_events_with_no_content( model = ChatOpenAI(model="gpt-3.5-turbo") - prompt = hub.pull( - "hwchase17/openai-functions-agent", - api_key=os.environ["LANGSMITH_API_KEY"], - ) + prompt = OPENAI_FUNCTIONS_AGENT_PROMPT agent = create_tool_calling_agent(model, tools, prompt) agent_executor = AgentExecutor(agent=agent, tools=tools) diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_tool_call_content.py b/packages/opentelemetry-instrumentation-langchain/tests/test_tool_call_content.py new file mode 100644 index 0000000000..b94017dcb0 --- /dev/null +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_tool_call_content.py @@ -0,0 +1,160 @@ +""" +Test for the fix of the issue where assistant message content is missing +when tool calls are present in LangGraph/LangChain instrumentation. + +This test reproduces the issue reported in GitHub where gen_ai.prompt.X.content +attributes were missing for assistant messages that contained tool_calls. +""" + +from unittest.mock import Mock +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from opentelemetry.instrumentation.langchain.span_utils import set_chat_request +from opentelemetry.semconv_ai import SpanAttributes + + +def test_assistant_message_with_tool_calls_includes_content(): + """ + Test that when an assistant message has both content and tool_calls, + both the content and tool_calls are included in the span attributes. + + This addresses the issue where content was missing when tool_calls were present. + """ + mock_span = Mock() + mock_span.set_attribute = Mock() + mock_span_holder = Mock() + mock_span_holder.request_model = None + messages = [ + [ + HumanMessage(content="what is the current time? First greet me."), + AIMessage( + content="Hello! Let me check the current time for you.", + tool_calls=[ + { + "id": "call_qU7pH3EdQvzwkPyKPOdpgaKA", + "name": "get_current_time", + "args": {}, + } + ], + ), + ToolMessage( + content="2025-08-15 08:15:21", + tool_call_id="call_qU7pH3EdQvzwkPyKPOdpgaKA", + ), + AIMessage(content="The current time is 2025-08-15 08:15:21"), + ] + ] + + set_chat_request(mock_span, {}, messages, {}, mock_span_holder) + + call_args = [call[0] for call in mock_span.set_attribute.call_args_list] + attributes = {args[0]: args[1] for args in call_args} + + assert f"{SpanAttributes.LLM_PROMPTS}.0.role" in attributes + assert attributes[f"{SpanAttributes.LLM_PROMPTS}.0.role"] == "user" + assert f"{SpanAttributes.LLM_PROMPTS}.0.content" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"] + == "what is the current time? First greet me." + ) + assert f"{SpanAttributes.LLM_PROMPTS}.1.role" in attributes + assert attributes[f"{SpanAttributes.LLM_PROMPTS}.1.role"] == "assistant" + assert f"{SpanAttributes.LLM_PROMPTS}.1.content" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.1.content"] + == "Hello! Let me check the current time for you." + ) + assert f"{SpanAttributes.LLM_PROMPTS}.1.tool_calls.0.id" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.1.tool_calls.0.id"] + == "call_qU7pH3EdQvzwkPyKPOdpgaKA" + ) + assert f"{SpanAttributes.LLM_PROMPTS}.1.tool_calls.0.name" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.1.tool_calls.0.name"] + == "get_current_time" + ) + assert f"{SpanAttributes.LLM_PROMPTS}.2.role" in attributes + assert attributes[f"{SpanAttributes.LLM_PROMPTS}.2.role"] == "tool" + assert f"{SpanAttributes.LLM_PROMPTS}.2.content" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.2.content"] == "2025-08-15 08:15:21" + ) + assert f"{SpanAttributes.LLM_PROMPTS}.2.tool_call_id" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.2.tool_call_id"] + == "call_qU7pH3EdQvzwkPyKPOdpgaKA" + ) + assert f"{SpanAttributes.LLM_PROMPTS}.3.role" in attributes + assert attributes[f"{SpanAttributes.LLM_PROMPTS}.3.role"] == "assistant" + assert f"{SpanAttributes.LLM_PROMPTS}.3.content" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.3.content"] + == "The current time is 2025-08-15 08:15:21" + ) + + +def test_assistant_message_with_only_tool_calls_no_content(): + """ + Test that when an assistant message has only tool_calls and no content, + the tool_calls are still included and no content attribute is set. + """ + mock_span = Mock() + mock_span.set_attribute = Mock() + mock_span_holder = Mock() + mock_span_holder.request_model = None + + messages = [ + [ + AIMessage( + content="", + tool_calls=[ + {"id": "call_123", "name": "some_tool", "args": {"param": "value"}} + ], + ) + ] + ] + + set_chat_request(mock_span, {}, messages, {}, mock_span_holder) + + call_args = [call[0] for call in mock_span.set_attribute.call_args_list] + attributes = {args[0]: args[1] for args in call_args} + + assert f"{SpanAttributes.LLM_PROMPTS}.0.role" in attributes + assert attributes[f"{SpanAttributes.LLM_PROMPTS}.0.role"] == "assistant" + assert f"{SpanAttributes.LLM_PROMPTS}.0.content" not in attributes + assert f"{SpanAttributes.LLM_PROMPTS}.0.tool_calls.0.id" in attributes + assert attributes[f"{SpanAttributes.LLM_PROMPTS}.0.tool_calls.0.id"] == "call_123" + assert f"{SpanAttributes.LLM_PROMPTS}.0.tool_calls.0.name" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.0.tool_calls.0.name"] == "some_tool" + ) + + +def test_assistant_message_with_only_content_no_tool_calls(): + """ + Test that when an assistant message has only content and no tool_calls, + the content is included and no tool_calls attributes are set. + """ + mock_span = Mock() + mock_span.set_attribute = Mock() + mock_span_holder = Mock() + mock_span_holder.request_model = None + + messages = [[AIMessage(content="Just a regular response with no tool calls")]] + + set_chat_request(mock_span, {}, messages, {}, mock_span_holder) + + call_args = [call[0] for call in mock_span.set_attribute.call_args_list] + + attributes = {args[0]: args[1] for args in call_args} + + assert f"{SpanAttributes.LLM_PROMPTS}.0.role" in attributes + assert attributes[f"{SpanAttributes.LLM_PROMPTS}.0.role"] == "assistant" + assert f"{SpanAttributes.LLM_PROMPTS}.0.content" in attributes + assert ( + attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"] + == "Just a regular response with no tool calls" + ) + + tool_call_attributes = [attr for attr in attributes.keys() if "tool_calls" in attr] + assert len(tool_call_attributes) == 0