From 36565da3b1f20141d330bec512592eabf739fd13 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 31 Oct 2025 13:05:13 -0400 Subject: [PATCH 1/4] fix tool call arguments parsing for v5 --- packages/dd-trace/src/llmobs/plugins/ai/index.js | 6 +++--- packages/dd-trace/test/llmobs/plugins/ai/index.spec.js | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/dd-trace/src/llmobs/plugins/ai/index.js b/packages/dd-trace/src/llmobs/plugins/ai/index.js index 0d4c9dc8091..f4f77dd8da9 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/index.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/index.js @@ -260,7 +260,7 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { const formattedToolCalls = [] for (const toolCall of outputMessageToolCalls) { - const toolCallArgs = getJsonStringValue(toolCall.args, {}) + const toolCallArgs = getJsonStringValue(toolCall.args) ?? getJsonStringValue(toolCall.input, {}) const toolDescription = toolsForModel?.find(tool => toolCall.toolName === tool.name)?.description const name = this.findToolName(toolDescription) this.#toolCallIdsToName[toolCall.toolCallId] = name @@ -269,7 +269,7 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { arguments: toolCallArgs, name, toolId: toolCall.toolCallId, - type: 'function' + type: toolCall.toolCallType ?? 'function' }) } @@ -320,7 +320,7 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { const name = this.findToolName(toolDescription) toolCalls.push({ - arguments: part.args, + arguments: part.args ?? part.input, name, toolId: part.toolCallId, type: 'function' diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index 318deca46d4..bf0a375090e 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -346,8 +346,7 @@ describe('Plugin', () => { }) }) - // TODO(sabrenner): Fix this test for v5.0.0 - tool "input" instead of "arguments" - it.skip('creates a span for a tool call', async () => { // eslint-disable-line mocha/no-pending-tests + it('creates a span for a tool call', async () => { let tools let additionalOptions = {} const toolSchema = ai.jsonSchema({ @@ -505,7 +504,7 @@ describe('Plugin', () => { }) // TODO(sabrenner): Fix this test for v5.0.0 - tool "input" instead of "arguments" & parsing, streaming - it.skip('created a span for a tool call from a stream', async () => { // eslint-disable-line mocha/no-pending-tests + it.skip('created a span for a tool call from a stream', async () => { let tools let additionalOptions = {} const toolSchema = ai.jsonSchema({ From e707b3807ce2a20e302369fda007316369d1b674 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 31 Oct 2025 14:44:52 -0400 Subject: [PATCH 2/4] fix tool call streaming and re-enable test --- packages/dd-trace/src/llmobs/plugins/ai/index.js | 3 ++- .../dd-trace/test/llmobs/plugins/ai/index.spec.js | 13 +++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/dd-trace/src/llmobs/plugins/ai/index.js b/packages/dd-trace/src/llmobs/plugins/ai/index.js index f4f77dd8da9..6c71e771c5b 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/index.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/index.js @@ -260,7 +260,8 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { const formattedToolCalls = [] for (const toolCall of outputMessageToolCalls) { - const toolCallArgs = getJsonStringValue(toolCall.args) ?? getJsonStringValue(toolCall.input, {}) + const toolArgs = toolCall.args ?? toolCall.input + const toolCallArgs = typeof toolArgs === 'string' ? getJsonStringValue(toolArgs, {}) : toolArgs const toolDescription = toolsForModel?.find(tool => toolCall.toolName === tool.name)?.description const name = this.findToolName(toolDescription) this.#toolCallIdsToName[toolCall.toolCallId] = name diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index bf0a375090e..f260d535a7b 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -503,8 +503,7 @@ describe('Plugin', () => { }) }) - // TODO(sabrenner): Fix this test for v5.0.0 - tool "input" instead of "arguments" & parsing, streaming - it.skip('created a span for a tool call from a stream', async () => { + it('created a span for a tool call from a stream', async () => { let tools let additionalOptions = {} const toolSchema = ai.jsonSchema({ @@ -628,15 +627,13 @@ describe('Plugin', () => { span: apmSpans[2], parentId: llmobsSpans[0].span_id, /** - * MOCK_STRING used as the stream implementation for ai does not finish the initial llm spans + * Before ai@4.0.2, the stream implementation did not finish the initial llm spans * first to associate the tool call id with the tool itself (by matching descriptions). * - * Usually, this would mean the tool call name is 'toolCall'. - * - * However, because we used mocked responses, the second time this test is called, the tool call - * will have the name 'weather' instead. We just assert that the name exists and is a string to simplify. + * Usually, this would mean the tool call name is 'toolCall'. This is a limitation with the older library + * versions. In v5+, this is resolved as the tool name is not its index in the tools array, but its actual name. */ - name: MOCK_STRING, + name: semifies(realVersion, '<4.0.2') ? 'toolCall' : MOCK_STRING, spanKind: 'tool', inputValue: JSON.stringify({ location: 'Tokyo' }), outputValue: JSON.stringify({ location: 'Tokyo', temperature: 72 }), From d635f8c5b91df014fb526f31bb42828186f574db Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 31 Oct 2025 15:08:50 -0400 Subject: [PATCH 3/4] update assertion --- packages/dd-trace/test/llmobs/plugins/ai/index.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index f260d535a7b..0a671fe61f0 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -633,7 +633,7 @@ describe('Plugin', () => { * Usually, this would mean the tool call name is 'toolCall'. This is a limitation with the older library * versions. In v5+, this is resolved as the tool name is not its index in the tools array, but its actual name. */ - name: semifies(realVersion, '<4.0.2') ? 'toolCall' : MOCK_STRING, + name: semifies(realVersion, '<4.0.2') ? 'toolCall' : 'weather', spanKind: 'tool', inputValue: JSON.stringify({ location: 'Tokyo' }), outputValue: JSON.stringify({ location: 'Tokyo', temperature: 72 }), From dba092015f993a5b92c980d4b0f6c227ec610144 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Mon, 3 Nov 2025 13:52:21 -0500 Subject: [PATCH 4/4] fix remaining tests --- packages/dd-trace/src/llmobs/plugins/ai/index.js | 8 +++++--- packages/dd-trace/test/llmobs/plugins/ai/index.spec.js | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/dd-trace/src/llmobs/plugins/ai/index.js b/packages/dd-trace/src/llmobs/plugins/ai/index.js index 6c71e771c5b..e732ce18ab9 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/index.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/index.js @@ -82,7 +82,9 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { * @param {string} toolDescription * @returns {string | undefined} */ - findToolName (toolDescription) { + findToolName (toolName, toolDescription) { + if (Number.isNaN(Number.parseInt(toolName))) return toolName + for (const availableTool of this.#availableTools) { const description = availableTool.description if (description === toolDescription && availableTool.id) { @@ -263,7 +265,7 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { const toolArgs = toolCall.args ?? toolCall.input const toolCallArgs = typeof toolArgs === 'string' ? getJsonStringValue(toolArgs, {}) : toolArgs const toolDescription = toolsForModel?.find(tool => toolCall.toolName === tool.name)?.description - const name = this.findToolName(toolDescription) + const name = this.findToolName(toolCall.toolName, toolDescription) this.#toolCallIdsToName[toolCall.toolCallId] = name formattedToolCalls.push({ @@ -318,7 +320,7 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { finalContent += part.text ?? part.data } else if (type === 'tool-call') { const toolDescription = toolsForModel?.find(tool => part.toolName === tool.name)?.description - const name = this.findToolName(toolDescription) + const name = this.findToolName(part.toolName, toolDescription) toolCalls.push({ arguments: part.args ?? part.input, diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index 0a671fe61f0..4bd6278d117 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -633,7 +633,7 @@ describe('Plugin', () => { * Usually, this would mean the tool call name is 'toolCall'. This is a limitation with the older library * versions. In v5+, this is resolved as the tool name is not its index in the tools array, but its actual name. */ - name: semifies(realVersion, '<4.0.2') ? 'toolCall' : 'weather', + name: semifies(realVersion, NODE_MAJOR < 22 ? '<=4.0.2' : '<4.0.2') ? 'toolCall' : 'weather', spanKind: 'tool', inputValue: JSON.stringify({ location: 'Tokyo' }), outputValue: JSON.stringify({ location: 'Tokyo', temperature: 72 }),