diff --git a/docs/design/gen-ai-arms-field-alignment.md b/docs/design/gen-ai-arms-field-alignment.md index 2094f6f149d..28085ec8f12 100644 --- a/docs/design/gen-ai-arms-field-alignment.md +++ b/docs/design/gen-ai-arms-field-alignment.md @@ -20,14 +20,15 @@ An upgrade to either baseline requires regenerating and reviewing this matrix. ## Field contract -| Span | Standard attributes emitted in this phase | Source and omission rule | -| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| LLM | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `gen_ai.request.model` | Written at span creation. Conversation ID is the existing session ID. | -| LLM response | `gen_ai.response.id`, `gen_ai.response.model`, `gen_ai.response.finish_reasons` | Provider response data only. Missing response model is omitted rather than replaced with the request model. All candidate finish reasons are ordered by candidate index. | -| LLM output | `gen_ai.output.type` | Gemini and Vertex AI only, and only when an explicit response MIME type or one unambiguous response modality is sent on the wire. | -| LLM usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.cache_creation.input_tokens` | Only provider-reported non-negative safe integers. Explicit zero is retained. When only a total is reported, input/output are omitted instead of estimated. | -| Tool | `gen_ai.operation.name=execute_tool`, `gen_ai.tool.name`, `gen_ai.tool.type=function`, `gen_ai.tool.call.id` | Tool call ID prefers the provider/model ID and falls back to Qwen Code's internal ID. | -| Agent | `gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`, `gen_ai.agent.description`, `gen_ai.conversation.id`, optional `gen_ai.request.model` | Description uses the existing 1024-UTF-16-code-unit truncation threshold and never splits surrogate pairs. Internal invocation IDs remain private. | +| Span | Standard attributes emitted in this phase | Source and omission rule | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| LLM | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `gen_ai.request.model` | Written at span creation. Conversation ID is the existing session ID. | +| LLM request | `gen_ai.request.choice.count`, `gen_ai.request.max_tokens`, `gen_ai.request.temperature`, `gen_ai.request.top_p`, `gen_ai.request.frequency_penalty`, `gen_ai.request.presence_penalty`, `gen_ai.request.stop_sequences` | Read from the first provider-final SDK request object. Invalid or unavailable values are omitted; no SDK or server defaults are inferred. | +| LLM response | `gen_ai.response.id`, `gen_ai.response.model`, `gen_ai.response.finish_reasons` | Provider response data only. Missing response model is omitted rather than replaced with the request model. All candidate finish reasons are ordered by candidate index. | +| LLM output | `gen_ai.output.type` | Gemini and Vertex AI only, and only when an explicit response MIME type or one unambiguous response modality is sent on the wire. | +| LLM usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.cache_creation.input_tokens` | Only provider-reported non-negative safe integers. Explicit zero is retained. When only a total is reported, input/output are omitted instead of estimated. | +| Tool | `gen_ai.operation.name=execute_tool`, `gen_ai.tool.name`, `gen_ai.tool.type=function`, `gen_ai.tool.call.id` | Tool call ID prefers the provider/model ID and falls back to Qwen Code's internal ID. | +| Agent | `gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`, `gen_ai.agent.description`, `gen_ai.conversation.id`, optional `gen_ai.request.model` | Description uses the existing 1024-UTF-16-code-unit truncation threshold and never splits surrogate pairs. Internal invocation IDs remain private. | Private attributes without an exact standard equivalent remain available for compatibility. Exact-equivalent private aliases and invalid GenAI aliases are @@ -65,6 +66,36 @@ from the model name. OpenAI-compatible, Anthropic, and Qwen OAuth requests use operation `chat`. Gemini and Vertex AI requests use `generate_content`. +## Request parameters + +Request attributes are collected after provider adapters have applied defaults, +overrides, unsupported-field removal, and output-window clamps, immediately +before calling the provider SDK. This is the final SDK request object visible +to Qwen Code, not the original logical configuration or the serialized HTTP +body. A logical LLM span records only its first such request snapshot. + +| Standard attribute | OpenAI-compatible and Qwen OAuth | Anthropic | Gemini and Vertex AI | +| ---------------------------------- | ---------------------------------------------------------- | ------------------ | ------------------------- | +| `gen_ai.request.choice.count` | `n` | Not applicable | `config.candidateCount` | +| `gen_ai.request.max_tokens` | `max_tokens`, `max_completion_tokens`, or `max_new_tokens` | `max_tokens` | `config.maxOutputTokens` | +| `gen_ai.request.temperature` | `temperature` | `temperature` | `config.temperature` | +| `gen_ai.request.top_p` | `top_p` | `top_p` | `config.topP` | +| `gen_ai.request.frequency_penalty` | `frequency_penalty` | Not currently sent | `config.frequencyPenalty` | +| `gen_ai.request.presence_penalty` | `presence_penalty` | Not currently sent | `config.presencePenalty` | +| `gen_ai.request.stop_sequences` | `stop` | `stop_sequences` | `config.stopSequences` | + +Finite numbers and safe integers are preserved exactly, including zero and +negative values on failed provider requests. Choice count is omitted when it is +one. Stop sequences must be a complete string array; OpenAI's single-string +form is normalized to a one-element array. Empty arrays are retained and mixed +arrays are omitted rather than filtered. Explicit adapter defaults are +recorded, while implicit SDK or server defaults are not inferred. + +When multiple OpenAI-compatible output-budget aliases are present, the standard +maximum is emitted only if all present values are valid safe integers and +equal. Conflicting values are omitted because compatible endpoints do not have +a common precedence rule. + ## Response and usage provenance Provider converters attach internal provenance to normalized Gemini usage @@ -104,8 +135,6 @@ Qwen Code does not inject that vendor-specific resource attribute or ## Deferred work -- Request sampling, choice, maximum-output, and stop fields need hooks after - provider adapters finalize the wire request. - `seed` and `top_k` have incompatible ARMS and GenAI types in the baselines. - Messages, instructions, tool definitions, arguments, and results require a standard JSON schema, privacy controls, and payload caps. diff --git a/docs/developers/development/telemetry.md b/docs/developers/development/telemetry.md index 1dc0f2906f9..3744d9d67b9 100644 --- a/docs/developers/development/telemetry.md +++ b/docs/developers/development/telemetry.md @@ -822,9 +822,10 @@ Distributed tracing spans form a tree rooted at `qwen-code.interaction`. Each in - **Attributes**: `session.id`, `qwen-code.prompt_id`, `qwen-code.message_type`, `qwen-code.model`, `qwen-code.approval_mode`, `interaction.sequence`, `interaction.duration_ms`, `qwen-code.turn_status` ("ok"/"error"/"cancelled") - `qwen-code.llm_request`: Wraps a single LLM API call. - - **GenAI attributes**: `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `gen_ai.request.model`, optional `gen_ai.output.type`, `gen_ai.response.id`, `gen_ai.response.model`, `gen_ai.response.finish_reasons`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.cache_creation.input_tokens` + - **GenAI attributes**: `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `gen_ai.request.model`, `gen_ai.request.choice.count`, `gen_ai.request.max_tokens`, `gen_ai.request.temperature`, `gen_ai.request.top_p`, `gen_ai.request.frequency_penalty`, `gen_ai.request.presence_penalty`, `gen_ai.request.stop_sequences`, optional `gen_ai.output.type`, `gen_ai.response.id`, `gen_ai.response.model`, `gen_ai.response.finish_reasons`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.cache_creation.input_tokens` - **Compatibility attributes**: `session.id`, `qwen-code.prompt_id`, `llm_request.context` ("subagent"/"interaction"/"standalone"), `duration_ms`, `ttft_ms`, `request_setup_ms`, `attempt`, `retry_total_delay_ms`, `sampling_ms`, `output_tokens_per_second`, `success`, `error`, `finish_reason`, `thoughts_token_count`, `subagent_name`, `error_type`, `error_status_code` - Standard response fields come from the provider response. Standard token fields are emitted only for provider-reported non-negative safe integers. If the provider reports only a total token count, input/output usage is omitted rather than estimated. + - Standard request-parameter fields come from the first provider-final SDK request object after adapter defaults, overrides, unsupported-field removal, and output-window clamps. Qwen Code does not infer SDK or server defaults. - `qwen-code.tool`: Wraps the full tool lifecycle (approval wait + execution). - **Attributes**: `session.id`, `gen_ai.operation.name` (`execute_tool`), `gen_ai.tool.name`, `gen_ai.tool.type` (`function`), `gen_ai.tool.call.id`, `tool.call_id`, `duration_ms`, `success`, `error` @@ -843,7 +844,7 @@ Distributed tracing spans form a tree rooted at `qwen-code.interaction`. Each in #### GenAI field migration and ARMS recognition -LLM spans now use `gen_ai.request.model`, `gen_ai.response.id`, and `gen_ai.usage.*` without the exact-equivalent private `qwen-code.model`, `response_id`, `input_tokens`, `output_tokens`, and `cached_input_tokens` aliases. Tool spans similarly use `gen_ai.tool.name` without `tool.name`; blocked-on-user and hook spans keep `tool.name` because they are not GenAI Tool spans. The invalid aliases `gen_ai.usage.cached_tokens`, `gen_ai.server.time_to_first_token`, and `gen_ai.usage.reasoning_tokens` are no longer emitted. Use `gen_ai.usage.cache_read.input_tokens` for provider-reported cache reads; continue using the private `ttft_ms` and `thoughts_token_count` fields where no GenAI/ARMS-common replacement exists. The full version-pinned contract and deferred fields are documented in [GenAI and ARMS field alignment](../../design/gen-ai-arms-field-alignment.md). +LLM spans now use standard `gen_ai.request.*`, `gen_ai.response.*`, and `gen_ai.usage.*` fields without exact-equivalent private aliases. Request sampling attributes are written only under their standard names; no bare `temperature`, `top_p`, `max_tokens`, penalty, choice-count, or stop-sequence aliases are emitted. Tool spans similarly use `gen_ai.tool.name` without `tool.name`; blocked-on-user and hook spans keep `tool.name` because they are not GenAI Tool spans. The invalid aliases `gen_ai.usage.cached_tokens`, `gen_ai.server.time_to_first_token`, and `gen_ai.usage.reasoning_tokens` are no longer emitted. Use `gen_ai.usage.cache_read.input_tokens` for provider-reported cache reads; continue using the private `ttft_ms` and `thoughts_token_count` fields where no GenAI/ARMS-common replacement exists. The full version-pinned contract and deferred fields are documented in [GenAI and ARMS field alignment](../../design/gen-ai-arms-field-alignment.md). To make ARMS recognize exported spans as a GenAI application, configure its resource feature explicitly: diff --git a/integration-tests/cli/gen-ai-telemetry.test.ts b/integration-tests/cli/gen-ai-telemetry.test.ts index 2de7d8cc7f4..95a45c2b207 100644 --- a/integration-tests/cli/gen-ai-telemetry.test.ts +++ b/integration-tests/cli/gen-ai-telemetry.test.ts @@ -106,7 +106,20 @@ describeLocal('GenAI telemetry fields', () => { rig.setup('gen-ai-telemetry', { settings: { security: { auth: { selectedType: 'openai' } }, - model: { name: 'request-model' }, + model: { + name: 'request-model', + generationConfig: { + samplingParams: { + n: 2, + max_tokens: 128, + temperature: 0, + top_p: 0.8, + frequency_penalty: -0.1, + presence_penalty: 0.2, + stop: ['END', 'DONE'], + }, + }, + }, ui: { enableFollowupSuggestions: false }, }, }); @@ -151,6 +164,13 @@ describeLocal('GenAI telemetry fields', () => { 'gen_ai.operation.name': 'chat', 'gen_ai.provider.name': 'openai', 'gen_ai.request.model': 'request-model', + 'gen_ai.request.choice.count': 2, + 'gen_ai.request.max_tokens': 128, + 'gen_ai.request.temperature': 0, + 'gen_ai.request.top_p': 0.8, + 'gen_ai.request.frequency_penalty': -0.1, + 'gen_ai.request.presence_penalty': 0.2, + 'gen_ai.request.stop_sequences': ['END', 'DONE'], 'gen_ai.response.model': 'provider-model-tool', 'gen_ai.response.finish_reasons': ['STOP'], 'gen_ai.usage.input_tokens': 20, @@ -160,6 +180,13 @@ describeLocal('GenAI telemetry fields', () => { expect(secondLlm).toMatchObject({ 'gen_ai.operation.name': 'chat', 'gen_ai.provider.name': 'openai', + 'gen_ai.request.choice.count': 2, + 'gen_ai.request.max_tokens': 128, + 'gen_ai.request.temperature': 0, + 'gen_ai.request.top_p': 0.8, + 'gen_ai.request.frequency_penalty': -0.1, + 'gen_ai.request.presence_penalty': 0.2, + 'gen_ai.request.stop_sequences': ['END', 'DONE'], 'gen_ai.response.model': 'provider-model-final', 'gen_ai.response.finish_reasons': ['STOP'], 'gen_ai.usage.input_tokens': 30, @@ -181,6 +208,26 @@ describeLocal('GenAI telemetry fields', () => { 'gen_ai.server.time_to_first_token', ); expect(attributes).not.toHaveProperty('gen_ai.usage.reasoning_tokens'); + expect(attributes).not.toHaveProperty('choice_count'); + expect(attributes).not.toHaveProperty('max_tokens'); + expect(attributes).not.toHaveProperty('temperature'); + expect(attributes).not.toHaveProperty('top_p'); + expect(attributes).not.toHaveProperty('frequency_penalty'); + expect(attributes).not.toHaveProperty('presence_penalty'); + expect(attributes).not.toHaveProperty('stop_sequences'); + } + + expect(server.requests).toHaveLength(2); + for (const { body } of server.requests) { + expect(body).toMatchObject({ + n: 2, + max_tokens: 128, + temperature: 0, + top_p: 0.8, + frequency_penalty: -0.1, + presence_penalty: 0.2, + stop: ['END', 'DONE'], + }); } const toolSpan = records.find( @@ -197,4 +244,70 @@ describeLocal('GenAI telemetry fields', () => { }); expect(toolSpan?.attributes).not.toHaveProperty('tool.name'); }); + + it('omits the default choice count from the exported span', async () => { + server = await startFakeOpenAIServer(() => ({ + model: 'provider-model', + content: 'Done.', + usage: { + prompt_tokens: 10, + completion_tokens: 2, + total_tokens: 12, + }, + })); + + rig = new TestRig(); + rig.setup('gen-ai-default-choice-count', { + settings: { + security: { auth: { selectedType: 'openai' } }, + model: { + name: 'request-model', + generationConfig: { + samplingParams: { n: 1 }, + }, + }, + ui: { enableFollowupSuggestions: false }, + }, + }); + + const restoreEnvironment = setEnvironment({ + HOME: rig.testDir!, + QWEN_HOME: join(rig.testDir!, '.qwen'), + OPENAI_API_KEY: 'fake-key', + OPENAI_BASE_URL: server.baseUrl, + OPENAI_MODEL: 'request-model', + QWEN_MODEL: 'request-model', + NO_PROXY: '127.0.0.1,localhost', + no_proxy: '127.0.0.1,localhost', + HTTP_PROXY: undefined, + HTTPS_PROXY: undefined, + ALL_PROXY: undefined, + http_proxy: undefined, + https_proxy: undefined, + all_proxy: undefined, + DASHSCOPE_PROXY_BASE_URL: undefined, + }); + + try { + await rig.run('Reply with done.', '--output-format', 'json'); + } finally { + restoreEnvironment(); + } + + expect(server.requests.length).toBeGreaterThan(0); + for (const { body } of server.requests) { + expect(body).toMatchObject({ n: 1 }); + } + + const records = parseTelemetry(rig.readFile('telemetry.log')); + const llmSpans = records.filter( + (record) => record.name === 'qwen-code.llm_request', + ); + expect(llmSpans).toHaveLength(server.requests.length); + for (const llmSpan of llmSpans) { + expect(llmSpan.attributes).not.toHaveProperty( + 'gen_ai.request.choice.count', + ); + } + }); }); diff --git a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts index 4f7a7e2e8bf..c4820146c03 100644 --- a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts +++ b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts @@ -22,10 +22,14 @@ const mockTokenizer = { calculateTokens: vi.fn(), dispose: vi.fn(), }; +const mockReportAnthropicRequest = vi.hoisted(() => vi.fn()); vi.mock('../../utils/request-tokenizer/index.js', () => ({ RequestTokenEstimator: vi.fn(() => mockTokenizer), })); +vi.mock('../../telemetry/gen-ai-request.js', () => ({ + reportAnthropicRequest: mockReportAnthropicRequest, +})); type AnthropicCreateArgs = [ unknown, @@ -1203,7 +1207,9 @@ describe('AnthropicContentGenerator', () => { // fallback (which would double latency + API cost). expect(anthropicState.createImpl).toHaveBeenCalledTimes(1); - const [, options] = anthropicState.lastCreateArgs as AnthropicCreateArgs; + const [streamingRequest, options] = + anthropicState.lastCreateArgs as AnthropicCreateArgs; + expect(mockReportAnthropicRequest).toHaveBeenCalledWith(streamingRequest); const headers = ((options as { headers?: Record }) ?.headers || {}) as Record; expect(headers['anthropic-beta']).toContain( @@ -1413,6 +1419,7 @@ describe('AnthropicContentGenerator', () => { output_config: { effort: 'high' }, }), ); + expect(mockReportAnthropicRequest).toHaveBeenCalledWith(anthropicRequest); expect(convertResponseSpy).toHaveBeenCalledTimes(1); }); @@ -3840,6 +3847,10 @@ describe('AnthropicContentGenerator', () => { expect.objectContaining({ stream: true }), ); expect(fallbackRequest).not.toHaveProperty('stream'); + expect(mockReportAnthropicRequest).toHaveBeenNthCalledWith( + 2, + fallbackRequest, + ); }); it('converts the non-streaming fallback response when an empty stream is recoverable', async () => { diff --git a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts index aaf123604e7..87ee08d63f8 100644 --- a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts +++ b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts @@ -49,6 +49,7 @@ import { parsePositiveIntegerEnvValue, } from '../tokenLimits.js'; import { setToolCallPreparations } from '../tool-call-preparation.js'; +import { reportAnthropicRequest } from '../../telemetry/gen-ai-request.js'; const debugLogger = createDebugLogger('ANTHROPIC'); @@ -347,6 +348,7 @@ export class AnthropicContentGenerator implements ContentGenerator { try { const anthropicRequest = await this.buildRequest(request); runtimeDiagnostics.recordAnthropicWireRequest(anthropicRequest); + reportAnthropicRequest(anthropicRequest); const headers = this.buildPerRequestHeaders(anthropicRequest); response = (await this.client.messages.create(anthropicRequest, { signal: perRequestAc?.signal, @@ -373,6 +375,7 @@ export class AnthropicContentGenerator implements ContentGenerator { stream: true, }; runtimeDiagnostics.recordAnthropicWireRequest(streamingRequest); + reportAnthropicRequest(streamingRequest); // Wrap the caller's signal in a per-request child so the Anthropic SDK's // leaked abort listener (core.mjs fetchWithTimeout registers one with no @@ -1394,6 +1397,7 @@ export class AnthropicContentGenerator implements ContentGenerator { let response: Message; try { runtimeDiagnostics.recordAnthropicWireRequest(fallbackRequest); + reportAnthropicRequest(fallbackRequest); response = (await this.client.messages.create(fallbackRequest, { signal: abortSignal, ...(headers ? { headers } : {}), diff --git a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts index 0f389111790..9d2edda3759 100644 --- a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts +++ b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts @@ -8,6 +8,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { GeminiContentGenerator } from './geminiContentGenerator.js'; import { GoogleGenAI } from '@google/genai'; +const mockReportGeminiRequest = vi.hoisted(() => vi.fn()); + vi.mock('@google/genai', () => { const mockGenerateContent = vi.fn(); const mockGenerateContentStream = vi.fn(); @@ -25,6 +27,9 @@ vi.mock('@google/genai', () => { })), }; }); +vi.mock('../../telemetry/gen-ai-request.js', () => ({ + reportGeminiRequest: mockReportGeminiRequest, +})); describe('GeminiContentGenerator', () => { let generator: GeminiContentGenerator; @@ -94,6 +99,9 @@ describe('GeminiContentGenerator', () => { }), }), ); + expect(mockReportGeminiRequest).toHaveBeenCalledWith( + mockGoogleGenAI.models.generateContent.mock.calls[0][0], + ); expect(response).toBe(expectedResponse); }); @@ -145,6 +153,9 @@ describe('GeminiContentGenerator', () => { }), }), ); + expect(mockReportGeminiRequest).toHaveBeenCalledWith( + mockGoogleGenAI.models.generateContentStream.mock.calls[0][0], + ); expect(stream).toBe(mockStream); }); diff --git a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts index 38d1df1467d..af010b14cca 100644 --- a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts +++ b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts @@ -22,6 +22,7 @@ import type { ContentGeneratorConfig, } from '../contentGenerator.js'; import { createDebugLogger } from '../../utils/debugLogger.js'; +import { reportGeminiRequest } from '../../telemetry/gen-ai-request.js'; const debugLogger = createDebugLogger('GEMINI'); @@ -193,6 +194,7 @@ export class GeminiContentGenerator implements ContentGenerator { contents: this.stripUnsupportedFields(request.contents), config: this.buildGenerateContentConfig(request), }; + reportGeminiRequest(finalRequest); return this.googleGenAI.models.generateContent(finalRequest); } @@ -205,6 +207,7 @@ export class GeminiContentGenerator implements ContentGenerator { contents: this.stripUnsupportedFields(request.contents), config: this.buildGenerateContentConfig(request), }; + reportGeminiRequest(finalRequest); return this.googleGenAI.models.generateContentStream(finalRequest); } diff --git a/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts b/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts index f0debb479c2..e83bbae8b9b 100644 --- a/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts +++ b/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts @@ -73,6 +73,7 @@ import { resolveGenAiProviderName, } from '../../telemetry/gen-ai-provider.js'; import { getGenAiUsageProvenance } from '../../telemetry/gen-ai-usage.js'; +import { createGenAiRequestObserverContext } from '../../telemetry/gen-ai-request.js'; /** * Phase 4b — read the active retry context once, default attempt to 1 when @@ -310,7 +311,10 @@ export class LoggingContentGenerator implements ContentGenerator { // Capture span context so the API call and logging activate it via // context.with(). Without this, nested OTel spans (HTTP instrumentation, // log-bridge spans) parent to session root instead of llm_request. - const spanContext = trace.setSpan(context.active(), llmSpan); + const spanContext = createGenAiRequestObserverContext( + trace.setSpan(context.active(), llmSpan), + llmSpan, + ); const startTime = Date.now(); const isInternal = isInternalPromptId(userPromptId); @@ -454,7 +458,10 @@ export class LoggingContentGenerator implements ContentGenerator { // Capture the span context so the stream wrapper can activate it // during iteration — not just during generator creation. - const spanContext = trace.setSpan(context.active(), llmSpan); + const spanContext = createGenAiRequestObserverContext( + trace.setSpan(context.active(), llmSpan), + llmSpan, + ); const startTime = Date.now(); const isInternal = isInternalPromptId(userPromptId); diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts index 2634746a002..2b2514cec1c 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts @@ -41,6 +41,8 @@ import { import { setToolCallPreparations } from '../tool-call-preparation.js'; // Mock dependencies +const mockReportOpenAiRequest = vi.hoisted(() => vi.fn()); + vi.mock('./converter.js', () => ({ OpenAIContentConverter: { convertGeminiRequestToOpenAI: vi.fn(), @@ -53,6 +55,9 @@ vi.mock('openai'); vi.mock('../../telemetry/loggers.js', () => ({ logProtocolTagSanitized: vi.fn(), })); +vi.mock('../../telemetry/gen-ai-request.js', () => ({ + reportOpenAiRequest: mockReportOpenAiRequest, +})); describe('ContentGenerationPipeline', () => { let pipeline: ContentGenerationPipeline; @@ -183,6 +188,9 @@ describe('ContentGenerationPipeline', () => { signal: undefined, }), ); + expect(mockReportOpenAiRequest).toHaveBeenCalledWith( + vi.mocked(mockClient.chat.completions.create).mock.calls[0]![0], + ); expect(mockConverter.convertOpenAIResponseToGemini).toHaveBeenCalledWith( mockOpenAIResponse, expect.objectContaining({ @@ -2078,6 +2086,8 @@ describe('ContentGenerationPipeline', () => { expect(calls).toHaveLength(2); expect(calls[0][0].enable_thinking).toBe(false); expect(calls[1][0].enable_thinking).toBe(true); + expect(mockReportOpenAiRequest).toHaveBeenNthCalledWith(1, calls[0][0]); + expect(mockReportOpenAiRequest).toHaveBeenNthCalledWith(2, calls[1][0]); expect(mockErrorHandler.handle).not.toHaveBeenCalled(); }); @@ -2165,6 +2175,9 @@ describe('ContentGenerationPipeline', () => { signal: expect.any(AbortSignal), }), ); + expect(mockReportOpenAiRequest).toHaveBeenCalledWith( + vi.mocked(mockClient.chat.completions.create).mock.calls[0]![0], + ); }); it('should filter empty responses', async () => { diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.ts b/packages/core/src/core/openaiContentGenerator/pipeline.ts index ce15a643941..01eba84deb8 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.ts @@ -33,6 +33,7 @@ import { logProtocolTagSanitized } from '../../telemetry/loggers.js'; import { ProtocolTagSanitizedEvent } from '../../telemetry/types.js'; import { getErrorMessage, getErrorStatus } from '../../utils/errors.js'; import { getRateLimitErrorDetails } from '../../utils/rateLimit.js'; +import { reportOpenAiRequest } from '../../telemetry/gen-ai-request.js'; const debugLogger = createDebugLogger('OPENAI_PIPELINE'); @@ -1119,6 +1120,7 @@ export class ContentGenerationPipeline { // so the logger sees the exact bytes sent on the wire. openaiRequestCaptureContext.getStore()?.(openaiRequest); runtimeDiagnostics.recordOpenAIWireRequest(openaiRequest); + reportOpenAiRequest(openaiRequest); return executor(openaiRequest, context); }; diff --git a/packages/core/src/telemetry/gen-ai-request.test.ts b/packages/core/src/telemetry/gen-ai-request.test.ts new file mode 100644 index 00000000000..afdcc7f66f6 --- /dev/null +++ b/packages/core/src/telemetry/gen-ai-request.test.ts @@ -0,0 +1,281 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { ROOT_CONTEXT, type Attributes, type Span } from '@opentelemetry/api'; +import { + createGenAiRequestObserverContext, + extractAnthropicRequestAttributes, + extractGeminiRequestAttributes, + extractOpenAiRequestAttributes, + reportOpenAiRequest, +} from './gen-ai-request.js'; + +function createSpan(options: { recording?: boolean; throws?: boolean } = {}): { + span: Span; + attributes: Attributes; +} { + const attributes: Attributes = {}; + const span = { + isRecording: () => options.recording ?? true, + setAttributes: (values: Attributes) => { + if (options.throws) throw new Error('setAttributes failed'); + Object.assign(attributes, values); + return span; + }, + } as unknown as Span; + return { span, attributes }; +} + +describe('GenAI request attribute extraction', () => { + it('extracts all OpenAI-compatible fields', () => { + expect( + extractOpenAiRequestAttributes({ + n: 3, + max_completion_tokens: 512, + temperature: 0, + top_p: 0.9, + frequency_penalty: -0.25, + presence_penalty: 0.5, + stop: 'done', + }), + ).toEqual({ + 'gen_ai.request.choice.count': 3, + 'gen_ai.request.max_tokens': 512, + 'gen_ai.request.temperature': 0, + 'gen_ai.request.top_p': 0.9, + 'gen_ai.request.frequency_penalty': -0.25, + 'gen_ai.request.presence_penalty': 0.5, + 'gen_ai.request.stop_sequences': ['done'], + }); + }); + + it.each([ + ['max_tokens', { max_tokens: 11 }], + ['max_completion_tokens', { max_completion_tokens: 11 }], + ['max_new_tokens', { max_new_tokens: 11 }], + [ + 'matching aliases', + { max_tokens: 11, max_completion_tokens: 11, max_new_tokens: 11 }, + ], + ])('maps the %s output budget', (_name, request) => { + expect(extractOpenAiRequestAttributes(request)).toEqual({ + 'gen_ai.request.max_tokens': 11, + }); + }); + + it.each([ + { max_tokens: 10, max_completion_tokens: 11 }, + { max_tokens: 10, max_completion_tokens: 10.5 }, + { max_tokens: 10, max_new_tokens: Number.NaN }, + { max_tokens: Number.MAX_SAFE_INTEGER + 1 }, + ])('omits ambiguous or invalid output budgets', (request) => { + expect(extractOpenAiRequestAttributes(request)).toEqual({}); + }); + + it('omits choice count one but preserves other safe integers', () => { + expect(extractOpenAiRequestAttributes({ n: 1 })).toEqual({}); + expect(extractOpenAiRequestAttributes({ n: 0 })).toEqual({ + 'gen_ai.request.choice.count': 0, + }); + expect(extractOpenAiRequestAttributes({ n: -1 })).toEqual({ + 'gen_ai.request.choice.count': -1, + }); + }); + + it('omits invalid numbers and nullish values', () => { + expect( + extractOpenAiRequestAttributes({ + temperature: Number.NaN, + top_p: Number.POSITIVE_INFINITY, + frequency_penalty: '0.1', + presence_penalty: null, + max_tokens: undefined, + }), + ).toEqual({}); + }); + + it('requires own properties', () => { + const request = Object.create({ + n: 2, + max_tokens: 50, + temperature: 0.2, + stop: ['inherited'], + }) as Record; + request['top_p'] = 0.8; + expect(extractOpenAiRequestAttributes(request)).toEqual({ + 'gen_ai.request.top_p': 0.8, + }); + }); + + it('preserves and copies valid stop sequences', () => { + const stop = ['one', 'two']; + const attributes = extractOpenAiRequestAttributes({ stop }); + expect(attributes).toEqual({ + 'gen_ai.request.stop_sequences': ['one', 'two'], + }); + expect(attributes['gen_ai.request.stop_sequences']).not.toBe(stop); + expect(extractOpenAiRequestAttributes({ stop: [] })).toEqual({ + 'gen_ai.request.stop_sequences': [], + }); + expect(extractOpenAiRequestAttributes({ stop: ['one', 2] })).toEqual({}); + }); + + it('extracts Anthropic fields without unsupported penalties', () => { + expect( + extractAnthropicRequestAttributes({ + max_tokens: 0, + temperature: -0.1, + top_p: 1, + stop_sequences: ['done'], + frequency_penalty: 0.2, + presence_penalty: 0.3, + }), + ).toEqual({ + 'gen_ai.request.max_tokens': 0, + 'gen_ai.request.temperature': -0.1, + 'gen_ai.request.top_p': 1, + 'gen_ai.request.stop_sequences': ['done'], + }); + }); + + it('extracts fields from the final Gemini config', () => { + expect( + extractGeminiRequestAttributes({ + candidateCount: 99, + config: { + candidateCount: 2, + maxOutputTokens: -1, + temperature: 1, + topP: 0.95, + frequencyPenalty: 0, + presencePenalty: -0.2, + stopSequences: [], + }, + }), + ).toEqual({ + 'gen_ai.request.choice.count': 2, + 'gen_ai.request.max_tokens': -1, + 'gen_ai.request.temperature': 1, + 'gen_ai.request.top_p': 0.95, + 'gen_ai.request.frequency_penalty': 0, + 'gen_ai.request.presence_penalty': -0.2, + 'gen_ai.request.stop_sequences': [], + }); + }); + + it('omits invalid Gemini config shapes', () => { + expect(extractGeminiRequestAttributes({ config: null })).toEqual({}); + expect(extractGeminiRequestAttributes({ config: 'invalid' })).toEqual({}); + }); +}); + +describe('GenAI request observer', () => { + it('records only the first request snapshot', () => { + const { span, attributes } = createSpan(); + const requestContext = createGenAiRequestObserverContext( + ROOT_CONTEXT, + span, + ); + + reportOpenAiRequest({}, requestContext); + reportOpenAiRequest({ temperature: 0.5 }, requestContext); + + expect(attributes).toEqual({}); + }); + + it('consumes the first snapshot when extraction fails', () => { + const { span, attributes } = createSpan(); + const requestContext = createGenAiRequestObserverContext( + ROOT_CONTEXT, + span, + ); + const brokenRequest = new Proxy( + {}, + { + getOwnPropertyDescriptor: () => { + throw new Error('property inspection failed'); + }, + }, + ); + + reportOpenAiRequest(brokenRequest, requestContext); + reportOpenAiRequest({ temperature: 0.5 }, requestContext); + + expect(attributes).toEqual({}); + }); + + it('keeps concurrent observer contexts isolated', () => { + const first = createSpan(); + const second = createSpan(); + const firstContext = createGenAiRequestObserverContext( + ROOT_CONTEXT, + first.span, + ); + const secondContext = createGenAiRequestObserverContext( + ROOT_CONTEXT, + second.span, + ); + + reportOpenAiRequest({ temperature: 0.1 }, firstContext); + reportOpenAiRequest({ temperature: 0.9 }, secondContext); + + expect(first.attributes).toEqual({ + 'gen_ai.request.temperature': 0.1, + }); + expect(second.attributes).toEqual({ + 'gen_ai.request.temperature': 0.9, + }); + }); + + it('does not install an observer for a non-recording span', () => { + const { span, attributes } = createSpan({ recording: false }); + const requestContext = createGenAiRequestObserverContext( + ROOT_CONTEXT, + span, + ); + reportOpenAiRequest({ temperature: 0.5 }, requestContext); + expect(attributes).toEqual({}); + }); + + it('does not let context failures escape', () => { + const { span } = createSpan(); + const brokenParent = { + setValue: () => { + throw new Error('setValue failed'); + }, + } as unknown as typeof ROOT_CONTEXT; + expect(createGenAiRequestObserverContext(brokenParent, span)).toBe( + brokenParent, + ); + + const brokenReporterContext = { + getValue: () => { + throw new Error('getValue failed'); + }, + } as unknown as typeof ROOT_CONTEXT; + expect(() => + reportOpenAiRequest({ temperature: 0.5 }, brokenReporterContext), + ).not.toThrow(); + }); + + it('does not let span failures escape', () => { + const { span } = createSpan({ throws: true }); + const requestContext = createGenAiRequestObserverContext( + ROOT_CONTEXT, + span, + ); + expect(() => + reportOpenAiRequest({ temperature: 0.5 }, requestContext), + ).not.toThrow(); + }); + + it('does nothing without an observer', () => { + expect(() => + reportOpenAiRequest({ temperature: 0.5 }, ROOT_CONTEXT), + ).not.toThrow(); + }); +}); diff --git a/packages/core/src/telemetry/gen-ai-request.ts b/packages/core/src/telemetry/gen-ai-request.ts new file mode 100644 index 00000000000..c8ccb4a76d6 --- /dev/null +++ b/packages/core/src/telemetry/gen-ai-request.ts @@ -0,0 +1,268 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + context, + createContextKey, + type Attributes, + type Context, + type Span, +} from '@opentelemetry/api'; + +type RequestObserver = (attributes: Attributes) => void; +type RequestRecord = Record; + +const requestObserverKey = createContextKey( + 'qwen-code.gen-ai-request-observer', +); + +function ownValue(record: RequestRecord, key: string): unknown | undefined { + return Object.hasOwn(record, key) ? record[key] : undefined; +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) + ? value + : undefined; +} + +function safeInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isSafeInteger(value) + ? value + : undefined; +} + +function stopSequences( + value: unknown, + allowSingleString: boolean, +): string[] | undefined { + if (allowSingleString && typeof value === 'string') return [value]; + if (!Array.isArray(value) || !value.every((item) => typeof item === 'string')) + return undefined; + return [...value]; +} + +function outputBudget(record: RequestRecord): number | undefined { + const values = ['max_tokens', 'max_completion_tokens', 'max_new_tokens'] + .map((key) => ownValue(record, key)) + .filter((value) => value !== undefined && value !== null); + if (values.length === 0) return undefined; + + const integers = values.map(safeInteger); + if (integers.some((value) => value === undefined)) return undefined; + const first = integers[0]!; + return integers.every((value) => value === first) ? first : undefined; +} + +function assignNumber( + attributes: Attributes, + key: string, + value: unknown, +): void { + const number = finiteNumber(value); + if (number !== undefined) attributes[key] = number; +} + +function assignInteger( + attributes: Attributes, + key: string, + value: unknown, +): void { + const integer = safeInteger(value); + if (integer !== undefined) attributes[key] = integer; +} + +function assignStopSequences( + attributes: Attributes, + value: unknown, + allowSingleString: boolean, +): void { + const sequences = stopSequences(value, allowSingleString); + if (sequences !== undefined) + attributes['gen_ai.request.stop_sequences'] = sequences; +} + +export function extractOpenAiRequestAttributes(request: object): Attributes { + const record = request as RequestRecord; + const attributes: Attributes = {}; + const choiceCount = safeInteger(ownValue(record, 'n')); + if (choiceCount !== undefined && choiceCount !== 1) { + attributes['gen_ai.request.choice.count'] = choiceCount; + } + const maxTokens = outputBudget(record); + if (maxTokens !== undefined) { + attributes['gen_ai.request.max_tokens'] = maxTokens; + } + assignNumber( + attributes, + 'gen_ai.request.temperature', + ownValue(record, 'temperature'), + ); + assignNumber(attributes, 'gen_ai.request.top_p', ownValue(record, 'top_p')); + assignNumber( + attributes, + 'gen_ai.request.frequency_penalty', + ownValue(record, 'frequency_penalty'), + ); + assignNumber( + attributes, + 'gen_ai.request.presence_penalty', + ownValue(record, 'presence_penalty'), + ); + assignStopSequences(attributes, ownValue(record, 'stop'), true); + return attributes; +} + +export function extractAnthropicRequestAttributes(request: object): Attributes { + const record = request as RequestRecord; + const attributes: Attributes = {}; + assignInteger( + attributes, + 'gen_ai.request.max_tokens', + ownValue(record, 'max_tokens'), + ); + assignNumber( + attributes, + 'gen_ai.request.temperature', + ownValue(record, 'temperature'), + ); + assignNumber(attributes, 'gen_ai.request.top_p', ownValue(record, 'top_p')); + assignStopSequences(attributes, ownValue(record, 'stop_sequences'), false); + return attributes; +} + +export function extractGeminiRequestAttributes(request: object): Attributes { + const record = request as RequestRecord; + const config = ownValue(record, 'config'); + if (typeof config !== 'object' || config === null) return {}; + const configRecord = config as RequestRecord; + const attributes: Attributes = {}; + const choiceCount = safeInteger(ownValue(configRecord, 'candidateCount')); + if (choiceCount !== undefined && choiceCount !== 1) { + attributes['gen_ai.request.choice.count'] = choiceCount; + } + assignInteger( + attributes, + 'gen_ai.request.max_tokens', + ownValue(configRecord, 'maxOutputTokens'), + ); + assignNumber( + attributes, + 'gen_ai.request.temperature', + ownValue(configRecord, 'temperature'), + ); + assignNumber( + attributes, + 'gen_ai.request.top_p', + ownValue(configRecord, 'topP'), + ); + assignNumber( + attributes, + 'gen_ai.request.frequency_penalty', + ownValue(configRecord, 'frequencyPenalty'), + ); + assignNumber( + attributes, + 'gen_ai.request.presence_penalty', + ownValue(configRecord, 'presencePenalty'), + ); + assignStopSequences( + attributes, + ownValue(configRecord, 'stopSequences'), + false, + ); + return attributes; +} + +export function createGenAiRequestObserverContext( + parent: Context, + span: Span, +): Context { + try { + if (!span.isRecording()) return parent; + } catch { + return parent; + } + + let consumed = false; + const observer: RequestObserver = (attributes) => { + if (consumed) return; + consumed = true; + try { + span.setAttributes(attributes); + } catch { + // Telemetry must not affect the provider request. + } + }; + try { + return parent.setValue(requestObserverKey, observer); + } catch { + return parent; + } +} + +function reportRequest( + request: object, + extract: (request: object) => Attributes, + requestContext: Context, +): void { + let observer: unknown; + try { + observer = requestContext.getValue(requestObserverKey); + } catch { + return; + } + if (typeof observer !== 'function') return; + + let attributes: Attributes = {}; + try { + attributes = extract(request); + } catch { + // Consume the first request snapshot even if extraction fails. + } + try { + (observer as RequestObserver)(attributes); + } catch { + // Telemetry must not affect the provider request. + } +} + +function reportActiveRequest( + request: object, + extract: (request: object) => Attributes, + requestContext?: Context, +): void { + try { + reportRequest(request, extract, requestContext ?? context.active()); + } catch { + // Telemetry must not affect the provider request. + } +} + +export function reportOpenAiRequest( + request: object, + requestContext?: Context, +): void { + reportActiveRequest(request, extractOpenAiRequestAttributes, requestContext); +} + +export function reportAnthropicRequest( + request: object, + requestContext?: Context, +): void { + reportActiveRequest( + request, + extractAnthropicRequestAttributes, + requestContext, + ); +} + +export function reportGeminiRequest( + request: object, + requestContext?: Context, +): void { + reportActiveRequest(request, extractGeminiRequestAttributes, requestContext); +}