diff --git a/docs/design/gen-ai-arms-field-alignment.md b/docs/design/gen-ai-arms-field-alignment.md index 28085ec8f12..ac4933ba96a 100644 --- a/docs/design/gen-ai-arms-field-alignment.md +++ b/docs/design/gen-ai-arms-field-alignment.md @@ -5,7 +5,7 @@ This design aligns the first set of Qwen Code span attributes whose names, types, and meanings agree between OpenTelemetry GenAI semantic conventions and Alibaba Cloud ARMS LLM Trace. It does not change span names, span kinds, -parenting, retry topology, or sensitive payload collection. +parenting, or retry topology. The OpenTelemetry GenAI convention is still Development status. This change is pinned to commit @@ -20,31 +20,43 @@ An upgrade to either baseline requires regenerating and reviewing this matrix. ## Field contract -| Span | Standard attributes emitted in this phase | Source and omission rule | -| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| LLM | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `gen_ai.request.model` | Written at span creation. Conversation ID is the existing session ID. | -| LLM request | `gen_ai.request.choice.count`, `gen_ai.request.max_tokens`, `gen_ai.request.temperature`, `gen_ai.request.top_p`, `gen_ai.request.frequency_penalty`, `gen_ai.request.presence_penalty`, `gen_ai.request.stop_sequences` | Read from the first provider-final SDK request object. Invalid or unavailable values are omitted; no SDK or server defaults are inferred. | -| LLM response | `gen_ai.response.id`, `gen_ai.response.model`, `gen_ai.response.finish_reasons` | Provider response data only. Missing response model is omitted rather than replaced with the request model. All candidate finish reasons are ordered by candidate index. | -| LLM output | `gen_ai.output.type` | Gemini and Vertex AI only, and only when an explicit response MIME type or one unambiguous response modality is sent on the wire. | -| LLM usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.cache_creation.input_tokens` | Only provider-reported non-negative safe integers. Explicit zero is retained. When only a total is reported, input/output are omitted instead of estimated. | -| Tool | `gen_ai.operation.name=execute_tool`, `gen_ai.tool.name`, `gen_ai.tool.type=function`, `gen_ai.tool.call.id` | Tool call ID prefers the provider/model ID and falls back to Qwen Code's internal ID. | -| Agent | `gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`, `gen_ai.agent.description`, `gen_ai.conversation.id`, optional `gen_ai.request.model` | Description uses the existing 1024-UTF-16-code-unit truncation threshold and never splits surrogate pairs. Internal invocation IDs remain private. | +| Span | Standard attributes emitted in this phase | Source and omission rule | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| LLM | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `gen_ai.request.model` | Written at span creation. Conversation ID is the existing session ID. | +| LLM request | `gen_ai.request.choice.count`, `gen_ai.request.max_tokens`, `gen_ai.request.temperature`, `gen_ai.request.top_p`, `gen_ai.request.frequency_penalty`, `gen_ai.request.presence_penalty`, `gen_ai.request.stop_sequences` | Read from the first provider-final SDK request object. Invalid or unavailable values are omitted; no SDK or server defaults are inferred. | +| LLM input | `gen_ai.input.messages`, `gen_ai.system_instructions`, `gen_ai.tool.definitions` | Sensitive compact JSON from the same first provider-final request. Each complete value is independently omitted if invalid or oversized. | +| LLM response | `gen_ai.response.id`, `gen_ai.response.model`, `gen_ai.response.finish_reasons` | Provider response data only. Missing response model is omitted rather than replaced with the request model. All candidate finish reasons are ordered by candidate index. | +| LLM output | `gen_ai.output.type`, `gen_ai.output.messages` | Output type is emitted for supported Gemini/Vertex request settings. Sensitive output messages come from the final physical request attempt and preserve every candidate. | +| LLM usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read.input_tokens`, `gen_ai.usage.cache_creation.input_tokens` | Only provider-reported non-negative safe integers. Explicit zero is retained. When only a total is reported, input/output are omitted instead of estimated. | +| Tool | `gen_ai.operation.name=execute_tool`, `gen_ai.tool.name`, `gen_ai.tool.description`, `gen_ai.tool.type=function`, `gen_ai.tool.call.id`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result` | Description is non-sensitive static registry metadata. Sensitive arguments reflect the executed invocation; result is emitted only for a successful tool call. | +| Agent | `gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`, `gen_ai.agent.description`, `gen_ai.conversation.id`, optional `gen_ai.request.model` | Description uses the existing 1024-UTF-16-code-unit truncation threshold and never splits surrogate pairs. Internal invocation IDs remain private. | Private attributes without an exact standard equivalent remain available for compatibility. Exact-equivalent private aliases and invalid GenAI aliases are removed without a dual-write period: -| Removed attribute | Replacement | -| ----------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| LLM `qwen-code.model` | `gen_ai.request.model`; interaction spans continue using `qwen-code.model` because they are not GenAI inference spans | -| LLM `response_id` | `gen_ai.response.id`; API response/error logs retain their existing `response_id` schema | -| LLM `input_tokens` | `gen_ai.usage.input_tokens` when the provider reports an input breakdown | -| LLM `output_tokens` | `gen_ai.usage.output_tokens` when the provider reports an output breakdown | -| LLM `cached_input_tokens` | `gen_ai.usage.cache_read.input_tokens` when the provider reports cache reads | -| `qwen-code.tool` Span `tool.name` | `gen_ai.tool.name`; blocked-on-user and hook spans continue using `tool.name` | -| `gen_ai.usage.cached_tokens` | `gen_ai.usage.cache_read.input_tokens` when the provider reports cache reads | -| `gen_ai.server.time_to_first_token` | No common attribute; continue querying private `ttft_ms` | -| `gen_ai.usage.reasoning_tokens` | No ARMS/GenAI common attribute in this baseline; continue querying private `thoughts_token_count` | +| Removed attribute | Replacement | +| ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | +| LLM `qwen-code.model` | `gen_ai.request.model`; interaction spans continue using `qwen-code.model` because they are not GenAI inference spans | +| LLM `response_id` | `gen_ai.response.id`; API response/error logs retain their existing `response_id` schema | +| LLM `input_tokens` | `gen_ai.usage.input_tokens` when the provider reports an input breakdown | +| LLM `output_tokens` | `gen_ai.usage.output_tokens` when the provider reports an output breakdown | +| LLM `cached_input_tokens` | `gen_ai.usage.cache_read.input_tokens` when the provider reports cache reads | +| `qwen-code.tool` Span `tool.name` | `gen_ai.tool.name`; blocked-on-user and hook spans continue using `tool.name` | +| `gen_ai.usage.cached_tokens` | `gen_ai.usage.cache_read.input_tokens` when the provider reports cache reads | +| `gen_ai.server.time_to_first_token` | No common attribute; continue querying private `ttft_ms` | +| `gen_ai.usage.reasoning_tokens` | No ARMS/GenAI common attribute in this baseline; continue querying private `thoughts_token_count` | +| LLM `system_prompt*` | `gen_ai.system_instructions`; OpenAI system/developer messages are represented in `gen_ai.input.messages` | +| LLM `tools`, `tool_schema` events | `gen_ai.tool.definitions` | +| LLM `response.model_output*` | `gen_ai.output.messages` | +| Tool `tool_input*` | `gen_ai.tool.call.arguments` | +| Tool `tool_result*` | `gen_ai.tool.call.result` | +| `tools_count`, hash/preview/length/truncation metadata | No standard equivalent; removed | + +`gen_ai.response.finish_reasons` now preserves the provider's raw strings for +all candidates instead of the previous Gemini-normalized values. Existing +queries that filter values such as `STOP` or `MAX_TOKENS` must migrate to the +provider values, such as `stop`, `length`, `tool_calls`, or `end_turn`. ## Provider and operation resolution @@ -96,6 +108,45 @@ maximum is emitted only if all present values are valid safe integers and equal. Conflicting values are omitted because compatible endpoints do not have a common precedence rule. +## Content and tool payloads + +Sensitive GenAI content is collected only when +`telemetry.includeSensitiveSpanAttributes` is enabled. Qwen Code does not read +`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`, so there is a single +content-capture switch. OpenAI-compatible, Anthropic, Gemini, and Vertex +adapters convert their provider-final SDK request and raw response structures +to the JSON schemas pinned with this design. + +The first physical request attempt supplies input messages, system +instructions, and tool definitions. Responses are generation-bound: a provider +fallback or required-thinking retry starts a new response accumulator, and +late chunks from an older attempt are ignored. Streaming accumulators retain +canonical parts rather than raw chunks. Partial failures mark unfinished +candidates with `error`; a successful response with a candidate that lacks an +explicit finish reason omits the complete output-message attribute. + +Each JSON attribute is compactly serialized and independently limited by +`telemetry.sensitiveSpanAttributeMaxLength`. Invalid, cyclic, incomplete, or +oversized attribute values are omitted as a whole; JSON is never truncated. +Within `gen_ai.tool.definitions`, `type` and `name` are required identities, so +an invalid identity omits the complete attribute. `parameters` is optional in +the standard schema; when a provider-supplied parameter schema cannot be +normalized to Draft-07, only that optional property is omitted while the +ordered tool identity list is retained. Empty arrays and objects are retained +when the provider explicitly sends or returns them. With the default 1 MiB +limit, the application-side theoretical maximum is about 4 MiB of sensitive +attributes per LLM span and 2 MiB per Tool span. Collectors and backends can +impose lower limits. + +Tool arguments are captured from the final invocation parameters immediately +before execution, after permission and edit hooks. A tool result is captured +only after a successful call and successful post-processing, from the final +`FunctionResponse.response` object returned to the model. Both roots must be +JSON objects. `gen_ai.tool.description` comes from the static registry +description and is not sensitive; it is limited to 4096 UTF-16 code units, +preserves surrogate pairs, and appends `…[truncated]` when shortened. Agent +descriptions and span errors retain their 1024-unit limits. + ## Response and usage provenance Provider converters attach internal provenance to normalized Gemini usage @@ -136,8 +187,6 @@ Qwen Code does not inject that vendor-specific resource attribute or ## Deferred work - `seed` and `top_k` have incompatible ARMS and GenAI types in the baselines. -- Messages, instructions, tool definitions, arguments, and results require a - standard JSON schema, privacy controls, and payload caps. - Embedding needs a correct requested-model lifecycle before tracing. - ARMS time-to-first-token and OpenTelemetry time-to-first-chunk differ in name, unit, and meaning, so private `ttft_ms` remains authoritative. diff --git a/docs/design/telemetry-llm-request-timing-design.md b/docs/design/telemetry-llm-request-timing-design.md index 7e9607a1437..491707a7369 100644 --- a/docs/design/telemetry-llm-request-timing-design.md +++ b/docs/design/telemetry-llm-request-timing-design.md @@ -542,7 +542,9 @@ Rollback path: revert the single PR (or each of 4a/4b/4c independently). All new - **After Phase 3 (#4410, in review)**: not a hard dependency. Phase 4 attributes attach to `qwen-code.llm_request` spans regardless of whether they're under a `qwen-code.subagent` (Phase 3) or `qwen-code.interaction` (Phase 1) parent. Recommend Phase 3 land first so per-attempt aggregation under subagent subtrees works naturally. - **Independent of #4384** (`traceparent` + `X-Qwen-Code-Session-Id` outbound propagation). They touch the HTTP layer; Phase 4 touches the stream/retry/metric layer. -- **Independent of `clearDetailedSpanState` chat-compression follow-up** (#4097 follow-up). Different surface. +- **Independent of GenAI content capture**. Chat compression no longer resets + process-global sensitive-attribute hash state because that state was removed; + request timing remains a separate surface. ## Open questions diff --git a/docs/developers/development/telemetry.md b/docs/developers/development/telemetry.md index 3744d9d67b9..c6c69297e37 100644 --- a/docs/developers/development/telemetry.md +++ b/docs/developers/development/telemetry.md @@ -66,21 +66,21 @@ observability framework — Qwen Code's observability system provides: All telemetry behavior is controlled through your `.qwen/settings.json` file. These settings can be overridden by environment variables or CLI flags. -| Setting | Environment Variable | CLI Flag | Description | Values | Default | -| --------------------------------- | ---------------------------------------------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | ----------------------- | -| `enabled` | `QWEN_TELEMETRY_ENABLED` | `--telemetry` / `--no-telemetry` | Enable or disable telemetry | `true`/`false` | `false` | -| `target` | `QWEN_TELEMETRY_TARGET` | `--telemetry-target ` _(deprecated)_ | Informational destination label; does not control exporter routing — set `otlpEndpoint` or `outfile` to configure where data is sent | `"gcp"`/`"local"` | `"local"` | -| `otlpEndpoint` | `QWEN_TELEMETRY_OTLP_ENDPOINT` | `--telemetry-otlp-endpoint ` | OTLP collector endpoint | URL string | `http://localhost:4317` | -| `otlpProtocol` | `QWEN_TELEMETRY_OTLP_PROTOCOL` | `--telemetry-otlp-protocol ` | OTLP transport protocol | `"grpc"`/`"http"` | `"grpc"` | -| `otlpTracesEndpoint` | `QWEN_TELEMETRY_OTLP_TRACES_ENDPOINT` | - | Per-signal endpoint override for traces (HTTP only) | URL string | - | -| `otlpLogsEndpoint` | `QWEN_TELEMETRY_OTLP_LOGS_ENDPOINT` | - | Per-signal endpoint override for logs (HTTP only) | URL string | - | -| `otlpMetricsEndpoint` | `QWEN_TELEMETRY_OTLP_METRICS_ENDPOINT` | - | Per-signal endpoint override for metrics (HTTP only) | URL string | - | -| `outfile` | `QWEN_TELEMETRY_OUTFILE` | `--telemetry-outfile ` | Save telemetry to file (overrides OTLP export) | file path | - | -| `logPrompts` | `QWEN_TELEMETRY_LOG_PROMPTS` | `--telemetry-log-prompts` / `--no-telemetry-log-prompts` | Include prompts in telemetry logs | `true`/`false` | `true` | -| `includeSensitiveSpanAttributes` | `QWEN_TELEMETRY_INCLUDE_SENSITIVE_SPAN_ATTRIBUTES` | - | Include user prompts, system prompts, tool I/O, and model output as native span attributes (in addition to log-to-span bridge spans) | `true`/`false` | `false` | -| `sensitiveSpanAttributeMaxLength` | `QWEN_TELEMETRY_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH` | - | Maximum JavaScript string length for each sensitive native span attribute content payload. Set lower if your backend rejects large attributes. | `1..104857600` | `1048576` | -| `resourceAttributes` | `OTEL_RESOURCE_ATTRIBUTES` (+ `OTEL_SERVICE_NAME`) | - | Static resource attributes attached to every exported span / log / metric. See [Resource attributes](#resource-attributes) below. | `key=value,…` | `{}` | -| `metrics.includeSessionId` | `QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID` | - | Include `session.id` on metric data points. **Disabled by default** to protect metric backends from time-series fan-out. | `true`/`false` | `false` | +| Setting | Environment Variable | CLI Flag | Description | Values | Default | +| --------------------------------- | ---------------------------------------------------- | -------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | ----------------------- | +| `enabled` | `QWEN_TELEMETRY_ENABLED` | `--telemetry` / `--no-telemetry` | Enable or disable telemetry | `true`/`false` | `false` | +| `target` | `QWEN_TELEMETRY_TARGET` | `--telemetry-target ` _(deprecated)_ | Informational destination label; does not control exporter routing — set `otlpEndpoint` or `outfile` to configure where data is sent | `"gcp"`/`"local"` | `"local"` | +| `otlpEndpoint` | `QWEN_TELEMETRY_OTLP_ENDPOINT` | `--telemetry-otlp-endpoint ` | OTLP collector endpoint | URL string | `http://localhost:4317` | +| `otlpProtocol` | `QWEN_TELEMETRY_OTLP_PROTOCOL` | `--telemetry-otlp-protocol ` | OTLP transport protocol | `"grpc"`/`"http"` | `"grpc"` | +| `otlpTracesEndpoint` | `QWEN_TELEMETRY_OTLP_TRACES_ENDPOINT` | - | Per-signal endpoint override for traces (HTTP only) | URL string | - | +| `otlpLogsEndpoint` | `QWEN_TELEMETRY_OTLP_LOGS_ENDPOINT` | - | Per-signal endpoint override for logs (HTTP only) | URL string | - | +| `otlpMetricsEndpoint` | `QWEN_TELEMETRY_OTLP_METRICS_ENDPOINT` | - | Per-signal endpoint override for metrics (HTTP only) | URL string | - | +| `outfile` | `QWEN_TELEMETRY_OUTFILE` | `--telemetry-outfile ` | Save telemetry to file (overrides OTLP export) | file path | - | +| `logPrompts` | `QWEN_TELEMETRY_LOG_PROMPTS` | `--telemetry-log-prompts` / `--no-telemetry-log-prompts` | Include prompts in telemetry logs | `true`/`false` | `true` | +| `includeSensitiveSpanAttributes` | `QWEN_TELEMETRY_INCLUDE_SENSITIVE_SPAN_ATTRIBUTES` | - | Include standard GenAI messages, instructions, tool definitions, tool arguments, and successful tool results as native span attributes | `true`/`false` | `false` | +| `sensitiveSpanAttributeMaxLength` | `QWEN_TELEMETRY_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH` | - | Maximum compact JSON string length for each sensitive native span attribute. Set lower if your backend rejects large attributes. | `1..104857600` | `1048576` | +| `resourceAttributes` | `OTEL_RESOURCE_ATTRIBUTES` (+ `OTEL_SERVICE_NAME`) | - | Static resource attributes attached to every exported span / log / metric. See [Resource attributes](#resource-attributes) below. | `key=value,…` | `{}` | +| `metrics.includeSessionId` | `QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID` | - | Include `session.id` on metric data points. **Disabled by default** to protect metric backends from time-series fan-out. | `true`/`false` | `false` | **Note on boolean environment variables:** For the boolean settings (`enabled`, `logPrompts`, `includeSensitiveSpanAttributes`), setting the @@ -91,31 +91,37 @@ other value will disable it. must be a positive integer when set. Invalid values fail telemetry configuration resolution instead of silently falling back. +`gen_ai.tool.description` is non-sensitive static registry metadata and is +emitted independently of `includeSensitiveSpanAttributes`. This includes +descriptions supplied by MCP servers and other workspace tool providers. The +value is limited to 4096 UTF-16 code units and never includes dynamic invocation +details. + **Sensitive span attributes:** When `includeSensitiveSpanAttributes` is enabled, two things happen: -1. **Native span attributes (`qwen-code.interaction`, `api.generateContent*`, - `tool.`)** carry verbatim conversation content: - - User prompts (`new_context`) - - System prompts (`system_prompt` — full text once per session, deduped by - SHA-256 hash; subsequent spans only carry `system_prompt_hash` + - `system_prompt_preview` + `system_prompt_length`) - - Tool schemas (emitted as `tool_schema` events, also hash-deduped) - - Tool inputs (`tool_input`) and tool results (`tool_result`) - - Model output (`response.model_output`) - - Each content payload is truncated at `sensitiveSpanAttributeMaxLength` - JavaScript string units. The default is 1 MiB (`1048576`), raised from the - previous 60 KiB default; set `61440` to preserve the old cap. The limit - must be between `1` and `104857600` (100 MiB). For labeled attributes, fixed - labels such as `[USER PROMPT]`, `[TOOL INPUT: ...]`, and - `[TOOL RESULT: ...]` count against the cap; the truncation marker also counts - against it. The limit is measured as JavaScript string length rather than - UTF-8 bytes. Non-ASCII content can therefore occupy more bytes after OTLP - export. For most payload types, truncation adds both `*_truncated` and - `*_original_length`. System prompts also set `system_prompt_truncated` when - truncated, but use the always-present `system_prompt_length` for the original - length. +1. **Native span attributes** carry standard OpenTelemetry GenAI JSON: + - LLM input messages (`gen_ai.input.messages`) + - System instructions (`gen_ai.system_instructions`) + - Tool definitions (`gen_ai.tool.definitions`) + - LLM output messages (`gen_ai.output.messages`) + - Final executed tool arguments (`gen_ai.tool.call.arguments`) + - Successful tool results (`gen_ai.tool.call.result`) + - Interaction spans continue to use `new_context` because they are not GenAI + inference spans. + + LLM values come from provider-final SDK request objects and raw provider + responses, not the original logical configuration. Tool values come from + the final invocation parameters and successful model-facing result. Each + standard GenAI value is compact JSON and must be complete and schema-valid. + A value that is invalid, cyclic, or longer than + `sensitiveSpanAttributeMaxLength` is omitted as a whole; JSON is never + truncated and no preview, hash, or truncation metadata is emitted. The + interaction-specific `new_context` attribute retains its existing + truncation behavior. The default maximum is 1 MiB (`1048576`) per attribute + and the accepted range is `1..104857600` (100 MiB). The limit is measured as + JavaScript string length rather than UTF-8 bytes. Non-ASCII content can + therefore occupy more bytes after OTLP export. 2. **Log-to-span bridge spans** (used when HTTP traces are exported without a logs endpoint) keep their existing `prompt`, `function_args`, and @@ -127,14 +133,13 @@ secrets in env vars or arguments), and model responses to the configured OTLP backend. Treat the backend as a privileged data sink. The flag defaults to `false`. -**Cost / payload size:** A heavy turn at the default limit (1 MiB system prompt -plus 10 tool calls, each up to 1 MiB input + 1 MiB result, plus 1 MiB model -output) can produce up to ~22 MiB of attribute payload before OTLP compression, -plus up to 1 MiB per emitted tool schema in workspaces with large tool -definitions. This is Qwen Code's application-side cap, not a guarantee that -every collector or backend accepts a single attribute that large. If spans are -rejected or dropped, lower `sensitiveSpanAttributeMaxLength` (for example, to -`61440`) and monitor exporter throughput. +**Cost / payload size:** At the default limit, one LLM span can carry at most +about 4 MiB across input, output, system instructions, and tool definitions; +one Tool span can carry about 2 MiB across arguments and result. This is Qwen +Code's application-side cap, not a guarantee that every collector or backend +accepts a single attribute that large. If spans are rejected or dropped, lower +`sensitiveSpanAttributeMaxLength` (for example, to `61440`) and monitor exporter +throughput. This setting does not disable sensitive data in OTel logs or other telemetry sinks; non-internal API response telemetry can populate `response_text`, so diff --git a/integration-tests/cli/gen-ai-telemetry.test.ts b/integration-tests/cli/gen-ai-telemetry.test.ts index 95a45c2b207..8cff3487dd5 100644 --- a/integration-tests/cli/gen-ai-telemetry.test.ts +++ b/integration-tests/cli/gen-ai-telemetry.test.ts @@ -16,6 +16,7 @@ import { type TelemetryRecord = { name?: string; attributes?: Record; + events?: Array<{ name?: string }>; }; const SKIP = @@ -93,7 +94,18 @@ describeLocal('GenAI telemetry fields', () => { } : { model: 'provider-model-final', - content: 'Tool completed.', + choices: [ + { + index: 0, + contentChunks: ['Tool ', 'completed.'], + finishReason: 'stop', + }, + { + index: 1, + content: 'Alternative final answer.', + finishReason: 'stop', + }, + ], usage: { prompt_tokens: 30, completion_tokens: 5, @@ -140,6 +152,7 @@ describeLocal('GenAI telemetry fields', () => { https_proxy: undefined, all_proxy: undefined, DASHSCOPE_PROXY_BASE_URL: undefined, + QWEN_TELEMETRY_INCLUDE_SENSITIVE_SPAN_ATTRIBUTES: 'true', }); try { @@ -172,7 +185,7 @@ describeLocal('GenAI telemetry fields', () => { 'gen_ai.request.presence_penalty': 0.2, 'gen_ai.request.stop_sequences': ['END', 'DONE'], 'gen_ai.response.model': 'provider-model-tool', - 'gen_ai.response.finish_reasons': ['STOP'], + 'gen_ai.response.finish_reasons': ['tool_calls'], 'gen_ai.usage.input_tokens': 20, 'gen_ai.usage.output_tokens': 4, 'gen_ai.usage.cache_read.input_tokens': 3, @@ -188,7 +201,7 @@ describeLocal('GenAI telemetry fields', () => { 'gen_ai.request.presence_penalty': 0.2, 'gen_ai.request.stop_sequences': ['END', 'DONE'], 'gen_ai.response.model': 'provider-model-final', - 'gen_ai.response.finish_reasons': ['STOP'], + 'gen_ai.response.finish_reasons': ['stop', 'stop'], 'gen_ai.usage.input_tokens': 30, 'gen_ai.usage.output_tokens': 5, }); @@ -197,6 +210,85 @@ describeLocal('GenAI telemetry fields', () => { ); expect(firstLlm['gen_ai.conversation.id']).toEqual(expect.any(String)); + const firstRequest = server.requests[0]!.body; + const firstInput = JSON.parse( + firstLlm['gen_ai.input.messages'] as string, + ) as Array>; + const secondInput = JSON.parse( + secondLlm['gen_ai.input.messages'] as string, + ) as Array>; + expect(firstInput.map((message) => message['role'])).toEqual( + (firstRequest['messages'] as Array>).map( + (message) => message['role'], + ), + ); + expect( + JSON.stringify(firstInput).includes( + 'Run the requested tool and then report completion.', + ), + ).toBe(true); + expect( + secondInput.some( + (message) => + message['role'] === 'tool' && + JSON.stringify(message).includes('provider-call-123'), + ), + ).toBe(true); + expect(secondInput.length).toBeGreaterThan(firstInput.length); + expect(firstLlm).not.toHaveProperty('gen_ai.system_instructions'); + expect(secondLlm).not.toHaveProperty('gen_ai.system_instructions'); + + const rawTools = firstRequest['tools'] as Array<{ + type: string; + function: { + name: string; + description?: string; + parameters?: object; + }; + }>; + const toolDefinitions = JSON.parse( + firstLlm['gen_ai.tool.definitions'] as string, + ) as Array>; + expect(toolDefinitions).toEqual( + rawTools.map((tool) => ({ + type: tool.type, + name: tool.function.name, + ...(tool.function.description !== undefined + ? { description: tool.function.description } + : {}), + ...(tool.function.parameters !== undefined + ? { parameters: tool.function.parameters } + : {}), + })), + ); + + expect(JSON.parse(firstLlm['gen_ai.output.messages'] as string)).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'provider-call-123', + name: 'run_shell_command', + arguments: { command: 'pwd' }, + }, + ], + finish_reason: 'tool_calls', + }, + ]); + expect(JSON.parse(secondLlm['gen_ai.output.messages'] as string)).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'Tool completed.' }], + finish_reason: 'stop', + }, + { + role: 'assistant', + parts: [{ type: 'text', content: 'Alternative final answer.' }], + finish_reason: 'stop', + }, + ]); + for (const attributes of [firstLlm, secondLlm]) { expect(attributes).not.toHaveProperty('qwen-code.model'); expect(attributes).not.toHaveProperty('response_id'); @@ -215,7 +307,14 @@ describeLocal('GenAI telemetry fields', () => { expect(attributes).not.toHaveProperty('frequency_penalty'); expect(attributes).not.toHaveProperty('presence_penalty'); expect(attributes).not.toHaveProperty('stop_sequences'); + expect(attributes).not.toHaveProperty('system_prompt'); + expect(attributes).not.toHaveProperty('tools'); + expect(attributes).not.toHaveProperty('tools_count'); + expect(attributes).not.toHaveProperty('response.model_output'); } + expect(records.flatMap((record) => record.events ?? [])).not.toContainEqual( + expect.objectContaining({ name: 'tool_schema' }), + ); expect(server.requests).toHaveLength(2); for (const { body } of server.requests) { @@ -242,19 +341,50 @@ describeLocal('GenAI telemetry fields', () => { 'gen_ai.tool.call.id': 'provider-call-123', 'tool.call_id': 'provider-call-123', }); + expect(toolSpan?.attributes?.['gen_ai.tool.description']).toEqual( + expect.any(String), + ); + expect( + JSON.parse( + toolSpan?.attributes?.['gen_ai.tool.call.arguments'] as string, + ), + ).toEqual({ command: 'pwd' }); + expect( + JSON.parse(toolSpan?.attributes?.['gen_ai.tool.call.result'] as string), + ).toMatchObject({ output: expect.any(String) }); expect(toolSpan?.attributes).not.toHaveProperty('tool.name'); + expect(toolSpan?.attributes).not.toHaveProperty('tool_input'); + expect(toolSpan?.attributes).not.toHaveProperty('tool_result'); }); - it('omits the default choice count from the exported span', async () => { - server = await startFakeOpenAIServer(() => ({ - model: 'provider-model', - content: 'Done.', - usage: { - prompt_tokens: 10, - completion_tokens: 2, - total_tokens: 12, - }, - })); + it('omits the default choice count and sensitive tool payloads', async () => { + server = await startFakeOpenAIServer(({ requestIndex }) => + requestIndex === 0 + ? { + model: 'provider-model-tool', + toolCalls: [ + fakeToolCall( + 'run_shell_command', + { command: 'pwd' }, + 'provider-call-sensitive-off', + ), + ], + usage: { + prompt_tokens: 10, + completion_tokens: 2, + total_tokens: 12, + }, + } + : { + model: 'provider-model-final', + content: 'Done.', + usage: { + prompt_tokens: 15, + completion_tokens: 2, + total_tokens: 17, + }, + }, + ); rig = new TestRig(); rig.setup('gen-ai-default-choice-count', { @@ -286,10 +416,15 @@ describeLocal('GenAI telemetry fields', () => { https_proxy: undefined, all_proxy: undefined, DASHSCOPE_PROXY_BASE_URL: undefined, + QWEN_TELEMETRY_INCLUDE_SENSITIVE_SPAN_ATTRIBUTES: 'false', }); try { - await rig.run('Reply with done.', '--output-format', 'json'); + await rig.run( + 'Run the requested tool and reply with done.', + '--output-format', + 'json', + ); } finally { restoreEnvironment(); } @@ -308,6 +443,24 @@ describeLocal('GenAI telemetry fields', () => { expect(llmSpan.attributes).not.toHaveProperty( 'gen_ai.request.choice.count', ); + expect(llmSpan.attributes).not.toHaveProperty('gen_ai.input.messages'); + expect(llmSpan.attributes).not.toHaveProperty('gen_ai.output.messages'); + expect(llmSpan.attributes).not.toHaveProperty( + 'gen_ai.system_instructions', + ); + expect(llmSpan.attributes).not.toHaveProperty('gen_ai.tool.definitions'); } + const toolSpan = records.find( + (record) => + record.name === 'qwen-code.tool' && + record.attributes?.['gen_ai.tool.name'] === 'run_shell_command', + ); + expect(toolSpan?.attributes?.['gen_ai.tool.description']).toEqual( + expect.any(String), + ); + expect(toolSpan?.attributes).not.toHaveProperty( + 'gen_ai.tool.call.arguments', + ); + expect(toolSpan?.attributes).not.toHaveProperty('gen_ai.tool.call.result'); }); }); diff --git a/integration-tests/fake-openai-server.ts b/integration-tests/fake-openai-server.ts index 57ce9f980e6..36ce5757ebb 100644 --- a/integration-tests/fake-openai-server.ts +++ b/integration-tests/fake-openai-server.ts @@ -31,6 +31,7 @@ export type FakeOpenAIResponse = { disconnectAfterContentChunks?: number; toolCalls?: FakeOpenAIToolCall[]; finishReason?: 'stop' | 'tool_calls' | 'length'; + choices?: FakeOpenAIChoice[]; usage?: { prompt_tokens: number; completion_tokens: number; @@ -41,6 +42,14 @@ export type FakeOpenAIResponse = { }; }; +export type FakeOpenAIChoice = { + index: number; + content?: string; + contentChunks?: string[]; + toolCalls?: FakeOpenAIToolCall[]; + finishReason?: 'stop' | 'tool_calls' | 'length'; +}; + export type FakeOpenAIRequest = { body: JsonObject; }; @@ -218,17 +227,15 @@ function writeNonStreamed( object: 'chat.completion', created: nowSeconds(), model: message.model ?? model, - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: message.content ?? message.contentChunks?.join('') ?? null, - ...(message.toolCalls ? { tool_calls: message.toolCalls } : {}), - }, - finish_reason: finishReason(message), + choices: responseChoices(message).map((choice) => ({ + index: choice.index, + message: { + role: 'assistant', + content: choice.content ?? choice.contentChunks?.join('') ?? null, + ...(choice.toolCalls ? { tool_calls: choice.toolCalls } : {}), }, - ], + finish_reason: finishReason(choice), + })), usage: message.usage ?? DEFAULT_USAGE, }), ); @@ -249,6 +256,7 @@ function writeStreamed( const created = nowSeconds(); const responseModel = message.model ?? model; const chunk = ( + index: number, delta: JsonObject, finish_reason: string | null = null, usage?: FakeOpenAIResponse['usage'], @@ -257,61 +265,101 @@ function writeStreamed( object: 'chat.completion.chunk', created, model: responseModel, - choices: [{ index: 0, delta, finish_reason }], + choices: [{ index, delta, finish_reason }], ...(usage ? { usage } : {}), }); const send = (payload: unknown, callback?: () => void) => { res.write(`data: ${JSON.stringify(payload)}\n\n`, callback); }; - send(chunk({ role: 'assistant' })); - for (const [index, content] of (message.contentChunks ?? []).entries()) { - if (message.disconnectAfterContentChunks === index + 1) { - send(chunk({ content }), () => res.destroy()); - return; + const choices = responseChoices(message); + for (const [choicePosition, choice] of choices.entries()) { + send(chunk(choice.index, { role: 'assistant' })); + for (const [contentIndex, content] of ( + choice.contentChunks ?? [] + ).entries()) { + if (message.disconnectAfterContentChunks === contentIndex + 1) { + send(chunk(choice.index, { content }), () => res.destroy()); + return; + } + send(chunk(choice.index, { content })); } - send(chunk({ content })); - } - if (!message.contentChunks && message.content) { - send(chunk({ content: message.content })); - } - for (const [index, toolCall] of (message.toolCalls ?? []).entries()) { - send( - chunk({ - tool_calls: [ - { - index, - id: toolCall.id, - type: toolCall.type, - function: { - name: toolCall.function.name, - arguments: '', - }, - }, - ], - }), - ); - if (toolCall.function.arguments) { + if (!choice.contentChunks && choice.content) { + send(chunk(choice.index, { content: choice.content })); + } + for (const [toolIndex, toolCall] of (choice.toolCalls ?? []).entries()) { send( - chunk({ + chunk(choice.index, { tool_calls: [ { - index, + index: toolIndex, + id: toolCall.id, + type: toolCall.type, function: { - arguments: toolCall.function.arguments, + name: toolCall.function.name, + arguments: '', }, }, ], }), ); + if (toolCall.function.arguments) { + send( + chunk(choice.index, { + tool_calls: [ + { + index: toolIndex, + function: { + arguments: toolCall.function.arguments, + }, + }, + ], + }), + ); + } } + send( + chunk( + choice.index, + {}, + finishReason(choice), + choices.length === 1 && choicePosition === choices.length - 1 + ? (message.usage ?? DEFAULT_USAGE) + : undefined, + ), + ); + } + if (choices.length > 1) { + send({ + id, + object: 'chat.completion.chunk', + created, + model: responseModel, + choices: [], + usage: message.usage ?? DEFAULT_USAGE, + }); } - send(chunk({}, finishReason(message), message.usage ?? DEFAULT_USAGE)); res.write('data: [DONE]\n\n'); res.end(); } -function finishReason(message: FakeOpenAIResponse): string { +function responseChoices(message: FakeOpenAIResponse): FakeOpenAIChoice[] { + return ( + message.choices ?? [ + { + index: 0, + content: message.content, + contentChunks: message.contentChunks, + toolCalls: message.toolCalls, + finishReason: message.finishReason, + }, + ] + ); +} + +function finishReason( + message: Pick, +): string { return message.finishReason ?? (message.toolCalls ? 'tool_calls' : 'stop'); } diff --git a/packages/cli/src/acp-integration/session/Session.test.ts b/packages/cli/src/acp-integration/session/Session.test.ts index 8e641c9390b..2af1ae0b3d0 100644 --- a/packages/cli/src/acp-integration/session/Session.test.ts +++ b/packages/cli/src/acp-integration/session/Session.test.ts @@ -54,6 +54,8 @@ const runVisionBridgeSpy = vi.hoisted(() => vi.fn()); const refreshMemoryAfterManagedWriteSpy = vi.hoisted(() => vi.fn()); const transcribeVoiceAudioSpy = vi.hoisted(() => vi.fn()); const startToolSpanSpy = vi.hoisted(() => vi.fn()); +const addToolArgumentsAttributesSpy = vi.hoisted(() => vi.fn()); +const addToolCallResultAttributesSpy = vi.hoisted(() => vi.fn()); // Records every LoopTickResolver construction's deps so a test can assert what // Session computed (e.g. the home confinement root) without a private-field peek. const loopTickResolverDepsSpy = vi.hoisted(() => vi.fn()); @@ -77,6 +79,18 @@ vi.mock('@qwen-code/qwen-code-core', async (importOriginal) => { startToolSpanSpy(...args); return actual.startToolSpan(...args); }, + addToolArgumentsAttributes: ( + ...args: Parameters + ) => { + addToolArgumentsAttributesSpy(...args); + return actual.addToolArgumentsAttributes(...args); + }, + addToolCallResultAttributes: ( + ...args: Parameters + ) => { + addToolCallResultAttributesSpy(...args); + return actual.addToolCallResultAttributes(...args); + }, // Transparent recording wrapper: records the constructor deps, then behaves // exactly like the real resolver (subclass → instanceof + methods preserved). LoopTickResolver: class extends actual.LoopTickResolver { @@ -443,6 +457,8 @@ describe('Session', () => { beforeEach(() => { startToolSpanSpy.mockClear(); + addToolArgumentsAttributesSpy.mockClear(); + addToolCallResultAttributesSpy.mockClear(); runVisionBridgeSpy.mockReset(); refreshMemoryAfterManagedWriteSpy.mockReset(); refreshMemoryAfterManagedWriteSpy.mockResolvedValue(false); @@ -11656,6 +11672,15 @@ describe('Session', () => { undefined, ); expect(executeSpy).toHaveBeenCalledOnce(); + expect(addToolArgumentsAttributesSpy).toHaveBeenCalledWith( + mockConfig, + expect.anything(), + invocation.params, + ); + expect(invocation.params).toMatchObject({ + command: rawCommand, + directory: expect.any(String), + }); }); it('reports ACP Plan shell approval request failures accurately', async () => { @@ -12288,6 +12313,8 @@ describe('Session', () => { expect(executeSpy).not.toHaveBeenCalled(); // No permission dialog should have been opened expect(mockClient.requestPermission).not.toHaveBeenCalled(); + expect(addToolArgumentsAttributesSpy).not.toHaveBeenCalled(); + expect(addToolCallResultAttributesSpy).not.toHaveBeenCalled(); }); it('respects permission-request hook allow decisions without opening ACP permission dialog', async () => { @@ -12362,6 +12389,11 @@ describe('Session', () => { ); expect(invocation.params).toEqual({ path: '/tmp/updated.txt' }); expect(executeSpy).toHaveBeenCalled(); + expect(addToolArgumentsAttributesSpy).toHaveBeenCalledWith( + mockConfig, + expect.anything(), + { path: '/tmp/updated.txt' }, + ); }); it('keeps exit_plan_mode in PLAN until ACP approval executes and then notifies once', async () => { @@ -15039,9 +15071,22 @@ describe('Session', () => { call_id: 'provider-call__qwen_dup_2', 'gen_ai.tool.call.id': 'provider-call', }), + 'read_file', + ); + expect(addToolArgumentsAttributesSpy).toHaveBeenCalledWith( + mockConfig, + expect.anything(), + {}, + ); + expect(addToolCallResultAttributesSpy).toHaveBeenCalledWith( + mockConfig, + expect.anything(), + { output: 'read' }, ); startToolSpanSpy.mockClear(); + addToolArgumentsAttributesSpy.mockClear(); + addToolCallResultAttributesSpy.mockClear(); await (session as unknown as ToolCallInternals).runToolCalls( new AbortController().signal, 'prompt-tool-span-fallback', @@ -15059,6 +15104,45 @@ describe('Session', () => { 'tool.call_id': 'internal-call', 'gen_ai.tool.call.id': 'internal-call', }), + 'read_file', + ); + }); + + it('does not fail ACP tool execution when telemetry helpers throw', async () => { + const execute = vi.fn().mockResolvedValue({ + llmContent: 'read', + returnDisplay: 'read', + }); + mockToolRegistry.getTool.mockReturnValue( + mockAllowedTool(core.ToolNames.READ_FILE, execute), + ); + addToolArgumentsAttributesSpy.mockImplementationOnce(() => { + throw new Error('arguments telemetry failed'); + }); + addToolCallResultAttributesSpy.mockImplementationOnce(() => { + throw new Error('result telemetry failed'); + }); + + await expect( + (session as unknown as ToolCallInternals).runToolCalls( + new AbortController().signal, + 'prompt-tool-telemetry-failure', + [ + { + id: 'internal-call', + name: core.ToolNames.READ_FILE, + args: { file_path: 'test.ts' }, + }, + ], + ), + ).resolves.toBeDefined(); + + expect(execute).toHaveBeenCalledOnce(); + expect(debugLoggerDebugSpy).toHaveBeenCalledWith( + '[Session.runTool] Failed to record tool arguments telemetry', + ); + expect(debugLoggerDebugSpy).toHaveBeenCalledWith( + '[Session.runTool] Failed to record tool result telemetry', ); }); diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index 4428e614c3f..4fe7e1178b3 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -127,6 +127,8 @@ import { SessionWriterError, startToolSpan, endToolSpan, + addToolArgumentsAttributes, + addToolCallResultAttributes, runInToolSpanContext, startToolExecutionSpan, endToolExecutionSpan, @@ -6589,15 +6591,19 @@ export class Session implements SessionContext { ? structuredClone(args) : args; - const toolSpan = startToolSpan(policyToolName, { - 'tool.call_id': callId, - 'gen_ai.tool.call.id': getProviderToolCallId(fc) ?? callId, - // Dual-emit the legacy call_id/tool_name aliases like CoreToolScheduler - // (coreToolScheduler.ts) so pre-Phase-2 dashboards keyed off call_id keep - // matching daemon/ACP tool spans during the migration window. - call_id: callId, - tool_name: policyToolName, - }); + const toolSpan = startToolSpan( + policyToolName, + { + 'tool.call_id': callId, + 'gen_ai.tool.call.id': getProviderToolCallId(fc) ?? callId, + // Dual-emit the legacy call_id/tool_name aliases like CoreToolScheduler + // (coreToolScheduler.ts) so pre-Phase-2 dashboards keyed off call_id keep + // matching daemon/ACP tool spans during the migration window. + call_id: callId, + tool_name: policyToolName, + }, + tool.description, + ); let spanSuccess = false; try { @@ -7418,6 +7424,17 @@ export class Session implements SessionContext { `Qwen Code is executing tool ${toolName}`, ); try { + try { + addToolArgumentsAttributes( + this.config, + toolSpan, + invocation.params, + ); + } catch { + debugLogger.debug( + '[Session.runTool] Failed to record tool arguments telemetry', + ); + } toolResult = await invocation.execute( activeToolAbortSignal, onToolProgress, @@ -7654,6 +7671,20 @@ export class Session implements SessionContext { }); spanSuccess = succeeded; + if (succeeded && !nestedPermissionCancelled) { + const result = responseParts.find( + (part) => part.functionResponse !== undefined, + )?.functionResponse?.response; + if (result !== undefined) { + try { + addToolCallResultAttributes(this.config, toolSpan, result); + } catch { + debugLogger.debug( + '[Session.runTool] Failed to record tool result telemetry', + ); + } + } + } if (toolResult.error) { spanError = toolResult.error.message; } else if (aborted) { diff --git a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts index c4820146c03..0297b74c47c 100644 --- a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts +++ b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.test.ts @@ -23,12 +23,18 @@ const mockTokenizer = { dispose: vi.fn(), }; const mockReportAnthropicRequest = vi.hoisted(() => vi.fn()); +const mockReportAnthropicFollowingRequest = vi.hoisted(() => vi.fn()); +const mockReportAnthropicResponse = vi.hoisted(() => vi.fn()); +const mockReportAnthropicEvent = vi.hoisted(() => vi.fn()); vi.mock('../../utils/request-tokenizer/index.js', () => ({ RequestTokenEstimator: vi.fn(() => mockTokenizer), })); vi.mock('../../telemetry/gen-ai-request.js', () => ({ reportAnthropicRequest: mockReportAnthropicRequest, + reportAnthropicFollowingRequest: mockReportAnthropicFollowingRequest, + reportAnthropicResponse: mockReportAnthropicResponse, + reportAnthropicEvent: mockReportAnthropicEvent, })); type AnthropicCreateArgs = [ @@ -1190,6 +1196,8 @@ describe('AnthropicContentGenerator', () => { { ...baseConfig, reasoning: { effort: 'medium' } }, mockConfig, ); + const telemetryAttempt = {}; + mockReportAnthropicRequest.mockReturnValueOnce(telemetryAttempt); const stream = await generator.generateContentStream({ model: 'models/ignored', contents: 'Hi', @@ -1210,6 +1218,10 @@ describe('AnthropicContentGenerator', () => { const [streamingRequest, options] = anthropicState.lastCreateArgs as AnthropicCreateArgs; expect(mockReportAnthropicRequest).toHaveBeenCalledWith(streamingRequest); + expect(mockReportAnthropicEvent).toHaveBeenCalledWith( + telemetryAttempt, + expect.objectContaining({ type: 'message_delta' }), + ); const headers = ((options as { headers?: Record }) ?.headers || {}) as Record; expect(headers['anthropic-beta']).toContain( @@ -1392,6 +1404,8 @@ describe('AnthropicContentGenerator', () => { }, }; + const telemetryAttempt = {}; + mockReportAnthropicRequest.mockReturnValueOnce(telemetryAttempt); const result = await generator.generateContent(request); expect(result.responseId).toBe('gemini-1'); @@ -1420,6 +1434,10 @@ describe('AnthropicContentGenerator', () => { }), ); expect(mockReportAnthropicRequest).toHaveBeenCalledWith(anthropicRequest); + expect(mockReportAnthropicResponse).toHaveBeenCalledWith( + telemetryAttempt, + expect.objectContaining({ id: 'anthropic-1' }), + ); expect(convertResponseSpy).toHaveBeenCalledTimes(1); }); @@ -3847,14 +3865,18 @@ describe('AnthropicContentGenerator', () => { expect.objectContaining({ stream: true }), ); expect(fallbackRequest).not.toHaveProperty('stream'); - expect(mockReportAnthropicRequest).toHaveBeenNthCalledWith( - 2, + expect(mockReportAnthropicFollowingRequest).toHaveBeenCalledWith( fallbackRequest, + undefined, ); }); it('converts the non-streaming fallback response when an empty stream is recoverable', async () => { const { AnthropicContentGenerator } = await importGenerator(); + const streamingAttempt = { generation: 1 }; + const fallbackAttempt = { generation: 2 }; + mockReportAnthropicRequest.mockReturnValueOnce(streamingAttempt); + mockReportAnthropicFollowingRequest.mockReturnValueOnce(fallbackAttempt); anthropicState.createImpl .mockResolvedValueOnce( (async function* () { @@ -3892,12 +3914,26 @@ describe('AnthropicContentGenerator', () => { } expect(anthropicState.createImpl).toHaveBeenCalledTimes(2); + const [fallbackRequest] = anthropicState.createImpl.mock + .calls[1] as AnthropicCreateArgs; expect(chunks).toHaveLength(1); expect(chunks[0]?.responseId).toBe('msg-fallback'); expect(chunks[0]?.candidates?.[0]?.content?.parts).toEqual([ { text: 'fallback ok' }, ]); expect(chunks[0]?.candidates?.[0]?.finishReason).toBe(FinishReason.STOP); + expect(mockReportAnthropicFollowingRequest).toHaveBeenCalledWith( + fallbackRequest, + streamingAttempt, + ); + expect(mockReportAnthropicResponse).toHaveBeenCalledWith( + fallbackAttempt, + expect.objectContaining({ id: 'msg-fallback' }), + ); + expect(mockReportAnthropicResponse).not.toHaveBeenCalledWith( + streamingAttempt, + expect.anything(), + ); }); }); diff --git a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts index 87ee08d63f8..a1f79938d8a 100644 --- a/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts +++ b/packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts @@ -49,7 +49,13 @@ import { parsePositiveIntegerEnvValue, } from '../tokenLimits.js'; import { setToolCallPreparations } from '../tool-call-preparation.js'; -import { reportAnthropicRequest } from '../../telemetry/gen-ai-request.js'; +import { + reportAnthropicEvent, + reportAnthropicFollowingRequest, + reportAnthropicRequest, + reportAnthropicResponse, + type GenAiAttemptHandle, +} from '../../telemetry/gen-ai-request.js'; const debugLogger = createDebugLogger('ANTHROPIC'); @@ -348,12 +354,13 @@ export class AnthropicContentGenerator implements ContentGenerator { try { const anthropicRequest = await this.buildRequest(request); runtimeDiagnostics.recordAnthropicWireRequest(anthropicRequest); - reportAnthropicRequest(anthropicRequest); + const telemetryAttempt = reportAnthropicRequest(anthropicRequest); const headers = this.buildPerRequestHeaders(anthropicRequest); response = (await this.client.messages.create(anthropicRequest, { signal: perRequestAc?.signal, ...(headers ? { headers } : {}), })) as Message; + reportAnthropicResponse(telemetryAttempt, response); } catch (error) { throw redactProxyError(error); } finally { @@ -375,7 +382,7 @@ export class AnthropicContentGenerator implements ContentGenerator { stream: true, }; runtimeDiagnostics.recordAnthropicWireRequest(streamingRequest); - reportAnthropicRequest(streamingRequest); + const telemetryAttempt = reportAnthropicRequest(streamingRequest); // Wrap the caller's signal in a per-request child so the Anthropic SDK's // leaked abort listener (core.mjs fetchWithTimeout registers one with no @@ -405,6 +412,7 @@ export class AnthropicContentGenerator implements ContentGenerator { anthropicRequest, perRequestAc.signal, headers, + telemetryAttempt, ); // Abort the child once the stream is fully drained or abandoned; this // releases the SDK request and detaches the child's listener from the @@ -1072,6 +1080,7 @@ export class AnthropicContentGenerator implements ContentGenerator { private async *processStream( stream: AsyncIterable, + telemetryAttempt: GenAiAttemptHandle | undefined, ): AsyncGenerator { let messageId: string | undefined; let model: string | undefined; @@ -1102,6 +1111,7 @@ export class AnthropicContentGenerator implements ContentGenerator { }; for await (const event of stream) { + reportAnthropicEvent(telemetryAttempt, event); switch (event.type) { case 'message_start': { messageId = event.message.id ?? messageId; @@ -1364,11 +1374,12 @@ export class AnthropicContentGenerator implements ContentGenerator { fallbackRequest: MessageCreateParamsWithThinking, abortSignal: AbortSignal | undefined, headers: Record | undefined, + telemetryAttempt: GenAiAttemptHandle | undefined, ): AsyncGenerator { let hasAssistantPayload = false; let hasFinishReason = false; - for await (const chunk of this.processStream(stream)) { + for await (const chunk of this.processStream(stream, telemetryAttempt)) { const candidates = chunk.candidates ?? []; hasFinishReason ||= candidates.some( (candidate) => candidate.finishReason !== undefined, @@ -1397,11 +1408,15 @@ export class AnthropicContentGenerator implements ContentGenerator { let response: Message; try { runtimeDiagnostics.recordAnthropicWireRequest(fallbackRequest); - reportAnthropicRequest(fallbackRequest); + const fallbackAttempt = reportAnthropicFollowingRequest( + fallbackRequest, + telemetryAttempt, + ); response = (await this.client.messages.create(fallbackRequest, { signal: abortSignal, ...(headers ? { headers } : {}), })) as Message; + reportAnthropicResponse(fallbackAttempt, response); yield this.converter.convertAnthropicResponseToGemini(response); } catch (error) { throw redactProxyError(error); diff --git a/packages/core/src/core/coreToolScheduler.test.ts b/packages/core/src/core/coreToolScheduler.test.ts index ade6837983a..7d50d25cae8 100644 --- a/packages/core/src/core/coreToolScheduler.test.ts +++ b/packages/core/src/core/coreToolScheduler.test.ts @@ -211,8 +211,16 @@ function createMockToolSpan( vi.mock('../telemetry/session-tracing.js', () => ({ startToolSpan: vi.fn( - (name: string, attrs?: Record) => - createMockToolSpan(`tool.${name}`, { tool_name: name, ...attrs }), + ( + name: string, + attrs?: Record, + description?: string, + ) => + createMockToolSpan(`tool.${name}`, { + tool_name: name, + ...attrs, + ...(description ? { 'gen_ai.tool.description': description } : {}), + }), ), endToolSpan: vi.fn( ( @@ -8832,6 +8840,46 @@ describe('CoreToolScheduler telemetry spans', () => { expect(fallbackSpan.attributes['gen_ai.tool.call.id']).toBe('span-call'); }); + it('records static description and final successful arguments/result', async () => { + mockTelemetrySdkState.initialized = true; + const { spanRecord } = await runSingleTool({ + includeSensitiveSpanAttributes: true, + }); + + expect(spanRecord.attributes['gen_ai.tool.description']).toBe('mockTool'); + expect( + JSON.parse( + spanRecord.spanAttributes['gen_ai.tool.call.arguments'] as string, + ), + ).toEqual({ input: '/secret/path' }); + expect( + JSON.parse( + spanRecord.spanAttributes['gen_ai.tool.call.result'] as string, + ), + ).toEqual({ output: 'ok' }); + expect(spanRecord.spanAttributes).not.toHaveProperty('tool_input'); + expect(spanRecord.spanAttributes).not.toHaveProperty('tool_result'); + }); + + it('keeps executed arguments but omits result for soft errors', async () => { + mockTelemetrySdkState.initialized = true; + const { spanRecord } = await runSingleTool({ + includeSensitiveSpanAttributes: true, + execute: vi.fn().mockResolvedValue({ + llmContent: 'failed', + returnDisplay: 'failed', + error: { message: 'failed', type: ToolErrorType.EXECUTION_FAILED }, + }), + }); + + expect( + spanRecord.spanAttributes['gen_ai.tool.call.arguments'], + ).toBeDefined(); + expect( + spanRecord.spanAttributes['gen_ai.tool.call.result'], + ).toBeUndefined(); + }); + it('acquires the sleep inhibitor around actual tool execution', async () => { mockAcquireSleepInhibitor.mockClear(); mockSleepInhibitorRelease.mockClear(); @@ -9412,14 +9460,12 @@ describe('CoreToolScheduler telemetry spans', () => { expect(completedCalls[0].status).toBe('success'); expect(spanRecord.ended).toBe(true); - expect(debugLoggerWarnSpy).toHaveBeenCalledWith( - 'Failed to add tool input span attributes:', - expect.any(TypeError), - ); - expect(debugLoggerWarnSpy).toHaveBeenCalledWith( - 'Failed to add tool result span attributes:', - expect.any(TypeError), - ); + expect( + spanRecord.spanAttributes['gen_ai.tool.call.arguments'], + ).toBeUndefined(); + expect( + spanRecord.spanAttributes['gen_ai.tool.call.result'], + ).toBeUndefined(); }); it('marks successful tool calls with OK status via endToolSpan', async () => { diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index bec27c73826..383b484cefa 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -147,15 +147,14 @@ import { endToolBlockedOnUserSpan, startHookSpan, endHookSpan, - addToolInputAttributes, - addToolResultAttributes, + addToolArgumentsAttributes, + addToolCallResultAttributes, truncateSpanError, type ToolBlockedDecision, type ToolBlockedSource, type StartHookSpanOptions, type HookSpanMetadata, } from '../telemetry/index.js'; -import { safeJsonStringify } from '../utils/safeJsonStringify.js'; import { acquireSleepInhibitor } from '../services/sleepInhibitor.js'; import { getRuntimeContentGenerator, @@ -2322,12 +2321,16 @@ export class CoreToolScheduler { // the same migration window (review-2 DeepSeek Suggestion) so // pre-Phase-2 dashboards filtering on it don't silently stop // matching during the rollout. - const toolSpan = startToolSpan(canonicalName, { - 'tool.call_id': reqInfo.callId, - 'gen_ai.tool.call.id': reqInfo.providerCallId ?? reqInfo.callId, - call_id: reqInfo.callId, - tool_name: canonicalName, - }); + const toolSpan = startToolSpan( + canonicalName, + { + 'tool.call_id': reqInfo.callId, + 'gen_ai.tool.call.id': reqInfo.providerCallId ?? reqInfo.callId, + call_id: reqInfo.callId, + tool_name: canonicalName, + }, + toolCall.tool.description, + ); this.toolSpans.set(reqInfo.callId, toolSpan); batchState.callIds.add(reqInfo.callId); this.callIdToBatch.set(reqInfo.callId, batchState); @@ -3699,12 +3702,16 @@ export class CoreToolScheduler { // grouping by span name don't see two entries for migrated/MCP tools // when this defensive fallback fires (#4321 review). const canonical = canonicalToolName(toolName); - toolSpan = startToolSpan(canonical, { - 'tool.call_id': callId, - 'gen_ai.tool.call.id': scheduledCall.request.providerCallId ?? callId, - call_id: callId, // legacy alias — see _schedule for context - tool_name: canonical, // legacy alias — see _schedule for context - }); + toolSpan = startToolSpan( + canonical, + { + 'tool.call_id': callId, + 'gen_ai.tool.call.id': scheduledCall.request.providerCallId ?? callId, + call_id: callId, // legacy alias — see _schedule for context + tool_name: canonical, // legacy alias — see _schedule for context + }, + scheduledCall.tool.description, + ); this.toolSpans.set(callId, toolSpan); } try { @@ -3855,27 +3862,22 @@ export class CoreToolScheduler { } } - private safelyAddToolInputAttributes( + private safelyAddToolArgumentsAttributes( span: Span, - toolName: string, - toolInput: string, + argumentsValue: unknown, ): void { try { - addToolInputAttributes(this.config, span, toolName, toolInput); + addToolArgumentsAttributes(this.config, span, argumentsValue); } catch (error) { - debugLogger.warn('Failed to add tool input span attributes:', error); + debugLogger.warn('Failed to add tool arguments span attribute:', error); } } - private safelyAddToolResultAttributes( - span: Span, - toolName: string, - toolResult: string, - ): void { + private safelyAddToolCallResultAttributes(span: Span, result: unknown): void { try { - addToolResultAttributes(this.config, span, toolName, toolResult); + addToolCallResultAttributes(this.config, span, result); } catch (error) { - debugLogger.warn('Failed to add tool result span attributes:', error); + debugLogger.warn('Failed to add tool result span attribute:', error); } } @@ -3908,17 +3910,6 @@ export class CoreToolScheduler { } } - // Guard the JSON serialization — addToolInputAttributes early-returns - // when sensitive attributes are off, but the argument is computed - // before the call. - if (this.config.getTelemetryIncludeSensitiveSpanAttributes?.()) { - this.safelyAddToolInputAttributes( - span, - toolName, - safeJsonStringify(toolInput) ?? '{}', - ); - } - // Generate unique tool_use_id for hook tracking. On a post-'ask' // re-execution, reuse the id from the first (bounced) attempt so the // PreToolUse event fired then pairs with the PostToolUse event fired now @@ -4005,11 +3996,6 @@ export class CoreToolScheduler { new Error(blockMessage), ToolErrorType.EXECUTION_DENIED, ); - this.safelyAddToolResultAttributes( - span, - toolName, - `BLOCKED: ${blockMessage}`, - ); this.setStatusInternal(callId, 'error', errorResponse); setToolSpanFailure( span, @@ -4144,6 +4130,7 @@ export class CoreToolScheduler { promotableShells[0]?.request.callId === callId ); }; + this.safelyAddToolArgumentsAttributes(span, invocation.params); promise = invocation.execute( execSignal, liveOutputCallback, @@ -4153,6 +4140,7 @@ export class CoreToolScheduler { canPromoteForegroundShell, ); } else { + this.safelyAddToolArgumentsAttributes(span, invocation.params); promise = invocation.execute( execSignal, liveOutputCallback, @@ -4251,11 +4239,6 @@ export class CoreToolScheduler { } failureHookArtifacts = failureHookResult.artifacts; } - this.safelyAddToolResultAttributes( - span, - toolName, - `CANCELLED: ${cancelMessage}`, - ); this.setStatusInternal( callId, 'cancelled', @@ -4353,11 +4336,6 @@ export class CoreToolScheduler { new Error(stopMessage), ToolErrorType.EXECUTION_DENIED, ); - this.safelyAddToolResultAttributes( - span, - toolName, - `STOPPED: ${stopMessage}`, - ); this.setStatusInternal(callId, 'error', errorResponse); setToolSpanFailure( span, @@ -4603,19 +4581,6 @@ export class CoreToolScheduler { } } - // Guard the JSON serialization for non-string content. Tool - // results can contain Part[] with large inlineData/media payloads - // that we don't want to serialize when telemetry is off. - if (this.config.getTelemetryIncludeSensitiveSpanAttributes?.()) { - this.safelyAddToolResultAttributes( - span, - toolName, - typeof content === 'string' - ? content - : (safeJsonStringify(content) ?? ''), - ); - } - // Recompute AFTER truncation so it reflects the model-facing length; // the final batch pass recomputes it again after aggregate reduction. contentLength = @@ -4655,6 +4620,12 @@ export class CoreToolScheduler { } catch { // OTel errors must not block API behavior. } + const result = response.find( + (part) => part.functionResponse !== undefined, + )?.functionResponse?.response; + if (result !== undefined) { + this.safelyAddToolCallResultAttributes(span, result); + } } else { // It is a failure // PostToolUseFailure Hook @@ -4724,11 +4695,6 @@ export class CoreToolScheduler { const error = part.functionResponse?.response?.['error']; return total + (typeof error === 'string' ? error.length : 0); }, 0); - this.safelyAddToolResultAttributes( - span, - toolName, - `ERROR: ${operationalErrorMessage}`, - ); const artifacts = [ ...(toolResult.artifacts ?? []), ...(failureHookArtifacts ?? []), @@ -4787,12 +4753,6 @@ export class CoreToolScheduler { ); } - this.safelyAddToolResultAttributes( - span, - toolName, - `ERROR: ${errorMessage}`, - ); - const error = new Error(errorMessage); const errorResponse = createErrorResponse( scheduledCall.request, @@ -4868,11 +4828,6 @@ export class CoreToolScheduler { } failureHookArtifacts = failureHookResult.artifacts; } - this.safelyAddToolResultAttributes( - span, - toolName, - `CANCELLED: ${cancelMessage}`, - ); this.setStatusInternal( callId, 'cancelled', @@ -4918,11 +4873,6 @@ export class CoreToolScheduler { } failureHookArtifacts = failureHookResult.artifacts; } - this.safelyAddToolResultAttributes( - span, - toolName, - `EXCEPTION: ${exceptionErrorMessage}`, - ); this.setStatusInternal( callId, 'error', diff --git a/packages/core/src/core/geminiChat.ts b/packages/core/src/core/geminiChat.ts index b9d497717aa..60410a8dd89 100644 --- a/packages/core/src/core/geminiChat.ts +++ b/packages/core/src/core/geminiChat.ts @@ -57,7 +57,6 @@ import { logApiRetry, logChatCompression, } from '../telemetry/loggers.js'; -import { clearDetailedSpanState } from '../telemetry/detailed-span-attributes.js'; import { subagentNameContext } from '../utils/subagentNameContext.js'; import { type ChatRecordingService } from '../services/chatRecordingService.js'; import { @@ -1735,7 +1734,6 @@ export class GeminiChat { this.setHistory(newHistory); debugLogger.debug('[FILE_READ_CACHE] clear after auto tryCompress'); this.config.getFileReadCache().clear(); - clearDetailedSpanState(); this.lastPromptTokenCount = info.newTokenCount; this.lastOutputTokenCount = 0; this.telemetryService?.setLastPromptTokenCount(info.newTokenCount); @@ -1832,7 +1830,6 @@ export class GeminiChat { }), ); this.setHistory(newHistory); - clearDetailedSpanState(); this.lastPromptTokenCount = adjustedTokenCount; this.telemetryService?.setLastPromptTokenCount(adjustedTokenCount); this.consecutiveFailures = 0; diff --git a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts index 9d2edda3759..0e3ef8a3280 100644 --- a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts +++ b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts @@ -9,6 +9,8 @@ import { GeminiContentGenerator } from './geminiContentGenerator.js'; import { GoogleGenAI } from '@google/genai'; const mockReportGeminiRequest = vi.hoisted(() => vi.fn()); +const mockReportGeminiResponse = vi.hoisted(() => vi.fn()); +const mockReportGeminiChunk = vi.hoisted(() => vi.fn()); vi.mock('@google/genai', () => { const mockGenerateContent = vi.fn(); @@ -29,6 +31,8 @@ vi.mock('@google/genai', () => { }); vi.mock('../../telemetry/gen-ai-request.js', () => ({ reportGeminiRequest: mockReportGeminiRequest, + reportGeminiResponse: mockReportGeminiResponse, + reportGeminiChunk: mockReportGeminiChunk, })); describe('GeminiContentGenerator', () => { @@ -83,6 +87,8 @@ describe('GeminiContentGenerator', () => { const request = { model: 'gemini-1.5-flash', contents: [] }; const expectedResponse = { responseId: 'test-id' }; mockGoogleGenAI.models.generateContent.mockResolvedValue(expectedResponse); + const telemetryAttempt = {}; + mockReportGeminiRequest.mockReturnValueOnce(telemetryAttempt); const response = await generator.generateContent(request, 'prompt-id'); @@ -102,6 +108,10 @@ describe('GeminiContentGenerator', () => { expect(mockReportGeminiRequest).toHaveBeenCalledWith( mockGoogleGenAI.models.generateContent.mock.calls[0][0], ); + expect(mockReportGeminiResponse).toHaveBeenCalledWith( + telemetryAttempt, + expectedResponse, + ); expect(response).toBe(expectedResponse); }); @@ -137,6 +147,8 @@ describe('GeminiContentGenerator', () => { yield { responseId: '1' }; })(); mockGoogleGenAI.models.generateContentStream.mockResolvedValue(mockStream); + const telemetryAttempt = {}; + mockReportGeminiRequest.mockReturnValueOnce(telemetryAttempt); const stream = await generator.generateContentStream(request, 'prompt-id'); @@ -156,7 +168,47 @@ describe('GeminiContentGenerator', () => { expect(mockReportGeminiRequest).toHaveBeenCalledWith( mockGoogleGenAI.models.generateContentStream.mock.calls[0][0], ); - expect(stream).toBe(mockStream); + expect(await stream.next()).toEqual({ + done: false, + value: { responseId: '1' }, + }); + expect(mockReportGeminiChunk).toHaveBeenCalledWith(telemetryAttempt, { + responseId: '1', + }); + }); + + it('forwards stream return without pre-consuming the SDK stream', async () => { + const next = vi.fn(); + const close = vi.fn().mockResolvedValue({ done: true, value: undefined }); + const sdkStream = { + [Symbol.asyncIterator]: () => ({ next, return: close }), + }; + mockGoogleGenAI.models.generateContentStream.mockResolvedValue(sdkStream); + + const stream = await generator.generateContentStream( + { model: 'gemini-1.5-flash', contents: [] }, + 'prompt-id', + ); + await stream.return(undefined); + + expect(next).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledTimes(1); + }); + + it('propagates SDK stream errors without reporting a chunk', async () => { + const failure = new Error('stream failed'); + const next = vi.fn().mockRejectedValue(failure); + mockGoogleGenAI.models.generateContentStream.mockResolvedValue({ + [Symbol.asyncIterator]: () => ({ next }), + }); + + const stream = await generator.generateContentStream( + { model: 'gemini-1.5-flash', contents: [] }, + 'prompt-id', + ); + + await expect(stream.next()).rejects.toBe(failure); + expect(mockReportGeminiChunk).not.toHaveBeenCalled(); }); it('should call countTokens on the underlying model', async () => { diff --git a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts index af010b14cca..b17cb6edbfc 100644 --- a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts +++ b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts @@ -22,10 +22,41 @@ import type { ContentGeneratorConfig, } from '../contentGenerator.js'; import { createDebugLogger } from '../../utils/debugLogger.js'; -import { reportGeminiRequest } from '../../telemetry/gen-ai-request.js'; +import { + reportGeminiChunk, + reportGeminiRequest, + reportGeminiResponse, + type GenAiAttemptHandle, +} from '../../telemetry/gen-ai-request.js'; const debugLogger = createDebugLogger('GEMINI'); +function observeGeminiStream( + stream: AsyncIterable, + telemetryAttempt: GenAiAttemptHandle | undefined, +): AsyncGenerator { + const iterator = stream[Symbol.asyncIterator](); + return { + async next() { + const result = await iterator.next(); + if (!result.done) reportGeminiChunk(telemetryAttempt, result.value); + return result; + }, + async return(value?: GenerateContentResponse) { + if (iterator.return) return iterator.return(value); + return { done: true, value }; + }, + async throw(error?: unknown) { + if (iterator.throw) return iterator.throw(error); + await iterator.return?.(); + throw error; + }, + [Symbol.asyncIterator]() { + return this; + }, + }; +} + /** * A wrapper for GoogleGenAI that implements the ContentGenerator interface. */ @@ -194,8 +225,11 @@ export class GeminiContentGenerator implements ContentGenerator { contents: this.stripUnsupportedFields(request.contents), config: this.buildGenerateContentConfig(request), }; - reportGeminiRequest(finalRequest); - return this.googleGenAI.models.generateContent(finalRequest); + const telemetryAttempt = reportGeminiRequest(finalRequest); + const response = + await this.googleGenAI.models.generateContent(finalRequest); + reportGeminiResponse(telemetryAttempt, response); + return response; } async generateContentStream( @@ -207,8 +241,10 @@ export class GeminiContentGenerator implements ContentGenerator { contents: this.stripUnsupportedFields(request.contents), config: this.buildGenerateContentConfig(request), }; - reportGeminiRequest(finalRequest); - return this.googleGenAI.models.generateContentStream(finalRequest); + const telemetryAttempt = reportGeminiRequest(finalRequest); + const stream = + await this.googleGenAI.models.generateContentStream(finalRequest); + return observeGeminiStream(stream, telemetryAttempt); } /** diff --git a/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.test.ts b/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.test.ts index 75f61a60d26..ed4b2c71ff9 100644 --- a/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.test.ts +++ b/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.test.ts @@ -22,10 +22,7 @@ import { logApiResponse, logApiError, } from '../../telemetry/loggers.js'; -import { - addModelOutputAttributes, - isTelemetrySdkInitialized, -} from '../../telemetry/index.js'; +import { isTelemetrySdkInitialized } from '../../telemetry/index.js'; import { OpenAILogger } from '../../utils/openaiLogger.js'; import type OpenAI from 'openai'; import { setGenAiUsageProvenance } from '../../telemetry/gen-ai-usage.js'; @@ -62,6 +59,15 @@ const loggingSpanRecords = vi.hoisted( const loggingSpanNamesWithSetStatusFailure = vi.hoisted( () => new Set(), ); +const genAiExchangeState = vi.hoisted( + (): { + controllers: Array<{ finalize: ReturnType }>; + finalizeResult: string[] | undefined; + } => ({ + controllers: [], + finalizeResult: undefined, + }), +); vi.mock('@opentelemetry/api', async (importOriginal) => { const actual = await importOriginal(); @@ -125,6 +131,16 @@ vi.mock('../../telemetry/tracer.js', () => ({ API_CALL_ABORTED_SPAN_STATUS_MESSAGE: 'API call aborted', })); +vi.mock('../../telemetry/gen-ai-request.js', () => ({ + createGenAiExchange: vi.fn((parent: unknown) => { + const controller = { + finalize: vi.fn(() => genAiExchangeState.finalizeResult), + }; + genAiExchangeState.controllers.push(controller); + return { context: parent, controller }; + }), +})); + vi.mock('../../telemetry/index.js', () => { const isTelemetrySdkInitialized = vi.fn(() => true); function createSpan( @@ -369,6 +385,8 @@ describe('LoggingContentGenerator', () => { activeOtelContext.current = 'root'; loggingSpanRecords.length = 0; loggingSpanNamesWithSetStatusFailure.clear(); + genAiExchangeState.controllers.length = 0; + genAiExchangeState.finalizeResult = undefined; }); afterEach(() => { @@ -529,6 +547,9 @@ describe('LoggingContentGenerator', () => { }); expect(spanRecord.statuses).toEqual([{ code: SpanStatusCode.OK }]); expect(spanRecord.ended).toBe(true); + expect( + genAiExchangeState.controllers.at(-1)?.finalize, + ).toHaveBeenCalledWith(true); }); it('emits output type only for Gemini and Vertex wire configurations', async () => { @@ -698,6 +719,7 @@ describe('LoggingContentGenerator', () => { }); it('forwards error metadata to endLLMRequestSpan on non-stream failure', async () => { + genAiExchangeState.finalizeResult = ['raw-error']; const wrapped = createWrappedGenerator( vi.fn().mockRejectedValue(new Error('upstream-down')), vi.fn(), @@ -720,18 +742,23 @@ describe('LoggingContentGenerator', () => { expect(spanRecord.endMetadata).toMatchObject({ success: false, error: 'API call failed', + finishReasons: ['raw-error'], }); + expect( + genAiExchangeState.controllers.at(-1)?.finalize, + ).toHaveBeenCalledWith(false); }); - it('forwards final lastUsageMetadata to endLLMRequestSpan on stream success', async () => { + it('forwards usage attached to the final response after it was yielded', async () => { + const response = createResponse('r1', 'test-model', [{ text: 'a' }]); const streamFn = vi.fn().mockResolvedValue( (async function* () { - yield createResponse('r1', 'test-model', [{ text: 'a' }]); - yield createResponse('r2', 'test-model', [{ text: 'b' }], { + yield response; + response.usageMetadata = { promptTokenCount: 100, candidatesTokenCount: 50, totalTokenCount: 150, - }); + }; })(), ); const wrapped = createWrappedGenerator(vi.fn(), streamFn); @@ -756,6 +783,52 @@ describe('LoggingContentGenerator', () => { inputTokens: 100, outputTokens: 50, }); + expect( + genAiExchangeState.controllers.at(-1)?.finalize, + ).toHaveBeenCalledWith(true); + }); + + it('retains late-attached usage when the stream subsequently fails', async () => { + const response = createResponse('r1', 'test-model', [{ text: 'a' }]); + const streamFn = vi.fn().mockResolvedValue( + (async function* () { + yield response; + response.usageMetadata = { + promptTokenCount: 10, + candidatesTokenCount: 5, + totalTokenCount: 15, + }; + throw new Error('late failure'); + })(), + ); + const generator = new LoggingContentGenerator( + createWrappedGenerator(vi.fn(), streamFn), + createConfig(), + { + model: 'test-model', + authType: AuthType.USE_OPENAI, + enableOpenAILogging: false, + }, + ); + const stream = await generator.generateContentStream( + { + model: 'test-model', + contents: 'Hello', + } as unknown as GenerateContentParameters, + 'prompt-late-usage-error', + ); + + await expect(async () => { + for await (const _ of stream) { + // consume + } + }).rejects.toThrow('late failure'); + + expect(getStreamSpanRecord().endMetadata).toMatchObject({ + success: false, + inputTokens: 10, + outputTokens: 5, + }); }); it('captures ttftMs on the first user-visible stream chunk (Phase 4a)', async () => { @@ -973,322 +1046,6 @@ describe('LoggingContentGenerator', () => { ); }); - it('passes uncapped response text to sensitive model output attributes', async () => { - const longText = 'x'.repeat(MAX_RESPONSE_TEXT_LENGTH + 100); - const wrapped = createWrappedGenerator( - vi - .fn() - .mockResolvedValue( - createResponse('resp-long', 'test-model', [{ text: longText }]), - ), - vi.fn(), - ); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ includeSensitiveSpanAttributes: true }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - await generator.generateContent(request, 'prompt-long'); - - expect(vi.mocked(addModelOutputAttributes).mock.calls.at(-1)?.[2]).toBe( - longText, - ); - const [, responseEvent] = vi.mocked(logApiResponse).mock.calls[0]; - expect(responseEvent.response_text).toHaveLength(MAX_RESPONSE_TEXT_LENGTH); - expect(responseEvent.response_text).toBe( - `${longText.slice( - 0, - MAX_RESPONSE_TEXT_LENGTH - RESPONSE_TEXT_TRUNCATION_SUFFIX.length, - )}${RESPONSE_TEXT_TRUNCATION_SUFFIX}`, - ); - }); - - it('excludes thought parts from sensitive model output attributes', async () => { - const wrapped = createWrappedGenerator( - vi - .fn() - .mockResolvedValue( - createResponse('resp-thought', 'test-model', [ - { text: 'visible' }, - { text: 'hidden thought', thought: true }, - ]), - ), - vi.fn(), - ); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ includeSensitiveSpanAttributes: true }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - await generator.generateContent(request, 'prompt-thought'); - - const sensitiveCall = vi.mocked(addModelOutputAttributes).mock.calls.at(-1); - expect(sensitiveCall?.[2]).toBe('visible'); - expect(sensitiveCall?.[3]).toBe('visible'.length); - }); - - it('bounds sensitive model output collection while preserving original length', async () => { - const longText = 'x'.repeat(MAX_RESPONSE_TEXT_LENGTH + 100); - const wrapped = createWrappedGenerator( - vi - .fn() - .mockResolvedValue( - createResponse('resp-long', 'test-model', [{ text: longText }]), - ), - vi.fn(), - ); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ - includeSensitiveSpanAttributes: true, - sensitiveSpanAttributeMaxLength: 10, - }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - await generator.generateContent(request, 'prompt-long'); - - const sensitiveCall = vi.mocked(addModelOutputAttributes).mock.calls.at(-1); - expect(sensitiveCall?.[2]).toBe('x'.repeat(10)); - expect(sensitiveCall?.[3]).toBe(longText.length); - const [, responseEvent] = vi.mocked(logApiResponse).mock.calls[0]; - expect(responseEvent.response_text).toHaveLength(MAX_RESPONSE_TEXT_LENGTH); - }); - - it('accumulates sensitive model output length across multiple parts', async () => { - const wrapped = createWrappedGenerator( - vi - .fn() - .mockResolvedValue( - createResponse('resp-multipart', 'test-model', [ - { text: 'a'.repeat(100) }, - { text: 'b'.repeat(100) }, - ]), - ), - vi.fn(), - ); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ - includeSensitiveSpanAttributes: true, - sensitiveSpanAttributeMaxLength: 150, - }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - await generator.generateContent(request, 'prompt-multipart'); - - const sensitiveCall = vi.mocked(addModelOutputAttributes).mock.calls.at(-1); - expect(sensitiveCall?.[2]).toBe('a'.repeat(100) + 'b'.repeat(50)); - expect(sensitiveCall?.[3]).toBe(200); - }); - - it('does not fail generateContent when sensitive span output attributes fail', async () => { - vi.mocked(addModelOutputAttributes).mockImplementationOnce(() => { - throw new TypeError('bad span attribute input'); - }); - const wrapped = createWrappedGenerator( - vi - .fn() - .mockResolvedValue( - createResponse('resp-safe', 'test-model', [{ text: 'visible' }]), - ), - vi.fn(), - ); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ includeSensitiveSpanAttributes: true }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - const response = await generator.generateContent(request, 'prompt-safe'); - - expect(response.responseId).toBe('resp-safe'); - expect(logApiResponse).toHaveBeenCalledTimes(1); - expect(getGenerateContentSpanRecord().statuses).toEqual([ - { code: SpanStatusCode.OK }, - ]); - }); - - it('skips sensitive model output attributes when the sensitive flag is disabled', async () => { - const wrapped = createWrappedGenerator( - vi - .fn() - .mockResolvedValue( - createResponse('resp-safe', 'test-model', [{ text: 'secret' }]), - ), - vi.fn(), - ); - const generator = new LoggingContentGenerator(wrapped, createConfig(), { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - await generator.generateContent(request, 'prompt-no-sensitive'); - - expect(addModelOutputAttributes).not.toHaveBeenCalled(); - }); - - it('skips sensitive model output attributes when telemetry SDK is not initialized', async () => { - vi.mocked(isTelemetrySdkInitialized).mockReturnValue(false); - const wrapped = createWrappedGenerator( - vi - .fn() - .mockResolvedValue( - createResponse('resp-safe', 'test-model', [{ text: 'secret' }]), - ), - vi.fn(), - ); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ includeSensitiveSpanAttributes: true }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - await generator.generateContent(request, 'prompt-no-sdk'); - - expect(addModelOutputAttributes).not.toHaveBeenCalled(); - }); - - it('passes uncapped stream response text to sensitive model output attributes', async () => { - const longText = 'x'.repeat(MAX_RESPONSE_TEXT_LENGTH + 100); - const streamFn = vi.fn().mockResolvedValue( - (async function* () { - yield createResponse('resp-long', 'test-model', [{ text: longText }]); - })(), - ); - const wrapped = createWrappedGenerator(vi.fn(), streamFn); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ includeSensitiveSpanAttributes: true }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - const stream = await generator.generateContentStream( - request, - 'prompt-long', - ); - for await (const _ of stream) { - // consume - } - - expect(vi.mocked(addModelOutputAttributes).mock.calls.at(-1)?.[2]).toBe( - longText, - ); - }); - - it('does not fail generateContentStream when sensitive span output attributes fail', async () => { - vi.mocked(addModelOutputAttributes).mockImplementationOnce(() => { - throw new TypeError('bad span attribute input'); - }); - const streamFn = vi.fn().mockResolvedValue( - (async function* () { - yield createResponse('resp-safe', 'test-model', [{ text: 'visible' }]); - })(), - ); - const wrapped = createWrappedGenerator(vi.fn(), streamFn); - const generator = new LoggingContentGenerator( - wrapped, - createConfig({ includeSensitiveSpanAttributes: true }), - { - model: 'test-model', - authType: AuthType.USE_OPENAI, - enableOpenAILogging: false, - }, - ); - - const request = { - model: 'test-model', - contents: 'Hello', - } as unknown as GenerateContentParameters; - - const stream = await generator.generateContentStream( - request, - 'prompt-safe', - ); - const responses: GenerateContentResponse[] = []; - for await (const response of stream) { - responses.push(response); - } - - expect(responses).toHaveLength(1); - expect(logApiResponse).toHaveBeenCalledTimes(1); - expect(getStreamSpanRecord().statuses).toEqual([ - { code: SpanStatusCode.OK }, - ]); - }); - it.each([ ['thought-only', [{ text: 'hidden thought', thought: true }]], [ @@ -1497,6 +1254,9 @@ describe('LoggingContentGenerator', () => { const spanRecord = getStreamSpanRecord(); expect(spanRecord.statuses).toEqual([{ code: SpanStatusCode.OK }]); expect(spanRecord.ended).toBe(true); + expect( + genAiExchangeState.controllers.at(-1)?.finalize, + ).toHaveBeenCalledWith(true); }); it('does not retain every metadata-only streaming response for logging', async () => { @@ -1969,6 +1729,9 @@ describe('LoggingContentGenerator', () => { finishReasons: ['MAX_TOKENS'], }); expect(spanRecord.ended).toBe(true); + expect( + genAiExchangeState.controllers.at(-1)?.finalize, + ).toHaveBeenCalledWith(false); releaseStream?.(); const done = await iterator.next(); @@ -2193,6 +1956,9 @@ describe('LoggingContentGenerator', () => { const spanRecord = getStreamSpanRecord(); expect(spanRecord.statuses).toEqual([{ code: SpanStatusCode.OK }]); expect(spanRecord.ended).toBe(true); + expect( + genAiExchangeState.controllers.at(-1)?.finalize, + ).toHaveBeenCalledWith(false); }); it('uses generator modalities when converting logged OpenAI requests', async () => { diff --git a/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts b/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts index e83bbae8b9b..9e348ca95c4 100644 --- a/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts +++ b/packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts @@ -53,9 +53,6 @@ import { import { startLLMRequestSpan, endLLMRequestSpan, - addSystemPromptAttributes, - addToolSchemaAttributes, - addModelOutputAttributes, areSensitiveSpanAttributesEnabled, } from '../../telemetry/index.js'; import { @@ -73,7 +70,10 @@ import { resolveGenAiProviderName, } from '../../telemetry/gen-ai-provider.js'; import { getGenAiUsageProvenance } from '../../telemetry/gen-ai-usage.js'; -import { createGenAiRequestObserverContext } from '../../telemetry/gen-ai-request.js'; +import { + createGenAiExchange, + type GenAiExchangeController, +} from '../../telemetry/gen-ai-request.js'; /** * Phase 4b — read the active retry context once, default attempt to 1 when @@ -311,31 +311,26 @@ export class LoggingContentGenerator implements ContentGenerator { // Capture span context so the API call and logging activate it via // context.with(). Without this, nested OTel spans (HTTP instrumentation, // log-bridge spans) parent to session root instead of llm_request. - const spanContext = createGenAiRequestObserverContext( + const isInternal = isInternalPromptId(userPromptId); + const exchange = createGenAiExchange( trace.setSpan(context.active(), llmSpan), llmSpan, + { + captureContent: + !isInternal && this.shouldCollectSensitiveSpanAttributes(), + sensitiveAttributeMaxLength: + this.config.getTelemetrySensitiveSpanAttributeMaxLength(), + }, ); + const spanContext = exchange.context; const startTime = Date.now(); - const isInternal = isInternalPromptId(userPromptId); const session = this.startCaptureSession(); try { runtimeDiagnostics.recordGenerateContentRequest(req, { stream: false, source: 'generateContent', }); - if (!isInternal) { - addSystemPromptAttributes( - this.config, - llmSpan, - req.config?.systemInstruction, - ); - addToolSchemaAttributes( - this.config, - llmSpan, - req.config?.tools as unknown[] | undefined, - ); - } const response = await context.with(spanContext, async () => { if (!isInternal) { this.logApiRequest( @@ -348,20 +343,9 @@ export class LoggingContentGenerator implements ContentGenerator { this.wrapped.generateContent(req, userPromptId), ); const durationMs = Date.now() - startTime; - const shouldCollectSensitiveSpanAttributes = - !isInternal && this.shouldCollectSensitiveSpanAttributes(); - const modelOutput = shouldCollectSensitiveSpanAttributes - ? this.extractResponseTextForSensitiveSpan( - result, - this.config.getTelemetrySensitiveSpanAttributeMaxLength(), - ) - : undefined; const responseText = isInternal ? undefined : this.extractResponseText(result, MAX_RESPONSE_TEXT_LENGTH); - if (shouldCollectSensitiveSpanAttributes) { - this.safelyAddModelOutputAttributes(llmSpan, modelOutput); - } this.safelyLogApiResponse( result.responseId ?? '', durationMs, @@ -382,13 +366,14 @@ export class LoggingContentGenerator implements ContentGenerator { } return result; }); + const observedFinishReasons = exchange.controller.finalize(true); endLLMRequestSpan(llmSpan, { success: true, ...usageSpanMetadata(response.usageMetadata), durationMs: Date.now() - startTime, responseId: response.responseId || undefined, responseModel: response.modelVersion || undefined, - finishReasons: orderedFinishReasons(response), + finishReasons: observedFinishReasons ?? orderedFinishReasons(response), thoughtsTokenCount: response.usageMetadata?.thoughtsTokenCount, subagentName: subagentNameContext.getStore() || undefined, ...retrySnapshot, @@ -397,6 +382,7 @@ export class LoggingContentGenerator implements ContentGenerator { return response; } catch (error) { const durationMs = Date.now() - startTime; + const observedFinishReasons = exchange.controller.finalize(false); // End the span BEFORE the (potentially-throwing) logging block, so a // logging-side rejection cannot prevent span finalization. Mirrors the // streaming path order. Use abort-specific status message when the @@ -411,6 +397,7 @@ export class LoggingContentGenerator implements ContentGenerator { : API_CALL_FAILED_SPAN_STATUS_MESSAGE, errorType: getErrorType(error), errorStatusCode: getErrorStatus(error), + finishReasons: observedFinishReasons, subagentName: subagentNameContext.getStore() || undefined, ...retrySnapshot, config: this.config, @@ -458,13 +445,20 @@ export class LoggingContentGenerator implements ContentGenerator { // Capture the span context so the stream wrapper can activate it // during iteration — not just during generator creation. - const spanContext = createGenAiRequestObserverContext( + const isInternal = isInternalPromptId(userPromptId); + const exchange = createGenAiExchange( trace.setSpan(context.active(), llmSpan), llmSpan, + { + captureContent: + !isInternal && this.shouldCollectSensitiveSpanAttributes(), + sensitiveAttributeMaxLength: + this.config.getTelemetrySensitiveSpanAttributeMaxLength(), + }, ); + const spanContext = exchange.context; const startTime = Date.now(); - const isInternal = isInternalPromptId(userPromptId); const session = this.startCaptureSession(); let stream: AsyncGenerator; @@ -473,18 +467,6 @@ export class LoggingContentGenerator implements ContentGenerator { stream: true, source: 'generateContentStream', }); - if (!isInternal) { - addSystemPromptAttributes( - this.config, - llmSpan, - req.config?.systemInstruction, - ); - addToolSchemaAttributes( - this.config, - llmSpan, - req.config?.tools as unknown[] | undefined, - ); - } stream = await context.with(spanContext, async () => { if (!isInternal) { this.logApiRequest( @@ -499,6 +481,7 @@ export class LoggingContentGenerator implements ContentGenerator { }); } catch (error) { const durationMs = Date.now() - startTime; + const observedFinishReasons = exchange.controller.finalize(false); context.with(spanContext, () => this.safelyLogApiError('', durationMs, error, req.model, userPromptId), ); @@ -511,6 +494,7 @@ export class LoggingContentGenerator implements ContentGenerator { : API_CALL_FAILED_SPAN_STATUS_MESSAGE, errorType: getErrorType(error), errorStatusCode: getErrorStatus(error), + finishReasons: observedFinishReasons, subagentName: subagentNameContext.getStore() || undefined, ...retrySnapshot, config: this.config, @@ -548,6 +532,7 @@ export class LoggingContentGenerator implements ContentGenerator { spanContext, req.config?.abortSignal, retrySnapshot, + exchange.controller, ), ); } @@ -589,6 +574,7 @@ export class LoggingContentGenerator implements ContentGenerator { // idle-timeout `setTimeout` callback sees the same values as the // entry-time read. retrySnapshot?: ReturnType, + exchangeController?: GenAiExchangeController, ): AsyncGenerator { const isInternal = isInternalPromptId(userPromptId); // Skip collecting full responses for internal prompts to avoid memory @@ -601,8 +587,15 @@ export class LoggingContentGenerator implements ContentGenerator { // values even when we skip collecting full responses for internal prompts. let firstResponseId = ''; let firstModelVersion = ''; + let lastResponse: GenerateContentResponse | undefined; let lastUsageMetadata: GenerateContentResponseUsageMetadata | undefined; + const refreshLateUsageMetadata = () => { + if (lastResponse?.usageMetadata) { + lastUsageMetadata = lastResponse.usageMetadata; + } + }; let errorOccurred = false; + let streamCompleted = false; const finishReasons = new Map(); let lastError: unknown; const subagentName = subagentNameContext.getStore(); @@ -637,11 +630,13 @@ export class LoggingContentGenerator implements ContentGenerator { if (spanEndedByTimeout) return; if (spanEndTimeout !== undefined) clearTimeout(spanEndTimeout); spanEndTimeout = setTimeout(() => { + refreshLateUsageMetadata(); try { span.setAttribute('stream.timed_out', true); } catch { // OTel errors must not interrupt the consumer. } + const observedFinishReasons = exchangeController?.finalize(false); endLLMRequestSpan(span, { success: false, ...usageSpanMetadata(lastUsageMetadata), @@ -650,11 +645,12 @@ export class LoggingContentGenerator implements ContentGenerator { responseId: firstResponseId || undefined, responseModel: firstModelVersion || undefined, finishReasons: - finishReasons.size > 0 + observedFinishReasons ?? + (finishReasons.size > 0 ? [...finishReasons.entries()] .sort(([left], [right]) => left - right) .map(([, reason]) => reason) - : undefined, + : undefined), subagentName: subagentName || undefined, ...retrySnapshot, config: this.config, @@ -668,6 +664,7 @@ export class LoggingContentGenerator implements ContentGenerator { try { for await (const response of stream) { + lastResponse = response; if (!firstResponseId && response.responseId) { firstResponseId = response.responseId; } @@ -707,6 +704,8 @@ export class LoggingContentGenerator implements ContentGenerator { resetSpanTimeout?.(); yield response; } + streamCompleted = true; + refreshLateUsageMetadata(); if (spanEndTimeout !== undefined) { clearTimeout(spanEndTimeout); spanEndTimeout = undefined; @@ -725,16 +724,6 @@ export class LoggingContentGenerator implements ContentGenerator { if (consolidatedResponse) { consolidatedResponse.usageMetadata = lastUsageMetadata; } - const shouldCollectSensitiveSpanAttributes = - !isInternal && - span !== undefined && - this.shouldCollectSensitiveSpanAttributes(); - const streamModelOutput = shouldCollectSensitiveSpanAttributes - ? this.extractResponseTextForSensitiveSpan( - consolidatedResponse, - this.config.getTelemetrySensitiveSpanAttributeMaxLength(), - ) - : undefined; const streamResponseText = isInternal ? undefined : this.extractResponseText( @@ -757,9 +746,6 @@ export class LoggingContentGenerator implements ContentGenerator { streamResponseText, ), ); - if (shouldCollectSensitiveSpanAttributes && span) { - this.safelyAddModelOutputAttributes(span, streamModelOutput); - } await runInSpan(() => this.safelyLogOpenAIInteraction( openaiRequest, @@ -802,12 +788,16 @@ export class LoggingContentGenerator implements ContentGenerator { if (spanEndTimeout !== undefined) { clearTimeout(spanEndTimeout); } + refreshLateUsageMetadata(); // If the idle timeout already ended the span, skip the redundant // endLLMRequestSpan call. The helper itself would no-op due to its // own ended guard, but we want to avoid pretending the final token // counts were recorded — they weren't, the span is the timeout one. if (span && !spanEndedByTimeout) { const aborted = abortSignal?.aborted ?? false; + const observedFinishReasons = exchangeController?.finalize( + !errorOccurred && streamCompleted, + ); endLLMRequestSpan(span, { success: !errorOccurred, ...usageSpanMetadata(lastUsageMetadata), @@ -821,11 +811,12 @@ export class LoggingContentGenerator implements ContentGenerator { responseId: firstResponseId || undefined, responseModel: firstModelVersion || undefined, finishReasons: - finishReasons.size > 0 + observedFinishReasons ?? + (finishReasons.size > 0 ? [...finishReasons.entries()] .sort(([left], [right]) => left - right) .map(([, reason]) => reason) - : undefined, + : undefined), thoughtsTokenCount: lastUsageMetadata?.thoughtsTokenCount, subagentName: subagentName || undefined, errorType: lastError ? getErrorType(lastError) : undefined, @@ -1069,27 +1060,6 @@ export class LoggingContentGenerator implements ContentGenerator { return truncated ? `${text}${RESPONSE_TEXT_TRUNCATION_SUFFIX}` : text; } - private extractResponseTextForSensitiveSpan( - response: GenerateContentResponse | undefined, - maxLength: number, - ): { text: string; originalLength: number } | undefined { - let text = ''; - let originalLength = 0; - const hasText = this.forEachVisibleResponseText(response, (partText) => { - originalLength += partText.length; - const remaining = maxLength - text.length; - if (remaining > 0) { - text += partText.slice(0, remaining); - } - }); - - if (!hasText) { - return undefined; - } - - return { text, originalLength }; - } - private forEachVisibleResponseText( response: GenerateContentResponse | undefined, onText: (text: string) => void, @@ -1126,22 +1096,6 @@ export class LoggingContentGenerator implements ContentGenerator { return undefined; } - private safelyAddModelOutputAttributes( - span: Span, - modelOutput: { text: string; originalLength: number } | undefined, - ): void { - try { - addModelOutputAttributes( - this.config, - span, - modelOutput?.text, - modelOutput?.originalLength, - ); - } catch (error) { - debugLogger.warn('Failed to add model output span attributes:', error); - } - } - private shouldCollectSensitiveSpanAttributes(): boolean { return areSensitiveSpanAttributesEnabled(this.config); } diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts index 3289f370f82..d3e8b03276a 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts @@ -42,6 +42,8 @@ import { setToolCallPreparations } from '../tool-call-preparation.js'; // Mock dependencies const mockReportOpenAiRequest = vi.hoisted(() => vi.fn()); +const mockReportOpenAiResponse = vi.hoisted(() => vi.fn()); +const mockReportOpenAiChunk = vi.hoisted(() => vi.fn()); vi.mock('./converter.js', () => ({ OpenAIContentConverter: { @@ -57,6 +59,8 @@ vi.mock('../../telemetry/loggers.js', () => ({ })); vi.mock('../../telemetry/gen-ai-request.js', () => ({ reportOpenAiRequest: mockReportOpenAiRequest, + reportOpenAiResponse: mockReportOpenAiResponse, + reportOpenAiChunk: mockReportOpenAiChunk, })); describe('ContentGenerationPipeline', () => { @@ -162,6 +166,8 @@ describe('ContentGenerationPipeline', () => { (mockClient.chat.completions.create as Mock).mockResolvedValue( mockOpenAIResponse, ); + const telemetryAttempt = {}; + mockReportOpenAiRequest.mockReturnValueOnce(telemetryAttempt); // Act const result = await pipeline.execute(request, userPromptId); @@ -191,6 +197,10 @@ describe('ContentGenerationPipeline', () => { expect(mockReportOpenAiRequest).toHaveBeenCalledWith( vi.mocked(mockClient.chat.completions.create).mock.calls[0]![0], ); + expect(mockReportOpenAiResponse).toHaveBeenCalledWith( + telemetryAttempt, + mockOpenAIResponse, + ); expect(mockConverter.convertOpenAIResponseToGemini).toHaveBeenCalledWith( mockOpenAIResponse, expect.objectContaining({ @@ -2134,6 +2144,8 @@ describe('ContentGenerationPipeline', () => { (mockClient.chat.completions.create as Mock).mockResolvedValue( mockStream, ); + const telemetryAttempt = {}; + mockReportOpenAiRequest.mockReturnValueOnce(telemetryAttempt); // Act const resultGenerator = await pipeline.executeStream( @@ -2178,6 +2190,16 @@ describe('ContentGenerationPipeline', () => { expect(mockReportOpenAiRequest).toHaveBeenCalledWith( vi.mocked(mockClient.chat.completions.create).mock.calls[0]![0], ); + expect(mockReportOpenAiChunk).toHaveBeenNthCalledWith( + 1, + telemetryAttempt, + mockChunk1, + ); + expect(mockReportOpenAiChunk).toHaveBeenNthCalledWith( + 2, + telemetryAttempt, + mockChunk2, + ); }); it('should filter empty responses', async () => { diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.ts b/packages/core/src/core/openaiContentGenerator/pipeline.ts index d55ec12a4ec..04b9837a434 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.ts @@ -33,7 +33,12 @@ import { logProtocolTagSanitized } from '../../telemetry/loggers.js'; import { ProtocolTagSanitizedEvent } from '../../telemetry/types.js'; import { getErrorMessage, getErrorStatus } from '../../utils/errors.js'; import { getRateLimitErrorDetails } from '../../utils/rateLimit.js'; -import { reportOpenAiRequest } from '../../telemetry/gen-ai-request.js'; +import { + reportOpenAiChunk, + reportOpenAiRequest, + reportOpenAiResponse, + type GenAiAttemptHandle, +} from '../../telemetry/gen-ai-request.js'; const debugLogger = createDebugLogger('OPENAI_PIPELINE'); @@ -315,7 +320,7 @@ export class ContentGenerationPipeline { request, userPromptId, false, - async (openaiRequest, context) => { + async (openaiRequest, context, telemetryAttempt) => { // Wrap in a per-request child so the OpenAI SDK's leaked abort // listener (client.mjs fetchWithTimeout — no {once:true}, no // removeEventListener) stays on a short-lived signal instead of @@ -331,6 +336,7 @@ export class ContentGenerationPipeline { signal: perRequestAc?.signal, }, )) as OpenAI.Chat.ChatCompletion; + reportOpenAiResponse(telemetryAttempt, openaiResponse); const geminiResponse = OpenAIContentConverter.convertOpenAIResponseToGemini( @@ -354,7 +360,7 @@ export class ContentGenerationPipeline { request, userPromptId, true, - async (openaiRequest, context) => { + async (openaiRequest, context, telemetryAttempt) => { // Always use a per-request controller so the inactivity watchdog can // abort the SDK request even when the caller did not provide a signal. const parentSignal = request.config?.abortSignal; @@ -459,6 +465,7 @@ export class ContentGenerationPipeline { context, request, userPromptId, + telemetryAttempt, ); async function* drainThenCleanup(): AsyncGenerator { try { @@ -485,6 +492,7 @@ export class ContentGenerationPipeline { context: RequestContext, request: GenerateContentParameters, userPromptId: string, + telemetryAttempt: GenAiAttemptHandle | undefined, ): AsyncGenerator { // State for handling chunk merging. // pendingFinishResponse holds a finish chunk waiting to be merged with @@ -525,6 +533,7 @@ export class ContentGenerationPipeline { try { // Stage 2a: Convert and yield each chunk while preserving original for await (const chunk of stream) { + reportOpenAiChunk(telemetryAttempt, chunk); // Detect API errors returned as stream content. // Some providers return errors (e.g., TPM throttling) as a normal SSE chunk // with finish_reason="error_finish" and the error in delta.content, @@ -1103,6 +1112,7 @@ export class ContentGenerationPipeline { executor: ( openaiRequest: OpenAI.Chat.ChatCompletionCreateParams, context: RequestContext, + telemetryAttempt: GenAiAttemptHandle | undefined, ) => Promise, ): Promise { const context = this.createRequestContext(request, isStreaming); @@ -1120,9 +1130,9 @@ export class ContentGenerationPipeline { // so the logger sees the exact bytes sent on the wire. openaiRequestCaptureContext.getStore()?.(openaiRequest); runtimeDiagnostics.recordOpenAIWireRequest(openaiRequest); - reportOpenAiRequest(openaiRequest); + const telemetryAttempt = reportOpenAiRequest(openaiRequest); - return executor(openaiRequest, context); + return executor(openaiRequest, context, telemetryAttempt); }; try { diff --git a/packages/core/src/telemetry/detailed-span-attributes.test.ts b/packages/core/src/telemetry/detailed-span-attributes.test.ts index 3a3a1845364..59fd89de515 100644 --- a/packages/core/src/telemetry/detailed-span-attributes.test.ts +++ b/packages/core/src/telemetry/detailed-span-attributes.test.ts @@ -4,8 +4,22 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { describe, it, expect, vi, beforeEach } from 'vitest'; -import type { Span, Attributes, SpanContext } from '@opentelemetry/api'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Attributes, Span, SpanContext } from '@opentelemetry/api'; +import type { Config } from '../config/config.js'; +import { + addModelOutputAttributes, + addSystemPromptAttributes, + addToolArgumentsAttributes, + addToolCallResultAttributes, + addToolInputAttributes, + addToolResultAttributes, + addToolSchemaAttributes, + addUserPromptAttributes, + clearDetailedSpanState, + truncateContent, +} from './detailed-span-attributes.js'; +import { DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH } from './constants.js'; const mockState = vi.hoisted(() => ({ sdkInitialized: true, @@ -17,20 +31,11 @@ vi.mock('./sdk.js', () => ({ isTelemetrySdkInitialized: () => mockState.sdkInitialized, })); -import type { Config } from '../config/config.js'; -import { - truncateContent, - addUserPromptAttributes, - addSystemPromptAttributes, - addToolSchemaAttributes, - addModelOutputAttributes, - addToolInputAttributes, - addToolResultAttributes, - clearDetailedSpanState, -} from './detailed-span-attributes.js'; -import { DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH } from './constants.js'; +interface MockSpan extends Span { + attrs: Record; +} -function createMockConfig(): Config { +function config(): Config { return { getTelemetryIncludeSensitiveSpanAttributes: () => mockState.sensitiveEnabled, @@ -38,31 +43,19 @@ function createMockConfig(): Config { } as unknown as Config; } -interface MockSpan extends Span { - attrs: Record; - events: Array<{ name: string; attributes: Record }>; -} - -function createMockSpan(): MockSpan { +function span(): MockSpan { const attrs: Record = {}; - const events: Array<{ name: string; attributes: Record }> = - []; return { attrs, - events, - setAttributes(a: Attributes) { - Object.assign(attrs, a); + setAttributes(values: Attributes) { + Object.assign(attrs, values); return this; }, setAttribute(key: string, value: unknown) { attrs[key] = value; return this; }, - addEvent(name: string, eventAttrs?: Attributes) { - events.push({ - name, - attributes: (eventAttrs ?? {}) as Record, - }); + addEvent() { return this; }, spanContext(): SpanContext { @@ -94,520 +87,201 @@ function createMockSpan(): MockSpan { }; } -describe('detailed-span-attributes', () => { +describe('detailed span attributes', () => { beforeEach(() => { mockState.sdkInitialized = true; mockState.sensitiveEnabled = true; - mockState.maxLength = 1024 * 1024; - clearDetailedSpanState(); + mockState.maxLength = DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH; }); describe('truncateContent', () => { - it('returns content as-is when under limit', () => { - const result = truncateContent( - 'hello', - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH, - ); - expect(result.content).toBe('hello'); - expect(result.truncated).toBe(false); + it('preserves content at or below the limit', () => { + expect(truncateContent('hello', 5)).toEqual({ + content: 'hello', + truncated: false, + }); }); - it('uses the default 1MiB limit when maxSize is omitted', () => { - const largeContent = 'a'.repeat(1024 * 1024 + 1); - const result = truncateContent(largeContent); - - expect(result.truncated).toBe(true); - expect(result.content.length).toBe( - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH, + it('uses the default 1 MiB limit', () => { + const content = 'x'.repeat( + DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH + 1, ); - expect(result.content).toContain( - 'configured limit of 1048576 characters', + const result = truncateContent(content); + expect(result.content).toHaveLength( + DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH, ); - }); - - it('uses the supplied original length for prebounded content', () => { - const result = truncateContent('abc', 3, 6); - expect(result.truncated).toBe(true); - expect(result.content).toBe('abc'); }); - it('throws when originalLength is shorter than the provided content', () => { - expect(() => truncateContent('abcd', 3, 2)).toThrow(TypeError); - }); - - it('truncates content over limit', () => { - const result = truncateContent('x'.repeat(200), 100); + it('bounds oversized content and marks it truncated', () => { + const result = truncateContent('x'.repeat(100), 50); + expect(result.content).toHaveLength(50); expect(result.truncated).toBe(true); - expect(result.content.length).toBe(100); - expect(result.content).toContain('[TRUNCATED'); - expect(result.content).toContain('configured limit of 100 characters'); }); - it('keeps truncated content within small configured limits', () => { - const result = truncateContent('x'.repeat(100), 50); - expect(result.truncated).toBe(true); - expect(result.content).toHaveLength(50); - expect(result.content).toMatch(/\.\.\.\[TRUNCATED\]$/); + it('keeps a visible marker within very small limits', () => { + expect(truncateContent('long', 3)).toEqual({ + content: '...', + truncated: true, + }); }); - it('uses a visible truncation marker even when only marker prefix fits', () => { - const result = truncateContent('x'.repeat(100), 3); - expect(result.truncated).toBe(true); - expect(result.content).toBe('...'); + it('accepts a prebounded prefix with a larger original length', () => { + expect(truncateContent('abc', 3, 6)).toEqual({ + content: 'abc', + truncated: true, + }); }); - it('does not truncate content exactly at the limit', () => { - const result = truncateContent('a'.repeat(50), 50); - expect(result.truncated).toBe(false); - expect(result.content).toBe('a'.repeat(50)); + it('rejects an original length shorter than the content', () => { + expect(() => truncateContent('abcd', 4, 3)).toThrow(TypeError); }); it.each([0, -1, Number.NaN, Number.POSITIVE_INFINITY])( - 'throws when maxSize is invalid: %s', - (maxSize) => { - expect(() => truncateContent('abc', maxSize)).toThrow(TypeError); + 'rejects invalid maximum %s', + (maxLength) => { + expect(() => truncateContent('x', maxLength)).toThrow(TypeError); }, ); - - it('does not truncate 70KB content with the default 1MiB limit', () => { - const largeContent = 'a'.repeat(70_000); - const result = truncateContent( - largeContent, - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH, - ); - expect(result.truncated).toBe(false); - expect(result.content.length).toBe(largeContent.length); - }); - - it('truncates content over the default 1MiB limit', () => { - const largeContent = 'a'.repeat(1024 * 1024 + 1); - const result = truncateContent( - largeContent, - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH, - ); - expect(result.truncated).toBe(true); - expect(result.content.length).toBe( - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH, - ); - expect(result.content).toContain('[TRUNCATED'); - }); }); - describe('addUserPromptAttributes', () => { - it('sets new_context with user prompt prefix', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addUserPromptAttributes(config, span, 'Hello world'); - - expect(span.attrs['new_context']).toBe('[USER PROMPT]\nHello world'); - }); - - it('no-ops when flag is disabled', () => { - mockState.sensitiveEnabled = false; - const config = createMockConfig(); - const span = createMockSpan(); - addUserPromptAttributes(config, span, 'Hello world'); - - expect(span.attrs['new_context']).toBeUndefined(); - }); - - it('no-ops when SDK is not initialized', () => { - mockState.sdkInitialized = false; - const config = createMockConfig(); - const span = createMockSpan(); - addUserPromptAttributes(config, span, 'Hello world'); - - expect(span.attrs['new_context']).toBeUndefined(); - }); - - it('no-ops when promptText is empty', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addUserPromptAttributes(config, span, ''); - - expect(span.attrs['new_context']).toBeUndefined(); - }); - - it('sets truncation attributes for content over the default limit', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const largePrompt = 'x'.repeat(1024 * 1024 + 1); - addUserPromptAttributes(config, span, largePrompt); - - expect(String(span.attrs['new_context'])).toHaveLength( - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH, - ); - expect(span.attrs['new_context_truncated']).toBe(true); - expect(span.attrs['new_context_original_length']).toBe(1024 * 1024 + 1); - }); - - it('uses configured max length for user prompt attributes', () => { - mockState.maxLength = 50; - const config = createMockConfig(); - const span = createMockSpan(); - const prompt = 'x'.repeat(100); - addUserPromptAttributes(config, span, prompt); - - expect(String(span.attrs['new_context'])).toHaveLength(50); - expect(span.attrs['new_context_truncated']).toBe(true); - expect(span.attrs['new_context_original_length']).toBe(100); - }); + it('keeps interaction new_context behavior unchanged', () => { + const target = span(); + addUserPromptAttributes(config(), target, 'hello'); + expect(target.attrs['new_context']).toBe('[USER PROMPT]\nhello'); }); - it('uses configured max length for all native sensitive span payloads', () => { - mockState.maxLength = 3; - const config = createMockConfig(); + it('keeps interaction truncation metadata', () => { + mockState.maxLength = 20; + const target = span(); + addUserPromptAttributes(config(), target, 'x'.repeat(100)); + expect(String(target.attrs['new_context'])).toHaveLength(20); + expect(target.attrs['new_context_truncated']).toBe(true); + expect(target.attrs['new_context_original_length']).toBe(100); + }); - const userSpan = createMockSpan(); - addUserPromptAttributes(config, userSpan, 'abcdef'); - expect(userSpan.attrs['new_context']).toBe('...'); - expect(userSpan.attrs['new_context_truncated']).toBe(true); - expect(userSpan.attrs['new_context_original_length']).toBe('abcdef'.length); + it('omits interaction content when disabled, uninitialized, or empty', () => { + const target = span(); + mockState.sensitiveEnabled = false; + addUserPromptAttributes(config(), target, 'disabled'); + mockState.sensitiveEnabled = true; + mockState.sdkInitialized = false; + addUserPromptAttributes(config(), target, 'uninitialized'); + mockState.sdkInitialized = true; + addUserPromptAttributes(config(), target, ''); + expect(target.attrs).toEqual({}); + }); - const systemSpan = createMockSpan(); - addSystemPromptAttributes(config, systemSpan, 'abcdef'); - expect(systemSpan.attrs['system_prompt']).toBe('...'); - expect(systemSpan.attrs['system_prompt_truncated']).toBe(true); + it('writes compatibility helpers only to standard GenAI keys', () => { + const target = span(); + addSystemPromptAttributes(config(), target, 'system'); + addToolSchemaAttributes(config(), target, [ + { type: 'function', name: 'read' }, + ]); + addModelOutputAttributes(config(), target, 'answer', 'stop'); + addToolInputAttributes(config(), target, 'read', '{"path":"a"}'); + addToolResultAttributes(config(), target, 'read', '{"output":"ok"}'); - const toolSchemaSpan = createMockSpan(); - const toolDeclaration = { name: 'Read', description: 'abcdef' }; - addToolSchemaAttributes(config, toolSchemaSpan, [toolDeclaration]); - expect(toolSchemaSpan.events[0]!.attributes['tool_definition']).toBe('...'); expect( - toolSchemaSpan.events[0]!.attributes['tool_definition_truncated'], - ).toBe(true); + JSON.parse(target.attrs['gen_ai.system_instructions'] as string), + ).toEqual([{ type: 'text', content: 'system' }]); expect( - toolSchemaSpan.events[0]!.attributes['tool_definition_original_length'], - ).toBe(JSON.stringify(toolDeclaration).length); - - const modelSpan = createMockSpan(); - addModelOutputAttributes(config, modelSpan, 'abcdef'); - expect(modelSpan.attrs['response.model_output']).toBe('...'); - expect(modelSpan.attrs['response.model_output_truncated']).toBe(true); - - const toolInputSpan = createMockSpan(); - addToolInputAttributes(config, toolInputSpan, 'Bash', 'abcdef'); - expect(toolInputSpan.attrs['tool_input']).toBe('...'); - expect(toolInputSpan.attrs['tool_input_truncated']).toBe(true); - expect(toolInputSpan.attrs['tool_input_original_length']).toBe( - 'abcdef'.length, - ); - - const toolResultSpan = createMockSpan(); - addToolResultAttributes(config, toolResultSpan, 'Read', 'abcdef'); - expect(toolResultSpan.attrs['tool_result']).toBe('...'); - expect(toolResultSpan.attrs['tool_result_truncated']).toBe(true); - expect(toolResultSpan.attrs['tool_result_original_length']).toBe( - 'abcdef'.length, + JSON.parse(target.attrs['gen_ai.tool.definitions'] as string), + ).toEqual([{ type: 'function', name: 'read' }]); + expect( + JSON.parse(target.attrs['gen_ai.output.messages'] as string), + ).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'answer' }], + finish_reason: 'stop', + }, + ]); + expect(target.attrs['gen_ai.tool.call.arguments']).toBe('{"path":"a"}'); + expect(target.attrs['gen_ai.tool.call.result']).toBe('{"output":"ok"}'); + expect(Object.keys(target.attrs)).not.toEqual( + expect.arrayContaining([ + 'system_prompt', + 'tools', + 'response.model_output', + 'tool_input', + 'tool_result', + ]), ); }); - describe('addSystemPromptAttributes', () => { - it('sets hash, preview, and length', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addSystemPromptAttributes(config, span, 'System prompt content'); - - expect(span.attrs['system_prompt_hash']).toMatch(/^sp_[a-f0-9]{12}$/); - expect(span.attrs['system_prompt_preview']).toBe('System prompt content'); - expect(span.attrs['system_prompt_length']).toBe(21); - expect(span.attrs['system_prompt']).toBe('System prompt content'); - }); - - it('deduplicates full content on same hash', () => { - const config = createMockConfig(); - const span1 = createMockSpan(); - const span2 = createMockSpan(); - - addSystemPromptAttributes(config, span1, 'Same prompt'); - addSystemPromptAttributes(config, span2, 'Same prompt'); - - expect(span1.attrs['system_prompt']).toBe('Same prompt'); - expect(span2.attrs['system_prompt']).toBeUndefined(); - expect(span2.attrs['system_prompt_hash']).toBeDefined(); - }); - - it('handles non-string systemInstruction', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addSystemPromptAttributes(config, span, { text: 'obj prompt' }); - - expect(span.attrs['system_prompt_hash']).toMatch(/^sp_/); - expect(span.attrs['system_prompt_length']).toBeGreaterThan(0); - }); - - it('sets system_prompt_truncated for content over the default limit', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const largePrompt = 'p'.repeat(1024 * 1024 + 1); - addSystemPromptAttributes(config, span, largePrompt); - - expect(span.attrs['system_prompt_truncated']).toBe(true); - expect(span.attrs['system_prompt_length']).toBe(1024 * 1024 + 1); - }); - - it('no-ops when flag is disabled', () => { - mockState.sensitiveEnabled = false; - const config = createMockConfig(); - const span = createMockSpan(); - addSystemPromptAttributes(config, span, 'prompt'); - - expect(span.attrs['system_prompt_hash']).toBeUndefined(); - }); + it('does not synthesize output without a finish reason', () => { + const target = span(); + addModelOutputAttributes(config(), target, 'answer'); + expect(target.attrs['gen_ai.output.messages']).toBeUndefined(); }); - describe('addToolSchemaAttributes', () => { - it('sets tools summary and count', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const tools = [ - { name: 'Read', description: 'Read a file' }, - { name: 'Bash', description: 'Execute command' }, - ]; - - addToolSchemaAttributes(config, span, tools); - - expect(span.attrs['tools_count']).toBe(2); - const toolsSummary = JSON.parse(span.attrs['tools'] as string); - expect(toolsSummary).toHaveLength(2); - expect(toolsSummary[0].name).toBe('Read'); - expect(toolsSummary[1].name).toBe('Bash'); - }); - - it('emits tool_schema events for first occurrence', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const tools = [{ name: 'Read', description: 'Read a file' }]; - - addToolSchemaAttributes(config, span, tools); - - expect(span.events).toHaveLength(1); - expect(span.events[0]!.name).toBe('tool_schema'); - expect(span.events[0]!.attributes['tool_name']).toBe('Read'); - expect( - span.events[0]!.attributes['tool_definition_original_length'], - ).toBeUndefined(); - expect( - span.events[0]!.attributes['tool_definition_truncated'], - ).toBeUndefined(); - }); - - it('deduplicates tool schema events', () => { - const config = createMockConfig(); - const span1 = createMockSpan(); - const span2 = createMockSpan(); - const tools = [{ name: 'Read', description: 'Read a file' }]; - - addToolSchemaAttributes(config, span1, tools); - addToolSchemaAttributes(config, span2, tools); - - expect(span1.events).toHaveLength(1); - expect(span2.events).toHaveLength(0); - }); - - it('falls back to unknown_tool when tool has no name', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addToolSchemaAttributes(config, span, [{ description: 'no name field' }]); - - expect(span.events).toHaveLength(1); - expect(span.events[0]!.attributes['tool_name']).toBe('unknown_tool'); - const toolsSummary = JSON.parse(span.attrs['tools'] as string); - expect(toolsSummary[0].name).toBe('unknown_tool'); - }); - - it('flattens functionDeclarations wrapper (Gemini API shape)', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const tools = [ - { - functionDeclarations: [ - { name: 'Read', description: 'Read a file' }, - { name: 'Bash', description: 'Execute command' }, - ], - }, - ]; - - addToolSchemaAttributes(config, span, tools); - - expect(span.attrs['tools_count']).toBe(2); - const toolsSummary = JSON.parse(span.attrs['tools'] as string); - expect(toolsSummary.map((t: { name: string }) => t.name)).toEqual([ - 'Read', - 'Bash', - ]); - expect(span.events).toHaveLength(2); - expect(span.events[0]!.attributes['tool_name']).toBe('Read'); - expect(span.events[1]!.attributes['tool_name']).toBe('Bash'); - }); - - it('no-ops on empty tools array', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addToolSchemaAttributes(config, span, []); - - expect(span.attrs['tools_count']).toBeUndefined(); - }); - - it('no-ops on undefined tools', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addToolSchemaAttributes(config, span, undefined); - - expect(span.attrs['tools_count']).toBeUndefined(); - }); + it('requires object roots for tool arguments and results', () => { + const target = span(); + addToolArgumentsAttributes(config(), target, []); + addToolCallResultAttributes(config(), target, 'result'); + expect(target.attrs['gen_ai.tool.call.arguments']).toBeUndefined(); + expect(target.attrs['gen_ai.tool.call.result']).toBeUndefined(); }); - describe('addModelOutputAttributes', () => { - it('sets response.model_output', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addModelOutputAttributes(config, span, 'Model says hello'); - - expect(span.attrs['response.model_output']).toBe('Model says hello'); - }); - - it('uses the supplied original length for prebounded output', () => { - const config = createMockConfig(); - const span = createMockSpan(); - mockState.maxLength = 3; - addModelOutputAttributes(config, span, 'abc', 6); - - expect(span.attrs['response.model_output']).toBe('abc'); - expect(span.attrs['response.model_output_truncated']).toBe(true); - expect(span.attrs['response.model_output_original_length']).toBe(6); - }); - - it('does not append a truncation suffix to prebounded output', () => { - const config = createMockConfig(); - const span = createMockSpan(); - mockState.maxLength = 100; - addModelOutputAttributes(config, span, 'x'.repeat(100), 200); - - expect(span.attrs['response.model_output']).toBe('x'.repeat(100)); - expect(span.attrs['response.model_output_truncated']).toBe(true); - expect(span.attrs['response.model_output_original_length']).toBe(200); - }); - - it('sets truncation attributes for output over the default limit', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const largeOutput = 'y'.repeat(1024 * 1024 + 1); - addModelOutputAttributes(config, span, largeOutput); - - expect(span.attrs['response.model_output_truncated']).toBe(true); - expect(span.attrs['response.model_output_original_length']).toBe( - 1024 * 1024 + 1, - ); - }); - - it('no-ops when responseText is undefined', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addModelOutputAttributes(config, span, undefined); - - expect(span.attrs['response.model_output']).toBeUndefined(); - }); + it('preserves empty tool objects', () => { + const target = span(); + addToolArgumentsAttributes(config(), target, {}); + addToolCallResultAttributes(config(), target, {}); + expect(target.attrs['gen_ai.tool.call.arguments']).toBe('{}'); + expect(target.attrs['gen_ai.tool.call.result']).toBe('{}'); }); - describe('addToolInputAttributes', () => { - it('sets tool_input with prefix', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addToolInputAttributes(config, span, 'Bash', '{"command":"ls"}'); - - expect(span.attrs['tool_input']).toBe( - '[TOOL INPUT: Bash]\n{"command":"ls"}', - ); - }); - - it('sets truncation attributes for input over the default limit', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const largeInput = 'i'.repeat( - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH + 1, - ); - addToolInputAttributes(config, span, 'Bash', largeInput); - - expect(span.attrs['tool_input_truncated']).toBe(true); - expect(span.attrs['tool_input_original_length']).toBe( - DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH + 1, - ); - }); - - it('keeps tool_input within the configured limit when tool name is long', () => { - const config = createMockConfig(); - const span = createMockSpan(); - mockState.maxLength = 50; - const toolName = 'n'.repeat(100); - addToolInputAttributes(config, span, toolName, '{"command":"ls"}'); - - expect(String(span.attrs['tool_input'])).toHaveLength(50); - expect(span.attrs['tool_input_truncated']).toBe(true); - expect(span.attrs['tool_input_original_length']).toBe( - '{"command":"ls"}'.length, - ); - }); - - it('no-ops when flag is disabled', () => { - mockState.sensitiveEnabled = false; - const config = createMockConfig(); - const span = createMockSpan(); - addToolInputAttributes(config, span, 'Bash', '{"command":"ls"}'); - - expect(span.attrs['tool_input']).toBeUndefined(); - }); + it('omits complete JSON attributes that exceed the configured limit', () => { + mockState.maxLength = 5; + const target = span(); + addToolArgumentsAttributes(config(), target, { value: 'too long' }); + expect(target.attrs['gen_ai.tool.call.arguments']).toBeUndefined(); }); - describe('addToolResultAttributes', () => { - it('sets tool_result with prefix', () => { - const config = createMockConfig(); - const span = createMockSpan(); - addToolResultAttributes(config, span, 'Read', 'file contents here'); - - expect(span.attrs['tool_result']).toBe( - '[TOOL RESULT: Read]\nfile contents here', - ); - }); - - it('sets truncation attributes for result over the default limit', () => { - const config = createMockConfig(); - const span = createMockSpan(); - const largeResult = 'z'.repeat(1024 * 1024 + 1); - addToolResultAttributes(config, span, 'Read', largeResult); - - expect(span.attrs['tool_result_truncated']).toBe(true); - expect(span.attrs['tool_result_original_length']).toBe(1024 * 1024 + 1); - }); - - it('keeps tool_result within the configured limit when tool name is long', () => { - const config = createMockConfig(); - const span = createMockSpan(); - mockState.maxLength = 50; - const toolName = 'n'.repeat(100); - addToolResultAttributes(config, span, toolName, 'file contents here'); - - expect(String(span.attrs['tool_result'])).toHaveLength(50); - expect(span.attrs['tool_result_truncated']).toBe(true); - expect(span.attrs['tool_result_original_length']).toBe( - 'file contents here'.length, - ); - }); + it('omits cyclic values without affecting the caller', () => { + const value: Record = {}; + value['self'] = value; + const target = span(); + expect(() => + addToolArgumentsAttributes(config(), target, value), + ).not.toThrow(); + expect(target.attrs['gen_ai.tool.call.arguments']).toBeUndefined(); }); - describe('clearDetailedSpanState', () => { - it('resets seenHashes so system prompt is emitted again', () => { - const config = createMockConfig(); - const span1 = createMockSpan(); - addSystemPromptAttributes(config, span1, 'Same prompt'); - expect(span1.attrs['system_prompt']).toBe('Same prompt'); + it('does not let serializer or Span API failures affect the caller', () => { + const uninspectable = new Proxy( + {}, + { + ownKeys() { + throw new Error('cannot inspect'); + }, + }, + ); + const target = span(); + target.setAttribute = () => { + throw new Error('cannot set'); + }; + + expect(() => + addToolArgumentsAttributes(config(), target, uninspectable), + ).not.toThrow(); + expect(() => + addToolCallResultAttributes(config(), target, { output: 'ok' }), + ).not.toThrow(); + }); - clearDetailedSpanState(); + it('honors the sensitive-data switch and SDK state', () => { + const target = span(); + mockState.sensitiveEnabled = false; + addToolArgumentsAttributes(config(), target, { secret: true }); + mockState.sensitiveEnabled = true; + mockState.sdkInitialized = false; + addToolCallResultAttributes(config(), target, { secret: true }); + expect(target.attrs).toEqual({}); + }); - const span2 = createMockSpan(); - addSystemPromptAttributes(config, span2, 'Same prompt'); - expect(span2.attrs['system_prompt']).toBe('Same prompt'); - }); + it('keeps the old state reset export as a no-op', () => { + expect(() => clearDetailedSpanState()).not.toThrow(); }); }); diff --git a/packages/core/src/telemetry/detailed-span-attributes.ts b/packages/core/src/telemetry/detailed-span-attributes.ts index e8e617d93c9..91a61cf7764 100644 --- a/packages/core/src/telemetry/detailed-span-attributes.ts +++ b/packages/core/src/telemetry/detailed-span-attributes.ts @@ -4,19 +4,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { createHash } from 'node:crypto'; import type { Span } from '@opentelemetry/api'; import type { Config } from '../config/config.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; import { isTelemetrySdkInitialized } from './sdk.js'; -import { safeJsonStringify } from '../utils/safeJsonStringify.js'; import { DEFAULT_SENSITIVE_SPAN_ATTRIBUTE_MAX_LENGTH } from './constants.js'; +import { extractGeminiContent, stringifyGenAiJson } from './gen-ai-content.js'; -const SYSTEM_PROMPT_PREVIEW_LENGTH = 500; const SHORT_TRUNCATION_SUFFIX = '...[TRUNCATED]'; - -// Process-global; intentionally never cleared in production. Bounded by the -// number of unique system prompts + tool schemas seen in one session. -const seenHashes = new Set(); +const debugLogger = createDebugLogger('GEN_AI_CONTENT'); export function areSensitiveSpanAttributesEnabled(config: Config): boolean { return ( @@ -92,15 +88,6 @@ function truncatePrefixedContent( }; } -function shortHash(content: string): string { - return createHash('sha256').update(content).digest('hex').slice(0, 12); -} - -function stringifyContentUnion(value: unknown): string { - if (typeof value === 'string') return value; - return safeJsonStringify(value) ?? ''; -} - // --- Interaction Span: User Prompt --- export function addUserPromptAttributes( @@ -124,172 +111,166 @@ export function addUserPromptAttributes( }); } -// --- LLM Request Span: System Prompt --- - +/** + * @deprecated Provider-final requests are captured by the GenAI exchange. + */ export function addSystemPromptAttributes( config: Config, span: Span, systemInstruction: unknown, ): void { - if (!areSensitiveSpanAttributesEnabled(config) || !systemInstruction) return; - - const text = stringifyContentUnion(systemInstruction); - if (!text) return; - - const hash = `sp_${shortHash(text)}`; - span.setAttributes({ - system_prompt_hash: hash, - system_prompt_preview: text.slice(0, SYSTEM_PROMPT_PREVIEW_LENGTH), - system_prompt_length: text.length, - }); - - if (!seenHashes.has(hash)) { - seenHashes.add(hash); - const { content, truncated } = truncateContent( - text, - getMaxContentSize(config), - ); - span.setAttribute('system_prompt', content); - if (truncated) { - span.setAttribute('system_prompt_truncated', true); - } + if (!areSensitiveSpanAttributesEnabled(config)) return; + const parts = extractGeminiContent({ + config: { systemInstruction }, + }).systemInstructions; + if (parts !== undefined) { + writeJsonAttribute(config, span, 'gen_ai.system_instructions', parts); } } -// --- LLM Request Span: Tool Schemas --- - +/** + * @deprecated Provider-final requests are captured by the GenAI exchange. + */ export function addToolSchemaAttributes( config: Config, span: Span, tools: unknown[] | undefined, ): void { - if (!areSensitiveSpanAttributesEnabled(config) || !tools?.length) return; - - // The Gemini API shape is `[{ functionDeclarations: [...] }]` — a single - // wrapper object whose inner array holds the actual per-tool schemas. - // Flatten that here so each declaration becomes its own summary entry and - // its own deduped tool_schema event, while still falling back to a flat - // input shape used by tests. - const declarations: unknown[] = []; - for (const tool of tools) { - const inner = (tool as Record)['functionDeclarations']; - if (Array.isArray(inner)) { - declarations.push(...inner); - } else { - declarations.push(tool); - } + if (!areSensitiveSpanAttributesEnabled(config) || tools === undefined) return; + const providerTools = tools.every( + (tool) => + typeof tool === 'object' && + tool !== null && + Object.hasOwn(tool, 'functionDeclarations'), + ) + ? tools + : [{ functionDeclarations: tools }]; + const definitions = extractGeminiContent({ + config: { tools: providerTools }, + }).toolDefinitions; + if (definitions !== undefined) { + writeJsonAttribute(config, span, 'gen_ai.tool.definitions', definitions); } - - const summary: Array<{ name: string; hash: string }> = []; - - for (const decl of declarations) { - const declObj = decl as Record; - const name = - typeof declObj['name'] === 'string' ? declObj['name'] : 'unknown_tool'; - const declJson = safeJsonStringify(decl) ?? `unstringifiable_${name}`; - const hash = shortHash(declJson); - summary.push({ name, hash }); - - const hashKey = `tool_${hash}`; - if (!seenHashes.has(hashKey)) { - seenHashes.add(hashKey); - const { content, truncated } = truncateContent( - declJson, - getMaxContentSize(config), - ); - span.addEvent('tool_schema', { - tool_name: name, - tool_hash: hash, - tool_definition: content, - ...(truncated && { - tool_definition_truncated: true, - tool_definition_original_length: declJson.length, - }), - }); - } - } - - span.setAttributes({ - tools: safeJsonStringify(summary) ?? '[]', - tools_count: summary.length, - }); } -// --- LLM Request Span: Model Output --- - +/** + * @deprecated Provider responses are captured by the GenAI exchange. + * This compatibility helper writes nothing without an explicit finish reason. + */ export function addModelOutputAttributes( config: Config, span: Span, responseText: string | undefined, - originalLength?: number, + originalLengthOrFinishReason?: number | string, + finishReason?: string, ): void { - if (!areSensitiveSpanAttributesEnabled(config) || !responseText) return; - - const responseTextOriginalLength = originalLength ?? responseText.length; - const { content, truncated } = truncateContent( - responseText, - getMaxContentSize(config), - responseTextOriginalLength, - ); - span.setAttributes({ - 'response.model_output': content, - ...(truncated && { - 'response.model_output_truncated': true, - 'response.model_output_original_length': responseTextOriginalLength, - }), - }); + if (!areSensitiveSpanAttributesEnabled(config) || responseText === undefined) + return; + const reason = + typeof originalLengthOrFinishReason === 'string' + ? originalLengthOrFinishReason + : finishReason; + if (!reason) return; + writeJsonAttribute(config, span, 'gen_ai.output.messages', [ + { + role: 'assistant', + parts: [{ type: 'text', content: responseText }], + finish_reason: reason, + }, + ]); } -// --- Tool Span: Input --- - +/** + * @deprecated Use addToolArgumentsAttributes with the final invocation params. + * Only JSON object strings produce a standard arguments attribute. + */ export function addToolInputAttributes( config: Config, span: Span, - toolName: string, + _toolName: string, toolInput: string, ): void { - if (!areSensitiveSpanAttributesEnabled(config)) return; - - const { content, truncated, originalLength } = truncatePrefixedContent( - `[TOOL INPUT: ${toolName}]\n`, - toolInput, - getMaxContentSize(config), - ); - span.setAttributes({ - tool_input: content, - ...(truncated && { - tool_input_truncated: true, - tool_input_original_length: originalLength, - }), - }); + let value: unknown; + try { + value = JSON.parse(toolInput); + } catch { + return; + } + addToolArgumentsAttributes(config, span, value); } -// --- Tool Span: Result --- - +/** + * @deprecated Use addToolCallResultAttributes with the final FunctionResponse. + * Only JSON object strings produce a standard result attribute. + */ export function addToolResultAttributes( config: Config, span: Span, - toolName: string, + _toolName: string, toolResult: string, ): void { - if (!areSensitiveSpanAttributesEnabled(config)) return; + let value: unknown; + try { + value = JSON.parse(toolResult); + } catch { + return; + } + addToolCallResultAttributes(config, span, value); +} - const { content, truncated, originalLength } = truncatePrefixedContent( - `[TOOL RESULT: ${toolName}]\n`, - toolResult, - getMaxContentSize(config), +export function addToolArgumentsAttributes( + config: Config, + span: Span, + argumentsValue: unknown, +): void { + if (!areSensitiveSpanAttributesEnabled(config)) return; + writeJsonAttribute( + config, + span, + 'gen_ai.tool.call.arguments', + argumentsValue, + true, ); - span.setAttributes({ - tool_result: content, - ...(truncated && { - tool_result_truncated: true, - tool_result_original_length: originalLength, - }), - }); } -// --- State Management --- +export function addToolCallResultAttributes( + config: Config, + span: Span, + result: unknown, +): void { + if (!areSensitiveSpanAttributesEnabled(config)) return; + writeJsonAttribute(config, span, 'gen_ai.tool.call.result', result, true); +} +/** + * @deprecated Sensitive GenAI attributes no longer use process-global state. + */ export function clearDetailedSpanState(): void { - seenHashes.clear(); + // Compatibility no-op. +} + +function writeJsonAttribute( + config: Config, + span: Span, + key: string, + value: unknown, + requireObject = false, +): void { + let serialized: string | undefined; + try { + serialized = stringifyGenAiJson( + value, + getMaxContentSize(config), + requireObject, + ); + } catch { + debugLogger.debug(`Failed to serialize ${key} span attribute`); + return; + } + if (serialized === undefined) return; + try { + span.setAttribute(key, serialized); + } catch { + debugLogger.debug(`Failed to set ${key} span attribute`); + } } diff --git a/packages/core/src/telemetry/gen-ai-content.test.ts b/packages/core/src/telemetry/gen-ai-content.test.ts new file mode 100644 index 00000000000..7438b2eef6e --- /dev/null +++ b/packages/core/src/telemetry/gen-ai-content.test.ts @@ -0,0 +1,941 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { Ajv } from 'ajv'; +import { + extractAnthropicContent, + extractGeminiContent, + extractOpenAiContent, + GenAiOutputAccumulator, + stringifyGenAiJson, +} from './gen-ai-content.js'; + +function fixture(name: string): object { + return JSON.parse( + readFileSync( + new URL(`./test-fixtures/gen-ai/${name}`, import.meta.url), + 'utf8', + ), + ) as object; +} + +describe('GenAI content conversion', () => { + it('converts final OpenAI messages and tool definitions', () => { + const content = extractOpenAiContent({ + messages: [ + { role: 'system', content: 'be helpful' }, + { role: 'user', content: [{ type: 'text', text: 'read a' }] }, + { + role: 'assistant', + content: 'calling', + reasoning_content: 'reasoning', + refusal: 'refused detail', + tool_calls: [ + { + id: 'call-1', + type: 'function', + function: { name: 'read', arguments: '{"path":"a"}' }, + }, + ], + }, + { + role: 'tool', + tool_call_id: 'call-1', + content: '{"output":"ok"}', + }, + ], + tools: [ + { + type: 'function', + function: { + name: 'read', + description: 'Read a file', + parameters: { + type: 'object', + properties: { path: { type: 'string' } }, + }, + }, + }, + ], + }); + + expect(content.inputMessages).toEqual([ + { + role: 'system', + parts: [{ type: 'text', content: 'be helpful' }], + }, + { + role: 'user', + parts: [{ type: 'text', content: 'read a' }], + }, + { + role: 'assistant', + parts: [ + { type: 'reasoning', content: 'reasoning' }, + { type: 'text', content: 'calling' }, + { type: 'refusal', content: 'refused detail' }, + { + type: 'tool_call', + id: 'call-1', + name: 'read', + arguments: { path: 'a' }, + }, + ], + }, + { + role: 'tool', + parts: [ + { + type: 'tool_call_response', + id: 'call-1', + response: '{"output":"ok"}', + }, + ], + }, + ]); + expect(content.systemInstructions).toBeUndefined(); + expect(content.toolDefinitions).toEqual([ + { + type: 'function', + name: 'read', + description: 'Read a file', + parameters: { + type: 'object', + properties: { path: { type: 'string' } }, + }, + }, + ]); + }); + + it('converts Anthropic system blocks, messages, and tools', () => { + const content = extractAnthropicContent({ + system: [{ type: 'text', text: 'be helpful' }], + messages: [ + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: [ + { type: 'thinking', thinking: 'reason' }, + { + type: 'tool_use', + id: 'call-1', + name: 'read', + input: { path: 'a' }, + }, + ], + }, + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call-1', + content: [{ type: 'text', text: 'ok' }], + }, + { type: 'text', text: 'continue' }, + ], + }, + ], + tools: [ + { + name: 'read', + description: 'Read', + input_schema: { type: 'object' }, + }, + ], + }); + + expect(content.systemInstructions).toEqual([ + { type: 'text', content: 'be helpful' }, + ]); + expect(content.inputMessages?.[1]).toEqual({ + role: 'assistant', + parts: [ + { type: 'reasoning', content: 'reason' }, + { + type: 'tool_call', + id: 'call-1', + name: 'read', + arguments: { path: 'a' }, + }, + ], + }); + expect(content.inputMessages?.slice(2)).toEqual([ + { + role: 'tool', + parts: [ + { + type: 'tool_call_response', + id: 'call-1', + response: [{ type: 'text', text: 'ok' }], + }, + ], + }, + { + role: 'user', + parts: [{ type: 'text', content: 'continue' }], + }, + ]); + expect(content.toolDefinitions).toEqual([ + { + type: 'function', + name: 'read', + description: 'Read', + parameters: { type: 'object' }, + }, + ]); + }); + + it('converts OpenAI-compatible media from the final adapter request', () => { + const content = extractOpenAiContent({ + messages: [ + { + role: 'user', + content: [ + { + type: 'file', + file: { + filename: 'document.pdf', + file_data: 'DATA:application/pdf;base64,cGRm', + }, + }, + { + type: 'file', + file: { + filename: 'remote.pdf', + file_data: 'https://example.com/document.pdf', + }, + }, + { + type: 'input_audio', + input_audio: { + data: 'data:audio/wav;base64,YXVkaW8=', + format: 'wav', + }, + }, + ], + }, + ], + }); + + expect(content.inputMessages).toEqual([ + { + role: 'user', + parts: [ + { + type: 'blob', + mime_type: 'application/pdf', + modality: 'document', + content: 'cGRm', + }, + { + type: 'uri', + mime_type: null, + modality: 'document', + uri: 'https://example.com/document.pdf', + }, + { + type: 'blob', + mime_type: 'audio/wav', + modality: 'audio', + content: 'YXVkaW8=', + }, + ], + }, + ]); + }); + + it('converts Gemini media and lowercases JSON Schema types', () => { + const content = extractGeminiContent({ + contents: [ + { + role: 'model', + parts: [ + { text: 'answer' }, + { + inlineData: { + mimeType: 'image/png', + data: 'YWJj', + }, + }, + ], + }, + ], + config: { + systemInstruction: { parts: [{ text: 'system' }] }, + tools: [ + { + functionDeclarations: [ + { + name: 'read', + parametersJsonSchema: { + type: 'object', + properties: { path: { type: 'STRING' } }, + examples: [{ type: 'filesystem-path' }], + }, + }, + ], + }, + ], + }, + }); + + expect(content.inputMessages).toEqual([ + { + role: 'assistant', + parts: [ + { type: 'text', content: 'answer' }, + { + type: 'blob', + mime_type: 'image/png', + modality: 'image', + content: 'YWJj', + }, + ], + }, + ]); + expect(content.systemInstructions).toEqual([ + { type: 'text', content: 'system' }, + ]); + expect(content.toolDefinitions).toEqual([ + { + type: 'function', + name: 'read', + parameters: { + type: 'object', + properties: { path: { type: 'string' } }, + examples: [{ type: 'filesystem-path' }], + }, + }, + ]); + }); + + it('omits only invalid optional parameters but rejects missing identity', () => { + expect( + extractGeminiContent({ + config: { + tools: [ + { + functionDeclarations: [ + { name: 'read', parameters: { type: 'NOT_A_SCHEMA_TYPE' } }, + { + name: 'write', + parametersJsonSchema: { + type: 'object', + required: 'path', + }, + }, + { + name: 'list', + parametersJsonSchema: { allOf: [] }, + }, + { + name: 'search', + parametersJsonSchema: { + type: 'object', + properties: { + query: { type: 'string', pattern: '[' }, + }, + }, + }, + ], + }, + ], + }, + }).toolDefinitions, + ).toEqual([ + { type: 'function', name: 'read' }, + { type: 'function', name: 'write' }, + { type: 'function', name: 'list' }, + { type: 'function', name: 'search' }, + ]); + + expect( + extractOpenAiContent({ + tools: [ + { type: 'function', function: { description: 'missing name' } }, + ], + }).toolDefinitions, + ).toBeUndefined(); + }); + + it('preserves boolean Draft-07 tool parameter schemas', () => { + expect( + extractGeminiContent({ + config: { + tools: [ + { + functionDeclarations: [ + { name: 'allowed', parametersJsonSchema: true }, + { name: 'impossible', parametersJsonSchema: false }, + ], + }, + ], + }, + }).toolDefinitions, + ).toEqual([ + { type: 'function', name: 'allowed', parameters: true }, + { type: 'function', name: 'impossible', parameters: false }, + ]); + }); + + it('rejects incomplete message snapshots and preserves generic parts', () => { + expect( + extractOpenAiContent({ + messages: [ + { role: 'user', content: 'ok' }, + { content: 'missing role' }, + ], + }).inputMessages, + ).toBeUndefined(); + expect( + extractOpenAiContent({ + messages: [ + { + role: 'user', + content: [{ type: 'provider_extension', value: 1 }], + }, + ], + }).inputMessages, + ).toEqual([ + { + role: 'user', + parts: [{ type: 'provider_extension', value: 1 }], + }, + ]); + expect( + extractOpenAiContent({ + messages: [{ role: 'assistant', content: null }], + }).inputMessages, + ).toEqual([{ role: 'assistant', parts: [] }]); + expect( + extractOpenAiContent({ + messages: [{ role: 'user', content: [null] }], + }).inputMessages, + ).toBeUndefined(); + }); + + it('preserves redacted Anthropic thinking as a generic part', () => { + expect( + extractAnthropicContent({ + messages: [ + { + role: 'assistant', + content: [{ type: 'redacted_thinking', data: 'ciphertext' }], + }, + ], + }).inputMessages, + ).toEqual([ + { + role: 'assistant', + parts: [{ type: 'redacted_thinking', data: 'ciphertext' }], + }, + ]); + }); +}); + +describe('GenAI JSON writer', () => { + it('preserves empty arrays and objects at the exact limit', () => { + expect(stringifyGenAiJson([], 2)).toBe('[]'); + expect(stringifyGenAiJson({}, 2, true)).toBe('{}'); + }); + + it('omits oversized, invalid-root, and cyclic values', () => { + expect(stringifyGenAiJson({ value: 1 }, 3)).toBeUndefined(); + expect(stringifyGenAiJson([], 100, true)).toBeUndefined(); + const cyclic: Record = {}; + cyclic['self'] = cyclic; + expect(stringifyGenAiJson(cyclic, 100)).toBeUndefined(); + }); + + it('preserves JSON keys that overlap object prototype accessors', () => { + const value = JSON.parse( + '{"__proto__":{"polluted":true},"constructor":"value"}', + ) as unknown; + + expect(stringifyGenAiJson(value, 1_000)).toBe( + '{"__proto__":{"polluted":true},"constructor":"value"}', + ); + expect( + (Object.prototype as unknown as Record)['polluted'], + ).toBeUndefined(); + }); + + it('uses nested toJSON values instead of fabricating empty objects', () => { + expect( + stringifyGenAiJson( + { + timestamp: new Date('2026-07-24T00:00:00.000Z'), + custom: { toJSON: () => ({ value: 1 }) }, + }, + 1_000, + ), + ).toBe('{"timestamp":"2026-07-24T00:00:00.000Z","custom":{"value":1}}'); + }); +}); + +describe('GenAI output accumulation', () => { + it('collects all non-streaming OpenAI choices in index order', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordOpenAiResponse({ + choices: [ + { + index: 1, + message: { role: 'assistant', content: 'B' }, + finish_reason: 'length', + }, + { + index: 0, + message: { role: 'assistant', content: 'A' }, + finish_reason: 'stop', + }, + ], + }); + + expect(JSON.parse(output.finalize(true)!)).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'A' }], + finish_reason: 'stop', + }, + { + role: 'assistant', + parts: [{ type: 'text', content: 'B' }], + finish_reason: 'length', + }, + ]); + }); + + it('collects all OpenAI choices in index order', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordOpenAiChunk({ + choices: [ + { index: 1, delta: { content: 'B' }, finish_reason: null }, + { + index: 0, + delta: { reasoning_content: 'R', content: 'A' }, + finish_reason: null, + }, + ], + }); + output.recordOpenAiChunk({ + choices: [ + { index: 1, delta: {}, finish_reason: 'length' }, + { index: 0, delta: {}, finish_reason: 'stop' }, + ], + }); + + expect(output.finishReasons).toEqual(['stop', 'length']); + expect(JSON.parse(output.finalize(true)!)).toEqual([ + { + role: 'assistant', + parts: [ + { type: 'reasoning', content: 'R' }, + { type: 'text', content: 'A' }, + ], + finish_reason: 'stop', + }, + { + role: 'assistant', + parts: [{ type: 'text', content: 'B' }], + finish_reason: 'length', + }, + ]); + }); + + it('merges fragmented tool-call arguments', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordOpenAiChunk({ + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: 'call-1', + function: { name: 'read', arguments: '{"path":' }, + }, + ], + }, + }, + ], + }); + output.recordOpenAiChunk({ + choices: [ + { + index: 0, + delta: { + tool_calls: [{ index: 0, function: { arguments: '"a"}' } }], + }, + finish_reason: 'tool_calls', + }, + ], + }); + expect(JSON.parse(output.finalize(true)!)[0].parts).toEqual([ + { + type: 'tool_call', + id: 'call-1', + name: 'read', + arguments: { path: 'a' }, + }, + ]); + }); + + it('does not invent missing tool-call arguments', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordOpenAiResponse({ + choices: [ + { + index: 0, + message: { + role: 'assistant', + tool_calls: [ + { + id: 'call-1', + type: 'function', + function: { name: 'read' }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + }); + + expect(JSON.parse(output.finalize(true)!)[0].parts).toEqual([ + { + type: 'tool_call', + id: 'call-1', + name: 'read', + }, + ]); + }); + + it('merges Anthropic content blocks by block index', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordAnthropicEvent({ + type: 'content_block_start', + index: 0, + content_block: { type: 'thinking', thinking: 'rea' }, + }); + output.recordAnthropicEvent({ + type: 'content_block_delta', + index: 0, + delta: { type: 'thinking_delta', thinking: 'son' }, + }); + output.recordAnthropicEvent({ + type: 'content_block_start', + index: 1, + content_block: { + type: 'tool_use', + id: 'call-1', + name: 'read', + input: {}, + }, + }); + output.recordAnthropicEvent({ + type: 'content_block_delta', + index: 1, + delta: { type: 'input_json_delta', partial_json: '{"path":"a"}' }, + }); + output.recordAnthropicEvent({ + type: 'message_delta', + delta: { stop_reason: 'tool_use' }, + }); + + expect(JSON.parse(output.finalize(true)!)).toEqual([ + { + role: 'assistant', + parts: [ + { type: 'reasoning', content: 'reason' }, + { + type: 'tool_call', + id: 'call-1', + name: 'read', + arguments: { path: 'a' }, + }, + ], + finish_reason: 'tool_use', + }, + ]); + }); + + it('preserves an empty Anthropic tool input when no deltas follow', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordAnthropicEvent({ + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id: 'call-1', + name: 'read', + input: {}, + }, + }); + output.recordAnthropicEvent({ + type: 'message_delta', + delta: { stop_reason: 'tool_use' }, + }); + + expect(JSON.parse(output.finalize(true)!)[0].parts).toEqual([ + { + type: 'tool_call', + id: 'call-1', + name: 'read', + arguments: {}, + }, + ]); + }); + + it('does not invent an output candidate for Anthropic keepalive events', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordAnthropicEvent({ type: 'ping' }); + expect(output.finalize(false)).toBeUndefined(); + expect(output.finishReasons).toBeUndefined(); + }); + + it('uses error for unfinished candidates only on failure', () => { + const failed = new GenAiOutputAccumulator(true, 10_000); + failed.recordGeminiResponse({ + candidates: [ + { index: 0, content: { role: 'model', parts: [{ text: 'a' }] } }, + ], + }); + expect(JSON.parse(failed.finalize(false)!)[0].finish_reason).toBe('error'); + expect(failed.finishReasons).toEqual(['error']); + + const successful = new GenAiOutputAccumulator(true, 10_000); + successful.recordGeminiResponse({ + candidates: [ + { index: 0, content: { role: 'model', parts: [{ text: 'a' }] } }, + ], + }); + expect(successful.finalize(true)).toBeUndefined(); + }); + + it('tracks non-streaming candidate failures when content capture is off', () => { + const responses: Array< + [GenAiOutputAccumulator, (accumulator: GenAiOutputAccumulator) => void] + > = [ + [ + new GenAiOutputAccumulator(false, 10_000), + (accumulator) => + accumulator.recordOpenAiResponse({ + choices: [ + { index: 0, message: { role: 'assistant', content: '' } }, + ], + }), + ], + [ + new GenAiOutputAccumulator(false, 10_000), + (accumulator) => accumulator.recordAnthropicResponse({ content: [] }), + ], + [ + new GenAiOutputAccumulator(false, 10_000), + (accumulator) => + accumulator.recordGeminiResponse({ + candidates: [{ index: 0, content: { role: 'model', parts: [] } }], + }), + ], + ]; + + for (const [accumulator, recordResponse] of responses) { + recordResponse(accumulator); + expect(accumulator.finalize(false)).toBeUndefined(); + expect(accumulator.finishReasons).toEqual(['error']); + } + }); + + it('accumulates Gemini text chunks without replacing earlier content', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordGeminiChunk({ + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [{ text: 'think', thought: true }], + }, + }, + ], + }); + output.recordGeminiChunk({ + candidates: [ + { + index: 0, + content: { role: 'model', parts: [{ text: 'hel' }] }, + }, + ], + }); + output.recordGeminiChunk({ + candidates: [ + { + index: 0, + content: { role: 'model', parts: [{ text: 'lo' }] }, + finishReason: 'STOP', + }, + ], + }); + + expect(JSON.parse(output.finalize(true)!)).toEqual([ + { + role: 'assistant', + parts: [ + { type: 'reasoning', content: 'think' }, + { type: 'text', content: 'hello' }, + ], + finish_reason: 'STOP', + }, + ]); + }); + + it('preserves explicit zero candidates and drops overflowed content', () => { + const empty = new GenAiOutputAccumulator(true, 10); + empty.recordOpenAiResponse({ choices: [] }); + expect(empty.finalize(true)).toBe('[]'); + + const tooSmallForEmpty = new GenAiOutputAccumulator(true, 1); + tooSmallForEmpty.recordOpenAiResponse({ choices: [] }); + expect(tooSmallForEmpty.finalize(true)).toBeUndefined(); + + const overflow = new GenAiOutputAccumulator(true, 20); + overflow.recordOpenAiChunk({ + choices: [ + { + index: 0, + delta: { content: 'too much content' }, + finish_reason: 'stop', + }, + ], + }); + expect(overflow.finishReasons).toEqual(['stop']); + expect(overflow.finalize(true)).toBeUndefined(); + }); + + it('omits an incomplete full snapshot instead of dropping invalid choices', () => { + const output = new GenAiOutputAccumulator(true, 10_000); + output.recordOpenAiResponse({ + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'valid' }, + finish_reason: 'stop', + }, + null, + ], + }); + expect(output.finalize(true)).toBeUndefined(); + + const missing = new GenAiOutputAccumulator(true, 10_000); + missing.recordGeminiResponse({}); + expect(missing.finalize(false)).toBeUndefined(); + }); + + it('budgets many small fragments by final content rather than chunk count', () => { + const expected = new GenAiOutputAccumulator(true, 10_000); + for (let index = 0; index < 100; index++) { + expected.recordOpenAiChunk({ + choices: [{ index: 0, delta: { content: 'a' } }], + }); + } + expected.recordOpenAiChunk({ + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + }); + const serialized = expected.finalize(true)!; + + const exact = new GenAiOutputAccumulator(true, serialized.length); + const oversized = new GenAiOutputAccumulator(true, serialized.length - 1); + for (let index = 0; index < 100; index++) { + const chunk = { choices: [{ index: 0, delta: { content: 'a' } }] }; + exact.recordOpenAiChunk(chunk); + oversized.recordOpenAiChunk(chunk); + } + const finish = { + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + }; + exact.recordOpenAiChunk(finish); + oversized.recordOpenAiChunk(finish); + + expect(exact.finalize(true)).toBe(serialized); + expect(oversized.finalize(true)).toBeUndefined(); + }); + + it('does not reject an exact-limit response with many small parts', () => { + const chunk = { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: Array.from({ length: 20 }, () => ({ text: '' })), + }, + finishReason: 'STOP', + }, + ], + }; + const expected = new GenAiOutputAccumulator(true, 10_000); + expected.recordGeminiChunk(chunk); + const serialized = expected.finalize(true)!; + + const exact = new GenAiOutputAccumulator(true, serialized.length); + exact.recordGeminiChunk(chunk); + expect(exact.finalize(true)).toBe(serialized); + }); +}); + +describe('pinned OpenTelemetry JSON Schemas', () => { + it('accepts the canonical values emitted by the converters', () => { + const ajv = new Ajv({ strict: false }); + const input = extractOpenAiContent({ + messages: [{ role: 'user', content: 'hello' }], + }).inputMessages; + const system = extractAnthropicContent({ + system: 'system', + }).systemInstructions; + const tools = extractOpenAiContent({ + tools: [ + { + type: 'function', + function: { + name: 'read', + parameters: { type: 'object' }, + }, + }, + ], + }).toolDefinitions; + const accumulator = new GenAiOutputAccumulator(true, 10_000); + accumulator.recordOpenAiResponse({ + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'answer' }, + finish_reason: 'stop', + }, + ], + }); + const output = JSON.parse(accumulator.finalize(true)!); + + const cases: Array<[string, unknown]> = [ + ['gen-ai-input-messages.json', input], + ['gen-ai-output-messages.json', output], + ['gen-ai-system-instructions.json', system], + ['gen-ai-tool-definitions.json', tools], + ['gen-ai-tool-call-arguments.json', { path: 'a' }], + ['gen-ai-tool-call-result.json', { output: 'ok' }], + ]; + for (const [name, value] of cases) { + const validate = ajv.compile(fixture(name)); + const valid = validate(value); + if (!valid) { + throw new Error(`${name}: ${JSON.stringify(validate.errors)}`); + } + expect(valid).toBe(true); + } + }); +}); diff --git a/packages/core/src/telemetry/gen-ai-content.ts b/packages/core/src/telemetry/gen-ai-content.ts new file mode 100644 index 00000000000..75b44ab7557 --- /dev/null +++ b/packages/core/src/telemetry/gen-ai-content.ts @@ -0,0 +1,1518 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createDebugLogger } from '../utils/debugLogger.js'; + +type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; +export interface JsonObject { + [key: string]: JsonValue; +} + +export interface GenAiContentAttributes { + inputMessages?: JsonObject[]; + systemInstructions?: JsonObject[]; + toolDefinitions?: JsonObject[]; +} + +interface CanonicalPart extends JsonObject { + type: string; +} + +interface PartWithRole { + part: CanonicalPart; + role?: string; +} + +interface StreamPart { + type: string; + content?: string; + id?: string; + name?: string; + arguments?: string; + argumentsFromStart?: boolean; + value?: CanonicalPart; +} + +interface StreamCandidate { + role: string; + parts: Map; + finishReason?: string; +} + +const DRAFT_07_TYPES = new Set([ + 'array', + 'boolean', + 'integer', + 'null', + 'number', + 'object', + 'string', +]); +const debugLogger = createDebugLogger('GEN_AI_CONTENT'); + +function record(value: unknown): Record | undefined { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function string(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function jsonValue( + value: unknown, + seen: Set = new Set(), +): JsonValue | undefined { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' + ) { + return value; + } + if (typeof value === 'number') { + return Number.isFinite(value) ? value : undefined; + } + if (typeof value !== 'object') return undefined; + if (seen.has(value)) return undefined; + seen.add(value); + try { + const toJSON = (value as { toJSON?: unknown }).toJSON; + if (typeof toJSON === 'function') { + return jsonValue(toJSON.call(value), seen); + } + if (Array.isArray(value)) { + const result: JsonValue[] = []; + for (const item of value) { + const converted = jsonValue(item, seen); + if (converted === undefined) return undefined; + result.push(converted); + } + return result; + } + const result: JsonObject = {}; + for (const [key, item] of Object.entries(value)) { + if (item === undefined) continue; + const converted = jsonValue(item, seen); + if (converted === undefined) return undefined; + Object.defineProperty(result, key, { + value: converted, + enumerable: true, + configurable: true, + writable: true, + }); + } + return result; + } finally { + seen.delete(value); + } +} + +function modality(mimeType: string | undefined): string | undefined { + if (mimeType?.startsWith('image/')) return 'image'; + if (mimeType?.startsWith('video/')) return 'video'; + if (mimeType?.startsWith('audio/')) return 'audio'; + if (mimeType?.startsWith('text/') || mimeType?.startsWith('application/')) { + return 'document'; + } + return undefined; +} + +function isDataUri(value: string): boolean { + return /^data:/i.test(value); +} + +function dataUriBlob( + uri: string, + contentModality: string, + fallbackMimeType: string | null, +): CanonicalPart | undefined { + if (!isDataUri(uri)) return undefined; + const separator = uri.indexOf(','); + if (separator < 0) return undefined; + const metadata = uri.slice(5, separator); + const metadataParts = metadata.split(';'); + if (!metadataParts.slice(1).some((part) => part.toLowerCase() === 'base64')) { + return undefined; + } + const mimeType = metadataParts[0] || fallbackMimeType; + return { + type: 'blob', + mime_type: mimeType, + modality: contentModality, + content: uri.slice(separator + 1), + }; +} + +function uriPart( + uri: string, + contentModality: string, + mimeType: string | null = null, +): CanonicalPart { + return { + type: 'uri', + mime_type: mimeType, + modality: contentModality, + uri, + }; +} + +function parseArguments(value: unknown): JsonValue | undefined { + if (typeof value !== 'string') return jsonValue(value); + try { + return jsonValue(JSON.parse(value)); + } catch { + return value; + } +} + +function genericPart( + value: Record, +): CanonicalPart | undefined { + const type = string(value['type']); + if (!type) return undefined; + const converted = jsonValue(value); + if (!converted || Array.isArray(converted) || typeof converted !== 'object') { + return undefined; + } + return { ...converted, type }; +} + +function openAiPart(value: unknown): PartWithRole | undefined { + if (typeof value === 'string') { + return { part: { type: 'text', content: value } }; + } + const item = record(value); + if (!item) return undefined; + const type = string(item['type']); + if (type === 'text' || type === 'input_text' || type === 'output_text') { + const content = string(item['text']) ?? string(item['content']); + return content === undefined + ? undefined + : { part: { type: 'text', content } }; + } + if ( + type === 'reasoning' || + type === 'thinking' || + type === 'reasoning_content' + ) { + const content = string(item['text']) ?? string(item['content']); + return content === undefined + ? undefined + : { part: { type: 'reasoning', content } }; + } + if (type === 'image_url') { + const image = record(item['image_url']); + const uri = string(image?.['url']) ?? string(item['image_url']); + if (!uri) return undefined; + const blob = dataUriBlob(uri, 'image', null); + if (isDataUri(uri) && !blob) return undefined; + return { + part: blob ?? uriPart(uri, 'image'), + }; + } + if (type === 'video_url') { + const video = record(item['video_url']); + const uri = string(video?.['url']) ?? string(item['video_url']); + if (!uri) return undefined; + const blob = dataUriBlob(uri, 'video', null); + if (isDataUri(uri) && !blob) return undefined; + return { + part: blob ?? uriPart(uri, 'video'), + }; + } + if (type === 'input_audio') { + const audio = record(item['input_audio']); + const content = string(audio?.['data']); + if (content === undefined) return undefined; + const format = string(audio?.['format']); + const mimeType = + format === 'mp3' ? 'audio/mpeg' : format ? `audio/${format}` : null; + const blob = dataUriBlob(content, 'audio', mimeType); + if (isDataUri(content) && !blob) return undefined; + return { + part: blob ?? { + type: 'blob', + mime_type: mimeType, + modality: 'audio', + content, + }, + }; + } + if (type === 'file') { + const file = record(item['file']); + const fileId = string(file?.['file_id']) ?? string(item['file_id']); + if (fileId) { + return { + part: { + type: 'file', + mime_type: null, + modality: 'document', + file_id: fileId, + }, + }; + } + const fileData = string(file?.['file_data']) ?? string(item['file_data']); + if (!fileData) return undefined; + const blob = dataUriBlob(fileData, 'document', null); + if (blob) return { part: blob }; + if (isDataUri(fileData)) return undefined; + if (/^[A-Za-z][A-Za-z0-9+.-]*:/.test(fileData)) { + return { part: uriPart(fileData, 'document') }; + } + return { + part: { + type: 'blob', + mime_type: null, + modality: 'document', + content: fileData, + }, + }; + } + if (type === 'tool_result' || type === 'tool_call_response') { + const response = jsonValue(item['response'] ?? item['content']); + if (response === undefined) return undefined; + return { + role: 'tool', + part: { + type: 'tool_call_response', + ...(string(item['tool_call_id']) || string(item['id']) + ? { id: string(item['tool_call_id']) ?? string(item['id'])! } + : {}), + response, + }, + }; + } + const part = genericPart(item); + return part ? { part } : undefined; +} + +function anthropicPart(value: unknown): PartWithRole | undefined { + if (typeof value === 'string') { + return { part: { type: 'text', content: value } }; + } + const item = record(value); + if (!item) return undefined; + const type = string(item['type']); + if (type === 'text') { + const content = string(item['text']); + return content === undefined + ? undefined + : { part: { type: 'text', content } }; + } + if (type === 'thinking') { + const content = string(item['thinking']) ?? string(item['text']); + return content === undefined + ? undefined + : { part: { type: 'reasoning', content } }; + } + if (type === 'redacted_thinking') { + const part = genericPart(item); + return part ? { part } : undefined; + } + if (type === 'tool_use') { + const name = string(item['name']); + if (!name) return undefined; + const argumentsValue = jsonValue(item['input']); + return { + part: { + type: 'tool_call', + ...(string(item['id']) ? { id: string(item['id'])! } : {}), + name, + ...(argumentsValue !== undefined ? { arguments: argumentsValue } : {}), + }, + }; + } + if (type === 'tool_result') { + const response = jsonValue(item['content']); + if (response === undefined) return undefined; + return { + role: 'tool', + part: { + type: 'tool_call_response', + ...(string(item['tool_use_id']) + ? { id: string(item['tool_use_id'])! } + : {}), + response, + }, + }; + } + if (type === 'image' || type === 'document') { + const source = record(item['source']); + const sourceType = string(source?.['type']); + const mimeType = string(source?.['media_type']); + const data = string(source?.['data']); + if (sourceType === 'base64' && data !== undefined) { + return { + part: { + type: 'blob', + mime_type: mimeType ?? null, + modality: type === 'image' ? 'image' : 'document', + content: data, + }, + }; + } + const uri = string(source?.['url']); + if (uri) { + return { + part: { + type: 'uri', + mime_type: mimeType ?? null, + modality: type === 'image' ? 'image' : 'document', + uri, + }, + }; + } + } + const part = genericPart(item); + return part ? { part } : undefined; +} + +function geminiPart(value: unknown): PartWithRole | undefined { + if (typeof value === 'string') { + return { part: { type: 'text', content: value } }; + } + const item = record(value); + if (!item) return undefined; + const text = string(item['text']); + if (text !== undefined) { + return { + part: { + type: item['thought'] === true ? 'reasoning' : 'text', + content: text, + }, + }; + } + const call = record(item['functionCall']); + if (call) { + const name = string(call['name']); + if (!name) return undefined; + const argumentsValue = jsonValue(call['args']); + return { + part: { + type: 'tool_call', + ...(string(call['id']) ? { id: string(call['id'])! } : {}), + name, + ...(argumentsValue !== undefined ? { arguments: argumentsValue } : {}), + }, + }; + } + const response = record(item['functionResponse']); + if (response) { + const responseValue = jsonValue(response['response']); + if (responseValue === undefined) return undefined; + return { + role: 'tool', + part: { + type: 'tool_call_response', + ...(string(response['id']) ? { id: string(response['id'])! } : {}), + response: responseValue, + }, + }; + } + const inlineData = record(item['inlineData']); + if (inlineData) { + const content = string(inlineData['data']); + if (content === undefined) return undefined; + const mimeType = string(inlineData['mimeType']); + const contentModality = modality(mimeType); + if (!contentModality) return undefined; + return { + part: { + type: 'blob', + mime_type: mimeType ?? null, + modality: contentModality, + content, + }, + }; + } + const fileData = record(item['fileData']); + if (fileData) { + const uri = string(fileData['fileUri']); + if (!uri) return undefined; + const mimeType = string(fileData['mimeType']); + const contentModality = modality(mimeType); + if (!contentModality) return undefined; + return { + part: { + type: 'uri', + mime_type: mimeType ?? null, + modality: contentModality, + uri, + }, + }; + } + const explicitType = string(item['type']); + const part = explicitType ? genericPart(item) : undefined; + return part ? { part } : undefined; +} + +function messages( + values: unknown, + convertPart: (value: unknown) => PartWithRole | undefined, + roleMap: (role: string) => string, +): JsonObject[] | undefined { + const items = Array.isArray(values) ? values : [values]; + const result: JsonObject[] = []; + for (const value of items) { + if (typeof value === 'string') { + result.push({ + role: 'user', + parts: [{ type: 'text', content: value }], + }); + continue; + } + const message = record(value); + const rawRole = string(message?.['role']); + if (!message || !rawRole) return undefined; + const role = roleMap(rawRole); + const content = message['content'] ?? message['parts']; + const rawParts = + content === undefined || content === null + ? [] + : Array.isArray(content) + ? content + : [content]; + let currentRole = role; + let currentParts: CanonicalPart[] = []; + let emitted = false; + const flush = () => { + if (currentParts.length === 0) return; + result.push({ + role: currentRole, + parts: currentParts, + ...(string(message['name']) ? { name: string(message['name'])! } : {}), + }); + currentParts = []; + emitted = true; + }; + const reasoning = + string(message['reasoning_content']) ?? + string(message['reasoning']) ?? + string(message['thinking']); + if (reasoning !== undefined) { + currentParts.push({ type: 'reasoning', content: reasoning }); + } + for (const rawPart of rawParts) { + const converted = convertPart(rawPart); + if (!converted) return undefined; + const nextRole = converted.role ?? role; + if (currentParts.length > 0 && nextRole !== currentRole) flush(); + currentRole = nextRole; + currentParts.push(converted.part); + } + const refusal = string(message['refusal']); + if (refusal !== undefined) { + currentParts.push({ type: 'refusal', content: refusal }); + } + const audio = record(message['audio']); + if (audio) { + const converted = record(jsonValue(audio)); + if (!converted) return undefined; + currentParts.push({ ...converted, type: 'audio' }); + } + if (Array.isArray(message['tool_calls'])) { + if (currentParts.length > 0 && currentRole !== role) flush(); + currentRole = role; + for (const value of message['tool_calls']) { + const toolCall = record(value); + const fn = record(toolCall?.['function']); + const name = string(fn?.['name']); + if (!toolCall || !fn || !name) return undefined; + const argumentsValue = parseArguments(fn['arguments']); + currentParts.push({ + type: 'tool_call', + ...(string(toolCall['id']) ? { id: string(toolCall['id'])! } : {}), + name, + ...(argumentsValue !== undefined + ? { arguments: argumentsValue } + : {}), + }); + } + } + if ( + role === 'tool' && + currentParts.length > 0 && + currentParts.every((part) => part.type === 'text') + ) { + currentParts = currentParts.map((part) => ({ + type: 'tool_call_response', + ...(string(message['tool_call_id']) + ? { id: string(message['tool_call_id'])! } + : {}), + response: part['content'] ?? null, + })); + } + flush(); + if (!emitted) { + result.push({ + role, + parts: [], + ...(string(message['name']) ? { name: string(message['name'])! } : {}), + }); + } + } + return result; +} + +function systemParts( + value: unknown, + convertPart: (value: unknown) => PartWithRole | undefined, +): JsonObject[] | undefined { + if (value === undefined || value === null) return undefined; + const content = record(value)?.['parts'] ?? value; + const values = Array.isArray(content) ? content : [content]; + const result: JsonObject[] = []; + for (const item of values) { + const part = convertPart(item); + if (!part || part.role) return undefined; + result.push(part.part); + } + return result; +} + +function jsonIdentity(value: JsonValue): string { + if (Array.isArray(value)) { + return `[${value.map(jsonIdentity).join(',')}]`; + } + if (value !== null && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${jsonIdentity(value[key]!)}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +function hasUniqueValues(values: JsonValue[]): boolean { + return new Set(values.map(jsonIdentity)).size === values.length; +} + +function isValidRegex(value: string): boolean { + try { + new RegExp(value, 'u'); + return true; + } catch { + return false; + } +} + +function isValidUri(value: string, allowReference: boolean): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code <= 0x20 || code === 0x7f) return false; + } + if (/%(?![0-9A-Fa-f]{2})/.test(value)) return false; + try { + new URL(value, allowReference ? 'https://example.invalid/' : undefined); + return true; + } catch { + return false; + } +} + +function normalizeSchema(value: unknown): JsonValue | undefined { + const converted = jsonValue(value); + if (converted === undefined) return undefined; + function visitSchema(schema: JsonValue): boolean { + return ( + typeof schema === 'boolean' || + (schema !== null && + !Array.isArray(schema) && + typeof schema === 'object' && + visit(schema)) + ); + } + function visit(item: JsonObject): boolean { + const schemaType = item['type']; + if (typeof schemaType === 'string') { + const lower = schemaType.toLowerCase(); + if (!DRAFT_07_TYPES.has(lower)) return false; + item['type'] = lower; + } else if (Array.isArray(schemaType)) { + if (schemaType.length === 0) return false; + const normalized: string[] = []; + for (const entry of schemaType) { + if (typeof entry !== 'string') return false; + const lower = entry.toLowerCase(); + if (!DRAFT_07_TYPES.has(lower)) return false; + normalized.push(lower); + } + if (new Set(normalized).size !== normalized.length) return false; + item['type'] = normalized; + } else if (schemaType !== undefined) { + return false; + } + + for (const key of [ + '$comment', + 'title', + 'description', + 'format', + 'contentMediaType', + 'contentEncoding', + ]) { + if (item[key] !== undefined && typeof item[key] !== 'string') + return false; + } + for (const key of ['readOnly', 'uniqueItems']) { + if (item[key] !== undefined && typeof item[key] !== 'boolean') { + return false; + } + } + if ( + (item['$id'] !== undefined && + (typeof item['$id'] !== 'string' || !isValidUri(item['$id'], true))) || + (item['$ref'] !== undefined && + (typeof item['$ref'] !== 'string' || + !isValidUri(item['$ref'], true))) || + (item['$schema'] !== undefined && + (typeof item['$schema'] !== 'string' || + !isValidUri(item['$schema'], false))) || + (item['pattern'] !== undefined && + (typeof item['pattern'] !== 'string' || !isValidRegex(item['pattern']))) + ) { + return false; + } + if (item['examples'] !== undefined && !Array.isArray(item['examples'])) { + return false; + } + for (const key of [ + 'maximum', + 'exclusiveMaximum', + 'minimum', + 'exclusiveMinimum', + ]) { + if (item[key] !== undefined && typeof item[key] !== 'number') + return false; + } + if ( + item['multipleOf'] !== undefined && + (typeof item['multipleOf'] !== 'number' || item['multipleOf'] <= 0) + ) { + return false; + } + for (const key of [ + 'maxLength', + 'minLength', + 'maxItems', + 'minItems', + 'maxProperties', + 'minProperties', + ]) { + const count = item[key]; + if ( + count !== undefined && + (typeof count !== 'number' || !Number.isInteger(count) || count < 0) + ) { + return false; + } + } + + const required = item['required']; + if ( + required !== undefined && + (!Array.isArray(required) || + !required.every((entry) => typeof entry === 'string') || + new Set(required).size !== required.length) + ) { + return false; + } + const enumValues = item['enum']; + if ( + enumValues !== undefined && + (!Array.isArray(enumValues) || + enumValues.length === 0 || + !hasUniqueValues(enumValues)) + ) { + return false; + } + + for (const key of ['properties', 'patternProperties', 'definitions']) { + const map = item[key]; + if (map === undefined) continue; + if (!map || Array.isArray(map) || typeof map !== 'object') return false; + if ( + key === 'patternProperties' && + !Object.keys(map).every(isValidRegex) + ) { + return false; + } + for (const schema of Object.values(map)) { + if (!visitSchema(schema)) return false; + } + } + + for (const key of [ + 'additionalProperties', + 'additionalItems', + 'contains', + 'propertyNames', + 'not', + 'if', + 'then', + 'else', + ]) { + const schema = item[key]; + if (schema !== undefined && !visitSchema(schema)) return false; + } + + const items = item['items']; + if (items !== undefined) { + const schemas = Array.isArray(items) ? items : [items]; + if (schemas.length === 0 || !schemas.every(visitSchema)) return false; + } + + for (const key of ['allOf', 'anyOf', 'oneOf']) { + const schemas = item[key]; + if (schemas === undefined) continue; + if ( + !Array.isArray(schemas) || + schemas.length === 0 || + !schemas.every(visitSchema) + ) { + return false; + } + } + + const dependencies = item['dependencies']; + if (dependencies !== undefined) { + if ( + !dependencies || + Array.isArray(dependencies) || + typeof dependencies !== 'object' + ) { + return false; + } + for (const dependency of Object.values(dependencies)) { + if (Array.isArray(dependency)) { + if ( + !dependency.every((entry) => typeof entry === 'string') || + new Set(dependency).size !== dependency.length + ) { + return false; + } + } else if (!visitSchema(dependency)) { + return false; + } + } + } + return true; + } + return visitSchema(converted) ? converted : undefined; +} + +function definition( + type: unknown, + name: unknown, + description: unknown, + parameters: unknown, +): JsonObject | undefined { + if (typeof type !== 'string' || !type || typeof name !== 'string' || !name) { + return undefined; + } + return { + type, + name, + ...(typeof description === 'string' ? { description } : {}), + ...(parameters !== undefined + ? { parameters: normalizeSchema(parameters) } + : {}), + }; +} + +function compactUndefined(value: JsonObject): JsonObject { + return Object.fromEntries( + Object.entries(value).filter(([, item]) => item !== undefined), + ) as JsonObject; +} + +function openAiTools(value: unknown): JsonObject[] | undefined { + if (!Array.isArray(value)) return undefined; + const result: JsonObject[] = []; + for (const item of value) { + const tool = record(item); + if (!tool) return undefined; + const type = string(tool['type']); + const fn = record(tool['function']); + const entry = + type === 'function' && fn + ? definition( + 'function', + fn['name'], + fn['description'], + fn['parameters'], + ) + : definition(type, tool['name'], undefined, undefined); + if (!entry) return undefined; + result.push(compactUndefined(entry)); + } + return result; +} + +function anthropicTools(value: unknown): JsonObject[] | undefined { + if (!Array.isArray(value)) return undefined; + const result: JsonObject[] = []; + for (const item of value) { + const tool = record(item); + const entry = tool + ? definition( + 'function', + tool['name'], + tool['description'], + tool['input_schema'], + ) + : undefined; + if (!entry) return undefined; + result.push(compactUndefined(entry)); + } + return result; +} + +function geminiTools(value: unknown): JsonObject[] | undefined { + if (!Array.isArray(value)) return undefined; + const result: JsonObject[] = []; + for (const wrapper of value) { + const declarations = record(wrapper)?.['functionDeclarations']; + if (!Array.isArray(declarations)) return undefined; + for (const item of declarations) { + const tool = record(item); + const hasParameters = tool ? Object.hasOwn(tool, 'parameters') : false; + const hasJsonSchema = tool + ? Object.hasOwn(tool, 'parametersJsonSchema') + : false; + const parameters = + hasParameters && hasJsonSchema + ? undefined + : hasJsonSchema + ? tool?.['parametersJsonSchema'] + : tool?.['parameters']; + const entry = tool + ? definition('function', tool['name'], tool['description'], parameters) + : undefined; + if (!entry) return undefined; + result.push(compactUndefined(entry)); + } + } + return result; +} + +export function extractOpenAiContent(request: object): GenAiContentAttributes { + const value = request as Record; + return { + inputMessages: Object.hasOwn(value, 'messages') + ? messages(value['messages'], openAiPart, (role) => + role === 'function' ? 'tool' : role, + ) + : undefined, + toolDefinitions: Object.hasOwn(value, 'tools') + ? openAiTools(value['tools']) + : undefined, + }; +} + +export function extractAnthropicContent( + request: object, +): GenAiContentAttributes { + const value = request as Record; + return { + inputMessages: Object.hasOwn(value, 'messages') + ? messages(value['messages'], anthropicPart, (role) => role) + : undefined, + systemInstructions: Object.hasOwn(value, 'system') + ? systemParts(value['system'], anthropicPart) + : undefined, + toolDefinitions: Object.hasOwn(value, 'tools') + ? anthropicTools(value['tools']) + : undefined, + }; +} + +export function extractGeminiContent(request: object): GenAiContentAttributes { + const value = request as Record; + const config = record(value['config']); + return { + inputMessages: Object.hasOwn(value, 'contents') + ? messages(value['contents'], geminiPart, (role) => + role === 'model' ? 'assistant' : role, + ) + : undefined, + systemInstructions: + config && Object.hasOwn(config, 'systemInstruction') + ? systemParts(config['systemInstruction'], geminiPart) + : undefined, + toolDefinitions: + config && Object.hasOwn(config, 'tools') + ? geminiTools(config['tools']) + : undefined, + }; +} + +export function stringifyGenAiJson( + value: unknown, + maxLength: number, + requireObject = false, +): string | undefined { + const converted = jsonValue(value); + if ( + converted === undefined || + (requireObject && + (converted === null || + Array.isArray(converted) || + typeof converted !== 'object')) + ) { + return undefined; + } + try { + const serialized = JSON.stringify(converted); + return serialized.length <= maxLength ? serialized : undefined; + } catch { + return undefined; + } +} + +export class GenAiOutputAccumulator { + private candidates = new Map(); + private overflow = false; + private observedResponse = false; + private explicitEmpty = false; + private estimatedLength = 2; + + constructor( + private readonly enabled: boolean, + private readonly maxLength: number, + ) {} + + get finishReasons(): string[] | undefined { + const values = [...this.candidates.entries()] + .sort(([left], [right]) => left - right) + .map(([, candidate]) => candidate.finishReason) + .filter((value): value is string => value !== undefined); + return values.length > 0 ? values : undefined; + } + + recordOpenAiResponse(response: object): void { + const choices = (response as Record)['choices']; + if (!Array.isArray(choices)) return; + this.observedResponse = true; + this.explicitEmpty = choices.length === 0; + for (const [position, rawChoice] of choices.entries()) { + const choice = record(rawChoice); + if (!choice) { + if (this.enabled) this.markOverflow(); + continue; + } + const index = + typeof choice['index'] === 'number' && + Number.isSafeInteger(choice['index']) + ? choice['index'] + : position; + const candidate = this.candidate(index); + const message = record(choice['message']); + if (message && this.enabled) { + const converted = messages([message], openAiPart, () => 'assistant'); + if (converted?.[0]) this.setComplete(index, converted[0]); + else this.markOverflow(); + } else if (this.enabled) { + this.markOverflow(); + } + const finishReason = string(choice['finish_reason']); + if (finishReason) candidate.finishReason = finishReason; + } + } + + recordOpenAiChunk(chunk: object): void { + const choices = (chunk as Record)['choices']; + if (!Array.isArray(choices)) return; + this.observedResponse = true; + if (choices.length === 0 && this.candidates.size === 0) { + this.explicitEmpty = true; + } + for (const [position, rawChoice] of choices.entries()) { + const choice = record(rawChoice); + if (!choice) { + if (this.enabled) this.markOverflow(); + continue; + } + const index = + typeof choice['index'] === 'number' && + Number.isSafeInteger(choice['index']) + ? choice['index'] + : position; + const candidate = this.candidate(index); + const delta = record(choice['delta']); + const reasoning = + string(delta?.['reasoning_content']) ?? + string(delta?.['reasoning']) ?? + string(delta?.['thinking']); + if (reasoning !== undefined) { + this.append(candidate, 'reasoning', 'reasoning', reasoning); + } + const content = string(delta?.['content']); + if (content !== undefined) { + this.append(candidate, 'text', 'text', content); + } + const refusal = string(delta?.['refusal']); + if (refusal !== undefined) { + this.append(candidate, 'refusal', 'refusal', refusal); + } + const audio = record(delta?.['audio']); + if (audio) { + const converted = record(jsonValue(audio)); + if (converted) { + this.setValue(candidate, 'audio', { + ...converted, + type: 'audio', + }); + } else { + this.markOverflow(); + } + } + const toolCalls = delta?.['tool_calls']; + if (Array.isArray(toolCalls)) { + for (const [toolPosition, rawToolCall] of toolCalls.entries()) { + const toolCall = record(rawToolCall); + if (!toolCall) { + this.markOverflow(); + continue; + } + const toolIndex = + typeof toolCall['index'] === 'number' && + Number.isSafeInteger(toolCall['index']) + ? toolCall['index'] + : toolPosition; + const key = `tool:${toolIndex}`; + const fn = record(toolCall['function']); + const existing = candidate.parts.get(key); + const part = existing ?? { + type: 'tool_call' as const, + }; + if ( + !existing && + !this.reserve(JSON.stringify({ type: 'tool_call' }).length) + ) { + continue; + } + const id = string(toolCall['id']); + if ( + id !== undefined && + id !== part.id && + !this.reserve(id.length - (part.id?.length ?? 0)) + ) { + continue; + } + const name = string(fn?.['name']); + if ( + name !== undefined && + name !== part.name && + !this.reserve(name.length - (part.name?.length ?? 0)) + ) { + continue; + } + part.id = id ?? part.id; + part.name = name ?? part.name; + const argumentsFragment = string(fn?.['arguments']); + if (argumentsFragment !== undefined) { + const next = (part.arguments ?? '') + argumentsFragment; + if (!this.reserve(argumentsFragment.length)) continue; + part.arguments = next; + } + candidate.parts.set(key, part); + } + } + const finishReason = string(choice['finish_reason']); + if (finishReason) candidate.finishReason = finishReason; + } + } + + recordAnthropicResponse(response: object): void { + this.observedResponse = true; + const value = response as Record; + const candidate = this.candidate(0); + const content = value['content']; + if (Array.isArray(content) && this.enabled) { + const parts: CanonicalPart[] = []; + for (const item of content) { + const converted = anthropicPart(item); + if (!converted || converted.role) { + this.markOverflow(); + break; + } + parts.push(converted.part); + } + if (!this.overflow) { + this.setComplete(0, { role: 'assistant', parts }); + } + } else if (this.enabled) { + this.markOverflow(); + } + const finishReason = string(value['stop_reason']); + if (finishReason) candidate.finishReason = finishReason; + } + + recordAnthropicEvent(event: object): void { + const value = event as Record; + const type = string(value['type']); + if ( + type !== 'message_start' && + type !== 'content_block_start' && + type !== 'content_block_delta' && + type !== 'message_delta' + ) { + return; + } + this.observedResponse = true; + const index = + typeof value['index'] === 'number' && Number.isSafeInteger(value['index']) + ? value['index'] + : 0; + const candidate = this.candidate(0); + if (type === 'content_block_start') { + const block = record(value['content_block']); + const blockType = string(block?.['type']); + if (blockType === 'text') { + this.append( + candidate, + `block:${index}`, + 'text', + string(block?.['text']), + ); + } else if (blockType === 'thinking') { + this.append( + candidate, + `block:${index}`, + 'reasoning', + string(block?.['thinking']) ?? string(block?.['data']), + ); + } else if (blockType === 'redacted_thinking' && block) { + const converted = anthropicPart(block); + if (converted && !converted.role) { + this.setValue(candidate, `block:${index}`, converted.part); + } else { + this.markOverflow(); + } + } else if (blockType === 'tool_use') { + if (!this.reserve(JSON.stringify({ type: 'tool_call' }).length)) return; + const id = string(block?.['id']); + const name = string(block?.['name']); + if (!this.reserve((id?.length ?? 0) + (name?.length ?? 0))) return; + const hasInput = block ? Object.hasOwn(block, 'input') : false; + const input = hasInput ? jsonValue(block?.['input']) : undefined; + if (hasInput && input === undefined) { + this.markOverflow(); + return; + } + const initialArguments = + input === undefined ? undefined : JSON.stringify(input); + if ( + initialArguments !== undefined && + !this.reserve(initialArguments.length) + ) { + return; + } + candidate.parts.set(`block:${index}`, { + type: 'tool_call', + id, + name, + arguments: initialArguments, + argumentsFromStart: initialArguments !== undefined, + }); + } else if (blockType && block) { + const converted = anthropicPart(block); + if (converted && !converted.role) { + this.setValue(candidate, `block:${index}`, converted.part); + } else { + this.markOverflow(); + } + } + } else if (type === 'content_block_delta') { + const delta = record(value['delta']); + const deltaType = string(delta?.['type']); + if (deltaType === 'text_delta') { + this.append( + candidate, + `block:${index}`, + 'text', + string(delta?.['text']), + ); + } else if ( + deltaType === 'thinking_delta' || + deltaType === 'signature_delta' + ) { + if (deltaType === 'thinking_delta') { + this.append( + candidate, + `block:${index}`, + 'reasoning', + string(delta?.['thinking']), + ); + } + } else if (deltaType === 'input_json_delta') { + const part = candidate.parts.get(`block:${index}`); + const fragment = string(delta?.['partial_json']); + if (part?.type === 'tool_call' && fragment !== undefined) { + const replacedLength = part.argumentsFromStart + ? (part.arguments?.length ?? 0) + : 0; + if (this.reserve(fragment.length - replacedLength)) { + if (part.argumentsFromStart) { + part.arguments = ''; + part.argumentsFromStart = false; + } + part.arguments = (part.arguments ?? '') + fragment; + } + } + } + } else if (type === 'message_delta') { + const delta = record(value['delta']); + const finishReason = string(delta?.['stop_reason']); + if (finishReason) candidate.finishReason = finishReason; + } + } + + recordGeminiResponse(response: object): void { + const candidates = (response as Record)['candidates']; + if (!Array.isArray(candidates)) return; + this.observedResponse = true; + this.explicitEmpty = candidates.length === 0; + for (const [position, rawCandidate] of candidates.entries()) { + const candidate = record(rawCandidate); + if (!candidate) { + if (this.enabled) this.markOverflow(); + continue; + } + const index = + typeof candidate['index'] === 'number' && + Number.isSafeInteger(candidate['index']) + ? candidate['index'] + : position; + const outputCandidate = this.candidate(index); + const content = record(candidate['content']); + if (content && this.enabled) { + const converted = messages([content], geminiPart, (role) => + role === 'model' ? 'assistant' : role, + ); + if (converted?.[0]) this.setComplete(index, converted[0]); + else this.markOverflow(); + } else if (this.enabled) { + this.setComplete(index, { role: 'assistant', parts: [] }); + } + const finishReason = string(candidate['finishReason']); + if (finishReason) outputCandidate.finishReason = finishReason; + } + } + + recordGeminiChunk(chunk: object): void { + const candidates = (chunk as Record)['candidates']; + if (!Array.isArray(candidates)) return; + this.observedResponse = true; + if (candidates.length === 0 && this.candidates.size === 0) { + this.explicitEmpty = true; + } + for (const [position, rawCandidate] of candidates.entries()) { + const value = record(rawCandidate); + if (!value) { + if (this.enabled) this.markOverflow(); + continue; + } + const index = + typeof value['index'] === 'number' && + Number.isSafeInteger(value['index']) + ? value['index'] + : position; + const candidate = this.candidate(index); + const content = record(value['content']); + const role = string(content?.['role']); + if (role) candidate.role = role === 'model' ? 'assistant' : role; + const parts = content?.['parts']; + if (Array.isArray(parts) && this.enabled && !this.overflow) { + for (const [partIndex, rawPart] of parts.entries()) { + const converted = geminiPart(rawPart); + if (!converted || converted.role) { + this.markOverflow(); + break; + } + const part = converted.part; + const type = string(part['type']); + const key = `part:${partIndex}:${type ?? 'unknown'}`; + if (type === 'text' || type === 'reasoning') { + this.append(candidate, key, type, string(part['content']) ?? ''); + } else { + this.setValue(candidate, key, part); + } + } + } + const finishReason = string(value['finishReason']); + if (finishReason) candidate.finishReason = finishReason; + } + } + + finalize(success: boolean): string | undefined { + if (!success) { + for (const candidate of this.candidates.values()) { + candidate.finishReason ??= 'error'; + } + } + if (!this.enabled || this.overflow || !this.observedResponse) { + return undefined; + } + if (this.explicitEmpty && this.candidates.size === 0) { + return stringifyGenAiJson([], this.maxLength); + } + const output: JsonObject[] = []; + for (const [, candidate] of [...this.candidates.entries()].sort( + ([left], [right]) => left - right, + )) { + const finishReason = candidate.finishReason; + if (!finishReason) { + debugLogger.debug( + 'Omitting GenAI output messages because a candidate has no finish reason', + ); + return undefined; + } + const parts: CanonicalPart[] = []; + for (const part of candidate.parts.values()) { + if (part.value) { + parts.push(part.value); + continue; + } + if (part.type === 'tool_call') { + if (!part.name) return undefined; + const argumentsValue = + part.arguments === undefined + ? undefined + : parseArguments(part.arguments); + parts.push({ + type: 'tool_call', + ...(part.id ? { id: part.id } : {}), + name: part.name, + ...(argumentsValue !== undefined + ? { arguments: argumentsValue } + : {}), + }); + } else { + parts.push({ + type: part.type, + content: part.content ?? '', + }); + } + } + output.push({ + role: candidate.role, + parts, + finish_reason: finishReason, + }); + } + return stringifyGenAiJson(output, this.maxLength); + } + + discardContent(): void { + this.markOverflow(); + } + + private candidate(index: number): StreamCandidate { + let candidate = this.candidates.get(index); + if (!candidate) { + candidate = { role: 'assistant', parts: new Map() }; + this.candidates.set(index, candidate); + } + return candidate; + } + + private setComplete(index: number, message: JsonObject): void { + const candidate = this.candidate(index); + candidate.role = string(message['role']) ?? 'assistant'; + candidate.parts.clear(); + const parts = message['parts']; + if (!Array.isArray(parts)) { + this.markOverflow(); + return; + } + for (const [partIndex, value] of parts.entries()) { + const part = record(value); + const type = string(part?.['type']); + if (!part || !type) { + this.markOverflow(); + return; + } + if (type === 'text' || type === 'reasoning') { + candidate.parts.set(`part:${partIndex}`, { + type, + content: string(part['content']) ?? '', + }); + } else if (type === 'tool_call') { + candidate.parts.set(`part:${partIndex}`, { + type: 'tool_call', + id: string(part['id']), + name: string(part['name']), + arguments: + part['arguments'] === undefined + ? undefined + : JSON.stringify(part['arguments']), + }); + } else { + candidate.parts.set(`part:${partIndex}`, { + type, + value: part as CanonicalPart, + }); + } + } + const serialized = stringifyGenAiJson(message, this.maxLength); + if (serialized === undefined) this.markOverflow(); + } + + private append( + candidate: StreamCandidate, + key: string, + type: 'text' | 'reasoning' | 'refusal', + fragment: string | undefined, + ): void { + if (fragment === undefined || !this.enabled || this.overflow) return; + const part = candidate.parts.get(key); + if (part && part.type !== type) { + this.markOverflow(); + return; + } + const newPartLength = part + ? 0 + : JSON.stringify({ type, content: '' }).length; + if (!this.reserve(fragment.length + newPartLength)) return; + if (part) { + part.content = (part.content ?? '') + fragment; + } else { + candidate.parts.set(key, { type, content: fragment }); + } + } + + private setValue( + candidate: StreamCandidate, + key: string, + value: CanonicalPart, + ): void { + const previous = candidate.parts.get(key)?.value; + const previousLength = previous ? JSON.stringify(previous).length : 0; + if (this.reserve(JSON.stringify(value).length - previousLength)) { + candidate.parts.set(key, { type: value.type, value }); + } + } + + private reserve(length: number): boolean { + if (!this.enabled || this.overflow) return false; + // This is a cheap lower-bound memory guard. finalize() applies the exact + // compact-JSON length limit, including string escaping. + this.estimatedLength += length; + if (this.estimatedLength <= this.maxLength) return true; + this.markOverflow(); + return false; + } + + private markOverflow(): void { + this.overflow = true; + for (const candidate of this.candidates.values()) candidate.parts.clear(); + } +} diff --git a/packages/core/src/telemetry/gen-ai-exchange.test.ts b/packages/core/src/telemetry/gen-ai-exchange.test.ts new file mode 100644 index 00000000000..0b7c79b45f4 --- /dev/null +++ b/packages/core/src/telemetry/gen-ai-exchange.test.ts @@ -0,0 +1,402 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + context, + ROOT_CONTEXT, + type Attributes, + type Context, + type Span, + type SpanContext, +} from '@opentelemetry/api'; +import { describe, expect, it } from 'vitest'; +import { + createGenAiExchange, + reportAnthropicFollowingRequest, + reportAnthropicRequest, + reportAnthropicResponse, + reportOpenAiChunk, + reportOpenAiRequest, + reportOpenAiResponse, +} from './gen-ai-request.js'; + +interface MockSpan extends Span { + attributes: Record; +} + +function span(recording = true): MockSpan { + const attributes: Record = {}; + return { + attributes, + setAttributes(values: Attributes) { + Object.assign(attributes, values); + return this; + }, + setAttribute(key: string, value: unknown) { + attributes[key] = value; + return this; + }, + isRecording: () => recording, + spanContext(): SpanContext { + return { + traceId: '0'.repeat(32), + spanId: '0'.repeat(16), + traceFlags: 0, + }; + }, + setStatus() { + return this; + }, + end() {}, + updateName() { + return this; + }, + recordException() { + return this; + }, + addEvent() { + return this; + }, + addLink() { + return this; + }, + addLinks() { + return this; + }, + }; +} + +function exchange(target: Span, captureContent = true) { + return createGenAiExchange(ROOT_CONTEXT, target, { + captureContent, + sensitiveAttributeMaxLength: 10_000, + }); +} + +describe('GenAI exchange observer', () => { + it('records provider-final request content and response content', () => { + const target = span(); + const observed = exchange(target); + const attempt = reportOpenAiRequest( + { + temperature: 0.2, + messages: [{ role: 'user', content: 'hello' }], + tools: [], + }, + observed.context, + ); + reportOpenAiResponse(attempt, { + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'answer' }, + finish_reason: 'stop', + }, + ], + }); + + expect(observed.controller.finalize(true)).toEqual(['stop']); + expect(target.attributes['gen_ai.request.temperature']).toBe(0.2); + expect( + JSON.parse(target.attributes['gen_ai.input.messages'] as string), + ).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'hello' }], + }, + ]); + expect(target.attributes['gen_ai.tool.definitions']).toBe('[]'); + expect( + JSON.parse(target.attributes['gen_ai.output.messages'] as string), + ).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'answer' }], + finish_reason: 'stop', + }, + ]); + }); + + it('keeps the first request snapshot and latest response attempt', () => { + const target = span(); + const observed = exchange(target); + const first = reportOpenAiRequest( + { messages: [{ role: 'user', content: 'first' }] }, + observed.context, + ); + reportOpenAiChunk(first, { + choices: [{ index: 0, delta: { content: 'old' } }], + }); + const second = reportOpenAiRequest( + { messages: [{ role: 'user', content: 'second' }] }, + observed.context, + ); + reportOpenAiChunk(first, { + choices: [ + { index: 0, delta: { content: 'late' }, finish_reason: 'stop' }, + ], + }); + reportOpenAiChunk(second, { + choices: [ + { index: 0, delta: { content: 'new' }, finish_reason: 'length' }, + ], + }); + observed.controller.finalize(true); + + expect(target.attributes['gen_ai.input.messages']).toContain('first'); + expect(target.attributes['gen_ai.input.messages']).not.toContain('second'); + expect(target.attributes['gen_ai.output.messages']).toContain('new'); + expect(target.attributes['gen_ai.output.messages']).not.toContain('old'); + expect(target.attributes['gen_ai.output.messages']).not.toContain('late'); + }); + + it('starts a fallback attempt from its handle after context exit', () => { + const target = span(); + const observed = exchange(target); + const streamingAttempt = reportAnthropicRequest( + { messages: [{ role: 'user', content: 'initial' }], stream: true }, + observed.context, + ); + const fallbackAttempt = reportAnthropicFollowingRequest( + { messages: [{ role: 'user', content: 'fallback' }] }, + streamingAttempt, + ); + reportAnthropicResponse(fallbackAttempt, { + content: [{ type: 'text', text: 'fallback answer' }], + stop_reason: 'end_turn', + }); + + expect(observed.controller.finalize(true)).toEqual(['end_turn']); + expect(target.attributes['gen_ai.input.messages']).toContain('initial'); + expect(target.attributes['gen_ai.input.messages']).not.toContain( + 'fallback', + ); + expect(target.attributes['gen_ai.output.messages']).toContain( + 'fallback answer', + ); + }); + + it('does not recover a missing fallback handle from the active context', () => { + const target = span(); + const outer = createGenAiExchange(ROOT_CONTEXT, target, { + captureContent: true, + sensitiveAttributeMaxLength: 10_000, + }); + + context.with(outer.context, () => { + expect( + reportAnthropicFollowingRequest( + { messages: [{ role: 'user', content: 'fallback' }] }, + undefined, + ), + ).toBeUndefined(); + }); + + expect(target.attributes).toEqual({}); + }); + + it('consumes an empty first snapshot', () => { + const target = span(); + const observed = exchange(target); + reportOpenAiRequest({}, observed.context); + reportOpenAiRequest( + { + temperature: 0.9, + messages: [{ role: 'user', content: 'later' }], + }, + observed.context, + ); + expect(target.attributes).toEqual({}); + }); + + it('shadows an outer observer for a non-recording nested span', () => { + const outerSpan = span(); + const outer = exchange(outerSpan); + const inner = createGenAiExchange(outer.context, span(false), { + captureContent: true, + sensitiveAttributeMaxLength: 10_000, + }); + + const innerAttempt = reportOpenAiRequest( + { temperature: 0.9 }, + inner.context, + ); + expect(innerAttempt).toBeUndefined(); + expect(outerSpan.attributes).toEqual({}); + + reportOpenAiRequest({ temperature: 0.1 }, outer.context); + expect(outerSpan.attributes['gen_ai.request.temperature']).toBe(0.1); + }); + + it('does not fall back to an outer observer when context installation fails', () => { + const outerSpan = span(); + const outer = exchange(outerSpan); + const brokenParent: Context = { + getValue: (key) => outer.context.getValue(key), + setValue: () => { + throw new Error('context write failed'); + }, + deleteValue: () => { + throw new Error('context write failed'); + }, + }; + const inner = createGenAiExchange(brokenParent, span(), { + captureContent: true, + sensitiveAttributeMaxLength: 10_000, + }); + + expect( + reportOpenAiRequest({ temperature: 0.9 }, inner.context), + ).toBeUndefined(); + expect(outerSpan.attributes).toEqual({}); + }); + + it('isolates concurrent exchange contexts and attempt handles', () => { + const leftSpan = span(); + const rightSpan = span(); + const left = exchange(leftSpan); + const right = exchange(rightSpan); + const leftAttempt = reportOpenAiRequest( + { messages: [{ role: 'user', content: 'left-input' }] }, + left.context, + ); + const rightAttempt = reportOpenAiRequest( + { messages: [{ role: 'user', content: 'right-input' }] }, + right.context, + ); + reportOpenAiChunk(rightAttempt, { + choices: [ + { index: 0, delta: { content: 'right-output' }, finish_reason: 'stop' }, + ], + }); + reportOpenAiChunk(leftAttempt, { + choices: [ + { index: 0, delta: { content: 'left-output' }, finish_reason: 'stop' }, + ], + }); + left.controller.finalize(true); + right.controller.finalize(true); + + expect(leftSpan.attributes['gen_ai.input.messages']).toContain( + 'left-input', + ); + expect(leftSpan.attributes['gen_ai.output.messages']).toContain( + 'left-output', + ); + expect(leftSpan.attributes['gen_ai.output.messages']).not.toContain( + 'right-output', + ); + expect(rightSpan.attributes['gen_ai.input.messages']).toContain( + 'right-input', + ); + expect(rightSpan.attributes['gen_ai.output.messages']).toContain( + 'right-output', + ); + }); + + it('uses an explicit handle after the creating context has exited', () => { + const target = span(); + const observed = exchange(target); + const attempt = reportOpenAiRequest({ messages: [] }, observed.context); + reportOpenAiChunk(attempt, { + choices: [ + { index: 0, delta: { content: 'outside' }, finish_reason: 'stop' }, + ], + }); + observed.controller.finalize(true); + expect(target.attributes['gen_ai.output.messages']).toContain('outside'); + }); + + it('invalidates handles and makes finalize idempotent', () => { + const target = span(); + const observed = exchange(target); + const attempt = reportOpenAiRequest({ messages: [] }, observed.context); + reportOpenAiChunk(attempt, { + choices: [ + { index: 0, delta: { content: 'before' }, finish_reason: 'stop' }, + ], + }); + expect(observed.controller.finalize(true)).toEqual(['stop']); + reportOpenAiChunk(attempt, { + choices: [ + { index: 0, delta: { content: 'after' }, finish_reason: 'length' }, + ], + }); + expect(observed.controller.finalize(false)).toBeUndefined(); + expect(target.attributes['gen_ai.output.messages']).not.toContain('after'); + }); + + it('does not let span API failures affect reporting', () => { + const target = span(); + target.setAttributes = () => { + throw new Error('setAttributes failed'); + }; + target.setAttribute = () => { + throw new Error('setAttribute failed'); + }; + const observed = exchange(target); + expect(() => + reportOpenAiRequest( + { + temperature: 0.1, + messages: [{ role: 'user', content: 'secret' }], + }, + observed.context, + ), + ).not.toThrow(); + expect(() => observed.controller.finalize(false)).not.toThrow(); + }); + + it('omits response content when conversion throws after a partial update', () => { + const target = span(); + const observed = exchange(target); + const attempt = reportOpenAiRequest({ messages: [] }, observed.context); + const brokenMessage = new Proxy( + {}, + { + get: () => { + throw new Error('response conversion failed'); + }, + }, + ); + reportOpenAiResponse(attempt, { + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'partial' }, + finish_reason: 'stop', + }, + { + index: 1, + message: brokenMessage, + finish_reason: 'stop', + }, + ], + }); + + expect(observed.controller.finalize(true)).toBeUndefined(); + expect(target.attributes['gen_ai.output.messages']).toBeUndefined(); + }); + + it('captures non-sensitive request fields while content capture is off', () => { + const target = span(); + const observed = exchange(target, false); + const attempt = reportOpenAiRequest( + { + temperature: 0.3, + messages: [{ role: 'user', content: 'secret' }], + }, + observed.context, + ); + reportOpenAiChunk(attempt, { + choices: [{ index: 0, delta: { content: 'partial' } }], + }); + expect(target.attributes['gen_ai.request.temperature']).toBe(0.3); + expect(target.attributes['gen_ai.input.messages']).toBeUndefined(); + expect(observed.controller.finalize(false)).toEqual(['error']); + expect(target.attributes['gen_ai.output.messages']).toBeUndefined(); + }); +}); diff --git a/packages/core/src/telemetry/gen-ai-request.test.ts b/packages/core/src/telemetry/gen-ai-request.test.ts index afdcc7f66f6..54ee77a0262 100644 --- a/packages/core/src/telemetry/gen-ai-request.test.ts +++ b/packages/core/src/telemetry/gen-ai-request.test.ts @@ -248,9 +248,9 @@ describe('GenAI request observer', () => { throw new Error('setValue failed'); }, } as unknown as typeof ROOT_CONTEXT; - expect(createGenAiRequestObserverContext(brokenParent, span)).toBe( - brokenParent, - ); + expect(() => + createGenAiRequestObserverContext(brokenParent, span), + ).not.toThrow(); const brokenReporterContext = { getValue: () => { diff --git a/packages/core/src/telemetry/gen-ai-request.ts b/packages/core/src/telemetry/gen-ai-request.ts index c8ccb4a76d6..571c44be618 100644 --- a/packages/core/src/telemetry/gen-ai-request.ts +++ b/packages/core/src/telemetry/gen-ai-request.ts @@ -7,17 +7,43 @@ import { context, createContextKey, + ROOT_CONTEXT, type Attributes, type Context, type Span, } from '@opentelemetry/api'; +import { + extractAnthropicContent, + extractGeminiContent, + extractOpenAiContent, + GenAiOutputAccumulator, + stringifyGenAiJson, + type GenAiContentAttributes, +} from './gen-ai-content.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; -type RequestObserver = (attributes: Attributes) => void; type RequestRecord = Record; +const debugLogger = createDebugLogger('GEN_AI_EXCHANGE'); const requestObserverKey = createContextKey( 'qwen-code.gen-ai-request-observer', ); +const DISABLED_OBSERVER = Symbol('disabled-gen-ai-exchange'); + +export interface GenAiExchangeOptions { + captureContent: boolean; + sensitiveAttributeMaxLength: number; +} + +export interface GenAiAttemptHandle { + readonly controller: GenAiExchangeController; + readonly generation: number; +} + +export interface GenAiExchange { + context: Context; + controller: GenAiExchangeController; +} function ownValue(record: RequestRecord, key: string): unknown | undefined { return Object.hasOwn(record, key) ? record[key] : undefined; @@ -177,92 +203,368 @@ export function extractGeminiRequestAttributes(request: object): Attributes { return attributes; } -export function createGenAiRequestObserverContext( - parent: Context, - span: Span, -): Context { - try { - if (!span.isRecording()) return parent; - } catch { - return parent; +export class GenAiExchangeController { + private requestConsumed = false; + private generation = 0; + private finalized = false; + private output: GenAiOutputAccumulator; + private responseConversionFailed = false; + + constructor( + private readonly span: Span, + private readonly options: GenAiExchangeOptions, + private readonly enabled: boolean, + ) { + this.output = this.newOutput(); } - let consumed = false; - const observer: RequestObserver = (attributes) => { - if (consumed) return; - consumed = true; + beginRequest( + request: object, + extractRequest: (request: object) => Attributes, + extractContent: (request: object) => GenAiContentAttributes, + ): GenAiAttemptHandle | undefined { + if (!this.enabled || this.finalized) return undefined; + const generation = ++this.generation; + this.output = this.newOutput(); + this.responseConversionFailed = false; + + if (!this.requestConsumed) { + this.requestConsumed = true; + const attributes: Attributes = {}; + try { + Object.assign(attributes, extractRequest(request)); + if (this.options.captureContent) { + const content = extractContent(request); + this.assignJsonAttribute( + attributes, + 'gen_ai.input.messages', + content.inputMessages, + ); + this.assignJsonAttribute( + attributes, + 'gen_ai.system_instructions', + content.systemInstructions, + ); + this.assignJsonAttribute( + attributes, + 'gen_ai.tool.definitions', + content.toolDefinitions, + ); + } + } catch { + debugLogger.debug('Failed to convert GenAI request attributes'); + // The first snapshot remains consumed when conversion fails. + } + try { + this.span.setAttributes(attributes); + } catch { + debugLogger.debug('Failed to set GenAI request span attributes'); + } + } + + return { controller: this, generation }; + } + + beginFollowingRequest( + handle: GenAiAttemptHandle, + request: object, + extractRequest: (request: object) => Attributes, + extractContent: (request: object) => GenAiContentAttributes, + ): GenAiAttemptHandle | undefined { + if ( + handle.controller !== this || + handle.generation !== this.generation || + this.finalized + ) { + return undefined; + } + return this.beginRequest(request, extractRequest, extractContent); + } + + record( + handle: GenAiAttemptHandle | undefined, + update: (output: GenAiOutputAccumulator) => void, + ): void { + if ( + !handle || + handle.controller !== this || + handle.generation !== this.generation || + this.finalized + ) { + return; + } try { - span.setAttributes(attributes); + update(this.output); } catch { - // Telemetry must not affect the provider request. + this.output.discardContent(); + this.responseConversionFailed = true; + debugLogger.debug('Failed to convert GenAI response content'); } - }; - try { - return parent.setValue(requestObserverKey, observer); - } catch { - return parent; + } + + finalize(success: boolean): string[] | undefined { + if (this.finalized) return undefined; + this.finalized = true; + let finishReasons: string[] | undefined; + try { + const outputMessages = this.output.finalize(success); + finishReasons = this.responseConversionFailed + ? undefined + : this.output.finishReasons; + if (outputMessages !== undefined) { + this.span.setAttribute('gen_ai.output.messages', outputMessages); + } + } catch { + debugLogger.debug('Failed to finalize GenAI response attributes'); + } finally { + this.output = this.newOutput(); + } + return finishReasons; + } + + private assignJsonAttribute( + attributes: Attributes, + key: string, + value: unknown, + ): void { + if (value === undefined) return; + let serialized: string | undefined; + try { + serialized = stringifyGenAiJson( + value, + this.options.sensitiveAttributeMaxLength, + ); + } catch { + debugLogger.debug(`Failed to serialize ${key} span attribute`); + return; + } + if (serialized !== undefined) attributes[key] = serialized; + } + + private newOutput(): GenAiOutputAccumulator { + return new GenAiOutputAccumulator( + this.options.captureContent, + this.options.sensitiveAttributeMaxLength, + ); } } -function reportRequest( - request: object, - extract: (request: object) => Attributes, - requestContext: Context, -): void { - let observer: unknown; +function disabledFallbackContext(parent: Context): Context { + // Preserve every other Context behavior through the parent prototype while + // shadowing key operations so a failed setValue cannot expose an ancestor + // observer to this exchange. + const fallback = Object.create(parent) as Context; + fallback.getValue = (key) => { + if (key === requestObserverKey) return DISABLED_OBSERVER; + try { + return parent.getValue(key); + } catch { + return undefined; + } + }; + fallback.setValue = (key, value) => { + if (key === requestObserverKey) return fallback; + try { + return disabledFallbackContext(parent.setValue(key, value)); + } catch { + return fallback; + } + }; + fallback.deleteValue = (key) => { + if (key === requestObserverKey) return fallback; + try { + return disabledFallbackContext(parent.deleteValue(key)); + } catch { + return fallback; + } + }; + return fallback; +} + +export function createGenAiExchange( + parent: Context, + span: Span, + options: GenAiExchangeOptions, +): GenAiExchange { + let enabled = false; try { - observer = requestContext.getValue(requestObserverKey); + enabled = span.isRecording(); } catch { - return; + enabled = false; } - if (typeof observer !== 'function') return; - - let attributes: Attributes = {}; + const controller = new GenAiExchangeController(span, options, enabled); try { - attributes = extract(request); + return { + context: parent.setValue( + requestObserverKey, + enabled ? controller : DISABLED_OBSERVER, + ), + controller, + }; } catch { - // Consume the first request snapshot even if extraction fails. + return { + context: + parent === ROOT_CONTEXT + ? ROOT_CONTEXT + : disabledFallbackContext(parent), + controller, + }; } +} + +/** + * @deprecated Use createGenAiExchange so response attempts can be finalized. + */ +export function createGenAiRequestObserverContext( + parent: Context, + span: Span, +): Context { + return createGenAiExchange(parent, span, { + captureContent: false, + sensitiveAttributeMaxLength: 1, + }).context; +} + +function activeController( + requestContext?: Context, +): GenAiExchangeController | undefined { + let observer: unknown; try { - (observer as RequestObserver)(attributes); + observer = (requestContext ?? context.active()).getValue( + requestObserverKey, + ); } catch { - // Telemetry must not affect the provider request. + return undefined; } + return observer instanceof GenAiExchangeController ? observer : undefined; } -function reportActiveRequest( +function reportRequest( request: object, - extract: (request: object) => Attributes, + extractRequest: (request: object) => Attributes, + extractContent: (request: object) => GenAiContentAttributes, requestContext?: Context, -): void { + previousAttempt?: GenAiAttemptHandle, +): GenAiAttemptHandle | undefined { try { - reportRequest(request, extract, requestContext ?? context.active()); + if (previousAttempt) { + return previousAttempt.controller.beginFollowingRequest( + previousAttempt, + request, + extractRequest, + extractContent, + ); + } + return activeController(requestContext)?.beginRequest( + request, + extractRequest, + extractContent, + ); } catch { - // Telemetry must not affect the provider request. + return undefined; } } export function reportOpenAiRequest( request: object, requestContext?: Context, -): void { - reportActiveRequest(request, extractOpenAiRequestAttributes, requestContext); +): GenAiAttemptHandle | undefined { + return reportRequest( + request, + extractOpenAiRequestAttributes, + extractOpenAiContent, + requestContext, + ); } export function reportAnthropicRequest( request: object, requestContext?: Context, -): void { - reportActiveRequest( +): GenAiAttemptHandle | undefined { + return reportRequest( request, extractAnthropicRequestAttributes, + extractAnthropicContent, requestContext, ); } +export function reportAnthropicFollowingRequest( + request: object, + previousAttempt: GenAiAttemptHandle | undefined, +): GenAiAttemptHandle | undefined { + if (!previousAttempt) return undefined; + return reportRequest( + request, + extractAnthropicRequestAttributes, + extractAnthropicContent, + undefined, + previousAttempt, + ); +} + export function reportGeminiRequest( request: object, requestContext?: Context, +): GenAiAttemptHandle | undefined { + return reportRequest( + request, + extractGeminiRequestAttributes, + extractGeminiContent, + requestContext, + ); +} + +export function reportOpenAiResponse( + handle: GenAiAttemptHandle | undefined, + response: object, +): void { + handle?.controller.record(handle, (output) => + output.recordOpenAiResponse(response), + ); +} + +export function reportOpenAiChunk( + handle: GenAiAttemptHandle | undefined, + chunk: object, +): void { + handle?.controller.record(handle, (output) => + output.recordOpenAiChunk(chunk), + ); +} + +export function reportAnthropicResponse( + handle: GenAiAttemptHandle | undefined, + response: object, +): void { + handle?.controller.record(handle, (output) => + output.recordAnthropicResponse(response), + ); +} + +export function reportAnthropicEvent( + handle: GenAiAttemptHandle | undefined, + event: object, +): void { + handle?.controller.record(handle, (output) => + output.recordAnthropicEvent(event), + ); +} + +export function reportGeminiResponse( + handle: GenAiAttemptHandle | undefined, + response: object, ): void { - reportActiveRequest(request, extractGeminiRequestAttributes, requestContext); + handle?.controller.record(handle, (output) => + output.recordGeminiResponse(response), + ); +} + +export function reportGeminiChunk( + handle: GenAiAttemptHandle | undefined, + chunk: object, +): void { + handle?.controller.record(handle, (output) => + output.recordGeminiChunk(chunk), + ); } diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index 5f6865b1f64..aea311b0044 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -239,6 +239,8 @@ export { addModelOutputAttributes, addToolInputAttributes, addToolResultAttributes, + addToolArgumentsAttributes, + addToolCallResultAttributes, areSensitiveSpanAttributesEnabled, truncateContent, } from './detailed-span-attributes.js'; diff --git a/packages/core/src/telemetry/session-tracing.test.ts b/packages/core/src/telemetry/session-tracing.test.ts index 63b91fc2550..ffc851efbd5 100644 --- a/packages/core/src/telemetry/session-tracing.test.ts +++ b/packages/core/src/telemetry/session-tracing.test.ts @@ -1093,6 +1093,25 @@ describe('session-tracing', () => { expect(mockSpans[0]!.statuses[0]!.code).toBe(SpanStatusCode.OK); }); + it('records and surrogate-safely bounds static tool descriptions', () => { + startToolSpan('Read', undefined, 'a'.repeat(4096)); + expect(mockSpans[0]!.attributes['gen_ai.tool.description']).toHaveLength( + 4096, + ); + + startToolSpan('Write', undefined, `${'a'.repeat(4095)}😀`); + expect(mockSpans[1]!.attributes['gen_ai.tool.description']).toBe( + `${'a'.repeat(4095)}…[truncated]`, + ); + }); + + it('omits an empty tool description', () => { + startToolSpan('Read', undefined, ''); + expect( + mockSpans[0]!.attributes['gen_ai.tool.description'], + ).toBeUndefined(); + }); + it('records error on tool failure', () => { const span = startToolSpan('Bash'); endToolSpan(span, { success: false, error: 'command failed' }); diff --git a/packages/core/src/telemetry/session-tracing.ts b/packages/core/src/telemetry/session-tracing.ts index 6841eee1e16..d87ff4cba7f 100644 --- a/packages/core/src/telemetry/session-tracing.ts +++ b/packages/core/src/telemetry/session-tracing.ts @@ -26,7 +26,6 @@ import { SPAN_TOOL_BLOCKED_ON_USER, SPAN_TOOL_EXECUTION, } from './constants.js'; -import { clearDetailedSpanState } from './detailed-span-attributes.js'; import { ApiRequestPhase, recordApiRequestBreakdown } from './metrics.js'; import { isTelemetrySdkInitialized } from './sdk.js'; import { getCurrentSessionId, setSessionContext } from './session-context.js'; @@ -366,6 +365,7 @@ function getSpanId(span: Span): string { } const SPAN_TEXT_MAX_CHARS = 1024; +const TOOL_DESCRIPTION_MAX_CHARS = 4096; /** * Bound the size of error strings written to span attributes / status @@ -381,13 +381,13 @@ const SPAN_TEXT_MAX_CHARS = 1024; * ~32KB), so we keep the simpler char-count bound rather than paying * the encoder cost on every endXSpan. */ -function truncateSpanText(s: string): string { - if (s.length <= SPAN_TEXT_MAX_CHARS) return s; +function truncateSpanText(s: string, maxChars = SPAN_TEXT_MAX_CHARS): string { + if (s.length <= maxChars) return s; // Back up one code unit if the cut lands on a high surrogate so we // don't emit a lone surrogate followed by the sentinel — strict // OTLP/gRPC collectors reject span batches with invalid UTF-8 // (a lone high surrogate encodes to an invalid byte sequence). - let end = SPAN_TEXT_MAX_CHARS; + let end = maxChars; const code = s.charCodeAt(end - 1); if (code >= 0xd800 && code <= 0xdbff) end--; return s.slice(0, end) + '…[truncated]'; @@ -826,6 +826,7 @@ export function endLLMRequestSpan( export function startToolSpan( toolName: string, attrs?: Record, + description?: string, ): Span { if (!isTelemetrySdkInitialized()) { return NOOP_SPAN; @@ -845,6 +846,14 @@ export function startToolSpan( 'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': toolName, 'gen_ai.tool.type': 'function', + ...(description + ? { + 'gen_ai.tool.description': truncateSpanText( + description, + TOOL_DESCRIPTION_MAX_CHARS, + ), + } + : {}), }; const span = getTracer().startSpan( @@ -1660,7 +1669,6 @@ export function clearSessionTracingForTesting(): void { subagentContext.enterWith(undefined); interactionSequence = 0; lastInteractionCtx = undefined; - clearDetailedSpanState(); // Reach into session-context module to prevent cross-test leakage. setSessionContext(undefined); } diff --git a/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-input-messages.json b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-input-messages.json new file mode 100644 index 00000000000..1841dbb9b0a --- /dev/null +++ b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-input-messages.json @@ -0,0 +1,492 @@ +{ + "$defs": { + "BlobPart": { + "description": "Represents blob binary data sent inline to the model", + "properties": { + "type": { + "const": "blob", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "content": { + "description": "Raw bytes of the attached data. This field SHOULD be encoded as a base64 string when serialized to JSON.", + "format": "binary", + "title": "Content", + "type": "string" + } + }, + "required": ["type", "modality", "content"], + "title": "BlobPart", + "type": "object" + }, + "ChatMessage": { + "additionalProperties": true, + "properties": { + "role": { + "anyOf": [ + { + "$ref": "#/$defs/Role" + }, + { + "type": "string" + } + ], + "description": "Role of the entity that created the message.", + "title": "Role" + }, + "parts": { + "description": "List of message parts that make up the message content.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TextPart" + }, + { + "$ref": "#/$defs/ToolCallRequestPart" + }, + { + "$ref": "#/$defs/ToolCallResponsePart" + }, + { + "$ref": "#/$defs/ServerToolCallPart" + }, + { + "$ref": "#/$defs/ServerToolCallResponsePart" + }, + { + "$ref": "#/$defs/BlobPart" + }, + { + "$ref": "#/$defs/FilePart" + }, + { + "$ref": "#/$defs/UriPart" + }, + { + "$ref": "#/$defs/ReasoningPart" + }, + { + "$ref": "#/$defs/CompactionPart" + }, + { + "$ref": "#/$defs/GenericPart" + } + ] + }, + "title": "Parts", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The name of the participant.", + "title": "Name" + } + }, + "required": ["role", "parts"], + "title": "ChatMessage", + "type": "object" + }, + "CompactionPart": { + "additionalProperties": true, + "description": "Represents compacted conversation state sent to or received from the model.", + "properties": { + "type": { + "const": "compaction", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Provider-assigned identifier for the compaction item or block.", + "title": "Id" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The unencrypted compacted conversation summary, when available.", + "title": "Content" + } + }, + "required": ["type"], + "title": "CompactionPart", + "type": "object" + }, + "FilePart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by file id", + "properties": { + "type": { + "const": "file", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "file_id": { + "description": "An identifier referencing a file that was pre-uploaded to the provider.", + "title": "File Id", + "type": "string" + } + }, + "required": ["type", "modality", "file_id"], + "title": "FilePart", + "type": "object" + }, + "GenericPart": { + "additionalProperties": true, + "description": "Represents an arbitrary message part with any type and properties.\nThis allows for extensibility with custom message part types.", + "properties": { + "type": { + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + } + }, + "required": ["type"], + "title": "GenericPart", + "type": "object" + }, + "GenericServerToolCall": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call with any type and properties.\nThis allows for extensibility with custom server tool types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call.", + "title": "Type", + "type": "string" + } + }, + "required": ["type"], + "title": "GenericServerToolCall", + "type": "object" + }, + "GenericServerToolCallResponse": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call response with any type and properties.\nThis allows for extensibility with custom server tool response types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call response.", + "title": "Type", + "type": "string" + } + }, + "required": ["type"], + "title": "GenericServerToolCallResponse", + "type": "object" + }, + "Modality": { + "enum": ["image", "video", "audio", "document"], + "title": "Modality", + "type": "string" + }, + "ReasoningPart": { + "additionalProperties": true, + "description": "Represents reasoning/thinking content received from the model.", + "properties": { + "type": { + "const": "reasoning", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Reasoning/thinking content received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": ["type", "content"], + "title": "ReasoningPart", + "type": "object" + }, + "Role": { + "enum": ["system", "user", "assistant", "tool"], + "title": "Role", + "type": "string" + }, + "ServerToolCallPart": { + "additionalProperties": true, + "description": "Represents a server-side tool call invocation. Server tool calls are executed by the model provider on the server side rather than by the client application. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the server tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the server tool.", + "title": "Name", + "type": "string" + }, + "server_tool_call": { + "$ref": "#/$defs/GenericServerToolCall", + "description": "Polymorphic server tool call details with type discriminator. The structure varies based on the tool type." + } + }, + "required": ["type", "name", "server_tool_call"], + "title": "ServerToolCallPart", + "type": "object" + }, + "ServerToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a server-side tool call response. Contains the outcome and details of a server tool execution. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined response schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique server tool call identifier matching the original call.", + "title": "Id" + }, + "server_tool_call_response": { + "$ref": "#/$defs/GenericServerToolCallResponse", + "description": "Polymorphic server tool call response with type discriminator. The structure varies based on the tool type." + } + }, + "required": ["type", "server_tool_call_response"], + "title": "ServerToolCallResponsePart", + "type": "object" + }, + "TextPart": { + "additionalProperties": true, + "description": "Represents text content sent to or received from the model.", + "properties": { + "type": { + "const": "text", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Text content sent to or received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": ["type", "content"], + "title": "TextPart", + "type": "object" + }, + "ToolCallRequestPart": { + "additionalProperties": true, + "description": "Represents a tool call requested by the model.", + "properties": { + "type": { + "const": "tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the tool.", + "title": "Name", + "type": "string" + }, + "arguments": { + "default": null, + "description": "Arguments for the tool call.", + "title": "Arguments" + } + }, + "required": ["type", "name"], + "title": "ToolCallRequestPart", + "type": "object" + }, + "ToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a tool call result sent to the model or a built-in tool call outcome and details.", + "properties": { + "type": { + "const": "tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique tool call identifier.", + "title": "Id" + }, + "response": { + "description": "Tool call response.", + "title": "Response" + } + }, + "required": ["type", "response"], + "title": "ToolCallResponsePart", + "type": "object" + }, + "UriPart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by URI", + "properties": { + "type": { + "const": "uri", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "uri": { + "description": "A URI referencing attached data. It should not be a base64 data URL, which should use the `blob` part instead. The URI may use a scheme known to the provider api (e.g. `gs://bucket/object.png`), or be a publicly accessible location.", + "title": "Uri", + "type": "string" + } + }, + "required": ["type", "modality", "uri"], + "title": "UriPart", + "type": "object" + } + }, + "description": "Represents the list of input messages sent to the model.", + "items": { + "$ref": "#/$defs/ChatMessage" + }, + "title": "InputMessages", + "type": "array" +} diff --git a/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-output-messages.json b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-output-messages.json new file mode 100644 index 00000000000..407312749d3 --- /dev/null +++ b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-output-messages.json @@ -0,0 +1,518 @@ +{ + "$defs": { + "BlobPart": { + "description": "Represents blob binary data sent inline to the model", + "properties": { + "type": { + "const": "blob", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "content": { + "description": "Raw bytes of the attached data. This field SHOULD be encoded as a base64 string when serialized to JSON.", + "format": "binary", + "title": "Content", + "type": "string" + } + }, + "required": ["type", "modality", "content"], + "title": "BlobPart", + "type": "object" + }, + "CompactionPart": { + "additionalProperties": true, + "description": "Represents compacted conversation state sent to or received from the model.", + "properties": { + "type": { + "const": "compaction", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Provider-assigned identifier for the compaction item or block.", + "title": "Id" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The unencrypted compacted conversation summary, when available.", + "title": "Content" + } + }, + "required": ["type"], + "title": "CompactionPart", + "type": "object" + }, + "FilePart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by file id", + "properties": { + "type": { + "const": "file", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "file_id": { + "description": "An identifier referencing a file that was pre-uploaded to the provider.", + "title": "File Id", + "type": "string" + } + }, + "required": ["type", "modality", "file_id"], + "title": "FilePart", + "type": "object" + }, + "FinishReason": { + "description": "Represents the reason for finishing the generation.", + "enum": [ + "stop", + "length", + "content_filter", + "tool_call", + "compaction", + "error" + ], + "title": "FinishReason", + "type": "string" + }, + "GenericPart": { + "additionalProperties": true, + "description": "Represents an arbitrary message part with any type and properties.\nThis allows for extensibility with custom message part types.", + "properties": { + "type": { + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + } + }, + "required": ["type"], + "title": "GenericPart", + "type": "object" + }, + "GenericServerToolCall": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call with any type and properties.\nThis allows for extensibility with custom server tool types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call.", + "title": "Type", + "type": "string" + } + }, + "required": ["type"], + "title": "GenericServerToolCall", + "type": "object" + }, + "GenericServerToolCallResponse": { + "additionalProperties": true, + "description": "Represents an arbitrary server tool call response with any type and properties.\nThis allows for extensibility with custom server tool response types.", + "properties": { + "type": { + "description": "Type identifier for the server tool call response.", + "title": "Type", + "type": "string" + } + }, + "required": ["type"], + "title": "GenericServerToolCallResponse", + "type": "object" + }, + "Modality": { + "enum": ["image", "video", "audio", "document"], + "title": "Modality", + "type": "string" + }, + "OutputMessage": { + "additionalProperties": true, + "description": "Represents an output message generated by the model or agent. The output message captures\nspecific response (choice, candidate).", + "properties": { + "role": { + "anyOf": [ + { + "$ref": "#/$defs/Role" + }, + { + "type": "string" + } + ], + "description": "Role of the entity that created the message.", + "title": "Role" + }, + "parts": { + "description": "List of message parts that make up the message content.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TextPart" + }, + { + "$ref": "#/$defs/ToolCallRequestPart" + }, + { + "$ref": "#/$defs/ToolCallResponsePart" + }, + { + "$ref": "#/$defs/ServerToolCallPart" + }, + { + "$ref": "#/$defs/ServerToolCallResponsePart" + }, + { + "$ref": "#/$defs/BlobPart" + }, + { + "$ref": "#/$defs/FilePart" + }, + { + "$ref": "#/$defs/UriPart" + }, + { + "$ref": "#/$defs/ReasoningPart" + }, + { + "$ref": "#/$defs/CompactionPart" + }, + { + "$ref": "#/$defs/GenericPart" + } + ] + }, + "title": "Parts", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The name of the participant.", + "title": "Name" + }, + "finish_reason": { + "anyOf": [ + { + "$ref": "#/$defs/FinishReason" + }, + { + "type": "string" + } + ], + "description": "Reason for finishing the generation.", + "title": "Finish Reason" + } + }, + "required": ["role", "parts", "finish_reason"], + "title": "OutputMessage", + "type": "object" + }, + "ReasoningPart": { + "additionalProperties": true, + "description": "Represents reasoning/thinking content received from the model.", + "properties": { + "type": { + "const": "reasoning", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Reasoning/thinking content received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": ["type", "content"], + "title": "ReasoningPart", + "type": "object" + }, + "Role": { + "enum": ["system", "user", "assistant", "tool"], + "title": "Role", + "type": "string" + }, + "ServerToolCallPart": { + "additionalProperties": true, + "description": "Represents a server-side tool call invocation. Server tool calls are executed by the model provider on the server side rather than by the client application. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the server tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the server tool.", + "title": "Name", + "type": "string" + }, + "server_tool_call": { + "$ref": "#/$defs/GenericServerToolCall", + "description": "Polymorphic server tool call details with type discriminator. The structure varies based on the tool type." + } + }, + "required": ["type", "name", "server_tool_call"], + "title": "ServerToolCallPart", + "type": "object" + }, + "ServerToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a server-side tool call response. Contains the outcome and details of a server tool execution. Provider-specific tools (e.g., code_interpreter, web_search) can have well-defined response schemas defined by the respective providers.", + "properties": { + "type": { + "const": "server_tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique server tool call identifier matching the original call.", + "title": "Id" + }, + "server_tool_call_response": { + "$ref": "#/$defs/GenericServerToolCallResponse", + "description": "Polymorphic server tool call response with type discriminator. The structure varies based on the tool type." + } + }, + "required": ["type", "server_tool_call_response"], + "title": "ServerToolCallResponsePart", + "type": "object" + }, + "TextPart": { + "additionalProperties": true, + "description": "Represents text content sent to or received from the model.", + "properties": { + "type": { + "const": "text", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Text content sent to or received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": ["type", "content"], + "title": "TextPart", + "type": "object" + }, + "ToolCallRequestPart": { + "additionalProperties": true, + "description": "Represents a tool call requested by the model.", + "properties": { + "type": { + "const": "tool_call", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique identifier for the tool call.", + "title": "Id" + }, + "name": { + "description": "Name of the tool.", + "title": "Name", + "type": "string" + }, + "arguments": { + "default": null, + "description": "Arguments for the tool call.", + "title": "Arguments" + } + }, + "required": ["type", "name"], + "title": "ToolCallRequestPart", + "type": "object" + }, + "ToolCallResponsePart": { + "additionalProperties": true, + "description": "Represents a tool call result sent to the model or a built-in tool call outcome and details.", + "properties": { + "type": { + "const": "tool_call_response", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique tool call identifier.", + "title": "Id" + }, + "response": { + "description": "Tool call response.", + "title": "Response" + } + }, + "required": ["type", "response"], + "title": "ToolCallResponsePart", + "type": "object" + }, + "UriPart": { + "additionalProperties": true, + "description": "Represents an external referenced file sent to the model by URI", + "properties": { + "type": { + "const": "uri", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "mime_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The IANA MIME type of the attached data.", + "title": "Mime Type" + }, + "modality": { + "anyOf": [ + { + "$ref": "#/$defs/Modality" + }, + { + "type": "string" + } + ], + "description": "The general modality of the data if it is known. Instrumentations SHOULD also set the mimeType field if the specific type is known.", + "title": "Modality" + }, + "uri": { + "description": "A URI referencing attached data. It should not be a base64 data URL, which should use the `blob` part instead. The URI may use a scheme known to the provider api (e.g. `gs://bucket/object.png`), or be a publicly accessible location.", + "title": "Uri", + "type": "string" + } + }, + "required": ["type", "modality", "uri"], + "title": "UriPart", + "type": "object" + } + }, + "description": "Represents the list of output messages generated by the model or agent.", + "items": { + "$ref": "#/$defs/OutputMessage" + }, + "title": "OutputMessages", + "type": "array" +} diff --git a/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-system-instructions.json b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-system-instructions.json new file mode 100644 index 00000000000..21171178aa4 --- /dev/null +++ b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-system-instructions.json @@ -0,0 +1,51 @@ +{ + "$defs": { + "GenericPart": { + "additionalProperties": true, + "description": "Represents an arbitrary message part with any type and properties.\nThis allows for extensibility with custom message part types.", + "properties": { + "type": { + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + } + }, + "required": ["type"], + "title": "GenericPart", + "type": "object" + }, + "TextPart": { + "additionalProperties": true, + "description": "Represents text content sent to or received from the model.", + "properties": { + "type": { + "const": "text", + "description": "The type of the content captured in this part.", + "title": "Type", + "type": "string" + }, + "content": { + "description": "Text content sent to or received from the model.", + "title": "Content", + "type": "string" + } + }, + "required": ["type", "content"], + "title": "TextPart", + "type": "object" + } + }, + "description": "Represents the system instructions provided to the model.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/TextPart" + }, + { + "$ref": "#/$defs/GenericPart" + } + ] + }, + "title": "SystemInstructions", + "type": "array" +} diff --git a/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-call-arguments.json b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-call-arguments.json new file mode 100644 index 00000000000..08990814c1c --- /dev/null +++ b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-call-arguments.json @@ -0,0 +1,6 @@ +{ + "additionalProperties": true, + "description": "Represents object-like arguments passed to a tool call.", + "title": "ToolCallArguments", + "type": "object" +} diff --git a/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-call-result.json b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-call-result.json new file mode 100644 index 00000000000..5ca1ac3efd2 --- /dev/null +++ b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-call-result.json @@ -0,0 +1,6 @@ +{ + "additionalProperties": true, + "description": "Represents object-like results returned by a tool call.", + "title": "ToolCallResult", + "type": "object" +} diff --git a/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-definitions.json b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-definitions.json new file mode 100644 index 00000000000..9138516fc18 --- /dev/null +++ b/packages/core/src/telemetry/test-fixtures/gen-ai/gen-ai-tool-definitions.json @@ -0,0 +1,82 @@ +{ + "$defs": { + "FunctionToolDefinition": { + "additionalProperties": true, + "description": "Represents a tool definition in the form of a function.", + "properties": { + "type": { + "const": "function", + "description": "The type of the tool.", + "title": "Type", + "type": "string" + }, + "name": { + "description": "The name of the tool.", + "title": "Name", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The description of the tool. Since this attribute could be large, it's NOT RECOMMENDED to be populated by default. Instrumentations MAY provide a way to enable populating this property.", + "title": "Description" + }, + "parameters": { + "anyOf": [ + { + "$ref": "http://json-schema.org/draft-07/schema#" + }, + { + "type": "null" + } + ], + "default": null, + "description": "JSON Schema document describing the parameters accepted by the tool. The value MUST conform to JSON Schema draft-07. Since this attribute could be large, it's NOT RECOMMENDED to be populated by default. Instrumentations MAY provide a way to enable populating this property.", + "title": "Parameters" + } + }, + "required": ["type", "name"], + "title": "FunctionToolDefinition", + "type": "object" + }, + "GenericToolDefinition": { + "additionalProperties": true, + "description": "Represents a tool definition in any form.", + "properties": { + "type": { + "description": "The type of the tool.", + "title": "Type", + "type": "string" + }, + "name": { + "description": "The name of the tool.", + "title": "Name", + "type": "string" + } + }, + "required": ["type", "name"], + "title": "GenericToolDefinition", + "type": "object" + } + }, + "description": "Represents the list of tool definitions available to the GenAI agent or model.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/FunctionToolDefinition" + }, + { + "$ref": "#/$defs/GenericToolDefinition" + } + ] + }, + "title": "ToolDefinitions", + "type": "array" +}