diff --git a/.github/skills/update-otel-genai-conventions/SKILL.md b/.github/skills/update-otel-genai-conventions/SKILL.md new file mode 100644 index 00000000000..7b5d4f531b8 --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/SKILL.md @@ -0,0 +1,202 @@ +--- +name: update-otel-genai-conventions +description: >- + Analyze OpenTelemetry semantic-conventions releases or PRs with gen-ai changes + and produce compensating change plans for dotnet/extensions. Use when asked to + "update OTel conventions", "check semantic-conventions release", "plan gen-ai + convention changes", review gen-ai convention PRs, or when given a release + version, URL, or PR number/URL from open-telemetry/semantic-conventions with + area:gen-ai changes. Also use when asked to "update OpenTelemetry", "bump + semconv version", or "what changed in semantic-conventions vX.Y". +agent: 'agent' +tools: ['github/*', 'sql'] +--- + +# Update OTel Gen-AI Conventions + +Analyze OpenTelemetry [semantic-conventions](https://github.com/open-telemetry/semantic-conventions) releases or PRs with `area:gen-ai` changes and produce compensating updates in `dotnet/extensions`. + +## Mode Detection + +Determine the operating mode from the user's request: + +| Signal | Mode | +|--------|------| +| User asks to "audit" current implementation or "check alignment" with conventions | **Mode 1: Audit** | +| User asks to "update for vX.Y" or "apply vX.Y changes" in autopilot / one-shot | **Mode 2: Autopilot** | +| User asks to "generate a prompt" or "delegate to Copilot" or "CCA prompt" | **Mode 3: CCA Prompt** | +| Running inside Copilot Coding Agent with a prompt referencing this skill | **Mode 4: CCA Implementation** | +| User is in `/plan` mode, asks to "plan" changes, or asks to "implement" / "apply" changes | **Mode 5: Plan-then-Implement** | +| User asks to `/review` or "review" convention changes | **Mode 6: Review** | + +If unclear, default to **Mode 5** (Plan-then-Implement) and offer Mode 3 as an alternative. + +## Input Handling + +The user provides one of: +- A **semantic-conventions release version** (e.g. `v1.40.0`) → fetch from `https://github.com/open-telemetry/semantic-conventions/releases/tag/{version}` +- A **release URL** → fetch the release notes directly +- One or more **PR references** from `open-telemetry/semantic-conventions` with `area:gen-ai` changes — as URLs, PR numbers (e.g. `#3598`), or `open-telemetry/semantic-conventions#3598` format + +When PR numbers are given without a full URL, resolve them against the `open-telemetry/semantic-conventions` repository. + +### Existing dotnet/extensions PR Preflight + +For **Mode 1: Audit** and **Mode 5: Plan-then-Implement**, after resolving the requested release or upstream PR identifiers but before doing deeper release analysis or creating a plan, search open pull requests in `dotnet/extensions` to determine whether another PR already appears to cover the requested GenAI/OpenTelemetry semantic-conventions update. + +Search using the requested release version, release URL, or upstream semantic-conventions PR numbers, plus relevant terms such as `gen-ai`, `GenAI`, `semantic conventions`, `OpenTelemetry`, and `OTel`. If one or more likely matching PRs are open, report the PR number, title, author, URL, and the signal that matched. Then stop and state that the audit or plan is not proceeding because an open PR already appears to cover the update. + +Do not silently ignore search failures. If GitHub search/listing is unavailable, report the problem and ask the user whether to proceed without the preflight. + +### Analyzing the Release / PRs + +1. **Fetch the release notes** or PR descriptions and identify all gen-ai changes +2. **Read** [references/file-inventory.md](references/file-inventory.md) to understand which files in this repo are affected +3. **Classify each change** using [references/change-classification.md](references/change-classification.md) +4. **Check current state** — read the current source files to determine what is already implemented vs. what needs new work +5. **Build a changes audit table** showing each semantic convention change, its classification, and required action + +For Step 4, read the source files listed in [references/file-inventory.md](references/file-inventory.md) (`OpenTelemetryConsts.cs`, `OpenTelemetryChatClient.cs`, `OpenTelemetryEmbeddingGenerator.cs`, `Common/FunctionInvocationProcessor.cs`, and any other OpenTelemetry* files). + +### PR Title and Description Guidance + +When creating or updating a PR after implementing semantic-conventions changes, follow [references/pr-description.md](references/pr-description.md) for the title format and the changes-table shape. + +--- + +## Mode 1: Audit + +Audit the current gen-ai semantic conventions implementation against the latest published conventions to identify gaps, inconsistencies, or missed updates. Produces a plan that can be implemented locally (Mode 5) or delegated to CCA (Mode 3). + +1. Complete the **Existing dotnet/extensions PR Preflight** above. If a matching open PR exists, report it and stop. +2. **Determine the current implemented version**: Read the version reference from `OpenTelemetryChatClient.cs` doc comment to identify which convention version the codebase claims to implement +3. **Check for version drift**: Verify every file with a gen-ai semantic conventions version reference uses the same version. Use the search command from [references/file-inventory.md](references/file-inventory.md#version-references). If files reference different versions, flag that as a critical gap requiring investigation. +4. **Fetch the latest convention spec**: Read the current gen-ai semantic conventions from the [published spec](https://opentelemetry.io/docs/specs/semconv/gen-ai/) and the latest release notes +5. **Read all current source files** listed in [references/file-inventory.md](references/file-inventory.md) to understand what is actually implemented +6. **Cross-reference**: For each attribute, metric, event, and operation name defined in the conventions: + - Is the constant defined in `OpenTelemetryConsts.cs`? + - Is it emitted in the relevant OpenTelemetry* client(s)? + - Are version references consistent across all files? + - Are tests covering the attribute/metric? +7. **Build an audit report** as a table: + + | Convention Item | Expected | Implemented | Gap | + |----------------|----------|-------------|-----| + | `gen_ai.request.attribute` | v1.XX | ✅ Yes / ❌ No / ⚠️ Partial | Description of gap | + +8. **Produce a remediation plan** covering all identified gaps — formatted as either: + - A **local plan** (Mode 5 format), or + - A **CCA prompt** (Mode 3 format) suitable for delegation + + Ask the user which format they prefer, or produce both if requested. +9. **Verify this skill is still accurate** (same as Mode 6, step 6): compare skill content against the current codebase and call out any discrepancies + +--- + +## Implementation Procedure + +Modes 2, 4, and 5 share the same implementation flow. See [references/implementation-procedure.md](references/implementation-procedure.md). + +--- + +## Mode 2: Autopilot + +One-shot mode that analyzes the release and implements all changes in a single pass without intermediate review. Best for end-to-end execution when the user does not need a plan checkpoint. + +1. Complete the **Input Handling** analysis above +2. Build an internal work plan in working memory (do not write `plan.md`): + - Changes audit table with classification for each gen-ai change + - Ordered list of implementation steps +3. Follow the **Implementation Procedure** above +4. Present a summary of all changes with the audit table showing what was implemented + +--- + +## Mode 3: Generate CCA Prompt + +Generate a structured prompt suitable for delegating to Copilot Coding Agent on github.com. + +1. Complete the **Input Handling** analysis above +2. Read [references/prompt-template.md](references/prompt-template.md) for the template structure +3. Generate the prompt following the template, filling in: + - Background with links to the upstream release/PRs + - Changes audit table + - Required changes with exact file paths and code context from the current source + - Test expectations referencing [references/testing-guide.md](references/testing-guide.md) + - Validation steps +4. Present the prompt to the user for review + +The generated prompt should reference this skill: +> Reference the `update-otel-genai-conventions` skill in `.github/skills/` for implementation patterns and testing guidance. + +--- + +## Mode 4: CCA Implementation + +When running inside Copilot Coding Agent (github.com) with a prompt that references this skill. + +1. Parse the prompt to identify the required changes +2. Follow the **Implementation Procedure** above + +--- + +## Mode 5: Plan-then-Implement + +Generate a plan and (after user review/approval) implement it. Best when the user wants a checkpoint between analysis and execution. The runtime decides how to track work items (e.g., a task list, an in-memory queue, or a SQL `todos` table — whichever the agent already uses). + +**Phase A: Plan** — + +1. Resolve the user's input to a semantic-conventions release or upstream PR identifiers +2. Complete the **Existing dotnet/extensions PR Preflight** above. If a matching open PR exists, report it and stop without creating a plan. +3. Complete the **Analyzing the Release / PRs** analysis above +4. Create `plan.md` with a problem statement linking to the upstream release, a changes audit table, and a numbered list of work items. Each work item should call out the file(s) to modify, what code/constants/attributes to add, and which tests to update. +5. Pause for user review/approval before proceeding to Phase B + +**Phase B: Implement** — + +6. Read the existing `plan.md` +7. Follow the **Implementation Procedure** above for each work item + +--- + +## Mode 6: Review + +Review changes to gen-ai conventions against past patterns and known gotchas. + +1. Identify the changes to review (local diff or PR diff) +2. Read [references/review-checklist.md](references/review-checklist.md) for the full checklist +3. Read [references/historical-releases.md](references/historical-releases.md) for past PR patterns. This file is point-in-time reference data from skill creation and may not include recent releases. +4. Check each item against the checklist: + - Sensitive data gating (`EnableSensitiveData`) + - Fluent Activity API chain style + - Code deduplication (shared `Common/` classes) + - Test augmentation vs. new tests + - Version reference completeness + - Exception recording approach (ILogger vs Activity.AddEvent) +5. Report findings with references to past PRs where similar feedback was given +6. **Verify this skill is still accurate**: Compare SKILL.md and all reference files against the current codebase (the codebase may have evolved — files moved, patterns changed). Recommend updates only for durable, cross-release guidance: workflow steps, validation commands, repository conventions, stable implementation patterns, file paths, test infrastructure. Do **not** pollute skill files with release-specific findings (per-version audits, one-off attribute mappings, etc.) — capture those in the review report, PR description, or implementation summary instead. Update `historical-releases.md` only when explicitly asked. + +--- + +## Gotchas + +Critical knowledge from past PR reviews that should inform all modes: + +- **Exception recording**: Use `ILogger` with `[LoggerMessage]`, NOT `Activity.AddEvent`. The OTel SDK handles `Exception` passed to `ILogger`. See `OpenTelemetryLog.cs` in `Common/`. +- **Sensitive data**: Attributes that could contain user data (e.g. `exception.message`, message content) must be gated behind `EnableSensitiveData`. When in doubt, gate it. +- **Fluent chains**: Use fluent Activity API chains (`.SetStatus(...).SetTag(...)`) rather than separate statements. +- **Shared code**: Cross-cutting concerns (like exception logging) shared across multiple OpenTelemetry* clients belong in `src/Libraries/Microsoft.Extensions.AI/Common/`. Before adding a new helper, method, or internal type, search `Common/`, `TelemetryHelpers.cs`, `OpenTelemetryLog.cs`, and sibling OpenTelemetry* clients for existing logic with the same purpose — reuse or extend instead of introducing a parallel implementation. When the same helper is needed in 2+ places, factor it into `Common/` from the start. The same applies to parallel internal types: if a sibling client already defines a type with the same shape (same properties, same role, e.g. `RealtimeOtelFunction` vs `OtelFunction`), unify them under a single shared definition rather than letting each client carry its own copy. +- **Test augmentation**: Prefer augmenting existing test assertions over creating new test methods. Check for existing tests that validate the same scenario. +- **Version references**: When bumping the convention version, update all files that match `grep -rn "Semantic Conventions for Generative AI systems v" src/Libraries/Microsoft.Extensions.AI/`. Not all OpenTelemetry* files contain this reference — only update the ones that do. +- **No CHANGELOGs**: This repository no longer maintains per-library CHANGELOG.md files. Do NOT create or update any CHANGELOG files. +- **Source-generated JSON**: Adding new OTel part types requires: (1) new inner class, (2) `[JsonSerializable]` registration on `OtelContext`, (3) switch case in `SerializeChatMessages()`. +- **LoggerMessage text**: When using `[LoggerMessage]`, the message text should match the OTel event name for console logger readability. +- **No orphan constants**: Never add a constant to `OpenTelemetryConsts.cs` unless the same PR also adds at least one emission site for it. If the convention defines an attribute that no current client populates, classify the change as 🟢 *Constant not yet emitted* and defer the constant — do not add it ahead of emission. Verify with `grep -rn NewConstantName src/Libraries/Microsoft.Extensions.AI/` before submitting. + +## Validation + +After implementing changes (Modes 2, 4, and 5): + +1. **Restore, build, and test** using the commands in [references/build-commands.md](references/build-commands.md) — pick the form (Windows or Linux/macOS) that matches your environment. Always remove any stale `SDK.sln*` files first; they cause build errors when present alongside a newly-generated filtered solution. +2. Verify no new build warnings in `artifacts/log/Build.binlog` +3. If the public API surface changed, regenerate the API baselines per [references/build-commands.md](references/build-commands.md) — then **discard baseline updates for unrelated libraries** (only keep baselines for libraries changed as part of the convention update) diff --git a/.github/skills/update-otel-genai-conventions/references/build-commands.md b/.github/skills/update-otel-genai-conventions/references/build-commands.md new file mode 100644 index 00000000000..f09b7bbc861 --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/build-commands.md @@ -0,0 +1,38 @@ +# Build and Test Commands + +The skill needs to restore, build, and test from a freshly-generated AI-filtered solution. Use the form that matches your environment. + +Always remove any stale `SDK.sln*` files first — they cause build errors when present alongside a newly-generated filtered solution. + +## Linux / macOS (Copilot Coding Agent runs here) + +```bash +rm -f SDK.sln* +./build.sh -vs AI +./build.sh -build -test +``` + +## Windows (local development) + +```powershell +Remove-Item SDK.sln* -Force -ErrorAction SilentlyContinue +.\build.cmd -vs AI -nolaunch +.\build.cmd -build -test +``` + +## Faster iteration (any platform) + +A full build + test takes 45-60+ minutes. For inner-loop iteration on a single test class, use: + +```bash +dotnet test test/Libraries/Microsoft.Extensions.AI.Tests/ --filter "FullyQualifiedName~OpenTelemetryChatClientTests" +``` + +## After implementation + +If the public API surface changed, regenerate the API baselines: + +- Linux / macOS: `pwsh ./scripts/MakeApiBaselines.ps1` +- Windows: `.\scripts\MakeApiBaselines.ps1` + +**Discard baseline updates for unrelated libraries** — only keep baselines for libraries that were changed as part of the convention update. diff --git a/.github/skills/update-otel-genai-conventions/references/change-classification.md b/.github/skills/update-otel-genai-conventions/references/change-classification.md new file mode 100644 index 00000000000..40ef2264651 --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/change-classification.md @@ -0,0 +1,90 @@ +# Change Classification + +Taxonomy for classifying gen-ai changes from semantic-conventions releases. Use this to assess each change's impact on dotnet/extensions. + +## Classification Categories + +### 🟢 No Action Required + +| Type | Description | Example | +|------|-------------|---------| +| **N/A — No client exists** | Change affects a capability we don't implement (e.g. `retrieval`, `memory`) | `gen_ai.retrieval.*` attributes | +| **Already implemented** | Change was already implemented in a prior PR | A change that was part of an earlier draft spec we adopted | +| **Server-side only** | Change affects server/provider-side instrumentation, not client-side | Server span attributes | +| **Documentation only** | Clarification of existing semantics with no behavioral change | Rewording of attribute descriptions | +| **Constant not yet emitted** | New attribute defined upstream, but no OpenTelemetry* client in this repo populates a value for it | `gen_ai.usage.cache_creation.input_tokens` — defer the constant until a future PR adds an emission site | + +> **No orphan constants.** A new constant in `OpenTelemetryConsts.cs` must only be added in a PR that also adds at least one emission site for it. If no client populates the attribute, classify the change as 🟢 *Constant not yet emitted* and defer adding the constant entirely — do not add it speculatively. + +### 🟡 Minor Action Required + +| Type | Description | Action | +|------|-------------|--------| +| **Version bump** | Convention version number changed | Update `v1.XX` in doc comments across all OpenTelemetry* files | +| **Stability promotion** | Attribute moved from experimental to stable | Usually no code change; note in audit table | + +### 🔴 Code Change Required + +| Type | Description | Action | +|------|-------------|--------| +| **New required attribute** | New attribute that should be emitted | Add constant, add emission code, add test assertion | +| **New metric** | New metric instrument defined | Add metric definition, emission, test | +| **Attribute rename** | Existing attribute renamed | Update constant value, verify backward compatibility | +| **New event** | New log/span event defined | Add event via `ILogger` + `[LoggerMessage]`, add test | +| **Behavioral change** | Change in how existing attributes are populated | Modify emission logic, update test expectations | +| **New operation name** | New `gen_ai.operation.name` value | Add detection logic, tests | +| **Schema change** | Change to JSON schema for serialized content (e.g. tool definitions) | Update serialization classes, `[JsonSerializable]` registration | + +## Indicator Mapping + +Use these indicators consistently in audit reports, implementation summaries, and PR descriptions: + +| Indicator | Category | Meaning | +|---|---|---| +| 🟢 | No action required | No compensating code change is needed; explain why. | +| 🟡 | Minor action required | Small metadata, stability, or version-reference update. | +| 🔴 | Code change required | Runtime behavior, emission logic, metrics, events, serialization, or tests must change. | + +## Impact Assessment Heuristic + +For each gen-ai change in a release: + +1. **Does it affect a capability we instrument?** Check the [file inventory](file-inventory.md) for matching client types. + - No → classify as "N/A — No client exists" +2. **Is it already implemented?** Search `OpenTelemetryConsts.cs` for the attribute name. + - Yes → classify as "Already implemented" +3. **Is it client-side or server-side?** Check the semantic convention's `span_kind` or context. + - Server-side only → classify as "Server-side only" +4. **What kind of change is it?** Match to the categories above. +5. **How many files need modification?** Count affected files from the file inventory. + - 1–2 files → Low complexity + - 3–5 files → Medium complexity + - 6+ files → High complexity (likely involves shared code or cross-cutting concern) + +## Audit Table Format + +When presenting the analysis, use this table format: + +```markdown +| Semantic Convention Change | Upstream PR | Classification | Action Required | Complexity | +|---|---|---|---|---| +| `gen_ai.new.attribute` | [#1234](link) | New required attribute | Add constant + emission + test | Low | +| `gen_ai.deferred.attribute` | [#2345](link) | Constant not yet emitted | Defer — no client populates this attribute in this PR | — | +| `retrieval` operation | [#5678](link) | N/A — No client | None | — | +| Version reference | — | Version bump | Update doc comments | Low | +``` + +## PR Description Table Format + +When preparing a PR description, adapt the audit table into a concise reviewer-facing table grouped or sorted by semantic-conventions version. Include every analyzed gen-ai change, not just changes that required code edits. + +```markdown +| Version | Indicator | Semantic-conventions change | Classification | Compensating change / rationale | +|---|:---:|---|---|---| +| v1.XX | 🔴 | `gen_ai.new.attribute` added | New required attribute | Added constant, emission, and tests in `{files}`. | +| v1.XX | 🟡 | Version reference update | Version bump | Updated OpenTelemetry* doc comments to v1.XX. | +| v1.XX | 🟢 | Provider server span clarified | Server-side only | No client-side instrumentation change needed. | +| v1.XX | 🟢 | `gen_ai.deferred.attribute` added upstream | Constant not yet emitted | No client populates this attribute today; constant will be added in the PR that adds emission. | +``` + +The final column should either describe the compensating change made or explain why no code change was made, such as "already implemented", "no local source exists", "no client exists", "server-side only", "documentation-only clarification", or "no client populates this attribute today; constant deferred until a PR adds emission". diff --git a/.github/skills/update-otel-genai-conventions/references/file-inventory.md b/.github/skills/update-otel-genai-conventions/references/file-inventory.md new file mode 100644 index 00000000000..b1d9aa977cc --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/file-inventory.md @@ -0,0 +1,73 @@ +# File Inventory + +Files that are typically inspected and/or modified when updating OpenTelemetry gen-ai semantic conventions. + +## Constants + +| File | Purpose | +|------|---------| +| `src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs` | All OTel attribute and metric name constants. New attributes/metrics always get constants added here first. | + +## Instrumentation Clients + +These files contain the actual telemetry emission logic. Each wraps a different AI capability with OTel spans, metrics, and logs. + +| File | Capability | Key Sections | +|------|-----------|--------------| +| `src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs` | Chat completion | Activity creation, attribute emission, message serialization, streaming support, metrics | +| `src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs` | Image generation | Activity creation, attribute emission | +| `src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs` | Embeddings | Activity creation, attribute emission | +| `src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs` | Speech-to-text | Activity creation, attribute emission | +| `src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs` | Text-to-speech | Activity creation, attribute emission | +| `src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs` | Realtime sessions | Activity creation, attribute emission | +| `src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClient.cs` | Realtime client wrapper | Delegates to session | +| `src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs` | Hosted file management | Activity creation, attribute emission | + +## Function Invocation / Tool Orchestration + +These files handle `execute_tool`, `invoke_agent`, and `invoke_workflow` spans: + +| File | Purpose | +|------|---------| +| `src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs` | Chat-based tool orchestration | +| `src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs` | Realtime tool orchestration | +| `src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs` | Shared function invocation logic (used by both chat and realtime) | +| `src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs` | Shared function invocation helpers | +| `src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationLogger.cs` | Shared function invocation logging | + +## Shared Code + +| File | Purpose | +|------|---------| +| `src/Libraries/Microsoft.Extensions.AI/Common/OpenTelemetryLog.cs` | Shared `[LoggerMessage]` definitions for OTel events (e.g. exception recording) | +| `src/Libraries/Microsoft.Extensions.AI/TelemetryHelpers.cs` | Shared telemetry helper methods (at library root, not Common/) | + +## Tests + +| File | Tests For | +|------|----------| +| `test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs` | Chat client telemetry | +| `test/Libraries/Microsoft.Extensions.AI.Tests/Image/OpenTelemetryImageGeneratorTests.cs` | Image generator telemetry | +| `test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs` | Embedding generator telemetry | +| `test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/OpenTelemetrySpeechToTextClientTests.cs` | Speech-to-text telemetry | +| `test/Libraries/Microsoft.Extensions.AI.Tests/TextToSpeech/OpenTelemetryTextToSpeechClientTests.cs` | Text-to-speech telemetry | +| `test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs` | Realtime session telemetry | +| `test/Libraries/Microsoft.Extensions.AI.Tests/Files/OpenTelemetryHostedFileClientTests.cs` | Hosted file client telemetry | + +To discover any additional test files: `dir test\Libraries\Microsoft.Extensions.AI.Tests\ -Recurse -Filter "OpenTelemetry*Tests.cs"` + +## Version References + +The semantic conventions version is referenced in a doc comment in specific OpenTelemetry* instrumentation client files. When bumping the version, update all files that match the grep below — not all OpenTelemetry* files contain the version reference. + +The reference looks like: + +```csharp +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.XX, +/// defined at . +``` + +Find all occurrences with: +``` +grep -rn "Semantic Conventions for Generative AI systems v" src/Libraries/Microsoft.Extensions.AI/ +``` diff --git a/.github/skills/update-otel-genai-conventions/references/historical-releases.md b/.github/skills/update-otel-genai-conventions/references/historical-releases.md new file mode 100644 index 00000000000..e6ed6af9c3e --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/historical-releases.md @@ -0,0 +1,60 @@ +# Historical Releases + +Mapping of OpenTelemetry semantic-conventions releases with gen-ai changes to dotnet/extensions PRs. + +> **Note**: This file is a point-in-time reference and is not intended to be kept up to date with every new release. It provides context for how past convention updates were handled. For the latest release history, consult the [semantic-conventions releases page](https://github.com/open-telemetry/semantic-conventions/releases) and search the dotnet/extensions PR history. + +## Release History + +### Pre-v1.31 (Pre-release Era) + +| Convention Version | dotnet/extensions PR | Description | +|-------------------|---------------------|-------------| +| Initial implementation | [#5532](https://github.com/dotnet/extensions/pull/5532) | Initial OpenTelemetry gen-ai instrumentation | +| v1.29 draft | [#5712](https://github.com/dotnet/extensions/pull/5712) | Align with v1.29 draft conventions | +| v1.30 | [#5815](https://github.com/dotnet/extensions/pull/5815) | Update to v1.30 conventions | + +### v1.31–v1.40 (Stable Release Era) + +| Convention Version | dotnet/extensions PR | Description | +|-------------------|---------------------|-------------| +| v1.31 | [#6073](https://github.com/dotnet/extensions/pull/6073) | Update to v1.31 conventions | +| v1.34 | [#6466](https://github.com/dotnet/extensions/pull/6466) | Update to v1.34 conventions | +| v1.35 | [#6557](https://github.com/dotnet/extensions/pull/6557) | Update to v1.35 conventions | +| v1.36 | [#6579](https://github.com/dotnet/extensions/pull/6579) | Bump version reference to v1.36 (CCA) | +| v1.37 | [#6767](https://github.com/dotnet/extensions/pull/6767) | Update to v1.37 conventions | +| v1.38 | [#6829](https://github.com/dotnet/extensions/pull/6829) | Update to v1.38 conventions | +| v1.38 update | [#6981](https://github.com/dotnet/extensions/pull/6981) | Additional v1.38 changes | +| v1.39 | [#7274](https://github.com/dotnet/extensions/pull/7274) | Bump version reference to v1.39 (CCA) | +| v1.40 | [#7322](https://github.com/dotnet/extensions/pull/7322) | Update to v1.40 conventions (CCA audit) | + +### Feature-Specific PRs (v1.38–v1.40 era) + +These PRs implemented specific gen-ai convention features rather than being tied to a single version bump: + +| PR | Feature | Convention Source | +|----|---------|-------------------| +| [#7240](https://github.com/dotnet/extensions/pull/7240) | Server-side tool call attributes | v1.37+ | +| [#7241](https://github.com/dotnet/extensions/pull/7241) | Metric computation fix | Bug fix | +| [#7325](https://github.com/dotnet/extensions/pull/7325) | Streaming metrics (time_to_first_chunk, time_per_output_chunk) | v1.39 | +| [#7379](https://github.com/dotnet/extensions/pull/7379) | Exception event recording (gen_ai.client.operation.exception) | v1.40 | +| [#7382](https://github.com/dotnet/extensions/pull/7382) | invoke_workflow operation name | v1.40 | + +## Typical Change Patterns by Release + +### Version-only releases (v1.36, v1.39) +- Only doc comment version bump needed +- Minimal code changes +- Quick turnaround + +### Attribute addition releases (v1.31, v1.34, v1.35, v1.37, v1.38) +- New constants in `OpenTelemetryConsts.cs` +- New attribute emission in one or more OpenTelemetry* clients +- Test updates +- Version bump + +### Behavioral change releases (v1.40 features) +- New code patterns (exception recording, streaming metrics) +- May require shared infrastructure (`Common/` classes) +- More extensive test changes +- Often split into multiple PRs per feature diff --git a/.github/skills/update-otel-genai-conventions/references/implementation-patterns.md b/.github/skills/update-otel-genai-conventions/references/implementation-patterns.md new file mode 100644 index 00000000000..3a9f0af7f75 --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/implementation-patterns.md @@ -0,0 +1,213 @@ +# Implementation Patterns + +Code patterns for common convention update change types. Use these as templates when implementing compensating changes. + +> **Reuse before adding.** Before applying any of the patterns below, search the touched libraries (`Common/`, `TelemetryHelpers.cs`, `OpenTelemetryLog.cs`, and sibling OpenTelemetry* client files) for an existing helper, method, or internal type that already does the same thing. Reuse or extend it instead of adding a parallel implementation. If the same logic will be needed in two or more places, factor it into `Common/` from the start rather than duplicating it per file. The same rule applies to parallel internal types — when a sibling client already defines a type with the same shape, unify under a single shared definition. See [review-checklist.md §3](review-checklist.md#3-code-deduplication) for what reviewers look for. + +## Pattern 1: Adding a New Constant + +Location: `src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs` + +Constants are organized into nested static classes by category. Find the appropriate section and add the new constant. + +```csharp +// In the appropriate nested class (e.g., GenAI, GenAI.Client, GenAI.Request, GenAI.Usage) +public const string NewAttributeName = "gen_ai.new.attribute"; +``` + +**Naming convention**: The C# constant name uses PascalCase, omitting the `gen_ai.` prefix where the parent class already implies it. For example: +- `gen_ai.request.stream` → in `GenAI.Request` class: `public const string Stream = "gen_ai.request.stream";` +- `gen_ai.usage.reasoning.output_tokens` → in `GenAI.Usage` class: `public const string ReasoningOutputTokens = "gen_ai.usage.reasoning.output_tokens";` + +## Pattern 2: Emitting a Span Attribute + +Location: Relevant OpenTelemetry* client file (e.g., `OpenTelemetryChatClient.cs`) + +Keep provider-agnostic and provider-specific instrumentation separated: + +- Generic `gen_ai.*` attributes belong in `src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs` or the relevant generic OpenTelemetry* client. +- Provider-specific attributes, such as `openai.*`, belong in the provider package (`src/Libraries/Microsoft.Extensions.AI.OpenAI/`) so `OpenTelemetryChatClient` remains provider-agnostic. +- For OpenAI-specific mappings, add helper logic near the existing `openai.api.type` handling in `OpenAIClientExtensions.cs`, invoke it from the provider client that exposes the SDK value, and test it in `test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/`. +- Use `SetTag` for provider-specific response attributes that can arrive on multiple streaming updates so repeated updates do not duplicate tags. + +### Request attributes (set before the call) + +In the `CreateAndConfigureActivity` or equivalent method, add the attribute after the activity is created: + +```csharp +activity?.SetTag(OpenTelemetryConsts.GenAI.Request.NewAttribute, value); +``` + +### Response attributes (set after the call) + +In the `TraceResponse` or equivalent method: + +```csharp +activity?.SetTag(OpenTelemetryConsts.GenAI.Response.NewAttribute, responseValue); +``` + +### Conditional attributes (only set when value is present) + +```csharp +if (someValue is not null) +{ + activity?.SetTag(OpenTelemetryConsts.GenAI.Request.NewAttribute, someValue); +} +``` + +### Boolean attributes + +```csharp +activity?.SetTag(OpenTelemetryConsts.GenAI.Request.Stream, true); +``` + +## Pattern 3: Adding a Usage Token Attribute + +Location: `OpenTelemetryChatClient.cs`, in the response tracing section + +Usage tokens follow a specific pattern where they're emitted as span attributes from response tracing: + +```csharp +// In TraceResponse or equivalent: +if (usage?.NewTokenCount is int newTokens and > 0) +{ + activity?.SetTag(OpenTelemetryConsts.GenAI.Usage.NewTokens, newTokens); +} +``` + +Only update `gen_ai.client.token.usage` metric recording when the convention adds or changes a token metric type. Do not add usage span attributes as metric tags. + +## Pattern 4: Adding a Metric + +Location: OpenTelemetry* client constructor for instrument creation, emission site for recording. + +### Instrument creation (in constructor) + +```csharp +private readonly Histogram _newMetric; + +// In constructor: +_newMetric = meter.CreateHistogram( + OpenTelemetryConsts.GenAI.Client.NewMetricName, + OpenTelemetryConsts.SecondsUnit, // or TokensUnit, or null + "Description of the metric."); +``` + +### Recording the metric + +```csharp +_newMetric.Record(value, tags); +``` + +## Pattern 5: Adding an Event via ILogger + +Location: `src/Libraries/Microsoft.Extensions.AI/Common/OpenTelemetryLog.cs` for the definition, emission site for the call. + +**IMPORTANT**: Use `ILogger` with `[LoggerMessage]`, NOT `Activity.AddEvent`. This is the established pattern per reviewer feedback. + +### Define the log message + +```csharp +// In OpenTelemetryLog.cs +[LoggerMessage( + EventName = "gen_ai.event.name", + Level = LogLevel.Warning, + Message = "gen_ai.event.name")] +internal static partial void EventName(ILogger logger, Exception error); +``` + +Note: The `Message` text should match the OTel event name. Parameters vary by event — use `Exception error` for exception events, add other parameters as needed. + +### Call the log method + +```csharp +if (_logger is not null) +{ + OpenTelemetryLog.EventName(_logger, exception); +} +``` + +## Pattern 6: Updating Version References + +When bumping the convention version (e.g. v1.39 → v1.40), update the doc comment in all matched OpenTelemetry* client files: + +```csharp +// Before: +/// Semantic Conventions for Generative AI systems v1.39, +// After: +/// Semantic Conventions for Generative AI systems v1.40, +``` + +Find all occurrences: +```bash +grep -rn "Semantic Conventions for Generative AI systems v" src/Libraries/Microsoft.Extensions.AI/ +``` + +## Pattern 7: Modifying Message Serialization + +Location: `OpenTelemetryChatClient.cs`, `SerializeChatMessages()` method and related inner classes. + +### Adding a new content part type + +1. Add a new inner class: +```csharp +private sealed class OtelNewPart +{ + public string? Type { get; set; } + public string? Value { get; set; } +} +``` + +2. Register with the JSON serializer context: +```csharp +[JsonSerializable(typeof(OtelNewPart))] +// Add to the OtelContext partial class +``` + +3. Add a case in `SerializeChatMessages()`: +```csharp +case NewContentType newContent: + writer.WriteRawValue(JsonSerializer.SerializeToUtf8Bytes( + new OtelNewPart { Type = "new_type", Value = newContent.Value }, + OtelContext.Default.OtelNewPart)); + break; +``` + +## Pattern 8: Sensitive Data Gating + +Any attribute that could contain user-generated content must be gated: + +```csharp +if (EnableSensitiveData) +{ + activity?.SetTag(OpenTelemetryConsts.GenAI.SensitiveAttribute, sensitiveValue); +} +``` + +Check the `EnableSensitiveData` property (set directly or from environment variable `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`). + +## Pattern 9: Span Naming for Tool Execution + +Location: `FunctionInvokingChatClient.cs` + +The span name format for tool execution follows the pattern: +```csharp +string spanName = $"execute_tool {toolCall.Name}"; +``` + +For `invoke_agent` or `invoke_workflow` operations, detect based on the function metadata and adjust the operation name accordingly. + +## Fluent API Style + +Always use fluent chains for Activity API calls: + +```csharp +// ✅ Correct — fluent chain +activity? + .SetStatus(ActivityStatusCode.Error, errorMessage) + .SetTag(OpenTelemetryConsts.Error.Type, errorType); + +// ❌ Incorrect — separate statements +activity?.SetStatus(ActivityStatusCode.Error, errorMessage); +activity?.SetTag(OpenTelemetryConsts.Error.Type, errorType); +``` diff --git a/.github/skills/update-otel-genai-conventions/references/implementation-procedure.md b/.github/skills/update-otel-genai-conventions/references/implementation-procedure.md new file mode 100644 index 00000000000..64ea28cb071 --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/implementation-procedure.md @@ -0,0 +1,14 @@ +# Implementation Procedure + +Used by Modes 2 (Autopilot), 4 (CCA Implementation), and 5 (Plan-then-Implement) when actually applying convention changes. + +1. Read [implementation-patterns.md](implementation-patterns.md) and [testing-guide.md](testing-guide.md) +2. Read [review-checklist.md](review-checklist.md) to anticipate review feedback +3. Apply changes in this order: + - Add new constants to `OpenTelemetryConsts.cs` **only for attributes whose emission is also added in this same PR**. Do not add constants speculatively — if no OpenTelemetry* client in this repo will populate the attribute, defer the constant until the PR that wires up emission and classify the change as 🟢 *Constant not yet emitted* per [change-classification.md](change-classification.md). + - **Before adding any new helper, method, or internal type**, search `Common/`, `TelemetryHelpers.cs`, `OpenTelemetryLog.cs`, and the sibling OpenTelemetry* client files for existing logic with the same purpose. Reuse or extend rather than introducing a parallel implementation. If the same logic is needed in two or more places, factor it into `Common/` from the start instead of duplicating it per file. The same applies to parallel internal types — unify types with identical shape under a single shared definition. + - Add attribute/metric emission to the relevant OpenTelemetry* client classes + - Update version references in doc comments across all files that reference the convention version + - Update or augment tests +4. Self-review against [review-checklist.md](review-checklist.md) +5. Validate per the **Validation** section in `SKILL.md` diff --git a/.github/skills/update-otel-genai-conventions/references/pr-description.md b/.github/skills/update-otel-genai-conventions/references/pr-description.md new file mode 100644 index 00000000000..091ace1b3ec --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/pr-description.md @@ -0,0 +1,33 @@ +# PR Title and Description Format + +When asked to create or update a PR after implementing semantic-conventions changes, use this guidance. + +## Title + +```text +Update OpenTelemetry gen-ai conventions to v{version} +``` + +Use the target semantic-conventions release version for `{version}`. If the PR also includes catch-up work from earlier releases, keep the title focused on the target version and explain the catch-up work in the description. + +## Description + +The description should include a changes table derived from the audit table and [change-classification.md](change-classification.md). Group or sort rows by semantic-conventions version and include every analyzed gen-ai change, not only the rows that produced code changes. Use the same red/yellow/green indicators as the classification guide: + +- 🟢 for no action required +- 🟡 for minor action required +- 🔴 for code change required + +Use this table shape: + +```markdown +| Version | Indicator | Semantic-conventions change | Classification | Compensating change / rationale | +|---|:---:|---|---|---| +| v1.XX | 🔴 | `gen_ai.example.attribute` added | New required attribute | Added constant, emission, and tests in `{files}`. | +| v1.XX | 🟡 | Convention version reference changed | Version bump | Updated OpenTelemetry* doc comments. | +| v1.XX | 🟢 | Server-side-only span attribute added | Server-side only | No client-side instrumentation change needed. | +``` + +For each row, describe the compensating change made, or explain why no change was made (already implemented, no local source, no client exists, server-side only, documentation only, etc.). + +Keep release-specific findings in the PR description or implementation summary; do not add them to the skill references unless they are durable cross-release guidance. diff --git a/.github/skills/update-otel-genai-conventions/references/prompt-template.md b/.github/skills/update-otel-genai-conventions/references/prompt-template.md new file mode 100644 index 00000000000..e0013b0260e --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/prompt-template.md @@ -0,0 +1,123 @@ +# CCA Prompt Template + +Template for generating a structured prompt suitable for delegating convention update work to Copilot Coding Agent (CCA) on github.com. + +## Template + +Fill in the bracketed sections based on the analysis of the semantic-conventions release. + +--- + +```markdown +## Background + +The OpenTelemetry semantic conventions {VERSION} release includes gen-ai changes that require compensating updates in dotnet/extensions. Release notes: {RELEASE_URL} + +Key upstream PRs: +{FOR_EACH_UPSTREAM_PR} +- [{PR_TITLE}]({PR_URL}) +{END_FOR_EACH} + +## Changes Audit + +| Semantic Convention Change | Upstream PR | Classification | Action Required | +|---|---|---|---| +{FOR_EACH_CHANGE} +| `{ATTRIBUTE_OR_CHANGE_NAME}` | [#{PR_NUMBER}]({PR_URL}) | {CLASSIFICATION} | {ACTION} | +{END_FOR_EACH} + +## Required Changes + +### 1. Version References + +Update the semantic conventions version reference from `v{OLD_VERSION}` to `v{NEW_VERSION}` in doc comments across ALL OpenTelemetry* client files: + +{LIST_ALL_FILES_WITH_VERSION_REFERENCE} + +The doc comment pattern to update: +```csharp +/// Semantic Conventions for Generative AI systems v{OLD_VERSION}, +``` +→ +```csharp +/// Semantic Conventions for Generative AI systems v{NEW_VERSION}, +``` + +### 2. New Constants + +Add these constants to `src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs`: + +{FOR_EACH_NEW_CONSTANT} +In the `{PARENT_CLASS}` nested class: +```csharp +public const string {CONSTANT_NAME} = "{ATTRIBUTE_NAME}"; +``` +{END_FOR_EACH} + +### 3. Attribute Emission + +{FOR_EACH_NEW_ATTRIBUTE} +#### 3.{N}. `{ATTRIBUTE_NAME}` in `{CLIENT_FILE}` + +{DESCRIPTION_OF_WHERE_AND_HOW_TO_EMIT} + +Current code context (around line {LINE_NUMBER}): +```csharp +{EXISTING_CODE_SNIPPET} +``` + +Add: +```csharp +{NEW_CODE_TO_ADD} +``` +{END_FOR_EACH} + +### 4. Tests + +Update tests in `{TEST_FILE_PATH}`: + +{FOR_EACH_TEST_UPDATE} +- {EXISTING_OR_NEW}: {DESCRIPTION_OF_ASSERTION_TO_ADD} +{END_FOR_EACH} + +Reference the `update-otel-genai-conventions` skill in `.github/skills/` for: +- Implementation patterns in `references/implementation-patterns.md` +- Testing guide in `references/testing-guide.md` +- Review checklist in `references/review-checklist.md` + +## Validation + +After implementing changes: +1. Restore, generate the AI-filtered solution, build, and run the tests using the Linux/macOS commands in `.github/skills/update-otel-genai-conventions/references/build-commands.md` +2. If the public API surface changed, run `pwsh ./scripts/MakeApiBaselines.ps1` and keep only the baselines for the libraries actually changed +3. Verify no remaining references to the old version: `grep -rn "v{OLD_VERSION}" src/Libraries/Microsoft.Extensions.AI/` +``` + +--- + +## Prompt Quality Guidelines + +Based on analysis of successful CCA prompts (PRs #7379, #7382, #7322): + +### What makes a good prompt + +1. **Exact file paths** — always include full relative paths from repo root +2. **Current code context** — show the existing code around the modification point with line numbers +3. **Expected code** — show what the new code should look like +4. **Constant values** — specify the exact string values for new OTel attribute names +5. **Test expectations** — specify which test file and whether to augment existing tests or create new ones +6. **Validation commands** — include the build/test commands to run + +### What to avoid + +1. **Vague instructions** — "update the tests" → specify exactly which assertions to add +2. **Missing files** — forgetting to update version references in all OpenTelemetry* files +3. **Wrong approach** — specifying `Activity.AddEvent` when `ILogger` should be used for events +4. **Incomplete scope** — only covering chat client when embedding generator also needs changes + +### Prompt size guidance + +- **Simple version bump** (few code changes): ~1,000–2,000 characters +- **New attributes/metrics** (moderate changes): ~3,000–5,000 characters +- **Behavioral changes** (complex): ~5,000–8,000 characters +- **Audit table only** (version bump with analysis): use the concise audit table format from PR #7322 diff --git a/.github/skills/update-otel-genai-conventions/references/review-checklist.md b/.github/skills/update-otel-genai-conventions/references/review-checklist.md new file mode 100644 index 00000000000..ea1d9124527 --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/review-checklist.md @@ -0,0 +1,92 @@ +# Review Checklist + +Review checklist for gen-ai convention changes. Based on patterns from past PR reviews by domain experts (@stephentoub, @tarekgh, @lmolkova, @CodeBlanch). + +## Critical Checks + +### 1. Exception Recording Approach +- [ ] Exception events use `ILogger` + `[LoggerMessage]`, NOT `Activity.AddEvent` +- [ ] Log message definitions are in `Common/OpenTelemetryLog.cs` +- [ ] `[LoggerMessage]` message text matches the OTel event name + +**Past feedback**: PR #7379 — tarekgh and CodeBlanch directed change from `Activity.AddEvent` to `ILogger`-based approach per OTel migration plan. + +### 2. Sensitive Data Gating +- [ ] Attributes that could contain user data are gated behind `EnableSensitiveData` +- [ ] `exception.message` is treated as potentially sensitive +- [ ] Message content serialization respects the sensitive data setting +- [ ] Test coverage for both `EnableSensitiveData = true` and `false` + +**Past feedback**: PR #7379 — stephentoub raised whether `exception.message` should be guarded. + +### 3. Code Deduplication +- [ ] Cross-cutting telemetry code is shared via `Common/` classes, not duplicated +- [ ] Similar patterns across multiple OpenTelemetry* clients use shared helpers +- [ ] New helper methods are added to `TelemetryHelpers.cs` or `OpenTelemetryLog.cs` as appropriate +- [ ] **Search before adding**: before introducing a new helper, method, or internal type, search `Common/`, `TelemetryHelpers.cs`, `OpenTelemetryLog.cs`, and sibling OpenTelemetry* client files for existing logic that does the same thing. Prefer reusing an existing helper or extending it over adding a parallel one. +- [ ] **Cross-file diff for new helpers**: when the same convention change introduces helper logic in more than one OpenTelemetry* client, diff the new blocks against each other. Byte-for-byte (or near-byte-for-byte) identical helpers must be factored into a shared helper in `Common/` rather than duplicated per file. +- [ ] **Parallel types with identical shape**: when defining a new internal/private type (e.g. for serialization, OTel mapping, tool definitions), check whether a sibling client already defines a type with the same shape (same properties, same purpose). Unify the two — either reuse the existing type or move both to a single shared definition under `Common/`. + +**Past feedback**: +- PR #7379 — tarekgh noted duplicated code across clients and requested consolidation to `Common/`. +- PR [#7497 r3161304243](https://github.com/dotnet/extensions/pull/7497#discussion_r3161304243) — reviewer flagged that the `chat` / `chat {name}` activity-name check was duplicated in several files; consolidated to a shared helper. +- PR [#7497 r3161364739](https://github.com/dotnet/extensions/pull/7497#discussion_r3161364739) / [r3162514449](https://github.com/dotnet/extensions/pull/7497#discussion_r3162514449) — reviewer flagged that `CreateOtelToolDefinition` returned a `RealtimeOtelFunction` in the realtime client and an `OtelFunction` in the chat client, with byte-for-byte identical logic and identical type shape (`Name`, `Description`, `Parameters`, `Type`). The two parallel types should have been unified from the start. + +### 4. Fluent API Style +- [ ] Activity API calls use fluent chains (`.SetStatus(...).SetTag(...)`) +- [ ] No separate statement for each Activity method call + +**Past feedback**: PR #7379 — stephentoub requested fluent chain continuation. + +### 5. Test Organization +- [ ] Existing tests augmented with new assertions rather than creating new test methods where possible +- [ ] Both streaming and non-streaming paths tested +- [ ] Sensitive data gating tested (both enabled and disabled) +- [ ] Missing/default value behavior tested + +**Past feedback**: PR #7379 — stephentoub asked "do we already have tests validating error.type? If so, can you just augment those". + +### 6. Version Reference Completeness +- [ ] All files with a gen-ai semantic conventions version reference use the same version before starting the update +- [ ] ALL OpenTelemetry* client files with a version reference have that reference updated +- [ ] Grep confirms no remaining references to the old version: `grep -rn "v1.OLD" src/Libraries/Microsoft.Extensions.AI/` + +### 7. Constants Organization +- [ ] New constants added to appropriate nested class in `OpenTelemetryConsts.cs` +- [ ] Constant names follow PascalCase convention +- [ ] String values match the semantic convention attribute names exactly +- [ ] **No orphan constants**: every newly added constant in `OpenTelemetryConsts.cs` is referenced by at least one emission site added in this PR. Verify with `grep -rn NewConstantName src/Libraries/Microsoft.Extensions.AI/`. If no client populates the attribute, the constant must be removed from this PR and deferred to the PR that adds emission (classify as 🟢 *Constant not yet emitted*). + +### 8. Scope Completeness +- [ ] Changes applied to ALL relevant OpenTelemetry* client classes (not just the chat client) +- [ ] If a change affects embeddings, image generation, speech, etc., those clients are also updated +- [ ] Function invocation changes apply to both `FunctionInvokingChatClient` and shared `Common/FunctionInvocationProcessor.cs` +- [ ] Realtime function invocation via `FunctionInvokingRealtimeClientSession` is also covered if applicable + +**Past feedback**: PR #7379 — stephentoub asked to extend changes to additional client types. + +### 9. JSON Serialization +- [ ] New content part types have proper inner classes +- [ ] `[JsonSerializable]` registration added to `OtelContext` +- [ ] Switch case added in `SerializeChatMessages()` for new types + +### 10. Metric Alignment +- [ ] New metrics have proper instrument creation (Histogram, Counter, etc.) +- [ ] Metric units use constants (`SecondsUnit`, `TokensUnit`) +- [ ] Metric tags align with span attributes where applicable + +## Common Mistakes + +| Mistake | Correct Approach | +|---------|-----------------| +| Using `Activity.AddEvent` for exceptions | Use `ILogger` + `[LoggerMessage]` | +| Separate Activity API statements | Use fluent chains | +| Creating new test methods for existing scenarios | Augment existing test assertions | +| Only updating `OpenTelemetryChatClient` | Update ALL relevant OpenTelemetry* clients | +| Missing `EnableSensitiveData` gate | Gate any attribute with user-generated content | +| Updating version in one file only | Check for version drift first, then update ALL files with version reference | +| Creating CHANGELOG entries | No CHANGELOGs — info goes in release notes only | +| Using `null` for optional metric units | Use the appropriate unit constant or omit | +| Adding a constant for an attribute no client emits | Defer the constant until the PR that adds the emission site (classify as 🟢 *Constant not yet emitted*) | +| Adding a new helper without searching for an existing one | Search `Common/`, `TelemetryHelpers.cs`, `OpenTelemetryLog.cs`, and sibling OpenTelemetry* clients first; reuse or extend rather than parallel-implement | +| Defining a parallel internal type with the same shape as one in a sibling client (e.g. `RealtimeOtelFunction` vs `OtelFunction`) | Unify the types — reuse the existing one or move a single shared definition to `Common/` | diff --git a/.github/skills/update-otel-genai-conventions/references/testing-guide.md b/.github/skills/update-otel-genai-conventions/references/testing-guide.md new file mode 100644 index 00000000000..cca6886028c --- /dev/null +++ b/.github/skills/update-otel-genai-conventions/references/testing-guide.md @@ -0,0 +1,147 @@ +# Testing Guide + +How to add and update tests when making convention changes. Tests for OpenTelemetry gen-ai instrumentation follow consistent patterns. + +## Test File Locations + +| Instrumentation Client | Test File | +|----------------------|-----------| +| `OpenTelemetryChatClient` | `test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs` | +| `OpenTelemetryImageGenerator` | `test/Libraries/Microsoft.Extensions.AI.Tests/Image/OpenTelemetryImageGeneratorTests.cs` | +| `OpenTelemetryEmbeddingGenerator` | `test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs` | +| `OpenTelemetrySpeechToTextClient` | `test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/OpenTelemetrySpeechToTextClientTests.cs` | +| `OpenTelemetryTextToSpeechClient` | `test/Libraries/Microsoft.Extensions.AI.Tests/TextToSpeech/OpenTelemetryTextToSpeechClientTests.cs` | +| `OpenTelemetryRealtimeClientSession` | `test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs` | +| `OpenTelemetryHostedFileClient` | `test/Libraries/Microsoft.Extensions.AI.Tests/Files/OpenTelemetryHostedFileClientTests.cs` | + +## Test Infrastructure + +### In-Memory Exporters + +Tests use in-memory OTel exporters to capture and assert on telemetry: + +```csharp +var activities = new List(); +using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddInMemoryExporter(activities) + .Build(); +``` + +### Metric Collection + +```csharp +using var meterCollector = new MetricCollector( + null, // meter provider + OpenTelemetryConsts.DefaultSourceName, + OpenTelemetryConsts.GenAI.Client.MetricName); +``` + +### Test Chat Client + +A `TestChatClient` is used to provide controlled responses: + +```csharp +var testClient = new TestChatClient +{ + GetResponseAsync = (messages, options, ct) => + { + return Task.FromResult(new ChatResponse(/* configured response */)); + } +}; +``` + +## Assertion Patterns + +### Asserting Span Attributes + +```csharp +var activity = Assert.Single(activities); +Assert.Equal("expected_value", activity.GetTagItem(OpenTelemetryConsts.GenAI.Request.AttributeName)); +``` + +### Asserting Optional Attributes (null when not present) + +```csharp +Assert.Null(activity.GetTagItem(OpenTelemetryConsts.GenAI.Request.OptionalAttribute)); +``` + +### Asserting Boolean Attributes + +```csharp +Assert.True(activity.GetTagItem(OpenTelemetryConsts.GenAI.Request.BoolAttribute) is true); +``` + +### Asserting Numeric Attributes + +```csharp +Assert.Equal(42L, activity.GetTagItem(OpenTelemetryConsts.GenAI.Usage.TokenCount)); +``` + +### Asserting Metric Values + +```csharp +var measurements = meterCollector.GetMeasurementSnapshot(); +var measurement = Assert.Single(measurements); +Assert.Equal(expectedValue, measurement.Value); +Assert.Equal("expected_tag_value", measurement.Tags[OpenTelemetryConsts.GenAI.Request.TagName]); +``` + +### JSON Content Assertions + +For serialized message content, tests use whitespace-normalized JSON comparison: + +```csharp +var events = activity.Events.ToList(); +var eventPayload = events[0].Tags.First(t => t.Key == "gen_ai.content").Value as string; +Assert.Equal( + NormalizeWhitespace(expectedJson), + NormalizeWhitespace(eventPayload)); +``` + +## Key Testing Principles + +### 1. Augment Existing Tests First + +Before creating new test methods, check if existing tests already exercise the scenario. Add new assertions to existing test methods when possible. This was explicit reviewer feedback on past PRs. + +For example, if adding a new response attribute, find the existing test that validates response attributes and add the new assertion there. + +### 2. Test Both Streaming and Non-Streaming + +The `OpenTelemetryChatClient` has two code paths: `GetResponseAsync` and `GetStreamingResponseAsync`. Both must be tested. Existing tests often use `[InlineData]` or `[Theory]` to parameterize across both paths. + +### 3. Test Sensitive Data Gating + +If an attribute is gated behind `EnableSensitiveData`, test both: +- **With sensitive data enabled**: attribute should be present +- **With sensitive data disabled**: attribute should be absent (null) + +```csharp +[Theory] +[InlineData(true)] +[InlineData(false)] +public async Task SensitiveAttribute_RespectsSetting(bool enableSensitiveData) +{ + // ... setup with enableSensitiveData + if (enableSensitiveData) + { + Assert.Equal(expected, activity.GetTagItem(...)); + } + else + { + Assert.Null(activity.GetTagItem(...)); + } +} +``` + +### 4. Test Default Values and Missing Values + +Test that attributes are omitted (not set to empty/default) when the source data doesn't include the relevant field. + +### 5. Verify Metric Tags Match Span Attributes + +When an attribute appears on both spans and metrics, ensure tests verify both emission points. + +## Build and Test Commands + +See [build-commands.md](build-commands.md) for the canonical Windows and Linux/macOS forms, including the faster `dotnet test --filter` invocation for inner-loop iteration. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index fb4c0c4c8e1..18db6289dac 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -354,6 +354,8 @@ internal static async IAsyncEnumerable FromOpenAIStreamingCh string? responseId = null; DateTimeOffset? createdAt = null; string? modelId = null; + string? serviceTier = null; + string? systemFingerprint = null; // Process each update as it arrives await foreach (StreamingChatCompletionUpdate update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -365,6 +367,11 @@ internal static async IAsyncEnumerable FromOpenAIStreamingCh createdAt ??= update.CreatedAt; modelId ??= update.Model; + // Record the service tier and system fingerprint each once if not yet recorded. + OpenAIClientExtensions.AddOpenAIResponseAttributes( + update.ServiceTier?.ToString(), update.SystemFingerprint, + ref serviceTier, ref systemFingerprint); + // Create the response content object. ChatResponseUpdate responseUpdate = new() { @@ -577,6 +584,8 @@ internal static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAICompl ResponseId = openAICompletion.Id, }; + OpenAIClientExtensions.AddOpenAIResponseAttributes(openAICompletion.ServiceTier?.ToString(), openAICompletion.SystemFingerprint); + if (openAICompletion.Usage is ChatTokenUsage tokenUsage) { response.Usage = FromOpenAIUsage(tokenUsage); diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs index 9a5ebd0d06a..f102c887541 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs @@ -334,6 +334,12 @@ internal sealed class ToolJson /// The "openai.api.type" tag name per the OpenTelemetry semantic conventions for OpenAI. internal const string OpenAIApiTypeTag = "openai.api.type"; + /// The "openai.response.service_tier" tag name per the OpenTelemetry semantic conventions for OpenAI. + internal const string OpenAIResponseServiceTierTag = "openai.response.service_tier"; + + /// The "openai.response.system_fingerprint" tag name per the OpenTelemetry semantic conventions for OpenAI. + internal const string OpenAIResponseSystemFingerprintTag = "openai.response.system_fingerprint"; + /// The "chat_completions" value for the "openai.api.type" tag. internal const string OpenAIApiTypeChatCompletions = "chat_completions"; @@ -348,16 +354,103 @@ internal sealed class ToolJson /// adds the "openai.api.type" tag with the specified value. /// internal static void AddOpenAIApiType(string apiType) + { + if (GetCurrentChatActivity() is { } activity) + { + _ = activity.AddTag(OpenAIApiTypeTag, apiType); + } + } + + /// + /// If the current represents a "chat" operation span, + /// adds OpenAI-specific response tags with the specified values. + /// + internal static void AddOpenAIResponseAttributes(string? serviceTier, string? systemFingerprint) + { + if (GetCurrentChatActivity() is { } activity) + { + if (!string.IsNullOrWhiteSpace(serviceTier)) + { + _ = activity.SetTag(OpenAIResponseServiceTierTag, serviceTier); + } + + if (!string.IsNullOrWhiteSpace(systemFingerprint)) + { + _ = activity.SetTag(OpenAIResponseSystemFingerprintTag, systemFingerprint); + } + } + } + + /// + /// Streaming-friendly overload of + /// that records each tag at most once per stream. Once a non-null value has been written for + /// either tag, subsequent calls short-circuit without performing the activity lookup or the + /// per-tag call. + /// + /// + /// Each tag is gated independently so a stream that never reports one of the two values still + /// captures the other on its first non-null arrival. + /// + /// The service tier value from the current update, if any. + /// The system fingerprint value from the current update, if any. + /// + /// A per-stream cache of the value already written for openai.response.service_tier. + /// Initialize to at the start of the stream. + /// + /// + /// A per-stream cache of the value already written for openai.response.system_fingerprint. + /// Initialize to at the start of the stream. + /// + internal static void AddOpenAIResponseAttributes( + string? serviceTier, + string? systemFingerprint, + ref string? capturedServiceTier, + ref string? capturedSystemFingerprint) + { + bool needsServiceTier = capturedServiceTier is null && !string.IsNullOrWhiteSpace(serviceTier); + bool needsSystemFingerprint = capturedSystemFingerprint is null && !string.IsNullOrWhiteSpace(systemFingerprint); + + if (!needsServiceTier && !needsSystemFingerprint) + { + return; + } + + if (GetCurrentChatActivity() is { } activity) + { + if (needsServiceTier) + { + capturedServiceTier = serviceTier; + _ = activity.SetTag(OpenAIResponseServiceTierTag, serviceTier); + } + + if (needsSystemFingerprint) + { + capturedSystemFingerprint = systemFingerprint; + _ = activity.SetTag(OpenAIResponseSystemFingerprintTag, systemFingerprint); + } + } + } + + /// + /// Returns if it has data requested and its + /// represents a gen_ai "chat" span + /// (the name is "chat" or "chat {name}"); otherwise . + /// + private static Activity? GetCurrentChatActivity() { Activity? activity = Activity.Current; if (activity is { IsAllDataRequested: true }) { + // Recognize an activity name of "chat" or "chat {name}". string name = activity.DisplayName; + if (name.StartsWith(ChatOperationName, StringComparison.Ordinal) && (name.Length == ChatOperationName.Length || name[ChatOperationName.Length] == ' ')) { - _ = activity.AddTag(OpenAIApiTypeTag, apiType); + return activity; } } + + return null; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index b6b3e3dad82..38695ae16f0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -125,6 +125,8 @@ public async Task GetResponseAsync( internal static ChatResponse FromOpenAIResponse(ResponseResult responseResult, CreateResponseOptions? openAIOptions, string? conversationId) { + OpenAIClientExtensions.AddOpenAIResponseAttributes(responseResult.ServiceTier?.ToString(), systemFingerprint: null); + // Convert and return the results. ChatResponse response = new() { @@ -368,6 +370,8 @@ internal static async IAsyncEnumerable FromOpenAIStreamingRe ChatRole? lastRole = null; bool anyFunctions = false; bool storedOutputDisabled = false; + string? serviceTier = null; + string? systemFingerprint = null; ResponseStatus? latestResponseStatus = null; Dictionary? mcpApprovalRequests = null; @@ -679,6 +683,11 @@ ChatResponseUpdate CreateUpdate(AIContent? content = null) => void UpdateConversationId(string? id, ResponseResult? response = null) { + // Record the service tier and system fingerprint each once if not yet recorded. + OpenAIClientExtensions.AddOpenAIResponseAttributes( + response?.ServiceTier?.ToString(), systemFingerprint: null, + ref serviceTier, ref systemFingerprint); + storedOutputDisabled |= IsStoredOutputDisabled(options, response); if (storedOutputDisabled) { diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs index 82da0ab4df8..326503bc04d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs @@ -7,11 +7,7 @@ using System.Diagnostics.Metrics; using System.Linq; using System.Runtime.CompilerServices; -using System.Text; -using System.Text.Encodings.Web; using System.Text.Json; -using System.Text.Json.Serialization; -using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -26,7 +22,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating chat client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.40, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// public sealed partial class OpenTelemetryChatClient : DelegatingChatClient @@ -74,19 +70,8 @@ public OpenTelemetryChatClient(IChatClient innerClient, ILogger? logger = null, _activitySource = new(name); _meter = new(name); - _tokenUsageHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, - OpenTelemetryConsts.TokensUnit, - OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries } - ); - - _operationDurationHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, - OpenTelemetryConsts.SecondsUnit, - OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } - ); + _tokenUsageHistogram = OtelMetricHelpers.CreateGenAITokenUsageHistogram(_meter); + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); _timeToFirstChunkHistogram = _meter.CreateHistogram( OpenTelemetryConsts.GenAI.Client.TimeToFirstChunk.Name, @@ -184,9 +169,9 @@ public override async IAsyncEnumerable GetStreamingResponseA _ = Throw.IfNull(messages); _jsonSerializerOptions.MakeReadOnly(); - using Activity? activity = CreateAndConfigureActivity(options); - bool trackChunkTimes = _timeToFirstChunkHistogram.Enabled || _timePerOutputChunkHistogram.Enabled; - Stopwatch? stopwatch = _operationDurationHistogram.Enabled || trackChunkTimes ? Stopwatch.StartNew() : null; + using Activity? activity = CreateAndConfigureActivity(options, streaming: true); + bool recordChunkHistograms = _timeToFirstChunkHistogram.Enabled || _timePerOutputChunkHistogram.Enabled; + Stopwatch? stopwatch = _operationDurationHistogram.Enabled || recordChunkHistograms || activity is not null ? Stopwatch.StartNew() : null; string? requestModelId = options?.ModelId ?? _defaultModelId; AddInputMessagesTags(messages, options, activity); @@ -207,8 +192,9 @@ public override async IAsyncEnumerable GetStreamingResponseA TimeSpan lastChunkElapsed = default; bool isFirstChunk = true; bool responseModelSet = false; + double? timeToFirstChunk = null; TagList chunkMetricTags = default; - if (trackChunkTimes) + if (recordChunkHistograms) { AddMetricTags(ref chunkMetricTags, requestModelId, response: null); } @@ -234,9 +220,9 @@ public override async IAsyncEnumerable GetStreamingResponseA throw; } - if (trackChunkTimes) + if (recordChunkHistograms) { - Debug.Assert(stopwatch is not null, "stopwatch should have been initialized when trackChunkTimes is true"); + Debug.Assert(stopwatch is not null, "stopwatch should have been initialized when recordChunkHistograms is true"); TimeSpan currentElapsed = stopwatch!.Elapsed; double delta = (currentElapsed - lastChunkElapsed).TotalSeconds; @@ -249,6 +235,7 @@ public override async IAsyncEnumerable GetStreamingResponseA if (isFirstChunk) { isFirstChunk = false; + timeToFirstChunk = delta; if (_timeToFirstChunkHistogram.Enabled) { _timeToFirstChunkHistogram.Record(delta, chunkMetricTags); @@ -261,6 +248,11 @@ public override async IAsyncEnumerable GetStreamingResponseA lastChunkElapsed = currentElapsed; } + else if (activity is not null && timeToFirstChunk is null) + { + Debug.Assert(stopwatch is not null, "stopwatch should have been initialized when activity is not null"); + timeToFirstChunk = stopwatch!.Elapsed.TotalSeconds; + } trackedUpdates.Add(update); yield return update; @@ -272,282 +264,14 @@ public override async IAsyncEnumerable GetStreamingResponseA } finally { - TraceResponse(activity, requestModelId, trackedUpdates.ToChatResponse(), error, stopwatch); + TraceResponse(activity, requestModelId, trackedUpdates.ToChatResponse(), error, stopwatch, timeToFirstChunk); await responseEnumerator.DisposeAsync(); } } - internal static string SerializeChatMessages( - IEnumerable messages, ChatFinishReason? chatFinishReason = null, JsonSerializerOptions? customContentSerializerOptions = null) - { - List output = []; - - string? finishReason = - chatFinishReason?.Value is null ? null : - chatFinishReason == ChatFinishReason.Length ? "length" : - chatFinishReason == ChatFinishReason.ContentFilter ? "content_filter" : - chatFinishReason == ChatFinishReason.ToolCalls ? "tool_call" : - "stop"; - - foreach (ChatMessage message in messages) - { - OtelMessage m = new() - { - FinishReason = finishReason, - Role = - message.Role == ChatRole.Assistant ? "assistant" : - message.Role == ChatRole.Tool ? "tool" : - message.Role == ChatRole.System || message.Role == new ChatRole("developer") ? "system" : - "user", - Name = message.AuthorName, - }; - - foreach (AIContent content in message.Contents) - { - switch (content) - { - // These are all specified in the convention: - - case TextContent tc when !string.IsNullOrWhiteSpace(tc.Text): - m.Parts.Add(new OtelGenericPart { Content = tc.Text }); - break; - - case TextReasoningContent trc when !string.IsNullOrWhiteSpace(trc.Text): - m.Parts.Add(new OtelGenericPart { Type = "reasoning", Content = trc.Text }); - break; - - case FunctionCallContent fcc: - m.Parts.Add(new OtelToolCallRequestPart - { - Id = fcc.CallId, - Name = fcc.Name, - Arguments = fcc.Arguments, - }); - break; - - case FunctionResultContent frc: - m.Parts.Add(new OtelToolCallResponsePart - { - Id = frc.CallId, - Response = frc.Result, - }); - break; - - case DataContent dc: - m.Parts.Add(new OtelBlobPart - { - Content = dc.Base64Data.ToString(), - MimeType = dc.MediaType, - Modality = DeriveModalityFromMediaType(dc.MediaType), - }); - break; - - case UriContent uc: - m.Parts.Add(new OtelUriPart - { - Uri = uc.Uri.AbsoluteUri, - MimeType = uc.MediaType, - Modality = DeriveModalityFromMediaType(uc.MediaType), - }); - break; - - case HostedFileContent fc: - m.Parts.Add(new OtelFilePart - { - FileId = fc.FileId, - MimeType = fc.MediaType, - Modality = DeriveModalityFromMediaType(fc.MediaType), - }); - break; - - // These are non-standard and are using the "generic" non-text part that provides an extensibility mechanism: - - case HostedVectorStoreContent vsc: - m.Parts.Add(new OtelGenericPart { Type = "vector_store", Content = vsc.VectorStoreId }); - break; - - case ErrorContent ec: - m.Parts.Add(new OtelGenericPart { Type = "error", Content = ec.Message }); - break; - - // Server tool call content types as specified in the OpenTelemetry semantic conventions: - - case CodeInterpreterToolCallContent citcc: - m.Parts.Add(new OtelServerToolCallPart - { - Id = citcc.CallId, - Name = "code_interpreter", - ServerToolCall = new OtelCodeInterpreterToolCall - { - Code = ExtractCodeFromInputs(citcc.Inputs), - }, - }); - break; - - case CodeInterpreterToolResultContent citrc: - m.Parts.Add(new OtelServerToolCallResponsePart - { - Id = citrc.CallId, - ServerToolCallResponse = new OtelCodeInterpreterToolCallResponse - { - Output = citrc.Outputs, - }, - }); - break; - - case ImageGenerationToolCallContent igtcc: - m.Parts.Add(new OtelServerToolCallPart - { - Id = igtcc.CallId, - Name = "image_generation", - ServerToolCall = new OtelImageGenerationToolCall(), - }); - break; - - case ImageGenerationToolResultContent igtrc: - m.Parts.Add(new OtelServerToolCallResponsePart - { - Id = igtrc.CallId, - ServerToolCallResponse = new OtelImageGenerationToolCallResponse - { - Output = igtrc.Outputs, - }, - }); - break; - - case McpServerToolCallContent mstcc: - m.Parts.Add(new OtelServerToolCallPart - { - Id = mstcc.CallId, - Name = mstcc.Name, - ServerToolCall = new OtelMcpToolCall - { - Arguments = mstcc.Arguments, - ServerName = mstcc.ServerName, - }, - }); - break; - - case McpServerToolResultContent mstrc: - m.Parts.Add(new OtelServerToolCallResponsePart - { - Id = mstrc.CallId, - ServerToolCallResponse = new OtelMcpToolCallResponse - { - Output = mstrc.Outputs, - }, - }); - break; - - case ToolApprovalRequestContent fareqc when fareqc.ToolCall is McpServerToolCallContent mcpToolCall: - m.Parts.Add(new OtelServerToolCallPart - { - Id = fareqc.RequestId, - Name = mcpToolCall.Name, - ServerToolCall = new OtelMcpApprovalRequest - { - Arguments = mcpToolCall.Arguments, - ServerName = mcpToolCall.ServerName, - }, - }); - break; - - case ToolApprovalResponseContent farespc when farespc.ToolCall is McpServerToolCallContent: - m.Parts.Add(new OtelServerToolCallResponsePart - { - Id = farespc.RequestId, - ServerToolCallResponse = new OtelMcpApprovalResponse - { - Approved = farespc.Approved, - }, - }); - break; - - default: - JsonElement element = _emptyObject; - try - { - JsonTypeInfo? unknownContentTypeInfo = - customContentSerializerOptions?.TryGetTypeInfo(content.GetType(), out JsonTypeInfo? ctsi) is true ? ctsi : - _defaultOptions.TryGetTypeInfo(content.GetType(), out JsonTypeInfo? dtsi) ? dtsi : - null; - - if (unknownContentTypeInfo is not null) - { - element = JsonSerializer.SerializeToElement(content, unknownContentTypeInfo); - } - } - catch - { - // Ignore the contents of any parts that can't be serialized. - } - - m.Parts.Add(new OtelGenericPart - { - Type = content.GetType().FullName!, - Content = element, - }); - break; - } - } - - output.Add(m); - } - - return JsonSerializer.Serialize(output, _defaultOptions.GetTypeInfo(typeof(IList))); - } - - private static string? DeriveModalityFromMediaType(string? mediaType) - { - if (mediaType is not null) - { - int pos = mediaType.IndexOf('/'); - if (pos >= 0) - { - ReadOnlySpan topLevel = mediaType.AsSpan(0, pos); - return - topLevel.Equals("image", StringComparison.OrdinalIgnoreCase) ? "image" : - topLevel.Equals("audio", StringComparison.OrdinalIgnoreCase) ? "audio" : - topLevel.Equals("video", StringComparison.OrdinalIgnoreCase) ? "video" : - null; - } - } - - return null; - } - - /// Extracts code text from code interpreter inputs. - /// - /// Code interpreter inputs typically contain a DataContent with a "text/x-python" or similar - /// media type representing the code to execute. - /// - private static string? ExtractCodeFromInputs(IList? inputs) - { - if (inputs is not null) - { - foreach (var input in inputs) - { - // Check for DataContent with text MIME types - if (input is DataContent dc && dc.HasTopLevelMediaType("text")) - { - // Return the data as a string (decode bytes as UTF8) - return Encoding.UTF8.GetString(dc.Data.ToArray()); - } - - // Check for TextContent - if (input is TextContent tc && !string.IsNullOrEmpty(tc.Text)) - { - return tc.Text; - } - } - } - - return null; - } - /// Creates an activity for a chat request, or returns if not enabled. - private Activity? CreateAndConfigureActivity(ChatOptions? options) + private Activity? CreateAndConfigureActivity(ChatOptions? options, bool streaming = false) { Activity? activity = null; if (_activitySource.HasListeners()) @@ -570,6 +294,11 @@ internal static string SerializeChatMessages( .AddTag(OpenTelemetryConsts.GenAI.Request.Model, modelId) .AddTag(OpenTelemetryConsts.GenAI.Provider.Name, _providerName); + if (streaming) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.Stream, true); + } + if (_serverAddress is not null) { _ = activity @@ -641,16 +370,7 @@ internal static string SerializeChatMessages( { _ = activity.AddTag( OpenTelemetryConsts.GenAI.Tool.Definitions, - JsonSerializer.Serialize(options.Tools.Select(t => t switch - { - _ when t.GetService() is { } af => new OtelFunction - { - Name = af.Name, - Description = af.Description, - Parameters = af.JsonSchema, - }, - _ => new OtelFunction { Type = t.Name }, - }), OtelContext.Default.IEnumerableOtelFunction)); + JsonSerializer.Serialize(options.Tools.Select(t => OtelFunction.Create(t, includeOptionalProperties: EnableSensitiveData)), OtelContext.Default.IEnumerableOtelFunction)); } if (EnableSensitiveData) @@ -678,7 +398,8 @@ private void TraceResponse( string? requestModelId, ChatResponse? response, Exception? error, - Stopwatch? stopwatch) + Stopwatch? stopwatch, + double? timeToFirstChunk = null) { if (_operationDurationHistogram.Enabled && stopwatch is not null) { @@ -712,17 +433,7 @@ private void TraceResponse( } } - if (error is not null) - { - _ = activity? - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - - if (_logger is not null) - { - OpenTelemetryLog.OperationException(_logger, error); - } - } + OpenTelemetryLog.RecordOperationError(activity, _logger, error); if (response is not null) { @@ -747,6 +458,11 @@ private void TraceResponse( _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Model, response.ModelId); } + if (timeToFirstChunk is double timeToFirstChunkValue) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.TimeToFirstChunk, timeToFirstChunkValue); + } + if (response.Usage?.InputTokenCount is long inputTokens) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); @@ -762,6 +478,11 @@ private void TraceResponse( _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.CacheReadInputTokens, (int)cachedInputTokens); } + if (response.Usage?.ReasoningTokenCount is long reasoningTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.ReasoningOutputTokens, (int)reasoningTokens); + } + // Log all additional response properties as raw values on the span. // Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data. if (EnableSensitiveData && response.AdditionalProperties is { } props) @@ -806,12 +527,12 @@ private void AddInputMessagesTags(IEnumerable messages, ChatOptions { _ = activity.AddTag( OpenTelemetryConsts.GenAI.SystemInstructions, - JsonSerializer.Serialize(new object[1] { new OtelGenericPart { Content = options!.Instructions } }, _defaultOptions.GetTypeInfo(typeof(IList)))); + JsonSerializer.Serialize(new object[1] { new OtelGenericPart { Content = options!.Instructions } }, OtelMessageSerializer.DefaultOptions.GetTypeInfo(typeof(IList)))); } _ = activity.AddTag( OpenTelemetryConsts.GenAI.Input.Messages, - SerializeChatMessages(messages, customContentSerializerOptions: _jsonSerializerOptions)); + OtelMessageSerializer.SerializeChatMessages(messages, customContentSerializerOptions: _jsonSerializerOptions)); } } @@ -821,174 +542,68 @@ private void AddOutputMessagesTags(ChatResponse response, Activity? activity) { _ = activity.AddTag( OpenTelemetryConsts.GenAI.Output.Messages, - SerializeChatMessages(response.Messages, response.FinishReason, customContentSerializerOptions: _jsonSerializerOptions)); + OtelMessageSerializer.SerializeChatMessages(response.Messages, response.FinishReason, customContentSerializerOptions: _jsonSerializerOptions)); } } - private sealed class OtelMessage - { - public string? Role { get; set; } - public string? Name { get; set; } - public List Parts { get; set; } = []; - public string? FinishReason { get; set; } - } - - private sealed class OtelGenericPart - { - public string Type { get; set; } = "text"; - public object? Content { get; set; } // should be a string when Type == "text" - } - - private sealed class OtelBlobPart - { - public string Type { get; set; } = "blob"; - public string? Content { get; set; } // base64-encoded binary data - public string? MimeType { get; set; } - public string? Modality { get; set; } - } - - private sealed class OtelUriPart - { - public string Type { get; set; } = "uri"; - public string? Uri { get; set; } - public string? MimeType { get; set; } - public string? Modality { get; set; } - } - - private sealed class OtelFilePart - { - public string Type { get; set; } = "file"; - public string? FileId { get; set; } - public string? MimeType { get; set; } - public string? Modality { get; set; } - } - - private sealed class OtelToolCallRequestPart - { - public string Type { get; set; } = "tool_call"; - public string? Id { get; set; } - public string? Name { get; set; } - public IDictionary? Arguments { get; set; } - } - - private sealed class OtelToolCallResponsePart - { - public string Type { get; set; } = "tool_call_response"; - public string? Id { get; set; } - public object? Response { get; set; } - } - - private sealed class OtelServerToolCallPart - where T : class - { - public string Type { get; set; } = "server_tool_call"; - public string? Id { get; set; } - public string? Name { get; set; } - public T? ServerToolCall { get; set; } - } - - private sealed class OtelServerToolCallResponsePart - where T : class - { - public string Type { get; set; } = "server_tool_call_response"; - public string? Id { get; set; } - public T? ServerToolCallResponse { get; set; } - } - - private sealed class OtelCodeInterpreterToolCall - { - public string Type { get; set; } = "code_interpreter"; - public string? Code { get; set; } - } - - private sealed class OtelCodeInterpreterToolCallResponse - { - public string Type { get; set; } = "code_interpreter"; - public object? Output { get; set; } - } - - private sealed class OtelImageGenerationToolCall - { - public string Type { get; set; } = "image_generation"; - } - - private sealed class OtelImageGenerationToolCallResponse - { - public string Type { get; set; } = "image_generation"; - public object? Output { get; set; } - } - - private sealed class OtelMcpToolCall - { - public string Type { get; set; } = "mcp"; - public string? ServerName { get; set; } - public IDictionary? Arguments { get; set; } - } - - private sealed class OtelMcpToolCallResponse - { - public string Type { get; set; } = "mcp"; - public object? Output { get; set; } - } + // Chat-specific OTel serialization POCOs. + // + // Types whose layout is shared 1:1 with OpenTelemetryRealtimeClientSession live in + // Common/OtelMessageParts.cs. The types below are either entirely chat-specific or + // contain chat-specific fields. The shared JsonSerializerContext lives in Common/OtelContext.cs, + // and the shared serialization helpers live in Common/OtelMessageSerializer.cs. +} - private sealed class OtelMcpApprovalRequest - { - public string Type { get; set; } = "mcp_approval_request"; - public string? ServerName { get; set; } - public IDictionary? Arguments { get; set; } - } +#pragma warning disable SA1402 // File may only contain a single type — chat-specific OTel POCOs are co-located with the chat client. - private sealed class OtelMcpApprovalResponse - { - public string Type { get; set; } = "mcp_approval_response"; - public bool Approved { get; set; } - } +internal sealed class OtelMessage +{ + public string? Role { get; set; } + public string? Name { get; set; } + public List Parts { get; set; } = []; + public string? FinishReason { get; set; } +} - private sealed class OtelFunction - { - public string Type { get; set; } = "function"; - public string? Name { get; set; } - public string? Description { get; set; } - public JsonElement? Parameters { get; set; } - } +internal sealed class OtelToolCallRequestPart +{ + public string Type { get; set; } = "tool_call"; + public string? Id { get; set; } + public string? Name { get; set; } + public IDictionary? Arguments { get; set; } +} - private static readonly JsonSerializerOptions _defaultOptions = CreateDefaultOptions(); - private static readonly JsonElement _emptyObject = JsonSerializer.SerializeToElement(new object(), _defaultOptions.GetTypeInfo(typeof(object))); +internal sealed class OtelCodeInterpreterToolCall +{ + public string Type { get; set; } = "code_interpreter"; + public string? Code { get; set; } +} - private static JsonSerializerOptions CreateDefaultOptions() - { - JsonSerializerOptions options = new(OtelContext.Default.Options) - { - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping - }; +internal sealed class OtelCodeInterpreterToolCallResponse +{ + public string Type { get; set; } = "code_interpreter"; + public object? Output { get; set; } +} - options.TypeInfoResolverChain.Add(AIJsonUtilities.DefaultOptions.TypeInfoResolver!); - options.MakeReadOnly(); +internal sealed class OtelImageGenerationToolCall +{ + public string Type { get; set; } = "image_generation"; +} - return options; - } +internal sealed class OtelImageGenerationToolCallResponse +{ + public string Type { get; set; } = "image_generation"; + public object? Output { get; set; } +} - [JsonSourceGenerationOptions( - PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, - WriteIndented = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] - [JsonSerializable(typeof(IList))] - [JsonSerializable(typeof(OtelMessage))] - [JsonSerializable(typeof(OtelGenericPart))] - [JsonSerializable(typeof(OtelBlobPart))] - [JsonSerializable(typeof(OtelUriPart))] - [JsonSerializable(typeof(OtelFilePart))] - [JsonSerializable(typeof(OtelToolCallRequestPart))] - [JsonSerializable(typeof(OtelToolCallResponsePart))] - [JsonSerializable(typeof(OtelServerToolCallPart))] - [JsonSerializable(typeof(OtelServerToolCallResponsePart))] - [JsonSerializable(typeof(OtelServerToolCallPart))] - [JsonSerializable(typeof(OtelServerToolCallResponsePart))] - [JsonSerializable(typeof(OtelServerToolCallPart))] - [JsonSerializable(typeof(OtelServerToolCallResponsePart))] - [JsonSerializable(typeof(OtelServerToolCallPart))] - [JsonSerializable(typeof(OtelServerToolCallResponsePart))] - [JsonSerializable(typeof(IEnumerable))] - private sealed partial class OtelContext : JsonSerializerContext; +internal sealed class OtelMcpApprovalRequest +{ + public string Type { get; set; } = "mcp_approval_request"; + public string? ServerName { get; set; } + public IDictionary? Arguments { get; set; } } +internal sealed class OtelMcpApprovalResponse +{ + public string Type { get; set; } = "mcp_approval_response"; + public bool Approved { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs index b50af4095bd..a20b512c7b0 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs @@ -20,7 +20,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating image generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.40, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// [Experimental(DiagnosticIds.Experiments.AIImageGeneration, UrlFormat = DiagnosticIds.UrlFormat)] @@ -62,19 +62,8 @@ public OpenTelemetryImageGenerator(IImageGenerator innerGenerator, ILogger? logg _activitySource = new(name); _meter = new(name); - _tokenUsageHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, - OpenTelemetryConsts.TokensUnit, - OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries } - ); - - _operationDurationHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, - OpenTelemetryConsts.SecondsUnit, - OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } - ); + _tokenUsageHistogram = OtelMetricHelpers.CreateGenAITokenUsageHistogram(_meter); + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); } /// @@ -198,7 +187,7 @@ public async override Task GenerateAsync( _ = activity.AddTag( OpenTelemetryConsts.GenAI.Input.Messages, - OpenTelemetryChatClient.SerializeChatMessages([new(ChatRole.User, content)])); + OtelMessageSerializer.SerializeChatMessages([new(ChatRole.User, content)])); if (options?.AdditionalProperties is { } props) { @@ -235,17 +224,7 @@ private void TraceResponse( _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); } - if (error is not null) - { - _ = activity? - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - - if (_logger is not null) - { - OpenTelemetryLog.OperationException(_logger, error); - } - } + OpenTelemetryLog.RecordOperationError(activity, _logger, error); if (response is not null) { @@ -255,7 +234,7 @@ private void TraceResponse( { _ = activity.AddTag( OpenTelemetryConsts.GenAI.Output.Messages, - OpenTelemetryChatClient.SerializeChatMessages([new(ChatRole.Assistant, contents)])); + OtelMessageSerializer.SerializeChatMessages([new(ChatRole.Assistant, contents)])); } if (response.Usage is { } usage) diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/OpenTelemetryLog.cs b/src/Libraries/Microsoft.Extensions.AI/Common/OpenTelemetryLog.cs index 4d4744076b2..4c5af90d7e4 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/OpenTelemetryLog.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/OpenTelemetryLog.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; using Microsoft.Extensions.Logging; namespace Microsoft.Extensions.AI; @@ -14,4 +15,24 @@ internal static partial class OpenTelemetryLog Level = LogLevel.Warning, Message = "gen_ai.client.operation.exception")] internal static partial void OperationException(ILogger logger, Exception error); + + /// Stamps the operation error tag/status on and logs the exception. + /// No-op when is . + internal static void RecordOperationError(Activity? activity, ILogger? logger, Exception? error) + { + if (error is null) + { + return; + } + + _ = activity? + .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, error.Message); + + if (logger is not null) + { + OperationException(logger, error); + } + } } + diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/OtelContext.cs b/src/Libraries/Microsoft.Extensions.AI/Common/OtelContext.cs new file mode 100644 index 00000000000..808a9872dc2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Common/OtelContext.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Extensions.AI; + +// Shared source-generated JsonSerializerContext for the OpenTelemetry* clients. +// Registers the union of all OTel message-part types serialized by both OpenTelemetryChatClient +// and OpenTelemetryRealtimeClientSession. + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(IList))] + +// Shared types (Common/OtelMessageParts.cs) +[JsonSerializable(typeof(OtelGenericPart))] +[JsonSerializable(typeof(OtelBlobPart))] +[JsonSerializable(typeof(OtelUriPart))] +[JsonSerializable(typeof(OtelFilePart))] +[JsonSerializable(typeof(OtelToolCallResponsePart))] +[JsonSerializable(typeof(IEnumerable))] + +// Chat-specific +[JsonSerializable(typeof(OtelMessage))] +[JsonSerializable(typeof(OtelToolCallRequestPart))] +[JsonSerializable(typeof(OtelServerToolCallPart))] +[JsonSerializable(typeof(OtelServerToolCallResponsePart))] +[JsonSerializable(typeof(OtelServerToolCallPart))] +[JsonSerializable(typeof(OtelServerToolCallResponsePart))] +[JsonSerializable(typeof(OtelServerToolCallPart))] +[JsonSerializable(typeof(OtelServerToolCallResponsePart))] +[JsonSerializable(typeof(OtelServerToolCallPart))] +[JsonSerializable(typeof(OtelServerToolCallResponsePart))] + +// Realtime-specific +[JsonSerializable(typeof(IEnumerable))] +[JsonSerializable(typeof(RealtimeOtelMessage))] +[JsonSerializable(typeof(RealtimeOtelToolCallPart))] +internal sealed partial class OtelContext : JsonSerializerContext; diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageParts.cs b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageParts.cs new file mode 100644 index 00000000000..f1ddd8e5c07 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageParts.cs @@ -0,0 +1,116 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json; + +#pragma warning disable SA1402 // File may only contain a single type — these POCOs are co-located on purpose. +#pragma warning disable SA1649 // File name should match first type name — this file holds the shared OTel POCOs as a group. + +namespace Microsoft.Extensions.AI; + +// Shared OTel message-part POCOs. +// +// Only types whose layout is byte-identical between the chat and realtime clients live here. Types +// that diverge remain in their respective client files. + +internal sealed class OtelGenericPart +{ + public string Type { get; set; } = "text"; + public object? Content { get; set; } // should be a string when Type == "text" +} + +internal sealed class OtelBlobPart +{ + public string Type { get; set; } = "blob"; + public string? Content { get; set; } // base64-encoded binary data + public string? MimeType { get; set; } + public string? Modality { get; set; } +} + +internal sealed class OtelUriPart +{ + public string Type { get; set; } = "uri"; + public string? Uri { get; set; } + public string? MimeType { get; set; } + public string? Modality { get; set; } +} + +internal sealed class OtelFilePart +{ + public string Type { get; set; } = "file"; + public string? FileId { get; set; } + public string? MimeType { get; set; } + public string? Modality { get; set; } +} + +internal sealed class OtelToolCallResponsePart +{ + public string Type { get; set; } = "tool_call_response"; + public string? Id { get; set; } + public object? Response { get; set; } +} + +internal sealed class OtelServerToolCallPart + where T : class +{ + public string Type { get; set; } = "server_tool_call"; + public string? Id { get; set; } + public string? Name { get; set; } + public T? ServerToolCall { get; set; } +} + +internal sealed class OtelServerToolCallResponsePart + where T : class +{ + public string Type { get; set; } = "server_tool_call_response"; + public string? Id { get; set; } + public T? ServerToolCallResponse { get; set; } +} + +internal sealed class OtelMcpToolCallResponse +{ + public string Type { get; set; } = "mcp"; + public object? Output { get; set; } +} + +internal sealed class OtelMcpToolCall +{ + public string Type { get; set; } = "mcp"; + public string? ServerName { get; set; } + public IReadOnlyDictionary? Arguments { get; set; } +} + +internal sealed class OtelFunction +{ + public string Type { get; set; } = "function"; + public string? Name { get; set; } + public string? Description { get; set; } + public JsonElement? Parameters { get; set; } + + /// Builds an from an . + /// The tool to describe. + /// + /// When , the optional and + /// properties will be set to , as they may contain sensitive, user-authored + /// values or large payloads. + /// + public static OtelFunction Create(AITool tool, bool includeOptionalProperties) + { + if (tool.GetService() is { } function) + { + return new() + { + Name = function.Name, + Description = includeOptionalProperties ? function.Description : null, + Parameters = includeOptionalProperties ? function.JsonSchema : null, + }; + } + + return new() + { + Type = tool.Name, + Name = tool.Name, + }; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageSerializer.cs b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageSerializer.cs new file mode 100644 index 00000000000..c5916379508 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageSerializer.cs @@ -0,0 +1,306 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; + +#pragma warning disable CA1307 // Specify StringComparison for clarity +#pragma warning disable CA1308 // Normalize strings to uppercase + +namespace Microsoft.Extensions.AI; + +/// Shared helpers for serializing chat messages to the OpenTelemetry gen-ai message-parts shape. +internal static class OtelMessageSerializer +{ + internal static readonly JsonSerializerOptions DefaultOptions = CreateDefaultOptions(); + + private static readonly JsonElement _emptyObject = + JsonSerializer.SerializeToElement(new object(), DefaultOptions.GetTypeInfo(typeof(object))); + + private static JsonSerializerOptions CreateDefaultOptions() + { + JsonSerializerOptions options = new(OtelContext.Default.Options) + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + options.TypeInfoResolverChain.Add(AIJsonUtilities.DefaultOptions.TypeInfoResolver!); + options.MakeReadOnly(); + + return options; + } + + internal static string SerializeChatMessages( + IEnumerable messages, ChatFinishReason? chatFinishReason = null, JsonSerializerOptions? customContentSerializerOptions = null) + { + List output = []; + + string? finishReason = + chatFinishReason?.Value is null ? null : + chatFinishReason == ChatFinishReason.Length ? "length" : + chatFinishReason == ChatFinishReason.ContentFilter ? "content_filter" : + chatFinishReason == ChatFinishReason.ToolCalls ? "tool_call" : + "stop"; + + foreach (ChatMessage message in messages) + { + OtelMessage m = new() + { + FinishReason = finishReason, + Role = + message.Role == ChatRole.Assistant ? "assistant" : + message.Role == ChatRole.Tool ? "tool" : + message.Role == ChatRole.System || message.Role == new ChatRole("developer") ? "system" : + "user", + Name = message.AuthorName, + }; + + foreach (AIContent content in message.Contents) + { + switch (content) + { + // These are all specified in the convention: + + case TextContent tc when !string.IsNullOrWhiteSpace(tc.Text): + m.Parts.Add(new OtelGenericPart { Content = tc.Text }); + break; + + case TextReasoningContent trc when !string.IsNullOrWhiteSpace(trc.Text): + m.Parts.Add(new OtelGenericPart { Type = "reasoning", Content = trc.Text }); + break; + + case FunctionCallContent fcc: + m.Parts.Add(new OtelToolCallRequestPart + { + Id = fcc.CallId, + Name = fcc.Name, + Arguments = fcc.Arguments, + }); + break; + + case FunctionResultContent frc: + m.Parts.Add(new OtelToolCallResponsePart + { + Id = frc.CallId, + Response = frc.Result, + }); + break; + + case DataContent dc: + m.Parts.Add(new OtelBlobPart + { + Content = dc.Base64Data.ToString(), + MimeType = dc.MediaType, + Modality = DeriveModalityFromMediaType(dc.MediaType), + }); + break; + + case UriContent uc: + m.Parts.Add(new OtelUriPart + { + Uri = uc.Uri.AbsoluteUri, + MimeType = uc.MediaType, + Modality = DeriveModalityFromMediaType(uc.MediaType), + }); + break; + + case HostedFileContent fc: + m.Parts.Add(new OtelFilePart + { + FileId = fc.FileId, + MimeType = fc.MediaType, + Modality = DeriveModalityFromMediaType(fc.MediaType), + }); + break; + + // These are non-standard and are using the "generic" non-text part that provides an extensibility mechanism: + + case HostedVectorStoreContent vsc: + m.Parts.Add(new OtelGenericPart { Type = "vector_store", Content = vsc.VectorStoreId }); + break; + + case ErrorContent ec: + m.Parts.Add(new OtelGenericPart { Type = "error", Content = ec.Message }); + break; + + // Server tool call content types as specified in the OpenTelemetry semantic conventions: + + case CodeInterpreterToolCallContent citcc: + m.Parts.Add(new OtelServerToolCallPart + { + Id = citcc.CallId, + Name = "code_interpreter", + ServerToolCall = new OtelCodeInterpreterToolCall + { + Code = ExtractCodeFromInputs(citcc.Inputs), + }, + }); + break; + + case CodeInterpreterToolResultContent citrc: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = citrc.CallId, + ServerToolCallResponse = new OtelCodeInterpreterToolCallResponse + { + Output = citrc.Outputs, + }, + }); + break; + + case ImageGenerationToolCallContent igtcc: + m.Parts.Add(new OtelServerToolCallPart + { + Id = igtcc.CallId, + Name = "image_generation", + ServerToolCall = new OtelImageGenerationToolCall(), + }); + break; + + case ImageGenerationToolResultContent igtrc: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = igtrc.CallId, + ServerToolCallResponse = new OtelImageGenerationToolCallResponse + { + Output = igtrc.Outputs, + }, + }); + break; + + case McpServerToolCallContent mstcc: + m.Parts.Add(new OtelServerToolCallPart + { + Id = mstcc.CallId, + Name = mstcc.Name, + ServerToolCall = new OtelMcpToolCall + { + Arguments = mstcc.Arguments as IReadOnlyDictionary ?? mstcc.Arguments?.ToDictionary(k => k.Key, v => v.Value), + ServerName = mstcc.ServerName, + }, + }); + break; + + case McpServerToolResultContent mstrc: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = mstrc.CallId, + ServerToolCallResponse = new OtelMcpToolCallResponse + { + Output = mstrc.Outputs, + }, + }); + break; + + case ToolApprovalRequestContent fareqc when fareqc.ToolCall is McpServerToolCallContent mcpToolCall: + m.Parts.Add(new OtelServerToolCallPart + { + Id = fareqc.RequestId, + Name = mcpToolCall.Name, + ServerToolCall = new OtelMcpApprovalRequest + { + Arguments = mcpToolCall.Arguments, + ServerName = mcpToolCall.ServerName, + }, + }); + break; + + case ToolApprovalResponseContent farespc when farespc.ToolCall is McpServerToolCallContent: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = farespc.RequestId, + ServerToolCallResponse = new OtelMcpApprovalResponse + { + Approved = farespc.Approved, + }, + }); + break; + + default: + JsonElement element = _emptyObject; + try + { + JsonTypeInfo? unknownContentTypeInfo = + customContentSerializerOptions?.TryGetTypeInfo(content.GetType(), out JsonTypeInfo? ctsi) is true ? ctsi : + DefaultOptions.TryGetTypeInfo(content.GetType(), out JsonTypeInfo? dtsi) ? dtsi : + null; + + if (unknownContentTypeInfo is not null) + { + element = JsonSerializer.SerializeToElement(content, unknownContentTypeInfo); + } + } + catch + { + // Ignore the contents of any parts that can't be serialized. + } + + m.Parts.Add(new OtelGenericPart + { + Type = content.GetType().FullName!, + Content = element, + }); + break; + } + } + + output.Add(m); + } + + return JsonSerializer.Serialize(output, DefaultOptions.GetTypeInfo(typeof(IList))); + } + + /// Derives the OTel modality classifier from a media type's top-level type. + internal static string? DeriveModalityFromMediaType(string? mediaType) + { + if (mediaType is not null) + { + int pos = mediaType.IndexOf('/'); + if (pos >= 0) + { + ReadOnlySpan topLevel = mediaType.AsSpan(0, pos); + return + topLevel.Equals("image", StringComparison.OrdinalIgnoreCase) ? "image" : + topLevel.Equals("audio", StringComparison.OrdinalIgnoreCase) ? "audio" : + topLevel.Equals("video", StringComparison.OrdinalIgnoreCase) ? "video" : + null; + } + } + + return null; + } + + /// Extracts code text from code interpreter inputs. + /// + /// Code interpreter inputs typically contain a DataContent with a "text/x-python" or similar + /// media type representing the code to execute. + /// + private static string? ExtractCodeFromInputs(IList? inputs) + { + if (inputs is not null) + { + foreach (var input in inputs) + { + // Check for DataContent with text MIME types + if (input is DataContent dc && dc.HasTopLevelMediaType("text")) + { + // Return the data as a string (decode bytes as UTF8) + return Encoding.UTF8.GetString(dc.Data.ToArray()); + } + + // Check for TextContent + if (input is TextContent tc && !string.IsNullOrEmpty(tc.Text)) + { + return tc.Text; + } + } + } + + return null; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/OtelMetricHelpers.cs b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMetricHelpers.cs new file mode 100644 index 00000000000..ca572ca515a --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMetricHelpers.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.Metrics; + +namespace Microsoft.Extensions.AI; + +/// Shared metric instrument factories for the OpenTelemetry* clients. +internal static class OtelMetricHelpers +{ + /// Creates the standard gen_ai.client.token.usage histogram on . + public static Histogram CreateGenAITokenUsageHistogram(Meter meter) => + meter.CreateHistogram( + OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, + OpenTelemetryConsts.TokensUnit, + OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, + advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries }); + + /// Creates the standard gen_ai.client.operation.duration histogram on . + public static Histogram CreateGenAIOperationDurationHistogram(Meter meter) => + meter.CreateHistogram( + OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, + OpenTelemetryConsts.SecondsUnit, + OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, + advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries }); +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs index 81a5eb4aa0a..090332b255f 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs @@ -18,7 +18,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating embedding generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.40, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// /// The type of input used to produce embeddings. @@ -66,19 +66,8 @@ public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator i _activitySource = new(name); _meter = new(name); - _tokenUsageHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, - OpenTelemetryConsts.TokensUnit, - OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries } - ); - - _operationDurationHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, - OpenTelemetryConsts.SecondsUnit, - OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } - ); + _tokenUsageHistogram = OtelMetricHelpers.CreateGenAITokenUsageHistogram(_meter); + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); } /// @@ -230,20 +219,10 @@ private void TraceResponse( _tokenUsageHistogram.Record(inputTokens.Value, tags); } + OpenTelemetryLog.RecordOperationError(activity, _logger, error); + if (activity is not null) { - if (error is not null) - { - _ = activity - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - - if (_logger is not null) - { - OpenTelemetryLog.OperationException(_logger, error); - } - } - if (inputTokens.HasValue) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, inputTokens); diff --git a/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs b/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs index 85a740cbec2..057edda6151 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs @@ -395,20 +395,8 @@ public override async Task DeleteAsync( } } - private void SetErrorStatus(Activity? activity, Exception? error) - { - if (error is not null) - { - _ = activity? - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - - if (_logger is not null) - { - OpenTelemetryLog.OperationException(_logger, error); - } - } - } + private void SetErrorStatus(Activity? activity, Exception? error) => + OpenTelemetryLog.RecordOperationError(activity, _logger, error); private void TagAdditionalProperties(Activity activity, HostedFileClientOptions? options) { diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 8ffbd0b9dec..804d0d9f684 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -128,6 +128,7 @@ public static class Request public const string PresencePenalty = "gen_ai.request.presence_penalty"; public const string Seed = "gen_ai.request.seed"; public const string StopSequences = "gen_ai.request.stop_sequences"; + public const string Stream = "gen_ai.request.stream"; public const string Temperature = "gen_ai.request.temperature"; public const string TopK = "gen_ai.request.top_k"; public const string TopP = "gen_ai.request.top_p"; @@ -138,6 +139,7 @@ public static class Response public const string FinishReasons = "gen_ai.response.finish_reasons"; public const string Id = "gen_ai.response.id"; public const string Model = "gen_ai.response.model"; + public const string TimeToFirstChunk = "gen_ai.response.time_to_first_chunk"; } public static class Token @@ -170,11 +172,12 @@ public static class Usage public const string InputTextTokens = "gen_ai.usage.input_text_tokens"; public const string OutputAudioTokens = "gen_ai.usage.output_audio_tokens"; public const string OutputTextTokens = "gen_ai.usage.output_text_tokens"; + public const string ReasoningOutputTokens = "gen_ai.usage.reasoning.output_tokens"; } /// /// Custom attributes for realtime sessions. - /// These attributes are NOT part of the OpenTelemetry GenAI semantic conventions (as of v1.40). + /// These attributes are NOT part of the OpenTelemetry GenAI semantic conventions (as of v1.41). /// They are custom extensions to capture realtime session-specific configuration. /// public static class Realtime diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs index bc16075e9da..a7f78f9e9a6 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Text.Json; -using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; @@ -22,10 +21,14 @@ namespace Microsoft.Extensions.AI; -/// Represents a delegating realtime session that implements the OpenTelemetry Semantic Conventions for Generative AI systems. +/// Represents a delegating realtime session that follows the OpenTelemetry Semantic Conventions for Generative AI systems where applicable. /// /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.38, defined at . +/// This class follows the patterns of the Semantic Conventions for Generative AI systems v1.41 where applicable, as defined at +/// , with custom extensions for realtime-specific behavior. +/// The specification does not currently define a realtime operation; a custom operation name is used. +/// +/// /// The specification is still experimental and subject to change; as such, the telemetry output by this session is also subject to change. /// /// @@ -33,15 +36,18 @@ namespace Microsoft.Extensions.AI; /// /// gen_ai.operation.name - Operation name ("chat") /// gen_ai.request.model - Model name from options +/// gen_ai.request.stream - Indicates streaming response requests; always as realtime is inherently streaming /// gen_ai.provider.name - Provider name from metadata /// gen_ai.response.id - Response ID from ResponseDone messages /// gen_ai.response.model - Model ID from response +/// gen_ai.response.time_to_first_chunk - Time to first streaming response chunk /// gen_ai.usage.input_tokens - Input token count /// gen_ai.usage.output_tokens - Output token count +/// gen_ai.usage.reasoning.output_tokens - Reasoning output token count /// gen_ai.request.max_tokens - Max output tokens from options /// gen_ai.system_instructions - Instructions from options (sensitive data) /// gen_ai.conversation.id - Conversation ID from response -/// gen_ai.tool.definitions - Tool definitions (sensitive data) +/// gen_ai.tool.definitions - Tool definitions /// gen_ai.input.messages - Input tool/MCP messages (sensitive data) /// gen_ai.output.messages - Output tool/MCP messages (sensitive data) /// server.address / server.port - Server endpoint info @@ -57,7 +63,7 @@ namespace Microsoft.Extensions.AI; /// /// /// -/// Additionally, the following custom attributes are supported (not part of OpenTelemetry GenAI semantic conventions as of v1.40): +/// Additionally, the following custom attributes are supported (not part of OpenTelemetry GenAI semantic conventions as of v1.41): /// /// gen_ai.request.tool_choice - Tool choice mode ("none", "auto", "required") or specific tool name /// gen_ai.realtime.voice - Voice setting from options @@ -114,19 +120,8 @@ public OpenTelemetryRealtimeClientSession(IRealtimeClientSession innerSession, I _activitySource = new(name); _meter = new(name); - _tokenUsageHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, - OpenTelemetryConsts.TokensUnit, - OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries } - ); - - _operationDurationHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, - OpenTelemetryConsts.SecondsUnit, - OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } - ); + _tokenUsageHistogram = OtelMetricHelpers.CreateGenAITokenUsageHistogram(_meter); + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); _jsonSerializerOptions = AIJsonUtilities.DefaultOptions; } @@ -207,7 +202,9 @@ public async IAsyncEnumerable GetStreamingResponseAsync( string? requestModelId = options?.Model ?? _defaultModelId; // Start timing from the beginning of the streaming operation - Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + bool trackStreamingResponseTime = _activitySource.HasListeners(); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled || trackStreamingResponseTime ? Stopwatch.StartNew() : null; + double? timeToFirstChunk = null; // Determine if we should capture messages for telemetry bool captureMessages = EnableSensitiveData && _activitySource.HasListeners(); @@ -220,8 +217,8 @@ public async IAsyncEnumerable GetStreamingResponseAsync( catch (Exception ex) { // Create an activity for the error case - using Activity? errorActivity = CreateAndConfigureActivity(options); - TraceStreamingResponse(errorActivity, requestModelId, response: null, ex, stopwatch); + using Activity? errorActivity = CreateAndConfigureActivity(options, streamingResponse: true); + TraceStreamingResponse(errorActivity, requestModelId, response: null, ex, stopwatch, timeToFirstChunk); throw; } @@ -249,6 +246,11 @@ public async IAsyncEnumerable GetStreamingResponseAsync( throw; } + if (timeToFirstChunk is null && stopwatch is not null) + { + timeToFirstChunk = stopwatch.Elapsed.TotalSeconds; + } + // Track output modalities if (outputModalities is not null) { @@ -273,12 +275,12 @@ public async IAsyncEnumerable GetStreamingResponseAsync( if (message is ResponseCreatedRealtimeServerMessage responseDoneMsg && responseDoneMsg.Type == RealtimeServerMessageType.ResponseDone) { - using Activity? responseActivity = CreateAndConfigureActivity(options); + using Activity? responseActivity = CreateAndConfigureActivity(options, streamingResponse: true); // Add output modalities and messages tags AddOutputModalitiesTag(responseActivity, outputModalities); AddOutputMessagesTag(responseActivity, outputMessages); - TraceStreamingResponse(responseActivity, requestModelId, responseDoneMsg, error, stopwatch); + TraceStreamingResponse(responseActivity, requestModelId, responseDoneMsg, error, stopwatch, timeToFirstChunk); } yield return message; @@ -289,10 +291,10 @@ public async IAsyncEnumerable GetStreamingResponseAsync( // Trace error if an exception was thrown during streaming if (error is not null) { - using Activity? errorActivity = CreateAndConfigureActivity(options); + using Activity? errorActivity = CreateAndConfigureActivity(options, streamingResponse: true); AddOutputModalitiesTag(errorActivity, outputModalities); AddOutputMessagesTag(errorActivity, outputMessages); - TraceStreamingResponse(errorActivity, requestModelId, response: null, error, stopwatch); + TraceStreamingResponse(errorActivity, requestModelId, response: null, error, stopwatch, timeToFirstChunk); } await responseEnumerator.DisposeAsync().ConfigureAwait(false); @@ -356,7 +358,7 @@ private static void AddOutputMessagesTag(Activity? activity, ListSerializes a single message to OTel format (as an array with one element). private static string SerializeMessage(RealtimeOtelMessage message) { - return JsonSerializer.Serialize(new[] { message }, RealtimeOtelContext.Default.IEnumerableRealtimeOtelMessage); + return JsonSerializer.Serialize(new[] { message }, OtelContext.Default.IEnumerableRealtimeOtelMessage); } /// Serializes content items to OTel format. private static string SerializeMessages(IEnumerable messages) { - return JsonSerializer.Serialize(messages, RealtimeOtelContext.Default.IEnumerableRealtimeOtelMessage); + return JsonSerializer.Serialize(messages, OtelContext.Default.IEnumerableRealtimeOtelMessage); } /// Extracts content from an AIContent list and converts to OTel format. @@ -526,11 +528,11 @@ private static string SerializeMessages(IEnumerable message { // Standard text content case TextContent tc when !string.IsNullOrEmpty(tc.Text): - message.Parts.Add(new RealtimeOtelGenericPart { Content = tc.Text }); + message.Parts.Add(new OtelGenericPart { Content = tc.Text }); break; case TextReasoningContent trc when !string.IsNullOrEmpty(trc.Text): - message.Parts.Add(new RealtimeOtelGenericPart { Type = "reasoning", Content = trc.Text }); + message.Parts.Add(new OtelGenericPart { Type = "reasoning", Content = trc.Text }); break; // Function call content @@ -544,7 +546,7 @@ private static string SerializeMessages(IEnumerable message break; case FunctionResultContent frc: - message.Parts.Add(new RealtimeOtelToolCallResponsePart + message.Parts.Add(new OtelToolCallResponsePart { Id = frc.CallId, Response = frc.Result, @@ -553,50 +555,50 @@ private static string SerializeMessages(IEnumerable message // Data content (binary data) case DataContent dc: - message.Parts.Add(new RealtimeOtelBlobPart + message.Parts.Add(new OtelBlobPart { Content = dc.Base64Data.ToString(), MimeType = dc.MediaType, - Modality = DeriveModalityFromMediaType(dc.MediaType), + Modality = OtelMessageSerializer.DeriveModalityFromMediaType(dc.MediaType), }); break; // URI content case UriContent uc: - message.Parts.Add(new RealtimeOtelUriPart + message.Parts.Add(new OtelUriPart { Uri = uc.Uri.AbsoluteUri, MimeType = uc.MediaType, - Modality = DeriveModalityFromMediaType(uc.MediaType), + Modality = OtelMessageSerializer.DeriveModalityFromMediaType(uc.MediaType), }); break; // Hosted file content case HostedFileContent fc: - message.Parts.Add(new RealtimeOtelFilePart + message.Parts.Add(new OtelFilePart { FileId = fc.FileId, MimeType = fc.MediaType, - Modality = DeriveModalityFromMediaType(fc.MediaType), + Modality = OtelMessageSerializer.DeriveModalityFromMediaType(fc.MediaType), }); break; // Non-standard "generic" parts case HostedVectorStoreContent vsc: - message.Parts.Add(new RealtimeOtelGenericPart { Type = "vector_store", Content = vsc.VectorStoreId }); + message.Parts.Add(new OtelGenericPart { Type = "vector_store", Content = vsc.VectorStoreId }); break; case ErrorContent ec: - message.Parts.Add(new RealtimeOtelGenericPart { Type = "error", Content = ec.Message }); + message.Parts.Add(new OtelGenericPart { Type = "error", Content = ec.Message }); break; // MCP server tool content case McpServerToolCallContent mstcc: - message.Parts.Add(new RealtimeOtelServerToolCallPart + message.Parts.Add(new OtelServerToolCallPart { Id = mstcc.CallId, Name = mstcc.Name, - ServerToolCall = new RealtimeOtelMcpToolCall + ServerToolCall = new OtelMcpToolCall { Arguments = mstcc.Arguments as IReadOnlyDictionary ?? mstcc.Arguments?.ToDictionary(k => k.Key, v => v.Value), ServerName = mstcc.ServerName, @@ -605,10 +607,10 @@ private static string SerializeMessages(IEnumerable message break; case McpServerToolResultContent mstrc: - message.Parts.Add(new RealtimeOtelServerToolCallResponsePart + message.Parts.Add(new OtelServerToolCallResponsePart { Id = mstrc.CallId, - ServerToolCallResponse = new RealtimeOtelMcpToolCallResponse + ServerToolCallResponse = new OtelMcpToolCallResponse { Output = mstrc.Outputs, }, @@ -642,7 +644,7 @@ private static string SerializeMessages(IEnumerable message if (element.ValueKind != JsonValueKind.Undefined) { - message.Parts.Add(new RealtimeOtelGenericPart + message.Parts.Add(new OtelGenericPart { Type = content.GetType().Name, Content = element, @@ -656,34 +658,8 @@ private static string SerializeMessages(IEnumerable message return message.Parts.Count > 0 ? message : null; } - /// Derives modality from media type for telemetry purposes. - private static string? DeriveModalityFromMediaType(string? mediaType) - { - if (string.IsNullOrEmpty(mediaType)) - { - return null; - } - - if (mediaType!.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) - { - return "image"; - } - - if (mediaType.StartsWith("audio/", StringComparison.OrdinalIgnoreCase)) - { - return "audio"; - } - - if (mediaType.StartsWith("video/", StringComparison.OrdinalIgnoreCase)) - { - return "video"; - } - - return null; - } - /// Creates an activity for a realtime session request, or returns if not enabled. - private Activity? CreateAndConfigureActivity(RealtimeSessionOptions? options) + private Activity? CreateAndConfigureActivity(RealtimeSessionOptions? options, bool streamingResponse = false) { Activity? activity = null; if (_activitySource.HasListeners()) @@ -701,6 +677,11 @@ private static string SerializeMessages(IEnumerable message .AddTag(OpenTelemetryConsts.GenAI.Request.Model, modelId) .AddTag(OpenTelemetryConsts.GenAI.Provider.Name, _providerName); + if (streamingResponse) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.Stream, true); + } + if (_serverAddress is not null) { _ = activity @@ -735,24 +716,16 @@ private static string SerializeMessages(IEnumerable message { _ = activity.AddTag( OpenTelemetryConsts.GenAI.SystemInstructions, - JsonSerializer.Serialize(new object[1] { new RealtimeOtelGenericPart { Content = options.Instructions } }, RealtimeOtelContext.Default.IListObject)); + JsonSerializer.Serialize(new object[1] { new OtelGenericPart { Content = options.Instructions } }, OtelContext.Default.IListObject)); } - if (options.Tools is { Count: > 0 }) - { - _ = activity.AddTag( - OpenTelemetryConsts.GenAI.Tool.Definitions, - JsonSerializer.Serialize(options.Tools.Select(t => t switch - { - _ when t.GetService() is { } af => new RealtimeOtelFunction - { - Name = af.Name, - Description = af.Description, - Parameters = af.JsonSchema, - }, - _ => new RealtimeOtelFunction { Type = t.Name }, - }), RealtimeOtelContext.Default.IEnumerableRealtimeOtelFunction)); - } + } + + if (options.Tools is { Count: > 0 }) + { + _ = activity.AddTag( + OpenTelemetryConsts.GenAI.Tool.Definitions, + JsonSerializer.Serialize(options.Tools.Select(t => OtelFunction.Create(t, includeOptionalProperties: EnableSensitiveData)), OtelContext.Default.IEnumerableOtelFunction)); } } } @@ -767,7 +740,8 @@ private void TraceStreamingResponse( string? requestModelId, ResponseCreatedRealtimeServerMessage? response, Exception? error, - Stopwatch? stopwatch) + Stopwatch? stopwatch, + double? timeToFirstChunk = null) { if (_operationDurationHistogram.Enabled && stopwatch is not null) { @@ -833,17 +807,7 @@ private void TraceStreamingResponse( } } - if (error is not null) - { - _ = activity? - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - - if (_logger is not null) - { - OpenTelemetryLog.OperationException(_logger, error); - } - } + OpenTelemetryLog.RecordOperationError(activity, _logger, error); if (response is not null && activity is not null) { @@ -861,6 +825,11 @@ private void TraceStreamingResponse( _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Id, response.ResponseId); } + if (timeToFirstChunk is double timeToFirstChunkValue) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.TimeToFirstChunk, timeToFirstChunkValue); + } + if (!string.IsNullOrWhiteSpace(response.Status)) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.FinishReasons, $"[\"{response.Status}\"]"); @@ -883,6 +852,11 @@ private void TraceStreamingResponse( _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.CacheReadInputTokens, (int)cachedInputTokens); } + if (responseUsage.ReasoningTokenCount is long reasoningTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.ReasoningOutputTokens, (int)reasoningTokens); + } + if (responseUsage.InputAudioTokenCount is long inputAudioTokens) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputAudioTokens, (int)inputAudioTokens); @@ -938,113 +912,27 @@ private void AddMetricTags(ref TagList tags, string? requestModelId, string? res #region OTel Serialization Types - private sealed class RealtimeOtelGenericPart - { - public string Type { get; set; } = "text"; - public object? Content { get; set; } - } + // Realtime-specific OTel serialization POCOs. + // + // Types whose layout is shared 1:1 with OpenTelemetryChatClient live in + // Common/OtelMessageParts.cs. The types below are either entirely realtime-specific or + // contain realtime-specific fields. The shared JsonSerializerContext lives in Common/OtelContext.cs. - private sealed class RealtimeOtelBlobPart - { - public string Type { get; set; } = "blob"; - public string? Content { get; set; } // base64-encoded binary data - public string? MimeType { get; set; } - public string? Modality { get; set; } - } - - private sealed class RealtimeOtelUriPart - { - public string Type { get; set; } = "uri"; - public string? Uri { get; set; } - public string? MimeType { get; set; } - public string? Modality { get; set; } - } - - private sealed class RealtimeOtelFilePart - { - public string Type { get; set; } = "file"; - public string? FileId { get; set; } - public string? MimeType { get; set; } - public string? Modality { get; set; } - } - - private sealed class RealtimeOtelFunction - { - public string Type { get; set; } = "function"; - public string? Name { get; set; } - public string? Description { get; set; } - public JsonElement? Parameters { get; set; } - } - - private sealed class RealtimeOtelMessage - { - public string? Role { get; set; } - public List Parts { get; set; } = []; - } - - private sealed class RealtimeOtelToolCallPart - { - public string Type { get; set; } = "tool_call"; - public string? Id { get; set; } - public string? Name { get; set; } - public IDictionary? Arguments { get; set; } - } - - private sealed class RealtimeOtelToolCallResponsePart - { - public string Type { get; set; } = "tool_call_response"; - public string? Id { get; set; } - public object? Response { get; set; } - } - - private sealed class RealtimeOtelServerToolCallPart - where T : class - { - public string Type { get; set; } = "server_tool_call"; - public string? Id { get; set; } - public string? Name { get; set; } - public T? ServerToolCall { get; set; } - } - - private sealed class RealtimeOtelServerToolCallResponsePart - where T : class - { - public string Type { get; set; } = "server_tool_call_response"; - public string? Id { get; set; } - public T? ServerToolCallResponse { get; set; } - } - - private sealed class RealtimeOtelMcpToolCall - { - public string Type { get; set; } = "mcp"; - public string? ServerName { get; set; } - public IReadOnlyDictionary? Arguments { get; set; } - } + #endregion +} - private sealed class RealtimeOtelMcpToolCallResponse - { - public string Type { get; set; } = "mcp"; - public object? Output { get; set; } - } +#pragma warning disable SA1402 // File may only contain a single type — realtime-specific OTel POCOs are co-located with the realtime session. - #endregion +internal sealed class RealtimeOtelMessage +{ + public string? Role { get; set; } + public List Parts { get; set; } = []; +} - [JsonSourceGenerationOptions( - PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, - WriteIndented = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] - [JsonSerializable(typeof(IList))] - [JsonSerializable(typeof(RealtimeOtelGenericPart))] - [JsonSerializable(typeof(RealtimeOtelBlobPart))] - [JsonSerializable(typeof(RealtimeOtelUriPart))] - [JsonSerializable(typeof(RealtimeOtelFilePart))] - [JsonSerializable(typeof(IEnumerable))] - [JsonSerializable(typeof(IEnumerable))] - [JsonSerializable(typeof(RealtimeOtelMessage))] - [JsonSerializable(typeof(RealtimeOtelToolCallPart))] - [JsonSerializable(typeof(RealtimeOtelToolCallResponsePart))] - [JsonSerializable(typeof(RealtimeOtelServerToolCallPart))] - [JsonSerializable(typeof(RealtimeOtelServerToolCallResponsePart))] - - private sealed partial class RealtimeOtelContext : JsonSerializerContext; +internal sealed class RealtimeOtelToolCallPart +{ + public string Type { get; set; } = "tool_call"; + public string? Id { get; set; } + public string? Name { get; set; } + public IDictionary? Arguments { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs index 20ffba484f2..82ece57f673 100644 --- a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs @@ -22,7 +22,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating speech-to-text client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.40, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// [Experimental(DiagnosticIds.Experiments.AISpeechToText, UrlFormat = DiagnosticIds.UrlFormat)] @@ -64,19 +64,8 @@ public OpenTelemetrySpeechToTextClient(ISpeechToTextClient innerClient, ILogger? _activitySource = new(name); _meter = new(name); - _tokenUsageHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, - OpenTelemetryConsts.TokensUnit, - OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries } - ); - - _operationDurationHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, - OpenTelemetryConsts.SecondsUnit, - OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } - ); + _tokenUsageHistogram = OtelMetricHelpers.CreateGenAITokenUsageHistogram(_meter); + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); } /// @@ -288,17 +277,7 @@ private void TraceResponse( } } - if (error is not null) - { - _ = activity? - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - - if (_logger is not null) - { - OpenTelemetryLog.OperationException(_logger, error); - } - } + OpenTelemetryLog.RecordOperationError(activity, _logger, error); if (response is not null) { @@ -368,7 +347,7 @@ private void AddOutputMessagesTags(SpeechToTextResponse response, Activity? acti { _ = activity.AddTag( OpenTelemetryConsts.GenAI.Output.Messages, - OpenTelemetryChatClient.SerializeChatMessages([new(ChatRole.Assistant, response.Contents)])); + OtelMessageSerializer.SerializeChatMessages([new(ChatRole.Assistant, response.Contents)])); } } } diff --git a/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs b/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs index b4ad4a663fc..3cf4eed611d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs @@ -21,7 +21,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating text-to-speech client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.40, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// [Experimental(DiagnosticIds.Experiments.AITextToSpeech, UrlFormat = DiagnosticIds.UrlFormat)] @@ -63,19 +63,8 @@ public OpenTelemetryTextToSpeechClient(ITextToSpeechClient innerClient, ILogger? _activitySource = new(name); _meter = new(name); - _tokenUsageHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.TokenUsage.Name, - OpenTelemetryConsts.TokensUnit, - OpenTelemetryConsts.GenAI.Client.TokenUsage.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.TokenUsage.ExplicitBucketBoundaries } - ); - - _operationDurationHistogram = _meter.CreateHistogram( - OpenTelemetryConsts.GenAI.Client.OperationDuration.Name, - OpenTelemetryConsts.SecondsUnit, - OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, - advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } - ); + _tokenUsageHistogram = OtelMetricHelpers.CreateGenAITokenUsageHistogram(_meter); + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); } /// @@ -287,17 +276,7 @@ private void TraceResponse( } } - if (error is not null) - { - _ = activity? - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - - if (_logger is not null) - { - OpenTelemetryLog.OperationException(_logger, error); - } - } + OpenTelemetryLog.RecordOperationError(activity, _logger, error); if (response is not null && activity is not null) { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs index 0c27ee27747..fe79396399d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs @@ -1976,7 +1976,7 @@ public async Task ReasoningContent_Streaming_SurfacedAsTextReasoningContent() [InlineData(true)] public async Task OpenAIApiTypeTag_SetToChatCompletions(bool streaming) { - const string Output = """ + const string NonStreamingOutput = """ { "id": "chatcmpl-test", "object": "chat.completion", @@ -1996,10 +1996,21 @@ public async Task OpenAIApiTypeTag_SetToChatCompletions(bool streaming) "prompt_tokens": 8, "completion_tokens": 2, "total_tokens": 10 - } + }, + "service_tier": "default", + "system_fingerprint": "fp_test" } """; + const string StreamingOutput = """ + data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1727888631,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_test","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello!"},"finish_reason":null}]} + + data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1727888631,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_test","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":8,"completion_tokens":2,"total_tokens":10}} + + data: [DONE] + + """; + var sourceName = Guid.NewGuid().ToString(); var activities = new List(); using var listener = new ActivityListener @@ -2010,7 +2021,7 @@ public async Task OpenAIApiTypeTag_SetToChatCompletions(bool streaming) }; ActivitySource.AddActivityListener(listener); - using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput(), Output); + using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput(), streaming ? StreamingOutput : NonStreamingOutput); using HttpClient httpClient = new(handler); using IChatClient client = new OpenAIClient(new ApiKeyCredential("apikey"), new OpenAIClientOptions { Transport = new HttpClientPipelineTransport(httpClient) }) .GetChatClient("gpt-4o-mini") @@ -2033,5 +2044,7 @@ public async Task OpenAIApiTypeTag_SetToChatCompletions(bool streaming) var activity = Assert.Single(activities); Assert.Equal("chat_completions", activity.GetTagItem("openai.api.type")); + Assert.Equal("default", activity.GetTagItem("openai.response.service_tier")); + Assert.Equal("fp_test", activity.GetTagItem("openai.response.system_fingerprint")); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs index d0d1541eaf7..90f8343d8f0 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs @@ -6625,13 +6625,14 @@ public async Task ReasoningOptions_EffortAndOutput_ProducesExpectedJson( [InlineData(true)] public async Task OpenAIApiTypeTag_SetToResponses(bool streaming) { - const string Output = """ + const string NonStreamingOutput = """ { "id": "resp_test", "object": "response", "created_at": 1741891428, "status": "completed", "model": "gpt-4o-mini", + "service_tier": "default", "output": [ { "id": "msg_test", @@ -6654,6 +6655,15 @@ public async Task OpenAIApiTypeTag_SetToResponses(bool streaming) } """; + const string StreamingOutput = """ + event: response.created + data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_test","object":"response","created_at":1741891428,"status":"in_progress","model":"gpt-4o-mini","service_tier":"default","output":[]}} + + event: response.completed + data: {"type":"response.completed","sequence_number":1,"response":{"id":"resp_test","object":"response","created_at":1741891428,"status":"completed","model":"gpt-4o-mini","service_tier":"default","output":[{"id":"msg_test","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Hello!"}]}],"usage":{"input_tokens":8,"output_tokens":2,"total_tokens":10}}} + + """; + var sourceName = Guid.NewGuid().ToString(); var activities = new List(); using var listener = new ActivityListener @@ -6664,7 +6674,7 @@ public async Task OpenAIApiTypeTag_SetToResponses(bool streaming) }; ActivitySource.AddActivityListener(listener); - using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput(), Output); + using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput(), streaming ? StreamingOutput : NonStreamingOutput); using HttpClient httpClient = new(handler); using IChatClient client = new OpenAIClient(new ApiKeyCredential("apikey"), new OpenAIClientOptions { Transport = new HttpClientPipelineTransport(httpClient) }) .GetResponsesClient() @@ -6687,6 +6697,7 @@ public async Task OpenAIApiTypeTag_SetToResponses(bool streaming) var activity = Assert.Single(activities); Assert.Equal("responses", activity.GetTagItem("openai.api.type")); + Assert.Equal("default", activity.GetTagItem("openai.response.service_tier")); } [Fact] @@ -8622,4 +8633,4 @@ public async Task ToolSearchTool_NamespaceDescription_FirstNonEmptyWins_NonStrea Assert.NotNull(response); Assert.Equal("Hello!", response.Text); } -} \ No newline at end of file +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs index c3437d8067d..6ef724f7c66 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs @@ -48,6 +48,7 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData, bool OutputTokenCount = 20, TotalTokenCount = 42, CachedInputTokenCount = 5, + ReasoningTokenCount = 8, }, AdditionalProperties = new() { @@ -89,6 +90,7 @@ async static IAsyncEnumerable CallbackAsync( OutputTokenCount = 20, TotalTokenCount = 42, CachedInputTokenCount = 5, + ReasoningTokenCount = 8, })], AdditionalProperties = new() { @@ -170,6 +172,7 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal("testservice", activity.GetTagItem("gen_ai.provider.name")); Assert.Equal("replacementmodel", activity.GetTagItem("gen_ai.request.model")); + Assert.Equal(streaming ? (object?)true : null, activity.GetTagItem("gen_ai.request.stream")); Assert.Equal(3.0f, activity.GetTagItem("gen_ai.request.frequency_penalty")); Assert.Equal(4.0f, activity.GetTagItem("gen_ai.request.top_p")); Assert.Equal(5.0f, activity.GetTagItem("gen_ai.request.presence_penalty")); @@ -186,9 +189,20 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal(10, activity.GetTagItem("gen_ai.usage.input_tokens")); Assert.Equal(20, activity.GetTagItem("gen_ai.usage.output_tokens")); Assert.Equal(5, activity.GetTagItem("gen_ai.usage.cache_read.input_tokens")); + Assert.Equal(8, activity.GetTagItem("gen_ai.usage.reasoning.output_tokens")); Assert.Equal(enableSensitiveData ? "abcdefgh" : null, activity.GetTagItem("system_fingerprint")); Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("AndSomethingElse")); + if (streaming) + { + var timeToFirstChunk = Assert.IsType(activity.GetTagItem("gen_ai.response.time_to_first_chunk")); + Assert.True(timeToFirstChunk >= 0); + } + else + { + Assert.Null(activity.GetTagItem("gen_ai.response.time_to_first_chunk")); + } + Assert.True(activity.Duration.TotalMilliseconds > 0); var tags = activity.Tags.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); @@ -300,16 +314,20 @@ async static IAsyncEnumerable CallbackAsync( } }, { - "type": "web_search" + "type": "web_search", + "name": "web_search" }, { - "type": "file_search" + "type": "file_search", + "name": "file_search" }, { - "type": "code_interpreter" + "type": "code_interpreter", + "name": "code_interpreter" }, { - "type": "mcp" + "type": "mcp", + "name": "mcp" }, { "type": "function", @@ -339,47 +357,27 @@ async static IAsyncEnumerable CallbackAsync( [ { "type": "function", - "name": "GetPersonAge", - "description": "Gets the age of a person by name.", - "parameters": { - "type": "object", - "properties": { - "personName": { - "type": "string" - } - }, - "required": [ - "personName" - ] - } + "name": "GetPersonAge" }, { - "type": "web_search" + "type": "web_search", + "name": "web_search" }, { - "type": "file_search" + "type": "file_search", + "name": "file_search" }, { - "type": "code_interpreter" + "type": "code_interpreter", + "name": "code_interpreter" }, { - "type": "mcp" + "type": "mcp", + "name": "mcp" }, { "type": "function", - "name": "GetCurrentWeather", - "description": "Gets the current weather for a location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - } + "name": "GetCurrentWeather" } ] """), ReplaceWhitespace(tags["gen_ai.tool.definitions"])); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs index 6537af4d29a..063d9658bc1 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs @@ -68,6 +68,7 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa OutputTokenCount = 25, TotalTokenCount = 40, CachedInputTokenCount = 3, + ReasoningTokenCount = 6, InputAudioTokenCount = 10, InputTextTokenCount = 5, OutputAudioTokenCount = 18, @@ -119,6 +120,7 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa Assert.Equal("chat", activity.GetTagItem("gen_ai.operation.name")); Assert.Equal("test-model", activity.GetTagItem("gen_ai.request.model")); + Assert.True(activity.GetTagItem("gen_ai.request.stream") is true); Assert.Equal(500, activity.GetTagItem("gen_ai.request.max_tokens")); // Realtime-specific attributes @@ -132,11 +134,15 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa Assert.Equal(15, activity.GetTagItem("gen_ai.usage.input_tokens")); Assert.Equal(25, activity.GetTagItem("gen_ai.usage.output_tokens")); Assert.Equal(3, activity.GetTagItem("gen_ai.usage.cache_read.input_tokens")); + Assert.Equal(6, activity.GetTagItem("gen_ai.usage.reasoning.output_tokens")); Assert.Equal(10, activity.GetTagItem("gen_ai.usage.input_audio_tokens")); Assert.Equal(5, activity.GetTagItem("gen_ai.usage.input_text_tokens")); Assert.Equal(18, activity.GetTagItem("gen_ai.usage.output_audio_tokens")); Assert.Equal(7, activity.GetTagItem("gen_ai.usage.output_text_tokens")); + var timeToFirstChunk = Assert.IsType(activity.GetTagItem("gen_ai.response.time_to_first_chunk")); + Assert.True(timeToFirstChunk >= 0); + Assert.True(activity.Duration.TotalMilliseconds > 0); var tags = activity.Tags.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); @@ -175,7 +181,14 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa else { Assert.False(tags.ContainsKey("gen_ai.system_instructions")); - Assert.False(tags.ContainsKey("gen_ai.tool.definitions")); + Assert.Equal(ReplaceWhitespace(""" + [ + { + "type": "function", + "name": "Search" + } + ] + """), ReplaceWhitespace(tags["gen_ai.tool.definitions"])); } }