diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/LogsHelper.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/LogsHelper.cs index 5e2a8e5b70fd..aef0a75e2f27 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/LogsHelper.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/LogsHelper.cs @@ -49,9 +49,10 @@ internal static class LogsHelper { if (!properties.ContainsKey(scopeItem.Key)) { - var stringValue = SchemaConstants.TruncationExemptProperties.Contains(scopeItem.Key) - ? Convert.ToString(scopeItem.Value, CultureInfo.InvariantCulture)! - : Convert.ToString(scopeItem.Value, CultureInfo.InvariantCulture)?.Truncate(SchemaConstants.MessageData_Properties_MaxValueLength)!; + var maxValueLength = SchemaConstants.GenAiProperties.Contains(scopeItem.Key) + ? SchemaConstants.GenAi_Properties_MaxValueLength + : SchemaConstants.MessageData_Properties_MaxValueLength; + var stringValue = Convert.ToString(scopeItem.Value, CultureInfo.InvariantCulture)?.Truncate(maxValueLength)!; properties.Add(scopeItem.Key, stringValue); } } @@ -190,9 +191,10 @@ internal static void ProcessLogRecordProperties(LogRecord logRecord, IDictionary { if (!properties.ContainsKey(item.Key)) { - var stringValue = SchemaConstants.TruncationExemptProperties.Contains(item.Key) - ? item.Value.ToString()! - : item.Value.ToString().Truncate(SchemaConstants.MessageData_Properties_MaxValueLength)!; + var maxValueLength = SchemaConstants.GenAiProperties.Contains(item.Key) + ? SchemaConstants.GenAi_Properties_MaxValueLength + : SchemaConstants.MessageData_Properties_MaxValueLength; + var stringValue = item.Value.ToString().Truncate(maxValueLength)!; properties.Add(item.Key, stringValue); } } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/SchemaConstants.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/SchemaConstants.cs index e1ef7310b08e..c8ad186b9d25 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/SchemaConstants.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/SchemaConstants.cs @@ -129,11 +129,16 @@ internal static class SchemaConstants public const int Tags_AiInternalSdkVersion_MaxLength = 64; /// - /// GenAI semantic convention property keys that are exempt from value truncation. + /// Maximum value length for GenAI semantic convention properties (256 KB). /// These properties may carry large payloads (e.g. full prompt/completion content) - /// and must not be truncated. + /// and are truncated to a higher limit than standard properties. /// - public static readonly HashSet TruncationExemptProperties = new HashSet(StringComparer.Ordinal) + public const int GenAi_Properties_MaxValueLength = 256 * 1024; + + /// + /// GenAI semantic convention property keys that receive a higher truncation limit. + /// + public static readonly HashSet GenAiProperties = new HashSet(StringComparer.Ordinal) { "gen_ai.input.messages", "gen_ai.output.messages", diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs index f08a30eec9c3..0867e29c20f2 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TraceHelper.cs @@ -109,9 +109,10 @@ internal static void AddKvpToDictionary(IDictionary destination, { // Note: if Key exceeds MaxLength or if Value is null, the entire KVP will be dropped. // In case of duplicate keys, only the first occurence will be exported. - var stringValue = SchemaConstants.TruncationExemptProperties.Contains(keyValuePair.Key) - ? Convert.ToString(keyValuePair.Value, CultureInfo.InvariantCulture) ?? "null" - : Convert.ToString(keyValuePair.Value, CultureInfo.InvariantCulture).Truncate(SchemaConstants.KVP_MaxValueLength) ?? "null"; + var maxValueLength = SchemaConstants.GenAiProperties.Contains(keyValuePair.Key) + ? SchemaConstants.GenAi_Properties_MaxValueLength + : SchemaConstants.KVP_MaxValueLength; + var stringValue = Convert.ToString(keyValuePair.Value, CultureInfo.InvariantCulture).Truncate(maxValueLength) ?? "null"; #if NET destination.TryAdd(keyValuePair.Key, stringValue); #else @@ -130,9 +131,10 @@ internal static void AddKvpToDictionary(IDictionary destination, { // Note: if Key exceeds MaxLength or if Value is null, the entire KVP will be dropped. // In case of duplicate keys, only the first occurence will be exported. - var sanitizedValue = SchemaConstants.TruncationExemptProperties.Contains(key) - ? value - : value.Truncate(SchemaConstants.KVP_MaxValueLength) ?? "null"; + var maxValueLength = SchemaConstants.GenAiProperties.Contains(key) + ? SchemaConstants.GenAi_Properties_MaxValueLength + : SchemaConstants.KVP_MaxValueLength; + var sanitizedValue = value.Truncate(maxValueLength) ?? "null"; #if NET destination.TryAdd(key, sanitizedValue); #else diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/GenAiTruncationExemptionTests.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/GenAiTruncationTests.cs similarity index 85% rename from sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/GenAiTruncationExemptionTests.cs rename to sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/GenAiTruncationTests.cs index 49d727dab4e8..54f93348c82d 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/GenAiTruncationExemptionTests.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/GenAiTruncationTests.cs @@ -17,13 +17,16 @@ namespace Azure.Monitor.OpenTelemetry.Exporter.Tests { - public class GenAiTruncationExemptionTests + public class GenAiTruncationTests { - private const int LargePayloadLength = 64_000; + /// + /// A payload larger than the 256 KB GenAI truncation limit to verify truncation occurs. + /// + private const int LargePayloadLength = 300_000; private static readonly string s_largePayload = new string('x', LargePayloadLength); - static GenAiTruncationExemptionTests() + static GenAiTruncationTests() { Activity.DefaultIdFormat = ActivityIdFormat.W3C; Activity.ForceDefaultIdFormat = true; @@ -45,7 +48,7 @@ static GenAiTruncationExemptionTests() [InlineData("gen_ai.tool.call.arguments")] [InlineData("gen_ai.tool.call.result")] [InlineData("gen_ai.evaluation.explanation")] - public void GenAiProperties_AreNotTruncated_InAddPropertiesToTelemetry(string propertyKey) + public void GenAiProperties_AreTruncatedTo256KB_InAddPropertiesToTelemetry(string propertyKey) { // Arrange IDictionary destination = new Dictionary(); @@ -57,8 +60,7 @@ public void GenAiProperties_AreNotTruncated_InAddPropertiesToTelemetry(string pr // Assert Assert.True(destination.TryGetValue(propertyKey, out var value)); - Assert.Equal(LargePayloadLength, value!.Length); - Assert.Equal(s_largePayload, value); + Assert.Equal(SchemaConstants.GenAi_Properties_MaxValueLength, value!.Length); } [Theory] @@ -69,7 +71,7 @@ public void GenAiProperties_AreNotTruncated_InAddPropertiesToTelemetry(string pr [InlineData("gen_ai.tool.call.arguments")] [InlineData("gen_ai.tool.call.result")] [InlineData("gen_ai.evaluation.explanation")] - public void GenAiProperties_AreNotTruncated_InAddKvpToDictionary_WithStringOverload(string propertyKey) + public void GenAiProperties_AreTruncatedTo256KB_InAddKvpToDictionary_WithStringOverload(string propertyKey) { // Arrange IDictionary destination = new Dictionary(); @@ -79,8 +81,7 @@ public void GenAiProperties_AreNotTruncated_InAddKvpToDictionary_WithStringOverl // Assert Assert.True(destination.TryGetValue(propertyKey, out var value)); - Assert.Equal(LargePayloadLength, value!.Length); - Assert.Equal(s_largePayload, value); + Assert.Equal(SchemaConstants.GenAi_Properties_MaxValueLength, value!.Length); } [Fact] @@ -121,7 +122,7 @@ public void NonGenAiProperties_AreTruncated_InAddKvpToDictionary_WithStringOverl [InlineData("gen_ai.tool.call.arguments")] [InlineData("gen_ai.tool.call.result")] [InlineData("gen_ai.evaluation.explanation")] - public void GenAiProperties_AreNotTruncated_InLogRecordAttributes(string propertyKey) + public void GenAiProperties_AreTruncatedTo256KB_InLogRecordAttributes(string propertyKey) { // Arrange var logRecords = new List(); @@ -132,10 +133,10 @@ public void GenAiProperties_AreNotTruncated_InLogRecordAttributes(string propert options.IncludeFormattedMessage = true; options.AddInMemoryExporter(logRecords); }); - builder.AddFilter(typeof(GenAiTruncationExemptionTests).FullName, LogLevel.Trace); + builder.AddFilter(typeof(GenAiTruncationTests).FullName, LogLevel.Trace); }); - var logger = loggerFactory.CreateLogger(); + var logger = loggerFactory.CreateLogger(); // Use LoggerMessage.Define pattern to add structured attributes var state = new List> @@ -151,8 +152,7 @@ public void GenAiProperties_AreNotTruncated_InLogRecordAttributes(string propert // Assert Assert.True(properties.TryGetValue(propertyKey, out var value), $"Property '{propertyKey}' should exist in the properties dictionary."); - Assert.Equal(LargePayloadLength, value!.Length); - Assert.Equal(s_largePayload, value); + Assert.Equal(SchemaConstants.GenAi_Properties_MaxValueLength, value!.Length); } [Fact] @@ -167,10 +167,10 @@ public void NonGenAiProperties_AreTruncated_InLogRecordAttributes() options.IncludeFormattedMessage = true; options.AddInMemoryExporter(logRecords); }); - builder.AddFilter(typeof(GenAiTruncationExemptionTests).FullName, LogLevel.Trace); + builder.AddFilter(typeof(GenAiTruncationTests).FullName, LogLevel.Trace); }); - var logger = loggerFactory.CreateLogger(); + var logger = loggerFactory.CreateLogger(); var state = new List> { @@ -196,10 +196,10 @@ public void NonGenAiProperties_AreTruncated_InLogRecordAttributes() [InlineData("gen_ai.tool.call.arguments")] [InlineData("gen_ai.tool.call.result")] [InlineData("gen_ai.evaluation.explanation")] - public void GenAiProperties_AreNotTruncated_InActivityCustomDimensions(string propertyKey) + public void GenAiProperties_AreTruncatedTo256KB_InActivityCustomDimensions(string propertyKey) { // Arrange - using var activitySource = new ActivitySource(nameof(GenAiTruncationExemptionTests)); + using var activitySource = new ActivitySource(nameof(GenAiTruncationTests)); using var activity = activitySource.StartActivity( "TestActivity", ActivityKind.Client, @@ -214,8 +214,7 @@ public void GenAiProperties_AreNotTruncated_InActivityCustomDimensions(string pr // Assert Assert.True(remoteDependencyData.Properties.TryGetValue(propertyKey, out var value), $"Property '{propertyKey}' should exist in dependency custom dimensions."); - Assert.Equal(LargePayloadLength, value!.Length); - Assert.Equal(s_largePayload, value); + Assert.Equal(SchemaConstants.GenAi_Properties_MaxValueLength, value!.Length); } [Theory] @@ -226,10 +225,10 @@ public void GenAiProperties_AreNotTruncated_InActivityCustomDimensions(string pr [InlineData("gen_ai.tool.call.arguments")] [InlineData("gen_ai.tool.call.result")] [InlineData("gen_ai.evaluation.explanation")] - public void GenAiProperties_AreNotTruncated_InRequestDataCustomDimensions(string propertyKey) + public void GenAiProperties_AreTruncatedTo256KB_InRequestDataCustomDimensions(string propertyKey) { // Arrange - using var activitySource = new ActivitySource(nameof(GenAiTruncationExemptionTests)); + using var activitySource = new ActivitySource(nameof(GenAiTruncationTests)); using var activity = activitySource.StartActivity( "TestActivity", ActivityKind.Server, @@ -244,8 +243,7 @@ public void GenAiProperties_AreNotTruncated_InRequestDataCustomDimensions(string // Assert Assert.True(requestData.Properties.TryGetValue(propertyKey, out var value), $"Property '{propertyKey}' should exist in request custom dimensions."); - Assert.Equal(LargePayloadLength, value!.Length); - Assert.Equal(s_largePayload, value); + Assert.Equal(SchemaConstants.GenAi_Properties_MaxValueLength, value!.Length); } } }