From 1df34077bafe0e49a22cb09af96d5a7c817a94ff Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 2 Jul 2024 14:08:47 -0700 Subject: [PATCH 01/11] Add instrumentation for convenience chat calls without stremaing --- src/Custom/Chat/ChatClient.cs | 45 ++- .../Common/Instrumentation/Constants.cs | 30 ++ .../Instrumentation/InstrumentationFactory.cs | 25 ++ .../Instrumentation/InstrumentationScope.cs | 218 +++++++++++++ src/OpenAI.csproj | 10 +- tests/Chat/ChatSmokeTests.cs | 58 ++++ .../ChatInstrumentationTests.cs | 291 ++++++++++++++++++ tests/Instrumentation/TestActivityListener.cs | 72 +++++ tests/Instrumentation/TestMeterListener.cs | 85 +++++ tests/OpenAI.Tests.csproj | 6 +- 10 files changed, 825 insertions(+), 15 deletions(-) create mode 100644 src/Custom/Common/Instrumentation/Constants.cs create mode 100644 src/Custom/Common/Instrumentation/InstrumentationFactory.cs create mode 100644 src/Custom/Common/Instrumentation/InstrumentationScope.cs create mode 100644 tests/Instrumentation/ChatInstrumentationTests.cs create mode 100644 tests/Instrumentation/TestActivityListener.cs create mode 100644 tests/Instrumentation/TestMeterListener.cs diff --git a/src/Custom/Chat/ChatClient.cs b/src/Custom/Chat/ChatClient.cs index 54c31bedb..9ecfb7d77 100644 --- a/src/Custom/Chat/ChatClient.cs +++ b/src/Custom/Chat/ChatClient.cs @@ -1,3 +1,4 @@ +using OpenAI.Custom.Common.Instrumentation; using System; using System.ClientModel; using System.ClientModel.Primitives; @@ -14,6 +15,7 @@ namespace OpenAI.Chat; public partial class ChatClient { private readonly string _model; + private readonly InstrumentationFactory _instrumentation; /// /// Initializes a new instance of that will use an API key when authenticating. @@ -62,6 +64,7 @@ protected internal ChatClient(ClientPipeline pipeline, string model, Uri endpoin _model = model; _pipeline = pipeline; _endpoint = endpoint; + _instrumentation = new InstrumentationFactory(model, endpoint); } /// @@ -77,11 +80,22 @@ public virtual async Task> CompleteChatAsync(IEnume options ??= new(); CreateChatCompletionOptions(messages, ref options); - - using BinaryContent content = options.ToBinaryContent(); - - ClientResult result = await CompleteChatAsync(content, cancellationToken.ToRequestOptions()).ConfigureAwait(false); - return ClientResult.FromValue(ChatCompletion.FromResponse(result.GetRawResponse()), result.GetRawResponse()); + using InstrumentationScope scope = _instrumentation.StartChatScope(options); + + try + { + using BinaryContent content = options.ToBinaryContent(); + + ClientResult result = await CompleteChatAsync(content, cancellationToken.ToRequestOptions()).ConfigureAwait(false); + ChatCompletion chatCompletion = ChatCompletion.FromResponse(result.GetRawResponse()); + scope?.RecordChatCompletion(chatCompletion); + return ClientResult.FromValue(chatCompletion, result.GetRawResponse()); + } + catch (Exception ex) + { + scope?.RecordException(ex); + throw; + } } /// @@ -105,11 +119,22 @@ public virtual ClientResult CompleteChat(IEnumerable diff --git a/src/Custom/Common/Instrumentation/Constants.cs b/src/Custom/Common/Instrumentation/Constants.cs new file mode 100644 index 000000000..98bf86923 --- /dev/null +++ b/src/Custom/Common/Instrumentation/Constants.cs @@ -0,0 +1,30 @@ +namespace OpenAI.Custom.Common.Instrumentation; + +internal class Constants +{ + public const string ErrorTypeKey = "error.type"; + public const string ServerAddressKey = "server.address"; + public const string ServerPortKey = "server.port"; + + public const string GenAiClientOperationDurationMetricName = "gen_ai.client.operation.duration"; + public const string GenAiClientTokenUsageMetricName = "gen_ai.client.token.usage"; + + public const string GenAiOperationNameKey = "gen_ai.operation.name"; + + public const string GenAiRequestMaxTokensKey = "gen_ai.request.max_tokens"; + public const string GenAiRequestModelKey = "gen_ai.request.model"; + public const string GenAiRequestTemperatureKey = "gen_ai.request.temperature"; + public const string GenAiRequestTopPKey = "gen_ai.request.top_p"; + + public const string GenAiResponseIdKey = "gen_ai.response.id"; + public const string GenAiResponseFinishReasonKey = "gen_ai.response.finish_reason"; + public const string GenAiResponseModelKey = "gen_ai.response.model"; + + public const string GenAiSystemKey = "gen_ai.system"; + public const string GenAiSystemValue = "openai"; + + public const string GenAiTokenTypeKey = "gen_ai.token_type"; + + public const string GenAiUsageInputTokensKey = "gen_ai.usage.input_tokens"; + public const string GenAiUsageOutputTokensKey = "gen_ai.usage.output_tokens"; +} diff --git a/src/Custom/Common/Instrumentation/InstrumentationFactory.cs b/src/Custom/Common/Instrumentation/InstrumentationFactory.cs new file mode 100644 index 000000000..12743d3d0 --- /dev/null +++ b/src/Custom/Common/Instrumentation/InstrumentationFactory.cs @@ -0,0 +1,25 @@ +using OpenAI.Chat; +using System; + +namespace OpenAI.Custom.Common.Instrumentation; + +internal class InstrumentationFactory +{ + private readonly string _serverAddress; + private readonly int _serverPort; + private readonly string _model; + + private const string ChatOperationName = "chat"; + + public InstrumentationFactory(string model, Uri endpoint) + { + _serverAddress = endpoint.Host; + _serverPort = endpoint.Port; + _model = model; + } + + public InstrumentationScope StartChatScope(ChatCompletionOptions completionsOptions) + { + return InstrumentationScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions); + } +} \ No newline at end of file diff --git a/src/Custom/Common/Instrumentation/InstrumentationScope.cs b/src/Custom/Common/Instrumentation/InstrumentationScope.cs new file mode 100644 index 000000000..5d0470f8a --- /dev/null +++ b/src/Custom/Common/Instrumentation/InstrumentationScope.cs @@ -0,0 +1,218 @@ +using OpenAI.Chat; +using System; +using System.ClientModel; +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace OpenAI.Custom.Common.Instrumentation; + +internal class InstrumentationScope : IDisposable +{ + private static readonly ActivitySource s_chatSource = new ActivitySource("OpenAI.ChatClient"); + private static readonly Meter s_chatMeter = new Meter("OpenAI.ChatClient"); + private static readonly Histogram s_duration = s_chatMeter.CreateHistogram(Constants.GenAiClientOperationDurationMetricName, "s", "Measures GenAI operation duration."); + private static readonly Histogram s_tokens = s_chatMeter.CreateHistogram(Constants.GenAiClientTokenUsageMetricName, "{token}", "Measures the number of input and output token used."); + + private readonly string _operationName; + private readonly string _serverAddress; + private readonly int _serverPort; + private readonly string _requestModel; + + private Stopwatch _duration; + private Activity _activity; + private TagList _commonTags; + + private InstrumentationScope( + string model, string operationName, + string serverAddress, int serverPort) + { + _requestModel = model; + _operationName = operationName; + _serverAddress = serverAddress; + _serverPort = serverPort; + } + + public static InstrumentationScope StartChat(string model, string operationName, + string serverAddress, int serverPort, ChatCompletionOptions options) + { + if (s_chatSource.HasListeners() || s_tokens.Enabled || s_duration.Enabled) + { + var scope = new InstrumentationScope(model, operationName, serverAddress, serverPort); + scope.StartChat(options); + return scope; + } + + return null; + } + + private void StartChat(ChatCompletionOptions options) + { + _duration = Stopwatch.StartNew(); + _commonTags = new TagList + { + { Constants.GenAiSystemKey, Constants.GenAiSystemValue }, + { Constants.GenAiRequestModelKey, _requestModel }, + { Constants.ServerAddressKey, _serverAddress }, + { Constants.ServerPortKey, _serverPort }, + { Constants.GenAiOperationNameKey, _operationName }, + }; + + _activity = s_chatSource.StartActivity(string.Concat(_operationName, " ", _requestModel), ActivityKind.Client); + if (_activity?.IsAllDataRequested == true) + { + RecordCommonAttributes(); + SetActivityTagIfNotNull(Constants.GenAiRequestMaxTokensKey, options?.MaxTokens); + SetActivityTagIfNotNull(Constants.GenAiRequestTemperatureKey, options?.Temperature); + SetActivityTagIfNotNull(Constants.GenAiRequestTopPKey, options?.TopP); + } + + return; + } + + public void RecordChatCompletion(ChatCompletion completion) + { + RecordMetrics(completion.Model, null, completion.Usage?.InputTokens, completion.Usage?.OutputTokens); + + if (_activity?.IsAllDataRequested == true) + { + RecordResponseAttributes(completion.Id, completion.Model, completion.FinishReason, completion.Usage); + } + } + + public void RecordException(Exception ex) + { + var errorType = GetErrorType(ex); + RecordMetrics(null, errorType, null, null); + SetActivityError(ex, errorType); + } + + public void Dispose() + { + _activity?.Stop(); + } + + private void RecordCommonAttributes() + { + _activity.SetTag(Constants.GenAiSystemKey, Constants.GenAiSystemValue); + _activity.SetTag(Constants.GenAiRequestModelKey, _requestModel); + _activity.SetTag(Constants.ServerAddressKey, _serverAddress); + _activity.SetTag(Constants.ServerPortKey, _serverPort); + _activity.SetTag(Constants.GenAiOperationNameKey, _operationName); + } + + private void RecordMetrics(string responseModel, string errorType, int? inputTokensUsage, int? outputTokensUsage) + { + TagList tags = ResponseTagsWithError(responseModel, errorType); + s_duration.Record(_duration.Elapsed.TotalSeconds, tags); + + if (inputTokensUsage != null) + { + // tags is a struct, let's copy them + TagList inputUsageTags = tags; + inputUsageTags.Add(Constants.GenAiTokenTypeKey, "input"); + s_tokens.Record(inputTokensUsage.Value, inputUsageTags); + } + + if (outputTokensUsage != null) + { + TagList outputUsageTags = tags; + outputUsageTags.Add(Constants.GenAiTokenTypeKey, "output"); + + s_tokens.Record(outputTokensUsage.Value, outputUsageTags); + } + } + + private TagList ResponseTagsWithError(string responseModel, string errorType) + { + // tags is a struct, let's copy them + var tags = _commonTags; + + if (responseModel != null) + { + tags.Add(Constants.GenAiResponseModelKey, responseModel); + } + + if (errorType != null) + { + tags.Add(Constants.ErrorTypeKey, errorType); + } + + return tags; + } + + private void RecordResponseAttributes(string responseId, string model, ChatFinishReason? finishReason, ChatTokenUsage usage) + { + SetActivityTagIfNotNull(Constants.GenAiResponseIdKey, responseId); + SetActivityTagIfNotNull(Constants.GenAiResponseModelKey, model); + SetActivityTagIfNotNull(Constants.GenAiResponseFinishReasonKey, GetFinishReason(finishReason)); + SetActivityTagIfNotNull(Constants.GenAiUsageInputTokensKey, usage?.InputTokens); + SetActivityTagIfNotNull(Constants.GenAiUsageOutputTokensKey, usage?.OutputTokens); + } + + private string GetFinishReason(ChatFinishReason? reason) => + reason switch + { + ChatFinishReason.ContentFilter => "content_filter", + ChatFinishReason.FunctionCall => "function_call", + ChatFinishReason.Length => "length", + ChatFinishReason.Stop => "stop", + ChatFinishReason.ToolCalls => "tool_calls", + _ => reason?.ToString(), + }; + + private string GetChatMessageRole(ChatMessageRole? role) => + role switch + { + ChatMessageRole.Assistant => "assistant", + ChatMessageRole.Function => "function", + ChatMessageRole.System => "system", + ChatMessageRole.Tool => "tool", + ChatMessageRole.User => "user", + _ => role?.ToString(), + }; + + private string GetErrorType(Exception exception) + { + if (exception is ClientResultException requestFailedException) + { + // TODO (limolkova) when we start targeting .NET 8 we should put + // requestFailedException.InnerException.HttpRequestError into error.type + return requestFailedException.Status.ToString(); + } + + return exception?.GetType()?.FullName; + } + + private void SetActivityError(Exception exception, string errorType) + { + if (exception != null || errorType != null) + { + _activity?.SetTag(Constants.ErrorTypeKey, errorType); + _activity?.SetStatus(ActivityStatusCode.Error, exception?.Message ?? errorType); + } + } + + private void SetActivityTagIfNotNull(string name, object value) + { + if (value != null) + { + _activity.SetTag(name, value); + } + } + + private void SetActivityTagIfNotNull(string name, int? value) + { + if (value.HasValue) + { + _activity.SetTag(name, value.Value); + } + } + + private void SetActivityTagIfNotNull(string name, float? value) + { + if (value.HasValue) + { + _activity.SetTag(name, value.Value); + } + } +} diff --git a/src/OpenAI.csproj b/src/OpenAI.csproj index eed204d55..8c02b1027 100644 --- a/src/OpenAI.csproj +++ b/src/OpenAI.csproj @@ -12,7 +12,7 @@ true - + true OpenAI.png @@ -21,7 +21,7 @@ true snupkg - + true @@ -29,7 +29,7 @@ $(NoWarn),1570,1573,1574,1591 - + $(NoWarn),0618 @@ -63,7 +63,7 @@ true - + @@ -73,5 +73,7 @@ + + diff --git a/tests/Chat/ChatSmokeTests.cs b/tests/Chat/ChatSmokeTests.cs index a3849d3a6..4940ca8f6 100644 --- a/tests/Chat/ChatSmokeTests.cs +++ b/tests/Chat/ChatSmokeTests.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using NUnit.Framework; using OpenAI.Chat; +using OpenAI.Tests.Instrumentation; using OpenAI.Tests.Utility; using System; using System.ClientModel; @@ -9,6 +10,8 @@ using System.Net; using System.Text.Json; using System.Threading.Tasks; +using static OpenAI.Tests.Instrumentation.TestMeterListener; +using static OpenAI.Tests.TestHelpers; namespace OpenAI.Tests.Chat; @@ -470,4 +473,59 @@ public void SerializeChatMessageContentPartAsImageBytes(bool fromRawJson) Assert.That(additionalPropertyProperty.ValueKind, Is.EqualTo(JsonValueKind.True)); } } + + [Test] + public async Task JsonResult() + { + ChatClient client = GetTestClient(TestScenario.Chat); + IEnumerable messages = [ + new UserChatMessage("Give me a JSON object with the following properties: red, green, and blue. The value " + + "of each property should be a string containing their RGB representation in hexadecimal.") + ]; + ChatCompletionOptions options = new() { ResponseFormat = ChatResponseFormat.JsonObject }; + ClientResult result = IsAsync + ? await client.CompleteChatAsync(messages, options) + : client.CompleteChat(messages, options); + + JsonDocument jsonDocument = JsonDocument.Parse(result.Value.Content[0].Text); + + Assert.That(jsonDocument.RootElement.TryGetProperty("red", out JsonElement redProperty)); + Assert.That(jsonDocument.RootElement.TryGetProperty("green", out JsonElement greenProperty)); + Assert.That(jsonDocument.RootElement.TryGetProperty("blue", out JsonElement blueProperty)); + Assert.That(redProperty.GetString().ToLowerInvariant(), Contains.Substring("ff0000")); + Assert.That(greenProperty.GetString().ToLowerInvariant(), Contains.Substring("00ff00")); + Assert.That(blueProperty.GetString().ToLowerInvariant(), Contains.Substring("0000ff")); + } + + [Test] + public async Task HelloWorldChatWithTracingAndMetrics() + { + using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient"); + using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient"); + + ChatClient client = GetTestClient(TestScenario.Chat); + IEnumerable messages = [new UserChatMessage("Hello, world!")]; + ClientResult result = IsAsync + ? await client.CompleteChatAsync(messages) + : client.CompleteChat(messages); + + Assert.AreEqual(1, activityListener.Activities.Count); + TestActivityListener.ValidateChatActivity(activityListener.Activities.Single(), result.Value); + + List durations = meterListener.GetMeasurements("gen_ai.client.operation.duration"); + Assert.AreEqual(1, durations.Count); + ValidateChatMetricTags(durations.Single(), result.Value); + + List usages = meterListener.GetMeasurements("gen_ai.client.token.usage"); + Assert.AreEqual(2, usages.Count); + + Assert.True(usages[0].tags.TryGetValue("gen_ai.token.type", out var type)); + Assert.IsInstanceOf(type); + + TestMeasurement input = (type is "input") ? usages[0] : usages[1]; + TestMeasurement output = (type is "input") ? usages[1] : usages[0]; + + Assert.AreEqual(result.Value.Usage.InputTokens, input.value); + Assert.AreEqual(result.Value.Usage.OutputTokens, output.value); + } } diff --git a/tests/Instrumentation/ChatInstrumentationTests.cs b/tests/Instrumentation/ChatInstrumentationTests.cs new file mode 100644 index 000000000..6535acfde --- /dev/null +++ b/tests/Instrumentation/ChatInstrumentationTests.cs @@ -0,0 +1,291 @@ +using NUnit.Framework; +using OpenAI.Chat; +using OpenAI.Custom.Common.Instrumentation; +using System; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Net.Sockets; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using static OpenAI.Tests.Instrumentation.TestMeterListener; +using static OpenAI.Tests.Instrumentation.TestActivityListener; + +namespace OpenAI.Tests.Instrumentation; + +[TestFixture] +[NonParallelizable] +public class ChatInstrumentationTests +{ + private const string RequestModel = "requestModel"; + private const string Host = "host"; + private const int Port = 42; + private static readonly string Endpoint = $"https://{Host}:{Port}/path"; + private const string CompletionId = "chatcmpl-9fG9OILMJnKZARXDwxoCnLcvDsDDX"; + private const string CompletionContent = "hello world"; + private const string ResponseModel = "responseModel"; + private const string FinishReason = "stop"; + private const int PromptTokens = 2; + private const int CompletionTokens = 42; + + private readonly InstrumentationFactory _instrumentationFactory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + + [Test] + public void AllTelemetryOff() + { + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + Assert.IsNull(factory.StartChatScope(new ChatCompletionOptions())); + Assert.IsNull(Activity.Current); + } + + [Test] + public void MetricsOnTracingOff() + { + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + + var elapsedMax = Stopwatch.StartNew(); + using var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions()); + var elapsedMin = Stopwatch.StartNew(); + + Assert.Null(Activity.Current); + Assert.NotNull(scope); + + // so we have some duration to measure + Thread.Sleep(20); + + elapsedMin.Stop(); + + ChatCompletion response = CreateChatCompletion(); + scope.RecordChatCompletion(response); + scope.Dispose(); + + ValidateDuration(meterListener, response, elapsedMin.Elapsed, elapsedMax.Elapsed); + ValidateUsage(meterListener, response, PromptTokens, CompletionTokens); + } + + [Test] + public void MetricsOnTracingOffException() + { + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + + using (var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions())) + { + scope.RecordException(new TaskCanceledException()); + } + + ValidateDuration(meterListener, null, TimeSpan.MinValue, TimeSpan.MaxValue); + Assert.IsNull(meterListener.GetMeasurements("gen_ai.client.token.usage")); + } + + [Test] + public void TracingOnMetricsOff() + { + using var listener = new TestActivityListener("OpenAI.ChatClient"); + + var chatCompletion = CreateChatCompletion(); + + Activity activity = null; + using (var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions())) + { + activity = Activity.Current; + Assert.IsNull(activity.GetTagItem("gen_ai.request.temperature")); + Assert.IsNull(activity.GetTagItem("gen_ai.request.top_p")); + Assert.IsNull(activity.GetTagItem("gen_ai.request.max_tokens")); + + Assert.NotNull(scope); + + scope.RecordChatCompletion(chatCompletion); + } + + Assert.Null(Activity.Current); + Assert.AreEqual(1, listener.Activities.Count); + + ValidateChatActivity(listener.Activities.Single(), chatCompletion, RequestModel, Host, Port); + } + + [Test] + public void ChatTracingAllAttributes() + { + using var listener = new TestActivityListener("OpenAI.ChatClient"); + var options = new ChatCompletionOptions() + { + Temperature = 0.42f, + MaxTokens = 200, + TopP = 0.9f + }; + SetMessages(options, new UserChatMessage("hello")); + + var chatCompletion = CreateChatCompletion(); + + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + using (var scope = factory.StartChatScope(options)) + { + Assert.AreEqual(options.Temperature.Value, (float)Activity.Current.GetTagItem("gen_ai.request.temperature"), 0.01); + Assert.AreEqual(options.TopP.Value, (float)Activity.Current.GetTagItem("gen_ai.request.top_p"), 0.01); + Assert.AreEqual(options.MaxTokens.Value, Activity.Current.GetTagItem("gen_ai.request.max_tokens")); + scope.RecordChatCompletion(chatCompletion); + } + Assert.Null(Activity.Current); + + ValidateChatActivity(listener.Activities.Single(), chatCompletion, RequestModel, Host, Port); + } + + [Test] + public void ChatTracingException() + { + using var listener = new TestActivityListener("OpenAI.ChatClient"); + + var error = new SocketException(42, "test error"); + using (var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions())) + { + scope.RecordException(error); + } + + Assert.Null(Activity.Current); + + ValidateChatActivity(listener.Activities.Single(), error, RequestModel, Host, Port); + } + + [Test] + public async Task ChatTracingAndMetricsMultiple() + { + using var activityListener = new TestActivityListener("OpenAI.ChatClient"); + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + + var options = new ChatCompletionOptions(); + SetMessages(options, new UserChatMessage("hello")); + + var tasks = new Task[5]; + int numberOfSuccessfulResponses = 3; + int totalPromptTokens = 0, totalCompletionTokens = 0; + for (int i = 0; i < tasks.Length; i ++) + { + int t = i; + // don't let Activity.Current escape the scope + tasks[i] = Task.Run(async () => + { + using var scope = _instrumentationFactory.StartChatScope(options); + await Task.Delay(10); + if (t < numberOfSuccessfulResponses) + { + var promptTokens = Random.Shared.Next(100); + var completionTokens = Random.Shared.Next(100); + + var completion = CreateChatCompletion(promptTokens, completionTokens); + totalPromptTokens += promptTokens; + totalCompletionTokens += completionTokens; + scope.RecordChatCompletion(completion); + } + else + { + scope.RecordException(new TaskCanceledException()); + } + }); + } + + await Task.WhenAll(tasks); + + Assert.AreEqual(tasks.Length, activityListener.Activities.Count); + + var durations = meterListener.GetMeasurements("gen_ai.client.operation.duration"); + Assert.AreEqual(tasks.Length, durations.Count); + Assert.AreEqual(numberOfSuccessfulResponses, durations.Count(d => !d.tags.ContainsKey("error.type"))); + + var usages = meterListener.GetMeasurements("gen_ai.client.token.usage"); + // we don't report usage if there was no response + Assert.AreEqual(numberOfSuccessfulResponses * 2, usages.Count); + Assert.IsEmpty(usages.Where(u => u.tags.ContainsKey("error.type"))); + + Assert.AreEqual(totalPromptTokens, usages + .Where(u => u.tags.Contains(new KeyValuePair("gen_ai.token.type", "input"))) + .Sum(u => (long)u.value)); + Assert.AreEqual(totalCompletionTokens, usages + .Where(u => u.tags.Contains(new KeyValuePair("gen_ai.token.type", "output"))) + .Sum(u => (long)u.value)); + } + + private void SetMessages(ChatCompletionOptions options, params ChatMessage[] messages) + { + var messagesProperty = typeof(ChatCompletionOptions).GetProperty("Messages", BindingFlags.Instance | BindingFlags.NonPublic); + messagesProperty.SetValue(options, messages.ToList()); + } + + private void ValidateDuration(TestMeterListener listener, ChatCompletion response, TimeSpan durationMin, TimeSpan durationMax) + { + var duration = listener.GetInstrument("gen_ai.client.operation.duration"); + Assert.IsNotNull(duration); + Assert.IsInstanceOf>(duration); + + var measurements = listener.GetMeasurements("gen_ai.client.operation.duration"); + Assert.IsNotNull(measurements); + Assert.AreEqual(1, measurements.Count); + + var measurement = measurements[0]; + Assert.IsInstanceOf(measurement.value); + Assert.GreaterOrEqual((double)measurement.value, durationMin.TotalSeconds); + Assert.LessOrEqual((double)measurement.value, durationMax.TotalSeconds); + + ValidateChatMetricTags(measurement, response, RequestModel, Host, Port); + } + + private void ValidateUsage(TestMeterListener listener, ChatCompletion response, int inputTokens, int outputTokens) + { + var usage = listener.GetInstrument("gen_ai.client.token.usage"); + Assert.IsNotNull(usage); + Assert.IsInstanceOf>(usage); + + var measurements = listener.GetMeasurements("gen_ai.client.token.usage"); + Assert.IsNotNull(measurements); + Assert.AreEqual(2, measurements.Count); + + foreach (var measurement in measurements) + { + Assert.IsInstanceOf(measurement.value); + ValidateChatMetricTags(measurement, response, RequestModel, Host, Port); + } + + Assert.True(measurements[0].tags.TryGetValue("gen_ai.token.type", out var type)); + Assert.IsInstanceOf(type); + + TestMeasurement input = (type is "input") ? measurements[0] : measurements[1]; + TestMeasurement output = (type is "input") ? measurements[1] : measurements[0]; + + Assert.AreEqual(inputTokens, input.value); + Assert.AreEqual(outputTokens, output.value); + } + + private static ChatCompletion CreateChatCompletion(int promptTokens = PromptTokens, int completionTokens = CompletionTokens) + { + var completion = BinaryData.FromString( + $$""" + { + "id": "{{CompletionId}}", + "created": 1719621282, + "choices": [ + { + "message": { + "role": "assistant", + "content": "{{CompletionContent}}" + }, + "logprobs": null, + "index": 0, + "finish_reason": "{{FinishReason}}" + } + ], + "model": "{{ResponseModel}}", + "system_fingerprint": "fp_7ec89fabc6", + "usage": { + "completion_tokens": {{completionTokens}}, + "prompt_tokens": {{promptTokens}}, + "total_tokens": 42 + } + } + """); + + return ModelReaderWriter.Read(completion); + } + +} diff --git a/tests/Instrumentation/TestActivityListener.cs b/tests/Instrumentation/TestActivityListener.cs new file mode 100644 index 000000000..208526500 --- /dev/null +++ b/tests/Instrumentation/TestActivityListener.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using NUnit.Framework; +using OpenAI.Chat; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace OpenAI.Tests.Instrumentation; + +internal class TestActivityListener : IDisposable +{ + private readonly ActivityListener _listener; + private readonly ConcurrentQueue stoppedActivities = new ConcurrentQueue(); + + public TestActivityListener(string sourceName) + { + _listener = new ActivityListener() + { + ActivityStopped = stoppedActivities.Enqueue, + ShouldListenTo = s => s.Name == sourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + }; + + ActivitySource.AddActivityListener(_listener); + } + + public List Activities => stoppedActivities.ToList(); + + public void Dispose() + { + _listener.Dispose(); + } + + public static void ValidateChatActivity(Activity activity, ChatCompletion response, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + Assert.NotNull(activity); + Assert.AreEqual($"chat {requestModel}", activity.DisplayName); + Assert.AreEqual("chat", activity.GetTagItem("gen_ai.operation.name")); + Assert.AreEqual("openai", activity.GetTagItem("gen_ai.system")); + Assert.AreEqual(requestModel, activity.GetTagItem("gen_ai.request.model")); + + Assert.AreEqual(host, activity.GetTagItem("server.address")); + Assert.AreEqual(port, activity.GetTagItem("server.port")); + + if (response != null) + { + Assert.AreEqual(response.Model, activity.GetTagItem("gen_ai.response.model")); + Assert.AreEqual(response.Id, activity.GetTagItem("gen_ai.response.id")); + Assert.AreEqual(response.FinishReason.ToString().ToLower(), activity.GetTagItem("gen_ai.response.finish_reason")); + Assert.AreEqual(response.Usage.OutputTokens, activity.GetTagItem("gen_ai.usage.output_tokens")); + Assert.AreEqual(response.Usage.InputTokens, activity.GetTagItem("gen_ai.usage.input_tokens")); + Assert.AreEqual(ActivityStatusCode.Unset, activity.Status); + Assert.Null(activity.StatusDescription); + Assert.Null(activity.GetTagItem("error.type")); + } + else + { + Assert.AreEqual(ActivityStatusCode.Error, activity.Status); + Assert.NotNull(activity.GetTagItem("error.type")); + } + } + + public static void ValidateChatActivity(Activity activity, Exception ex, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + ValidateChatActivity(activity, (ChatCompletion)null, requestModel, host, port); + Assert.AreEqual(ex.GetType().FullName, activity.GetTagItem("error.type")); + } +} diff --git a/tests/Instrumentation/TestMeterListener.cs b/tests/Instrumentation/TestMeterListener.cs new file mode 100644 index 000000000..2bbd9c7e3 --- /dev/null +++ b/tests/Instrumentation/TestMeterListener.cs @@ -0,0 +1,85 @@ +using NUnit.Framework; +using OpenAI.Chat; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace OpenAI.Tests.Instrumentation; + +internal class TestMeterListener : IDisposable +{ + public record TestMeasurement(object value, Dictionary tags); + + private readonly ConcurrentDictionary> _measurements = new (); + private readonly ConcurrentDictionary _instruments = new (); + private readonly MeterListener _listener; + public TestMeterListener(string meterName) + { + _listener = new MeterListener(); + _listener.InstrumentPublished = (i, l) => + { + if (i.Meter.Name == meterName) + { + l.EnableMeasurementEvents(i); + } + }; + _listener.SetMeasurementEventCallback(OnMeasurementRecorded); + _listener.SetMeasurementEventCallback(OnMeasurementRecorded); + _listener.Start(); + } + + public List GetMeasurements(string instrumentName) + { + _measurements.TryGetValue(instrumentName, out var list); + return list; + } + + public Instrument GetInstrument(string instrumentName) + { + _instruments.TryGetValue(instrumentName, out var instrument); + return instrument; + } + + private void OnMeasurementRecorded(Instrument instrument, T measurement, ReadOnlySpan> tags, object state) + { + _instruments.TryAdd(instrument.Name, instrument); + + var testMeasurement = new TestMeasurement(measurement, new Dictionary(tags.ToArray())); + _measurements.AddOrUpdate(instrument.Name, + k => new() { testMeasurement }, + (k, l) => + { + l.Add(testMeasurement); + return l; + }); + } + + public void Dispose() + { + _listener.Dispose(); + } + + public static void ValidateChatMetricTags(TestMeasurement measurement, ChatCompletion response, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + Assert.AreEqual("openai", measurement.tags["gen_ai.system"]); + Assert.AreEqual("chat", measurement.tags["gen_ai.operation.name"]); + Assert.AreEqual(host, measurement.tags["server.address"]); + Assert.AreEqual(requestModel, measurement.tags["gen_ai.request.model"]); + Assert.AreEqual(port, measurement.tags["server.port"]); + + if (response != null) + { + Assert.AreEqual(response.Model, measurement.tags["gen_ai.response.model"]); + Assert.False(measurement.tags.ContainsKey("error.type")); + } + } + + public static void ValidateChatMetricTags(TestMeasurement measurement, Exception ex, string requestModel = "gpt-3.5-turbo", string host = "api.openai.com", int port = 443) + { + ValidateChatMetricTags(measurement, (ChatCompletion)null, requestModel, host, port); + Assert.True(measurement.tags.ContainsKey("error.type")); + Assert.AreEqual(ex.GetType().FullName, measurement.tags["error.type"]); + } +} diff --git a/tests/OpenAI.Tests.csproj b/tests/OpenAI.Tests.csproj index b33b036ea..9fc6818fe 100644 --- a/tests/OpenAI.Tests.csproj +++ b/tests/OpenAI.Tests.csproj @@ -1,4 +1,4 @@ - + net8.0 @@ -15,4 +15,8 @@ + + + + \ No newline at end of file From 333c1f8539897fcb679f447902fb62199b470de6 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 2 Jul 2024 14:41:06 -0700 Subject: [PATCH 02/11] nits --- src/Custom/Common/Instrumentation/Constants.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Custom/Common/Instrumentation/Constants.cs b/src/Custom/Common/Instrumentation/Constants.cs index 98bf86923..f71d3adc3 100644 --- a/src/Custom/Common/Instrumentation/Constants.cs +++ b/src/Custom/Common/Instrumentation/Constants.cs @@ -2,6 +2,9 @@ internal class Constants { + // follows OpenTelemetry GenAI semantic conventions: + // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai + public const string ErrorTypeKey = "error.type"; public const string ServerAddressKey = "server.address"; public const string ServerPortKey = "server.port"; @@ -23,7 +26,7 @@ internal class Constants public const string GenAiSystemKey = "gen_ai.system"; public const string GenAiSystemValue = "openai"; - public const string GenAiTokenTypeKey = "gen_ai.token_type"; + public const string GenAiTokenTypeKey = "gen_ai.token.type"; public const string GenAiUsageInputTokensKey = "gen_ai.usage.input_tokens"; public const string GenAiUsageOutputTokensKey = "gen_ai.usage.output_tokens"; From a8e9b9d8773c96d6e49b344382803099b8450979 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 5 Jul 2024 15:09:16 -0700 Subject: [PATCH 03/11] add readme otel section --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index b0333937c..d3ca324ff 100644 --- a/README.md +++ b/README.md @@ -749,3 +749,44 @@ By default, the client classes will automatically retry the following errors up - 502 Bad Gateway - 503 Service Unavailable - 504 Gateway Timeout + +## Observability with OpenTelemetry + +> Note: +> OpenAI .NET SDK instrumentation is in development and is not complete. See [Available sources and meters](#available-sources-and-meters) section for the list of covered operations. + +OpenAI .NET library is instrumented with distributed tracing and metrics using .NET [tracing](https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing) +and [metrics](https://learn.microsoft.com/dotnet/core/diagnostics/metrics-instrumentation) API and supports [OpenTelemetry](https://learn.microsoft.com/dotnet/core/diagnostics/observability-with-otel). + +OpenAI .NET library follows [OpenTelemetry Semantic Conventions for Generative AI systems](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai). + +You can enable the instrumentation by configuring OpenTelemetry to record telemetry from OpenAI sources and meters: + +```csharp +builder.Services.AddOpenTelemetry() + .WithTracing(b => + { + b.AddSource("OpenAI.*") + ... + .AddOtlpExporter(); + }) + .WithMetrics(b => + { + b.AddMeter("OpenAI.*") + ... + .AddOtlpExporter(); + }); +``` + +Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. + +Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client +calls made by your application including those done by the OpenAI SDK. + + + +### Available sources and meters + +The following sources and meters are available: + +- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) \ No newline at end of file From 5bddb897aed22dc6a2dd5be8d3cfce4e2c667d7e Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 5 Jul 2024 16:06:31 -0700 Subject: [PATCH 04/11] feature-flag --- README.md | 50 ++++++++++++------- .../Instrumentation/InstrumentationFactory.cs | 41 +++++++++++++-- tests/Chat/ChatSmokeTests.cs | 2 + .../ChatInstrumentationTests.cs | 44 ++++++++++++---- 4 files changed, 106 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index d3ca324ff..f6144fe01 100644 --- a/README.md +++ b/README.md @@ -760,31 +760,45 @@ and [metrics](https://learn.microsoft.com/dotnet/core/diagnostics/metrics-instru OpenAI .NET library follows [OpenTelemetry Semantic Conventions for Generative AI systems](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai). -You can enable the instrumentation by configuring OpenTelemetry to record telemetry from OpenAI sources and meters: +### How to enable -```csharp -builder.Services.AddOpenTelemetry() - .WithTracing(b => - { - b.AddSource("OpenAI.*") - ... - .AddOtlpExporter(); - }) - .WithMetrics(b => - { - b.AddMeter("OpenAI.*") - ... - .AddOtlpExporter(); - }); -``` +The instrumentation is **experimental** - names of activity sources and meters, volume and semantics of the telemetry items may change. + +To enable the instrumentation: + +1. Set instrumentation feature-flag using one of the following options: + + - set the `AZURE_EXPERIMENTAL_ENABLE_ACTIVITY_SOURCE` environment variable to `"true"` + - set the `Azure.Experimental.EnableActivitySource` context switch to true in your application code when application + is starting and before initializing any OpenAI clients. For example: + + ```csharp + AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true); + ``` + +2. Configuring OpenTelemetry to record telemetry from OpenAI sources and meters: + + ```csharp + builder.Services.AddOpenTelemetry() + .WithTracing(b => + { + b.AddSource("OpenAI.*") + ... + .AddOtlpExporter(); + }) + .WithMetrics(b => + { + b.AddMeter("OpenAI.*") + ... + .AddOtlpExporter(); + }); + ``` Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client calls made by your application including those done by the OpenAI SDK. - - ### Available sources and meters The following sources and meters are available: diff --git a/src/Custom/Common/Instrumentation/InstrumentationFactory.cs b/src/Custom/Common/Instrumentation/InstrumentationFactory.cs index 12743d3d0..3b7538903 100644 --- a/src/Custom/Common/Instrumentation/InstrumentationFactory.cs +++ b/src/Custom/Common/Instrumentation/InstrumentationFactory.cs @@ -5,12 +5,14 @@ namespace OpenAI.Custom.Common.Instrumentation; internal class InstrumentationFactory { + private const string ChatOperationName = "chat"; + private readonly bool IsInstrumentationEnabled = AppContextSwitchHelper + .GetConfigValue("OpenAI.Experimental.EnableInstrumentation", "OPENAI_EXPERIMENTAL_ENABLE_INSTRUMENTATION"); + private readonly string _serverAddress; private readonly int _serverPort; private readonly string _model; - private const string ChatOperationName = "chat"; - public InstrumentationFactory(string model, Uri endpoint) { _serverAddress = endpoint.Host; @@ -20,6 +22,39 @@ public InstrumentationFactory(string model, Uri endpoint) public InstrumentationScope StartChatScope(ChatCompletionOptions completionsOptions) { - return InstrumentationScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions); + return IsInstrumentationEnabled + ? InstrumentationScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions) + : null; } + + internal static class AppContextSwitchHelper + { + /// + /// Determines if either an AppContext switch or its corresponding Environment Variable is set + /// + /// Name of the AppContext switch. + /// Name of the Environment variable. + /// If the AppContext switch has been set, returns the value of the switch. + /// If the AppContext switch has not been set, returns the value of the environment variable. + /// False if neither is set. + /// + public static bool GetConfigValue(string appContexSwitchName, string environmentVariableName) + { + // First check for the AppContext switch, giving it priority over the environment variable. + if (AppContext.TryGetSwitch(appContexSwitchName, out bool value)) + { + return value; + } + // AppContext switch wasn't used. Check the environment variable. + string envVar = Environment.GetEnvironmentVariable(environmentVariableName); + if (envVar != null && (envVar.Equals("true", StringComparison.OrdinalIgnoreCase) || envVar.Equals("1"))) + { + return true; + } + + // Default to false. + return false; + } + } + } \ No newline at end of file diff --git a/tests/Chat/ChatSmokeTests.cs b/tests/Chat/ChatSmokeTests.cs index 4940ca8f6..930a0e9e1 100644 --- a/tests/Chat/ChatSmokeTests.cs +++ b/tests/Chat/ChatSmokeTests.cs @@ -498,8 +498,10 @@ public async Task JsonResult() } [Test] + [NonParallelizable] public async Task HelloWorldChatWithTracingAndMetrics() { + using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient"); using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient"); diff --git a/tests/Instrumentation/ChatInstrumentationTests.cs b/tests/Instrumentation/ChatInstrumentationTests.cs index 6535acfde..bc26a2094 100644 --- a/tests/Instrumentation/ChatInstrumentationTests.cs +++ b/tests/Instrumentation/ChatInstrumentationTests.cs @@ -13,6 +13,7 @@ using System.Threading.Tasks; using static OpenAI.Tests.Instrumentation.TestMeterListener; using static OpenAI.Tests.Instrumentation.TestActivityListener; +using OpenAI.Tests.Utility; namespace OpenAI.Tests.Instrumentation; @@ -31,8 +32,6 @@ public class ChatInstrumentationTests private const int PromptTokens = 2; private const int CompletionTokens = 42; - private readonly InstrumentationFactory _instrumentationFactory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); - [Test] public void AllTelemetryOff() { @@ -41,13 +40,27 @@ public void AllTelemetryOff() Assert.IsNull(Activity.Current); } + [Test] + public void SwitchOffAllTelemetryOn() + { + using var activityListener = new TestActivityListener("OpenAI.ChatClient"); + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + Assert.IsNull(factory.StartChatScope(new ChatCompletionOptions())); + Assert.IsNull(Activity.Current); + } + [Test] public void MetricsOnTracingOff() { + using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + using var meterListener = new TestMeterListener("OpenAI.ChatClient"); var elapsedMax = Stopwatch.StartNew(); - using var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions()); + using var scope = factory.StartChatScope(new ChatCompletionOptions()); var elapsedMin = Stopwatch.StartNew(); Assert.Null(Activity.Current); @@ -69,9 +82,12 @@ public void MetricsOnTracingOff() [Test] public void MetricsOnTracingOffException() { + using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); using var meterListener = new TestMeterListener("OpenAI.ChatClient"); - using (var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions())) + using (var scope = factory.StartChatScope(new ChatCompletionOptions())) { scope.RecordException(new TaskCanceledException()); } @@ -83,12 +99,15 @@ public void MetricsOnTracingOffException() [Test] public void TracingOnMetricsOff() { + using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); using var listener = new TestActivityListener("OpenAI.ChatClient"); var chatCompletion = CreateChatCompletion(); Activity activity = null; - using (var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions())) + using (var scope = factory.StartChatScope(new ChatCompletionOptions())) { activity = Activity.Current; Assert.IsNull(activity.GetTagItem("gen_ai.request.temperature")); @@ -109,6 +128,8 @@ public void TracingOnMetricsOff() [Test] public void ChatTracingAllAttributes() { + using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); using var listener = new TestActivityListener("OpenAI.ChatClient"); var options = new ChatCompletionOptions() { @@ -120,7 +141,6 @@ public void ChatTracingAllAttributes() var chatCompletion = CreateChatCompletion(); - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); using (var scope = factory.StartChatScope(options)) { Assert.AreEqual(options.Temperature.Value, (float)Activity.Current.GetTagItem("gen_ai.request.temperature"), 0.01); @@ -136,10 +156,13 @@ public void ChatTracingAllAttributes() [Test] public void ChatTracingException() { + using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + + var instrumentationFactory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); using var listener = new TestActivityListener("OpenAI.ChatClient"); var error = new SocketException(42, "test error"); - using (var scope = _instrumentationFactory.StartChatScope(new ChatCompletionOptions())) + using (var scope = instrumentationFactory.StartChatScope(new ChatCompletionOptions())) { scope.RecordException(error); } @@ -152,11 +175,13 @@ public void ChatTracingException() [Test] public async Task ChatTracingAndMetricsMultiple() { + using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + using var activityListener = new TestActivityListener("OpenAI.ChatClient"); using var meterListener = new TestMeterListener("OpenAI.ChatClient"); var options = new ChatCompletionOptions(); - SetMessages(options, new UserChatMessage("hello")); var tasks = new Task[5]; int numberOfSuccessfulResponses = 3; @@ -167,7 +192,7 @@ public async Task ChatTracingAndMetricsMultiple() // don't let Activity.Current escape the scope tasks[i] = Task.Run(async () => { - using var scope = _instrumentationFactory.StartChatScope(options); + using var scope = factory.StartChatScope(options); await Task.Delay(10); if (t < numberOfSuccessfulResponses) { @@ -287,5 +312,4 @@ private static ChatCompletion CreateChatCompletion(int promptTokens = PromptToke return ModelReaderWriter.Read(completion); } - } From 27be386d613ed3b4693b5ea770f0c0edfedcb24d Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 5 Jul 2024 16:14:27 -0700 Subject: [PATCH 05/11] toc and missing file --- README.md | 19 +++++++++------- .../InstrumentationAppContextHelper.cs | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 tests/Utility/InstrumentationAppContextHelper.cs diff --git a/README.md b/README.md index f6144fe01..ef48597c6 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,13 @@ [![NuGet version](https://img.shields.io/nuget/vpre/openai.svg)](https://www.nuget.org/packages/OpenAI/absoluteLatest) -The OpenAI .NET library provides convenient access to the OpenAI REST API from .NET applications. +The OpenAI .NET library provides convenient access to the OpenAI REST API from .NET applications. It is generated from our [OpenAPI specification](https://github.com/openai/openai-openapi) in collaboration with Microsoft. ## Table of Contents +- [Table of Contents](#table-of-contents) - [Getting started](#getting-started) - [Prerequisites](#prerequisites) - [Install the NuGet package](#install-the-nuget-package) @@ -26,6 +27,9 @@ It is generated from our [OpenAPI specification](https://github.com/openai/opena - [Advanced scenarios](#advanced-scenarios) - [Using protocol methods](#using-protocol-methods) - [Automatically retrying errors](#automatically-retrying-errors) +- [Observability with OpenTelemetry](#observability-with-opentelemetry) + - [How to enable](#how-to-enable) + - [Available sources and meters](#available-sources-and-meters) ## Getting started @@ -714,7 +718,7 @@ For example, to use the protocol method variant of the `ChatClient`'s `CompleteC ChatClient client = new("gpt-4o", Environment.GetEnvironmentVariable("OPENAI_API_KEY")); BinaryData input = BinaryData.FromBytes(""" -{ +{ "model": "gpt-4o", "messages": [ { @@ -755,7 +759,7 @@ By default, the client classes will automatically retry the following errors up > Note: > OpenAI .NET SDK instrumentation is in development and is not complete. See [Available sources and meters](#available-sources-and-meters) section for the list of covered operations. -OpenAI .NET library is instrumented with distributed tracing and metrics using .NET [tracing](https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing) +OpenAI .NET library is instrumented with distributed tracing and metrics using .NET [tracing](https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing) and [metrics](https://learn.microsoft.com/dotnet/core/diagnostics/metrics-instrumentation) API and supports [OpenTelemetry](https://learn.microsoft.com/dotnet/core/diagnostics/observability-with-otel). OpenAI .NET library follows [OpenTelemetry Semantic Conventions for Generative AI systems](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai). @@ -775,7 +779,7 @@ To enable the instrumentation: ```csharp AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true); ``` - + 2. Configuring OpenTelemetry to record telemetry from OpenAI sources and meters: ```csharp @@ -794,13 +798,12 @@ To enable the instrumentation: }); ``` -Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. - -Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client +Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client calls made by your application including those done by the OpenAI SDK. +Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. ### Available sources and meters The following sources and meters are available: -- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) \ No newline at end of file +- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) diff --git a/tests/Utility/InstrumentationAppContextHelper.cs b/tests/Utility/InstrumentationAppContextHelper.cs new file mode 100644 index 000000000..e95896d5d --- /dev/null +++ b/tests/Utility/InstrumentationAppContextHelper.cs @@ -0,0 +1,22 @@ +using System; + +namespace OpenAI.Tests.Utility; + +internal class InstrumentationAppContextHelper : IDisposable +{ + private const string SwitchName = "OpenAI.Experimental.EnableInstrumentation"; + private InstrumentationAppContextHelper() + { + AppContext.SetSwitch(SwitchName, true); + } + + public static IDisposable EnableInstrumentation() + { + return new InstrumentationAppContextHelper(); + } + + public void Dispose() + { + AppContext.SetSwitch(SwitchName, false); + } +} From 9c1bf9e9e7ab2c47e49af4284431fbb18780e88e Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 26 Jul 2024 12:22:46 -0700 Subject: [PATCH 06/11] address review comments --- README.md | 58 +-------------- docs/observability.md | 57 +++++++++++++++ src/Custom/Chat/ChatClient.cs | 4 +- .../Instrumentation/Constants.cs | 4 +- .../Instrumentation/InstrumentationFactory.cs | 4 +- .../Instrumentation/InstrumentationScope.cs | 71 ++++++++++--------- tests/Chat/ChatSmokeTests.cs | 2 + .../ChatInstrumentationTests.cs | 8 +-- tests/Instrumentation/TestActivityListener.cs | 2 +- tests/OpenAI.Tests.csproj | 2 +- 10 files changed, 110 insertions(+), 102 deletions(-) create mode 100644 docs/observability.md rename src/{Custom/Common => Utility}/Instrumentation/Constants.cs (95%) rename src/{Custom/Common => Utility}/Instrumentation/InstrumentationFactory.cs (96%) rename src/{Custom/Common => Utility}/Instrumentation/InstrumentationScope.cs (86%) diff --git a/README.md b/README.md index ef48597c6..7ca492a44 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,7 @@ It is generated from our [OpenAPI specification](https://github.com/openai/opena - [Advanced scenarios](#advanced-scenarios) - [Using protocol methods](#using-protocol-methods) - [Automatically retrying errors](#automatically-retrying-errors) -- [Observability with OpenTelemetry](#observability-with-opentelemetry) - - [How to enable](#how-to-enable) - - [Available sources and meters](#available-sources-and-meters) +- [Observability](#observability) ## Getting started @@ -754,56 +752,6 @@ By default, the client classes will automatically retry the following errors up - 503 Service Unavailable - 504 Gateway Timeout -## Observability with OpenTelemetry +## Observability -> Note: -> OpenAI .NET SDK instrumentation is in development and is not complete. See [Available sources and meters](#available-sources-and-meters) section for the list of covered operations. - -OpenAI .NET library is instrumented with distributed tracing and metrics using .NET [tracing](https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing) -and [metrics](https://learn.microsoft.com/dotnet/core/diagnostics/metrics-instrumentation) API and supports [OpenTelemetry](https://learn.microsoft.com/dotnet/core/diagnostics/observability-with-otel). - -OpenAI .NET library follows [OpenTelemetry Semantic Conventions for Generative AI systems](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai). - -### How to enable - -The instrumentation is **experimental** - names of activity sources and meters, volume and semantics of the telemetry items may change. - -To enable the instrumentation: - -1. Set instrumentation feature-flag using one of the following options: - - - set the `AZURE_EXPERIMENTAL_ENABLE_ACTIVITY_SOURCE` environment variable to `"true"` - - set the `Azure.Experimental.EnableActivitySource` context switch to true in your application code when application - is starting and before initializing any OpenAI clients. For example: - - ```csharp - AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true); - ``` - -2. Configuring OpenTelemetry to record telemetry from OpenAI sources and meters: - - ```csharp - builder.Services.AddOpenTelemetry() - .WithTracing(b => - { - b.AddSource("OpenAI.*") - ... - .AddOtlpExporter(); - }) - .WithMetrics(b => - { - b.AddMeter("OpenAI.*") - ... - .AddOtlpExporter(); - }); - ``` - -Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client -calls made by your application including those done by the OpenAI SDK. -Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. - -### Available sources and meters - -The following sources and meters are available: - -- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) +OpenAI .NET library supports experimental distributed tracing and metrics with OpenTelemetry. Check out [Observability with OpenTelemetry](./docs/observability.md) for more details. diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 000000000..f04f94c6e --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,57 @@ +## Observability with OpenTelemetry + +> Note: +> OpenAI .NET SDK instrumentation is in development and is not complete. See [Available sources and meters](#available-sources-and-meters) section for the list of covered operations. + +OpenAI .NET library is instrumented with distributed tracing and metrics using .NET [tracing](https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing) +and [metrics](https://learn.microsoft.com/dotnet/core/diagnostics/metrics-instrumentation) API and supports [OpenTelemetry](https://learn.microsoft.com/dotnet/core/diagnostics/observability-with-otel). + +OpenAI .NET instrumentation follows [OpenTelemetry Semantic Conventions for Generative AI systems](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai). + +### How to enable + +The instrumentation is **experimental** - volume and semantics of the telemetry items may change. + +To enable the instrumentation: + +1. Set instrumentation feature-flag using one of the following options: + + - set the `OPENAI_EXPERIMENTAL_ENABLE_INSTRUMENTATION` environment variable to `"true"` + - set the `OpenAI.Experimental.EnableInstrumentation` context switch to true in your application code when application + is starting and before initializing any OpenAI clients. For example: + + ```csharp + AppContext.SetSwitch("OpenAI.Experimental.EnableInstrumentation", true); + ``` + +2. Enable OpenAI telemetry: + + ```csharp + builder.Services.AddOpenTelemetry() + .WithTracing(b => + { + b.AddSource("OpenAI.*") + ... + .AddOtlpExporter(); + }) + .WithMetrics(b => + { + b.AddMeter("OpenAI.*") + ... + .AddOtlpExporter(); + }); + ``` + + Distributed tracing is enabled with `AddSource("OpenAI.*")` which tells OpenTelemetry to listen to all [ActivitySources](https://learn.microsoft.com/dotnet/api/system.diagnostics.activitysource) with names starting with `OpenAI.*`. + + Similarly, metrics are configured with `AddMeter("OpenAI.*")` which enables all OpenAI-related [Meters](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.meter). + +Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client +calls made by your application including those done by the OpenAI SDK. +Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. + +### Available sources and meters + +The following sources and meters are available: + +- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) diff --git a/src/Custom/Chat/ChatClient.cs b/src/Custom/Chat/ChatClient.cs index 9ecfb7d77..a8d221b61 100644 --- a/src/Custom/Chat/ChatClient.cs +++ b/src/Custom/Chat/ChatClient.cs @@ -1,4 +1,4 @@ -using OpenAI.Custom.Common.Instrumentation; +using OpenAI.Instrumentation; using System; using System.ClientModel; using System.ClientModel.Primitives; @@ -225,7 +225,7 @@ private void CreateChatCompletionOptions(IEnumerable messages, ref { options.Messages = messages.ToList(); options.Model = _model; - options.Stream = stream + options.Stream = stream ? true : null; options.StreamOptions = stream ? options.StreamOptions : null; diff --git a/src/Custom/Common/Instrumentation/Constants.cs b/src/Utility/Instrumentation/Constants.cs similarity index 95% rename from src/Custom/Common/Instrumentation/Constants.cs rename to src/Utility/Instrumentation/Constants.cs index f71d3adc3..1f1ee419c 100644 --- a/src/Custom/Common/Instrumentation/Constants.cs +++ b/src/Utility/Instrumentation/Constants.cs @@ -1,4 +1,4 @@ -namespace OpenAI.Custom.Common.Instrumentation; +namespace OpenAI.Instrumentation; internal class Constants { @@ -20,7 +20,7 @@ internal class Constants public const string GenAiRequestTopPKey = "gen_ai.request.top_p"; public const string GenAiResponseIdKey = "gen_ai.response.id"; - public const string GenAiResponseFinishReasonKey = "gen_ai.response.finish_reason"; + public const string GenAiResponseFinishReasonKey = "gen_ai.response.finish_reasons"; public const string GenAiResponseModelKey = "gen_ai.response.model"; public const string GenAiSystemKey = "gen_ai.system"; diff --git a/src/Custom/Common/Instrumentation/InstrumentationFactory.cs b/src/Utility/Instrumentation/InstrumentationFactory.cs similarity index 96% rename from src/Custom/Common/Instrumentation/InstrumentationFactory.cs rename to src/Utility/Instrumentation/InstrumentationFactory.cs index 3b7538903..871120d14 100644 --- a/src/Custom/Common/Instrumentation/InstrumentationFactory.cs +++ b/src/Utility/Instrumentation/InstrumentationFactory.cs @@ -1,7 +1,7 @@ using OpenAI.Chat; using System; -namespace OpenAI.Custom.Common.Instrumentation; +namespace OpenAI.Instrumentation; internal class InstrumentationFactory { @@ -22,7 +22,7 @@ public InstrumentationFactory(string model, Uri endpoint) public InstrumentationScope StartChatScope(ChatCompletionOptions completionsOptions) { - return IsInstrumentationEnabled + return IsInstrumentationEnabled ? InstrumentationScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions) : null; } diff --git a/src/Custom/Common/Instrumentation/InstrumentationScope.cs b/src/Utility/Instrumentation/InstrumentationScope.cs similarity index 86% rename from src/Custom/Common/Instrumentation/InstrumentationScope.cs rename to src/Utility/Instrumentation/InstrumentationScope.cs index 5d0470f8a..a610fa2f0 100644 --- a/src/Custom/Common/Instrumentation/InstrumentationScope.cs +++ b/src/Utility/Instrumentation/InstrumentationScope.cs @@ -1,15 +1,18 @@ using OpenAI.Chat; using System; +using System.Buffers; using System.ClientModel; using System.Diagnostics; using System.Diagnostics.Metrics; -namespace OpenAI.Custom.Common.Instrumentation; +namespace OpenAI.Instrumentation; internal class InstrumentationScope : IDisposable { private static readonly ActivitySource s_chatSource = new ActivitySource("OpenAI.ChatClient"); private static readonly Meter s_chatMeter = new Meter("OpenAI.ChatClient"); + + // TODO: add explicit histogram buckets once System.Diagnostics.DiagnosticSource 9.0 is used private static readonly Histogram s_duration = s_chatMeter.CreateHistogram(Constants.GenAiClientOperationDurationMetricName, "s", "Measures GenAI operation duration."); private static readonly Histogram s_tokens = s_chatMeter.CreateHistogram(Constants.GenAiClientTokenUsageMetricName, "{token}", "Measures the number of input and output token used."); @@ -83,7 +86,11 @@ public void RecordException(Exception ex) { var errorType = GetErrorType(ex); RecordMetrics(null, errorType, null, null); - SetActivityError(ex, errorType); + if (_activity?.IsAllDataRequested == true) + { + _activity?.SetTag(Constants.ErrorTypeKey, errorType); + _activity?.SetStatus(ActivityStatusCode.Error, ex?.Message ?? errorType); + } } public void Dispose() @@ -102,64 +109,67 @@ private void RecordCommonAttributes() private void RecordMetrics(string responseModel, string errorType, int? inputTokensUsage, int? outputTokensUsage) { - TagList tags = ResponseTagsWithError(responseModel, errorType); - s_duration.Record(_duration.Elapsed.TotalSeconds, tags); + // tags is a struct, let's copy and modify them + var tags = _commonTags; + + if (responseModel != null) + { + tags.Add(Constants.GenAiResponseModelKey, responseModel); + } if (inputTokensUsage != null) { - // tags is a struct, let's copy them - TagList inputUsageTags = tags; + var inputUsageTags = tags; inputUsageTags.Add(Constants.GenAiTokenTypeKey, "input"); s_tokens.Record(inputTokensUsage.Value, inputUsageTags); } if (outputTokensUsage != null) - { - TagList outputUsageTags = tags; + { + var outputUsageTags = tags; outputUsageTags.Add(Constants.GenAiTokenTypeKey, "output"); - s_tokens.Record(outputTokensUsage.Value, outputUsageTags); } - } - - private TagList ResponseTagsWithError(string responseModel, string errorType) - { - // tags is a struct, let's copy them - var tags = _commonTags; - - if (responseModel != null) - { - tags.Add(Constants.GenAiResponseModelKey, responseModel); - } if (errorType != null) { tags.Add(Constants.ErrorTypeKey, errorType); } - return tags; + s_duration.Record(_duration.Elapsed.TotalSeconds, tags); } private void RecordResponseAttributes(string responseId, string model, ChatFinishReason? finishReason, ChatTokenUsage usage) { SetActivityTagIfNotNull(Constants.GenAiResponseIdKey, responseId); SetActivityTagIfNotNull(Constants.GenAiResponseModelKey, model); - SetActivityTagIfNotNull(Constants.GenAiResponseFinishReasonKey, GetFinishReason(finishReason)); SetActivityTagIfNotNull(Constants.GenAiUsageInputTokensKey, usage?.InputTokens); SetActivityTagIfNotNull(Constants.GenAiUsageOutputTokensKey, usage?.OutputTokens); + SetFinishReasonAttribute(finishReason); } - private string GetFinishReason(ChatFinishReason? reason) => - reason switch + private void SetFinishReasonAttribute(ChatFinishReason? finishReason) + { + if (finishReason == null) + { + return; + } + + var reasonStr = finishReason switch { ChatFinishReason.ContentFilter => "content_filter", ChatFinishReason.FunctionCall => "function_call", ChatFinishReason.Length => "length", ChatFinishReason.Stop => "stop", ChatFinishReason.ToolCalls => "tool_calls", - _ => reason?.ToString(), + _ => finishReason.ToString(), }; + // There could be multiple finish reasons, so semantic conventions use array type for the corrresponding attribute. + // It's likely to change, but for now let's report it as array. + _activity.SetTag(Constants.GenAiResponseFinishReasonKey, new[] { reasonStr }); + } + private string GetChatMessageRole(ChatMessageRole? role) => role switch { @@ -175,7 +185,7 @@ private string GetErrorType(Exception exception) { if (exception is ClientResultException requestFailedException) { - // TODO (limolkova) when we start targeting .NET 8 we should put + // TODO (lmolkova) when we start targeting .NET 8 we should put // requestFailedException.InnerException.HttpRequestError into error.type return requestFailedException.Status.ToString(); } @@ -183,15 +193,6 @@ private string GetErrorType(Exception exception) return exception?.GetType()?.FullName; } - private void SetActivityError(Exception exception, string errorType) - { - if (exception != null || errorType != null) - { - _activity?.SetTag(Constants.ErrorTypeKey, errorType); - _activity?.SetStatus(ActivityStatusCode.Error, exception?.Message ?? errorType); - } - } - private void SetActivityTagIfNotNull(string name, object value) { if (value != null) diff --git a/tests/Chat/ChatSmokeTests.cs b/tests/Chat/ChatSmokeTests.cs index 930a0e9e1..bc678e773 100644 --- a/tests/Chat/ChatSmokeTests.cs +++ b/tests/Chat/ChatSmokeTests.cs @@ -6,7 +6,9 @@ using System; using System.ClientModel; using System.ClientModel.Primitives; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Text.Json; using System.Threading.Tasks; diff --git a/tests/Instrumentation/ChatInstrumentationTests.cs b/tests/Instrumentation/ChatInstrumentationTests.cs index bc26a2094..c12a42f40 100644 --- a/tests/Instrumentation/ChatInstrumentationTests.cs +++ b/tests/Instrumentation/ChatInstrumentationTests.cs @@ -1,6 +1,6 @@ using NUnit.Framework; using OpenAI.Chat; -using OpenAI.Custom.Common.Instrumentation; +using OpenAI.Instrumentation; using System; using System.ClientModel.Primitives; using System.Collections.Generic; @@ -62,7 +62,7 @@ public void MetricsOnTracingOff() var elapsedMax = Stopwatch.StartNew(); using var scope = factory.StartChatScope(new ChatCompletionOptions()); var elapsedMin = Stopwatch.StartNew(); - + Assert.Null(Activity.Current); Assert.NotNull(scope); @@ -149,7 +149,7 @@ public void ChatTracingAllAttributes() scope.RecordChatCompletion(chatCompletion); } Assert.Null(Activity.Current); - + ValidateChatActivity(listener.Activities.Single(), chatCompletion, RequestModel, Host, Port); } @@ -214,7 +214,7 @@ public async Task ChatTracingAndMetricsMultiple() await Task.WhenAll(tasks); Assert.AreEqual(tasks.Length, activityListener.Activities.Count); - + var durations = meterListener.GetMeasurements("gen_ai.client.operation.duration"); Assert.AreEqual(tasks.Length, durations.Count); Assert.AreEqual(numberOfSuccessfulResponses, durations.Count(d => !d.tags.ContainsKey("error.type"))); diff --git a/tests/Instrumentation/TestActivityListener.cs b/tests/Instrumentation/TestActivityListener.cs index 208526500..0c4f0c5fb 100644 --- a/tests/Instrumentation/TestActivityListener.cs +++ b/tests/Instrumentation/TestActivityListener.cs @@ -50,7 +50,7 @@ public static void ValidateChatActivity(Activity activity, ChatCompletion respon { Assert.AreEqual(response.Model, activity.GetTagItem("gen_ai.response.model")); Assert.AreEqual(response.Id, activity.GetTagItem("gen_ai.response.id")); - Assert.AreEqual(response.FinishReason.ToString().ToLower(), activity.GetTagItem("gen_ai.response.finish_reason")); + Assert.AreEqual(new[] { response.FinishReason.ToString().ToLower() }, activity.GetTagItem("gen_ai.response.finish_reasons")); Assert.AreEqual(response.Usage.OutputTokens, activity.GetTagItem("gen_ai.usage.output_tokens")); Assert.AreEqual(response.Usage.InputTokens, activity.GetTagItem("gen_ai.usage.input_tokens")); Assert.AreEqual(ActivityStatusCode.Unset, activity.Status); diff --git a/tests/OpenAI.Tests.csproj b/tests/OpenAI.Tests.csproj index 9fc6818fe..f9da73399 100644 --- a/tests/OpenAI.Tests.csproj +++ b/tests/OpenAI.Tests.csproj @@ -17,6 +17,6 @@ - + \ No newline at end of file From 1c2174d717996d6ffebe816c77cf7af254191df5 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 26 Jul 2024 12:41:36 -0700 Subject: [PATCH 07/11] clean up --- README.md | 1 - src/OpenAI.csproj | 2 -- src/Utility/Instrumentation/InstrumentationFactory.cs | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 7ca492a44..405b5d5a0 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ It is generated from our [OpenAPI specification](https://github.com/openai/opena ## Table of Contents -- [Table of Contents](#table-of-contents) - [Getting started](#getting-started) - [Prerequisites](#prerequisites) - [Install the NuGet package](#install-the-nuget-package) diff --git a/src/OpenAI.csproj b/src/OpenAI.csproj index 8c02b1027..7d19ce117 100644 --- a/src/OpenAI.csproj +++ b/src/OpenAI.csproj @@ -63,7 +63,6 @@ true - @@ -75,5 +74,4 @@ - diff --git a/src/Utility/Instrumentation/InstrumentationFactory.cs b/src/Utility/Instrumentation/InstrumentationFactory.cs index 871120d14..1d863f9c9 100644 --- a/src/Utility/Instrumentation/InstrumentationFactory.cs +++ b/src/Utility/Instrumentation/InstrumentationFactory.cs @@ -57,4 +57,4 @@ public static bool GetConfigValue(string appContexSwitchName, string environment } } -} \ No newline at end of file +} From 69dfb11eefdf3d594e5dbcc757b38a1051099718 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 26 Jul 2024 12:59:27 -0700 Subject: [PATCH 08/11] clean up --- tests/Chat/ChatSmokeTests.cs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/tests/Chat/ChatSmokeTests.cs b/tests/Chat/ChatSmokeTests.cs index bc678e773..69c062fdb 100644 --- a/tests/Chat/ChatSmokeTests.cs +++ b/tests/Chat/ChatSmokeTests.cs @@ -476,29 +476,6 @@ public void SerializeChatMessageContentPartAsImageBytes(bool fromRawJson) } } - [Test] - public async Task JsonResult() - { - ChatClient client = GetTestClient(TestScenario.Chat); - IEnumerable messages = [ - new UserChatMessage("Give me a JSON object with the following properties: red, green, and blue. The value " - + "of each property should be a string containing their RGB representation in hexadecimal.") - ]; - ChatCompletionOptions options = new() { ResponseFormat = ChatResponseFormat.JsonObject }; - ClientResult result = IsAsync - ? await client.CompleteChatAsync(messages, options) - : client.CompleteChat(messages, options); - - JsonDocument jsonDocument = JsonDocument.Parse(result.Value.Content[0].Text); - - Assert.That(jsonDocument.RootElement.TryGetProperty("red", out JsonElement redProperty)); - Assert.That(jsonDocument.RootElement.TryGetProperty("green", out JsonElement greenProperty)); - Assert.That(jsonDocument.RootElement.TryGetProperty("blue", out JsonElement blueProperty)); - Assert.That(redProperty.GetString().ToLowerInvariant(), Contains.Substring("ff0000")); - Assert.That(greenProperty.GetString().ToLowerInvariant(), Contains.Substring("00ff00")); - Assert.That(blueProperty.GetString().ToLowerInvariant(), Contains.Substring("0000ff")); - } - [Test] [NonParallelizable] public async Task HelloWorldChatWithTracingAndMetrics() From 4ca1edbbabaf4876eb6e7adf266fb4d1aa9d70f0 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Mon, 5 Aug 2024 14:29:10 -0700 Subject: [PATCH 09/11] review: rename and move files and other fixes --- docs/observability.md | 6 +- src/Custom/Chat/ChatClient.cs | 10 +-- src/Utility/AppContextSwitchHelper.cs | 33 +++++++++ .../Instrumentation/InstrumentationFactory.cs | 60 ----------------- .../OpenTelemetryConstants.cs} | 8 +-- .../OpenTelemetryScope.cs} | 67 ++++++++++--------- src/Utility/Telemetry/OpenTelemetrySource.cs | 30 +++++++++ tests/Chat/ChatSmokeTests.cs | 6 +- tests/OpenAI.Tests.csproj | 3 +- .../ChatTelemetryTests.cs} | 57 ++++++++-------- .../TestActivityListener.cs | 2 +- tests/Telemetry/TestAppContextSwitchHelper.cs | 25 +++++++ .../TestMeterListener.cs | 3 +- .../InstrumentationAppContextHelper.cs | 22 ------ 14 files changed, 170 insertions(+), 162 deletions(-) create mode 100644 src/Utility/AppContextSwitchHelper.cs delete mode 100644 src/Utility/Instrumentation/InstrumentationFactory.cs rename src/Utility/{Instrumentation/Constants.cs => Telemetry/OpenTelemetryConstants.cs} (86%) rename src/Utility/{Instrumentation/InstrumentationScope.cs => Telemetry/OpenTelemetryScope.cs} (69%) create mode 100644 src/Utility/Telemetry/OpenTelemetrySource.cs rename tests/{Instrumentation/ChatInstrumentationTests.cs => Telemetry/ChatTelemetryTests.cs} (82%) rename tests/{Instrumentation => Telemetry}/TestActivityListener.cs (98%) create mode 100644 tests/Telemetry/TestAppContextSwitchHelper.cs rename tests/{Instrumentation => Telemetry}/TestMeterListener.cs (97%) delete mode 100644 tests/Utility/InstrumentationAppContextHelper.cs diff --git a/docs/observability.md b/docs/observability.md index f04f94c6e..8dd3290aa 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -16,12 +16,12 @@ To enable the instrumentation: 1. Set instrumentation feature-flag using one of the following options: - - set the `OPENAI_EXPERIMENTAL_ENABLE_INSTRUMENTATION` environment variable to `"true"` - - set the `OpenAI.Experimental.EnableInstrumentation` context switch to true in your application code when application + - set the `OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY` environment variable to `"true"` + - set the `OpenAI.Experimental.EnableOpenTelemetry` context switch to true in your application code when application is starting and before initializing any OpenAI clients. For example: ```csharp - AppContext.SetSwitch("OpenAI.Experimental.EnableInstrumentation", true); + AppContext.SetSwitch("OpenAI.Experimental.EnableOpenTelemetry", true); ``` 2. Enable OpenAI telemetry: diff --git a/src/Custom/Chat/ChatClient.cs b/src/Custom/Chat/ChatClient.cs index a8d221b61..2f366b9a1 100644 --- a/src/Custom/Chat/ChatClient.cs +++ b/src/Custom/Chat/ChatClient.cs @@ -1,4 +1,4 @@ -using OpenAI.Instrumentation; +using OpenAI.Telemetry; using System; using System.ClientModel; using System.ClientModel.Primitives; @@ -15,7 +15,7 @@ namespace OpenAI.Chat; public partial class ChatClient { private readonly string _model; - private readonly InstrumentationFactory _instrumentation; + private readonly OpenTelemetrySource _telemetry; /// /// Initializes a new instance of that will use an API key when authenticating. @@ -64,7 +64,7 @@ protected internal ChatClient(ClientPipeline pipeline, string model, Uri endpoin _model = model; _pipeline = pipeline; _endpoint = endpoint; - _instrumentation = new InstrumentationFactory(model, endpoint); + _telemetry = new OpenTelemetrySource(model, endpoint); } /// @@ -80,7 +80,7 @@ public virtual async Task> CompleteChatAsync(IEnume options ??= new(); CreateChatCompletionOptions(messages, ref options); - using InstrumentationScope scope = _instrumentation.StartChatScope(options); + using OpenTelemetryScope scope = _telemetry.StartChatScope(options); try { @@ -119,7 +119,7 @@ public virtual ClientResult CompleteChat(IEnumerable + /// Determines if either an AppContext switch or its corresponding Environment Variable is set + /// + /// Name of the AppContext switch. + /// Name of the Environment variable. + /// If the AppContext switch has been set, returns the value of the switch. + /// If the AppContext switch has not been set, returns the value of the environment variable. + /// False if neither is set. + /// + public static bool GetConfigValue(string appContexSwitchName, string environmentVariableName) + { + // First check for the AppContext switch, giving it priority over the environment variable. + if (AppContext.TryGetSwitch(appContexSwitchName, out bool value)) + { + return value; + } + // AppContext switch wasn't used. Check the environment variable. + string envVar = Environment.GetEnvironmentVariable(environmentVariableName); + if (envVar != null && (envVar.Equals("true", StringComparison.OrdinalIgnoreCase) || envVar.Equals("1"))) + { + return true; + } + + // Default to false. + return false; + } +} diff --git a/src/Utility/Instrumentation/InstrumentationFactory.cs b/src/Utility/Instrumentation/InstrumentationFactory.cs deleted file mode 100644 index 1d863f9c9..000000000 --- a/src/Utility/Instrumentation/InstrumentationFactory.cs +++ /dev/null @@ -1,60 +0,0 @@ -using OpenAI.Chat; -using System; - -namespace OpenAI.Instrumentation; - -internal class InstrumentationFactory -{ - private const string ChatOperationName = "chat"; - private readonly bool IsInstrumentationEnabled = AppContextSwitchHelper - .GetConfigValue("OpenAI.Experimental.EnableInstrumentation", "OPENAI_EXPERIMENTAL_ENABLE_INSTRUMENTATION"); - - private readonly string _serverAddress; - private readonly int _serverPort; - private readonly string _model; - - public InstrumentationFactory(string model, Uri endpoint) - { - _serverAddress = endpoint.Host; - _serverPort = endpoint.Port; - _model = model; - } - - public InstrumentationScope StartChatScope(ChatCompletionOptions completionsOptions) - { - return IsInstrumentationEnabled - ? InstrumentationScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions) - : null; - } - - internal static class AppContextSwitchHelper - { - /// - /// Determines if either an AppContext switch or its corresponding Environment Variable is set - /// - /// Name of the AppContext switch. - /// Name of the Environment variable. - /// If the AppContext switch has been set, returns the value of the switch. - /// If the AppContext switch has not been set, returns the value of the environment variable. - /// False if neither is set. - /// - public static bool GetConfigValue(string appContexSwitchName, string environmentVariableName) - { - // First check for the AppContext switch, giving it priority over the environment variable. - if (AppContext.TryGetSwitch(appContexSwitchName, out bool value)) - { - return value; - } - // AppContext switch wasn't used. Check the environment variable. - string envVar = Environment.GetEnvironmentVariable(environmentVariableName); - if (envVar != null && (envVar.Equals("true", StringComparison.OrdinalIgnoreCase) || envVar.Equals("1"))) - { - return true; - } - - // Default to false. - return false; - } - } - -} diff --git a/src/Utility/Instrumentation/Constants.cs b/src/Utility/Telemetry/OpenTelemetryConstants.cs similarity index 86% rename from src/Utility/Instrumentation/Constants.cs rename to src/Utility/Telemetry/OpenTelemetryConstants.cs index 1f1ee419c..d5a0906a7 100644 --- a/src/Utility/Instrumentation/Constants.cs +++ b/src/Utility/Telemetry/OpenTelemetryConstants.cs @@ -1,9 +1,9 @@ -namespace OpenAI.Instrumentation; +namespace OpenAI.Telemetry; -internal class Constants +internal class OpenTelemetryConstants { - // follows OpenTelemetry GenAI semantic conventions: - // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai + // follow OpenTelemetry GenAI semantic conventions: + // https://github.com/open-telemetry/semantic-conventions/tree/v1.27.0/docs/gen-ai public const string ErrorTypeKey = "error.type"; public const string ServerAddressKey = "server.address"; diff --git a/src/Utility/Instrumentation/InstrumentationScope.cs b/src/Utility/Telemetry/OpenTelemetryScope.cs similarity index 69% rename from src/Utility/Instrumentation/InstrumentationScope.cs rename to src/Utility/Telemetry/OpenTelemetryScope.cs index a610fa2f0..8cdda7e21 100644 --- a/src/Utility/Instrumentation/InstrumentationScope.cs +++ b/src/Utility/Telemetry/OpenTelemetryScope.cs @@ -1,20 +1,21 @@ using OpenAI.Chat; using System; -using System.Buffers; using System.ClientModel; using System.Diagnostics; using System.Diagnostics.Metrics; -namespace OpenAI.Instrumentation; +using static OpenAI.Telemetry.OpenTelemetryConstants; -internal class InstrumentationScope : IDisposable +namespace OpenAI.Telemetry; + +internal class OpenTelemetryScope : IDisposable { private static readonly ActivitySource s_chatSource = new ActivitySource("OpenAI.ChatClient"); private static readonly Meter s_chatMeter = new Meter("OpenAI.ChatClient"); // TODO: add explicit histogram buckets once System.Diagnostics.DiagnosticSource 9.0 is used - private static readonly Histogram s_duration = s_chatMeter.CreateHistogram(Constants.GenAiClientOperationDurationMetricName, "s", "Measures GenAI operation duration."); - private static readonly Histogram s_tokens = s_chatMeter.CreateHistogram(Constants.GenAiClientTokenUsageMetricName, "{token}", "Measures the number of input and output token used."); + private static readonly Histogram s_duration = s_chatMeter.CreateHistogram(GenAiClientOperationDurationMetricName, "s", "Measures GenAI operation duration."); + private static readonly Histogram s_tokens = s_chatMeter.CreateHistogram(GenAiClientTokenUsageMetricName, "{token}", "Measures the number of input and output token used."); private readonly string _operationName; private readonly string _serverAddress; @@ -25,7 +26,7 @@ internal class InstrumentationScope : IDisposable private Activity _activity; private TagList _commonTags; - private InstrumentationScope( + private OpenTelemetryScope( string model, string operationName, string serverAddress, int serverPort) { @@ -35,12 +36,14 @@ private InstrumentationScope( _serverPort = serverPort; } - public static InstrumentationScope StartChat(string model, string operationName, + private static bool IsChatEnabled => s_chatSource.HasListeners() || s_tokens.Enabled || s_duration.Enabled; + + public static OpenTelemetryScope StartChat(string model, string operationName, string serverAddress, int serverPort, ChatCompletionOptions options) { - if (s_chatSource.HasListeners() || s_tokens.Enabled || s_duration.Enabled) + if (IsChatEnabled) { - var scope = new InstrumentationScope(model, operationName, serverAddress, serverPort); + var scope = new OpenTelemetryScope(model, operationName, serverAddress, serverPort); scope.StartChat(options); return scope; } @@ -53,20 +56,20 @@ private void StartChat(ChatCompletionOptions options) _duration = Stopwatch.StartNew(); _commonTags = new TagList { - { Constants.GenAiSystemKey, Constants.GenAiSystemValue }, - { Constants.GenAiRequestModelKey, _requestModel }, - { Constants.ServerAddressKey, _serverAddress }, - { Constants.ServerPortKey, _serverPort }, - { Constants.GenAiOperationNameKey, _operationName }, + { GenAiSystemKey, GenAiSystemValue }, + { GenAiRequestModelKey, _requestModel }, + { ServerAddressKey, _serverAddress }, + { ServerPortKey, _serverPort }, + { GenAiOperationNameKey, _operationName }, }; _activity = s_chatSource.StartActivity(string.Concat(_operationName, " ", _requestModel), ActivityKind.Client); if (_activity?.IsAllDataRequested == true) { RecordCommonAttributes(); - SetActivityTagIfNotNull(Constants.GenAiRequestMaxTokensKey, options?.MaxTokens); - SetActivityTagIfNotNull(Constants.GenAiRequestTemperatureKey, options?.Temperature); - SetActivityTagIfNotNull(Constants.GenAiRequestTopPKey, options?.TopP); + SetActivityTagIfNotNull(GenAiRequestMaxTokensKey, options?.MaxTokens); + SetActivityTagIfNotNull(GenAiRequestTemperatureKey, options?.Temperature); + SetActivityTagIfNotNull(GenAiRequestTopPKey, options?.TopP); } return; @@ -88,7 +91,7 @@ public void RecordException(Exception ex) RecordMetrics(null, errorType, null, null); if (_activity?.IsAllDataRequested == true) { - _activity?.SetTag(Constants.ErrorTypeKey, errorType); + _activity?.SetTag(OpenTelemetryConstants.ErrorTypeKey, errorType); _activity?.SetStatus(ActivityStatusCode.Error, ex?.Message ?? errorType); } } @@ -100,11 +103,11 @@ public void Dispose() private void RecordCommonAttributes() { - _activity.SetTag(Constants.GenAiSystemKey, Constants.GenAiSystemValue); - _activity.SetTag(Constants.GenAiRequestModelKey, _requestModel); - _activity.SetTag(Constants.ServerAddressKey, _serverAddress); - _activity.SetTag(Constants.ServerPortKey, _serverPort); - _activity.SetTag(Constants.GenAiOperationNameKey, _operationName); + _activity.SetTag(GenAiSystemKey, GenAiSystemValue); + _activity.SetTag(GenAiRequestModelKey, _requestModel); + _activity.SetTag(ServerAddressKey, _serverAddress); + _activity.SetTag(ServerPortKey, _serverPort); + _activity.SetTag(GenAiOperationNameKey, _operationName); } private void RecordMetrics(string responseModel, string errorType, int? inputTokensUsage, int? outputTokensUsage) @@ -114,26 +117,26 @@ private void RecordMetrics(string responseModel, string errorType, int? inputTok if (responseModel != null) { - tags.Add(Constants.GenAiResponseModelKey, responseModel); + tags.Add(GenAiResponseModelKey, responseModel); } if (inputTokensUsage != null) { var inputUsageTags = tags; - inputUsageTags.Add(Constants.GenAiTokenTypeKey, "input"); + inputUsageTags.Add(GenAiTokenTypeKey, "input"); s_tokens.Record(inputTokensUsage.Value, inputUsageTags); } if (outputTokensUsage != null) { var outputUsageTags = tags; - outputUsageTags.Add(Constants.GenAiTokenTypeKey, "output"); + outputUsageTags.Add(GenAiTokenTypeKey, "output"); s_tokens.Record(outputTokensUsage.Value, outputUsageTags); } if (errorType != null) { - tags.Add(Constants.ErrorTypeKey, errorType); + tags.Add(ErrorTypeKey, errorType); } s_duration.Record(_duration.Elapsed.TotalSeconds, tags); @@ -141,10 +144,10 @@ private void RecordMetrics(string responseModel, string errorType, int? inputTok private void RecordResponseAttributes(string responseId, string model, ChatFinishReason? finishReason, ChatTokenUsage usage) { - SetActivityTagIfNotNull(Constants.GenAiResponseIdKey, responseId); - SetActivityTagIfNotNull(Constants.GenAiResponseModelKey, model); - SetActivityTagIfNotNull(Constants.GenAiUsageInputTokensKey, usage?.InputTokens); - SetActivityTagIfNotNull(Constants.GenAiUsageOutputTokensKey, usage?.OutputTokens); + SetActivityTagIfNotNull(GenAiResponseIdKey, responseId); + SetActivityTagIfNotNull(GenAiResponseModelKey, model); + SetActivityTagIfNotNull(GenAiUsageInputTokensKey, usage?.InputTokens); + SetActivityTagIfNotNull(GenAiUsageOutputTokensKey, usage?.OutputTokens); SetFinishReasonAttribute(finishReason); } @@ -167,7 +170,7 @@ private void SetFinishReasonAttribute(ChatFinishReason? finishReason) // There could be multiple finish reasons, so semantic conventions use array type for the corrresponding attribute. // It's likely to change, but for now let's report it as array. - _activity.SetTag(Constants.GenAiResponseFinishReasonKey, new[] { reasonStr }); + _activity.SetTag(GenAiResponseFinishReasonKey, new[] { reasonStr }); } private string GetChatMessageRole(ChatMessageRole? role) => diff --git a/src/Utility/Telemetry/OpenTelemetrySource.cs b/src/Utility/Telemetry/OpenTelemetrySource.cs new file mode 100644 index 000000000..a0ac1fe47 --- /dev/null +++ b/src/Utility/Telemetry/OpenTelemetrySource.cs @@ -0,0 +1,30 @@ +using OpenAI.Chat; +using System; + +namespace OpenAI.Telemetry; + +internal class OpenTelemetrySource +{ + private const string ChatOperationName = "chat"; + private readonly bool IsOTelEnabled = AppContextSwitchHelper + .GetConfigValue("OpenAI.Experimental.EnableOpenTelemetry", "OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY"); + + private readonly string _serverAddress; + private readonly int _serverPort; + private readonly string _model; + + public OpenTelemetrySource(string model, Uri endpoint) + { + _serverAddress = endpoint.Host; + _serverPort = endpoint.Port; + _model = model; + } + + public OpenTelemetryScope StartChatScope(ChatCompletionOptions completionsOptions) + { + return IsOTelEnabled + ? OpenTelemetryScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions) + : null; + } + +} diff --git a/tests/Chat/ChatSmokeTests.cs b/tests/Chat/ChatSmokeTests.cs index 69c062fdb..650191a20 100644 --- a/tests/Chat/ChatSmokeTests.cs +++ b/tests/Chat/ChatSmokeTests.cs @@ -1,7 +1,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using NUnit.Framework; using OpenAI.Chat; -using OpenAI.Tests.Instrumentation; +using OpenAI.Tests.Telemetry; using OpenAI.Tests.Utility; using System; using System.ClientModel; @@ -12,7 +12,7 @@ using System.Net; using System.Text.Json; using System.Threading.Tasks; -using static OpenAI.Tests.Instrumentation.TestMeterListener; +using static OpenAI.Tests.Telemetry.TestMeterListener; using static OpenAI.Tests.TestHelpers; namespace OpenAI.Tests.Chat; @@ -480,7 +480,7 @@ public void SerializeChatMessageContentPartAsImageBytes(bool fromRawJson) [NonParallelizable] public async Task HelloWorldChatWithTracingAndMetrics() { - using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient"); using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient"); diff --git a/tests/OpenAI.Tests.csproj b/tests/OpenAI.Tests.csproj index f9da73399..27d23a6f8 100644 --- a/tests/OpenAI.Tests.csproj +++ b/tests/OpenAI.Tests.csproj @@ -17,6 +17,7 @@ - + + \ No newline at end of file diff --git a/tests/Instrumentation/ChatInstrumentationTests.cs b/tests/Telemetry/ChatTelemetryTests.cs similarity index 82% rename from tests/Instrumentation/ChatInstrumentationTests.cs rename to tests/Telemetry/ChatTelemetryTests.cs index c12a42f40..8bf070b96 100644 --- a/tests/Instrumentation/ChatInstrumentationTests.cs +++ b/tests/Telemetry/ChatTelemetryTests.cs @@ -1,6 +1,6 @@ using NUnit.Framework; using OpenAI.Chat; -using OpenAI.Instrumentation; +using OpenAI.Telemetry; using System; using System.ClientModel.Primitives; using System.Collections.Generic; @@ -11,15 +11,14 @@ using System.Reflection; using System.Threading; using System.Threading.Tasks; -using static OpenAI.Tests.Instrumentation.TestMeterListener; -using static OpenAI.Tests.Instrumentation.TestActivityListener; -using OpenAI.Tests.Utility; +using static OpenAI.Tests.Telemetry.TestMeterListener; +using static OpenAI.Tests.Telemetry.TestActivityListener; -namespace OpenAI.Tests.Instrumentation; +namespace OpenAI.Tests.Telemetry; [TestFixture] [NonParallelizable] -public class ChatInstrumentationTests +public class ChatTelemetryTests { private const string RequestModel = "requestModel"; private const string Host = "host"; @@ -35,8 +34,8 @@ public class ChatInstrumentationTests [Test] public void AllTelemetryOff() { - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); - Assert.IsNull(factory.StartChatScope(new ChatCompletionOptions())); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + Assert.IsNull(telemetry.StartChatScope(new ChatCompletionOptions())); Assert.IsNull(Activity.Current); } @@ -45,22 +44,22 @@ public void SwitchOffAllTelemetryOn() { using var activityListener = new TestActivityListener("OpenAI.ChatClient"); using var meterListener = new TestMeterListener("OpenAI.ChatClient"); - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); - Assert.IsNull(factory.StartChatScope(new ChatCompletionOptions())); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); + Assert.IsNull(telemetry.StartChatScope(new ChatCompletionOptions())); Assert.IsNull(Activity.Current); } [Test] public void MetricsOnTracingOff() { - using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); using var meterListener = new TestMeterListener("OpenAI.ChatClient"); var elapsedMax = Stopwatch.StartNew(); - using var scope = factory.StartChatScope(new ChatCompletionOptions()); + using var scope = telemetry.StartChatScope(new ChatCompletionOptions()); var elapsedMin = Stopwatch.StartNew(); Assert.Null(Activity.Current); @@ -71,7 +70,7 @@ public void MetricsOnTracingOff() elapsedMin.Stop(); - ChatCompletion response = CreateChatCompletion(); + var response = CreateChatCompletion(); scope.RecordChatCompletion(response); scope.Dispose(); @@ -82,12 +81,12 @@ public void MetricsOnTracingOff() [Test] public void MetricsOnTracingOffException() { - using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); using var meterListener = new TestMeterListener("OpenAI.ChatClient"); - using (var scope = factory.StartChatScope(new ChatCompletionOptions())) + using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) { scope.RecordException(new TaskCanceledException()); } @@ -99,15 +98,15 @@ public void MetricsOnTracingOffException() [Test] public void TracingOnMetricsOff() { - using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); using var listener = new TestActivityListener("OpenAI.ChatClient"); var chatCompletion = CreateChatCompletion(); Activity activity = null; - using (var scope = factory.StartChatScope(new ChatCompletionOptions())) + using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) { activity = Activity.Current; Assert.IsNull(activity.GetTagItem("gen_ai.request.temperature")); @@ -128,8 +127,8 @@ public void TracingOnMetricsOff() [Test] public void ChatTracingAllAttributes() { - using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); using var listener = new TestActivityListener("OpenAI.ChatClient"); var options = new ChatCompletionOptions() { @@ -141,7 +140,7 @@ public void ChatTracingAllAttributes() var chatCompletion = CreateChatCompletion(); - using (var scope = factory.StartChatScope(options)) + using (var scope = telemetry.StartChatScope(options)) { Assert.AreEqual(options.Temperature.Value, (float)Activity.Current.GetTagItem("gen_ai.request.temperature"), 0.01); Assert.AreEqual(options.TopP.Value, (float)Activity.Current.GetTagItem("gen_ai.request.top_p"), 0.01); @@ -156,13 +155,13 @@ public void ChatTracingAllAttributes() [Test] public void ChatTracingException() { - using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var instrumentationFactory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); using var listener = new TestActivityListener("OpenAI.ChatClient"); var error = new SocketException(42, "test error"); - using (var scope = instrumentationFactory.StartChatScope(new ChatCompletionOptions())) + using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) { scope.RecordException(error); } @@ -175,8 +174,8 @@ public void ChatTracingException() [Test] public async Task ChatTracingAndMetricsMultiple() { - using var _ = InstrumentationAppContextHelper.EnableInstrumentation(); - var factory = new InstrumentationFactory(RequestModel, new Uri(Endpoint)); + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + var source = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); using var activityListener = new TestActivityListener("OpenAI.ChatClient"); using var meterListener = new TestMeterListener("OpenAI.ChatClient"); @@ -192,7 +191,7 @@ public async Task ChatTracingAndMetricsMultiple() // don't let Activity.Current escape the scope tasks[i] = Task.Run(async () => { - using var scope = factory.StartChatScope(options); + using var scope = source.StartChatScope(options); await Task.Delay(10); if (t < numberOfSuccessfulResponses) { diff --git a/tests/Instrumentation/TestActivityListener.cs b/tests/Telemetry/TestActivityListener.cs similarity index 98% rename from tests/Instrumentation/TestActivityListener.cs rename to tests/Telemetry/TestActivityListener.cs index 0c4f0c5fb..a236691b7 100644 --- a/tests/Instrumentation/TestActivityListener.cs +++ b/tests/Telemetry/TestActivityListener.cs @@ -9,7 +9,7 @@ using System.Diagnostics; using System.Linq; -namespace OpenAI.Tests.Instrumentation; +namespace OpenAI.Tests.Telemetry; internal class TestActivityListener : IDisposable { diff --git a/tests/Telemetry/TestAppContextSwitchHelper.cs b/tests/Telemetry/TestAppContextSwitchHelper.cs new file mode 100644 index 000000000..5faf5eca0 --- /dev/null +++ b/tests/Telemetry/TestAppContextSwitchHelper.cs @@ -0,0 +1,25 @@ +using System; + +namespace OpenAI.Tests.Telemetry; + +internal class TestAppContextSwitchHelper : IDisposable +{ + private const string OpenTelemetrySwitchName = "OpenAI.Experimental.EnableOpenTelemetry"; + + private string _switchName; + private TestAppContextSwitchHelper(string switchName) + { + _switchName = switchName; + AppContext.SetSwitch(_switchName, true); + } + + public static IDisposable EnableOpenTelemetry() + { + return new TestAppContextSwitchHelper(OpenTelemetrySwitchName); + } + + public void Dispose() + { + AppContext.SetSwitch(_switchName, false); + } +} diff --git a/tests/Instrumentation/TestMeterListener.cs b/tests/Telemetry/TestMeterListener.cs similarity index 97% rename from tests/Instrumentation/TestMeterListener.cs rename to tests/Telemetry/TestMeterListener.cs index 2bbd9c7e3..187f94015 100644 --- a/tests/Instrumentation/TestMeterListener.cs +++ b/tests/Telemetry/TestMeterListener.cs @@ -3,10 +3,9 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.Diagnostics.Metrics; -namespace OpenAI.Tests.Instrumentation; +namespace OpenAI.Tests.Telemetry; internal class TestMeterListener : IDisposable { diff --git a/tests/Utility/InstrumentationAppContextHelper.cs b/tests/Utility/InstrumentationAppContextHelper.cs deleted file mode 100644 index e95896d5d..000000000 --- a/tests/Utility/InstrumentationAppContextHelper.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; - -namespace OpenAI.Tests.Utility; - -internal class InstrumentationAppContextHelper : IDisposable -{ - private const string SwitchName = "OpenAI.Experimental.EnableInstrumentation"; - private InstrumentationAppContextHelper() - { - AppContext.SetSwitch(SwitchName, true); - } - - public static IDisposable EnableInstrumentation() - { - return new InstrumentationAppContextHelper(); - } - - public void Dispose() - { - AppContext.SetSwitch(SwitchName, false); - } -} From fcbefa24a9b6ae249d261624b0e5a68f99808bd3 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Mon, 5 Aug 2024 14:40:34 -0700 Subject: [PATCH 10/11] nit --- tests/OpenAI.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/OpenAI.Tests.csproj b/tests/OpenAI.Tests.csproj index 27d23a6f8..c05ab9f8d 100644 --- a/tests/OpenAI.Tests.csproj +++ b/tests/OpenAI.Tests.csproj @@ -18,6 +18,6 @@ - + \ No newline at end of file From 245624a828c7fa972d2dfc33cf8b1d09c4ad18f8 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 9 Aug 2024 13:52:24 -0700 Subject: [PATCH 11/11] Fix tests --- tests/Chat/ChatSmokeTests.cs | 34 ------------------------ tests/Chat/ChatTests.cs | 37 +++++++++++++++++++++++++++ tests/Telemetry/ChatTelemetryTests.cs | 1 + 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/tests/Chat/ChatSmokeTests.cs b/tests/Chat/ChatSmokeTests.cs index 650191a20..80d7e6b1e 100644 --- a/tests/Chat/ChatSmokeTests.cs +++ b/tests/Chat/ChatSmokeTests.cs @@ -475,38 +475,4 @@ public void SerializeChatMessageContentPartAsImageBytes(bool fromRawJson) Assert.That(additionalPropertyProperty.ValueKind, Is.EqualTo(JsonValueKind.True)); } } - - [Test] - [NonParallelizable] - public async Task HelloWorldChatWithTracingAndMetrics() - { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient"); - using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient"); - - ChatClient client = GetTestClient(TestScenario.Chat); - IEnumerable messages = [new UserChatMessage("Hello, world!")]; - ClientResult result = IsAsync - ? await client.CompleteChatAsync(messages) - : client.CompleteChat(messages); - - Assert.AreEqual(1, activityListener.Activities.Count); - TestActivityListener.ValidateChatActivity(activityListener.Activities.Single(), result.Value); - - List durations = meterListener.GetMeasurements("gen_ai.client.operation.duration"); - Assert.AreEqual(1, durations.Count); - ValidateChatMetricTags(durations.Single(), result.Value); - - List usages = meterListener.GetMeasurements("gen_ai.client.token.usage"); - Assert.AreEqual(2, usages.Count); - - Assert.True(usages[0].tags.TryGetValue("gen_ai.token.type", out var type)); - Assert.IsInstanceOf(type); - - TestMeasurement input = (type is "input") ? usages[0] : usages[1]; - TestMeasurement output = (type is "input") ? usages[1] : usages[0]; - - Assert.AreEqual(result.Value.Usage.InputTokens, input.value); - Assert.AreEqual(result.Value.Usage.OutputTokens, output.value); - } } diff --git a/tests/Chat/ChatTests.cs b/tests/Chat/ChatTests.cs index 5527c2d3d..05fe667f5 100644 --- a/tests/Chat/ChatTests.cs +++ b/tests/Chat/ChatTests.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using NUnit.Framework; using OpenAI.Chat; +using OpenAI.Tests.Telemetry; using OpenAI.Tests.Utility; using System; using System.ClientModel; @@ -12,6 +13,7 @@ using System.Net; using System.Text.Json; using System.Threading.Tasks; +using static OpenAI.Tests.Telemetry.TestMeterListener; using static OpenAI.Tests.TestHelpers; namespace OpenAI.Tests.Chat; @@ -334,4 +336,39 @@ public async Task JsonResult() Assert.That(greenProperty.GetString().ToLowerInvariant(), Contains.Substring("00ff00")); Assert.That(blueProperty.GetString().ToLowerInvariant(), Contains.Substring("0000ff")); } + + + [Test] + [NonParallelizable] + public async Task HelloWorldChatWithTracingAndMetrics() + { + using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); + using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient"); + using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient"); + + ChatClient client = GetTestClient(TestScenario.Chat); + IEnumerable messages = [new UserChatMessage("Hello, world!")]; + ClientResult result = IsAsync + ? await client.CompleteChatAsync(messages) + : client.CompleteChat(messages); + + Assert.AreEqual(1, activityListener.Activities.Count); + TestActivityListener.ValidateChatActivity(activityListener.Activities.Single(), result.Value); + + List durations = meterListener.GetMeasurements("gen_ai.client.operation.duration"); + Assert.AreEqual(1, durations.Count); + ValidateChatMetricTags(durations.Single(), result.Value); + + List usages = meterListener.GetMeasurements("gen_ai.client.token.usage"); + Assert.AreEqual(2, usages.Count); + + Assert.True(usages[0].tags.TryGetValue("gen_ai.token.type", out var type)); + Assert.IsInstanceOf(type); + + TestMeasurement input = (type is "input") ? usages[0] : usages[1]; + TestMeasurement output = (type is "input") ? usages[1] : usages[0]; + + Assert.AreEqual(result.Value.Usage.InputTokens, input.value); + Assert.AreEqual(result.Value.Usage.OutputTokens, output.value); + } } diff --git a/tests/Telemetry/ChatTelemetryTests.cs b/tests/Telemetry/ChatTelemetryTests.cs index 8bf070b96..d3b043a7c 100644 --- a/tests/Telemetry/ChatTelemetryTests.cs +++ b/tests/Telemetry/ChatTelemetryTests.cs @@ -18,6 +18,7 @@ namespace OpenAI.Tests.Telemetry; [TestFixture] [NonParallelizable] +[Category("Smoke")] public class ChatTelemetryTests { private const string RequestModel = "requestModel";