Add support for retrieving File Search result content in Run Steps and other fixes

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Release History
 
+## 2.1.0-beta.3 (Unreleased)
+
+### Features added
+
+- OpenAI.Assistants:
+  - Added a `Content` property to `RunStepFileSearchResult` ([`step_details.tool_calls.file_search.results.content` in the REST API](https://platform.openai.com/docs/api-reference/run-steps/step-object)). (commit_hash)
+    - When using an Assistant with the File Search tool, you can use this property to retrieve the contents of the File Search results that were used by the model.
+  - Added `FileSearchRankingOptions` and `FileSearchResults` properties to `RunStepDetailsUpdate`. (commit_hash)
+
+### Breaking Changes in Preview APIs
+
+- OpenAI.RealtimeConversation:
+  - Renamed the `From*()` factory methods on `ConversationContentPart` to `Create*Part()` for consistency. (commit_hash)
+  - Removed an extraneous `toolCallId` parameter from `ConversationItem.CreateSystemMessage()`. (commit_hash)
+- OpenAI.Assistants:
+  - Renamed `RunStepType` to `RunStepKind`. (commit_hash)
+  - Changed `RunStepKind` from an "extensible enum" to a regular enum. (commit_hash)
+  - Renamed the `ToolCallId` property of `RunStepToolCall` to `Id`. (commit_hash)
+  - Renamed the `ToolKind` property of `RunStepToolCall` to `Kind`. (commit_hash)
+  - Replaced the `FileSearchRanker` and `FileSearchScoreThreshold` properties of `RunStepToolCall` with a new `FileSearchRankingOptions` property that contains both values to make it clearer how they are related. (commit_hash)
+
+### Bugs fixed
+
+- OpenAI.RealtimeConversation:
+  - Fixed serialization issues with `ConversationItem` creation of system and assistant messages. (commit_hash)
+
 ## 2.1.0-beta.2 (2024-11-04)
 
 ### Features added

api/OpenAI.netstandard2.0.cs

@@ -666,12 +666,12 @@ public class RunStep : IJsonModel<RunStep>, IPersistableModel<RunStep> {
         public DateTimeOffset? ExpiredAt { get; }
         public DateTimeOffset? FailedAt { get; }
         public string Id { get; }
+        public RunStepKind Kind { get; }
         public RunStepError LastError { get; }
         public IReadOnlyDictionary<string, string> Metadata { get; }
         public string RunId { get; }
         public RunStepStatus Status { get; }
         public string ThreadId { get; }
-        public RunStepType Type { get; }
         public RunStepTokenUsage Usage { get; }
         RunStep IJsonModel<RunStep>.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options);
         void IJsonModel<RunStep>.Write(Utf8JsonWriter writer, ModelReaderWriterOptions options);
@@ -723,6 +723,8 @@ public class RunStepDetailsUpdate : StreamingUpdate {
         public string CodeInterpreterInput { get; }
         public IReadOnlyList<RunStepUpdateCodeInterpreterOutput> CodeInterpreterOutputs { get; }
         public string CreatedMessageId { get; }
+        public FileSearchRankingOptions FileSearchRankingOptions { get; }
+        public IReadOnlyList<RunStepFileSearchResult> FileSearchResults { get; }
         public string FunctionArguments { get; }
         public string FunctionName { get; }
         public string FunctionOutput { get; }
@@ -756,6 +758,7 @@ public class RunStepError : IJsonModel<RunStepError>, IPersistableModel<RunStepE
         public override readonly string ToString();
     }
     public class RunStepFileSearchResult : IJsonModel<RunStepFileSearchResult>, IPersistableModel<RunStepFileSearchResult> {
+        public IReadOnlyList<RunStepFileSearchResultContent> Content { get; }
         public string FileId { get; }
         public string FileName { get; }
         public float Score { get; }
@@ -765,6 +768,22 @@ public class RunStepFileSearchResult : IJsonModel<RunStepFileSearchResult>, IPer
         string IPersistableModel<RunStepFileSearchResult>.GetFormatFromOptions(ModelReaderWriterOptions options);
         BinaryData IPersistableModel<RunStepFileSearchResult>.Write(ModelReaderWriterOptions options);
     }
+    public class RunStepFileSearchResultContent : IJsonModel<RunStepFileSearchResultContent>, IPersistableModel<RunStepFileSearchResultContent> {
+        public RunStepFileSearchResultContentKind Kind { get; }
+        public string Text { get; }
+        RunStepFileSearchResultContent IJsonModel<RunStepFileSearchResultContent>.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options);
+        void IJsonModel<RunStepFileSearchResultContent>.Write(Utf8JsonWriter writer, ModelReaderWriterOptions options);
+        RunStepFileSearchResultContent IPersistableModel<RunStepFileSearchResultContent>.Create(BinaryData data, ModelReaderWriterOptions options);
+        string IPersistableModel<RunStepFileSearchResultContent>.GetFormatFromOptions(ModelReaderWriterOptions options);
+        BinaryData IPersistableModel<RunStepFileSearchResultContent>.Write(ModelReaderWriterOptions options);
+    }
+    public enum RunStepFileSearchResultContentKind {
+        Text = 0
+    }
+    public enum RunStepKind {
+        CreatedMessage = 0,
+        ToolCall = 1
+    }
     public readonly partial struct RunStepStatus : IEquatable<RunStepStatus> {
         private readonly object _dummy;
         private readonly int _dummyPrimitive;
@@ -797,41 +816,23 @@ public class RunStepTokenUsage : IJsonModel<RunStepTokenUsage>, IPersistableMode
     public abstract class RunStepToolCall : IJsonModel<RunStepToolCall>, IPersistableModel<RunStepToolCall> {
         public string CodeInterpreterInput { get; }
         public IReadOnlyList<RunStepCodeInterpreterOutput> CodeInterpreterOutputs { get; }
-        public FileSearchRanker? FileSearchRanker { get; }
+        public FileSearchRankingOptions FileSearchRankingOptions { get; }
         public IReadOnlyList<RunStepFileSearchResult> FileSearchResults { get; }
-        public float? FileSearchScoreThreshold { get; }
         public string FunctionArguments { get; }
         public string FunctionName { get; }
         public string FunctionOutput { get; }
-        public string ToolCallId { get; }
-        public RunStepToolCallKind ToolKind { get; }
+        public string Id { get; }
+        public RunStepToolCallKind Kind { get; }
         RunStepToolCall IJsonModel<RunStepToolCall>.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options);
         void IJsonModel<RunStepToolCall>.Write(Utf8JsonWriter writer, ModelReaderWriterOptions options);
         RunStepToolCall IPersistableModel<RunStepToolCall>.Create(BinaryData data, ModelReaderWriterOptions options);
         string IPersistableModel<RunStepToolCall>.GetFormatFromOptions(ModelReaderWriterOptions options);
         BinaryData IPersistableModel<RunStepToolCall>.Write(ModelReaderWriterOptions options);
     }
     public enum RunStepToolCallKind {
-        Unknown = 0,
-        CodeInterpreter = 1,
-        FileSearch = 2,
-        Function = 3
-    }
-    public readonly partial struct RunStepType : IEquatable<RunStepType> {
-        private readonly object _dummy;
-        private readonly int _dummyPrimitive;
-        public RunStepType(string value);
-        public static RunStepType MessageCreation { get; }
-        public static RunStepType ToolCalls { get; }
-        public readonly bool Equals(RunStepType other);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        public override readonly bool Equals(object obj);
-        [EditorBrowsable(EditorBrowsableState.Never)]
-        public override readonly int GetHashCode();
-        public static bool operator ==(RunStepType left, RunStepType right);
-        public static implicit operator RunStepType(string value);
-        public static bool operator !=(RunStepType left, RunStepType right);
-        public override readonly string ToString();
+        CodeInterpreter = 0,
+        FileSearch = 1,
+        Function = 2
     }
     public class RunStepUpdate : StreamingUpdate<RunStep> {
     }
@@ -1456,8 +1457,8 @@ public class ChatFunctionChoice : IJsonModel<ChatFunctionChoice>, IPersistableMo
         public override readonly string ToString();
     }
     public class ChatInputTokenUsageDetails : IJsonModel<ChatInputTokenUsageDetails>, IPersistableModel<ChatInputTokenUsageDetails> {
-        public int? AudioTokenCount { get; }
-        public int? CachedTokenCount { get; }
+        public int AudioTokenCount { get; }
+        public int CachedTokenCount { get; }
         ChatInputTokenUsageDetails IJsonModel<ChatInputTokenUsageDetails>.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options);
         void IJsonModel<ChatInputTokenUsageDetails>.Write(Utf8JsonWriter writer, ModelReaderWriterOptions options);
         ChatInputTokenUsageDetails IPersistableModel<ChatInputTokenUsageDetails>.Create(BinaryData data, ModelReaderWriterOptions options);
@@ -1528,7 +1529,7 @@ public enum ChatMessageRole {
         Function = 4
     }
     public class ChatOutputTokenUsageDetails : IJsonModel<ChatOutputTokenUsageDetails>, IPersistableModel<ChatOutputTokenUsageDetails> {
-        public int? AudioTokenCount { get; }
+        public int AudioTokenCount { get; }
         public int ReasoningTokenCount { get; }
         ChatOutputTokenUsageDetails IJsonModel<ChatOutputTokenUsageDetails>.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options);
         void IJsonModel<ChatOutputTokenUsageDetails>.Write(Utf8JsonWriter writer, ModelReaderWriterOptions options);
@@ -1633,8 +1634,8 @@ public class FunctionChatMessage : ChatMessage, IJsonModel<FunctionChatMessage>,
     }
     public static class OpenAIChatModelFactory {
         public static ChatCompletion ChatCompletion(string id = null, ChatFinishReason finishReason = ChatFinishReason.Stop, ChatMessageContent content = null, string refusal = null, IEnumerable<ChatToolCall> toolCalls = null, ChatMessageRole role = ChatMessageRole.System, ChatFunctionCall functionCall = null, IEnumerable<ChatTokenLogProbabilityDetails> contentTokenLogProbabilities = null, IEnumerable<ChatTokenLogProbabilityDetails> refusalTokenLogProbabilities = null, DateTimeOffset createdAt = default, string model = null, string systemFingerprint = null, ChatTokenUsage usage = null);
-        public static ChatInputTokenUsageDetails ChatInputTokenUsageDetails(int? audioTokenCount = null, int? cachedTokenCount = null);
-        public static ChatOutputTokenUsageDetails ChatOutputTokenUsageDetails(int reasoningTokenCount = 0, int? audioTokenCount = null);
+        public static ChatInputTokenUsageDetails ChatInputTokenUsageDetails(int audioTokenCount = 0, int cachedTokenCount = 0);
+        public static ChatOutputTokenUsageDetails ChatOutputTokenUsageDetails(int reasoningTokenCount = 0, int audioTokenCount = 0);
         public static ChatTokenLogProbabilityDetails ChatTokenLogProbabilityDetails(string token = null, float logProbability = 0, ReadOnlyMemory<byte>? utf8Bytes = null, IEnumerable<ChatTokenTopLogProbabilityDetails> topLogProbabilities = null);
         public static ChatTokenTopLogProbabilityDetails ChatTokenTopLogProbabilityDetails(string token = null, float logProbability = 0, ReadOnlyMemory<byte>? utf8Bytes = null);
         public static ChatTokenUsage ChatTokenUsage(int outputTokenCount = 0, int inputTokenCount = 0, int totalTokenCount = 0, ChatOutputTokenUsageDetails outputTokenDetails = null, ChatInputTokenUsageDetails inputTokenDetails = null);
@@ -2259,10 +2260,10 @@ public enum ConversationContentModalities {
     public abstract class ConversationContentPart : IJsonModel<ConversationContentPart>, IPersistableModel<ConversationContentPart> {
         public string AudioTranscript { get; }
         public string Text { get; }
-        public static ConversationContentPart FromInputAudioTranscript(string transcript = null);
-        public static ConversationContentPart FromInputText(string text);
-        public static ConversationContentPart FromOutputAudioTranscript(string transcript = null);
-        public static ConversationContentPart FromOutputText(string text);
+        public static ConversationContentPart CreateInputAudioTranscriptPart(string transcript = null);
+        public static ConversationContentPart CreateInputTextPart(string text);
+        public static ConversationContentPart CreateOutputAudioTranscriptPart(string transcript = null);
+        public static ConversationContentPart CreateOutputTextPart(string text);
         public static implicit operator ConversationContentPart(string text);
         ConversationContentPart IJsonModel<ConversationContentPart>.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options);
         void IJsonModel<ConversationContentPart>.Write(Utf8JsonWriter writer, ModelReaderWriterOptions options);
@@ -2395,7 +2396,7 @@ public abstract class ConversationItem : IJsonModel<ConversationItem>, IPersista
         public static ConversationItem CreateAssistantMessage(IEnumerable<ConversationContentPart> contentItems);
         public static ConversationItem CreateFunctionCall(string name, string callId, string arguments);
         public static ConversationItem CreateFunctionCallOutput(string callId, string output);
-        public static ConversationItem CreateSystemMessage(string toolCallId, IEnumerable<ConversationContentPart> contentItems);
+        public static ConversationItem CreateSystemMessage(IEnumerable<ConversationContentPart> contentItems);
         public static ConversationItem CreateUserMessage(IEnumerable<ConversationContentPart> contentItems);
         ConversationItem IJsonModel<ConversationItem>.Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options);
         void IJsonModel<ConversationItem>.Write(Utf8JsonWriter writer, ModelReaderWriterOptions options);

examples/Assistants/Example04_AllTheTools.cs

@@ -180,7 +180,7 @@ CollectionResult<ThreadMessage> messages
                 Console.WriteLine($"Run step: {step.Status}");
                 foreach (RunStepToolCall toolCall in step.Details.ToolCalls)
                 {
-                    Console.WriteLine($" --> Tool call: {toolCall.ToolKind}");
+                    Console.WriteLine($" --> Tool call: {toolCall.Kind}");
                     foreach (RunStepCodeInterpreterOutput output in toolCall.CodeInterpreterOutputs)
                     {
                         Console.WriteLine($"    --> Output: {output.ImageFileId}");

examples/OpenAI.Examples.csproj

@@ -17,7 +17,7 @@
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
     <PackageReference Include="NUnit" Version="3.13.2" />
     <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageReference Include="Moq" Version="[4.18.2]" />
   </ItemGroup>
 </Project>

src/Custom/Assistants/AssistantClient.cs

@@ -1142,50 +1142,50 @@ public virtual AsyncCollectionResult<RunStep> GetRunStepsAsync(
     }
 
     /// <summary>
-    /// Rehydrates a page collection holding <see cref="RunStep"/> instances from a page token.
+    /// Gets a page collection holding <see cref="RunStep"/> instances associated with a <see cref="ThreadRun"/>.
     /// </summary>
-    /// <param name="firstPageToken"> Page token corresponding to the first page of the collection to rehydrate. </param>
+    /// <param name="threadId"> The ID of the thread associated with the run. </param>
+    /// <param name="runId"> The ID of the run to list run steps from. </param>
+    /// <param name="options"></param>
     /// <param name="cancellationToken">A token that can be used to cancel this method call.</param>
     /// <returns> A collection of <see cref="RunStep"/>. </returns>
-    public virtual AsyncCollectionResult<RunStep> GetRunStepsAsync(
-        ContinuationToken firstPageToken,
+    public virtual CollectionResult<RunStep> GetRunSteps(
+        string threadId,
+        string runId,
+        RunStepCollectionOptions options = default,
         CancellationToken cancellationToken = default)
     {
-        Argument.AssertNotNull(firstPageToken, nameof(firstPageToken));
+        Argument.AssertNotNullOrEmpty(threadId, nameof(threadId));
+        Argument.AssertNotNullOrEmpty(runId, nameof(runId));
 
-        RunStepCollectionPageToken pageToken = RunStepCollectionPageToken.FromToken(firstPageToken);
-        AsyncCollectionResult result = GetRunStepsAsync(pageToken?.ThreadId, pageToken?.RunId, pageToken?.Limit, pageToken?.Order, pageToken?.After, pageToken?.Before, cancellationToken.ToRequestOptions());
+        CollectionResult result = GetRunSteps(threadId, runId, options?.PageSizeLimit, options?.Order?.ToString(), options?.AfterId, options?.BeforeId, cancellationToken.ToRequestOptions());
 
-        if (result is not AsyncCollectionResult<RunStep> collection)
+        if (result is not CollectionResult<RunStep> collection)
         {
-            throw new InvalidOperationException("Failed to cast protocol return type to expected collection type 'AsyncCollectionResult<RunStep>'.");
+            throw new InvalidOperationException("Failed to cast protocol return type to expected collection type 'CollectionResult<RunStep>'.");
         }
 
         return collection;
     }
 
     /// <summary>
-    /// Gets a page collection holding <see cref="RunStep"/> instances associated with a <see cref="ThreadRun"/>.
+    /// Rehydrates a page collection holding <see cref="RunStep"/> instances from a page token.
     /// </summary>
-    /// <param name="threadId"> The ID of the thread associated with the run. </param>
-    /// <param name="runId"> The ID of the run to list run steps from. </param>
-    /// <param name="options"></param>
+    /// <param name="firstPageToken"> Page token corresponding to the first page of the collection to rehydrate. </param>
     /// <param name="cancellationToken">A token that can be used to cancel this method call.</param>
     /// <returns> A collection of <see cref="RunStep"/>. </returns>
-    public virtual CollectionResult<RunStep> GetRunSteps(
-        string threadId,
-        string runId,
-        RunStepCollectionOptions options = default,
+    public virtual AsyncCollectionResult<RunStep> GetRunStepsAsync(
+        ContinuationToken firstPageToken,
         CancellationToken cancellationToken = default)
     {
-        Argument.AssertNotNullOrEmpty(threadId, nameof(threadId));
-        Argument.AssertNotNullOrEmpty(runId, nameof(runId));
+        Argument.AssertNotNull(firstPageToken, nameof(firstPageToken));
 
-        CollectionResult result = GetRunSteps(threadId, runId, options?.PageSizeLimit, options?.Order?.ToString(), options?.AfterId, options?.BeforeId, cancellationToken.ToRequestOptions());
+        RunStepCollectionPageToken pageToken = RunStepCollectionPageToken.FromToken(firstPageToken);
+        AsyncCollectionResult result = GetRunStepsAsync(pageToken?.ThreadId, pageToken?.RunId, pageToken?.Limit, pageToken?.Order, pageToken?.After, pageToken?.Before, cancellationToken.ToRequestOptions());
 
-        if (result is not CollectionResult<RunStep> collection)
+        if (result is not AsyncCollectionResult<RunStep> collection)
         {
-            throw new InvalidOperationException("Failed to cast protocol return type to expected collection type 'CollectionResult<RunStep>'.");
+            throw new InvalidOperationException("Failed to cast protocol return type to expected collection type 'AsyncCollectionResult<RunStep>'.");
         }
 
         return collection;

src/Custom/Assistants/GeneratorStubs.cs

@@ -34,10 +34,6 @@ public readonly partial struct RunErrorCode { }
 [CodeGenModel("RunObjectIncompleteDetails")]
 public partial class RunIncompleteDetails { }
 
-[Experimental("OPENAI001")]
-[CodeGenModel("RunStepObjectType")]
-public readonly partial struct RunStepType { }
-
 [Experimental("OPENAI001")]
 [CodeGenModel("RunStepObjectStatus")]
 public readonly partial struct RunStepStatus { }
@@ -53,3 +49,4 @@ public readonly partial struct RunStepErrorCode { }
 [Experimental("OPENAI001")]
 [CodeGenModel("RunStepDetailsToolCallsCodeObjectCodeInterpreterOutputsObject")]
 public partial class RunStepCodeInterpreterOutput { }
+