[.Net] Agent as service: Run an IAgent as openai chat completion endpoint (#2633)

* update

* add test

* clean up

* update

* Delete dotnet/src/AutoGen.Server/AutoGen.Service.csproj.user

* implement streaming

* add sample project

* rename AutoGen.Service to AutoGen.WebAPI

* rename AutoGen.Service to AutoGen.WebAPI

Author: LittleLittleCloud

dotnet/AutoGen.sln

@@ -1,4 +1,3 @@
-
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.8.34322.80
@@ -33,6 +32,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Mistral", "src\Auto
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Mistral.Tests", "test\AutoGen.Mistral.Tests\AutoGen.Mistral.Tests.csproj", "{15441693-3659-4868-B6C1-B106F52FF3BA}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI", "src\AutoGen.WebAPI\AutoGen.WebAPI.csproj", "{257FFD71-08E5-40C7-AB04-6A81A78EB410}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI.Tests", "test\AutoGen.WebAPI.Tests\AutoGen.WebAPI.Tests.csproj", "{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.SemanticKernel.Tests", "test\AutoGen.SemanticKernel.Tests\AutoGen.SemanticKernel.Tests.csproj", "{1DFABC4A-8458-4875-8DCB-59F3802DAC65}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.OpenAI.Tests", "test\AutoGen.OpenAI.Tests\AutoGen.OpenAI.Tests.csproj", "{D36A85F9-C172-487D-8192-6BFE5D05B4A7}"
@@ -61,7 +64,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Gemini.Sample", "sa
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.AotCompatibility.Tests", "test\AutoGen.AotCompatibility.Tests\AutoGen.AotCompatibility.Tests.csproj", "{6B82F26D-5040-4453-B21B-C8D1F913CE4C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.OpenAI.Sample", "sample\AutoGen.OpenAI.Sample\AutoGen.OpenAI.Sample.csproj", "{0E635268-351C-4A6B-A28D-593D868C2CA4}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.OpenAI.Sample", "sample\AutoGen.OpenAI.Sample\AutoGen.OpenAI.Sample.csproj", "{0E635268-351C-4A6B-A28D-593D868C2CA4}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI.Sample", "sample\AutoGen.WebAPI.Sample\AutoGen.WebAPI.Sample.csproj", "{12079C18-A519-403F-BBFD-200A36A0C083}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -117,6 +122,14 @@ Global
 		{15441693-3659-4868-B6C1-B106F52FF3BA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{15441693-3659-4868-B6C1-B106F52FF3BA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{15441693-3659-4868-B6C1-B106F52FF3BA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{257FFD71-08E5-40C7-AB04-6A81A78EB410}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{257FFD71-08E5-40C7-AB04-6A81A78EB410}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{257FFD71-08E5-40C7-AB04-6A81A78EB410}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{257FFD71-08E5-40C7-AB04-6A81A78EB410}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Release|Any CPU.Build.0 = Release|Any CPU
 		{1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -177,6 +190,10 @@ Global
 		{0E635268-351C-4A6B-A28D-593D868C2CA4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{12079C18-A519-403F-BBFD-200A36A0C083}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{12079C18-A519-403F-BBFD-200A36A0C083}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{12079C18-A519-403F-BBFD-200A36A0C083}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{12079C18-A519-403F-BBFD-200A36A0C083}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -194,6 +211,8 @@ Global
 		{63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
 		{6585D1A4-3D97-4D76-A688-1933B61AEB19} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
 		{15441693-3659-4868-B6C1-B106F52FF3BA} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
+		{257FFD71-08E5-40C7-AB04-6A81A78EB410} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
+		{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
 		{1DFABC4A-8458-4875-8DCB-59F3802DAC65} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
 		{D36A85F9-C172-487D-8192-6BFE5D05B4A7} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
 		{B61388CA-DC73-4B7F-A7B2-7B9A86C9229E} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
@@ -209,8 +228,9 @@ Global
 		{19679B75-CE3A-4DF0-A3F0-CA369D2760A4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9}
 		{6B82F26D-5040-4453-B21B-C8D1F913CE4C} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
 		{0E635268-351C-4A6B-A28D-593D868C2CA4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9}
+		{12079C18-A519-403F-BBFD-200A36A0C083} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B}
 	EndGlobalSection
-EndGlobal
+EndGlobal

dotnet/eng/Version.props

@@ -12,6 +12,7 @@
         <MicrosoftNETTestSdkVersion>17.7.0</MicrosoftNETTestSdkVersion>
         <MicrosoftDotnetInteractive>1.0.0-beta.24229.4</MicrosoftDotnetInteractive>
         <MicrosoftSourceLinkGitHubVersion>8.0.0</MicrosoftSourceLinkGitHubVersion>
+        <MicrosoftASPNETCoreVersion>8.0.4</MicrosoftASPNETCoreVersion>
         <GoogleCloudAPIPlatformVersion>3.0.0</GoogleCloudAPIPlatformVersion>
         <JsonSchemaVersion>4.3.0.2</JsonSchemaVersion>
     </PropertyGroup>

dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\AutoGen.WebAPI\AutoGen.WebAPI.csproj" />
+  </ItemGroup>
+
+</Project>

dotnet/sample/AutoGen.WebAPI.Sample/Program.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Program.cs
+
+using System.Runtime.CompilerServices;
+using AutoGen.Core;
+using AutoGen.Service;
+
+var alice = new DummyAgent("alice");
+var bob = new DummyAgent("bob");
+
+var builder = WebApplication.CreateBuilder(args);
+// Add services to the container.
+
+// run endpoint at port 5000
+builder.WebHost.UseUrls("http://localhost:5000");
+var app = builder.Build();
+
+app.UseAgentAsOpenAIChatCompletionEndpoint(alice);
+app.UseAgentAsOpenAIChatCompletionEndpoint(bob);
+
+app.Run();
+
+public class DummyAgent : IStreamingAgent
+{
+    public DummyAgent(string name = "dummy")
+    {
+        Name = name;
+    }
+
+    public string Name { get; }
+
+    public async Task<IMessage> GenerateReplyAsync(IEnumerable<IMessage> messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default)
+    {
+        return new TextMessage(Role.Assistant, $"I am dummy {this.Name}", this.Name);
+    }
+
+    public async IAsyncEnumerable<IMessage> GenerateStreamingReplyAsync(IEnumerable<IMessage> messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        var reply = $"I am dummy {this.Name}";
+        foreach (var c in reply)
+        {
+            yield return new TextMessageUpdate(Role.Assistant, c.ToString(), this.Name);
+        };
+    }
+}

dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <NoWarn>$(NoWarn);CS1591;CS1573</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" Version="$(MicrosoftASPNETCoreVersion)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\AutoGen.Core\AutoGen.Core.csproj" />
+  </ItemGroup>
+</Project>

dotnet/src/AutoGen.WebAPI/Extension.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Extension.cs
+
+using AutoGen.Core;
+using Microsoft.AspNetCore.Builder;
+
+namespace AutoGen.Service;
+
+public static class Extension
+{
+    /// <summary>
+    /// Serve the agent as an OpenAI chat completion endpoint using <see cref="OpenAIChatCompletionMiddleware"/>.
+    /// If the request path is /v1/chat/completions and model name is the same as the agent name,
+    /// the request will be handled by the agent.
+    /// otherwise, the request will be passed to the next middleware.
+    /// </summary>
+    /// <param name="app">application builder</param>
+    /// <param name="agent"><see cref="IAgent"/></param>
+    public static IApplicationBuilder UseAgentAsOpenAIChatCompletionEndpoint(this IApplicationBuilder app, IAgent agent)
+    {
+        var middleware = new OpenAIChatCompletionMiddleware(agent);
+        return app.Use(middleware.InvokeAsync);
+    }
+}

dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// OpenAIMessageConverter.cs
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AutoGen.Service.OpenAI.DTO;
+
+internal class OpenAIMessageConverter : JsonConverter<OpenAIMessage>
+{
+    public override OpenAIMessage Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        using JsonDocument document = JsonDocument.ParseValue(ref reader);
+        var root = document.RootElement;
+        var role = root.GetProperty("role").GetString();
+        var contentDocument = root.GetProperty("content");
+        var isContentDocumentString = contentDocument.ValueKind == JsonValueKind.String;
+        switch (role)
+        {
+            case "system":
+                return JsonSerializer.Deserialize<OpenAISystemMessage>(root.GetRawText()) ?? throw new JsonException();
+            case "user" when isContentDocumentString:
+                return JsonSerializer.Deserialize<OpenAIUserMessage>(root.GetRawText()) ?? throw new JsonException();
+            case "user" when !isContentDocumentString:
+                return JsonSerializer.Deserialize<OpenAIUserMultiModalMessage>(root.GetRawText()) ?? throw new JsonException();
+            case "assistant":
+                return JsonSerializer.Deserialize<OpenAIAssistantMessage>(root.GetRawText()) ?? throw new JsonException();
+            case "tool":
+                return JsonSerializer.Deserialize<OpenAIToolMessage>(root.GetRawText()) ?? throw new JsonException();
+            default:
+                throw new JsonException();
+        }
+    }
+
+    public override void Write(Utf8JsonWriter writer, OpenAIMessage value, JsonSerializerOptions options)
+    {
+        switch (value)
+        {
+            case OpenAISystemMessage systemMessage:
+                JsonSerializer.Serialize(writer, systemMessage, options);
+                break;
+            case OpenAIUserMessage userMessage:
+                JsonSerializer.Serialize(writer, userMessage, options);
+                break;
+            case OpenAIAssistantMessage assistantMessage:
+                JsonSerializer.Serialize(writer, assistantMessage, options);
+                break;
+            case OpenAIToolMessage toolMessage:
+                JsonSerializer.Serialize(writer, toolMessage, options);
+                break;
+            default:
+                throw new JsonException();
+        }
+    }
+}

dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// OpenAIAssistantMessage.cs
+
+using System.Text.Json.Serialization;
+
+namespace AutoGen.Service.OpenAI.DTO;
+
+internal class OpenAIAssistantMessage : OpenAIMessage
+{
+    [JsonPropertyName("role")]
+    public override string? Role { get; } = "assistant";
+
+    [JsonPropertyName("content")]
+    public string? Content { get; set; }
+
+    [JsonPropertyName("name")]
+    public string? Name { get; set; }
+
+    [JsonPropertyName("tool_calls")]
+    public OpenAIToolCallObject[]? ToolCalls { get; set; }
+}

dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs

@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// OpenAIChatCompletion.cs
+
+using System.Text.Json.Serialization;
+
+namespace AutoGen.Service.OpenAI.DTO;
+
+internal class OpenAIChatCompletion
+{
+    [JsonPropertyName("id")]
+    public string? ID { get; set; }
+
+    [JsonPropertyName("created")]
+    public long Created { get; set; }
+
+    [JsonPropertyName("choices")]
+    public OpenAIChatCompletionChoice[]? Choices { get; set; }
+
+    [JsonPropertyName("model")]
+    public string? Model { get; set; }
+
+    [JsonPropertyName("system_fingerprint")]
+    public string? SystemFingerprint { get; set; }
+
+    [JsonPropertyName("object")]
+    public string Object { get; set; } = "chat.completion";
+
+    [JsonPropertyName("usage")]
+    public OpenAIChatCompletionUsage? Usage { get; set; }
+}

dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// OpenAIChatCompletionChoice.cs
+
+using System.Text.Json.Serialization;
+
+namespace AutoGen.Service.OpenAI.DTO;
+
+internal class OpenAIChatCompletionChoice
+{
+    [JsonPropertyName("finish_reason")]
+    public string? FinishReason { get; set; }
+
+    [JsonPropertyName("index")]
+    public int Index { get; set; }
+
+    [JsonPropertyName("message")]
+    public OpenAIChatCompletionMessage? Message { get; set; }
+
+    [JsonPropertyName("delta")]
+    public OpenAIChatCompletionMessage? Delta { get; set; }
+}

dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// OpenAIChatCompletionMessage.cs
+
+using System.Text.Json.Serialization;
+
+namespace AutoGen.Service.OpenAI.DTO;
+
+internal class OpenAIChatCompletionMessage
+{
+    [JsonPropertyName("role")]
+    public string Role { get; } = "assistant";
+
+    [JsonPropertyName("content")]
+    public string? Content { get; set; }
+}

dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// OpenAIChatCompletionOption.cs
+
+using System.Text.Json.Serialization;
+
+namespace AutoGen.Service.OpenAI.DTO;
+
+internal class OpenAIChatCompletionOption
+{
+    [JsonPropertyName("messages")]
+    public OpenAIMessage[]? Messages { get; set; }
+
+    [JsonPropertyName("model")]
+    public string? Model { get; set; }
+
+    [JsonPropertyName("max_tokens")]
+    public int? MaxTokens { get; set; }
+
+    [JsonPropertyName("temperature")]
+    public float Temperature { get; set; } = 1;
+
+    /// <summary>
+    /// If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only server-sent events as they become available, with the stream terminated by a data: [DONE] message
+    /// </summary>
+    [JsonPropertyName("stream")]
+    public bool? Stream { get; set; } = false;
+
+    [JsonPropertyName("stream_options")]
+    public OpenAIStreamOptions? StreamOptions { get; set; }
+
+    [JsonPropertyName("stop")]
+    public string[]? Stop { get; set; }
+}

dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// OpenAIChatCompletionUsage.cs
+
+using System.Text.Json.Serialization;
+
+namespace AutoGen.Service.OpenAI.DTO;
+
+internal class OpenAIChatCompletionUsage
+{
+    [JsonPropertyName("completion_tokens")]
+    public int CompletionTokens { get; set; }
+
+    [JsonPropertyName("prompt_tokens")]
+    public int PromptTokens { get; set; }
+
+    [JsonPropertyName("total_tokens")]
+    public int TotalTokens { get; set; }
+}