Enable human interaction in AutoGenStudio

autogen/coding/local_commandline_code_executor.py

@@ -221,7 +221,12 @@ def _setup_functions(self) -> None:
             cmd = [py_executable, "-m", "pip", "install"] + required_packages
             try:
                 result = subprocess.run(
-                    cmd, cwd=self._work_dir, capture_output=True, text=True, timeout=float(self._timeout)
+                    cmd,
+                    cwd=self._work_dir,
+                    capture_output=True,
+                    text=True,
+                    timeout=float(self._timeout),
+                    encoding="utf-8",
                 )
             except subprocess.TimeoutExpired as e:
                 raise ValueError("Pip install timed out") from e
@@ -303,7 +308,13 @@ def _execute_code_dont_check_setup(self, code_blocks: List[CodeBlock]) -> Comman
 
             try:
                 result = subprocess.run(
-                    cmd, cwd=self._work_dir, capture_output=True, text=True, timeout=float(self._timeout), env=env
+                    cmd,
+                    cwd=self._work_dir,
+                    capture_output=True,
+                    text=True,
+                    timeout=float(self._timeout),
+                    env=env,
+                    encoding="utf-8",
                 )
             except subprocess.TimeoutExpired:
                 logs_all += "\n" + TIMEOUT_MSG

autogen/oai/cohere.py

@@ -6,6 +6,7 @@
             "api_type": "cohere",
             "model": "command-r-plus",
             "api_key": os.environ.get("COHERE_API_KEY")
+            "client_name": "autogen-cohere", # Optional parameter
             }
     ]}
 
@@ -144,7 +145,7 @@ def parse_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
     def create(self, params: Dict) -> ChatCompletion:
 
         messages = params.get("messages", [])
-
+        client_name = params.get("client_name") or "autogen-cohere"
         # Parse parameters to the Cohere API's parameters
         cohere_params = self.parse_params(params)
 
@@ -156,7 +157,7 @@ def create(self, params: Dict) -> ChatCompletion:
         cohere_params["preamble"] = preamble
 
         # We use chat model by default
-        client = Cohere(api_key=self.api_key)
+        client = Cohere(api_key=self.api_key, client_name=client_name)
 
         # Token counts will be returned
         prompt_tokens = 0
@@ -285,6 +286,23 @@ def create(self, params: Dict) -> ChatCompletion:
         return response_oai
 
 
+def extract_to_cohere_tool_results(tool_call_id: str, content_output: str, all_tool_calls) -> List[Dict[str, Any]]:
+    temp_tool_results = []
+
+    for tool_call in all_tool_calls:
+        if tool_call["id"] == tool_call_id:
+
+            call = {
+                "name": tool_call["function"]["name"],
+                "parameters": json.loads(
+                    tool_call["function"]["arguments"] if not tool_call["function"]["arguments"] == "" else "{}"
+                ),
+            }
+            output = [{"value": content_output}]
+            temp_tool_results.append(ToolResult(call=call, outputs=output))
+    return temp_tool_results
+
+
 def oai_messages_to_cohere_messages(
     messages: list[Dict[str, Any]], params: Dict[str, Any], cohere_params: Dict[str, Any]
 ) -> tuple[list[dict[str, Any]], str, str]:
@@ -352,7 +370,8 @@ def oai_messages_to_cohere_messages(
     # 'content' field renamed to 'message'
     # tools go into tools parameter
     # tool_results go into tool_results parameter
-    for message in messages:
+    messages_length = len(messages)
+    for index, message in enumerate(messages):
 
         if "role" in message and message["role"] == "system":
             # System message
@@ -369,34 +388,34 @@ def oai_messages_to_cohere_messages(
             new_message = {
                 "role": "CHATBOT",
                 "message": message["content"],
-                # Not including tools in this message, may need to. Testing required.
+                "tool_calls": [
+                    {
+                        "name": tool_call_.get("function", {}).get("name"),
+                        "parameters": json.loads(tool_call_.get("function", {}).get("arguments") or "null"),
+                    }
+                    for tool_call_ in message["tool_calls"]
+                ],
             }
 
             cohere_messages.append(new_message)
         elif "role" in message and message["role"] == "tool":
-            if "tool_call_id" in message:
-                # Convert the tool call to a result
+            if not (tool_call_id := message.get("tool_call_id")):
+                continue
+
+            # Convert the tool call to a result
+            content_output = message["content"]
+            tool_results_chat_turn = extract_to_cohere_tool_results(tool_call_id, content_output, tool_calls)
+            if (index == messages_length - 1) or (messages[index + 1].get("role", "").lower() in ("user", "tool")):
+                # If the tool call is the last message or the next message is a user/tool message, this is a recent tool call.
+                # So, we pass it into tool_results.
+                tool_results.extend(tool_results_chat_turn)
+                continue
 
-                tool_call_id = message["tool_call_id"]
-                content_output = message["content"]
-
-                # Find the original tool
-                for tool_call in tool_calls:
-                    if tool_call["id"] == tool_call_id:
-
-                        call = {
-                            "name": tool_call["function"]["name"],
-                            "parameters": json.loads(
-                                tool_call["function"]["arguments"]
-                                if not tool_call["function"]["arguments"] == ""
-                                else "{}"
-                            ),
-                        }
-                        output = [{"value": content_output}]
-
-                        tool_results.append(ToolResult(call=call, outputs=output))
+            else:
+                # If its not the current tool call, we pass it as a tool message in the chat history.
+                new_message = {"role": "TOOL", "tool_results": tool_results_chat_turn}
+                cohere_messages.append(new_message)
 
-                        break
         elif "content" in message and isinstance(message["content"], str):
             # Standard text message
             new_message = {
@@ -416,7 +435,7 @@ def oai_messages_to_cohere_messages(
         # If we're adding tool_results, like we are, the last message can't be a USER message
         # So, we add a CHATBOT 'continue' message, if so.
         # Changed key from "content" to "message" (jaygdesai/autogen_Jay)
-        if cohere_messages[-1]["role"] == "USER":
+        if cohere_messages[-1]["role"].lower() == "user":
             cohere_messages.append({"role": "CHATBOT", "message": "Please continue."})
 
         # We return a blank message when we have tool results

dotnet/sample/AutoGen.OpenAI.Sample/Structural_Output.cs

@@ -9,11 +9,10 @@
 using Json.Schema;
 using Json.Schema.Generation;
 using OpenAI;
-using OpenAI.Chat;
 
 namespace AutoGen.OpenAI.Sample;
 
-internal class Structural_Output
+public class Structural_Output
 {
     public static async Task RunAsync()
     {
@@ -23,24 +22,25 @@ public static async Task RunAsync()
 
         var schemaBuilder = new JsonSchemaBuilder().FromType<Person>();
         var schema = schemaBuilder.Build();
-
-        var personSchemaFormat = ChatResponseFormat.CreateJsonSchemaFormat(
-            name: "Person",
-            jsonSchema: BinaryData.FromObjectAsJson(schema),
-            description: "Person schema");
-
         var openAIClient = new OpenAIClient(apiKey);
         var openAIClientAgent = new OpenAIChatAgent(
             chatClient: openAIClient.GetChatClient(model),
             name: "assistant",
-            systemMessage: "You are a helpful assistant",
-            responseFormat: personSchemaFormat) // structural output by passing schema to response format
+            systemMessage: "You are a helpful assistant")
             .RegisterMessageConnector()
             .RegisterPrintMessage();
         #endregion create_agent
 
         #region chat_with_agent
-        var reply = await openAIClientAgent.SendAsync("My name is John, I am 25 years old, and I live in Seattle. I like to play soccer and read books.");
+        var prompt = new TextMessage(Role.User, """
+            My name is John, I am 25 years old, and I live in Seattle. I like to play soccer and read books.
+            """);
+        var reply = await openAIClientAgent.GenerateReplyAsync(
+            messages: [prompt],
+            options: new GenerateReplyOptions
+            {
+                OutputSchema = schema,
+            });
 
         var person = JsonSerializer.Deserialize<Person>(reply.GetContent());
         Console.WriteLine($"Name: {person.Name}");
@@ -60,31 +60,34 @@ public static async Task RunAsync()
         person.City.Should().Be("Seattle");
         person.Hobbies.Count.Should().Be(2);
     }
-}
 
-#region person_class
-public class Person
-{
-    [JsonPropertyName("name")]
-    [Description("Name of the person")]
-    [Required]
-    public string Name { get; set; }
-
-    [JsonPropertyName("age")]
-    [Description("Age of the person")]
-    [Required]
-    public int Age { get; set; }
-
-    [JsonPropertyName("city")]
-    [Description("City of the person")]
-    public string? City { get; set; }
-
-    [JsonPropertyName("address")]
-    [Description("Address of the person")]
-    public string? Address { get; set; }
-
-    [JsonPropertyName("hobbies")]
-    [Description("Hobbies of the person")]
-    public List<string>? Hobbies { get; set; }
+
+    #region person_class
+    [Title("Person")]
+    public class Person
+    {
+        [JsonPropertyName("name")]
+        [Description("Name of the person")]
+        [Required]
+        public string Name { get; set; }
+
+        [JsonPropertyName("age")]
+        [Description("Age of the person")]
+        [Required]
+        public int Age { get; set; }
+
+        [JsonPropertyName("city")]
+        [Description("City of the person")]
+        public string? City { get; set; }
+
+        [JsonPropertyName("address")]
+        [Description("Address of the person")]
+        public string? Address { get; set; }
+
+        [JsonPropertyName("hobbies")]
+        [Description("Hobbies of the person")]
+        public List<string>? Hobbies { get; set; }
+    }
+    #endregion person_class
+
 }
-#endregion person_class

dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs

@@ -4,13 +4,12 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using AutoGen.Core;
-using AutoGen.OpenAI;
 using AutoGen.OpenAI.Extension;
 using FluentAssertions;
 using OpenAI;
 using OpenAI.Chat;
 
-namespace AutoGen.BasicSample;
+namespace AutoGen.OpenAI.Sample;
 
 public class Use_Json_Mode
 {
@@ -50,18 +49,20 @@ public static async Task RunAsync()
         person.Age.Should().Be(25);
         person.Address.Should().BeNullOrEmpty();
     }
-}
 
-#region person_class
-public class Person
-{
-    [JsonPropertyName("name")]
-    public string Name { get; set; }
 
-    [JsonPropertyName("age")]
-    public int Age { get; set; }
+    #region person_class
+    public class Person
+    {
+        [JsonPropertyName("name")]
+        public string Name { get; set; }
+
+        [JsonPropertyName("age")]
+        public int Age { get; set; }
 
-    [JsonPropertyName("address")]
-    public string Address { get; set; }
+        [JsonPropertyName("address")]
+        public string Address { get; set; }
+    }
+    #endregion person_class
 }
-#endregion person_class
+

dotnet/src/AutoGen.Core/Agent/IAgent.cs

@@ -5,6 +5,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Json.Schema;
 
 namespace AutoGen.Core;
 
@@ -42,6 +43,7 @@ public GenerateReplyOptions(GenerateReplyOptions other)
         this.MaxToken = other.MaxToken;
         this.StopSequence = other.StopSequence?.Select(s => s)?.ToArray();
         this.Functions = other.Functions?.Select(f => f)?.ToArray();
+        this.OutputSchema = other.OutputSchema;
     }
 
     public float? Temperature { get; set; }
@@ -51,4 +53,9 @@ public GenerateReplyOptions(GenerateReplyOptions other)
     public string[]? StopSequence { get; set; }
 
     public FunctionContract[]? Functions { get; set; }
+
+    /// <summary>
+    /// Structural schema for the output. This property only applies to certain LLMs.
+    /// </summary>
+    public JsonSchema? OutputSchema { get; set; }
 }

dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs

@@ -10,6 +10,7 @@
 using AutoGen.OpenAI.Extension;
 using global::OpenAI;
 using global::OpenAI.Chat;
+using Json.Schema;
 
 namespace AutoGen.OpenAI;
 
@@ -179,6 +180,14 @@ private ChatCompletionOptions CreateChatCompletionsOptions(GenerateReplyOptions?
             }
         }
 
+        if (options?.OutputSchema is not null)
+        {
+            option.ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat(
+                name: options.OutputSchema.GetTitle() ?? throw new ArgumentException("Output schema must have a title"),
+                jsonSchema: BinaryData.FromObjectAsJson(options.OutputSchema),
+                description: options.OutputSchema.GetDescription());
+        }
+
         return option;
     }
 

dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj

@@ -12,6 +12,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\sample\AutoGen.OpenAI.Sample\AutoGen.OpenAI.Sample.csproj" />
     <ProjectReference Include="..\..\src\AutoGen.OpenAI\AutoGen.OpenAI.csproj" />
     <ProjectReference Include="..\..\src\AutoGen.SourceGenerator\AutoGen.SourceGenerator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
     <ProjectReference Include="..\AutoGen.Test.Share\AutoGen.Tests.Share.csproj" />

dotnet/test/AutoGen.OpenAI.Tests/OpenAIChatAgentTest.cs

@@ -246,7 +246,6 @@ public async Task ItCreateOpenAIChatAgentWithChatCompletionOptionAsync()
         respond.GetContent()?.Should().NotBeNullOrEmpty();
     }
 
-
     private OpenAIClient CreateOpenAIClientFromAzureOpenAI()
     {
         var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new Exception("Please set AZURE_OPENAI_ENDPOINT environment variable.");