From 4be98d04c2f047ace092b292d21c6ce129851a90 Mon Sep 17 00:00:00 2001 From: jspv Date: Wed, 18 Dec 2024 16:07:20 -0500 Subject: [PATCH 1/7] Adding ToolCallResultSummaryMessage --- .../src/autogen_agentchat/agents/_assistant_agent.py | 5 +++-- .../autogen-agentchat/src/autogen_agentchat/messages.py | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py index 74a80ac7bbd3..921dd99ac328 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py @@ -28,6 +28,7 @@ TextMessage, ToolCallMessage, ToolCallResultMessage, + ToolCallResultSummaryMessage, ) from ..state import AssistantAgentState from ._base_chat_agent import BaseChatAgent @@ -62,7 +63,7 @@ class AssistantAgent(BaseChatAgent): * If the model returns no tool call, then the response is immediately returned as a :class:`~autogen_agentchat.messages.TextMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. * When the model returns tool calls, they will be executed right away: - - When `reflect_on_tool_use` is False (default), the tool call results are returned as a :class:`~autogen_agentchat.messages.TextMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. `tool_call_summary_format` can be used to customize the tool call summary. + - When `reflect_on_tool_use` is False (default), the tool call results are returned as a :class:`~autogen_agentchat.messages.ToolCallResultSummaryMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. `tool_call_summary_format` can be used to customize the tool call summary. - When `reflect_on_tool_use` is True, the another model inference is made using the tool calls and results, and the text response is returned as a :class:`~autogen_agentchat.messages.TextMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. Hand off behavior: @@ -379,7 +380,7 @@ async def on_messages_stream( ) tool_call_summary = "\n".join(tool_call_summaries) yield Response( - chat_message=TextMessage(content=tool_call_summary, source=self.name), + chat_message=ToolCallResultSummaryMessage(content=tool_call_summary, source=self.name), inner_messages=inner_messages, ) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py index fddd6bea5bf8..aab3659b7ede 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py @@ -81,6 +81,15 @@ class ToolCallResultMessage(BaseMessage): type: Literal["ToolCallResultMessage"] = "ToolCallResultMessage" +class ToolCallResultSummaryMessage(BaseMessage): + """A message signaling the summary of tool call results.""" + + content: str + """Summary of the the tool call results.""" + + type: Literal["ToolCallResultSummaryMessage"] = "ToolCallResultSummaryMessage" + + ChatMessage = Annotated[TextMessage | MultiModalMessage | StopMessage | HandoffMessage, Field(discriminator="type")] """Messages for agent-to-agent communication.""" From bd3ee608faa7192baeddab17a5bba8578c978618 Mon Sep 17 00:00:00 2001 From: jspv Date: Wed, 18 Dec 2024 16:57:04 -0500 Subject: [PATCH 2/7] Support for ToolCallResultSummaryMessage --- .../autogen-agentchat/src/autogen_agentchat/messages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py index aab3659b7ede..fddeb89020ff 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py @@ -95,7 +95,7 @@ class ToolCallResultSummaryMessage(BaseMessage): AgentMessage = Annotated[ - TextMessage | MultiModalMessage | StopMessage | HandoffMessage | ToolCallMessage | ToolCallResultMessage, + TextMessage | MultiModalMessage | StopMessage | HandoffMessage | ToolCallMessage | ToolCallResultMessage | ToolCallResultSummaryMessage, Field(discriminator="type"), ] """All message types.""" @@ -109,6 +109,7 @@ class ToolCallResultSummaryMessage(BaseMessage): "HandoffMessage", "ToolCallMessage", "ToolCallResultMessage", + "ToolCallResultSummaryMessage", "ChatMessage", "AgentMessage", ] From b26bcb0af3e9978854fdb3f802a2587751e5dd75 Mon Sep 17 00:00:00 2001 From: jspv Date: Wed, 18 Dec 2024 20:15:33 -0500 Subject: [PATCH 3/7] Added ToolCallSummaryMessage --- .../src/autogen_agentchat/agents/_assistant_agent.py | 6 +++--- .../autogen-agentchat/src/autogen_agentchat/messages.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py index b6433eb03371..b29a58a15b56 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py @@ -28,7 +28,7 @@ TextMessage, ToolCallExecutionEvent, ToolCallRequestEvent, - ToolCallResultSummaryMessage, + ToolCallSummaryMessage, ) from ..state import AssistantAgentState from ._base_chat_agent import BaseChatAgent @@ -63,7 +63,7 @@ class AssistantAgent(BaseChatAgent): * If the model returns no tool call, then the response is immediately returned as a :class:`~autogen_agentchat.messages.TextMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. * When the model returns tool calls, they will be executed right away: - - When `reflect_on_tool_use` is False (default), the tool call results are returned as a :class:`~autogen_agentchat.messages.ToolCallResultSummaryMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. `tool_call_summary_format` can be used to customize the tool call summary. + - When `reflect_on_tool_use` is False (default), the tool call results are returned as a :class:`~autogen_agentchat.messages.ToolCallSummaryMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. `tool_call_summary_format` can be used to customize the tool call summary. - When `reflect_on_tool_use` is True, the another model inference is made using the tool calls and results, and the text response is returned as a :class:`~autogen_agentchat.messages.TextMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. Hand off behavior: @@ -380,7 +380,7 @@ async def on_messages_stream( ) tool_call_summary = "\n".join(tool_call_summaries) yield Response( - chat_message=ToolCallResultSummaryMessage(content=tool_call_summary, source=self.name), + chat_message=ToolCallSummaryMessage(content=tool_call_summary, source=self.name), inner_messages=inner_messages, ) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py index ff51198c6f06..e2e6585345dd 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py @@ -101,13 +101,13 @@ class ToolCallExecutionEvent(BaseMessage): type: Literal["ToolCallExecutionEvent"] = "ToolCallExecutionEvent" -class ToolCallResultSummaryMessage(BaseMessage): +class ToolCallSummaryMessage(BaseMessage): """A message signaling the summary of tool call results.""" content: str """Summary of the the tool call results.""" - type: Literal["ToolCallResultSummaryMessage"] = "ToolCallResultSummaryMessage" + type: Literal["ToolCallSummaryMessage"] = "ToolCallSummaryMessage" ChatMessage = Annotated[TextMessage | MultiModalMessage | StopMessage | HandoffMessage, Field(discriminator="type")] @@ -119,7 +119,7 @@ class ToolCallResultSummaryMessage(BaseMessage): AgentMessage = Annotated[ - TextMessage | MultiModalMessage | StopMessage | HandoffMessage | ToolCallRequestEvent | ToolCallExecutionEvent | ToolCallResultSummaryMessage, + TextMessage | MultiModalMessage | StopMessage | HandoffMessage | ToolCallRequestEvent | ToolCallExecutionEvent | ToolCallSummaryMessage, Field(discriminator="type"), ] """(Deprecated, will be removed in 0.4.0) All message and event types.""" @@ -135,7 +135,7 @@ class ToolCallResultSummaryMessage(BaseMessage): "ToolCallExecutionEvent", "ToolCallMessage", "ToolCallResultMessage", - "ToolCallResultSummaryMessage", + "ToolCallSummaryMessage", "ChatMessage", "AgentEvent", "AgentMessage", From aef903ec18650de1b0632a1838212407eb680758 Mon Sep 17 00:00:00 2001 From: jspv Date: Thu, 19 Dec 2024 06:25:48 -0500 Subject: [PATCH 4/7] ruff format --- .../autogen-agentchat/src/autogen_agentchat/messages.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py index e2e6585345dd..3baea02dbce6 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py @@ -119,7 +119,13 @@ class ToolCallSummaryMessage(BaseMessage): AgentMessage = Annotated[ - TextMessage | MultiModalMessage | StopMessage | HandoffMessage | ToolCallRequestEvent | ToolCallExecutionEvent | ToolCallSummaryMessage, + TextMessage + | MultiModalMessage + | StopMessage + | HandoffMessage + | ToolCallRequestEvent + | ToolCallExecutionEvent + | ToolCallSummaryMessage, Field(discriminator="type"), ] """(Deprecated, will be removed in 0.4.0) All message and event types.""" From b4eb3abe43ba24033519bc2155cf7868041a1134 Mon Sep 17 00:00:00 2001 From: jspv Date: Thu, 19 Dec 2024 06:32:43 -0500 Subject: [PATCH 5/7] Add ToolCallSummaryMessage to ChatMessage --- .../autogen-agentchat/src/autogen_agentchat/messages.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py index 3baea02dbce6..7237812ca72f 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/messages.py @@ -110,7 +110,9 @@ class ToolCallSummaryMessage(BaseMessage): type: Literal["ToolCallSummaryMessage"] = "ToolCallSummaryMessage" -ChatMessage = Annotated[TextMessage | MultiModalMessage | StopMessage | HandoffMessage, Field(discriminator="type")] +ChatMessage = Annotated[ + TextMessage | MultiModalMessage | StopMessage | ToolCallSummaryMessage | HandoffMessage, Field(discriminator="type") +] """Messages for agent-to-agent communication only.""" From 285228a29b0bfeb79b858a09c20ed9ee3784f139 Mon Sep 17 00:00:00 2001 From: jspv Date: Thu, 19 Dec 2024 15:21:43 -0500 Subject: [PATCH 6/7] typing and tests for ToolCallSummaryMessage --- .../src/autogen_agentchat/agents/_assistant_agent.py | 7 +++++-- .../_magentic_one/_magentic_one_orchestrator.py | 3 ++- .../teams/_group_chat/_selector_group_chat.py | 3 ++- .../autogen-agentchat/tests/test_assistant_agent.py | 3 ++- python/packages/autogen-agentchat/tests/test_group_chat.py | 4 +++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py index b29a58a15b56..d421d7ccb2ce 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py @@ -281,9 +281,12 @@ def __init__( @property def produced_message_types(self) -> List[type[ChatMessage]]: """The types of messages that the assistant agent produces.""" + message_types: List[type[ChatMessage]] = [TextMessage] if self._handoffs: - return [TextMessage, HandoffMessage] - return [TextMessage] + message_types.append(HandoffMessage) + if self._tools: + message_types.append(ToolCallSummaryMessage) + return message_types async def on_messages(self, messages: Sequence[ChatMessage], cancellation_token: CancellationToken) -> Response: async for message in self.on_messages_stream(messages, cancellation_token): diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py index 6e0d12f7f754..4f06f5d1a938 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py @@ -21,6 +21,7 @@ TextMessage, ToolCallExecutionEvent, ToolCallRequestEvent, + ToolCallSummaryMessage, ) from ....state import MagenticOneOrchestratorState from .._base_group_chat_manager import BaseGroupChatManager @@ -427,7 +428,7 @@ def _thread_to_context(self) -> List[LLMMessage]: """Convert the message thread to a context for the model.""" context: List[LLMMessage] = [] for m in self._message_thread: - if isinstance(m, ToolCallRequestEvent | ToolCallExecutionEvent): + if isinstance(m, ToolCallRequestEvent | ToolCallExecutionEvent | ToolCallSummaryMessage): # Ignore tool call messages. continue elif isinstance(m, StopMessage | HandoffMessage): diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py index 735e533c908e..d7029a12619a 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py @@ -15,6 +15,7 @@ TextMessage, ToolCallExecutionEvent, ToolCallRequestEvent, + ToolCallSummaryMessage, ) from ...state import SelectorManagerState from ._base_group_chat import BaseGroupChat @@ -95,7 +96,7 @@ async def select_speaker(self, thread: List[AgentEvent | ChatMessage]) -> str: # Construct the history of the conversation. history_messages: List[str] = [] for msg in thread: - if isinstance(msg, ToolCallRequestEvent | ToolCallExecutionEvent): + if isinstance(msg, ToolCallRequestEvent | ToolCallExecutionEvent | ToolCallSummaryMessage): # Ignore tool call messages. continue # The agent type must be the same as the topic type, which we use as the agent name. diff --git a/python/packages/autogen-agentchat/tests/test_assistant_agent.py b/python/packages/autogen-agentchat/tests/test_assistant_agent.py index 67969cfce8a7..9065d5139180 100644 --- a/python/packages/autogen-agentchat/tests/test_assistant_agent.py +++ b/python/packages/autogen-agentchat/tests/test_assistant_agent.py @@ -14,6 +14,7 @@ TextMessage, ToolCallExecutionEvent, ToolCallRequestEvent, + ToolCallSummaryMessage, ) from autogen_core import Image from autogen_core.tools import FunctionTool @@ -142,7 +143,7 @@ async def test_run_with_tools(monkeypatch: pytest.MonkeyPatch) -> None: assert result.messages[1].models_usage.prompt_tokens == 10 assert isinstance(result.messages[2], ToolCallExecutionEvent) assert result.messages[2].models_usage is None - assert isinstance(result.messages[3], TextMessage) + assert isinstance(result.messages[3], ToolCallSummaryMessage) assert result.messages[3].content == "pass" assert result.messages[3].models_usage is None diff --git a/python/packages/autogen-agentchat/tests/test_group_chat.py b/python/packages/autogen-agentchat/tests/test_group_chat.py index d3a0f2e56f98..6d2fe29beb8f 100644 --- a/python/packages/autogen-agentchat/tests/test_group_chat.py +++ b/python/packages/autogen-agentchat/tests/test_group_chat.py @@ -22,6 +22,7 @@ TextMessage, ToolCallExecutionEvent, ToolCallRequestEvent, + ToolCallSummaryMessage, ) from autogen_agentchat.teams import ( RoundRobinGroupChat, @@ -325,7 +326,8 @@ async def test_round_robin_group_chat_with_tools(monkeypatch: pytest.MonkeyPatch assert isinstance(result.messages[0], TextMessage) # task assert isinstance(result.messages[1], ToolCallRequestEvent) # tool call assert isinstance(result.messages[2], ToolCallExecutionEvent) # tool call result - assert isinstance(result.messages[3], TextMessage) # tool use agent response + assert isinstance(result.messages[3], ToolCallSummaryMessage) # tool use agent response + assert result.messages[3].content == "pass" # ensure the tool call was executed assert isinstance(result.messages[4], TextMessage) # echo agent response assert isinstance(result.messages[5], TextMessage) # tool use agent response assert isinstance(result.messages[6], TextMessage) # echo agent response From a6cab09bed5379ed3f58675b5c56f44bc17af033 Mon Sep 17 00:00:00 2001 From: jspv Date: Thu, 19 Dec 2024 16:36:16 -0500 Subject: [PATCH 7/7] PR Feedback --- .../_group_chat/_magentic_one/_magentic_one_orchestrator.py | 4 ++-- .../teams/_group_chat/_selector_group_chat.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py index 4f06f5d1a938..d405bab5b13e 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py @@ -428,13 +428,13 @@ def _thread_to_context(self) -> List[LLMMessage]: """Convert the message thread to a context for the model.""" context: List[LLMMessage] = [] for m in self._message_thread: - if isinstance(m, ToolCallRequestEvent | ToolCallExecutionEvent | ToolCallSummaryMessage): + if isinstance(m, ToolCallRequestEvent | ToolCallExecutionEvent): # Ignore tool call messages. continue elif isinstance(m, StopMessage | HandoffMessage): context.append(UserMessage(content=m.content, source=m.source)) elif m.source == self._name: - assert isinstance(m, TextMessage) + assert isinstance(m, TextMessage | ToolCallSummaryMessage) context.append(AssistantMessage(content=m.content, source=m.source)) else: assert isinstance(m, TextMessage) or isinstance(m, MultiModalMessage) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py index d7029a12619a..be8ec726c301 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_selector_group_chat.py @@ -96,12 +96,12 @@ async def select_speaker(self, thread: List[AgentEvent | ChatMessage]) -> str: # Construct the history of the conversation. history_messages: List[str] = [] for msg in thread: - if isinstance(msg, ToolCallRequestEvent | ToolCallExecutionEvent | ToolCallSummaryMessage): + if isinstance(msg, ToolCallRequestEvent | ToolCallExecutionEvent): # Ignore tool call messages. continue # The agent type must be the same as the topic type, which we use as the agent name. message = f"{msg.source}:" - if isinstance(msg, TextMessage | StopMessage | HandoffMessage): + if isinstance(msg, TextMessage | StopMessage | HandoffMessage | ToolCallSummaryMessage): message += f" {msg.content}" elif isinstance(msg, MultiModalMessage): for item in msg.content: