From ea51382fdb611afdd950d7acfdef29b73297b954 Mon Sep 17 00:00:00 2001 From: Hk669 Date: Thu, 11 Apr 2024 20:14:21 +0530 Subject: [PATCH 1/6] gather_usage_summary has been updated --- autogen/agentchat/utils.py | 6 +++--- autogen/code_utils.py | 2 +- test/agentchat/test_agent_usage.py | 11 ++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py index eef3741605d8..6e43c4146782 100644 --- a/autogen/agentchat/utils.py +++ b/autogen/agentchat/utils.py @@ -26,14 +26,14 @@ def consolidate_chat_info(chat_info, uniform_sender=None) -> None: ), "llm client must be set in either the recipient or sender when summary_method is reflection_with_llm." -def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str, any]]: +def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, any], Dict[str, any]]: r"""Gather usage summary from all agents. Args: agents: (list): List of agents. Returns: - tuple: (total_usage_summary, actual_usage_summary) + dictionary: (total_usage_summary, actual_usage_summary) Example: @@ -77,7 +77,7 @@ def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, An aggregate_summary(total_usage_summary, agent.client.total_usage_summary) aggregate_summary(actual_usage_summary, agent.client.actual_usage_summary) - return total_usage_summary, actual_usage_summary + return {"total_usage_summary": total_usage_summary, "actual_usage_summary": actual_usage_summary} def parse_tags_from_content(tag: str, content: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Dict[str, str]]]: diff --git a/autogen/code_utils.py b/autogen/code_utils.py index d4b2ae99cf01..a63de473352e 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -40,7 +40,7 @@ def content_str(content: Union[str, List[Union[UserMessageTextContentPart, UserMessageImageContentPart]], None]) -> str: - """Converts the `content` field of an OpenAI merssage into a string format. + """Converts the `content` field of an OpenAI message into a string format. This function processes content that may be a string, a list of mixed text and image URLs, or None, and converts it into a string. Text is directly appended to the result string, while image URLs are diff --git a/test/agentchat/test_agent_usage.py b/test/agentchat/test_agent_usage.py index 212c5513e62f..2357b8b48327 100755 --- a/test/agentchat/test_agent_usage.py +++ b/test/agentchat/test_agent_usage.py @@ -62,11 +62,11 @@ def test_gathering(): "gpt-4": {"cost": 0.3, "prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}, } - total_usage, _ = gather_usage_summary([assistant1, assistant2, assistant3]) + total_usage = gather_usage_summary([assistant1, assistant2, assistant3]) - assert round(total_usage["total_cost"], 8) == 0.6 - assert round(total_usage["gpt-35-turbo"]["cost"], 8) == 0.3 - assert round(total_usage["gpt-4"]["cost"], 8) == 0.3 + assert round(total_usage["total_usage_summary"]["total_cost"], 8) == 0.6 + assert round(total_usage["total_usage_summary"]["gpt-35-turbo"]["cost"], 8) == 0.3 + assert round(total_usage["total_usage_summary"]["gpt-4"]["cost"], 8) == 0.3 # test when agent doesn't have client user_proxy = UserProxyAgent( @@ -77,7 +77,8 @@ def test_gathering(): default_auto_reply="That's all. Thank you.", ) - total_usage, acutal_usage = gather_usage_summary([user_proxy]) + total_usage = gather_usage_summary([user_proxy]) + total_usage_summary = total_usage["total_usage_summary"] @pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") From 56e032ccf1164604691b3ef6f58bc8a50d60e231 Mon Sep 17 00:00:00 2001 From: Hk669 Date: Sun, 14 Apr 2024 11:49:48 +0530 Subject: [PATCH 2/6] updated cost info to 'usage_including_cached_inference' and 'usage_excluding_cached_inference' --- autogen/agentchat/chat.py | 6 +-- autogen/agentchat/utils.py | 39 ++++++++++++-------- notebook/agentchat_agentoptimizer.ipynb | 2 +- notebook/agentchat_cost_token_tracking.ipynb | 5 ++- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.py index a07f3302ae9a..9cb823a435af 100644 --- a/autogen/agentchat/chat.py +++ b/autogen/agentchat/chat.py @@ -25,9 +25,9 @@ class ChatResult: """The chat history.""" summary: str = None """A summary obtained from the chat.""" - cost: tuple = None # (dict, dict) - (total_cost, actual_cost_with_cache) - """The cost of the chat. a tuple of (total_cost, total_actual_cost), where total_cost is a - dictionary of cost information, and total_actual_cost is a dictionary of information on + cost: Dict[dict,dict] = None # {dict, dict} - {usage_including_cached_inference, usage_excluding_cached_inference} + """The cost of the chat. a dictionary of (usage_including_cached_inference, usage_excluding_cached_inference), where usage_including_cached_inference is a + dictionary of cost information, and usage_excluding_cached_inference is a dictionary of information on the actual incurred cost with cache.""" human_input: List[str] = None """A list of human input solicited during the chat.""" diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py index 6e43c4146782..14750f138056 100644 --- a/autogen/agentchat/utils.py +++ b/autogen/agentchat/utils.py @@ -33,26 +33,35 @@ def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, any], Dict[str, agents: (list): List of agents. Returns: - dictionary: (total_usage_summary, actual_usage_summary) + dictionary: {usage_including_cached_inference, usage_excluding_cached_inference} Example: ```python - total_usage_summary = { - "total_cost": 0.0006090000000000001, - "gpt-35-turbo": { - "cost": 0.0006090000000000001, - "prompt_tokens": 242, - "completion_tokens": 123, - "total_tokens": 365 + { + usage_including_cached_inference : { + "total_cost": 0.0006090000000000001, + "gpt-35-turbo": { + "cost": 0.0006090000000000001, + "prompt_tokens": 242, + "completion_tokens": 123, + "total_tokens": 365 + }, + + usage_excluding_cached_inference : { + "total_cost": 0.0006090000000000001, + "gpt-35-turbo": { + "cost": 0.0006090000000000001, + "prompt_tokens": 242, + "completion_tokens": 123, + "total_tokens": 365 } } ``` Note: - `actual_usage_summary` follows the same format. - If none of the agents incurred any cost (not having a client), then the total_usage_summary and actual_usage_summary will be `{'total_cost': 0}`. + If none of the agents incurred any cost (not having a client), then the usage_including_cached_inference and usage_excluding_cached_inference will be `{'total_cost': 0}`. """ def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, Any]) -> None: @@ -69,15 +78,15 @@ def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, An usage_summary[model]["completion_tokens"] += data.get("completion_tokens", 0) usage_summary[model]["total_tokens"] += data.get("total_tokens", 0) - total_usage_summary = {"total_cost": 0} - actual_usage_summary = {"total_cost": 0} + usage_including_cached_inference = {"total_cost": 0} + usage_excluding_cached_inference = {"total_cost": 0} for agent in agents: if getattr(agent, "client", None): - aggregate_summary(total_usage_summary, agent.client.total_usage_summary) - aggregate_summary(actual_usage_summary, agent.client.actual_usage_summary) + aggregate_summary(usage_excluding_cached_inference, agent.client.total_usage_summary) + aggregate_summary(usage_including_cached_inference, agent.client.actual_usage_summary) - return {"total_usage_summary": total_usage_summary, "actual_usage_summary": actual_usage_summary} + return {"usage_including_cached_inference": usage_including_cached_inference, "usage_excluding_cached_inference": usage_excluding_cached_inference} def parse_tags_from_content(tag: str, content: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Dict[str, str]]]: diff --git a/notebook/agentchat_agentoptimizer.ipynb b/notebook/agentchat_agentoptimizer.ipynb index 9b45a6c3b827..ff5d0742d210 100644 --- a/notebook/agentchat_agentoptimizer.ipynb +++ b/notebook/agentchat_agentoptimizer.ipynb @@ -454,7 +454,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.13" } }, "nbformat": 4, diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index 7334f2e36bbf..93046b751d77 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -494,8 +494,9 @@ } ], "source": [ - "total_usage_summary, actual_usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n", - "total_usage_summary" + "usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n", + "usage_summary['usage_including_cached_inference']\n", + "usage_summary['usage_excluding_cached_inference']" ] } ], From fadbde4cdee58b657bee9341980165d1af0b853a Mon Sep 17 00:00:00 2001 From: Hk669 Date: Sun, 14 Apr 2024 12:13:46 +0530 Subject: [PATCH 3/6] fix: pre-commit formatting for cost_info --- autogen/agentchat/chat.py | 2 +- autogen/agentchat/utils.py | 5 ++++- .../agentchat_auto_feedback_from_code_execution.ipynb | 1 + notebook/agentchat_cost_token_tracking.ipynb | 4 ++-- notebook/agentchat_groupchat_stateflow.ipynb | 1 - notebook/agentchat_image_generation_capability.ipynb | 1 + test/agentchat/test_agent_usage.py | 10 ++++++---- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.py index 9cb823a435af..708c0ad9a5db 100644 --- a/autogen/agentchat/chat.py +++ b/autogen/agentchat/chat.py @@ -25,7 +25,7 @@ class ChatResult: """The chat history.""" summary: str = None """A summary obtained from the chat.""" - cost: Dict[dict,dict] = None # {dict, dict} - {usage_including_cached_inference, usage_excluding_cached_inference} + cost: Dict[dict, dict] = None # {dict, dict} - {usage_including_cached_inference, usage_excluding_cached_inference} """The cost of the chat. a dictionary of (usage_including_cached_inference, usage_excluding_cached_inference), where usage_including_cached_inference is a dictionary of cost information, and usage_excluding_cached_inference is a dictionary of information on the actual incurred cost with cache.""" diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py index 14750f138056..42eb20813bc9 100644 --- a/autogen/agentchat/utils.py +++ b/autogen/agentchat/utils.py @@ -86,7 +86,10 @@ def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, An aggregate_summary(usage_excluding_cached_inference, agent.client.total_usage_summary) aggregate_summary(usage_including_cached_inference, agent.client.actual_usage_summary) - return {"usage_including_cached_inference": usage_including_cached_inference, "usage_excluding_cached_inference": usage_excluding_cached_inference} + return { + "usage_including_cached_inference": usage_including_cached_inference, + "usage_excluding_cached_inference": usage_excluding_cached_inference, + } def parse_tags_from_content(tag: str, content: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Dict[str, str]]]: diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynb index 834e1e7df1df..37584acdcc92 100644 --- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb +++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb @@ -692,6 +692,7 @@ " file_content = \"No data found.\"\n", " return \"Analyze the data and write a brief but engaging blog post. \\n Data: \\n\" + file_content\n", "\n", + "\n", "# followup of the previous question\n", "chat_res = user_proxy.initiate_chat(\n", " recipient=assistant,\n", diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index 93046b751d77..aaae878849d4 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -495,8 +495,8 @@ ], "source": [ "usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n", - "usage_summary['usage_including_cached_inference']\n", - "usage_summary['usage_excluding_cached_inference']" + "usage_summary[\"usage_including_cached_inference\"]\n", + "usage_summary[\"usage_excluding_cached_inference\"]" ] } ], diff --git a/notebook/agentchat_groupchat_stateflow.ipynb b/notebook/agentchat_groupchat_stateflow.ipynb index 6205e1147ee4..c824046f73af 100644 --- a/notebook/agentchat_groupchat_stateflow.ipynb +++ b/notebook/agentchat_groupchat_stateflow.ipynb @@ -112,7 +112,6 @@ ")\n", "\n", "\n", - "\n", "coder = autogen.AssistantAgent(\n", " name=\"Retrieve_Action_1\",\n", " llm_config=gpt4_config,\n", diff --git a/notebook/agentchat_image_generation_capability.ipynb b/notebook/agentchat_image_generation_capability.ipynb index 7c0c366a5f00..b5d298d7f4d2 100644 --- a/notebook/agentchat_image_generation_capability.ipynb +++ b/notebook/agentchat_image_generation_capability.ipynb @@ -135,6 +135,7 @@ " return content[\"text\"].rstrip().endswith(\"TERMINATE\")\n", " return False\n", "\n", + "\n", "def critic_agent() -> autogen.ConversableAgent:\n", " return autogen.ConversableAgent(\n", " name=\"critic\",\n", diff --git a/test/agentchat/test_agent_usage.py b/test/agentchat/test_agent_usage.py index 2357b8b48327..2711a3661b56 100755 --- a/test/agentchat/test_agent_usage.py +++ b/test/agentchat/test_agent_usage.py @@ -64,9 +64,9 @@ def test_gathering(): total_usage = gather_usage_summary([assistant1, assistant2, assistant3]) - assert round(total_usage["total_usage_summary"]["total_cost"], 8) == 0.6 - assert round(total_usage["total_usage_summary"]["gpt-35-turbo"]["cost"], 8) == 0.3 - assert round(total_usage["total_usage_summary"]["gpt-4"]["cost"], 8) == 0.3 + assert round(total_usage["usage_including_cached_inference"]["total_cost"], 8) == 0.6 + assert round(total_usage["usage_including_cached_inference"]["gpt-35-turbo"]["cost"], 8) == 0.3 + assert round(total_usage["usage_including_cached_inference"]["gpt-4"]["cost"], 8) == 0.3 # test when agent doesn't have client user_proxy = UserProxyAgent( @@ -78,7 +78,9 @@ def test_gathering(): ) total_usage = gather_usage_summary([user_proxy]) - total_usage_summary = total_usage["total_usage_summary"] + total_usage_summary = total_usage["usage_including_cached_inference"] + + print("Total usage summary:", total_usage_summary) @pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") From 87f65937ef394a7ae6c243b16543bad1f9266bf3 Mon Sep 17 00:00:00 2001 From: Hk669 Date: Sun, 14 Apr 2024 19:50:39 +0530 Subject: [PATCH 4/6] improved cost explanation and doc --- autogen/agentchat/chat.py | 10 ++++++---- autogen/agentchat/utils.py | 12 ++++++++---- notebook/agentchat_cost_token_tracking.ipynb | 3 +-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.py index 708c0ad9a5db..ba68f456aca3 100644 --- a/autogen/agentchat/chat.py +++ b/autogen/agentchat/chat.py @@ -25,10 +25,12 @@ class ChatResult: """The chat history.""" summary: str = None """A summary obtained from the chat.""" - cost: Dict[dict, dict] = None # {dict, dict} - {usage_including_cached_inference, usage_excluding_cached_inference} - """The cost of the chat. a dictionary of (usage_including_cached_inference, usage_excluding_cached_inference), where usage_including_cached_inference is a - dictionary of cost information, and usage_excluding_cached_inference is a dictionary of information on - the actual incurred cost with cache.""" + cost: Dict[str, dict] = None # keys: "usage_including_cached_inference", "usage_excluding_cached_inference" + """The cost of the chat. + The value for each usage type is a dictionary containing cost information for that specific type. + - "usage_including_cached_inference": Cost information on the actual incurred cost with cache. + - "usage_excluding_cached_inference": Cost information on the incurred cost without cache. + """ human_input: List[str] = None """A list of human input solicited during the chat.""" diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py index 42eb20813bc9..a9d780ad71cb 100644 --- a/autogen/agentchat/utils.py +++ b/autogen/agentchat/utils.py @@ -26,20 +26,22 @@ def consolidate_chat_info(chat_info, uniform_sender=None) -> None: ), "llm client must be set in either the recipient or sender when summary_method is reflection_with_llm." -def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, any], Dict[str, any]]: +def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, Dict], Dict[str, Dict]]: r"""Gather usage summary from all agents. Args: agents: (list): List of agents. Returns: - dictionary: {usage_including_cached_inference, usage_excluding_cached_inference} + dictionary: A dictionary containing two keys: + - "usage_including_cached_inference": Usage summary including cached inference. + - "usage_excluding_cached_inference": Usage summary excluding cached inference. Example: ```python { - usage_including_cached_inference : { + "usage_including_cached_inference" : { "total_cost": 0.0006090000000000001, "gpt-35-turbo": { "cost": 0.0006090000000000001, @@ -47,14 +49,16 @@ def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, any], Dict[str, "completion_tokens": 123, "total_tokens": 365 }, + }, - usage_excluding_cached_inference : { + "usage_excluding_cached_inference" : { "total_cost": 0.0006090000000000001, "gpt-35-turbo": { "cost": 0.0006090000000000001, "prompt_tokens": 242, "completion_tokens": 123, "total_tokens": 365 + }, } } ``` diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index aaae878849d4..457d585178f3 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -495,8 +495,7 @@ ], "source": [ "usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n", - "usage_summary[\"usage_including_cached_inference\"]\n", - "usage_summary[\"usage_excluding_cached_inference\"]" + "usage_summary[\"usage_including_cached_inference\"]" ] } ], From 8edd704a54a623ad19b9c4b136f1d10489d1627d Mon Sep 17 00:00:00 2001 From: Hk669 Date: Sun, 14 Apr 2024 22:21:15 +0530 Subject: [PATCH 5/6] improved cost info doc --- autogen/agentchat/chat.py | 4 ++-- autogen/agentchat/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.py index ba68f456aca3..10ad8014ed95 100644 --- a/autogen/agentchat/chat.py +++ b/autogen/agentchat/chat.py @@ -28,8 +28,8 @@ class ChatResult: cost: Dict[str, dict] = None # keys: "usage_including_cached_inference", "usage_excluding_cached_inference" """The cost of the chat. The value for each usage type is a dictionary containing cost information for that specific type. - - "usage_including_cached_inference": Cost information on the actual incurred cost with cache. - - "usage_excluding_cached_inference": Cost information on the incurred cost without cache. + - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference. + - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference". """ human_input: List[str] = None """A list of human input solicited during the chat.""" diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py index a9d780ad71cb..3b7387763e94 100644 --- a/autogen/agentchat/utils.py +++ b/autogen/agentchat/utils.py @@ -34,8 +34,8 @@ def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, Dict], Dict[str, Returns: dictionary: A dictionary containing two keys: - - "usage_including_cached_inference": Usage summary including cached inference. - - "usage_excluding_cached_inference": Usage summary excluding cached inference. + - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference. + - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference". Example: From 97856b98f1c4fcfe0074b55f6559892d74cbfc05 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Sun, 14 Apr 2024 20:06:27 +0000 Subject: [PATCH 6/6] include - exclude --- autogen/agentchat/utils.py | 6 +++--- test/agentchat/test_agent_usage.py | 11 ++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py index 3b7387763e94..b32c2f5f0a07 100644 --- a/autogen/agentchat/utils.py +++ b/autogen/agentchat/utils.py @@ -1,5 +1,5 @@ import re -from typing import Any, Callable, Dict, List, Tuple, Union +from typing import Any, Callable, Dict, List, Union from .agent import Agent @@ -87,8 +87,8 @@ def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, An for agent in agents: if getattr(agent, "client", None): - aggregate_summary(usage_excluding_cached_inference, agent.client.total_usage_summary) - aggregate_summary(usage_including_cached_inference, agent.client.actual_usage_summary) + aggregate_summary(usage_including_cached_inference, agent.client.total_usage_summary) + aggregate_summary(usage_excluding_cached_inference, agent.client.actual_usage_summary) return { "usage_including_cached_inference": usage_including_cached_inference, diff --git a/test/agentchat/test_agent_usage.py b/test/agentchat/test_agent_usage.py index 2711a3661b56..e31672cddd76 100755 --- a/test/agentchat/test_agent_usage.py +++ b/test/agentchat/test_agent_usage.py @@ -1,21 +1,18 @@ #!/usr/bin/env python3 -m pytest import io +import os +import sys from contextlib import redirect_stdout import pytest -from conftest import skip_openai from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST import autogen from autogen import AssistantAgent, UserProxyAgent, gather_usage_summary -try: - import openai -except ImportError: - skip = True -else: - skip = False or skip_openai +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from conftest import skip_openai as skip # noqa: E402 @pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")