diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.py index a07f3302ae9a..10ad8014ed95 100644 --- a/autogen/agentchat/chat.py +++ b/autogen/agentchat/chat.py @@ -25,10 +25,12 @@ class ChatResult: """The chat history.""" summary: str = None """A summary obtained from the chat.""" - cost: tuple = None # (dict, dict) - (total_cost, actual_cost_with_cache) - """The cost of the chat. a tuple of (total_cost, total_actual_cost), where total_cost is a - dictionary of cost information, and total_actual_cost is a dictionary of information on - the actual incurred cost with cache.""" + cost: Dict[str, dict] = None # keys: "usage_including_cached_inference", "usage_excluding_cached_inference" + """The cost of the chat. + The value for each usage type is a dictionary containing cost information for that specific type. + - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference. + - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference". + """ human_input: List[str] = None """A list of human input solicited during the chat.""" diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py index eef3741605d8..b32c2f5f0a07 100644 --- a/autogen/agentchat/utils.py +++ b/autogen/agentchat/utils.py @@ -1,5 +1,5 @@ import re -from typing import Any, Callable, Dict, List, Tuple, Union +from typing import Any, Callable, Dict, List, Union from .agent import Agent @@ -26,33 +26,46 @@ def consolidate_chat_info(chat_info, uniform_sender=None) -> None: ), "llm client must be set in either the recipient or sender when summary_method is reflection_with_llm." -def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str, any]]: +def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, Dict], Dict[str, Dict]]: r"""Gather usage summary from all agents. Args: agents: (list): List of agents. Returns: - tuple: (total_usage_summary, actual_usage_summary) + dictionary: A dictionary containing two keys: + - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference. + - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference". Example: ```python - total_usage_summary = { - "total_cost": 0.0006090000000000001, - "gpt-35-turbo": { - "cost": 0.0006090000000000001, - "prompt_tokens": 242, - "completion_tokens": 123, - "total_tokens": 365 + { + "usage_including_cached_inference" : { + "total_cost": 0.0006090000000000001, + "gpt-35-turbo": { + "cost": 0.0006090000000000001, + "prompt_tokens": 242, + "completion_tokens": 123, + "total_tokens": 365 + }, + }, + + "usage_excluding_cached_inference" : { + "total_cost": 0.0006090000000000001, + "gpt-35-turbo": { + "cost": 0.0006090000000000001, + "prompt_tokens": 242, + "completion_tokens": 123, + "total_tokens": 365 + }, } } ``` Note: - `actual_usage_summary` follows the same format. - If none of the agents incurred any cost (not having a client), then the total_usage_summary and actual_usage_summary will be `{'total_cost': 0}`. + If none of the agents incurred any cost (not having a client), then the usage_including_cached_inference and usage_excluding_cached_inference will be `{'total_cost': 0}`. """ def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, Any]) -> None: @@ -69,15 +82,18 @@ def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, An usage_summary[model]["completion_tokens"] += data.get("completion_tokens", 0) usage_summary[model]["total_tokens"] += data.get("total_tokens", 0) - total_usage_summary = {"total_cost": 0} - actual_usage_summary = {"total_cost": 0} + usage_including_cached_inference = {"total_cost": 0} + usage_excluding_cached_inference = {"total_cost": 0} for agent in agents: if getattr(agent, "client", None): - aggregate_summary(total_usage_summary, agent.client.total_usage_summary) - aggregate_summary(actual_usage_summary, agent.client.actual_usage_summary) + aggregate_summary(usage_including_cached_inference, agent.client.total_usage_summary) + aggregate_summary(usage_excluding_cached_inference, agent.client.actual_usage_summary) - return total_usage_summary, actual_usage_summary + return { + "usage_including_cached_inference": usage_including_cached_inference, + "usage_excluding_cached_inference": usage_excluding_cached_inference, + } def parse_tags_from_content(tag: str, content: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Dict[str, str]]]: diff --git a/autogen/code_utils.py b/autogen/code_utils.py index aa75756e04a4..5057c8615ea9 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -41,7 +41,7 @@ def content_str(content: Union[str, List[Union[UserMessageTextContentPart, UserMessageImageContentPart]], None]) -> str: - """Converts the `content` field of an OpenAI merssage into a string format. + """Converts the `content` field of an OpenAI message into a string format. This function processes content that may be a string, a list of mixed text and image URLs, or None, and converts it into a string. Text is directly appended to the result string, while image URLs are diff --git a/notebook/agentchat_agentoptimizer.ipynb b/notebook/agentchat_agentoptimizer.ipynb index dd56244588a8..924f1a557395 100644 --- a/notebook/agentchat_agentoptimizer.ipynb +++ b/notebook/agentchat_agentoptimizer.ipynb @@ -459,7 +459,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.13" } }, "nbformat": 4, diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynb index 834e1e7df1df..37584acdcc92 100644 --- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb +++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb @@ -692,6 +692,7 @@ " file_content = \"No data found.\"\n", " return \"Analyze the data and write a brief but engaging blog post. \\n Data: \\n\" + file_content\n", "\n", + "\n", "# followup of the previous question\n", "chat_res = user_proxy.initiate_chat(\n", " recipient=assistant,\n", diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index 7334f2e36bbf..457d585178f3 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -494,8 +494,8 @@ } ], "source": [ - "total_usage_summary, actual_usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n", - "total_usage_summary" + "usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n", + "usage_summary[\"usage_including_cached_inference\"]" ] } ], diff --git a/notebook/agentchat_groupchat_stateflow.ipynb b/notebook/agentchat_groupchat_stateflow.ipynb index 6205e1147ee4..c824046f73af 100644 --- a/notebook/agentchat_groupchat_stateflow.ipynb +++ b/notebook/agentchat_groupchat_stateflow.ipynb @@ -112,7 +112,6 @@ ")\n", "\n", "\n", - "\n", "coder = autogen.AssistantAgent(\n", " name=\"Retrieve_Action_1\",\n", " llm_config=gpt4_config,\n", diff --git a/notebook/agentchat_image_generation_capability.ipynb b/notebook/agentchat_image_generation_capability.ipynb index 7c0c366a5f00..b5d298d7f4d2 100644 --- a/notebook/agentchat_image_generation_capability.ipynb +++ b/notebook/agentchat_image_generation_capability.ipynb @@ -135,6 +135,7 @@ " return content[\"text\"].rstrip().endswith(\"TERMINATE\")\n", " return False\n", "\n", + "\n", "def critic_agent() -> autogen.ConversableAgent:\n", " return autogen.ConversableAgent(\n", " name=\"critic\",\n", diff --git a/test/agentchat/test_agent_usage.py b/test/agentchat/test_agent_usage.py index 212c5513e62f..e31672cddd76 100755 --- a/test/agentchat/test_agent_usage.py +++ b/test/agentchat/test_agent_usage.py @@ -1,21 +1,18 @@ #!/usr/bin/env python3 -m pytest import io +import os +import sys from contextlib import redirect_stdout import pytest -from conftest import skip_openai from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST import autogen from autogen import AssistantAgent, UserProxyAgent, gather_usage_summary -try: - import openai -except ImportError: - skip = True -else: - skip = False or skip_openai +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from conftest import skip_openai as skip # noqa: E402 @pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") @@ -62,11 +59,11 @@ def test_gathering(): "gpt-4": {"cost": 0.3, "prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}, } - total_usage, _ = gather_usage_summary([assistant1, assistant2, assistant3]) + total_usage = gather_usage_summary([assistant1, assistant2, assistant3]) - assert round(total_usage["total_cost"], 8) == 0.6 - assert round(total_usage["gpt-35-turbo"]["cost"], 8) == 0.3 - assert round(total_usage["gpt-4"]["cost"], 8) == 0.3 + assert round(total_usage["usage_including_cached_inference"]["total_cost"], 8) == 0.6 + assert round(total_usage["usage_including_cached_inference"]["gpt-35-turbo"]["cost"], 8) == 0.3 + assert round(total_usage["usage_including_cached_inference"]["gpt-4"]["cost"], 8) == 0.3 # test when agent doesn't have client user_proxy = UserProxyAgent( @@ -77,7 +74,10 @@ def test_gathering(): default_auto_reply="That's all. Thank you.", ) - total_usage, acutal_usage = gather_usage_summary([user_proxy]) + total_usage = gather_usage_summary([user_proxy]) + total_usage_summary = total_usage["usage_including_cached_inference"] + + print("Total usage summary:", total_usage_summary) @pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")