diff --git a/autogen/agentchat/contrib/capabilities/transforms.py b/autogen/agentchat/contrib/capabilities/transforms.py index bc56efd74d2..dad3fc335ed 100644 --- a/autogen/agentchat/contrib/capabilities/transforms.py +++ b/autogen/agentchat/contrib/capabilities/transforms.py @@ -1,5 +1,4 @@ import copy -import json import sys from typing import Any, Dict, List, Optional, Protocol, Tuple, Union @@ -8,8 +7,9 @@ from autogen import token_count_utils from autogen.cache import AbstractCache, Cache -from autogen.oai.openai_utils import filter_config +from autogen.types import MessageContentType +from . import transforms_util from .text_compressors import LLMLingua, TextCompressor @@ -169,7 +169,7 @@ def apply_transform(self, messages: List[Dict]) -> List[Dict]: assert self._min_tokens is not None # if the total number of tokens in the messages is less than the min_tokens, return the messages as is - if not _min_tokens_reached(messages, self._min_tokens): + if not transforms_util.min_tokens_reached(messages, self._min_tokens): return messages temp_messages = copy.deepcopy(messages) @@ -178,13 +178,13 @@ def apply_transform(self, messages: List[Dict]) -> List[Dict]: for msg in reversed(temp_messages): # Some messages may not have content. - if not _is_content_right_type(msg.get("content")): + if not transforms_util.is_content_right_type(msg.get("content")): processed_messages.insert(0, msg) continue - if not _should_transform_message(msg, self._filter_dict, self._exclude_filter): + if not transforms_util.should_transform_message(msg, self._filter_dict, self._exclude_filter): processed_messages.insert(0, msg) - processed_messages_tokens += _count_tokens(msg["content"]) + processed_messages_tokens += transforms_util.count_text_tokens(msg["content"]) continue expected_tokens_remained = self._max_tokens - processed_messages_tokens - self._max_tokens_per_message @@ -199,7 +199,7 @@ def apply_transform(self, messages: List[Dict]) -> List[Dict]: break msg["content"] = self._truncate_str_to_tokens(msg["content"], self._max_tokens_per_message) - msg_tokens = _count_tokens(msg["content"]) + msg_tokens = transforms_util.count_text_tokens(msg["content"]) # prepend the message to the list to preserve order processed_messages_tokens += msg_tokens @@ -209,10 +209,10 @@ def apply_transform(self, messages: List[Dict]) -> List[Dict]: def get_logs(self, pre_transform_messages: List[Dict], post_transform_messages: List[Dict]) -> Tuple[str, bool]: pre_transform_messages_tokens = sum( - _count_tokens(msg["content"]) for msg in pre_transform_messages if "content" in msg + transforms_util.count_text_tokens(msg["content"]) for msg in pre_transform_messages if "content" in msg ) post_transform_messages_tokens = sum( - _count_tokens(msg["content"]) for msg in post_transform_messages if "content" in msg + transforms_util.count_text_tokens(msg["content"]) for msg in post_transform_messages if "content" in msg ) if post_transform_messages_tokens < pre_transform_messages_tokens: @@ -349,31 +349,32 @@ def apply_transform(self, messages: List[Dict]) -> List[Dict]: return messages # if the total number of tokens in the messages is less than the min_tokens, return the messages as is - if not _min_tokens_reached(messages, self._min_tokens): + if not transforms_util.min_tokens_reached(messages, self._min_tokens): return messages total_savings = 0 processed_messages = messages.copy() for message in processed_messages: # Some messages may not have content. - if not _is_content_right_type(message.get("content")): + if not transforms_util.is_content_right_type(message.get("content")): continue - if not _should_transform_message(message, self._filter_dict, self._exclude_filter): + if not transforms_util.should_transform_message(message, self._filter_dict, self._exclude_filter): continue - if _is_content_text_empty(message["content"]): + if transforms_util.is_content_text_empty(message["content"]): continue - cached_content = self._cache_get(message["content"]) + cache_key = transforms_util.cache_key(message["content"], self._min_tokens) + cached_content = transforms_util.cache_content_get(self._cache, cache_key) if cached_content is not None: - savings, compressed_content = cached_content + message["content"], savings = cached_content else: - savings, compressed_content = self._compress(message["content"]) + message["content"], savings = self._compress(message["content"]) - self._cache_set(message["content"], compressed_content, savings) + transforms_util.cache_content_set(self._cache, cache_key, message["content"], savings) - message["content"] = compressed_content + assert isinstance(savings, int) total_savings += savings self._recent_tokens_savings = total_savings @@ -385,24 +386,29 @@ def get_logs(self, pre_transform_messages: List[Dict], post_transform_messages: else: return "No tokens saved with text compression.", False - def _compress(self, content: Union[str, List[Dict]]) -> Tuple[int, Union[str, List[Dict]]]: + def _compress(self, content: MessageContentType) -> Tuple[MessageContentType, int]: """Compresses the given text or multimodal content using the specified compression method.""" if isinstance(content, str): return self._compress_text(content) elif isinstance(content, list): return self._compress_multimodal(content) else: - return 0, content + return content, 0 - def _compress_multimodal(self, content: List[Dict]) -> Tuple[int, List[Dict]]: + def _compress_multimodal(self, content: MessageContentType) -> Tuple[MessageContentType, int]: tokens_saved = 0 - for msg in content: - if "text" in msg: - savings, msg["text"] = self._compress_text(msg["text"]) + for item in content: + if isinstance(item, dict) and "text" in item: + item["text"], savings = self._compress_text(item["text"]) + tokens_saved += savings + + elif isinstance(item, str): + item, savings = self._compress_text(item) tokens_saved += savings - return tokens_saved, content - def _compress_text(self, text: str) -> Tuple[int, str]: + return content, tokens_saved + + def _compress_text(self, text: str) -> Tuple[str, int]: """Compresses the given text using the specified compression method.""" compressed_text = self._text_compressor.compress_text(text, **self._compression_args) @@ -410,63 +416,8 @@ def _compress_text(self, text: str) -> Tuple[int, str]: if "origin_tokens" in compressed_text and "compressed_tokens" in compressed_text: savings = compressed_text["origin_tokens"] - compressed_text["compressed_tokens"] - return savings, compressed_text["compressed_prompt"] - - def _cache_get(self, content: Union[str, List[Dict]]) -> Optional[Tuple[int, Union[str, List[Dict]]]]: - if self._cache: - cached_value = self._cache.get(self._cache_key(content)) - if cached_value: - return cached_value - - def _cache_set( - self, content: Union[str, List[Dict]], compressed_content: Union[str, List[Dict]], tokens_saved: int - ): - if self._cache: - value = (tokens_saved, compressed_content) - self._cache.set(self._cache_key(content), value) - - def _cache_key(self, content: Union[str, List[Dict]]) -> str: - return f"{json.dumps(content)}_{self._min_tokens}" + return compressed_text["compressed_prompt"], savings def _validate_min_tokens(self, min_tokens: Optional[int]): if min_tokens is not None and min_tokens <= 0: raise ValueError("min_tokens must be greater than 0 or None") - - -def _min_tokens_reached(messages: List[Dict], min_tokens: Optional[int]) -> bool: - """Returns True if the total number of tokens in the messages is greater than or equal to the specified value.""" - if not min_tokens: - return True - - messages_tokens = sum(_count_tokens(msg["content"]) for msg in messages if "content" in msg) - return messages_tokens >= min_tokens - - -def _count_tokens(content: Union[str, List[Dict[str, Any]]]) -> int: - token_count = 0 - if isinstance(content, str): - token_count = token_count_utils.count_token(content) - elif isinstance(content, list): - for item in content: - token_count += _count_tokens(item.get("text", "")) - return token_count - - -def _is_content_right_type(content: Any) -> bool: - return isinstance(content, (str, list)) - - -def _is_content_text_empty(content: Union[str, List[Dict[str, Any]]]) -> bool: - if isinstance(content, str): - return content == "" - elif isinstance(content, list): - return all(_is_content_text_empty(item.get("text", "")) for item in content) - else: - return False - - -def _should_transform_message(message: Dict[str, Any], filter_dict: Optional[Dict[str, Any]], exclude: bool) -> bool: - if not filter_dict: - return True - - return len(filter_config([message], filter_dict, exclude)) > 0 diff --git a/autogen/agentchat/contrib/capabilities/transforms_util.py b/autogen/agentchat/contrib/capabilities/transforms_util.py new file mode 100644 index 00000000000..8678dec654c --- /dev/null +++ b/autogen/agentchat/contrib/capabilities/transforms_util.py @@ -0,0 +1,114 @@ +from typing import Any, Dict, Hashable, List, Optional, Tuple + +from autogen import token_count_utils +from autogen.cache.abstract_cache_base import AbstractCache +from autogen.oai.openai_utils import filter_config +from autogen.types import MessageContentType + + +def cache_key(content: MessageContentType, *args: Hashable) -> str: + """Calculates the cache key for the given message content and any other hashable args. + + Args: + content (MessageContentType): The message content to calculate the cache key for. + *args: Any additional hashable args to include in the cache key. + """ + str_keys = [str(key) for key in (content, *args)] + return "".join(str_keys) + + +def cache_content_get(cache: Optional[AbstractCache], key: str) -> Optional[Tuple[MessageContentType, ...]]: + """Retrieves cachedd content from the cache. + + Args: + cache (None or AbstractCache): The cache to retrieve the content from. If None, the cache is ignored. + key (str): The key to retrieve the content from. + """ + if cache: + cached_value = cache.get(key) + if cached_value: + return cached_value + + +def cache_content_set(cache: Optional[AbstractCache], key: str, content: MessageContentType, *extra_values): + """Sets content into the cache. + + Args: + cache (None or AbstractCache): The cache to set the content into. If None, the cache is ignored. + key (str): The key to set the content into. + content (MessageContentType): The message content to set into the cache. + *extra_values: Additional values to be passed to the cache. + """ + if cache: + cache_value = (content, *extra_values) + cache.set(key, cache_value) + + +def min_tokens_reached(messages: List[Dict], min_tokens: Optional[int]) -> bool: + """Returns True if the total number of tokens in the messages is greater than or equal to the specified value. + + Args: + messages (List[Dict]): A list of messages to check. + """ + if not min_tokens: + return True + + messages_tokens = sum(count_text_tokens(msg["content"]) for msg in messages if "content" in msg) + return messages_tokens >= min_tokens + + +def count_text_tokens(content: MessageContentType) -> int: + """Calculates the number of text tokens in the given message content. + + Args: + content (MessageContentType): The message content to calculate the number of text tokens for. + """ + token_count = 0 + if isinstance(content, str): + token_count = token_count_utils.count_token(content) + elif isinstance(content, list): + for item in content: + if isinstance(item, str): + token_count += token_count_utils.count_token(item) + else: + token_count += count_text_tokens(item.get("text", "")) + return token_count + + +def is_content_right_type(content: Any) -> bool: + """A helper function to check if the passed in content is of the right type.""" + return isinstance(content, (str, list)) + + +def is_content_text_empty(content: MessageContentType) -> bool: + """Checks if the content of the message does not contain any text. + + Args: + content (MessageContentType): The message content to check. + """ + if isinstance(content, str): + return content == "" + elif isinstance(content, list): + texts = [] + for item in content: + if isinstance(item, str): + texts.append(item) + elif isinstance(item, dict): + texts.append(item.get("text", "")) + return not any(texts) + else: + return True + + +def should_transform_message(message: Dict[str, Any], filter_dict: Optional[Dict[str, Any]], exclude: bool) -> bool: + """Validates whether the transform should be applied according to the filter dictionary. + + Args: + message (Dict[str, Any]): The message to validate. + filter_dict (None or Dict[str, Any]): The filter dictionary to validate against. If None, the transform is always applied. + exclude (bool): Whether to exclude messages that match the filter dictionary. + """ + if not filter_dict: + return True + + return len(filter_config([message], filter_dict, exclude)) > 0 diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index aeea27cff67..b434fc648eb 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -31,7 +31,7 @@ from ..function_utils import get_function_schema, load_basemodels_if_needed, serialize_to_str from ..io.base import IOStream from ..oai.client import ModelClient, OpenAIWrapper -from ..runtime_logging import log_event, log_new_agent, logging_enabled +from ..runtime_logging import log_event, log_function_use, log_new_agent, logging_enabled from .agent import Agent, LLMAgent from .chat import ChatResult, a_initiate_chats, initiate_chats from .utils import consolidate_chat_info, gather_usage_summary @@ -1357,9 +1357,7 @@ def _generate_oai_reply_from_client(self, llm_client, messages, cache) -> Union[ # TODO: #1143 handle token limit exceeded error response = llm_client.create( - context=messages[-1].pop("context", None), - messages=all_messages, - cache=cache, + context=messages[-1].pop("context", None), messages=all_messages, cache=cache, agent=self ) extracted_response = llm_client.extract_text_or_completion_object(response)[0] @@ -2528,13 +2526,14 @@ def _wrap_function(self, func: F) -> F: @functools.wraps(func) def _wrapped_func(*args, **kwargs): retval = func(*args, **kwargs) - + log_function_use(self, func, kwargs, retval) return serialize_to_str(retval) @load_basemodels_if_needed @functools.wraps(func) async def _a_wrapped_func(*args, **kwargs): retval = await func(*args, **kwargs) + log_function_use(self, func, kwargs, retval) return serialize_to_str(retval) wrapped_func = _a_wrapped_func if inspect.iscoroutinefunction(func) else _wrapped_func diff --git a/autogen/agentchat/groupchat.py b/autogen/agentchat/groupchat.py index f6e5d30264b..3e01139a1cb 100644 --- a/autogen/agentchat/groupchat.py +++ b/autogen/agentchat/groupchat.py @@ -1160,7 +1160,7 @@ async def a_run_chat( def resume( self, messages: Union[List[Dict], str], - remove_termination_string: str = None, + remove_termination_string: Union[str, Callable[[str], str]] = None, silent: Optional[bool] = False, ) -> Tuple[ConversableAgent, Dict]: """Resumes a group chat using the previous messages as a starting point. Requires the agents, group chat, and group chat manager to be established @@ -1168,7 +1168,9 @@ def resume( Args: - messages Union[List[Dict], str]: The content of the previous chat's messages, either as a Json string or a list of message dictionaries. - - remove_termination_string str: Remove the provided string from the last message to prevent immediate termination + - remove_termination_string (str or function): Remove the termination string from the last message to prevent immediate termination + If a string is provided, this string will be removed from last message. + If a function is provided, the last message will be passed to this function. - silent (bool or None): (Experimental) whether to print the messages for this conversation. Default is False. Returns: @@ -1263,7 +1265,7 @@ def resume( async def a_resume( self, messages: Union[List[Dict], str], - remove_termination_string: str = None, + remove_termination_string: Union[str, Callable[[str], str]], silent: Optional[bool] = False, ) -> Tuple[ConversableAgent, Dict]: """Resumes a group chat using the previous messages as a starting point, asynchronously. Requires the agents, group chat, and group chat manager to be established @@ -1271,7 +1273,9 @@ async def a_resume( Args: - messages Union[List[Dict], str]: The content of the previous chat's messages, either as a Json string or a list of message dictionaries. - - remove_termination_string str: Remove the provided string from the last message to prevent immediate termination + - remove_termination_string (str or function): Remove the termination string from the last message to prevent immediate termination + If a string is provided, this string will be removed from last message. + If a function is provided, the last message will be passed to this function, and the function returns the string after processing. - silent (bool or None): (Experimental) whether to print the messages for this conversation. Default is False. Returns: @@ -1390,11 +1394,15 @@ def _valid_resume_messages(self, messages: List[Dict]): ): raise Exception(f"Agent name in message doesn't exist as agent in group chat: {message['name']}") - def _process_resume_termination(self, remove_termination_string: str, messages: List[Dict]): + def _process_resume_termination( + self, remove_termination_string: Union[str, Callable[[str], str]], messages: List[Dict] + ): """Removes termination string, if required, and checks if termination may occur. args: - remove_termination_string (str): termination string to remove from the last message + remove_termination_string (str or function): Remove the termination string from the last message to prevent immediate termination + If a string is provided, this string will be removed from last message. + If a function is provided, the last message will be passed to this function, and the function returns the string after processing. returns: None @@ -1403,9 +1411,17 @@ def _process_resume_termination(self, remove_termination_string: str, messages: last_message = messages[-1] # Replace any given termination string in the last message - if remove_termination_string: - if messages[-1].get("content") and remove_termination_string in messages[-1]["content"]: - messages[-1]["content"] = messages[-1]["content"].replace(remove_termination_string, "") + if isinstance(remove_termination_string, str): + + def _remove_termination_string(content: str) -> str: + return content.replace(remove_termination_string, "") + + else: + _remove_termination_string = remove_termination_string + + if _remove_termination_string: + if messages[-1].get("content"): + messages[-1]["content"] = _remove_termination_string(messages[-1]["content"]) # Check if the last message meets termination (if it has one) if self._is_termination_msg: diff --git a/autogen/logger/base_logger.py b/autogen/logger/base_logger.py index 7c35f8a5091..c5c236fa4ae 100644 --- a/autogen/logger/base_logger.py +++ b/autogen/logger/base_logger.py @@ -3,7 +3,7 @@ import sqlite3 import uuid from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Dict, List, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, TypeVar, Union from openai import AzureOpenAI, OpenAI from openai.types.chat import ChatCompletion @@ -11,6 +11,7 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper +F = TypeVar("F", bound=Callable[..., Any]) ConfigItem = Dict[str, Union[str, List[str]]] LLMConfig = Dict[str, Union[None, float, int, ConfigItem, List[ConfigItem]]] @@ -32,6 +33,7 @@ def log_chat_completion( invocation_id: uuid.UUID, client_id: int, wrapper_id: int, + source: Union[str, Agent], request: Dict[str, Union[float, str, List[Dict[str, str]]]], response: Union[str, ChatCompletion], is_cached: int, @@ -49,9 +51,10 @@ def log_chat_completion( invocation_id (uuid): A unique identifier for the invocation to the OpenAIWrapper.create method call client_id (int): A unique identifier for the underlying OpenAI client instance wrapper_id (int): A unique identifier for the OpenAIWrapper instance - request (dict): A dictionary representing the the request or call to the OpenAI client endpoint + source (str or Agent): The source/creator of the event as a string name or an Agent instance + request (dict): A dictionary representing the request or call to the OpenAI client endpoint response (str or ChatCompletion): The response from OpenAI - is_chached (int): 1 if the response was a cache hit, 0 otherwise + is_cached (int): 1 if the response was a cache hit, 0 otherwise cost(float): The cost for OpenAI response start_time (str): A string representing the moment the request was initiated """ @@ -104,6 +107,18 @@ def log_new_client( """ ... + @abstractmethod + def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[str, Any], returns: Any) -> None: + """ + Log the use of a registered function (could be a tool) + + Args: + source (str or Agent): The source/creator of the event as a string name or an Agent instance + function (F): The function information + args (dict): The function args to log + returns (any): The return + """ + @abstractmethod def stop(self) -> None: """ diff --git a/autogen/logger/file_logger.py b/autogen/logger/file_logger.py index fedcc134ab9..15b2c457e42 100644 --- a/autogen/logger/file_logger.py +++ b/autogen/logger/file_logger.py @@ -5,7 +5,7 @@ import os import threading import uuid -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union from openai import AzureOpenAI, OpenAI from openai.types.chat import ChatCompletion @@ -21,9 +21,21 @@ logger = logging.getLogger(__name__) +F = TypeVar("F", bound=Callable[..., Any]) + __all__ = ("FileLogger",) +def safe_serialize(obj: Any) -> str: + def default(o: Any) -> str: + if hasattr(o, "to_json"): + return str(o.to_json()) + else: + return f"<>" + + return json.dumps(obj, default=default) + + class FileLogger(BaseLogger): def __init__(self, config: Dict[str, Any]): self.config = config @@ -59,6 +71,7 @@ def log_chat_completion( invocation_id: uuid.UUID, client_id: int, wrapper_id: int, + source: Union[str, Agent], request: Dict[str, Union[float, str, List[Dict[str, str]]]], response: Union[str, ChatCompletion], is_cached: int, @@ -69,6 +82,11 @@ def log_chat_completion( Log a chat completion. """ thread_id = threading.get_ident() + source_name = None + if isinstance(source, str): + source_name = source + else: + source_name = source.name try: log_data = json.dumps( { @@ -82,6 +100,7 @@ def log_chat_completion( "start_time": start_time, "end_time": get_current_ts(), "thread_id": thread_id, + "source_name": source_name, } ) @@ -204,6 +223,29 @@ def log_new_client( except Exception as e: self.logger.error(f"[file_logger] Failed to log event {e}") + def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[str, Any], returns: Any) -> None: + """ + Log a registered function(can be a tool) use from an agent or a string source. + """ + thread_id = threading.get_ident() + + try: + log_data = json.dumps( + { + "source_id": id(source), + "source_name": str(source.name) if hasattr(source, "name") else source, + "agent_module": source.__module__, + "agent_class": source.__class__.__name__, + "timestamp": get_current_ts(), + "thread_id": thread_id, + "input_args": safe_serialize(args), + "returns": safe_serialize(returns), + } + ) + self.logger.info(log_data) + except Exception as e: + self.logger.error(f"[file_logger] Failed to log event {e}") + def get_connection(self) -> None: """Method is intentionally left blank because there is no specific connection needed for the FileLogger.""" pass diff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.py index 42db83d849d..4ebea32cc19 100644 --- a/autogen/logger/sqlite_logger.py +++ b/autogen/logger/sqlite_logger.py @@ -6,7 +6,7 @@ import sqlite3 import threading import uuid -from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, TypeVar, Union from openai import AzureOpenAI, OpenAI from openai.types.chat import ChatCompletion @@ -25,6 +25,18 @@ __all__ = ("SqliteLogger",) +F = TypeVar("F", bound=Callable[..., Any]) + + +def safe_serialize(obj: Any) -> str: + def default(o: Any) -> str: + if hasattr(o, "to_json"): + return str(o.to_json()) + else: + return f"<>" + + return json.dumps(obj, default=default) + class SqliteLogger(BaseLogger): schema_version = 1 @@ -49,6 +61,7 @@ def start(self) -> str: client_id INTEGER, wrapper_id INTEGER, session_id TEXT, + source_name TEXT, request TEXT, response TEXT, is_cached INEGER, @@ -118,6 +131,18 @@ class TEXT, -- type or class name of cli """ self._run_query(query=query) + query = """ + CREATE TABLE IF NOT EXISTS function_calls ( + source_id INTEGER, + source_name TEXT, + function_name TEXT, + args TEXT DEFAULT NULL, + returns TEXT DEFAULT NULL, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP + ); + """ + self._run_query(query=query) + current_verion = self._get_current_db_version() if current_verion is None: self._run_query( @@ -192,6 +217,7 @@ def log_chat_completion( invocation_id: uuid.UUID, client_id: int, wrapper_id: int, + source: Union[str, Agent], request: Dict[str, Union[float, str, List[Dict[str, str]]]], response: Union[str, ChatCompletion], is_cached: int, @@ -208,10 +234,16 @@ def log_chat_completion( else: response_messages = json.dumps(to_dict(response), indent=4) + source_name = None + if isinstance(source, str): + source_name = source + else: + source_name = source.name + query = """ INSERT INTO chat_completions ( - invocation_id, client_id, wrapper_id, session_id, request, response, is_cached, cost, start_time, end_time - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + invocation_id, client_id, wrapper_id, session_id, request, response, is_cached, cost, start_time, end_time, source_name + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ args = ( invocation_id, @@ -224,6 +256,7 @@ def log_chat_completion( cost, start_time, end_time, + source_name, ) self._run_query(query=query, args=args) @@ -236,7 +269,16 @@ def log_new_agent(self, agent: ConversableAgent, init_args: Dict[str, Any]) -> N args = to_dict( init_args, - exclude=("self", "__class__", "api_key", "organization", "base_url", "azure_endpoint"), + exclude=( + "self", + "__class__", + "api_key", + "organization", + "base_url", + "azure_endpoint", + "azure_ad_token", + "azure_ad_token_provider", + ), no_recursive=(Agent,), ) @@ -301,7 +343,17 @@ def log_new_wrapper(self, wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLM return args = to_dict( - init_args, exclude=("self", "__class__", "api_key", "organization", "base_url", "azure_endpoint") + init_args, + exclude=( + "self", + "__class__", + "api_key", + "organization", + "base_url", + "azure_endpoint", + "azure_ad_token", + "azure_ad_token_provider", + ), ) query = """ @@ -316,6 +368,24 @@ def log_new_wrapper(self, wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLM ) self._run_query(query=query, args=args) + def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[str, Any], returns: Any) -> None: + + if self.con is None: + return + + query = """ + INSERT INTO function_calls (source_id, source_name, function_name, args, returns, timestamp) VALUES (?, ?, ?, ?, ?, ?) + """ + query_args: Tuple[Any, ...] = ( + id(source), + source.name if hasattr(source, "name") else source, + function.__name__, + safe_serialize(args), + safe_serialize(returns), + get_current_ts(), + ) + self._run_query(query=query, args=query_args) + def log_new_client( self, client: Union[AzureOpenAI, OpenAI, GeminiClient], wrapper: OpenAIWrapper, init_args: Dict[str, Any] ) -> None: @@ -323,7 +393,17 @@ def log_new_client( return args = to_dict( - init_args, exclude=("self", "__class__", "api_key", "organization", "base_url", "azure_endpoint") + init_args, + exclude=( + "self", + "__class__", + "api_key", + "organization", + "base_url", + "azure_endpoint", + "azure_ad_token", + "azure_ad_token_provider", + ), ) query = """ diff --git a/autogen/oai/client.py b/autogen/oai/client.py index 982d1c0d57f..4c1da7a3931 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -319,6 +319,7 @@ class OpenAIWrapper: """A wrapper class for openai client.""" extra_kwargs = { + "agent", "cache", "cache_seed", "filter_func", @@ -407,6 +408,14 @@ def _configure_azure_openai(self, config: Dict[str, Any], openai_config: Dict[st openai_config["azure_deployment"] = openai_config["azure_deployment"].replace(".", "") openai_config["azure_endpoint"] = openai_config.get("azure_endpoint", openai_config.pop("base_url", None)) + # Create a default Azure token provider if requested + if openai_config.get("azure_ad_token_provider") == "DEFAULT": + import azure.identity + + openai_config["azure_ad_token_provider"] = azure.identity.get_bearer_token_provider( + azure.identity.DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default" + ) + def _register_default_client(self, config: Dict[str, Any], openai_config: Dict[str, Any]) -> None: """Create a client with the given config to override openai_config, after removing extra kwargs. @@ -534,6 +543,7 @@ def create(self, **config: Any) -> ModelClient.ModelClientResponseProtocol: Note that the cache argument overrides the legacy cache_seed argument: if this argument is provided, then the cache_seed argument is ignored. If this argument is not provided or None, then the cache_seed argument is used. + - agent (AbstractAgent | None): The object responsible for creating a completion if an agent. - (Legacy) cache_seed (int | None) for using the DiskCache. Default to 41. An integer cache_seed is useful when implementing "controlled randomness" for the completion. None for no caching. @@ -581,6 +591,7 @@ def yes_or_no_filter(context, response): cache = extra_kwargs.get("cache") filter_func = extra_kwargs.get("filter_func") context = extra_kwargs.get("context") + agent = extra_kwargs.get("agent") total_usage = None actual_usage = None @@ -618,6 +629,7 @@ def yes_or_no_filter(context, response): invocation_id=invocation_id, client_id=id(client), wrapper_id=id(self), + agent=agent, request=params, response=response, is_cached=1, @@ -650,6 +662,7 @@ def yes_or_no_filter(context, response): invocation_id=invocation_id, client_id=id(client), wrapper_id=id(self), + agent=agent, request=params, response=f"error_code:{error_code}, config {i} failed", is_cached=0, @@ -680,6 +693,7 @@ def yes_or_no_filter(context, response): invocation_id=invocation_id, client_id=id(client), wrapper_id=id(self), + agent=agent, request=params, response=response, is_cached=0, diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 1ed347f6271..a676e964390 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -13,7 +13,7 @@ from openai.types.beta.assistant import Assistant from packaging.version import parse -NON_CACHE_KEY = ["api_key", "base_url", "api_type", "api_version"] +NON_CACHE_KEY = ["api_key", "base_url", "api_type", "api_version", "azure_ad_token", "azure_ad_token_provider"] DEFAULT_AZURE_API_VERSION = "2024-02-15-preview" OAI_PRICE1K = { # https://openai.com/api/pricing/ diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.py index ffc741482e6..6ce207a2f33 100644 --- a/autogen/runtime_logging.py +++ b/autogen/runtime_logging.py @@ -3,7 +3,7 @@ import logging import sqlite3 import uuid -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, TypeVar, Union from openai import AzureOpenAI, OpenAI from openai.types.chat import ChatCompletion @@ -20,6 +20,8 @@ autogen_logger = None is_logging = False +F = TypeVar("F", bound=Callable[..., Any]) + def start( logger: Optional[BaseLogger] = None, @@ -56,6 +58,7 @@ def log_chat_completion( invocation_id: uuid.UUID, client_id: int, wrapper_id: int, + agent: Union[str, Agent], request: Dict[str, Union[float, str, List[Dict[str, str]]]], response: Union[str, ChatCompletion], is_cached: int, @@ -67,7 +70,7 @@ def log_chat_completion( return autogen_logger.log_chat_completion( - invocation_id, client_id, wrapper_id, request, response, is_cached, cost, start_time + invocation_id, client_id, wrapper_id, agent, request, response, is_cached, cost, start_time ) @@ -87,6 +90,14 @@ def log_event(source: Union[str, Agent], name: str, **kwargs: Dict[str, Any]) -> autogen_logger.log_event(source, name, **kwargs) +def log_function_use(agent: Union[str, Agent], function: F, args: Dict[str, Any], returns: any): + if autogen_logger is None: + logger.error("[runtime logging] log_function_use: autogen logger is None") + return + + autogen_logger.log_function_use(agent, function, args, returns) + + def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLMConfig, List[LLMConfig]]]) -> None: if autogen_logger is None: logger.error("[runtime logging] log_new_wrapper: autogen logger is None") diff --git a/autogen/types.py b/autogen/types.py index 77ca70b70b9..461765a6adc 100644 --- a/autogen/types.py +++ b/autogen/types.py @@ -1,5 +1,7 @@ from typing import Dict, List, Literal, TypedDict, Union +MessageContentType = Union[str, List[Union[Dict, str]], None] + class UserMessageTextContentPart(TypedDict): type: Literal["text"] diff --git a/dotnet/website/template/public/main.js b/dotnet/website/template/public/main.js new file mode 100644 index 00000000000..df5fb0b8343 --- /dev/null +++ b/dotnet/website/template/public/main.js @@ -0,0 +1,9 @@ +export default { + iconLinks: [ + { + icon: 'github', + href: 'https://github.com/microsoft/autogen', + title: 'GitHub' + } + ] + } \ No newline at end of file diff --git a/notebook/agentchat_agentops.ipynb b/notebook/agentchat_agentops.ipynb new file mode 100644 index 00000000000..293efa8e4bd --- /dev/null +++ b/notebook/agentchat_agentops.ipynb @@ -0,0 +1,533 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "abb8a01d85d8b146", + "metadata": { + "collapsed": false + }, + "source": [ + "# AgentOps" + ] + }, + { + "cell_type": "markdown", + "id": "a447802c88c8a240", + "metadata": {}, + "source": [ + "![logo](https://raw.githubusercontent.com/AgentOps-AI/agentops/35d5682866921a9e28d8ef66ae3c3b3d92d8fa6b/img/logo.png)\n", + "\n", + "[AgentOps](https://agentops.ai/?=autogen) provides session replays, metrics, and monitoring for AI agents.\n", + "\n", + "At a high level, AgentOps gives you the ability to monitor LLM calls, costs, latency, agent failures, multi-agent interactions, tool usage, session-wide statistics, and more. For more info, check out the [AgentOps Repo](https://github.com/AgentOps-AI/agentops).\n" + ] + }, + { + "cell_type": "markdown", + "id": "b354c068", + "metadata": {}, + "source": [ + "### Dashboard\n", + "![Agent Dashboard](https://github.com/AgentOps-AI/agentops/assets/14807319/158e082a-9a7d-49b7-9b41-51a49a1f7d3d)" + ] + }, + { + "cell_type": "markdown", + "id": "38182a5296dceb34", + "metadata": {}, + "source": [ + "## Adding AgentOps to an existing Autogen service.\n", + "To get started, you'll need to install the AgentOps package and set an API key.\n", + "\n", + "AgentOps automatically configures itself when it's initialized. This means your agents will be tracked and logged to your AgentOps account right away." + ] + }, + { + "cell_type": "markdown", + "id": "8d9451f4", + "metadata": {}, + "source": [ + "````{=mdx}\n", + ":::info Requirements\n", + "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", + "\n", + "```bash\n", + "pip install pyautogen agentops\n", + "```\n", + "\n", + "For more information, please refer to the [installation guide](/docs/installation/).\n", + ":::\n", + "````" + ] + }, + { + "cell_type": "markdown", + "id": "6be9e11620b0e8d6", + "metadata": {}, + "source": [ + "### Set an API key\n", + "\n", + "By default, the AgentOps `init()` function will look for an environment variable named `AGENTOPS_API_KEY`. Alternatively, you can pass one in as an optional parameter.\n", + "\n", + "Create an account and API key at [AgentOps.ai](https://agentops.ai/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "f31a28d20a13b377", + "metadata": { + "ExecuteTime": { + "end_time": "2024-05-31T22:48:27.679318Z", + "start_time": "2024-05-31T22:48:26.192071Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001B[0m\u001B[0m\n" + ] + }, + { + "data": { + "text/plain": "UUID('8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4')" + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import agentops\n", + "\n", + "from autogen import ConversableAgent, UserProxyAgent, config_list_from_json\n", + "\n", + "agentops.init(api_key=\"7c94212b-b89d-47a6-a20c-23b2077d3226\") # or agentops.init(api_key=\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "4dd8f461ccd9cbef", + "metadata": {}, + "source": [ + "Autogen will now start automatically tracking\n", + "- LLM prompts and completions\n", + "- Token usage and costs\n", + "- Agent names and actions\n", + "- Correspondence between agents\n", + "- Tool usage\n", + "- Errors" + ] + }, + { + "cell_type": "markdown", + "id": "712315c520536eb8", + "metadata": {}, + "source": [ + "# Simple Chat Example" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "66d68e66e9f4a677", + "metadata": { + "ExecuteTime": { + "end_time": "2024-05-31T22:48:32.813123Z", + "start_time": "2024-05-31T22:48:27.677564Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33magent\u001B[0m (to user):\n", + "\n", + "How can I help you today?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33muser\u001B[0m (to agent):\n", + "\n", + "2+2\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33magent\u001B[0m (to user):\n", + "\n", + "2 + 2 equals 4.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "๐Ÿ–‡ AgentOps: This run's cost $0.000960\n", + "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001B[0m\u001B[0m\n" + ] + } + ], + "source": [ + "import agentops\n", + "\n", + "# When initializing AgentOps, you can pass in optional tags to help filter sessions\n", + "agentops.init(tags=[\"simple-autogen-example\"])\n", + "\n", + "# Create the agent that uses the LLM.\n", + "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n", + "assistant = ConversableAgent(\"agent\", llm_config={\"config_list\": config_list})\n", + "\n", + "# Create the agent that represents the user in the conversation.\n", + "user_proxy = UserProxyAgent(\"user\", code_execution_config=False)\n", + "\n", + "# Let the assistant start the conversation. It will end when the user types exit.\n", + "assistant.initiate_chat(user_proxy, message=\"How can I help you today?\")\n", + "\n", + "# Close your AgentOps session to indicate that it completed.\n", + "agentops.end_session(\"Success\")" + ] + }, + { + "cell_type": "markdown", + "id": "2217ed0f930cfcaa", + "metadata": {}, + "source": [ + "You can view data on this run at [app.agentops.ai](https://app.agentops.ai). \n", + "\n", + "The dashboard will display LLM events for each message sent by each agent, including those made by the human user." + ] + }, + { + "cell_type": "markdown", + "source": [ + "![Session_Overview](https://github.com/AgentOps-AI/agentops/assets/14807319/d7228019-1488-40d3-852f-a61e998658ad)" + ], + "metadata": { + "collapsed": false + }, + "id": "cbd689b0f5617013" + }, + { + "cell_type": "markdown", + "id": "fd78f1a816276cb7", + "metadata": {}, + "source": [ + "# Tool Example\n", + "AgentOps also tracks when Autogen agents use tools. You can find more information on this example in [tool-use.ipynb](https://github.com/microsoft/autogen/blob/main/website/docs/tutorial/tool-use.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3498aa6176c799ff", + "metadata": { + "ExecuteTime": { + "end_time": "2024-05-31T22:48:35.808674Z", + "start_time": "2024-05-31T22:48:32.813225Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001B[0m\u001B[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "What is (1423 - 123) / 3 + (32 + 23) * 5?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "\u001B[32m***** Suggested tool call (call_aINcGyo0Xkrh9g7buRuhyCz0): calculator *****\u001B[0m\n", + "Arguments: \n", + "{\n", + " \"a\": 1423,\n", + " \"b\": 123,\n", + " \"operator\": \"-\"\n", + "}\n", + "\u001B[32m***************************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[32m***** Response from calling tool (call_aINcGyo0Xkrh9g7buRuhyCz0) *****\u001B[0m\n", + "1300\n", + "\u001B[32m**********************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "\u001B[32m***** Suggested tool call (call_prJGf8V0QVT7cbD91e0Fcxpb): calculator *****\u001B[0m\n", + "Arguments: \n", + "{\n", + " \"a\": 1300,\n", + " \"b\": 3,\n", + " \"operator\": \"/\"\n", + "}\n", + "\u001B[32m***************************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[32m***** Response from calling tool (call_prJGf8V0QVT7cbD91e0Fcxpb) *****\u001B[0m\n", + "433\n", + "\u001B[32m**********************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/braelynboynton/Developer/agentops/autogen/autogen/agentchat/conversable_agent.py:2489: UserWarning: Function 'calculator' is being overridden.\n", + " warnings.warn(f\"Function '{tool_sig['function']['name']}' is being overridden.\", UserWarning)\n", + "/Users/braelynboynton/Developer/agentops/autogen/autogen/agentchat/conversable_agent.py:2408: UserWarning: Function 'calculator' is being overridden.\n", + " warnings.warn(f\"Function '{name}' is being overridden.\", UserWarning)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "\u001B[32m***** Suggested tool call (call_CUIgHRsySLjayDKuUphI1TGm): calculator *****\u001B[0m\n", + "Arguments: \n", + "{\n", + " \"a\": 32,\n", + " \"b\": 23,\n", + " \"operator\": \"+\"\n", + "}\n", + "\u001B[32m***************************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[32m***** Response from calling tool (call_CUIgHRsySLjayDKuUphI1TGm) *****\u001B[0m\n", + "55\n", + "\u001B[32m**********************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "\u001B[32m***** Suggested tool call (call_L7pGtBLUf9V0MPL90BASyesr): calculator *****\u001B[0m\n", + "Arguments: \n", + "{\n", + " \"a\": 55,\n", + " \"b\": 5,\n", + " \"operator\": \"*\"\n", + "}\n", + "\u001B[32m***************************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[32m***** Response from calling tool (call_L7pGtBLUf9V0MPL90BASyesr) *****\u001B[0m\n", + "275\n", + "\u001B[32m**********************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "\u001B[32m***** Suggested tool call (call_Ygo6p4XfcxRjkYBflhG3UVv6): calculator *****\u001B[0m\n", + "Arguments: \n", + "{\n", + " \"a\": 433,\n", + " \"b\": 275,\n", + " \"operator\": \"+\"\n", + "}\n", + "\u001B[32m***************************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[32m***** Response from calling tool (call_Ygo6p4XfcxRjkYBflhG3UVv6) *****\u001B[0m\n", + "708\n", + "\u001B[32m**********************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "The result of the calculation is 708.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "๐Ÿ–‡ AgentOps: This run's cost $0.001800\n", + "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001B[0m\u001B[0m\n" + ] + } + ], + "source": [ + "from typing import Annotated, Literal\n", + "\n", + "from autogen import ConversableAgent, config_list_from_json, register_function\n", + "\n", + "agentops.start_session(tags=[\"autogen-tool-example\"])\n", + "\n", + "Operator = Literal[\"+\", \"-\", \"*\", \"/\"]\n", + "\n", + "\n", + "def calculator(a: int, b: int, operator: Annotated[Operator, \"operator\"]) -> int:\n", + " if operator == \"+\":\n", + " return a + b\n", + " elif operator == \"-\":\n", + " return a - b\n", + " elif operator == \"*\":\n", + " return a * b\n", + " elif operator == \"/\":\n", + " return int(a / b)\n", + " else:\n", + " raise ValueError(\"Invalid operator\")\n", + "\n", + "\n", + "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n", + "\n", + "# Create the agent that uses the LLM.\n", + "assistant = ConversableAgent(\n", + " name=\"Assistant\",\n", + " system_message=\"You are a helpful AI assistant. \"\n", + " \"You can help with simple calculations. \"\n", + " \"Return 'TERMINATE' when the task is done.\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# The user proxy agent is used for interacting with the assistant agent\n", + "# and executes tool calls.\n", + "user_proxy = ConversableAgent(\n", + " name=\"User\",\n", + " llm_config=False,\n", + " is_termination_msg=lambda msg: msg.get(\"content\") is not None and \"TERMINATE\" in msg[\"content\"],\n", + " human_input_mode=\"NEVER\",\n", + ")\n", + "\n", + "assistant.register_for_llm(name=\"calculator\", description=\"A simple calculator\")(calculator)\n", + "user_proxy.register_for_execution(name=\"calculator\")(calculator)\n", + "\n", + "# Register the calculator function to the two agents.\n", + "register_function(\n", + " calculator,\n", + " caller=assistant, # The assistant agent can suggest calls to the calculator.\n", + " executor=user_proxy, # The user proxy agent can execute the calculator calls.\n", + " name=\"calculator\", # By default, the function name is used as the tool name.\n", + " description=\"A simple calculator\", # A description of the tool.\n", + ")\n", + "\n", + "# Let the assistant start the conversation. It will end when the user types exit.\n", + "user_proxy.initiate_chat(assistant, message=\"What is (1423 - 123) / 3 + (32 + 23) * 5?\")\n", + "\n", + "agentops.end_session(\"Success\")" + ] + }, + { + "cell_type": "markdown", + "id": "2b4edf8e70d17267", + "metadata": {}, + "source": [ + "You can see your run in action at [app.agentops.ai](https://app.agentops.ai). In this example, the AgentOps dashboard will show:\n", + "- Agents talking to each other\n", + "- Each use of the `calculator` tool\n", + "- Each call to OpenAI for LLM use" + ] + }, + { + "cell_type": "markdown", + "source": [ + "![Session Drilldown](https://github.com/AgentOps-AI/agentops/assets/14807319/561d59f3-c441-4066-914b-f6cfe32a598c)" + ], + "metadata": { + "collapsed": false + }, + "id": "a922a52ab5fce31" + } + ], + "metadata": { + "front_matter": { + "description": "Use AgentOps to simplify the development process and monitor your agents in production.", + "tags": [ + "monitoring", + "debugging" + ] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynb index bf784889d61..6ea6f662b93 100644 --- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb +++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb @@ -40,7 +40,7 @@ " filter_dict={\"tags\": [\"gpt-4\"]}, # comment out to get all\n", ")\n", "# When using a single openai endpoint, you can use the following:\n", - "# config_list = [{\"model\": \"gpt-4\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]\n" + "# config_list = [{\"model\": \"gpt-4\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]" ] }, { diff --git a/notebook/agentchat_logging.ipynb b/notebook/agentchat_logging.ipynb index 7eb4138b4cc..eb5a6e752e4 100644 --- a/notebook/agentchat_logging.ipynb +++ b/notebook/agentchat_logging.ipynb @@ -25,12 +25,12 @@ "output_type": "stream", "text": [ "Logging session ID: 6e08f3e0-392b-434e-8b69-4ab36c4fcf99\n", - "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\u001B[33muser_proxy\u001B[0m (to assistant):\n", "\n", "What is the height of the Eiffel Tower? Only respond with the answer and terminate\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\u001B[33massistant\u001B[0m (to user_proxy):\n", "\n", "The height of the Eiffel Tower is approximately 330 meters.\n", "\n", @@ -313,12 +313,12 @@ "output_type": "stream", "text": [ "Logging session ID: ed493ebf-d78e-49f0-b832-69557276d557\n", - "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\u001B[33muser_proxy\u001B[0m (to assistant):\n", "\n", "What is the height of the Eiffel Tower? Only respond with the answer and terminate\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\u001B[33massistant\u001B[0m (to user_proxy):\n", "\n", "The height of the Eiffel Tower is 330 meters.\n", "TERMINATE\n", @@ -328,7 +328,6 @@ } ], "source": [ - "\n", "import pandas as pd\n", "\n", "import autogen\n", diff --git a/samples/apps/autogen-studio/README.md b/samples/apps/autogen-studio/README.md index 1e60b5362db..05a2a58f800 100644 --- a/samples/apps/autogen-studio/README.md +++ b/samples/apps/autogen-studio/README.md @@ -12,24 +12,14 @@ Code for AutoGen Studio is on GitHub at [microsoft/autogen](https://github.com/m > **Note**: AutoGen Studio is meant to help you rapidly prototype multi-agent workflows and demonstrate an example of end user interfaces built with AutoGen. It is not meant to be a production-ready app. > [!WARNING] -> AutoGen Studio is currently under active development and we are iterating quickly. Kindly consider that we may introduce breaking changes in the releases during the upcoming weeks, and also the `README` might be outdated. We'll update the `README` as soon as we stabilize the API. +> AutoGen Studio is currently under active development and we are iterating quickly. Kindly consider that we may introduce breaking changes in the releases during the upcoming weeks, and also the `README` might be outdated. Please see the AutoGen Studio [docs](https://microsoft.github.io/autogen/docs/autogen-studio/getting-started) page for the most up-to-date information. + +**Updates** -> [!NOTE] Updates > April 17: AutoGen Studio database layer is now rewritten to use [SQLModel](https://sqlmodel.tiangolo.com/) (Pydantic + SQLAlchemy). This provides entity linking (skills, models, agents and workflows are linked via association tables) and supports multiple [database backend dialects](https://docs.sqlalchemy.org/en/20/dialects/) supported in SQLAlchemy (SQLite, PostgreSQL, MySQL, Oracle, Microsoft SQL Server). The backend database can be specified a `--database-uri` argument when running the application. For example, `autogenstudio ui --database-uri sqlite:///database.sqlite` for SQLite and `autogenstudio ui --database-uri postgresql+psycopg://user:password@localhost/dbname` for PostgreSQL. > March 12: Default directory for AutoGen Studio is now /home//.autogenstudio. You can also specify this directory using the `--appdir` argument when running the application. For example, `autogenstudio ui --appdir /path/to/folder`. This will store the database and other files in the specified directory e.g. `/path/to/folder/database.sqlite`. `.env` files in that directory will be used to set environment variables for the app. -### Capabilities / Roadmap - -Some of the capabilities supported by the app frontend include the following: - -- [x] Build / Configure agents (currently supports two agent workflows based on `UserProxyAgent` and `AssistantAgent`), modify their configuration (e.g. skills, temperature, model, agent system message, model etc) and compose them into workflows. -- [x] Chat with agent works and specify tasks. -- [x] View agent messages and output files in the UI from agent runs. -- [x] Add interaction sessions to a gallery. -- [ ] Support for more complex agent workflows (e.g. `GroupChat` workflows). -- [ ] Improved user experience (e.g., streaming intermediate model output, better summarization of agent responses, etc). - Project Structure: - _autogenstudio/_ code for the backend classes and web api (FastAPI) @@ -97,32 +87,6 @@ AutoGen Studio also takes several parameters to customize the application: Now that you have AutoGen Studio installed and running, you are ready to explore its capabilities, including defining and modifying agent workflows, interacting with agents and sessions, and expanding agent skills. -## Capabilities - -AutoGen Studio proposes some high-level concepts. - -**Agent Workflow**: An agent workflow is a specification of a set of agents that can work together to accomplish a task. The simplest version of this is a setup with two agents โ€“ a user proxy agent (that represents a user i.e. it compiles code and prints result) and an assistant that can address task requests (e.g., generating plans, writing code, evaluating responses, proposing error recovery steps, etc.). A more complex flow could be a group chat where even more agents work towards a solution. - -**Session**: A session refers to a period of continuous interaction or engagement with an agent workflow, typically characterized by a sequence of activities or operations aimed at achieving specific objectives. It includes the agent workflow configuration, the interactions between the user and the agents. A session can be โ€œpublishedโ€ to a โ€œgalleryโ€. - -**Skills**: Skills are functions (e.g., Python functions) that describe how to solve a task. In general, a good skill has a descriptive name (e.g. `generate_images`), extensive docstrings and good defaults (e.g., writing out files to disk for persistence and reuse). You can add new skills AutoGen Studio app via the provided UI. At inference time, these skills are made available to the assistant agent as they address your tasks. - -## Example Usage - -Consider the following query. - -``` -Plot a chart of NVDA and TESLA stock price YTD. Save the result to a file named nvda_tesla.png -``` - -The agent workflow responds by _writing and executing code_ to create a python program to generate the chart with the stock prices. - -> Note than there could be multiple turns between the `AssistantAgent` and the `UserProxyAgent` to produce and execute the code in order to complete the task. - -![ARA](./docs/ara_stockprices.png) - -> Note: You can also view the debug console that generates useful information to see how the agents are interacting in the background. - ## Contribution Guide We welcome contributions to AutoGen Studio. We recommend the following general steps to contribute to the project: @@ -137,29 +101,7 @@ We welcome contributions to AutoGen Studio. We recommend the following general s ## FAQ -**Q: How do I specify the directory where files(e.g. database) are stored?** - -A: You can specify the directory where files are stored by setting the `--appdir` argument when running the application. For example, `autogenstudio ui --appdir /path/to/folder`. This will store the database (default) and other files in the specified directory e.g. `/path/to/folder/database.sqlite`. - -**Q: Where can I adjust the default skills, agent and workflow configurations?** -A: You can modify agent configurations directly from the UI or by editing the [dbdefaults.json](autogenstudio/utils/dbdefaults.json) file which is used to initialize the database. - -**Q: If I want to reset the entire conversation with an agent, how do I go about it?** -A: To reset your conversation history, you can delete the `database.sqlite` file in the `--appdir` directory. This will reset the entire conversation history. To delete user files, you can delete the `files` directory in the `--appdir` directory. - -**Q: Is it possible to view the output and messages generated by the agents during interactions?** -A: Yes, you can view the generated messages in the debug console of the web UI, providing insights into the agent interactions. Alternatively, you can inspect the `database.sqlite` file for a comprehensive record of messages. - -**Q: Can I use other models with AutoGen Studio?** -Yes. AutoGen standardizes on the openai model api format, and you can use any api server that offers an openai compliant endpoint. In the AutoGen Studio UI, each agent has an `llm_config` field where you can input your model endpoint details including `model`, `api key`, `base url`, `model type` and `api version`. For Azure OpenAI models, you can find these details in the Azure portal. Note that for Azure OpenAI, the `model name` is the deployment id or engine, and the `model type` is "azure". -For other OSS models, we recommend using a server such as vllm to instantiate an openai compliant endpoint. - -**Q: The server starts but I can't access the UI** -A: If you are running the server on a remote machine (or a local machine that fails to resolve localhost correstly), you may need to specify the host address. By default, the host address is set to `localhost`. You can specify the host address using the `--host ` argument. For example, to start the server on port 8081 and local address such that it is accessible from other machines on the network, you can run the following command: - -```bash -autogenstudio ui --port 8081 --host 0.0.0.0 -``` +Please refer to the AutoGen Studio [FAQs](https://microsoft.github.io/autogen/docs/autogen-studio/faqs) page for more information. ## Acknowledgements diff --git a/test/agentchat/contrib/capabilities/test_transforms_util.py b/test/agentchat/contrib/capabilities/test_transforms_util.py new file mode 100644 index 00000000000..089ebbdc8db --- /dev/null +++ b/test/agentchat/contrib/capabilities/test_transforms_util.py @@ -0,0 +1,72 @@ +import itertools +import tempfile +from typing import Dict, Tuple + +import pytest + +from autogen.agentchat.contrib.capabilities import transforms_util +from autogen.cache.cache import Cache +from autogen.types import MessageContentType + +MESSAGES = { + "message1": { + "content": [{"text": "Hello"}, {"image_url": {"url": "https://example.com/image.jpg"}}], + "text_tokens": 1, + }, + "message2": {"content": [{"image_url": {"url": "https://example.com/image.jpg"}}], "text_tokens": 0}, + "message3": {"content": [{"text": "Hello"}, {"text": "World"}], "text_tokens": 2}, + "message4": {"content": None, "text_tokens": 0}, + "message5": {"content": "Hello there!", "text_tokens": 3}, + "message6": {"content": ["Hello there!", "Hello there!"], "text_tokens": 6}, +} + + +@pytest.mark.parametrize("message", MESSAGES.values()) +def test_cache_content(message: Dict[str, MessageContentType]) -> None: + with tempfile.TemporaryDirectory() as tmpdirname: + cache = Cache.disk(tmpdirname) + cache_key_1 = "test_string" + + transforms_util.cache_content_set(cache, cache_key_1, message["content"]) + assert transforms_util.cache_content_get(cache, cache_key_1) == (message["content"],) + + cache_key_2 = "test_list" + cache_value_2 = [message["content"], 1, "some_string", {"new_key": "new_value"}] + transforms_util.cache_content_set(cache, cache_key_2, *cache_value_2) + assert transforms_util.cache_content_get(cache, cache_key_2) == tuple(cache_value_2) + assert isinstance(cache_value_2[1], int) + assert isinstance(cache_value_2[2], str) + assert isinstance(cache_value_2[3], dict) + + cache_key_3 = "test_None" + transforms_util.cache_content_set(None, cache_key_3, message["content"]) + assert transforms_util.cache_content_get(cache, cache_key_3) is None + assert transforms_util.cache_content_get(None, cache_key_3) is None + + +@pytest.mark.parametrize("messages", itertools.product(MESSAGES.values(), MESSAGES.values())) +def test_cache_key(messages: Tuple[Dict[str, MessageContentType], Dict[str, MessageContentType]]) -> None: + message_1, message_2 = messages + cache_1 = transforms_util.cache_key(message_1["content"], 10) + cache_2 = transforms_util.cache_key(message_2["content"], 10) + if message_1 == message_2: + assert cache_1 == cache_2 + else: + assert cache_1 != cache_2 + + +@pytest.mark.parametrize("message", MESSAGES.values()) +def test_min_tokens_reached(message: Dict[str, MessageContentType]): + assert transforms_util.min_tokens_reached([message], None) + assert transforms_util.min_tokens_reached([message], 0) + assert not transforms_util.min_tokens_reached([message], message["text_tokens"] + 1) + + +@pytest.mark.parametrize("message", MESSAGES.values()) +def test_count_text_tokens(message: Dict[str, MessageContentType]): + assert transforms_util.count_text_tokens(message["content"]) == message["text_tokens"] + + +@pytest.mark.parametrize("message", MESSAGES.values()) +def test_is_content_text_empty(message: Dict[str, MessageContentType]): + assert transforms_util.is_content_text_empty(message["content"]) == (message["text_tokens"] == 0) diff --git a/test/agentchat/test_agent_file_logging.py b/test/agentchat/test_agent_file_logging.py index 9b930ba4ac9..1818b85fa9d 100644 --- a/test/agentchat/test_agent_file_logging.py +++ b/test/agentchat/test_agent_file_logging.py @@ -3,6 +3,7 @@ import sys import tempfile import uuid +from typing import Any, Callable import pytest @@ -19,6 +20,10 @@ is_windows = sys.platform.startswith("win") +def dummy_function(param1: str, param2: int) -> Any: + return param1 * param2 + + @pytest.mark.skipif(is_windows, reason="Skipping file logging tests on Windows") @pytest.fixture def logger() -> FileLogger: @@ -49,8 +54,19 @@ def test_log_chat_completion(logger: FileLogger): is_cached = 0 cost = 0.5 start_time = "2024-05-06 15:20:21.263231" - - logger.log_chat_completion(invocation_id, client_id, wrapper_id, request, response, is_cached, cost, start_time) + agent = autogen.AssistantAgent(name="TestAgent", code_execution_config=False) + + logger.log_chat_completion( + invocation_id=invocation_id, + client_id=client_id, + wrapper_id=wrapper_id, + request=request, + response=response, + is_cached=is_cached, + cost=cost, + start_time=start_time, + source=agent, + ) with open(logger.log_file, "r") as f: lines = f.readlines() @@ -63,6 +79,26 @@ def test_log_chat_completion(logger: FileLogger): assert log_data["is_cached"] == is_cached assert log_data["cost"] == cost assert log_data["start_time"] == start_time + assert log_data["source_name"] == "TestAgent" + assert isinstance(log_data["thread_id"], int) + + +@pytest.mark.skipif(is_windows, reason="Skipping file logging tests on Windows") +def test_log_function_use(logger: FileLogger): + source = autogen.AssistantAgent(name="TestAgent", code_execution_config=False) + func: Callable[[str, int], Any] = dummy_function + args = {"foo": "bar"} + returns = True + + logger.log_function_use(source=source, function=func, args=args, returns=returns) + + with open(logger.log_file, "r") as f: + lines = f.readlines() + assert len(lines) == 1 + log_data = json.loads(lines[0]) + assert log_data["source_name"] == "TestAgent" + assert log_data["input_args"] == json.dumps(args) + assert log_data["returns"] == json.dumps(returns) assert isinstance(log_data["thread_id"], int) diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index 933b4ce7df2..20a83685178 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -1916,6 +1916,51 @@ def test_manager_resume_functions(): # TERMINATE should be removed assert messages[-1]["content"] == final_msg.replace("TERMINATE", "") + # Tests termination message replacement with function + def termination_func(x: str) -> str: + if "APPROVED" in x: + x = x.replace("APPROVED", "") + else: + x = x.replace("TERMINATE", "") + return x + + final_msg1 = "Product_Manager has created 3 new product ideas. APPROVED" + messages1 = [ + { + "content": "You are an expert at finding the next speaker.", + "role": "system", + }, + { + "content": final_msg1, + "name": "Coder", + "role": "assistant", + }, + ] + + manager._process_resume_termination(remove_termination_string=termination_func, messages=messages1) + + # APPROVED should be removed + assert messages1[-1]["content"] == final_msg1.replace("APPROVED", "") + + final_msg2 = "Idea has been approved. TERMINATE" + messages2 = [ + { + "content": "You are an expert at finding the next speaker.", + "role": "system", + }, + { + "content": final_msg2, + "name": "Coder", + "role": "assistant", + }, + ] + + manager._process_resume_termination(remove_termination_string=termination_func, messages=messages2) + + # TERMINATE should be removed, "approved" should still be present as the termination_func only replaces upper-cased "APPROVED". + assert messages2[-1]["content"] == final_msg2.replace("TERMINATE", "") + assert "approved" in messages2[-1]["content"] + # Check if the termination string doesn't exist there's no replacing of content final_msg = ( "Let's get this meeting started. First the Product_Manager will create 3 new product ideas. TERMINATE this." @@ -2027,7 +2072,7 @@ def test_manager_resume_messages(): # test_clear_agents_history() # test_custom_speaker_selection_overrides_transition_graph() # test_role_for_select_speaker_messages() - test_select_speaker_message_and_prompt_templates() + # test_select_speaker_message_and_prompt_templates() # test_speaker_selection_agent_name_match() # test_role_for_reflection_summary() # test_speaker_selection_auto_process_result() @@ -2036,7 +2081,7 @@ def test_manager_resume_messages(): # test_select_speaker_auto_messages() # test_manager_messages_to_string() # test_manager_messages_from_string() - # test_manager_resume_functions() + test_manager_resume_functions() # test_manager_resume_returns() # test_manager_resume_messages() pass diff --git a/test/test_logging.py b/test/test_logging.py index d824cc91400..c6f7a182c5c 100644 --- a/test/test_logging.py +++ b/test/test_logging.py @@ -1,6 +1,7 @@ import json import sqlite3 import uuid +from typing import Any, Callable from unittest.mock import Mock, patch import pytest @@ -61,6 +62,11 @@ """ ) + +def dummy_function(param1: str, param2: int) -> Any: + return param1 * param2 + + ############################################################### @@ -84,6 +90,7 @@ def get_sample_chat_completion(response): "is_cached": 0, "cost": 0.347, "start_time": get_current_ts(), + "agent": autogen.AssistantAgent(name="TestAgent", code_execution_config=False), } @@ -103,7 +110,7 @@ def test_log_completion(response, expected_logged_response, db_connection): query = """ SELECT invocation_id, client_id, wrapper_id, request, response, is_cached, - cost, start_time FROM chat_completions + cost, start_time, source_name FROM chat_completions """ for row in cur.execute(query): @@ -115,6 +122,28 @@ def test_log_completion(response, expected_logged_response, db_connection): assert row["is_cached"] == sample_completion["is_cached"] assert row["cost"] == sample_completion["cost"] assert row["start_time"] == sample_completion["start_time"] + assert row["source_name"] == "TestAgent" + + +def test_log_function_use(db_connection): + cur = db_connection.cursor() + + source = autogen.AssistantAgent(name="TestAgent", code_execution_config=False) + func: Callable[[str, int], Any] = dummy_function + args = {"foo": "bar"} + returns = True + + autogen.runtime_logging.log_function_use(agent=source, function=func, args=args, returns=returns) + + query = """ + SELECT source_id, source_name, function_name, args, returns, timestamp + FROM function_calls + """ + + for row in cur.execute(query): + assert row["source_name"] == "TestAgent" + assert row["args"] == json.dumps(args) + assert row["returns"] == json.dumps(returns) def test_log_new_agent(db_connection): diff --git a/website/blog/2023-12-01-AutoGenStudio/index.mdx b/website/blog/2023-12-01-AutoGenStudio/index.mdx index 014aa870e0c..49151f7b355 100644 --- a/website/blog/2023-12-01-AutoGenStudio/index.mdx +++ b/website/blog/2023-12-01-AutoGenStudio/index.mdx @@ -25,6 +25,9 @@ To help you rapidly prototype multi-agent solutions for your tasks, we are intro - Explicitly add skills to your agents and accomplish more tasks. - Publish your sessions to a local gallery. + +See the official AutoGen Studio documentation [here](https://microsoft.github.io/autogen/docs/autogen-studio/getting-started) for more details. + AutoGen Studio is open source [code here](https://github.com/microsoft/autogen/tree/main/samples/apps/autogen-studio), and can be installed via pip. Give it a try! ```bash diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 5062cc5deff..2ec83d1e0f2 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -100,6 +100,9 @@ Links to notebook examples: - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) - Automatically Build Multi-agent System from Agent Library - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb) +### Observability +- Track LLM calls, tool usage, actions and errors using AgentOps - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentops.ipynb) + ## Enhanced Inferences ### Utilities diff --git a/website/docs/autogen-studio/faqs.md b/website/docs/autogen-studio/faqs.md new file mode 100644 index 00000000000..82426578daa --- /dev/null +++ b/website/docs/autogen-studio/faqs.md @@ -0,0 +1,86 @@ +# AutoGen Studio FAQs + +## Q: How do I specify the directory where files(e.g. database) are stored? + +A: You can specify the directory where files are stored by setting the `--appdir` argument when running the application. For example, `autogenstudio ui --appdir /path/to/folder`. This will store the database (default) and other files in the specified directory e.g. `/path/to/folder/database.sqlite`. + +## Q: Where can I adjust the default skills, agent and workflow configurations? + +A: You can modify agent configurations directly from the UI or by editing the `init_db_samples` function in the `autogenstudio/database/utils.py` file which is used to initialize the database. + +## Q: If I want to reset the entire conversation with an agent, how do I go about it? + +A: To reset your conversation history, you can delete the `database.sqlite` file in the `--appdir` directory. This will reset the entire conversation history. To delete user files, you can delete the `files` directory in the `--appdir` directory. + +## Q: Is it possible to view the output and messages generated by the agents during interactions? + +A: Yes, you can view the generated messages in the debug console of the web UI, providing insights into the agent interactions. Alternatively, you can inspect the `database.sqlite` file for a comprehensive record of messages. + +## Q: Can I use other models with AutoGen Studio? + +Yes. AutoGen standardizes on the openai model api format, and you can use any api server that offers an openai compliant endpoint. In the AutoGen Studio UI, each agent has an `llm_config` field where you can input your model endpoint details including `model`, `api key`, `base url`, `model type` and `api version`. For Azure OpenAI models, you can find these details in the Azure portal. Note that for Azure OpenAI, the `model name` is the deployment id or engine, and the `model type` is "azure". +For other OSS models, we recommend using a server such as vllm, LMStudio, Ollama, to instantiate an openai compliant endpoint. + +## Q: The server starts but I can't access the UI + +A: If you are running the server on a remote machine (or a local machine that fails to resolve localhost correctly), you may need to specify the host address. By default, the host address is set to `localhost`. You can specify the host address using the `--host ` argument. For example, to start the server on port 8081 and local address such that it is accessible from other machines on the network, you can run the following command: + +```bash +autogenstudio ui --port 8081 --host 0.0.0.0 +``` + +## Q: Can I export my agent workflows for use in a python app? + +Yes. In the Build view, you can click the export button to save your agent workflow as a JSON file. This file can be imported in a python application using the `WorkflowManager` class. For example: + +```python + +from autogenstudio import WorkflowManager +# load workflow from exported json workflow file. +workflow_manager = WorkflowManager(workflow="path/to/your/workflow_.json") + +# run the workflow on a task +task_query = "What is the height of the Eiffel Tower?. Dont write code, just respond to the question." +workflow_manager.run(message=task_query) + +``` + +## Q: Can I deploy my agent workflows as APIs? + +Yes. You can launch the workflow as an API endpoint from the command line using the `autogenstudio` commandline tool. For example: + +```bash +autogenstudio serve --workflow=workflow.json --port=5000 +``` + +Similarly, the workflow launch command above can be wrapped into a Dockerfile that can be deployed on cloud services like Azure Container Apps or Azure Web Apps. + +## Q: Can I run AutoGen Studio in a Docker container? + +A: Yes, you can run AutoGen Studio in a Docker container. You can build the Docker image using the provided [Dockerfile](https://github.com/microsoft/autogen/blob/autogenstudio/samples/apps/autogen-studio/Dockerfile) and run the container using the following commands: + +```bash +FROM python:3.10 + +WORKDIR /code + +RUN pip install -U gunicorn autogenstudio + +RUN useradd -m -u 1000 user +USER user +ENV HOME=/home/user \ + PATH=/home/user/.local/bin:$PATH \ + AUTOGENSTUDIO_APPDIR=/home/user/app + +WORKDIR $HOME/app + +COPY --chown=user . $HOME/app + +CMD gunicorn -w $((2 * $(getconf _NPROCESSORS_ONLN) + 1)) --timeout 12600 -k uvicorn.workers.UvicornWorker autogenstudio.web.app:app --bind "0.0.0.0:8081" +``` + +Using Gunicorn as the application server for improved performance is recommended. To run AutoGen Studio with Gunicorn, you can use the following command: + +```bash +gunicorn -w $((2 * $(getconf _NPROCESSORS_ONLN) + 1)) --timeout 12600 -k uvicorn.workers.UvicornWorker autogenstudio.web.app:app --bind +``` diff --git a/website/docs/autogen-studio/getting-started.md b/website/docs/autogen-studio/getting-started.md new file mode 100644 index 00000000000..eee1707b7cc --- /dev/null +++ b/website/docs/autogen-studio/getting-started.md @@ -0,0 +1,121 @@ +# AutoGen Studio - Getting Started + +[![PyPI version](https://badge.fury.io/py/autogenstudio.svg)](https://badge.fury.io/py/autogenstudio) +[![Downloads](https://static.pepy.tech/badge/autogenstudio/week)](https://pepy.tech/project/autogenstudio) + +![ARA](./img/ara_stockprices.png) + +AutoGen Studio is an low-code interface built to help you rapidly prototype AI agents, enhance them with skills, compose them into workflows and interact with them to accomplish tasks. It is built on top of the [AutoGen](https://microsoft.github.io/autogen) framework, which is a toolkit for building AI agents. + +Code for AutoGen Studio is on GitHub at [microsoft/autogen](https://github.com/microsoft/autogen/tree/main/samples/apps/autogen-studio) + +> **Note**: AutoGen Studio is meant to help you rapidly prototype multi-agent workflows and demonstrate an example of end user interfaces built with AutoGen. It is not meant to be a production-ready app. Developers are encouraged to use the AutoGen framework to build their own applications, implementing authentication, security and other features required for deployed applications. + +**Updates** + +- April 17: AutoGen Studio database layer is now rewritten to use [SQLModel](https://sqlmodel.tiangolo.com/) (Pydantic + SQLAlchemy). This provides entity linking (skills, models, agents and workflows are linked via association tables) and supports multiple [database backend dialects](https://docs.sqlalchemy.org/en/20/dialects/) supported in SQLAlchemy (SQLite, PostgreSQL, MySQL, Oracle, Microsoft SQL Server). The backend database can be specified with a `--database-uri` argument when running the application. For example, `autogenstudio ui --database-uri sqlite:///database.sqlite` for SQLite and `autogenstudio ui --database-uri postgresql+psycopg://user:password@localhost/dbname` for PostgreSQL. + +- March 12: Default directory for AutoGen Studio is now /home//.autogenstudio. You can also specify this directory using the `--appdir` argument when running the application. For example, `autogenstudio ui --appdir /path/to/folder`. This will store the database and other files in the specified directory e.g. `/path/to/folder/database.sqlite`. `.env` files in that directory will be used to set environment variables for the app. + +### Installation + +There are two ways to install AutoGen Studio - from PyPi or from source. We **recommend installing from PyPi** unless you plan to modify the source code. + +1. **Install from PyPi** + + We recommend using a virtual environment (e.g., conda) to avoid conflicts with existing Python packages. With Python 3.10 or newer active in your virtual environment, use pip to install AutoGen Studio: + + ```bash + pip install autogenstudio + ``` + +2. **Install from Source** + + > Note: This approach requires some familiarity with building interfaces in React. + + If you prefer to install from source, ensure you have Python 3.10+ and Node.js (version above 14.15.0) installed. Here's how you get started: + + - Clone the AutoGen Studio repository and install its Python dependencies: + + ```bash + pip install -e . + ``` + + - Navigate to the `samples/apps/autogen-studio/frontend` directory, install dependencies, and build the UI: + + ```bash + npm install -g gatsby-cli + npm install --global yarn + cd frontend + yarn install + yarn build + ``` + +For Windows users, to build the frontend, you may need alternative commands to build the frontend. + +```bash + + gatsby clean && rmdir /s /q ..\\autogenstudio\\web\\ui 2>nul & (set \"PREFIX_PATH_VALUE=\" || ver>nul) && gatsby build --prefix-paths && xcopy /E /I /Y public ..\\autogenstudio\\web\\ui + +``` + +### Running the Application + +Once installed, run the web UI by entering the following in your terminal: + +```bash +autogenstudio ui --port 8081 +``` + +This will start the application on the specified port. Open your web browser and go to `http://localhost:8081/` to begin using AutoGen Studio. + +AutoGen Studio also takes several parameters to customize the application: + +- `--host ` argument to specify the host address. By default, it is set to `localhost`. Y +- `--appdir ` argument to specify the directory where the app files (e.g., database and generated user files) are stored. By default, it is set to the a `.autogenstudio` directory in the user's home directory. +- `--port ` argument to specify the port number. By default, it is set to `8080`. +- `--reload` argument to enable auto-reloading of the server when changes are made to the code. By default, it is set to `False`. +- `--database-uri` argument to specify the database URI. Example values include `sqlite:///database.sqlite` for SQLite and `postgresql+psycopg://user:password@localhost/dbname` for PostgreSQL. If this is not specified, the database URI defaults to a `database.sqlite` file in the `--appdir` directory. + +Now that you have AutoGen Studio installed and running, you are ready to explore its capabilities, including defining and modifying agent workflows, interacting with agents and sessions, and expanding agent skills. + +### Capabilities / Roadmap + +Some of the capabilities supported by the app frontend include the following: + +- [x] Build / Configure agents (currently supports two agent workflows based on `UserProxyAgent` and `AssistantAgent`), modify their configuration (e.g. skills, temperature, model, agent system message, model etc) and compose them into workflows. +- [x] Chat with agent workflows and specify tasks. +- [x] View agent messages and output files in the UI from agent runs. +- [x] Support for more complex agent workflows (e.g. `GroupChat` and `Sequential` workflows). +- [x] Improved user experience (e.g., streaming intermediate model output, better summarization of agent responses, etc). + +Review project roadmap and issues [here](https://github.com/microsoft/autogen/issues/737) . + +Project Structure: + +- _autogenstudio/_ code for the backend classes and web api (FastAPI) +- _frontend/_ code for the webui, built with Gatsby and TailwindCSS + +## Contribution Guide + +We welcome contributions to AutoGen Studio. We recommend the following general steps to contribute to the project: + +- Review the overall AutoGen project [contribution guide](https://github.com/microsoft/autogen?tab=readme-ov-file#contributing) +- Please review the AutoGen Studio [roadmap](https://github.com/microsoft/autogen/issues/737) to get a sense of the current priorities for the project. Help is appreciated especially with Studio issues tagged with `help-wanted` +- Please initiate a discussion on the roadmap issue or a new issue to discuss your proposed contribution. +- Please review the autogenstudio dev branch here [dev branch](https://github.com/microsoft/autogen/tree/autogenstudio) and use as a base for your contribution. This way, your contribution will be aligned with the latest changes in the AutoGen Studio project. +- Submit a pull request with your contribution! +- If you are modifying AutoGen Studio, it has its own devcontainer. See instructions in `.devcontainer/README.md` to use it +- Please use the tag `studio` for any issues, questions, and PRs related to Studio + +## A Note on Security + +AutoGen Studio is a research prototype and is not meant to be used in a production environment. Some baseline practices are encouraged e.g., using Docker code execution environment for your agents. + +However, other considerations such as rigorous tests related to jailbreaking, ensuring LLMs only have access to the right keys of data given the end user's permissions, and other security features are not implemented in AutoGen Studio. + +If you are building a production application, please use the AutoGen framework and implement the necessary security features. + +## Acknowledgements + +AutoGen Studio is Based on the [AutoGen](https://microsoft.github.io/autogen) project. It was adapted from a research prototype built in October 2023 (original credits: Gagan Bansal, Adam Fourney, Victor Dibia, Piali Choudhury, Saleema Amershi, Ahmed Awadallah, Chi Wang). diff --git a/website/docs/autogen-studio/img/agent_assistant.png b/website/docs/autogen-studio/img/agent_assistant.png new file mode 100644 index 00000000000..e34e90e3665 --- /dev/null +++ b/website/docs/autogen-studio/img/agent_assistant.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd8eff59d97c9fbdf118eefe071894125d6421cad6b428c3427d61630d57a3e8 +size 133246 diff --git a/website/docs/autogen-studio/img/agent_groupchat.png b/website/docs/autogen-studio/img/agent_groupchat.png new file mode 100644 index 00000000000..516cc28c6f9 --- /dev/null +++ b/website/docs/autogen-studio/img/agent_groupchat.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:060d2bfb9c38da7535015718202e17cdd6c51545a0a986dedfe6c91bd8accb52 +size 146086 diff --git a/website/docs/autogen-studio/img/agent_new.png b/website/docs/autogen-studio/img/agent_new.png new file mode 100644 index 00000000000..696e794320a --- /dev/null +++ b/website/docs/autogen-studio/img/agent_new.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18de683a302d4cfaf218b3d57d6f72d17925b589c15ea0604e4b3bd03f6b464c +size 141037 diff --git a/website/docs/autogen-studio/img/agent_skillsmodel.png b/website/docs/autogen-studio/img/agent_skillsmodel.png new file mode 100644 index 00000000000..fea9113e503 --- /dev/null +++ b/website/docs/autogen-studio/img/agent_skillsmodel.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d85a85e931123b404ab1f3d20e2fe52a0e874479f5b36a6d56cd3ffaa0f9991b +size 147060 diff --git a/website/docs/autogen-studio/img/ara_stockprices.png b/website/docs/autogen-studio/img/ara_stockprices.png new file mode 100644 index 00000000000..f5adf6256e5 --- /dev/null +++ b/website/docs/autogen-studio/img/ara_stockprices.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e3340a765da6dff6585c8b2e8a4014df0c94b537d62d341d2d0d45627bbc345 +size 198222 diff --git a/website/docs/autogen-studio/img/model_new.png b/website/docs/autogen-studio/img/model_new.png new file mode 100644 index 00000000000..424c7e43727 --- /dev/null +++ b/website/docs/autogen-studio/img/model_new.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82cf098881c1b318aeec3858aedbc80dea3e80e6d34c0dbd36d721a8e14cc058 +size 94667 diff --git a/website/docs/autogen-studio/img/model_openai.png b/website/docs/autogen-studio/img/model_openai.png new file mode 100644 index 00000000000..9b107c60439 --- /dev/null +++ b/website/docs/autogen-studio/img/model_openai.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:696397efe3a289f5dd084a5a7fbfe3f151adb21a19be617d3e66255acc4a404d +size 90123 diff --git a/website/docs/autogen-studio/img/skill.png b/website/docs/autogen-studio/img/skill.png new file mode 100644 index 00000000000..5357fff4c05 --- /dev/null +++ b/website/docs/autogen-studio/img/skill.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f675444d66c0f6dbca9756b92d2bd166cd29eb645efadc38b7331ab891bef204 +size 232801 diff --git a/website/docs/autogen-studio/img/workflow_chat.png b/website/docs/autogen-studio/img/workflow_chat.png new file mode 100644 index 00000000000..a83462146b3 --- /dev/null +++ b/website/docs/autogen-studio/img/workflow_chat.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b14764c16149a1094ba95612e84fd28ef778485cc026a1cb4904a8c3f0b7815 +size 127639 diff --git a/website/docs/autogen-studio/img/workflow_export.png b/website/docs/autogen-studio/img/workflow_export.png new file mode 100644 index 00000000000..b8ef14c4c11 --- /dev/null +++ b/website/docs/autogen-studio/img/workflow_export.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db5e6f7171a4de9ddfeb6d1e29d7dac2464f727720438ae3433cf78ffe8b75ce +size 204265 diff --git a/website/docs/autogen-studio/img/workflow_new.png b/website/docs/autogen-studio/img/workflow_new.png new file mode 100644 index 00000000000..52f864016e1 --- /dev/null +++ b/website/docs/autogen-studio/img/workflow_new.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64614cd603aa384270075788253566a8035bd0d0011c28af0476f6e484111e4c +size 90426 diff --git a/website/docs/autogen-studio/img/workflow_profile.png b/website/docs/autogen-studio/img/workflow_profile.png new file mode 100644 index 00000000000..0464bccfc48 --- /dev/null +++ b/website/docs/autogen-studio/img/workflow_profile.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ad630cdf09112be8831c830f516a2ec061de1d0097e03d205eda982ab408a63 +size 283288 diff --git a/website/docs/autogen-studio/img/workflow_sequential.png b/website/docs/autogen-studio/img/workflow_sequential.png new file mode 100644 index 00000000000..2fe6c76f006 --- /dev/null +++ b/website/docs/autogen-studio/img/workflow_sequential.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:670715663ec78b47d53e2689ad2853e07f99ac498ed890f9bdd36c309e52758f +size 117232 diff --git a/website/docs/autogen-studio/img/workflow_test.png b/website/docs/autogen-studio/img/workflow_test.png new file mode 100644 index 00000000000..1146ecfa1f9 --- /dev/null +++ b/website/docs/autogen-studio/img/workflow_test.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0567731649d732d1bacd557b94b5eec87b8d491fa4207f4a8e29170ee56419d +size 258139 diff --git a/website/docs/autogen-studio/usage.md b/website/docs/autogen-studio/usage.md new file mode 100644 index 00000000000..e63c040fb32 --- /dev/null +++ b/website/docs/autogen-studio/usage.md @@ -0,0 +1,114 @@ +# Using AutoGen Studio + +AutoGen Studio supports the declarative creation of an agent workflow and tasks can be specified and run in a chat interface for the agents to complete. The expected usage behavior is that developers can create skills and models, _attach_ them to agents, and compose agents into workflows that can be tested interactively in the chat interface. + +## Building an Agent Workflow + +AutoGen Studio implements several entities that are ultimately composed into a workflow. + +### Skills + +A skill is a python function that implements the solution to a task. In general, a good skill has a descriptive name (e.g. generate*images), extensive docstrings and good defaults (e.g., writing out files to disk for persistence and reuse). Skills can be \_associated with* or _attached to_ agent specifications. + +![AutoGen Studio Skill Interface](./img/skill.png) + +### Models + +A model refers to the configuration of an LLM. Similar to skills, a model can be attached to an agent specification. +The AutoGen Studio interface supports multiple model types including OpenAI models (and any other model endpoint provider that supports the OpenAI endpoint specification), Azure OpenAI models and Gemini Models. + +![AutoGen Studio Create new model](./img/model_new.png) +![AutoGen Studio Create new model](./img/model_openai.png) + +### Agents + +An agent entity declaratively specifies properties for an AutoGen agent (mirrors most but not all of the members of a base AutoGen Conversable agent class). Currently `UserProxyAgent` and `AssistantAgent` and `GroupChat` agent abstractions are supported. + +![AutoGen Studio Create new agent](./img/agent_new.png) +![AutoGen Studio Createan assistant agent](./img/agent_groupchat.png) + +Once agents have been created, existing models or skills can be _added_ to the agent. + +![AutoGen Studio Add skills and models to agent](./img/agent_skillsmodel.png) + +### Workflows + +An agent workflow is a specification of a set of agents (team of agents) that can work together to accomplish a task. AutoGen Studio supports two types of high level workflow patterns: + +#### Autonomous Chat : + +This workflow implements a paradigm where agents are defined and a chat is initiated between the agents to accomplish a task. AutoGen simplifies this into defining an `initiator` agent and a `receiver` agent where the receiver agent is selected from a list of previously created agents. Note that when the receiver is a `GroupChat` agent (i.e., contains multiple agents), the communication pattern between those agents is determined by the `speaker_selection_method` parameter in the `GroupChat` agent configuration. + +![AutoGen Studio Autonomous Chat Workflow](./img/workflow_chat.png) + +#### Sequential Chat + +This workflow allows users to specify a list of `AssistantAgent` agents that are executed in sequence to accomplish a task. The runtime behavior here follows the following pattern: at each step, each `AssistantAgent` is _paired_ with a `UserProxyAgent` and chat initiated between this pair to process the input task. The result of this exchange is summarized and provided to the next `AssistantAgent` which is also paired with a `UserProxyAgent` and their summarized result is passed to the next `AssistantAgent` in the sequence. This continues until the last `AssistantAgent` in the sequence is reached. + +![AutoGen Studio Sequential Workflow](./img/workflow_sequential.png) + + + + + +## Testing an Agent Workflow + +AutoGen Studio allows users to interactively test workflows on tasks and review resulting artifacts (such as images, code, and documents). + +![AutoGen Studio Test Workflow](./img/workflow_test.png) + +Users can also review the โ€œinner monologueโ€ of agent workflows as they address tasks, and view profiling information such as costs associated with the run (such as number of turns, number of tokens etc.), and agent actions (such as whether tools were called and the outcomes of code execution). + +![AutoGen Studio Profile Workflow Results](./img/workflow_profile.png) + +## Exporting Agent Workflows + +Users can download the skills, agents, and workflow configurations they create as well as share and reuse these artifacts. AutoGen Studio also offers a seamless process to export workflows and deploy them as application programming interfaces (APIs) that can be consumed in other applications deploying workflows as APIs. + +### Export Workflow + +AutoGen Studio allows you to export a selected workflow as a JSON configuration file. + +Build -> Workflows -> (On workflow card) -> Export + +![AutoGen Studio Export Workflow](./img/workflow_export.png) + +### Using AutoGen Studio Workflows in a Python Application + +An exported workflow can be easily integrated into any Python application using the `WorkflowManager` class with just two lines of code. Underneath, the WorkflowManager rehydrates the workflow specification into AutoGen agents that are subsequently used to address tasks. + +```python + +from autogenstudio import WorkflowManager +# load workflow from exported json workflow file. +workflow_manager = WorkflowManager(workflow="path/to/your/workflow_.json") + +# run the workflow on a task +task_query = "What is the height of the Eiffel Tower?. Dont write code, just respond to the question." +workflow_manager.run(message=task_query) + +``` + +### Deploying AutoGen Studio Workflows as APIs + +The workflow can be launched as an API endpoint from the command line using the autogenstudio commandline tool. + +```bash +autogenstudio serve --workflow=workflow.json --port=5000 +``` + +Similarly, the workflow launch command above can be wrapped into a Dockerfile that can be deployed on cloud services like Azure Container Apps or Azure Web Apps. diff --git a/website/docs/ecosystem/agentops.md b/website/docs/ecosystem/agentops.md new file mode 100644 index 00000000000..76995b6eb5e --- /dev/null +++ b/website/docs/ecosystem/agentops.md @@ -0,0 +1,83 @@ +# AgentOps ๐Ÿ–‡๏ธ + +![logo](https://raw.githubusercontent.com/AgentOps-AI/agentops/35d5682866921a9e28d8ef66ae3c3b3d92d8fa6b/img/logo.png) + +[AgentOps](https://agentops.ai/?=autogen) provides session replays, metrics, and monitoring for agents. + +At a high level, AgentOps gives you the ability to monitor LLM calls, costs, latency, agent failures, multi-agent interactions, tool usage, session-wide statistics, and more. For more info, check out the [AgentOps Repo](https://github.com/AgentOps-AI/agentops). + +
+ Agent Dashboard + + Agent Dashboard + +
+ +
+ Session Analytics + + Session Analytics + +
+ +
+ Session Replays + + Session Replays + +
+ + +## Installation + +AgentOps works seamlessly with applications built using Autogen. + +1. **Install AgentOps** +```bash +pip install agentops +``` + +2. **Create an API Key:** +Create a user API key here: [Create API Key](https://app.agentops.ai/account) + +3. **Configure Your Environment:** +Add your API key to your environment variables + +``` +AGENTOPS_API_KEY= +``` + +4. **Initialize AgentOps** + +To start tracking all available data on Autogen runs, simply add two lines of code before implementing Autogen. + +```python +import agentops +agentops.init() # Or: agentops.init(api_key="your-api-key-here") +``` + +After initializing AgentOps, Autogen will now start automatically tracking your agent runs. + +## Features + +- **LLM Costs**: Track spend with foundation model providers +- **Replay Analytics**: Watch step-by-step agent execution graphs +- **Recursive Thought Detection**: Identify when agents fall into infinite loops +- **Custom Reporting:** Create custom analytics on agent performance +- **Analytics Dashboard:** Monitor high level statistics about agents in development and production +- **Public Model Testing**: Test your agents against benchmarks and leaderboards +- **Custom Tests:** Run your agents against domain specific tests +- **Time Travel Debugging**: Save snapshots of session states to rewind and replay agent runs from chosen checkpoints. +- **Compliance and Security**: Create audit logs and detect potential threats such as profanity and PII leaks +- **Prompt Injection Detection**: Identify potential code injection and secret leaks + +## Autogen + AgentOps examples +* [AgentChat with AgentOps Notebook](/docs/notebooks/agentchat_agentops) +* [More AgentOps Examples](https://docs.agentops.ai/v1/quickstart) + +## Extra links + +- [๐Ÿฆ Twitter](https://twitter.com/agentopsai/) +- [๐Ÿ“ข Discord](https://discord.gg/JHPt4C7r) +- [๐Ÿ–‡๏ธ AgentOps Dashboard](https://app.agentops.ai/ref?=autogen) +- [๐Ÿ“™ Documentation](https://docs.agentops.ai/introduction) diff --git a/website/docs/topics/llm-observability.md b/website/docs/topics/llm-observability.md new file mode 100644 index 00000000000..6a95d185f97 --- /dev/null +++ b/website/docs/topics/llm-observability.md @@ -0,0 +1,42 @@ +# LLM Observability + +AutoGen supports advanced LLM observability and monitoring through built-in logging and partner providers. + +## What is LLM Observability +AI agent observability is the ability to monitor, measure, and understand the internal states and behaviors of AI agent systems. +Observability is crucial for ensuring transparency, reliability, and accountability in your agent systems. + + +## Development + +### Agent Development in Terminal is Limited +- Lose track of what your agents did in between executions +- Parsing through terminal output searching for LLM completions +- Printing โ€œtool calledโ€ + +### Agent Development Dashboards Enable More +- Visual dashboard so you can see what your agents did in human-readable format +- LLM calls are magically recorded - prompt, completion, timestamps for each - with one line of code +- Agents and their events (including tool calls) are recorded with one more line of code +- Errors are magically associated to its causal event +- Record any other events to your session with two more lines of code +- Tons of other useful data if youโ€™re developing with supported agent frameworks: SDK version + +## Compliance + +Observability and monitoring is critical to ensure AI agent systems adhere to laws and regulations in industries like finance and healthcare, preventing violations such as data breaches and privacy issues. + +- Insights into AI decision-making, allowing organizations to explain outcomes and build trust with stakeholders. +- Helps detect anomalies and unintended behaviors early, mitigating operational, financial, and reputational risks. +- Ensures compliance with data privacy regulations, preventing unauthorized access and misuse of sensitive information. +- Quick identification and response to compliance violations, supporting incident analysis and prevention. + +## Available Observability Integrations + +### Logging +- Autogen SQLite and File Logger - [Tutorial](/docs/notebooks/agentchat_logging) + +### Full-Service Partners +Autogen is currently partnered with [AgentOps](https://agentops.ai) for seamless observability integration. + +[Learn how to install AgentOps](/docs/notebooks/agentchat_agentops) diff --git a/website/docs/topics/llm_configuration.ipynb b/website/docs/topics/llm_configuration.ipynb index 51abf1f4622..c0a1b7e74a9 100644 --- a/website/docs/topics/llm_configuration.ipynb +++ b/website/docs/topics/llm_configuration.ipynb @@ -279,6 +279,7 @@ " def __deepcopy__(self, memo):\n", " return self\n", "\n", + "\n", "config_list = [\n", " {\n", " \"model\": \"my-gpt-4-deployment\",\n", diff --git a/website/docs/tutorial/tool-use.ipynb b/website/docs/tutorial/tool-use.ipynb index 12d0f5a8066..b5e21559225 100644 --- a/website/docs/tutorial/tool-use.ipynb +++ b/website/docs/tutorial/tool-use.ipynb @@ -39,8 +39,13 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:48.858694Z", + "start_time": "2024-04-25T01:49:48.854420Z" + } + }, "outputs": [], "source": [ "from typing import Annotated, Literal\n", @@ -91,8 +96,24 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:48.946697Z", + "start_time": "2024-04-25T01:49:48.857869Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + " int>" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import os\n", "\n", @@ -160,8 +181,13 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:48.953345Z", + "start_time": "2024-04-25T01:49:48.947026Z" + } + }, "outputs": [], "source": [ "from autogen import register_function\n", @@ -189,124 +215,128 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:57.947530Z", + "start_time": "2024-04-25T01:49:48.953943Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", "What is (44232 + 13312 / (232 - 32)) * 5?\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_bACquf0OreI0VHh7rWiP6ZE7): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_4rElPoLggOYJmkUutbGaSTX1): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", - " \"a\": 13312,\n", - " \"b\": 232 - 32,\n", - " \"operator\": \"/\"\n", + " \"a\": 232,\n", + " \"b\": 32,\n", + " \"operator\": \"-\"\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_bACquf0OreI0VHh7rWiP6ZE7\" *****\u001b[0m\n", - "Error: Expecting ',' delimiter: line 1 column 26 (char 25)\n", - " You argument should follow json format.\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_4rElPoLggOYJmkUutbGaSTX1) *****\u001B[0m\n", + "200\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_2c0H5gzX9SWsJ05x7nEOVbav): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_SGtr8tK9A4iOCJGdCqkKR2Ov): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", " \"a\": 13312,\n", " \"b\": 200,\n", " \"operator\": \"/\"\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_2c0H5gzX9SWsJ05x7nEOVbav\" *****\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_SGtr8tK9A4iOCJGdCqkKR2Ov) *****\u001B[0m\n", "66\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_ioceLhuKMpfU131E7TSQ8wCD): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_YsR95CM1Ice2GZ7ZoStYXI6M): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", " \"a\": 44232,\n", " \"b\": 66,\n", " \"operator\": \"+\"\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_ioceLhuKMpfU131E7TSQ8wCD\" *****\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_YsR95CM1Ice2GZ7ZoStYXI6M) *****\u001B[0m\n", "44298\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_0rhx9vrbigcbqLssKLh4sS7j): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_oqZn4rTjyvXYcmjAXkvVaJm1): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", " \"a\": 44298,\n", " \"b\": 5,\n", " \"operator\": \"*\"\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_0rhx9vrbigcbqLssKLh4sS7j\" *****\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_oqZn4rTjyvXYcmjAXkvVaJm1) *****\u001B[0m\n", "221490\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", - "\n", - "The result of the calculation (44232 + 13312 / (232 - 32)) * 5 is 221490. \n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "TERMINATE\n", + "The result of the calculation is 221490. TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n" ] @@ -325,8 +355,13 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:57.956063Z", + "start_time": "2024-04-25T01:49:57.948882Z" + } + }, "outputs": [ { "data": { @@ -334,7 +369,7 @@ "221490" ] }, - "execution_count": 5, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -369,8 +404,13 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:57.956696Z", + "start_time": "2024-04-25T01:49:57.952767Z" + } + }, "outputs": [ { "data": { @@ -387,7 +427,7 @@ " 'required': ['a', 'b', 'operator']}}}]" ] }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -417,8 +457,13 @@ }, { "cell_type": "code", - "execution_count": 33, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:57.965069Z", + "start_time": "2024-04-25T01:49:57.958274Z" + } + }, "outputs": [], "source": [ "from pydantic import BaseModel, Field\n", @@ -459,8 +504,24 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:57.990811Z", + "start_time": "2024-04-25T01:49:57.962315Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + " int>" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "assistant.register_for_llm(name=\"calculator\", description=\"A calculator tool that accepts nested expression as input\")(\n", " calculator\n", @@ -477,8 +538,13 @@ }, { "cell_type": "code", - "execution_count": 28, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:49:57.991342Z", + "start_time": "2024-04-25T01:49:57.972554Z" + } + }, "outputs": [ { "data": { @@ -504,7 +570,7 @@ " 'required': ['input']}}}]" ] }, - "execution_count": 28, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -522,159 +588,192 @@ }, { "cell_type": "code", - "execution_count": 32, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:50:17.808416Z", + "start_time": "2024-04-25T01:49:57.975143Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", "What is (1423 - 123) / 3 + (32 + 23) * 5?\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_t9By3vewGRoSLWsvdTR7p8Zo): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_Uu4diKtxlTfkwXuY6MmJEb4E): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", - " \"input\": {\n", - " \"a\": 1423,\n", - " \"b\": 123,\n", - " \"operator\": \"-\"\n", - " }\n", + " \"input\": {\n", + " \"a\": (1423 - 123) / 3,\n", + " \"b\": (32 + 23) * 5,\n", + " \"operator\": \"+\"\n", + " }\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_t9By3vewGRoSLWsvdTR7p8Zo\" *****\u001b[0m\n", - "1300\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_Uu4diKtxlTfkwXuY6MmJEb4E) *****\u001B[0m\n", + "Error: Expecting value: line 1 column 29 (char 28)\n", + " You argument should follow json format.\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "I apologize for the confusion, I seem to have made a mistake. Let me recalculate the expression properly.\n", "\n", - "\u001b[32m***** Suggested tool Call (call_rhecyhVCo0Y8HPL193xOUPE6): calculator *****\u001b[0m\n", + "First, we need to do the calculations within the brackets. So, calculating (1423 - 123), (32 + 23), and then performing remaining operations.\n", + "\u001B[32m***** Suggested tool call (call_mx3M3fNOwikFNoqSojDH1jIr): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", - " \"input\": {\n", - " \"a\": 1300,\n", - " \"b\": 3,\n", - " \"operator\": \"/\"\n", - " }\n", + " \"input\": {\n", + " \"a\": 1423,\n", + " \"b\": 123,\n", + " \"operator\": \"-\"\n", + " }\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_rhecyhVCo0Y8HPL193xOUPE6\" *****\u001b[0m\n", - "433\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_mx3M3fNOwikFNoqSojDH1jIr) *****\u001B[0m\n", + "1300\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_zDpq9J5MYAsL7uS8cobOwa7S): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_hBAL2sYi6Y5ZtTHCNPCmxdN3): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", - " \"input\": {\n", - " \"a\": 32,\n", - " \"b\": 23,\n", - " \"operator\": \"+\"\n", - " }\n", + " \"input\": {\n", + " \"a\": 32,\n", + " \"b\": 23,\n", + " \"operator\": \"+\"\n", + " }\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_zDpq9J5MYAsL7uS8cobOwa7S\" *****\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_hBAL2sYi6Y5ZtTHCNPCmxdN3) *****\u001B[0m\n", "55\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_mjDuVMojOIdaxmvDUIF4QtVi): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_wO3AP7EDeJvsVLCpvv5LohUa): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", - " \"input\": {\n", - " \"a\": 55,\n", - " \"b\": 5,\n", - " \"operator\": \"*\"\n", - " }\n", + " \"input\": {\n", + " \"a\": 1300,\n", + " \"b\": 3,\n", + " \"operator\": \"/\"\n", + " }\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_mjDuVMojOIdaxmvDUIF4QtVi\" *****\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_wO3AP7EDeJvsVLCpvv5LohUa) *****\u001B[0m\n", + "433\n", + "\u001B[32m**********************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", + "\n", + "\u001B[32m***** Suggested tool call (call_kQ2hDhqem8BHNlaHaE9ezvvQ): calculator *****\u001B[0m\n", + "Arguments: \n", + "{\n", + " \"input\": {\n", + " \"a\": 55,\n", + " \"b\": 5,\n", + " \"operator\": \"*\"\n", + " }\n", + "}\n", + "\u001B[32m***************************************************************************\u001B[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\n", + "\u001B[32m***** Response from calling tool (call_kQ2hDhqem8BHNlaHaE9ezvvQ) *****\u001B[0m\n", "275\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", - "\u001b[32m***** Suggested tool Call (call_hpirkAGKOewZstsDOxL2sYNW): calculator *****\u001b[0m\n", + "\u001B[32m***** Suggested tool call (call_1FLDUdvAZmjlSD7g5GFFJOpO): calculator *****\u001B[0m\n", "Arguments: \n", "{\n", - " \"input\": {\n", - " \"a\": 433,\n", - " \"b\": 275,\n", - " \"operator\": \"+\"\n", - " }\n", + " \"input\": {\n", + " \"a\": 433,\n", + " \"b\": 275,\n", + " \"operator\": \"+\"\n", + " }\n", "}\n", - "\u001b[32m***************************************************************************\u001b[0m\n", + "\u001B[32m***************************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[33mUser\u001b[0m (to Assistant):\n", + "\u001B[33mUser\u001B[0m (to Assistant):\n", "\n", - "\u001b[32m***** Response from calling tool \"call_hpirkAGKOewZstsDOxL2sYNW\" *****\u001b[0m\n", + "\u001B[32m***** Response from calling tool (call_1FLDUdvAZmjlSD7g5GFFJOpO) *****\u001B[0m\n", "708\n", - "\u001b[32m**********************************************************************\u001b[0m\n", + "\u001B[32m**********************************************************************\u001B[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", - "\u001b[33mAssistant\u001b[0m (to User):\n", - "\n", - "The result of the calculation is 708. \n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mAssistant\u001B[0m (to User):\n", "\n", + "The calculation result of the expression (1423 - 123) / 3 + (32 + 23) * 5 is 708. Let's proceed to the next task.\n", "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n" @@ -694,8 +793,13 @@ }, { "cell_type": "code", - "execution_count": 31, - "metadata": {}, + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-25T01:50:17.818095Z", + "start_time": "2024-04-25T01:50:17.808502Z" + } + }, "outputs": [ { "data": { @@ -703,7 +807,7 @@ "708" ] }, - "execution_count": 31, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index f60ac41fd61..efc13096b0f 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -8,9 +8,9 @@ customPostCssPlugin = () => { configurePostCss(options) { options.plugins.push(require("postcss-preset-env")); return options; - } + }, }; -} +}; module.exports = { title: "AutoGen", @@ -24,13 +24,13 @@ module.exports = { projectName: "AutoGen", // Usually your repo name. scripts: [ { - src: '/autogen/js/custom.js', + src: "/autogen/js/custom.js", async: true, defer: true, }, ], markdown: { - format: 'detect', // Support for MD files with .md extension + format: "detect", // Support for MD files with .md extension }, themeConfig: { docs: { @@ -80,6 +80,11 @@ module.exports = { docId: "FAQ", label: "FAQs", }, + { + type: "doc", + docId: "autogen-studio/getting-started", + label: "AutoGen Studio", + }, { type: "doc", docId: "ecosystem", @@ -127,9 +132,8 @@ module.exports = { { label: "Dotnet", href: "https://microsoft.github.io/autogen-for-net/", - } + }, ], - }, { to: "blog", @@ -175,7 +179,6 @@ module.exports = { { label: "Discord", href: "https://aka.ms/autogen-dc", - }, { label: "Twitter", @@ -187,17 +190,17 @@ module.exports = { copyright: `Copyright ยฉ ${new Date().getFullYear()} AutoGen Authors | Privacy and Cookies`, }, announcementBar: { - id: 'whats_new', + id: "whats_new", content: 'What\'s new in AutoGen? Read this blog for an overview of updates', - backgroundColor: '#fafbfc', - textColor: '#091E42', + backgroundColor: "#fafbfc", + textColor: "#091E42", isCloseable: true, }, /* Clarity Config */ clarity: { ID: "lnxpe6skj1", // The Tracking ID provided by Clarity - } + }, }, presets: [ [ @@ -281,14 +284,10 @@ module.exports = { { to: "/docs/contributor-guide/contributing", from: ["/docs/Contribute"], - } + }, ], }, ], - [ - 'docusaurus-plugin-clarity', - { - } - ], + ["docusaurus-plugin-clarity", {}], ], }; diff --git a/website/sidebars.js b/website/sidebars.js index 49d8fbf87e9..589c0ee9ba4 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -9,114 +9,143 @@ Create as many sidebars as you want. */ - module.exports = { +module.exports = { docsSidebar: [ - 'Getting-Started', + "Getting-Started", { type: "category", label: "Installation", collapsed: true, items: ["installation/Docker", "installation/Optional-Dependencies"], link: { - type: 'doc', - id: "installation/Installation" + type: "doc", + id: "installation/Installation", }, }, { - type: 'category', - label: 'Tutorial', + type: "category", + label: "Tutorial", collapsed: false, link: { - type: 'generated-index', - title: 'Tutorial', - description: 'Tutorial on the basic concepts of AutoGen', - slug: 'tutorial', + type: "generated-index", + title: "Tutorial", + description: "Tutorial on the basic concepts of AutoGen", + slug: "tutorial", }, items: [ { - type: 'doc', - id: 'tutorial/introduction', - label: 'Introduction', + type: "doc", + id: "tutorial/introduction", + label: "Introduction", }, { - type: 'doc', - id: 'tutorial/chat-termination', - label: 'Chat Termination', + type: "doc", + id: "tutorial/chat-termination", + label: "Chat Termination", }, { - type: 'doc', - id: 'tutorial/human-in-the-loop', - label: 'Human in the Loop', + type: "doc", + id: "tutorial/human-in-the-loop", + label: "Human in the Loop", }, { - type: 'doc', - id: 'tutorial/code-executors', - label: 'Code Executors', + type: "doc", + id: "tutorial/code-executors", + label: "Code Executors", }, { - type: 'doc', - id: 'tutorial/tool-use', - label: 'Tool Use', + type: "doc", + id: "tutorial/tool-use", + label: "Tool Use", }, { - type: 'doc', - id: 'tutorial/conversation-patterns', - label: 'Conversation Patterns', + type: "doc", + id: "tutorial/conversation-patterns", + label: "Conversation Patterns", }, { - type: 'doc', - id: 'tutorial/what-next', - label: 'What Next?', - } + type: "doc", + id: "tutorial/what-next", + label: "What Next?", + }, ], }, - {'Use Cases': [{type: 'autogenerated', dirName: 'Use-Cases'}]}, + { "Use Cases": [{ type: "autogenerated", dirName: "Use-Cases" }] }, { - type: 'category', - label: 'User Guide', + type: "category", + label: "User Guide", collapsed: false, link: { - type: 'generated-index', - title: 'User Guide', - slug: 'topics', + type: "generated-index", + title: "User Guide", + slug: "topics", }, - items: [{type: 'autogenerated', dirName: 'topics'}] + items: [{ type: "autogenerated", dirName: "topics" }], }, { - type: 'link', - label: 'API Reference', - href: '/docs/reference/agentchat/conversable_agent', + type: "link", + label: "API Reference", + href: "/docs/reference/agentchat/conversable_agent", }, { - type: 'doc', - label: 'FAQs', - id: 'FAQ', + type: "doc", + label: "FAQs", + id: "FAQ", }, + { - 'type': 'category', - 'label': 'Ecosystem', - 'link': { - type: 'generated-index', - title: 'Ecosystem', - description: 'Learn about the ecosystem of AutoGen', - slug: 'ecosystem', + type: "category", + label: "AutoGen Studio", + collapsed: true, + items: [ + { + type: "doc", + id: "autogen-studio/getting-started", + label: "Getting Started", + }, + { + type: "doc", + id: "autogen-studio/usage", + label: "Using AutoGen Studio", + }, + { + type: "doc", + id: "autogen-studio/faqs", + label: "AutoGen Studio FAQs", + }, + ], + link: { + type: "generated-index", + title: "AutoGen Studio", + description: "Learn about AutoGen Studio", + slug: "autogen-studio", + }, + }, + { + type: "category", + label: "Ecosystem", + link: { + type: "generated-index", + title: "Ecosystem", + description: "Learn about the ecosystem of AutoGen", + slug: "ecosystem", }, - 'items': [{type: 'autogenerated', dirName: 'ecosystem'}] + items: [{ type: "autogenerated", dirName: "ecosystem" }], }, { type: "category", label: "Contributor Guide", collapsed: true, - items: [{type: 'autogenerated', dirName: 'contributor-guide'}], + items: [{ type: "autogenerated", dirName: "contributor-guide" }], link: { - type: 'generated-index', - title: 'Contributor Guide', - description: 'Learn how to contribute to AutoGen', - slug: 'contributor-guide', + type: "generated-index", + title: "Contributor Guide", + description: "Learn how to contribute to AutoGen", + slug: "contributor-guide", }, }, - 'Research', - 'Migration-Guide' + "Research", + "Migration-Guide", ], // pydoc-markdown auto-generated markdowns from docstrings referenceSideBar: [require("./docs/reference/sidebar.json")], @@ -124,14 +153,16 @@ { type: "category", label: "Notebooks", - items: [{ - type: "autogenerated", - dirName: "notebooks", - },], + items: [ + { + type: "autogenerated", + dirName: "notebooks", + }, + ], link: { - type: 'doc', - id: "notebooks" + type: "doc", + id: "notebooks", }, }, - ] + ], };