From 5a496b34ea66840126baa5a93a67ccde2ef4b3ec Mon Sep 17 00:00:00 2001 From: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> Date: Wed, 19 Jun 2024 11:45:12 +0530 Subject: [PATCH] Anthropic Client (#2931) * intial setup for the anthropic client with cost config * update: other methods added * fix: formatting * fix: config unused * update: changes made in the client * update: test added to the workflow * update: added tests to the anthropic client * fix: errors in workflows and client * fix * fix: anthropic tools type * update: notebook anthropic * Nonetype fixed * fix-tests config * update: tests and client issues * logger support * remove sys path * updated the functioning of the client * update: type hints and stream * skip tests- importerror * fix: anthropic client and tests * none fix * Alternating roles, parameter keywords, cost on response, * update: anthropic notebook * update: notebook with more details * devcontainer * update: added validate_params from the client_utils * fix: formatting * fix: minor comment --------- Co-authored-by: Mark Sze --- .github/workflows/contrib-tests.yml | 43 ++ autogen/logger/file_logger.py | 6 +- autogen/logger/sqlite_logger.py | 6 +- autogen/oai/anthropic.py | 274 +++++++++++ autogen/oai/client.py | 12 + autogen/runtime_logging.py | 3 +- autogen/token_count_utils.py | 3 + setup.py | 1 + test/oai/test_anthropic.py | 97 ++++ .../non-openai-models/cloud-anthropic.ipynb | 442 ++++-------------- 10 files changed, 541 insertions(+), 346 deletions(-) create mode 100644 autogen/oai/anthropic.py create mode 100644 test/oai/test_anthropic.py diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index ebce2cd4e335..05e7349d111b 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -475,3 +475,46 @@ jobs: with: file: ./coverage.xml flags: unittests + + + AnthropicTest: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + + - name: Install packages and dependencies for Anthropic + run: | + pip install -e .[test] + pip install -e .[anthropic] + + - name: Set AUTOGEN_USE_DOCKER based on OS + shell: bash + run: | + if [[ ${{ matrix.os }} != ubuntu-latest ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi + + - name: Coverage + run: | + pytest test/oai/test_anthropic.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests diff --git a/autogen/logger/file_logger.py b/autogen/logger/file_logger.py index 15b2c457e420..4e1513478374 100644 --- a/autogen/logger/file_logger.py +++ b/autogen/logger/file_logger.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper + from autogen.oai.anthropic import AnthropicClient from autogen.oai.gemini import GeminiClient logger = logging.getLogger(__name__) @@ -200,7 +201,10 @@ def log_new_wrapper( self.logger.error(f"[file_logger] Failed to log event {e}") def log_new_client( - self, client: AzureOpenAI | OpenAI | GeminiClient, wrapper: OpenAIWrapper, init_args: Dict[str, Any] + self, + client: AzureOpenAI | OpenAI | GeminiClient | AnthropicClient, + wrapper: OpenAIWrapper, + init_args: Dict[str, Any], ) -> None: """ Log a new client instance. diff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.py index 4ebea32cc199..1e80bc8751ef 100644 --- a/autogen/logger/sqlite_logger.py +++ b/autogen/logger/sqlite_logger.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper + from autogen.oai.anthropic import AnthropicClient from autogen.oai.gemini import GeminiClient logger = logging.getLogger(__name__) @@ -387,7 +388,10 @@ def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[st self._run_query(query=query, args=query_args) def log_new_client( - self, client: Union[AzureOpenAI, OpenAI, GeminiClient], wrapper: OpenAIWrapper, init_args: Dict[str, Any] + self, + client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient], + wrapper: OpenAIWrapper, + init_args: Dict[str, Any], ) -> None: if self.con is None: return diff --git a/autogen/oai/anthropic.py b/autogen/oai/anthropic.py new file mode 100644 index 000000000000..27fbc36d8fc7 --- /dev/null +++ b/autogen/oai/anthropic.py @@ -0,0 +1,274 @@ +""" +Create an OpenAI-compatible client for the Anthropic API. + +Example usage: +Install the `anthropic` package by running `pip install --upgrade anthropic`. +- https://docs.anthropic.com/en/docs/quickstart-guide + +import autogen + +config_list = [ + { + "model": "claude-3-sonnet-20240229", + "api_key": os.getenv("ANTHROPIC_API_KEY"), + "api_type": "anthropic", + } +] + +assistant = autogen.AssistantAgent("assistant", llm_config={"config_list": config_list}) +""" + +from __future__ import annotations + +import copy +import inspect +import json +import os +import warnings +from typing import Any, Dict, List, Tuple, Union + +from anthropic import Anthropic +from anthropic import __version__ as anthropic_version +from anthropic.types import Completion, Message +from client_utils import validate_parameter +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall +from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from typing_extensions import Annotated + +TOOL_ENABLED = anthropic_version >= "0.23.1" +if TOOL_ENABLED: + from anthropic.types.tool_use_block_param import ( + ToolUseBlockParam, + ) + + +ANTHROPIC_PRICING_1k = { + "claude-3-sonnet-20240229": (0.003, 0.015), + "claude-3-opus-20240229": (0.015, 0.075), + "claude-2.0": (0.008, 0.024), + "claude-2.1": (0.008, 0.024), + "claude-3.0-opus": (0.015, 0.075), + "claude-3.0-haiku": (0.00025, 0.00125), +} + + +class AnthropicClient: + def __init__(self, **kwargs: Any): + """ + Initialize the Anthropic API client. + Args: + api_key (str): The API key for the Anthropic API or set the `ANTHROPIC_API_KEY` environment variable. + """ + self._api_key = kwargs.get("api_key", None) + + if not self._api_key: + self._api_key = os.getenv("ANTHROPIC_API_KEY") + + if self._api_key is None: + raise ValueError("API key is required to use the Anthropic API.") + + self._client = Anthropic(api_key=self._api_key) + self._last_tooluse_status = {} + + def load_config(self, params: Dict[str, Any]): + """Load the configuration for the Anthropic API client.""" + anthropic_params = {} + + anthropic_params["model"] = params.get("model", None) + assert anthropic_params["model"], "Please provide a `model` in the config_list to use the Anthropic API." + + anthropic_params["temperature"] = validate_parameter( + params, "temperature", (float, int), False, 1.0, (0.0, 1.0), None + ) + anthropic_params["max_tokens"] = validate_parameter(params, "max_tokens", int, False, 4096, (1, None), None) + anthropic_params["top_k"] = validate_parameter(params, "top_k", int, True, None, (1, None), None) + anthropic_params["top_p"] = validate_parameter(params, "top_p", (float, int), True, None, (0.0, 1.0), None) + anthropic_params["stop_sequences"] = validate_parameter(params, "stop_sequences", list, True, None, None, None) + anthropic_params["stream"] = validate_parameter(params, "stream", bool, False, False, None, None) + + if anthropic_params["stream"]: + warnings.warn( + "Streaming is not currently supported, streaming will be disabled.", + UserWarning, + ) + anthropic_params["stream"] = False + + return anthropic_params + + def cost(self, response) -> float: + """Calculate the cost of the completion using the Anthropic pricing.""" + return response.cost + + @property + def api_key(self): + return self._api_key + + def create(self, params: Dict[str, Any]) -> Completion: + """Create a completion for a given config. + + Args: + params: The params for the completion. + + Returns: + The completion. + """ + if "tools" in params: + converted_functions = self.convert_tools_to_functions(params["tools"]) + params["functions"] = params.get("functions", []) + converted_functions + + raw_contents = params["messages"] + anthropic_params = self.load_config(params) + + processed_messages = [] + for message in raw_contents: + + if message["role"] == "system": + params["system"] = message["content"] + elif message["role"] == "function": + processed_messages.append(self.return_function_call_result(message["content"])) + elif "function_call" in message: + processed_messages.append(self.restore_last_tooluse_status()) + elif message["content"] == "": + message["content"] = "I'm done. Please send TERMINATE" # Not sure about this one. + processed_messages.append(message) + else: + processed_messages.append(message) + + # Check for interleaving roles and correct, for Anthropic must be: user, assistant, user, etc. + for i, message in enumerate(processed_messages): + if message["role"] is not ("user" if i % 2 == 0 else "assistant"): + message["role"] = "user" if i % 2 == 0 else "assistant" + + # Note: When using reflection_with_llm we may end up with an "assistant" message as the last message + if processed_messages[-1]["role"] != "user": + # If the last role is not user, add a continue message at the end + continue_message = {"content": "continue", "role": "user"} + processed_messages.append(continue_message) + + params["messages"] = processed_messages + + # TODO: support stream + params = params.copy() + if "functions" in params: + tools_configs = params.pop("functions") + tools_configs = [self.openai_func_to_anthropic(tool) for tool in tools_configs] + params["tools"] = tools_configs + + # Anthropic doesn't accept None values, so we need to use keyword argument unpacking instead of setting parameters. + # Copy params we need into anthropic_params + # Remove any that don't have values + anthropic_params["messages"] = params["messages"] + if "system" in params: + anthropic_params["system"] = params["system"] + if "tools" in params: + anthropic_params["tools"] = params["tools"] + if anthropic_params["top_k"] is None: + del anthropic_params["top_k"] + if anthropic_params["top_p"] is None: + del anthropic_params["top_p"] + if anthropic_params["stop_sequences"] is None: + del anthropic_params["stop_sequences"] + + response = self._client.messages.create(**anthropic_params) + + # Calculate and save the cost onto the response + prompt_tokens = response.usage.input_tokens + completion_tokens = response.usage.output_tokens + response.cost = _calculate_cost(prompt_tokens, completion_tokens, anthropic_params["model"]) + + return response + + def message_retrieval(self, response: Union[Message]) -> Union[List[str], List[ChatCompletionMessage]]: + """Retrieve the messages from the response.""" + messages = response.content + if len(messages) == 0: + return [None] + res = [] + if TOOL_ENABLED: + for choice in messages: + if choice.type == "tool_use": + res.insert(0, self.response_to_openai_message(choice)) + self._last_tooluse_status["tool_use"] = choice.model_dump() + else: + res.append(choice.text) + self._last_tooluse_status["think"] = choice.text + + return res + + else: + return [ # type: ignore [return-value] + choice.text if choice.message.function_call is not None else choice.message.content # type: ignore [union-attr] + for choice in messages + ] + + def response_to_openai_message(self, response) -> ChatCompletionMessage: + """Convert the client response to OpenAI ChatCompletion Message""" + dict_response = response.model_dump() + return ChatCompletionMessage( + content=None, + role="assistant", + function_call={"name": dict_response["name"], "arguments": json.dumps(dict_response["input"])}, + ) + + def restore_last_tooluse_status(self) -> Dict: + cached_content = [] + if "think" in self._last_tooluse_status: + cached_content.append({"type": "text", "text": self._last_tooluse_status["think"]}) + cached_content.append(self._last_tooluse_status["tool_use"]) + res = {"role": "assistant", "content": cached_content} + return res + + def return_function_call_result(self, result: str) -> Dict: + return { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": self._last_tooluse_status["tool_use"]["id"], + "content": result, + } + ], + } + + @staticmethod + def openai_func_to_anthropic(openai_func: dict) -> dict: + res = openai_func.copy() + res["input_schema"] = res.pop("parameters") + return res + + @staticmethod + def get_usage(response: Message) -> Dict: + """Get the usage of tokens and their cost information.""" + return { + "prompt_tokens": response.usage.input_tokens if response.usage is not None else 0, + "completion_tokens": response.usage.output_tokens if response.usage is not None else 0, + "total_tokens": ( + response.usage.input_tokens + response.usage.output_tokens if response.usage is not None else 0 + ), + "cost": response.cost if hasattr(response, "cost") else 0.0, + "model": response.model, + } + + @staticmethod + def convert_tools_to_functions(tools: List) -> List: + functions = [] + for tool in tools: + if tool.get("type") == "function" and "function" in tool: + functions.append(tool["function"]) + + return functions + + +def _calculate_cost(input_tokens: int, output_tokens: int, model: str) -> float: + """Calculate the cost of the completion using the Anthropic pricing.""" + total = 0.0 + + if model in ANTHROPIC_PRICING_1k: + input_cost_per_1k, output_cost_per_1k = ANTHROPIC_PRICING_1k[model] + input_cost = (input_tokens / 1000) * input_cost_per_1k + output_cost = (output_tokens / 1000) * output_cost_per_1k + total = input_cost + output_cost + else: + warnings.warn(f"Cost calculation not available for model {model}", UserWarning) + + return total diff --git a/autogen/oai/client.py b/autogen/oai/client.py index fb5946b381d2..e86ecc51282c 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -49,6 +49,13 @@ except ImportError as e: gemini_import_exception = e +try: + from autogen.oai.anthropic import AnthropicClient + + anthropic_import_exception: Optional[ImportError] = None +except ImportError as e: + anthropic_import_exception = e + logger = logging.getLogger(__name__) if not logger.handlers: # Add the console handler. @@ -449,6 +456,11 @@ def _register_default_client(self, config: Dict[str, Any], openai_config: Dict[s raise ImportError("Please install `google-generativeai` to use Google OpenAI API.") client = GeminiClient(**openai_config) self._clients.append(client) + elif api_type is not None and api_type.startswith("anthropic"): + if anthropic_import_exception: + raise ImportError("Please install `anthropic` to use Anthropic API.") + client = AnthropicClient(**openai_config) + self._clients.append(client) else: client = OpenAI(**openai_config) self._clients.append(OpenAIClient(client)) diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.py index 6ce207a2f330..3d238580bb0d 100644 --- a/autogen/runtime_logging.py +++ b/autogen/runtime_logging.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper + from autogen.oai.anthropic import AnthropicClient from autogen.oai.gemini import GeminiClient logger = logging.getLogger(__name__) @@ -107,7 +108,7 @@ def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLMConfig def log_new_client( - client: Union[AzureOpenAI, OpenAI, GeminiClient], wrapper: OpenAIWrapper, init_args: Dict[str, Any] + client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient], wrapper: OpenAIWrapper, init_args: Dict[str, Any] ) -> None: if autogen_logger is None: logger.error("[runtime logging] log_new_client: autogen logger is None") diff --git a/autogen/token_count_utils.py b/autogen/token_count_utils.py index b71dbc428a13..6da6429d7e75 100644 --- a/autogen/token_count_utils.py +++ b/autogen/token_count_utils.py @@ -119,6 +119,9 @@ def _num_token_from_messages(messages: Union[List, Dict], model="gpt-3.5-turbo-0 elif "gemini" in model: logger.info("Gemini is not supported in tiktoken. Returning num tokens assuming gpt-4-0613.") return _num_token_from_messages(messages, model="gpt-4-0613") + elif "claude" in model: + logger.info("Claude is not supported in tiktoken. Returning num tokens assuming gpt-4-0613.") + return _num_token_from_messages(messages, model="gpt-4-0613") else: raise NotImplementedError( f"""_num_token_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""" diff --git a/setup.py b/setup.py index d52a4e7ef5b7..20dc5e1bf70c 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,7 @@ "jupyter-executor": jupyter_executor, "types": ["mypy==1.9.0", "pytest>=6.1.1,<8"] + jupyter_executor, "long-context": ["llmlingua<0.3"], + "anthropic": ["anthropic>=0.23.1"], } setuptools.setup( diff --git a/test/oai/test_anthropic.py b/test/oai/test_anthropic.py new file mode 100644 index 000000000000..379ab47f6756 --- /dev/null +++ b/test/oai/test_anthropic.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 -m pytest + +import os +from unittest.mock import MagicMock, patch + +import pytest + +try: + from autogen.oai.anthropic import AnthropicClient, _calculate_cost + + skip = False +except ImportError: + AnthropicClient = object + skip = True + +from typing_extensions import Literal + +reason = "Anthropic dependency not installed!" + + +@pytest.fixture() +def mock_completion(): + class MockCompletion: + def __init__( + self, + id="msg_013Zva2CMHLNnXjNJJKqJ2EF", + completion="Hi! My name is Claude.", + model="claude-3-opus-20240229", + stop_reason="end_turn", + role="assistant", + type: Literal["completion"] = "completion", + usage={"input_tokens": 10, "output_tokens": 25}, + ): + self.id = id + self.role = role + self.completion = completion + self.model = model + self.stop_reason = stop_reason + self.type = type + self.usage = usage + + return MockCompletion + + +@pytest.fixture() +def anthropic_client(): + return AnthropicClient(api_key="dummy_api_key") + + +@pytest.mark.skipif(skip, reason=reason) +def test_initialization_missing_api_key(): + os.environ.pop("ANTHROPIC_API_KEY", None) + with pytest.raises(ValueError, match="API key is required to use the Anthropic API."): + AnthropicClient() + + AnthropicClient(api_key="dummy_api_key") + + +@pytest.mark.skipif(skip, reason=reason) +def test_intialization(anthropic_client): + assert anthropic_client.api_key == "dummy_api_key", "`api_key` should be correctly set in the config" + + +# Test cost calculation +@pytest.mark.skipif(skip, reason=reason) +def test_cost_calculation(mock_completion): + completion = mock_completion( + completion="Hi! My name is Claude.", + usage={"prompt_tokens": 10, "completion_tokens": 25, "total_tokens": 35}, + model="claude-3-opus-20240229", + ) + assert ( + _calculate_cost(completion.usage["prompt_tokens"], completion.usage["completion_tokens"], completion.model) + == 0.002025 + ), "Cost should be $0.002025" + + +@pytest.mark.skipif(skip, reason=reason) +def test_load_config(anthropic_client): + params = { + "model": "claude-3-sonnet-20240229", + "stream": False, + "temperature": 1, + "top_p": 0.8, + "max_tokens": 100, + } + expected_params = { + "model": "claude-3-sonnet-20240229", + "stream": False, + "temperature": 1, + "top_p": 0.8, + "max_tokens": 100, + "stop_sequences": None, + "top_k": None, + } + result = anthropic_client.load_config(params) + assert result == expected_params, "Config should be correctly loaded" diff --git a/website/docs/topics/non-openai-models/cloud-anthropic.ipynb b/website/docs/topics/non-openai-models/cloud-anthropic.ipynb index ec8793be4bcb..a1e520850e75 100644 --- a/website/docs/topics/non-openai-models/cloud-anthropic.ipynb +++ b/website/docs/topics/non-openai-models/cloud-anthropic.ipynb @@ -10,10 +10,18 @@ "source": [ "# Anthropic Claude\n", "\n", - "In this notebook, we demonstrate how a to use Anthropic Claude model for AgentChat.\n", + "In the v0.2.30 release of AutoGen we support Anthropic Client.\n", + "\n", + "Claude is a family of large language models developed by Anthropic and designed to revolutionize the way you interact with AI. Claude excels at a wide variety of tasks involving language, reasoning, analysis, coding, and more. The models are highly capable, easy to use, and can be customized to suit your needs.\n", + "\n", + "In this notebook, we demonstrate how to use Anthropic Claude model for AgentChat in AutoGen.\n", + "\n", + "## Features\n", + "\n", + "Additionally, this client class provides support for function/tool calling and will track token usage and cost correctly as per Anthropic's API costs (as of June 2024).\n", "\n", "## Requirements\n", - "To use Anthropic Claude with AutoGen, first you need to install the `pyautogen` and `anthropic` package.\n", + "To use Anthropic Claude with AutoGen, first you need to install the `pyautogen[\"anthropic]` package.\n", "\n", "To try out the function call feature of Claude model, you need to install `anthropic>=0.23.1`.\n" ] @@ -25,291 +33,83 @@ "outputs": [], "source": [ "# !pip install pyautogen\n", - "!pip install \"anthropic>=0.23.1\"" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "import inspect\n", - "import json\n", - "from typing import Any, Dict, List, Union\n", - "\n", - "from anthropic import Anthropic\n", - "from anthropic import __version__ as anthropic_version\n", - "from anthropic.types import Completion, Message\n", - "from openai.types.chat.chat_completion import ChatCompletionMessage\n", - "from typing_extensions import Annotated\n", - "\n", - "import autogen\n", - "from autogen import AssistantAgent, UserProxyAgent\n", - "\n", - "TOOL_ENABLED = anthropic_version >= \"0.23.1\"\n", - "if TOOL_ENABLED:\n", - " from anthropic.types.beta.tools import ToolsBetaMessage\n", - "else:\n", - " ToolsBetaMessage = object" + "!pip install pyautogen[\"anthropic\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Create Anthropic Model Client following ModelClient Protocol\n", - "\n", - "We will implement our Anthropic client adhere to the `ModelClient` protocol and response structure which is defined in client.py and shown below.\n", - "\n", - "\n", - "```python\n", - "class ModelClient(Protocol):\n", - " \"\"\"\n", - " A client class must implement the following methods:\n", - " - create must return a response object that implements the ModelClientResponseProtocol\n", - " - cost must return the cost of the response\n", - " - get_usage must return a dict with the following keys:\n", - " - prompt_tokens\n", - " - completion_tokens\n", - " - total_tokens\n", - " - cost\n", - " - model\n", - "\n", - " This class is used to create a client that can be used by OpenAIWrapper.\n", - " The response returned from create must adhere to the ModelClientResponseProtocol but can be extended however needed.\n", - " The message_retrieval method must be implemented to return a list of str or a list of messages from the response.\n", - " \"\"\"\n", - "\n", - " RESPONSE_USAGE_KEYS = [\"prompt_tokens\", \"completion_tokens\", \"total_tokens\", \"cost\", \"model\"]\n", - "\n", - " class ModelClientResponseProtocol(Protocol):\n", - " class Choice(Protocol):\n", - " class Message(Protocol):\n", - " content: Optional[str]\n", - "\n", - " message: Message\n", - "\n", - " choices: List[Choice]\n", - " model: str\n", - "\n", - " def create(self, params) -> ModelClientResponseProtocol:\n", - " ...\n", - "\n", - " def message_retrieval(\n", - " self, response: ModelClientResponseProtocol\n", - " ) -> Union[List[str], List[ModelClient.ModelClientResponseProtocol.Choice.Message]]:\n", - " \"\"\"\n", - " Retrieve and return a list of strings or a list of Choice.Message from the response.\n", + "## Set the config for the Anthropic API\n", "\n", - " NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object,\n", - " since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used.\n", - " \"\"\"\n", - " ...\n", + "You can add any parameters that are needed for the custom model loading in the same configuration list.\n", "\n", - " def cost(self, response: ModelClientResponseProtocol) -> float:\n", - " ...\n", + "It is important to add the `api_type` field and set it to a string that corresponds to the client type used: `anthropic`.\n", "\n", - " @staticmethod\n", - " def get_usage(response: ModelClientResponseProtocol) -> Dict:\n", - " \"\"\"Return usage summary of the response using RESPONSE_USAGE_KEYS.\"\"\"\n", - " ...\n", - "```\n" + "Example:\n", + "```\n", + "[\n", + " {\n", + " \"model\": \"claude-3-sonnet-20240229\",\n", + " \"api_key\": \"your Anthropic API Key goes here\",\n", + " \"api_type\": \"anthropic\",\n", + " \"temperature\": 0.5,\n", + " \"top_p\": 0.2, # Note: It is recommended to set temperature or top_p but not both.\n", + " \"max_tokens\": 10000,\n", + " },\n", + " {\n", + " \"model\":\"claude-3-opus-20240229\",\n", + " \"api_key\":\"your api key\",\n", + " \"api_type\":\"anthropic\",\n", + " },\n", + " {\n", + " \"model\":\"claude-2.0\",\n", + " \"api_key\":\"your api key\",\n", + " \"api_type\":\"anthropic\",\n", + " },\n", + " {\n", + " \"model\":\"claude-2.1\",\n", + " \"api_key\":\"your api key\",\n", + " \"api_type\":\"anthropic\",\n", + " },\n", + " {\n", + " \"model\":\"claude-3.0-haiku\",\n", + " \"api_key\":\"your api key\",\n", + " \"api_type\":\"anthropic\",\n", + " },\n", + "]\n", + "```" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Implementation of AnthropicClient\n", - "\n", - "You can find the introduction to Claude-3-Opus model [here](https://docs.anthropic.com/claude/docs/intro-to-claude). \n", - "\n", - "Since anthropic provides their Python SDK with similar structure as OpenAI's, we will following the implementation from `autogen.oai.client.OpenAIClient`.\n", - "\n" + "### Alternative\n", + "\n", + "As an alternative to the api_key key and value in the config, you can set the environment variable `ANTHROPIC_API_KEY` to your Anthropic API key.\n", + "\n", + "Linux/Mac:\n", + "```\n", + "export ANTHROPIC_API_KEY=\"your Anthropic API key here\"\n", + "```\n", + "Windows:\n", + "```\n", + "set ANTHROPIC_API_KEY=your_anthropic_api_key_here\n", + "```" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "class AnthropicClient:\n", - " def __init__(self, config: Dict[str, Any]):\n", - " self._config = config\n", - " self.model = config[\"model\"]\n", - " anthropic_kwargs = set(inspect.getfullargspec(Anthropic.__init__).kwonlyargs)\n", - " filter_dict = {k: v for k, v in config.items() if k in anthropic_kwargs}\n", - " self._client = Anthropic(**filter_dict)\n", - "\n", - " self._last_tooluse_status = {}\n", - "\n", - " def message_retrieval(\n", - " self, response: Union[Message, ToolsBetaMessage]\n", - " ) -> Union[List[str], List[ChatCompletionMessage]]:\n", - " \"\"\"Retrieve the messages from the response.\"\"\"\n", - " messages = response.content\n", - " if len(messages) == 0:\n", - " return [None]\n", - " res = []\n", - " if TOOL_ENABLED:\n", - " for choice in messages:\n", - " if choice.type == \"tool_use\":\n", - " res.insert(0, self.response_to_openai_message(choice))\n", - " self._last_tooluse_status[\"tool_use\"] = choice.model_dump()\n", - " else:\n", - " res.append(choice.text)\n", - " self._last_tooluse_status[\"think\"] = choice.text\n", - "\n", - " return res\n", - "\n", - " else:\n", - " return [ # type: ignore [return-value]\n", - " choice.text if choice.message.function_call is not None else choice.message.content # type: ignore [union-attr]\n", - " for choice in messages\n", - " ]\n", - "\n", - " def create(self, params: Dict[str, Any]) -> Completion:\n", - " \"\"\"Create a completion for a given config.\n", - "\n", - " Args:\n", - " params: The params for the completion.\n", - "\n", - " Returns:\n", - " The completion.\n", - " \"\"\"\n", - " if \"tools\" in params:\n", - " converted_functions = self.convert_tools_to_functions(params[\"tools\"])\n", - " params[\"functions\"] = params.get(\"functions\", []) + converted_functions\n", - "\n", - " raw_contents = params[\"messages\"]\n", - " processed_messages = []\n", - " for message in raw_contents:\n", - "\n", - " if message[\"role\"] == \"system\":\n", - " params[\"system\"] = message[\"content\"]\n", - " elif message[\"role\"] == \"function\":\n", - " processed_messages.append(self.return_function_call_result(message[\"content\"]))\n", - " elif \"function_call\" in message:\n", - " processed_messages.append(self.restore_last_tooluse_status())\n", - " elif message[\"content\"] == \"\":\n", - " # I'm not sure how to elegantly terminate the conversation, please give me some advice about this.\n", - " message[\"content\"] = \"I'm done. Please send TERMINATE\"\n", - " processed_messages.append(message)\n", - " else:\n", - " processed_messages.append(message)\n", - "\n", - " params[\"messages\"] = processed_messages\n", - "\n", - " if TOOL_ENABLED and \"functions\" in params:\n", - " completions: Completion = self._client.beta.tools.messages\n", - " else:\n", - " completions: Completion = self._client.messages # type: ignore [attr-defined]\n", - "\n", - " # Not yet support stream\n", - " params = params.copy()\n", - " params[\"stream\"] = False\n", - " params.pop(\"model_client_cls\")\n", - " params[\"max_tokens\"] = params.get(\"max_tokens\", 4096)\n", - " if \"functions\" in params:\n", - " tools_configs = params.pop(\"functions\")\n", - " tools_configs = [self.openai_func_to_anthropic(tool) for tool in tools_configs]\n", - " params[\"tools\"] = tools_configs\n", - " response = completions.create(**params)\n", - "\n", - " return response\n", - "\n", - " def cost(self, response: Completion) -> float:\n", - " \"\"\"Calculate the cost of the response.\"\"\"\n", - " total = 0.0\n", - " tokens = {\n", - " \"input\": response.usage.input_tokens if response.usage is not None else 0,\n", - " \"output\": response.usage.output_tokens if response.usage is not None else 0,\n", - " }\n", - " price_per_million = {\n", - " \"input\": 15,\n", - " \"output\": 75,\n", - " }\n", - " for key, value in tokens.items():\n", - " total += value * price_per_million[key] / 1_000_000\n", - "\n", - " return total\n", - "\n", - " def response_to_openai_message(self, response) -> ChatCompletionMessage:\n", - " dict_response = response.model_dump()\n", - " return ChatCompletionMessage(\n", - " content=None,\n", - " role=\"assistant\",\n", - " function_call={\"name\": dict_response[\"name\"], \"arguments\": json.dumps(dict_response[\"input\"])},\n", - " )\n", - "\n", - " def restore_last_tooluse_status(self) -> Dict:\n", - " cached_content = []\n", - " if \"think\" in self._last_tooluse_status:\n", - " cached_content.append({\"type\": \"text\", \"text\": self._last_tooluse_status[\"think\"]})\n", - " cached_content.append(self._last_tooluse_status[\"tool_use\"])\n", - " res = {\"role\": \"assistant\", \"content\": cached_content}\n", - " return res\n", - "\n", - " def return_function_call_result(self, result: str) -> Dict:\n", - " return {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_result\",\n", - " \"tool_use_id\": self._last_tooluse_status[\"tool_use\"][\"id\"],\n", - " \"content\": result,\n", - " }\n", - " ],\n", - " }\n", - "\n", - " @staticmethod\n", - " def openai_func_to_anthropic(openai_func: dict) -> dict:\n", - " res = openai_func.copy()\n", - " res[\"input_schema\"] = res.pop(\"parameters\")\n", - " return res\n", - "\n", - " @staticmethod\n", - " def get_usage(response: Completion) -> Dict:\n", - " return {\n", - " \"prompt_tokens\": response.usage.input_tokens if response.usage is not None else 0,\n", - " \"completion_tokens\": response.usage.output_tokens if response.usage is not None else 0,\n", - " \"total_tokens\": (\n", - " response.usage.input_tokens + response.usage.output_tokens if response.usage is not None else 0\n", - " ),\n", - " \"cost\": response.cost if hasattr(response, \"cost\") else 0,\n", - " \"model\": response.model,\n", - " }\n", - "\n", - " @staticmethod\n", - " def convert_tools_to_functions(tools: List) -> List:\n", - " functions = []\n", - " for tool in tools:\n", - " if tool.get(\"type\") == \"function\" and \"function\" in tool:\n", - " functions.append(tool[\"function\"])\n", - "\n", - " return functions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Set the config for the Anthropic API\n", + "import os\n", "\n", - "You can add any parameters that are needed for the custom model loading in the same configuration list.\n", + "from typing_extensions import Annotated\n", "\n", - "It is important to add the `model_client_cls` field and set it to a string that corresponds to the class name: `\"CustomModelClient\"`." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", + "import autogen\n", "\n", "config_list_claude = [\n", " {\n", @@ -317,13 +117,18 @@ " \"model\": \"claude-3-sonnet-20240229\",\n", " # You need to provide your API key here.\n", " \"api_key\": os.getenv(\"ANTHROPIC_API_KEY\"),\n", - " \"base_url\": \"https://api.anthropic.com\",\n", " \"api_type\": \"anthropic\",\n", - " \"model_client_cls\": \"AnthropicClient\",\n", " }\n", "]" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Coding Example with Two Agent" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -336,26 +141,18 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[autogen.oai.client: 04-08 22:15:59] {419} INFO - Detected custom model client in config: AnthropicClient, model client can not be used until register_model_client is called.\n" - ] - } - ], + "outputs": [], "source": [ - "assistant = AssistantAgent(\n", + "assistant = autogen.AssistantAgent(\n", " \"assistant\",\n", " llm_config={\n", " \"config_list\": config_list_claude,\n", " },\n", ")\n", "\n", - "user_proxy = UserProxyAgent(\n", + "user_proxy = autogen.UserProxyAgent(\n", " \"user_proxy\",\n", " human_input_mode=\"NEVER\",\n", " code_execution_config={\n", @@ -371,95 +168,54 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Function Call in Latest Anthropic API \n", - "Anthropic just announced that tool use is now in public beta in the Anthropic API. To use this feature, please install `anthropic>=0.23.1`." + "## Initiate Chat" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[autogen.oai.client: 04-08 22:15:59] {419} INFO - Detected custom model client in config: AnthropicClient, model client can not be used until register_model_client is called.\n" - ] - } - ], + "outputs": [], "source": [ - "@user_proxy.register_for_execution()\n", - "@assistant.register_for_llm(name=\"get_weather\", description=\"Get the current weather in a given location.\")\n", - "def preprocess(location: Annotated[str, \"The city and state, e.g. Toronto, ON.\"]) -> str:\n", - " return \"Absolutely cloudy and rainy\"" + "user_proxy.initiate_chat(\n", + " assistant, message=\"Write a python program to print the first 10 numbers of the Fibonacci sequence.\"\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Register the custom client class to the assistant agent" + "# Function Call in Latest Anthropic API \n", + "Anthropic just announced that tool use is now supported in the Anthropic API. To use this feature, please install `anthropic>=0.23.1`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Register the function" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "assistant.register_model_client(model_client_cls=AnthropicClient)" + "@user_proxy.register_for_execution() # Decorator factory for registering a function to be executed by an agent\n", + "@assistant.register_for_llm(\n", + " name=\"get_weather\", description=\"Get the current weather in a given location.\"\n", + ") # Decorator factory for registering a function to be used by an agent\n", + "def preprocess(location: Annotated[str, \"The city and state, e.g. Toronto, ON.\"]) -> str:\n", + " return \"Absolutely cloudy and rainy\"" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "user_proxy (to assistant):\n", - "\n", - "What's the weather in Toronto?\n", - "\n", - "--------------------------------------------------------------------------------\n", - "assistant (to user_proxy):\n", - "\n", - "***** Suggested function call: get_weather *****\n", - "Arguments: \n", - "{\"location\": \"Toronto, ON\"}\n", - "************************************************\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\n", - ">>>>>>>> EXECUTING FUNCTION get_weather...\n", - "user_proxy (to assistant):\n", - "\n", - "***** Response from calling function (get_weather) *****\n", - "Absolutely cloudy and rainy\n", - "********************************************************\n", - "\n", - "--------------------------------------------------------------------------------\n", - "assistant (to user_proxy):\n", - "\n", - "The tool returned that the current weather in Toronto, ON is absolutely cloudy and rainy.\n", - "\n", - "--------------------------------------------------------------------------------\n" - ] - }, - { - "data": { - "text/plain": [ - "ChatResult(chat_id=None, chat_history=[{'content': \"What's the weather in Toronto?\", 'role': 'assistant'}, {'function_call': {'arguments': '{\"location\": \"Toronto, ON\"}', 'name': 'get_weather'}, 'content': None, 'role': 'assistant'}, {'content': 'Absolutely cloudy and rainy', 'name': 'get_weather', 'role': 'function'}, {'content': 'The tool returned that the current weather in Toronto, ON is absolutely cloudy and rainy.', 'role': 'user'}], summary='The tool returned that the current weather in Toronto, ON is absolutely cloudy and rainy.', cost=({'total_cost': 0.030494999999999998, 'claude-3-sonnet-20240229': {'cost': 0.030494999999999998, 'prompt_tokens': 1533, 'completion_tokens': 100, 'total_tokens': 1633}}, {'total_cost': 0}), human_input=[])" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "user_proxy.initiate_chat(\n", " assistant,\n", @@ -490,7 +246,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.13" }, "vscode": { "interpreter": {