diff --git a/python/packages/autogen-core/src/autogen_core/models/_model_client.py b/python/packages/autogen-core/src/autogen_core/models/_model_client.py index efe8627b1faf..2a32189583a3 100644 --- a/python/packages/autogen-core/src/autogen_core/models/_model_client.py +++ b/python/packages/autogen-core/src/autogen_core/models/_model_client.py @@ -31,9 +31,9 @@ class ModelFamily: CLAUDE_3_HAIKU = "claude-3-haiku" CLAUDE_3_SONNET = "claude-3-sonnet" CLAUDE_3_OPUS = "claude-3-opus" - CLAUDE_3_5_HAIKU = "claude-3.5-haiku" - CLAUDE_3_5_SONNET = "claude-3.5-sonnet" - CLAUDE_3_7_SONNET = "claude-3.7-sonnet" + CLAUDE_3_5_HAIKU = "claude-3-5-haiku" + CLAUDE_3_5_SONNET = "claude-3-5-sonnet" + CLAUDE_3_7_SONNET = "claude-3-7-sonnet" UNKNOWN = "unknown" ANY: TypeAlias = Literal[ @@ -50,8 +50,9 @@ class ModelFamily: "claude-3-haiku", "claude-3-sonnet", "claude-3-opus", - "claude-3.5-haiku", - "claude-3.5-sonnet", + "claude-3-5-haiku", + "claude-3-5-sonnet", + "claude-3-7-sonnet", "unknown", ] @@ -66,6 +67,7 @@ def is_claude(family: str) -> bool: ModelFamily.CLAUDE_3_OPUS, ModelFamily.CLAUDE_3_5_HAIKU, ModelFamily.CLAUDE_3_5_SONNET, + ModelFamily.CLAUDE_3_7_SONNET, ) @staticmethod diff --git a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py index 11b8777dfe88..9cb80e48fd8f 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py @@ -12,6 +12,7 @@ AsyncGenerator, Coroutine, Dict, + Iterable, List, Literal, Mapping, @@ -20,6 +21,7 @@ Set, Union, cast, + overload, ) import tiktoken @@ -142,20 +144,41 @@ def get_mime_type_from_image(image: Image) -> Literal["image/jpeg", "image/png", return "image/jpeg" +@overload +def __empty_content_to_whitespace(content: str) -> str: ... + + +@overload +def __empty_content_to_whitespace(content: List[Any]) -> Iterable[Any]: ... + + +def __empty_content_to_whitespace( + content: Union[str, List[Union[str, Image]]], +) -> Union[str, Iterable[Any]]: + if isinstance(content, str) and not content.strip(): + return " " + elif isinstance(content, list) and not any(isinstance(x, str) and not x.strip() for x in content): + for idx, message in enumerate(content): + if isinstance(message, str) and not message.strip(): + content[idx] = " " + + return content + + def user_message_to_anthropic(message: UserMessage) -> MessageParam: assert_valid_name(message.source) if isinstance(message.content, str): return { "role": "user", - "content": message.content, + "content": __empty_content_to_whitespace(message.content), } else: blocks: List[Union[TextBlockParam, ImageBlockParam]] = [] for part in message.content: if isinstance(part, str): - blocks.append(TextBlockParam(type="text", text=part)) + blocks.append(TextBlockParam(type="text", text=__empty_content_to_whitespace(part))) elif isinstance(part, Image): blocks.append( ImageBlockParam( @@ -177,7 +200,7 @@ def user_message_to_anthropic(message: UserMessage) -> MessageParam: def system_message_to_anthropic(message: SystemMessage) -> str: - return message.content + return __empty_content_to_whitespace(message.content) def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam: @@ -190,6 +213,7 @@ def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam: for func_call in message.content: # Parse the arguments and convert to dict if it's a JSON string args = func_call.arguments + args = __empty_content_to_whitespace(args) if isinstance(args, str): try: args_dict = json.loads(args) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py index e09ff22d3ab7..2241f663af26 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py @@ -1,3 +1,4 @@ +from . import _message_transform from ._openai_client import ( AZURE_OPENAI_USER_AGENT, AzureOpenAIChatCompletionClient, @@ -20,4 +21,5 @@ "BaseOpenAIClientConfigurationConfigModel", "CreateArgumentsConfigModel", "AZURE_OPENAI_USER_AGENT", + "_message_transform", ] diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py new file mode 100644 index 000000000000..64dc9e0e0df5 --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -0,0 +1,357 @@ +from typing import Any, Callable, Dict, List, cast, get_args + +from autogen_core import ( + FunctionCall, + Image, +) +from autogen_core.models import ( + AssistantMessage, + FunctionExecutionResultMessage, + LLMMessage, + ModelFamily, + SystemMessage, + UserMessage, +) +from openai.types.chat import ( + ChatCompletionAssistantMessageParam, + ChatCompletionContentPartImageParam, + ChatCompletionContentPartParam, + ChatCompletionContentPartTextParam, + ChatCompletionMessageToolCallParam, + ChatCompletionSystemMessageParam, + ChatCompletionToolMessageParam, + ChatCompletionUserMessageParam, +) + +from ._transformation import ( + LLMMessageContent, + TransformerMap, + TrasformerReturnType, + build_conditional_transformer_func, + build_transformer_func, + register_transformer, +) +from ._utils import assert_valid_name + +EMPTY: Dict[str, Any] = {} + + +def func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: + return ChatCompletionMessageToolCallParam( + id=message.id, + function={ + "arguments": message.arguments, + "name": message.name, + }, + type="function", + ) + + +# ===Mini Transformers=== +def _assert_valid_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, None]: + assert isinstance(message, (UserMessage, AssistantMessage)) + assert_valid_name(message.source) + return EMPTY + + +def _set_role(role: str) -> Callable[[LLMMessage, Dict[str, Any]], Dict[str, str]]: + def inner(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str]: + return {"role": role} + + return inner + + +def _set_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str]: + assert isinstance(message, (UserMessage, AssistantMessage)) + assert_valid_name(message.source) + return {"name": message.source} + + +def _set_content_direct(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, LLMMessageContent]: + return {"content": message.content} + + +def _set_prepend_text_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str]: + assert isinstance(message, (UserMessage, AssistantMessage)) + assert isinstance(message.content, str) + prepend = context.get("prepend_name", False) + prefix = f"{message.source} said:\n" if prepend else "" + return {"content": prefix + message.content} + + +def _set_multimodal_content( + message: LLMMessage, context: Dict[str, Any] +) -> Dict[str, List[ChatCompletionContentPartParam]]: + assert isinstance(message, (UserMessage, AssistantMessage)) + prepend = context.get("prepend_name", False) + parts: List[ChatCompletionContentPartParam] = [] + + for idx, part in enumerate(message.content): + if isinstance(part, str): + # If prepend, Append the name to the first text part + text = f"{message.source} said:\n" + part if prepend and idx == 0 else part + parts.append(ChatCompletionContentPartTextParam(type="text", text=text)) + elif isinstance(part, Image): + # TODO: support url based images + # TODO: support specifying details + parts.append(cast(ChatCompletionContentPartImageParam, part.to_openai_format())) + else: + raise ValueError(f"Unknown content part: {part}") + + return {"content": parts} + + +def _set_tool_calls( + message: LLMMessage, context: Dict[str, Any] +) -> Dict[str, List[ChatCompletionMessageToolCallParam]]: + assert isinstance(message.content, list) + assert isinstance(message, AssistantMessage) + return { + "tool_calls": [func_call_to_oai(x) for x in message.content], + } + + +def _set_thought_as_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str | None]: + assert isinstance(message, AssistantMessage) + return {"content": message.thought} + + +def _set_thought_as_content_gemini(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str | None]: + assert isinstance(message, AssistantMessage) + return {"content": message.thought or " "} + + +def _set_empty_to_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, LLMMessageContent]: + return {"content": message.content or " "} + + +def _set_pass_message_when_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, bool]: + if isinstance(message.content, str) and (message.content.isspace() or not message.content): + return {"pass_message": True} + return {} + + +# === Base Transformers list === +base_system_message_transformers: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = [ + _set_content_direct, + _set_role("system"), +] + +base_user_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = [ + _assert_valid_name, + _set_name, + _set_role("user"), +] + +base_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = [ + _assert_valid_name, + _set_name, + _set_role("assistant"), +] + + +# === Transformers list === +system_message_transformers: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_system_message_transformers +) + +single_user_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_user_transformer_funcs + + [ + _set_prepend_text_content, + ] +) + +multimodal_user_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_user_transformer_funcs + + [ + _set_multimodal_content, + ] +) + +single_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_assistant_transformer_funcs + + [ + _set_content_direct, + ] +) + +tools_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_assistant_transformer_funcs + + [ + _set_tool_calls, + ] +) + +thought_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + tools_assistant_transformer_funcs + + [ + _set_thought_as_content, + ] +) + +thought_assistant_transformer_funcs_gemini: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + tools_assistant_transformer_funcs + + [ + _set_thought_as_content_gemini, + ] +) + + +# === Specific message param functions === + + +# === Transformer maps === +user_transformer_funcs: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_user_transformer_funcs, + "multimodal": multimodal_user_transformer_funcs, +} +user_transformer_constructors: Dict[str, Callable[..., Any]] = { + "text": ChatCompletionUserMessageParam, + "multimodal": ChatCompletionUserMessageParam, +} + + +def user_condition(message: LLMMessage, context: Dict[str, Any]) -> str: + if isinstance(message.content, str): + return "text" + else: + return "multimodal" + + +assistant_transformer_funcs: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_assistant_transformer_funcs, + "tools": tools_assistant_transformer_funcs, + "thought": thought_assistant_transformer_funcs, +} +assistant_transformer_constructors: Dict[str, Callable[..., Any]] = { + "text": ChatCompletionAssistantMessageParam, + "tools": ChatCompletionAssistantMessageParam, + "thought": ChatCompletionAssistantMessageParam, +} + + +def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: + assert isinstance(message, AssistantMessage) + if isinstance(message.content, list): + if message.thought is not None: + return "thought" + else: + return "tools" + else: + return "text" + + +user_transformer_funcs_gemini: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_user_transformer_funcs + [_set_empty_to_whitespace], + "multimodal": multimodal_user_transformer_funcs + [_set_empty_to_whitespace], +} + + +assistant_transformer_funcs_gemini: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_assistant_transformer_funcs + [_set_empty_to_whitespace], + "tools": tools_assistant_transformer_funcs, # that case, message.content is a list of FunctionCall + "thought": thought_assistant_transformer_funcs_gemini, # that case, message.content is a list of FunctionCall +} + + +user_transformer_funcs_claude: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_user_transformer_funcs + [_set_pass_message_when_whitespace], + "multimodal": multimodal_user_transformer_funcs + [_set_pass_message_when_whitespace], +} + + +assistant_transformer_funcs_claude: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_assistant_transformer_funcs + [_set_pass_message_when_whitespace], + "tools": tools_assistant_transformer_funcs, # that case, message.content is a list of FunctionCall + "thought": thought_assistant_transformer_funcs_gemini, # that case, message.content is a list of FunctionCall +} + + +def function_execution_result_message(message: LLMMessage, context: Dict[str, Any]) -> TrasformerReturnType: + assert isinstance(message, FunctionExecutionResultMessage) + return [ + ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content + ] + + +# === Transformers === + +__BASE_TRANSFORMER_MAP: TransformerMap = { + SystemMessage: build_transformer_func( + funcs=system_message_transformers, + message_param_func=ChatCompletionSystemMessageParam, + ), + UserMessage: build_conditional_transformer_func( + funcs_map=user_transformer_funcs, + message_param_func_map=user_transformer_constructors, + condition_func=user_condition, + ), + AssistantMessage: build_conditional_transformer_func( + funcs_map=assistant_transformer_funcs, + message_param_func_map=assistant_transformer_constructors, + condition_func=assistant_condition, + ), + FunctionExecutionResultMessage: function_execution_result_message, +} + +__GEMINI_TRANSFORMER_MAP: TransformerMap = { + SystemMessage: build_transformer_func( + funcs=system_message_transformers + [_set_empty_to_whitespace], + message_param_func=ChatCompletionSystemMessageParam, + ), + UserMessage: build_conditional_transformer_func( + funcs_map=user_transformer_funcs_gemini, + message_param_func_map=user_transformer_constructors, + condition_func=user_condition, + ), + AssistantMessage: build_conditional_transformer_func( + funcs_map=assistant_transformer_funcs_gemini, + message_param_func_map=assistant_transformer_constructors, + condition_func=assistant_condition, + ), + FunctionExecutionResultMessage: function_execution_result_message, +} + +__CLAUDE_TRANSFORMER_MAP: TransformerMap = { + SystemMessage: build_transformer_func( + funcs=system_message_transformers + [_set_empty_to_whitespace], + message_param_func=ChatCompletionSystemMessageParam, + ), + UserMessage: build_conditional_transformer_func( + funcs_map=user_transformer_funcs_claude, + message_param_func_map=user_transformer_constructors, + condition_func=user_condition, + ), + AssistantMessage: build_conditional_transformer_func( + funcs_map=assistant_transformer_funcs_claude, + message_param_func_map=assistant_transformer_constructors, + condition_func=assistant_condition, + ), + FunctionExecutionResultMessage: function_execution_result_message, +} + + +# set openai models to use the transformer map +total_models = get_args(ModelFamily.ANY) +__openai_models = [model for model in total_models if ModelFamily.is_openai(model)] + +__claude_models = [model for model in total_models if ModelFamily.is_claude(model)] + +__gemini_models = [model for model in total_models if ModelFamily.is_gemini(model)] + +__unknown_models = list(set(total_models) - set(__openai_models) - set(__claude_models) - set(__gemini_models)) + +for model in __openai_models: + register_transformer("openai", model, __BASE_TRANSFORMER_MAP) + +for model in __claude_models: + register_transformer("openai", model, __CLAUDE_TRANSFORMER_MAP) + +for model in __gemini_models: + register_transformer("openai", model, __GEMINI_TRANSFORMER_MAP) + +for model in __unknown_models: + register_transformer("openai", model, __BASE_TRANSFORMER_MAP) + +register_transformer("openai", "default", __BASE_TRANSFORMER_MAP) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py index c36ea42bf557..ae537fd6c71b 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py @@ -21,9 +21,9 @@ "claude-3-haiku": "claude-3-haiku-20240307", "claude-3-sonnet": "claude-3-sonnet-20240229", "claude-3-opus": "claude-3-opus-20240229", - "claude-3.5-haiku": "claude-3-5-haiku-20241022", - "claude-3.5-sonnet": "claude-3-5-sonnet-20241022", - "claude-3.7-sonnet": "claude-3-7-sonnet-20250219", + "claude-3-5-haiku": "claude-3-5-haiku-20241022", + "claude-3-5-sonnet": "claude-3-5-sonnet-20241022", + "claude-3-7-sonnet": "claude-3-7-sonnet-20250219", } _MODEL_INFO: Dict[str, ModelInfo] = { diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index 9608aeb2348b..e79c49ffe2f0 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -12,6 +12,7 @@ from typing import ( Any, AsyncGenerator, + Callable, Dict, List, Mapping, @@ -38,7 +39,6 @@ ChatCompletionClient, ChatCompletionTokenLogprob, CreateResult, - FunctionExecutionResultMessage, LLMMessage, ModelCapabilities, # type: ignore ModelFamily, @@ -53,18 +53,11 @@ from openai import NOT_GIVEN, AsyncAzureOpenAI, AsyncOpenAI from openai.types.chat import ( ChatCompletion, - ChatCompletionAssistantMessageParam, ChatCompletionChunk, - ChatCompletionContentPartImageParam, ChatCompletionContentPartParam, - ChatCompletionContentPartTextParam, ChatCompletionMessageParam, - ChatCompletionMessageToolCallParam, ChatCompletionRole, - ChatCompletionSystemMessageParam, - ChatCompletionToolMessageParam, ChatCompletionToolParam, - ChatCompletionUserMessageParam, ParsedChatCompletion, ParsedChoice, completion_create_params, @@ -82,6 +75,10 @@ from .._utils.normalize_stop_reason import normalize_stop_reason from .._utils.parse_r1_content import parse_r1_content from . import _model_info +from ._transformation import ( + get_transformer, +) +from ._utils import assert_valid_name from .config import ( AzureOpenAIClientConfiguration, AzureOpenAIClientConfigurationConfigModel, @@ -164,105 +161,22 @@ def type_to_role(message: LLMMessage) -> ChatCompletionRole: return "tool" -def user_message_to_oai(message: UserMessage, prepend_name: bool = False) -> ChatCompletionUserMessageParam: - assert_valid_name(message.source) - if isinstance(message.content, str): - return ChatCompletionUserMessageParam( - content=(f"{message.source} said:\n" if prepend_name else "") + message.content, - role="user", - name=message.source, - ) - else: - parts: List[ChatCompletionContentPartParam] = [] - for part in message.content: - if isinstance(part, str): - if prepend_name: - # Append the name to the first text part - oai_part = ChatCompletionContentPartTextParam( - text=f"{message.source} said:\n" + part, - type="text", - ) - prepend_name = False - else: - oai_part = ChatCompletionContentPartTextParam( - text=part, - type="text", - ) - parts.append(oai_part) - elif isinstance(part, Image): - # TODO: support url based images - # TODO: support specifying details - parts.append(cast(ChatCompletionContentPartImageParam, part.to_openai_format())) - else: - raise ValueError(f"Unknown content type: {part}") - return ChatCompletionUserMessageParam( - content=parts, - role="user", - name=message.source, - ) - - -def system_message_to_oai(message: SystemMessage) -> ChatCompletionSystemMessageParam: - return ChatCompletionSystemMessageParam( - content=message.content, - role="system", - ) +def to_oai_type( + message: LLMMessage, prepend_name: bool = False, model_family: str = "gpt-4o" +) -> Sequence[ChatCompletionMessageParam]: + context = { + "prepend_name": prepend_name, + } + transformers = get_transformer("openai", model_family) + def raise_value_error(message: LLMMessage, context: Dict[str, Any]) -> Sequence[ChatCompletionMessageParam]: + raise ValueError(f"Unknown message type: {type(message)}") -def func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: - return ChatCompletionMessageToolCallParam( - id=message.id, - function={ - "arguments": message.arguments, - "name": message.name, - }, - type="function", + transformer: Callable[[LLMMessage, Dict[str, Any]], Sequence[ChatCompletionMessageParam]] = transformers.get( + type(message), raise_value_error ) - - -def tool_message_to_oai( - message: FunctionExecutionResultMessage, -) -> Sequence[ChatCompletionToolMessageParam]: - return [ - ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content - ] - - -def assistant_message_to_oai( - message: AssistantMessage, -) -> ChatCompletionAssistantMessageParam: - assert_valid_name(message.source) - if isinstance(message.content, list): - if message.thought is not None: - return ChatCompletionAssistantMessageParam( - content=message.thought, - tool_calls=[func_call_to_oai(x) for x in message.content], - role="assistant", - name=message.source, - ) - else: - return ChatCompletionAssistantMessageParam( - tool_calls=[func_call_to_oai(x) for x in message.content], - role="assistant", - name=message.source, - ) - else: - return ChatCompletionAssistantMessageParam( - content=message.content, - role="assistant", - name=message.source, - ) - - -def to_oai_type(message: LLMMessage, prepend_name: bool = False) -> Sequence[ChatCompletionMessageParam]: - if isinstance(message, SystemMessage): - return [system_message_to_oai(message)] - elif isinstance(message, UserMessage): - return [user_message_to_oai(message, prepend_name)] - elif isinstance(message, AssistantMessage): - return [assistant_message_to_oai(message)] - else: - return tool_message_to_oai(message) + result = transformer(message, context) + return result def calculate_vision_tokens(image: Image, detail: str = "auto") -> int: @@ -360,19 +274,6 @@ def normalize_name(name: str) -> str: return re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:64] -def assert_valid_name(name: str) -> str: - """ - Ensure that configured names are valid, raises ValueError if not. - - For munging LLM responses use _normalize_name to ensure LLM specified names don't break the API. - """ - if not re.match(r"^[a-zA-Z0-9_-]+$", name): - raise ValueError(f"Invalid name: {name}. Only letters, numbers, '_' and '-' are allowed.") - if len(name) > 64: - raise ValueError(f"Invalid name: {name}. Name must be less than 64 characters.") - return name - - def count_tokens_openai( messages: Sequence[LLMMessage], model: str, @@ -392,7 +293,7 @@ def count_tokens_openai( # Message tokens. for message in messages: num_tokens += tokens_per_message - oai_message = to_oai_type(message, prepend_name=add_name_prefixes) + oai_message = to_oai_type(message, prepend_name=add_name_prefixes, model_family=model) for oai_message_part in oai_message: for key, value in oai_message_part.items(): if value is None: @@ -638,7 +539,11 @@ def _process_create_args( _messages.insert(_first_system_message_idx, system_message) messages = _messages - oai_messages_nested = [to_oai_type(m, prepend_name=self._add_name_prefixes) for m in messages] + oai_messages_nested = [ + to_oai_type(m, prepend_name=self._add_name_prefixes, model_family=create_args.get("model", "unknown")) + for m in messages + ] + oai_messages = [item for sublist in oai_messages_nested for item in sublist] if self.model_info["function_calling"] is False and len(tools) > 0: diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py new file mode 100644 index 000000000000..dc21b9c10815 --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py @@ -0,0 +1,27 @@ +from .registry import ( + MESSAGE_TRANSFORMERS, + build_conditional_transformer_func, + build_transformer_func, + get_transformer, + register_transformer, +) +from .types import ( + LLMMessageContent, + MessageParam, + TransformerFunc, + TransformerMap, + TrasformerReturnType, +) + +__all__ = [ + "register_transformer", + "get_transformer", + "build_transformer_func", + "build_conditional_transformer_func", + "MESSAGE_TRANSFORMERS", + "TransformerMap", + "TransformerFunc", + "MessageParam", + "LLMMessageContent", + "TrasformerReturnType", +] diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py new file mode 100644 index 000000000000..b2e659158256 --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py @@ -0,0 +1,117 @@ +from collections import defaultdict +from typing import Any, Callable, Dict, List + +from autogen_core.models import LLMMessage + +from .types import ( + TransformerFunc, + TransformerMap, +) + +# Global registry of model family → message transformer map +# Each model family (e.g. "gpt-4o", "gemini-1.5-flash") maps to a dict of LLMMessage type → transformer function +MESSAGE_TRANSFORMERS: Dict[str, Dict[str, TransformerMap]] = defaultdict(dict) + + +def build_transformer_func( + funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]], message_param_func: Callable[..., Any] +) -> TransformerFunc: + """ + Combines multiple transformer functions into a single transformer. + + Each `func` must accept a message and a context dict, and return a partial dict + of keyword arguments. These are merged and passed to `message_param_func`. + + This structure allows flexible transformation pipelines and future extensibility + (e.g., prepend name, insert metadata, etc). + + message_param_func: A model-specific constructor (e.g. ChatCompletionMessageParam). + Signature is intentionally open: Callable[..., Any]. + """ + + def transformer_func(message: LLMMessage, context: Any) -> Any: + kwargs: Dict[str, Any] = {} + for func in funcs: + kwargs.update(func(message, context)) + return [message_param_func(**kwargs)] + + return transformer_func + + +def build_conditional_transformer_func( + funcs_map: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]], + message_param_func_map: Dict[str, Callable[..., Any]], + condition_func: Callable[[LLMMessage, Dict[str, Any]], str], +) -> TransformerFunc: + """ + Combines multiple transformer functions into a single transformer, with a conditional constructor. + + Each `func` must accept a message and a context dict, and return a partial dict + of keyword arguments. These are merged and passed to the constructor selected by `condition_func`. + + This structure allows flexible transformation pipelines and future extensibility + (e.g., prepend name, insert metadata, etc). + + message_param_func_map: A mapping of condition → constructor function. + condition_func: A function that returns the condition for selecting the constructor. + """ + + def transformer(message: LLMMessage, context: Dict[str, Any]) -> Any: + condition = condition_func(message, context) + message_param_func = message_param_func_map[condition] + kwargs: Dict[str, Any] = {} + for func in funcs_map[condition]: + kwargs.update(func(message, context)) + if kwargs.get("pass_message", False): + return [] + return [message_param_func(**kwargs)] + + return transformer + + +def register_transformer(api: str, model_family: str, transformer_map: TransformerMap) -> None: + """ + Registers a transformer map for a given model family. + + Example: + register_transformer("gpt-4o", { + UserMessage: user_message_to_oai, + SystemMessage: system_message_to_oai, + }) + """ + MESSAGE_TRANSFORMERS[api][model_family] = transformer_map + + +def _find_model_family(api: str, model: str) -> str: + """ + Finds the best matching model family for the given model. + Search via prefix matching (e.g. "gpt-4o" → "gpt-4o-1.0"). + """ + for family in MESSAGE_TRANSFORMERS[api].keys(): + if model.startswith(family): + return family + return "default" + + +def get_transformer(api: str, model_family: str) -> TransformerMap: + """ + Returns the registered transformer map for the given model family. + + This is a thin wrapper around `MESSAGE_TRANSFORMERS.get(...)`, but serves as + an abstraction layer to allow future enhancements such as: + + - Providing fallback transformers for unknown model families + - Injecting mock transformers during testing + - Adding logging, metrics, or versioning later + + Keeping this as a function (instead of direct dict access) improves long-term flexibility. + """ + + model = _find_model_family(api, model_family) + + transformer = MESSAGE_TRANSFORMERS.get(api, {}).get(model, {}) + + if not transformer: + raise ValueError(f"No transformer found for model family '{model_family}'") + + return transformer diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py new file mode 100644 index 000000000000..9cfb28e040cc --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py @@ -0,0 +1,22 @@ +from typing import Any, Callable, Dict, List, Sequence, Type, Union + +from autogen_core import FunctionCall, Image +from autogen_core.models import LLMMessage +from autogen_core.models._types import FunctionExecutionResult +from openai.types.chat import ChatCompletionMessageParam + +MessageParam = Union[ChatCompletionMessageParam] # If that transformation move to global, add other message params here +TrasformerReturnType = Sequence[MessageParam] +TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], TrasformerReturnType] +TransformerMap = Dict[Type[LLMMessage], TransformerFunc] + +LLMMessageContent = Union[ + # SystemMessage.content + str, + # UserMessage.content + List[Union[str, Image]], + # AssistantMessage.content + List[FunctionCall], + # FunctionExecutionResultMessage.content + List[FunctionExecutionResult], +] diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py new file mode 100644 index 000000000000..8c1df22961d7 --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py @@ -0,0 +1,14 @@ +import re + + +def assert_valid_name(name: str) -> str: + """ + Ensure that configured names are valid, raises ValueError if not. + + For munging LLM responses use _normalize_name to ensure LLM specified names don't break the API. + """ + if not re.match(r"^[a-zA-Z0-9_-]+$", name): + raise ValueError(f"Invalid name: {name}. Only letters, numbers, '_' and '-' are allowed.") + if len(name) > 64: + raise ValueError(f"Invalid name: {name}. Name must be less than 64 characters.") + return name diff --git a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py index b72a57f91e55..5fd8e812feaf 100644 --- a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py @@ -339,7 +339,6 @@ async def test_anthropic_serialization() -> None: @pytest.mark.asyncio async def test_anthropic_muliple_system_message() -> None: """Test multiple system messages in a single request.""" - api_key = os.getenv("ANTHROPIC_API_KEY") if not api_key: pytest.skip("ANTHROPIC_API_KEY not found in environment variables") @@ -560,3 +559,28 @@ def test_merge_system_messages_no_duplicates() -> None: assert isinstance(merged_messages[0], SystemMessage) # 중복된 내용도 그대로 병합됨 assert merged_messages[0].content == "Same instruction\nSame instruction" + + +@pytest.mark.asyncio +async def test_empty_assistant_content_string_with_anthropic() -> None: + """Test that an empty assistant content string is handled correctly.""" + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + pytest.skip("ANTHROPIC_API_KEY not found in environment variables") + + client = AnthropicChatCompletionClient( + model="claude-3-haiku-20240307", + api_key=api_key, + ) + + # Test empty assistant content string + result = await client.create( + messages=[ + UserMessage(content="Say something", source="user"), + AssistantMessage(content="", source="assistant"), + ] + ) + + # Verify we got a response + assert isinstance(result.content, str) + assert len(result.content) > 0 diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py index 91f07d7f0a3d..abb270a712ef 100644 --- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py @@ -1602,6 +1602,10 @@ def openai_client(request: pytest.FixtureRequest) -> OpenAIChatCompletionClient: api_key = os.getenv("GEMINI_API_KEY") if not api_key: pytest.skip("GEMINI_API_KEY not found in environment variables") + elif model.startswith("claude"): + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + pytest.skip("ANTHROPIC_API_KEY not found in environment variables") else: api_key = os.getenv("OPENAI_API_KEY") if not api_key: @@ -1616,7 +1620,7 @@ def openai_client(request: pytest.FixtureRequest) -> OpenAIChatCompletionClient: @pytest.mark.asyncio @pytest.mark.parametrize( "model", - ["gpt-4o-mini", "gemini-1.5-flash"], + ["gpt-4o-mini", "gemini-1.5-flash", "claude-3-5-haiku-20241022"], ) async def test_model_client_basic_completion(model: str, openai_client: OpenAIChatCompletionClient) -> None: # Test basic completion @@ -1633,7 +1637,7 @@ async def test_model_client_basic_completion(model: str, openai_client: OpenAICh @pytest.mark.asyncio @pytest.mark.parametrize( "model", - ["gpt-4o-mini", "gemini-1.5-flash"], + ["gpt-4o-mini", "gemini-1.5-flash", "claude-3-5-haiku-20241022"], ) async def test_model_client_with_function_calling(model: str, openai_client: OpenAIChatCompletionClient) -> None: # Test tool calling @@ -2065,7 +2069,7 @@ async def test_add_name_prefixes(monkeypatch: pytest.MonkeyPatch) -> None: [ "gpt-4o-mini", "gemini-1.5-flash", - # TODO: Add anthropic models when available. + "claude-3-5-haiku-20241022", ], ) async def test_muliple_system_message(model: str, openai_client: OpenAIChatCompletionClient) -> None: @@ -2303,4 +2307,64 @@ async def test_single_system_message_for_gemini_model() -> None: assert system_messages[0]["content"] == "I am the only system message" +def noop(input: str) -> str: + return "done" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gemini-1.5-flash"]) +async def test_empty_assistant_content_with_gemini(model: str, openai_client: OpenAIChatCompletionClient) -> None: + # Test tool calling + tool = FunctionTool(noop, name="noop", description="No-op tool") + messages: List[LLMMessage] = [UserMessage(content="Call noop", source="user")] + result = await openai_client.create(messages=messages, tools=[tool]) + assert isinstance(result.content, list) + tool_call = result.content[0] + assert isinstance(tool_call, FunctionCall) + + # reply with empty string as thought (== content) + messages.append(AssistantMessage(content=result.content, thought="", source="assistant")) + messages.append( + FunctionExecutionResultMessage( + content=[ + FunctionExecutionResult( + content="done", + call_id=tool_call.id, + is_error=False, + name=tool_call.name, + ) + ] + ) + ) + + # This will crash if _set_empty_to_whitespace is not applied to "thought" + result = await openai_client.create(messages=messages) + assert isinstance(result.content, str) + assert result.content.strip() != "" or result.content == " " + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model", + [ + "gpt-4o-mini", + "gemini-1.5-flash", + "claude-3-5-haiku-20241022", + ], +) +async def test_empty_assistant_content_string_with_some_model( + model: str, openai_client: OpenAIChatCompletionClient +) -> None: + # message: assistant is response empty content + messages: list[LLMMessage] = [ + UserMessage(content="Say something", source="user"), + AssistantMessage(content="test", source="assistant"), + UserMessage(content="", source="user"), + ] + + # This will crash if _set_empty_to_whitespace is not applied to "content" + result = await openai_client.create(messages=messages) + assert isinstance(result.content, str) + + # TODO: add integration tests for Azure OpenAI using AAD token.