From 7d00b2390983487c78ddf5437ca040c9887592e9 Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Sun, 23 Mar 2025 20:23:21 +0900 Subject: [PATCH 01/15] FEAT: transformation registy --- .../autogen_ext/transformation/__init__.py | 19 ++++++ .../autogen_ext/transformation/registry.py | 64 +++++++++++++++++++ .../src/autogen_ext/transformation/types.py | 5 ++ 3 files changed, 88 insertions(+) create mode 100644 python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py create mode 100644 python/packages/autogen-ext/src/autogen_ext/transformation/registry.py create mode 100644 python/packages/autogen-ext/src/autogen_ext/transformation/types.py diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py new file mode 100644 index 000000000000..2c81afff4378 --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py @@ -0,0 +1,19 @@ +from .registry import ( + register_transformer, + get_transformer, + build_transformer_func, + MESSAGE_TRANSFORMERS, +) +from .types import ( + TransformerMap, + TransformerFunc, +) + +__all__ = [ + "register_transformer", + "get_transformer", + "build_transformer_func", + "MESSAGE_TRANSFORMERS", + "TransformerMap", + "TransformerFunc", +] \ No newline at end of file diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py new file mode 100644 index 000000000000..3718ecf4ccca --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py @@ -0,0 +1,64 @@ +from typing import Dict, List, Callable, Type, Any +from autogen_core.models import LLMMessage, ModelFamily +from .types import ( + TransformerMap, + TransformerFunc, +) + +# Global registry of model family → message transformer map +# Each model family (e.g. "gpt-4o", "gemini-1.5-flash") maps to a dict of LLMMessage type → transformer function +MESSAGE_TRANSFORMERS: Dict[str, TransformerMap] = {} + + +def build_transformer_func( + funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]], + message_param_func: Callable[..., Any] +) -> TransformerFunc: + """ + Combines multiple transformer functions into a single transformer. + + Each `func` must accept a message and a context dict, and return a partial dict + of keyword arguments. These are merged and passed to `message_param_func`. + + This structure allows flexible transformation pipelines and future extensibility + (e.g., prepend name, insert metadata, etc). + + message_param_func: A model-specific constructor (e.g. ChatCompletionMessageParam). + Signature is intentionally open: Callable[..., Any]. + """ + def transformer_func(message: LLMMessage, context: Any) -> Any: + kwargs: Dict[str, Any] = {} + for func in funcs: + kwargs.update(func(message, context)) + return message_param_func(**kwargs) + + return transformer_func + + +def register_transformer(model_family: str, transformer_map: TransformerMap): + """ + Registers a transformer map for a given model family. + + Example: + register_transformer("gpt-4o", { + UserMessage: user_message_to_oai, + SystemMessage: system_message_to_oai, + }) + """ + MESSAGE_TRANSFORMERS[model_family] = transformer_map + + +def get_transformer(model_family: str) -> TransformerMap: + """ + Returns the registered transformer map for the given model family. + + This is a thin wrapper around `MESSAGE_TRANSFORMERS.get(...)`, but serves as + an abstraction layer to allow future enhancements such as: + + - Providing fallback transformers for unknown model families + - Injecting mock transformers during testing + - Adding logging, metrics, or versioning later + + Keeping this as a function (instead of direct dict access) improves long-term flexibility. + """ + return MESSAGE_TRANSFORMERS.get(model_family, {}) diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py new file mode 100644 index 000000000000..904ecc22e742 --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py @@ -0,0 +1,5 @@ +from typing import Dict, Callable, Type, Any +from autogen_core.models import LLMMessage, ModelFamily + +TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], Any] +TransformerMap = Dict[Type[LLMMessage],TransformerFunc] \ No newline at end of file From b972f706e0083f7735d0f2b6d03163ed54df88f3 Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Sun, 23 Mar 2025 23:23:21 +0900 Subject: [PATCH 02/15] FEAT: dispatcher init --- .../autogen_ext/transformation/__init__.py | 10 +++ .../autogen_ext/transformation/dispatcher.py | 47 ++++++++++++++ .../autogen_ext/transformation/registry.py | 62 ++++++++++++++++++- .../src/autogen_ext/transformation/types.py | 7 ++- 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py index 2c81afff4378..7bd027d17404 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py @@ -1,19 +1,29 @@ from .registry import ( register_transformer, get_transformer, + register_builder, + get_builder build_transformer_func, + build_conditional_transformer_func, MESSAGE_TRANSFORMERS, ) from .types import ( TransformerMap, TransformerFunc, + BuilderMap, + BuilderFunc, ) __all__ = [ "register_transformer", "get_transformer", + "register_builder", + "get_builder", "build_transformer_func", + "build_conditional_transformer_func", "MESSAGE_TRANSFORMERS", "TransformerMap", "TransformerFunc", + "BuilderMap", + "BuilderFunc", ] \ No newline at end of file diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py b/python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py new file mode 100644 index 000000000000..8feac69eb67e --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py @@ -0,0 +1,47 @@ +from typing import Any, Callable, Dict, List +from autogen_core.models import LLMMessage + +from autogen_ext.transformation import ( + # Types + TransformerFunc, + TransformerMap, + BuilderMap, + BuilderFunc, + # Functions + get_transformer, +) + + +def dispatch_transformation( + model_family: str, + message: LLMMessage, + funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]], + context: Dict[str, Any], +) -> Any: + """ + Dispatches a transformation for a given model family. + + The transformation is selected based on the model family and the message type. + + The context dict is passed to the transformer functions. + + Returns the transformed message. + """ + transformer_map: TransformerMap = get_transformer(model_family) + transformer: TransformerFunc = transformer_map.get(type(message), lambda x, y: {}) + + builder_map: BuilderMap = get_builder(model_family) + builder: BuilderFunc = builder_map.get(type(message), lambda x, y: x) + + parts: List[Any] = [message] if isinstance(message.content, str) else message.content + contents: List[Any] = [] + for part in parts: + message: Any = transformer(part, context) + contents.append(message) + + kwargs: Dict[str, Any] = {} + for func in funcs: + kwargs.update(func(contents, context)) + + return builder(contents, kwargs) + \ No newline at end of file diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py index 3718ecf4ccca..d9464570c787 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py @@ -1,14 +1,15 @@ from typing import Dict, List, Callable, Type, Any from autogen_core.models import LLMMessage, ModelFamily -from .types import ( +from autogen_ext.transformation import ( TransformerMap, TransformerFunc, + BuilderMap, ) # Global registry of model family → message transformer map # Each model family (e.g. "gpt-4o", "gemini-1.5-flash") maps to a dict of LLMMessage type → transformer function MESSAGE_TRANSFORMERS: Dict[str, TransformerMap] = {} - +MESSAGE_BUILDERS: Dict[str, BuilderMap] = {} def build_transformer_func( funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]], @@ -35,6 +36,34 @@ def transformer_func(message: LLMMessage, context: Any) -> Any: return transformer_func +def build_conditional_transformer_func( + funcs_map: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]], + message_param_func_map: Dict[str, Callable[..., Any]], + condition_func: Callable[[LLMMessage, Dict[str, Any]], str], +) -> TransformerFunc: + """ + Combines multiple transformer functions into a single transformer, with a conditional constructor. + + Each `func` must accept a message and a context dict, and return a partial dict + of keyword arguments. These are merged and passed to the constructor selected by `condition_func`. + + This structure allows flexible transformation pipelines and future extensibility + (e.g., prepend name, insert metadata, etc). + + message_param_func_map: A mapping of condition → constructor function. + condition_func: A function that returns the condition for selecting the constructor. + """ + def transformer(message: LLMMessage, context: Dict[str, Any]) -> Any: + condition = condition_func(message, context) + constructor = message_param_func_map[condition] + kwargs: Dict[str, Any] = {} + for func in funcs_map[condition]: + kwargs.update(func(message, context)) + return constructor(**kwargs) + + return transformer + + def register_transformer(model_family: str, transformer_map: TransformerMap): """ Registers a transformer map for a given model family. @@ -62,3 +91,32 @@ def get_transformer(model_family: str) -> TransformerMap: Keeping this as a function (instead of direct dict access) improves long-term flexibility. """ return MESSAGE_TRANSFORMERS.get(model_family, {}) + + +def register_builder(model_family: str, builder_map: BuilderMap): + """ + Registers a builder map for a given model family. + + Example: + register_builder("gpt-4o", { + UserMessage: user_message_to_oai, + SystemMessage: system_message_to_oai, + }) + """ + MESSAGE_BUILDERS[model_family] = builder_map + + +def get_builder(model_family: str) -> BuilderMap: + """ + Returns the registered builder map for the given model family. + + This is a thin wrapper around `MESSAGE_BUILDERS.get(...)`, but serves as + an abstraction layer to allow future enhancements such as: + + - Providing fallback builders for unknown model families + - Injecting mock builders during testing + - Adding logging, metrics, or versioning later + + Keeping this as a function (instead of direct dict access) improves long-term flexibility. + """ + return MESSAGE_BUILDERS.get(model_family, {}) \ No newline at end of file diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py index 904ecc22e742..8e3fa2897477 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py @@ -1,5 +1,8 @@ -from typing import Dict, Callable, Type, Any +from typing import Dict, List, Callable, Type, Any from autogen_core.models import LLMMessage, ModelFamily TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], Any] -TransformerMap = Dict[Type[LLMMessage],TransformerFunc] \ No newline at end of file +TransformerMap = Dict[Type[LLMMessage],TransformerFunc] + +BuilderFunc = Callable[[List[Any], Dict[str, Any]], Any] +BuilderMap = Dict[Type[LLMMessage], BuilderFunc] From d8b0ac0041120ca800e937bea11cb528399b3ce7 Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Mon, 24 Mar 2025 01:51:08 +0900 Subject: [PATCH 03/15] FEAT: new message transformation done --- .../src/autogen_ext/models/openai/__init__.py | 2 + .../models/openai/_message_transform.py | 285 ++++++++++++++++++ .../models/openai/_openai_client.py | 34 ++- .../src/autogen_ext/models/openai/_utils.py | 29 ++ .../autogen_ext/transformation/__init__.py | 20 +- .../autogen_ext/transformation/dispatcher.py | 47 --- .../autogen_ext/transformation/registry.py | 61 ++-- .../src/autogen_ext/transformation/types.py | 5 +- 8 files changed, 379 insertions(+), 104 deletions(-) create mode 100644 python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py create mode 100644 python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py delete mode 100644 python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py index 366ad831175e..6e63bfc04cd3 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py @@ -1,3 +1,4 @@ +from . import _message_transform from ._openai_client import AzureOpenAIChatCompletionClient, BaseOpenAIChatCompletionClient, OpenAIChatCompletionClient from .config import ( AzureOpenAIClientConfigurationConfigModel, @@ -14,4 +15,5 @@ "OpenAIClientConfigurationConfigModel", "BaseOpenAIClientConfigurationConfigModel", "CreateArgumentsConfigModel", + "_message_transform", ] diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py new file mode 100644 index 000000000000..8ea269c412ad --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -0,0 +1,285 @@ +from typing import Any, Callable, Dict, List, cast, get_args + +from autogen_core import ( + Image, +) +from autogen_core.models import ( + AssistantMessage, + FunctionExecutionResultMessage, + LLMMessage, + ModelFamily, + SystemMessage, + UserMessage, +) +from openai.types.chat import ( + ChatCompletionAssistantMessageParam, + ChatCompletionContentPartImageParam, + ChatCompletionContentPartParam, + ChatCompletionContentPartTextParam, + ChatCompletionSystemMessageParam, + ChatCompletionToolMessageParam, + ChatCompletionUserMessageParam, +) + +from autogen_ext.transformation import ( + TransformerMap, + build_conditional_transformer_func, + build_transformer_func, + register_transformer, +) + +from ._utils import assert_valid_name, func_call_to_oai + + +# ===Mini Transformers=== +def _assert_valid_name(message: LLMMessage, context: Dict[str, Any]): + assert_valid_name(message.source) + result: Dict[str, Any] = {} + return result + + +def _set_role(role: str): + def inner(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + return {"role": role} + + return inner + + +def _set_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert_valid_name(message.source) + return {"name": message.source} + + +def _set_content_direct(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + return {"content": message.content} + + +def _set_prepend_text_content(message: UserMessage, context: Dict[str, Any]) -> Dict[str, Any]: + prepend = context.get("prepend_name", False) + prefix = f"{message.source} said:\n" if prepend else "" + return {"content": prefix + message.content} + + +def _set_multimodal_content(message: UserMessage, context: Dict[str, Any]) -> Dict[str, Any]: + prepend = context.get("prepend_name", False) + parts: List[ChatCompletionContentPartParam] = [] + + for idx, part in enumerate(message.content): + if isinstance(part, str): + # If prepend, Append the name to the first text part + text = f"{message.source} said:\n" + part if prepend and idx == 0 else part + parts.append(ChatCompletionContentPartTextParam(type="text", text=text)) + elif isinstance(part, Image): + # TODO: support url based images + # TODO: support specifying details + parts.append(cast(ChatCompletionContentPartImageParam, part.to_openai_format())) + else: + raise ValueError(f"Unknown content part: {part}") + + return {"content": parts} + + +def _set_tool_calls(message: AssistantMessage, context: Dict[str, Any]) -> Dict[str, Any]: + return { + "tool_calls": [func_call_to_oai(x) for x in message.content], + } + + +def _set_thought_as_content(message: AssistantMessage, context: Dict[str, Any]) -> Dict[str, Any]: + return {"content": message.thought} + + +def _set_empty_to_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + return {"content": message.content or " "} + + +# === Base Transformers list === +base_system_message_transformers: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = [ + _set_content_direct, + _set_role("system"), +] + +base_user_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = [ + _assert_valid_name, + _set_name, + _set_role("user"), +] + +base_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = [ + _assert_valid_name, + _set_name, + _set_role("assistant"), +] + + +# === Transformers list === +system_message_transformers: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_system_message_transformers +) + +single_user_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_user_transformer_funcs + + [ + _set_prepend_text_content, + ] +) + +multimodal_user_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_user_transformer_funcs + + [ + _set_multimodal_content, + ] +) + +single_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_assistant_transformer_funcs + + [ + _set_content_direct, + ] +) + +tools_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + base_assistant_transformer_funcs + + [ + _set_tool_calls, + ] +) + +thought_assistant_transformer_funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + tools_assistant_transformer_funcs + + [ + _set_thought_as_content, + ] +) + + +# === Specific message param functions === + + +# === Transformer maps === +user_transformer_funcs: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_user_transformer_funcs, + "multimodal": multimodal_user_transformer_funcs, +} +user_transformer_constructors: Dict[str, Callable[..., Any]] = { + "text": ChatCompletionUserMessageParam, + "multimodal": ChatCompletionUserMessageParam, +} + + +def user_condition(message: LLMMessage, context: Dict[str, Any]) -> str: + if isinstance(message.content, str): + return "text" + else: + return "multimodal" + + +assistant_transformer_funcs: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_assistant_transformer_funcs, + "tools": tools_assistant_transformer_funcs, + "thought": thought_assistant_transformer_funcs, +} +assistant_transformer_constructors: Dict[str, Callable[..., Any]] = { + "text": ChatCompletionAssistantMessageParam, + "tools": ChatCompletionAssistantMessageParam, + "thought": ChatCompletionAssistantMessageParam, +} + + +def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: + if isinstance(message.content, list): + if message.thought is not None: + return "thought" + else: + return "tools" + else: + return "text" + + +user_transformer_funcs_gemini: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_user_transformer_funcs + [_set_empty_to_whitespace], + "multimodal": multimodal_user_transformer_funcs + [_set_empty_to_whitespace], +} + +assistant_transformer_funcs_gemini: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_assistant_transformer_funcs + [_set_empty_to_whitespace], + "tools": tools_assistant_transformer_funcs + [_set_empty_to_whitespace], + "thought": thought_assistant_transformer_funcs + [_set_empty_to_whitespace], +} + + +def function_execution_result_message(message: LLMMessage, context: Dict[str, Any]) -> List[Dict[str, Any]]: + return [ + ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content + ] + + +def function_execution_result_message_gemini(message: LLMMessage, context: Dict[str, Any]) -> List[Dict[str, Any]]: + return [ + ChatCompletionToolMessageParam(content=x.content if x.content else " ", role="tool", tool_call_id=x.call_id) + for x in message.content + ] + + +# === Transformers === + +__BASE_TRANSFORMER_MAP: TransformerMap = { + SystemMessage: build_transformer_func( + funcs=system_message_transformers, + message_param_func=ChatCompletionSystemMessageParam, + ), + UserMessage: build_conditional_transformer_func( + funcs_map=user_transformer_funcs, + message_param_func_map=user_transformer_constructors, + condition_func=user_condition, + ), + AssistantMessage: build_conditional_transformer_func( + funcs_map=assistant_transformer_funcs, + message_param_func_map=assistant_transformer_constructors, + condition_func=assistant_condition, + ), + FunctionExecutionResultMessage: function_execution_result_message, +} + +__GEMINI_TRANSFORMER_MAP: TransformerMap = { + SystemMessage: build_transformer_func( + funcs=system_message_transformers + [_set_empty_to_whitespace], + message_param_func=ChatCompletionSystemMessageParam, + ), + UserMessage: build_conditional_transformer_func( + funcs_map=user_transformer_funcs_gemini, + message_param_func_map=user_transformer_constructors, + condition_func=user_condition, + ), + AssistantMessage: build_conditional_transformer_func( + funcs_map=assistant_transformer_funcs_gemini, + message_param_func_map=assistant_transformer_constructors, + condition_func=assistant_condition, + ), + FunctionExecutionResultMessage: function_execution_result_message_gemini, +} + + +# set openai models to use the transformer map +total_models = get_args(ModelFamily.ANY) +__openai_models = [model for model in total_models if ModelFamily.is_openai(model)] + +__claude_models = [model for model in total_models if ModelFamily.is_claude(model)] + +__gemini_models = [model for model in total_models if ModelFamily.is_gemini(model)] + +__unknown_models = list(set(total_models) - set(__openai_models) - set(__claude_models) - set(__gemini_models)) + +for model in __openai_models: + register_transformer("openai", model, __BASE_TRANSFORMER_MAP) + +for model in __claude_models: + register_transformer("openai", model, __BASE_TRANSFORMER_MAP) + +for model in __gemini_models: + register_transformer("openai", model, __GEMINI_TRANSFORMER_MAP) + +for model in __unknown_models: + register_transformer("openai", model, __BASE_TRANSFORMER_MAP) + +register_transformer("openai", "default", __BASE_TRANSFORMER_MAP) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index 4d3b1a5b49da..080dc5b86ca4 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -79,9 +79,14 @@ from pydantic import BaseModel, SecretStr from typing_extensions import Self, Unpack +from autogen_ext.transformation import ( + get_transformer, +) + from .._utils.normalize_stop_reason import normalize_stop_reason from .._utils.parse_r1_content import parse_r1_content from . import _model_info +from ._utils import assert_valid_name, func_call_to_oai from .config import ( AzureOpenAIClientConfiguration, AzureOpenAIClientConfigurationConfigModel, @@ -190,7 +195,7 @@ def system_message_to_oai(message: SystemMessage) -> ChatCompletionSystemMessage ) -def func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: +def _old_func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: return ChatCompletionMessageToolCallParam( id=message.id, function={ @@ -235,7 +240,25 @@ def assistant_message_to_oai( ) -def to_oai_type(message: LLMMessage, prepend_name: bool = False) -> Sequence[ChatCompletionMessageParam]: +def to_oai_type( + message: LLMMessage, prepend_name: bool = False, model_family: str = "gpt-4o" +) -> Sequence[ChatCompletionMessageParam]: + context = { + "prepend_name": prepend_name, + } + transformers = get_transformer("openai", model_family) + + def raise_value_error(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + raise ValueError(f"Unknown message type: {type(message)}") + + transformer = transformers.get(type(message), raise_value_error) + result = transformer(message, context) + if isinstance(result, list): + return result + return [result] + + +def _old_to_oai_type(message: LLMMessage, prepend_name: bool = False) -> Sequence[ChatCompletionMessageParam]: if isinstance(message, SystemMessage): return [system_message_to_oai(message)] elif isinstance(message, UserMessage): @@ -341,7 +364,7 @@ def normalize_name(name: str) -> str: return re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:64] -def assert_valid_name(name: str) -> str: +def _old_assert_valid_name(name: str) -> str: """ Ensure that configured names are valid, raises ValueError if not. @@ -498,7 +521,10 @@ def _process_create_args( if self.model_info["json_output"] is False and json_output is True: raise ValueError("Model does not support JSON output.") - oai_messages_nested = [to_oai_type(m, prepend_name=self._add_name_prefixes) for m in messages] + oai_messages_nested = [ + to_oai_type(m, prepend_name=self._add_name_prefixes, model_family=create_args.get("model", "unknown")) + for m in messages + ] oai_messages = [item for sublist in oai_messages_nested for item in sublist] if self.model_info["function_calling"] is False and len(tools) > 0: diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py new file mode 100644 index 000000000000..ed4566a797ca --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py @@ -0,0 +1,29 @@ +import re +from typing import Any, Dict, List, Union + +from autogen_core import FunctionCall +from openai.types.chat import ChatCompletionMessageToolCallParam + + +def assert_valid_name(name: str) -> str: + """ + Ensure that configured names are valid, raises ValueError if not. + + For munging LLM responses use _normalize_name to ensure LLM specified names don't break the API. + """ + if not re.match(r"^[a-zA-Z0-9_-]+$", name): + raise ValueError(f"Invalid name: {name}. Only letters, numbers, '_' and '-' are allowed.") + if len(name) > 64: + raise ValueError(f"Invalid name: {name}. Name must be less than 64 characters.") + return name + + +def func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: + return ChatCompletionMessageToolCallParam( + id=message.id, + function={ + "arguments": message.arguments, + "name": message.name, + }, + type="function", + ) diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py index 7bd027d17404..a23309f483b9 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py @@ -1,24 +1,20 @@ from .registry import ( - register_transformer, - get_transformer, - register_builder, - get_builder - build_transformer_func, - build_conditional_transformer_func, MESSAGE_TRANSFORMERS, + build_conditional_transformer_func, + build_transformer_func, + get_transformer, + register_transformer, ) from .types import ( - TransformerMap, - TransformerFunc, - BuilderMap, BuilderFunc, + BuilderMap, + TransformerFunc, + TransformerMap, ) __all__ = [ "register_transformer", "get_transformer", - "register_builder", - "get_builder", "build_transformer_func", "build_conditional_transformer_func", "MESSAGE_TRANSFORMERS", @@ -26,4 +22,4 @@ "TransformerFunc", "BuilderMap", "BuilderFunc", -] \ No newline at end of file +] diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py b/python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py deleted file mode 100644 index 8feac69eb67e..000000000000 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/dispatcher.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import Any, Callable, Dict, List -from autogen_core.models import LLMMessage - -from autogen_ext.transformation import ( - # Types - TransformerFunc, - TransformerMap, - BuilderMap, - BuilderFunc, - # Functions - get_transformer, -) - - -def dispatch_transformation( - model_family: str, - message: LLMMessage, - funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]], - context: Dict[str, Any], -) -> Any: - """ - Dispatches a transformation for a given model family. - - The transformation is selected based on the model family and the message type. - - The context dict is passed to the transformer functions. - - Returns the transformed message. - """ - transformer_map: TransformerMap = get_transformer(model_family) - transformer: TransformerFunc = transformer_map.get(type(message), lambda x, y: {}) - - builder_map: BuilderMap = get_builder(model_family) - builder: BuilderFunc = builder_map.get(type(message), lambda x, y: x) - - parts: List[Any] = [message] if isinstance(message.content, str) else message.content - contents: List[Any] = [] - for part in parts: - message: Any = transformer(part, context) - contents.append(message) - - kwargs: Dict[str, Any] = {} - for func in funcs: - kwargs.update(func(contents, context)) - - return builder(contents, kwargs) - \ No newline at end of file diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py index d9464570c787..80880a91400b 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py @@ -1,19 +1,22 @@ -from typing import Dict, List, Callable, Type, Any +from collections import defaultdict +from typing import Any, Callable, Dict, List, Type + from autogen_core.models import LLMMessage, ModelFamily -from autogen_ext.transformation import ( - TransformerMap, - TransformerFunc, + +from .types import ( BuilderMap, + TransformerFunc, + TransformerMap, ) # Global registry of model family → message transformer map # Each model family (e.g. "gpt-4o", "gemini-1.5-flash") maps to a dict of LLMMessage type → transformer function -MESSAGE_TRANSFORMERS: Dict[str, TransformerMap] = {} +MESSAGE_TRANSFORMERS: Dict[str, Dict[str, TransformerMap]] = defaultdict(dict) MESSAGE_BUILDERS: Dict[str, BuilderMap] = {} + def build_transformer_func( - funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]], - message_param_func: Callable[..., Any] + funcs: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]], message_param_func: Callable[..., Any] ) -> TransformerFunc: """ Combines multiple transformer functions into a single transformer. @@ -27,6 +30,7 @@ def build_transformer_func( message_param_func: A model-specific constructor (e.g. ChatCompletionMessageParam). Signature is intentionally open: Callable[..., Any]. """ + def transformer_func(message: LLMMessage, context: Any) -> Any: kwargs: Dict[str, Any] = {} for func in funcs: @@ -53,6 +57,7 @@ def build_conditional_transformer_func( message_param_func_map: A mapping of condition → constructor function. condition_func: A function that returns the condition for selecting the constructor. """ + def transformer(message: LLMMessage, context: Dict[str, Any]) -> Any: condition = condition_func(message, context) constructor = message_param_func_map[condition] @@ -64,7 +69,7 @@ def transformer(message: LLMMessage, context: Dict[str, Any]) -> Any: return transformer -def register_transformer(model_family: str, transformer_map: TransformerMap): +def register_transformer(api: str, model_family: str, transformer_map: TransformerMap): """ Registers a transformer map for a given model family. @@ -74,13 +79,13 @@ def register_transformer(model_family: str, transformer_map: TransformerMap): SystemMessage: system_message_to_oai, }) """ - MESSAGE_TRANSFORMERS[model_family] = transformer_map + MESSAGE_TRANSFORMERS[api][model_family] = transformer_map -def get_transformer(model_family: str) -> TransformerMap: +def get_transformer(api: str, model_family: str) -> TransformerMap: """ Returns the registered transformer map for the given model family. - + This is a thin wrapper around `MESSAGE_TRANSFORMERS.get(...)`, but serves as an abstraction layer to allow future enhancements such as: @@ -90,33 +95,11 @@ def get_transformer(model_family: str) -> TransformerMap: Keeping this as a function (instead of direct dict access) improves long-term flexibility. """ - return MESSAGE_TRANSFORMERS.get(model_family, {}) + transformer = MESSAGE_TRANSFORMERS.get(api, {}).get(model_family, {}) + if not transformer: + transformer = MESSAGE_TRANSFORMERS.get("default", {}).get("default", {}) + if not transformer: + raise ValueError(f"No transformer found for model family '{model_family}'") -def register_builder(model_family: str, builder_map: BuilderMap): - """ - Registers a builder map for a given model family. - - Example: - register_builder("gpt-4o", { - UserMessage: user_message_to_oai, - SystemMessage: system_message_to_oai, - }) - """ - MESSAGE_BUILDERS[model_family] = builder_map - - -def get_builder(model_family: str) -> BuilderMap: - """ - Returns the registered builder map for the given model family. - - This is a thin wrapper around `MESSAGE_BUILDERS.get(...)`, but serves as - an abstraction layer to allow future enhancements such as: - - - Providing fallback builders for unknown model families - - Injecting mock builders during testing - - Adding logging, metrics, or versioning later - - Keeping this as a function (instead of direct dict access) improves long-term flexibility. - """ - return MESSAGE_BUILDERS.get(model_family, {}) \ No newline at end of file + return transformer diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py index 8e3fa2897477..f57df3b32cb8 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py @@ -1,8 +1,9 @@ -from typing import Dict, List, Callable, Type, Any +from typing import Any, Callable, Dict, List, Type + from autogen_core.models import LLMMessage, ModelFamily TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], Any] -TransformerMap = Dict[Type[LLMMessage],TransformerFunc] +TransformerMap = Dict[Type[LLMMessage], TransformerFunc] BuilderFunc = Callable[[List[Any], Dict[str, Any]], Any] BuilderMap = Dict[Type[LLMMessage], BuilderFunc] From 82c50ce346058522e812de868190206a0aecf9d7 Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Mon, 24 Mar 2025 03:14:57 +0900 Subject: [PATCH 04/15] FIX: fix errors and pyright mypy issues --- .../models/openai/_message_transform.py | 38 ++++++++++----- .../models/openai/_openai_client.py | 47 +++---------------- .../src/autogen_ext/models/openai/_utils.py | 1 - .../autogen_ext/transformation/__init__.py | 4 -- .../autogen_ext/transformation/registry.py | 26 ++++++---- .../src/autogen_ext/transformation/types.py | 7 +-- 6 files changed, 54 insertions(+), 69 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py index 8ea269c412ad..48932f54bd0f 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -30,15 +30,17 @@ from ._utils import assert_valid_name, func_call_to_oai +EMPTY: Dict[str, Any] = {} + # ===Mini Transformers=== -def _assert_valid_name(message: LLMMessage, context: Dict[str, Any]): +def _assert_valid_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert isinstance(message, (UserMessage, AssistantMessage)) assert_valid_name(message.source) - result: Dict[str, Any] = {} - return result + return EMPTY -def _set_role(role: str): +def _set_role(role: str) -> Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]: def inner(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: return {"role": role} @@ -46,6 +48,7 @@ def inner(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: def _set_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert isinstance(message, (UserMessage, AssistantMessage)) assert_valid_name(message.source) return {"name": message.source} @@ -54,13 +57,16 @@ def _set_content_direct(message: LLMMessage, context: Dict[str, Any]) -> Dict[st return {"content": message.content} -def _set_prepend_text_content(message: UserMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_prepend_text_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert isinstance(message, (UserMessage, AssistantMessage)) + assert isinstance(message.content, str) prepend = context.get("prepend_name", False) prefix = f"{message.source} said:\n" if prepend else "" return {"content": prefix + message.content} -def _set_multimodal_content(message: UserMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_multimodal_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert isinstance(message, (UserMessage, AssistantMessage)) prepend = context.get("prepend_name", False) parts: List[ChatCompletionContentPartParam] = [] @@ -79,13 +85,16 @@ def _set_multimodal_content(message: UserMessage, context: Dict[str, Any]) -> Di return {"content": parts} -def _set_tool_calls(message: AssistantMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_tool_calls(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert isinstance(message.content, list) + assert isinstance(message, AssistantMessage) return { "tool_calls": [func_call_to_oai(x) for x in message.content], } -def _set_thought_as_content(message: AssistantMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_thought_as_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert isinstance(message, AssistantMessage) return {"content": message.thought} @@ -187,6 +196,7 @@ def user_condition(message: LLMMessage, context: Dict[str, Any]) -> str: def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: + assert isinstance(message, AssistantMessage) if isinstance(message.content, list): if message.thought is not None: return "thought" @@ -208,13 +218,19 @@ def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: } -def function_execution_result_message(message: LLMMessage, context: Dict[str, Any]) -> List[Dict[str, Any]]: +def function_execution_result_message( + message: LLMMessage, context: Dict[str, Any] +) -> list[ChatCompletionToolMessageParam]: + assert isinstance(message, FunctionExecutionResultMessage) return [ ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content ] -def function_execution_result_message_gemini(message: LLMMessage, context: Dict[str, Any]) -> List[Dict[str, Any]]: +def function_execution_result_message_gemini( + message: LLMMessage, context: Dict[str, Any] +) -> list[ChatCompletionToolMessageParam]: + assert isinstance(message, FunctionExecutionResultMessage) return [ ChatCompletionToolMessageParam(content=x.content if x.content else " ", role="tool", tool_call_id=x.call_id) for x in message.content @@ -282,4 +298,4 @@ def function_execution_result_message_gemini(message: LLMMessage, context: Dict[ for model in __unknown_models: register_transformer("openai", model, __BASE_TRANSFORMER_MAP) -register_transformer("openai", "default", __BASE_TRANSFORMER_MAP) +# register_transformer("openai", "default", __BASE_TRANSFORMER_MAP) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index 080dc5b86ca4..b9f6a3cd08a5 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -11,6 +11,7 @@ from typing import ( Any, AsyncGenerator, + Callable, Dict, List, Mapping, @@ -58,7 +59,6 @@ ChatCompletionContentPartParam, ChatCompletionContentPartTextParam, ChatCompletionMessageParam, - ChatCompletionMessageToolCallParam, ChatCompletionRole, ChatCompletionSystemMessageParam, ChatCompletionToolMessageParam, @@ -195,17 +195,6 @@ def system_message_to_oai(message: SystemMessage) -> ChatCompletionSystemMessage ) -def _old_func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: - return ChatCompletionMessageToolCallParam( - id=message.id, - function={ - "arguments": message.arguments, - "name": message.name, - }, - type="function", - ) - - def tool_message_to_oai( message: FunctionExecutionResultMessage, ) -> Sequence[ChatCompletionToolMessageParam]: @@ -248,25 +237,16 @@ def to_oai_type( } transformers = get_transformer("openai", model_family) - def raise_value_error(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + def raise_value_error(message: LLMMessage, context: Dict[str, Any]) -> ChatCompletionMessageParam: raise ValueError(f"Unknown message type: {type(message)}") - transformer = transformers.get(type(message), raise_value_error) + transformer: Callable[ + [LLMMessage, Dict[str, Any]], Union[ChatCompletionMessageParam, Sequence[ChatCompletionMessageParam]] + ] = transformers.get(type(message), raise_value_error) result = transformer(message, context) if isinstance(result, list): - return result - return [result] - - -def _old_to_oai_type(message: LLMMessage, prepend_name: bool = False) -> Sequence[ChatCompletionMessageParam]: - if isinstance(message, SystemMessage): - return [system_message_to_oai(message)] - elif isinstance(message, UserMessage): - return [user_message_to_oai(message, prepend_name)] - elif isinstance(message, AssistantMessage): - return [assistant_message_to_oai(message)] - else: - return tool_message_to_oai(message) + return cast(List[ChatCompletionMessageParam], result) + return cast(List[ChatCompletionMessageParam], [result]) def calculate_vision_tokens(image: Image, detail: str = "auto") -> int: @@ -364,19 +344,6 @@ def normalize_name(name: str) -> str: return re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:64] -def _old_assert_valid_name(name: str) -> str: - """ - Ensure that configured names are valid, raises ValueError if not. - - For munging LLM responses use _normalize_name to ensure LLM specified names don't break the API. - """ - if not re.match(r"^[a-zA-Z0-9_-]+$", name): - raise ValueError(f"Invalid name: {name}. Only letters, numbers, '_' and '-' are allowed.") - if len(name) > 64: - raise ValueError(f"Invalid name: {name}. Name must be less than 64 characters.") - return name - - @dataclass class CreateParams: messages: List[ChatCompletionMessageParam] diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py index ed4566a797ca..d89571c2e8f1 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py @@ -1,5 +1,4 @@ import re -from typing import Any, Dict, List, Union from autogen_core import FunctionCall from openai.types.chat import ChatCompletionMessageToolCallParam diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py index a23309f483b9..54e59d8172d7 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py @@ -6,8 +6,6 @@ register_transformer, ) from .types import ( - BuilderFunc, - BuilderMap, TransformerFunc, TransformerMap, ) @@ -20,6 +18,4 @@ "MESSAGE_TRANSFORMERS", "TransformerMap", "TransformerFunc", - "BuilderMap", - "BuilderFunc", ] diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py index 80880a91400b..6d463fb1ba90 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py @@ -1,10 +1,9 @@ from collections import defaultdict -from typing import Any, Callable, Dict, List, Type +from typing import Any, Callable, Dict, List -from autogen_core.models import LLMMessage, ModelFamily +from autogen_core.models import LLMMessage from .types import ( - BuilderMap, TransformerFunc, TransformerMap, ) @@ -12,7 +11,6 @@ # Global registry of model family → message transformer map # Each model family (e.g. "gpt-4o", "gemini-1.5-flash") maps to a dict of LLMMessage type → transformer function MESSAGE_TRANSFORMERS: Dict[str, Dict[str, TransformerMap]] = defaultdict(dict) -MESSAGE_BUILDERS: Dict[str, BuilderMap] = {} def build_transformer_func( @@ -69,7 +67,7 @@ def transformer(message: LLMMessage, context: Dict[str, Any]) -> Any: return transformer -def register_transformer(api: str, model_family: str, transformer_map: TransformerMap): +def register_transformer(api: str, model_family: str, transformer_map: TransformerMap) -> None: """ Registers a transformer map for a given model family. @@ -82,6 +80,17 @@ def register_transformer(api: str, model_family: str, transformer_map: Transform MESSAGE_TRANSFORMERS[api][model_family] = transformer_map +def _find_model_family(api: str, model: str) -> str: + """ + Finds the best matching model family for the given model. + Search via prefix matching (e.g. "gpt-4o" → "gpt-4o-1.0"). + """ + for family in MESSAGE_TRANSFORMERS[api].keys(): + if model.startswith(family): + return family + return "default" + + def get_transformer(api: str, model_family: str) -> TransformerMap: """ Returns the registered transformer map for the given model family. @@ -95,9 +104,10 @@ def get_transformer(api: str, model_family: str) -> TransformerMap: Keeping this as a function (instead of direct dict access) improves long-term flexibility. """ - transformer = MESSAGE_TRANSFORMERS.get(api, {}).get(model_family, {}) - if not transformer: - transformer = MESSAGE_TRANSFORMERS.get("default", {}).get("default", {}) + + model = _find_model_family(api, model_family) + + transformer = MESSAGE_TRANSFORMERS.get(api, {}).get(model, {}) if not transformer: raise ValueError(f"No transformer found for model family '{model_family}'") diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py index f57df3b32cb8..91c9943dfa86 100644 --- a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py +++ b/python/packages/autogen-ext/src/autogen_ext/transformation/types.py @@ -1,9 +1,6 @@ -from typing import Any, Callable, Dict, List, Type +from typing import Any, Callable, Dict, Type -from autogen_core.models import LLMMessage, ModelFamily +from autogen_core.models import LLMMessage TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], Any] TransformerMap = Dict[Type[LLMMessage], TransformerFunc] - -BuilderFunc = Callable[[List[Any], Dict[str, Any]], Any] -BuilderMap = Dict[Type[LLMMessage], BuilderFunc] From 4def743644652ba674ea6967965fda688edd284a Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Mon, 24 Mar 2025 04:44:50 +0900 Subject: [PATCH 05/15] FIX: Json dump error when function tools --- .../models/openai/_message_transform.py | 35 ++++++-- .../models/openai/_openai_client.py | 88 +------------------ .../src/autogen_ext/models/openai/_utils.py | 14 --- 3 files changed, 31 insertions(+), 106 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py index 48932f54bd0f..fd1dbfe07a7a 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -1,6 +1,7 @@ from typing import Any, Callable, Dict, List, cast, get_args from autogen_core import ( + FunctionCall, Image, ) from autogen_core.models import ( @@ -16,6 +17,7 @@ ChatCompletionContentPartImageParam, ChatCompletionContentPartParam, ChatCompletionContentPartTextParam, + ChatCompletionMessageToolCallParam, ChatCompletionSystemMessageParam, ChatCompletionToolMessageParam, ChatCompletionUserMessageParam, @@ -28,11 +30,22 @@ register_transformer, ) -from ._utils import assert_valid_name, func_call_to_oai +from ._utils import assert_valid_name EMPTY: Dict[str, Any] = {} +def func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: + return ChatCompletionMessageToolCallParam( + id=message.id, + function={ + "arguments": message.arguments, + "name": message.name, + }, + type="function", + ) + + # ===Mini Transformers=== def _assert_valid_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: assert isinstance(message, (UserMessage, AssistantMessage)) @@ -98,6 +111,11 @@ def _set_thought_as_content(message: LLMMessage, context: Dict[str, Any]) -> Dic return {"content": message.thought} +def _set_thought_as_content_gemini(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: + assert isinstance(message, AssistantMessage) + return {"content": message.thought or " "} + + def _set_empty_to_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: return {"content": message.content or " "} @@ -161,6 +179,13 @@ def _set_empty_to_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Di ] ) +thought_assistant_transformer_funcs_gemini: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = ( + tools_assistant_transformer_funcs + + [ + _set_thought_as_content_gemini, + ] +) + # === Specific message param functions === @@ -213,8 +238,8 @@ def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: assistant_transformer_funcs_gemini: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { "text": single_assistant_transformer_funcs + [_set_empty_to_whitespace], - "tools": tools_assistant_transformer_funcs + [_set_empty_to_whitespace], - "thought": thought_assistant_transformer_funcs + [_set_empty_to_whitespace], + "tools": tools_assistant_transformer_funcs, # that case, message.content is a list of FunctionCall + "thought": thought_assistant_transformer_funcs_gemini, # that case, message.content is a list of FunctionCall } @@ -272,7 +297,7 @@ def function_execution_result_message_gemini( message_param_func_map=assistant_transformer_constructors, condition_func=assistant_condition, ), - FunctionExecutionResultMessage: function_execution_result_message_gemini, + FunctionExecutionResultMessage: function_execution_result_message, } @@ -298,4 +323,4 @@ def function_execution_result_message_gemini( for model in __unknown_models: register_transformer("openai", model, __BASE_TRANSFORMER_MAP) -# register_transformer("openai", "default", __BASE_TRANSFORMER_MAP) +register_transformer("openai", "default", __BASE_TRANSFORMER_MAP) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index b9f6a3cd08a5..1e6a1d2ca496 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -38,7 +38,6 @@ ChatCompletionClient, ChatCompletionTokenLogprob, CreateResult, - FunctionExecutionResultMessage, LLMMessage, ModelCapabilities, # type: ignore ModelFamily, @@ -53,17 +52,11 @@ from openai import NOT_GIVEN, AsyncAzureOpenAI, AsyncOpenAI from openai.types.chat import ( ChatCompletion, - ChatCompletionAssistantMessageParam, ChatCompletionChunk, - ChatCompletionContentPartImageParam, ChatCompletionContentPartParam, - ChatCompletionContentPartTextParam, ChatCompletionMessageParam, ChatCompletionRole, - ChatCompletionSystemMessageParam, - ChatCompletionToolMessageParam, ChatCompletionToolParam, - ChatCompletionUserMessageParam, ParsedChatCompletion, ParsedChoice, completion_create_params, @@ -86,7 +79,7 @@ from .._utils.normalize_stop_reason import normalize_stop_reason from .._utils.parse_r1_content import parse_r1_content from . import _model_info -from ._utils import assert_valid_name, func_call_to_oai +from ._utils import assert_valid_name from .config import ( AzureOpenAIClientConfiguration, AzureOpenAIClientConfigurationConfigModel, @@ -150,85 +143,6 @@ def type_to_role(message: LLMMessage) -> ChatCompletionRole: return "tool" -def user_message_to_oai(message: UserMessage, prepend_name: bool = False) -> ChatCompletionUserMessageParam: - assert_valid_name(message.source) - if isinstance(message.content, str): - return ChatCompletionUserMessageParam( - content=(f"{message.source} said:\n" if prepend_name else "") + message.content, - role="user", - name=message.source, - ) - else: - parts: List[ChatCompletionContentPartParam] = [] - for part in message.content: - if isinstance(part, str): - if prepend_name: - # Append the name to the first text part - oai_part = ChatCompletionContentPartTextParam( - text=f"{message.source} said:\n" + part, - type="text", - ) - prepend_name = False - else: - oai_part = ChatCompletionContentPartTextParam( - text=part, - type="text", - ) - parts.append(oai_part) - elif isinstance(part, Image): - # TODO: support url based images - # TODO: support specifying details - parts.append(cast(ChatCompletionContentPartImageParam, part.to_openai_format())) - else: - raise ValueError(f"Unknown content type: {part}") - return ChatCompletionUserMessageParam( - content=parts, - role="user", - name=message.source, - ) - - -def system_message_to_oai(message: SystemMessage) -> ChatCompletionSystemMessageParam: - return ChatCompletionSystemMessageParam( - content=message.content, - role="system", - ) - - -def tool_message_to_oai( - message: FunctionExecutionResultMessage, -) -> Sequence[ChatCompletionToolMessageParam]: - return [ - ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content - ] - - -def assistant_message_to_oai( - message: AssistantMessage, -) -> ChatCompletionAssistantMessageParam: - assert_valid_name(message.source) - if isinstance(message.content, list): - if message.thought is not None: - return ChatCompletionAssistantMessageParam( - content=message.thought, - tool_calls=[func_call_to_oai(x) for x in message.content], - role="assistant", - name=message.source, - ) - else: - return ChatCompletionAssistantMessageParam( - tool_calls=[func_call_to_oai(x) for x in message.content], - role="assistant", - name=message.source, - ) - else: - return ChatCompletionAssistantMessageParam( - content=message.content, - role="assistant", - name=message.source, - ) - - def to_oai_type( message: LLMMessage, prepend_name: bool = False, model_family: str = "gpt-4o" ) -> Sequence[ChatCompletionMessageParam]: diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py index d89571c2e8f1..8c1df22961d7 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_utils.py @@ -1,8 +1,5 @@ import re -from autogen_core import FunctionCall -from openai.types.chat import ChatCompletionMessageToolCallParam - def assert_valid_name(name: str) -> str: """ @@ -15,14 +12,3 @@ def assert_valid_name(name: str) -> str: if len(name) > 64: raise ValueError(f"Invalid name: {name}. Name must be less than 64 characters.") return name - - -def func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallParam: - return ChatCompletionMessageToolCallParam( - id=message.id, - function={ - "arguments": message.arguments, - "name": message.name, - }, - type="function", - ) From 0b994814d4749b53c96162920b51fa373234c88f Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Mon, 24 Mar 2025 05:03:37 +0900 Subject: [PATCH 06/15] FEAT: white space test code --- .../tests/models/test_openai_model_client.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py index 6fcc6dc3b072..bf9584a19e6a 100644 --- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py @@ -2047,4 +2047,57 @@ async def test_add_name_prefixes(monkeypatch: pytest.MonkeyPatch) -> None: assert str(converted_mm["content"][0]["text"]) == "Adam said:\n" + str(oai_mm["content"][0]["text"]) +def noop(input: str) -> str: + return "done" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gemini-1.5-flash"]) +async def test_empty_assistant_content_with_gemini(model: str, openai_client: OpenAIChatCompletionClient) -> None: + # Test tool calling + tool = FunctionTool(noop, name="noop", description="No-op tool") + messages: List[LLMMessage] = [UserMessage(content="Call noop", source="user")] + result = await openai_client.create(messages=messages, tools=[tool]) + assert isinstance(result.content, list) + tool_call = result.content[0] + assert isinstance(tool_call, FunctionCall) + + # reply with empty string as thought (== content) + messages.append(AssistantMessage(content=result.content, thought="", source="assistant")) + messages.append( + FunctionExecutionResultMessage( + content=[ + FunctionExecutionResult( + content="done", + call_id=tool_call.id, + is_error=False, + name=tool_call.name, + ) + ] + ) + ) + + # This will crash if _set_empty_to_whitespace is not applied to "thought" + result = await openai_client.create(messages=messages) + assert isinstance(result.content, str) + assert result.content.strip() != "" or result.content == " " + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gemini-1.5-flash"]) +async def test_empty_assistant_content_string_with_gemini( + model: str, openai_client: OpenAIChatCompletionClient +) -> None: + # message: assistant is response empty content + messages: list[LLMMessage] = [ + UserMessage(content="Say something", source="user"), + AssistantMessage(content="", source="assistant"), + ] + + # This will crash if _set_empty_to_whitespace is not applied to "content" + result = await openai_client.create(messages=messages) + assert isinstance(result.content, str) + assert result.content.strip() != "" or result.content == " " + + # TODO: add integration tests for Azure OpenAI using AAD token. From f470252a4857e72dbbe5bb893eda78faf36bf2be Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Wed, 26 Mar 2025 16:27:33 +0900 Subject: [PATCH 07/15] CHOR: change transfrom location to under openai --- .../src/autogen_ext/models/openai/_message_transform.py | 2 +- .../autogen-ext/src/autogen_ext/models/openai/_openai_client.py | 2 +- .../openai/_transformation}/__init__.py | 0 .../openai/_transformation}/registry.py | 0 .../{transformation => models/openai/_transformation}/types.py | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename python/packages/autogen-ext/src/autogen_ext/{transformation => models/openai/_transformation}/__init__.py (100%) rename python/packages/autogen-ext/src/autogen_ext/{transformation => models/openai/_transformation}/registry.py (100%) rename python/packages/autogen-ext/src/autogen_ext/{transformation => models/openai/_transformation}/types.py (100%) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py index fd1dbfe07a7a..92f9783deec3 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -23,7 +23,7 @@ ChatCompletionUserMessageParam, ) -from autogen_ext.transformation import ( +from ._transformation import ( TransformerMap, build_conditional_transformer_func, build_transformer_func, diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index 1e6a1d2ca496..50b88f6350e2 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -72,7 +72,7 @@ from pydantic import BaseModel, SecretStr from typing_extensions import Self, Unpack -from autogen_ext.transformation import ( +from ._transformation import ( get_transformer, ) diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py similarity index 100% rename from python/packages/autogen-ext/src/autogen_ext/transformation/__init__.py rename to python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py similarity index 100% rename from python/packages/autogen-ext/src/autogen_ext/transformation/registry.py rename to python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py diff --git a/python/packages/autogen-ext/src/autogen_ext/transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py similarity index 100% rename from python/packages/autogen-ext/src/autogen_ext/transformation/types.py rename to python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py From a3b0f24e024d90bee8c092c1e86c4106c26a20aa Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Wed, 26 Mar 2025 19:06:41 +0900 Subject: [PATCH 08/15] FIX: pass the mypy and pyright --- .../models/anthropic/_anthropic_client.py | 21 +++++++-- .../models/openai/_message_transform.py | 43 ++++++++++++++----- .../models/openai/_openai_client.py | 7 ++- .../models/test_anthropic_model_client.py | 25 +++++++++++ .../tests/models/test_openai_model_client.py | 5 ++- 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py index 5788f1bf44e3..77088d15da62 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py @@ -12,6 +12,7 @@ AsyncGenerator, Coroutine, Dict, + Iterable, List, Literal, Mapping, @@ -142,20 +143,33 @@ def get_mime_type_from_image(image: Image) -> Literal["image/jpeg", "image/png", return "image/jpeg" +def __empty_content_to_whitespace( + content: Union[str, List[Union[str, Image]]], +) -> Union[str, Iterable[Any]]: + if isinstance(content, str) and not content.strip(): + return " " + elif isinstance(content, list) and not any(isinstance(x, str) and not x.strip() for x in content): + for idx, message in enumerate(content): + if isinstance(message, str) and not message.strip(): + content[idx] = " " + + return content + + def user_message_to_anthropic(message: UserMessage) -> MessageParam: assert_valid_name(message.source) if isinstance(message.content, str): return { "role": "user", - "content": message.content, + "content": __empty_content_to_whitespace(message.content), } else: blocks: List[Union[TextBlockParam, ImageBlockParam]] = [] for part in message.content: if isinstance(part, str): - blocks.append(TextBlockParam(type="text", text=part)) + blocks.append(TextBlockParam(type="text", text=cast(str, __empty_content_to_whitespace(part)))) elif isinstance(part, Image): blocks.append( ImageBlockParam( @@ -177,7 +191,7 @@ def user_message_to_anthropic(message: UserMessage) -> MessageParam: def system_message_to_anthropic(message: SystemMessage) -> str: - return message.content + return cast(str, __empty_content_to_whitespace(message.content)) def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam: @@ -190,6 +204,7 @@ def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam: for func_call in message.content: # Parse the arguments and convert to dict if it's a JSON string args = func_call.arguments + args = cast(str, __empty_content_to_whitespace(args)) if isinstance(args, str): try: args_dict = json.loads(args) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py index 92f9783deec3..0ea2248c2a50 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -29,7 +29,6 @@ build_transformer_func, register_transformer, ) - from ._utils import assert_valid_name EMPTY: Dict[str, Any] = {} @@ -236,6 +235,7 @@ def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: "multimodal": multimodal_user_transformer_funcs + [_set_empty_to_whitespace], } + assistant_transformer_funcs_gemini: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { "text": single_assistant_transformer_funcs + [_set_empty_to_whitespace], "tools": tools_assistant_transformer_funcs, # that case, message.content is a list of FunctionCall @@ -243,22 +243,25 @@ def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: } -def function_execution_result_message( - message: LLMMessage, context: Dict[str, Any] -) -> list[ChatCompletionToolMessageParam]: - assert isinstance(message, FunctionExecutionResultMessage) - return [ - ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content - ] +user_transformer_funcs_claude: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_user_transformer_funcs + [_set_empty_to_whitespace], + "multimodal": multimodal_user_transformer_funcs + [_set_empty_to_whitespace], +} + + +assistant_transformer_funcs_claude: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { + "text": single_assistant_transformer_funcs + [_set_empty_to_whitespace], + "tools": tools_assistant_transformer_funcs, # that case, message.content is a list of FunctionCall + "thought": thought_assistant_transformer_funcs_gemini, # that case, message.content is a list of FunctionCall +} -def function_execution_result_message_gemini( +def function_execution_result_message( message: LLMMessage, context: Dict[str, Any] ) -> list[ChatCompletionToolMessageParam]: assert isinstance(message, FunctionExecutionResultMessage) return [ - ChatCompletionToolMessageParam(content=x.content if x.content else " ", role="tool", tool_call_id=x.call_id) - for x in message.content + ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content ] @@ -300,6 +303,24 @@ def function_execution_result_message_gemini( FunctionExecutionResultMessage: function_execution_result_message, } +__CLAUDE_TRANSFORMER_MAP: TransformerMap = { + SystemMessage: build_transformer_func( + funcs=system_message_transformers + [_set_empty_to_whitespace], + message_param_func=ChatCompletionSystemMessageParam, + ), + UserMessage: build_conditional_transformer_func( + funcs_map=user_transformer_funcs_claude, + message_param_func_map=user_transformer_constructors, + condition_func=user_condition, + ), + AssistantMessage: build_conditional_transformer_func( + funcs_map=assistant_transformer_funcs_claude, + message_param_func_map=assistant_transformer_constructors, + condition_func=assistant_condition, + ), + FunctionExecutionResultMessage: function_execution_result_message, +} + # set openai models to use the transformer map total_models = get_args(ModelFamily.ANY) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index af4456470781..9f37eebf8d4e 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -71,13 +71,12 @@ from pydantic import BaseModel, SecretStr from typing_extensions import Self, Unpack -from ._transformation import ( - get_transformer, -) - from .._utils.normalize_stop_reason import normalize_stop_reason from .._utils.parse_r1_content import parse_r1_content from . import _model_info +from ._transformation import ( + get_transformer, +) from ._utils import assert_valid_name from .config import ( AzureOpenAIClientConfiguration, diff --git a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py index 4f4135b4b0c8..05fcb5407ee4 100644 --- a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py @@ -334,3 +334,28 @@ async def test_anthropic_serialization() -> None: loaded_model_client = AnthropicChatCompletionClient.load_component(model_client_config) assert loaded_model_client is not None assert isinstance(loaded_model_client, AnthropicChatCompletionClient) + + +@pytest.mark.asyncio +async def test_empty_assistant_content_string_with_anthropic() -> None: + """Test that an empty assistant content string is handled correctly.""" + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + pytest.skip("ANTHROPIC_API_KEY not found in environment variables") + + client = AnthropicChatCompletionClient( + model="claude-3-haiku-20240307", + api_key=api_key, + ) + + # Test empty assistant content string + result = await client.create( + messages=[ + UserMessage(content="Say something", source="user"), + AssistantMessage(content="", source="assistant"), + ] + ) + + # Verify we got a response + assert isinstance(result.content, str) + assert len(result.content) > 0 diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py index 25e3f307833b..41bc00f837ed 100644 --- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py @@ -2095,8 +2095,9 @@ async def test_empty_assistant_content_with_gemini(model: str, openai_client: Op @pytest.mark.asyncio -@pytest.mark.parametrize("model", ["gemini-1.5-flash"]) -async def test_empty_assistant_content_string_with_gemini( +# @pytest.mark.parametrize("model", ["gemini-1.5-flash", "claude-3-5-haiku-latest"]) # +@pytest.mark.parametrize("model", ["gemini-1.5-flash"]) # before add claude-3-5-haiku-latest at openai/_model_info.py +async def test_empty_assistant_content_string_with_some_model( model: str, openai_client: OpenAIChatCompletionClient ) -> None: # message: assistant is response empty content From a2f41c08d52edee091724a557eb08c09bb2e0cfa Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Thu, 27 Mar 2025 22:46:04 +0900 Subject: [PATCH 09/15] FIX: Using overide instead of cast --- .../models/anthropic/_anthropic_client.py | 15 ++++++++++++--- .../src/autogen_ext/models/openai/__init__.py | 3 +-- .../autogen_ext/models/openai/_openai_client.py | 3 +-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py index 77088d15da62..69192cc10420 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py @@ -21,6 +21,7 @@ Set, Union, cast, + overload, ) import tiktoken @@ -143,6 +144,14 @@ def get_mime_type_from_image(image: Image) -> Literal["image/jpeg", "image/png", return "image/jpeg" +@overload +def __empty_content_to_whitespace(content: str) -> str: ... + + +@overload +def __empty_content_to_whitespace(content: List[Any]) -> Iterable[Any]: ... + + def __empty_content_to_whitespace( content: Union[str, List[Union[str, Image]]], ) -> Union[str, Iterable[Any]]: @@ -169,7 +178,7 @@ def user_message_to_anthropic(message: UserMessage) -> MessageParam: for part in message.content: if isinstance(part, str): - blocks.append(TextBlockParam(type="text", text=cast(str, __empty_content_to_whitespace(part)))) + blocks.append(TextBlockParam(type="text", text=__empty_content_to_whitespace(part))) elif isinstance(part, Image): blocks.append( ImageBlockParam( @@ -191,7 +200,7 @@ def user_message_to_anthropic(message: UserMessage) -> MessageParam: def system_message_to_anthropic(message: SystemMessage) -> str: - return cast(str, __empty_content_to_whitespace(message.content)) + return __empty_content_to_whitespace(message.content) def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam: @@ -204,7 +213,7 @@ def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam: for func_call in message.content: # Parse the arguments and convert to dict if it's a JSON string args = func_call.arguments - args = cast(str, __empty_content_to_whitespace(args)) + args = __empty_content_to_whitespace(args) if isinstance(args, str): try: args_dict = json.loads(args) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py index 24ad3fde8e9b..2241f663af26 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py @@ -1,10 +1,9 @@ - from . import _message_transform from ._openai_client import ( + AZURE_OPENAI_USER_AGENT, AzureOpenAIChatCompletionClient, BaseOpenAIChatCompletionClient, OpenAIChatCompletionClient, - AZURE_OPENAI_USER_AGENT, ) from .config import ( AzureOpenAIClientConfigurationConfigModel, diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index a4cb9c98f30b..fd466274763c 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -8,6 +8,7 @@ import warnings from asyncio import Task from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version from typing import ( Any, AsyncGenerator, @@ -84,8 +85,6 @@ OpenAIClientConfiguration, OpenAIClientConfigurationConfigModel, ) -from importlib.metadata import PackageNotFoundError, version - logger = logging.getLogger(EVENT_LOGGER_NAME) trace_logger = logging.getLogger(TRACE_LOGGER_NAME) From 6c5cac61fce9f4d05394eca613771f0b3fd88d8d Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Fri, 28 Mar 2025 21:25:53 +0900 Subject: [PATCH 10/15] FIX: do not using cast --- .../models/openai/_message_transform.py | 5 ++-- .../models/openai/_openai_client.py | 27 +++++-------------- .../models/openai/_transformation/__init__.py | 4 +++ .../models/openai/_transformation/registry.py | 6 ++--- .../models/openai/_transformation/types.py | 7 +++-- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py index 0ea2248c2a50..88fbae27639e 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -25,6 +25,7 @@ from ._transformation import ( TransformerMap, + TrasformerReturnType, build_conditional_transformer_func, build_transformer_func, register_transformer, @@ -256,9 +257,7 @@ def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: } -def function_execution_result_message( - message: LLMMessage, context: Dict[str, Any] -) -> list[ChatCompletionToolMessageParam]: +def function_execution_result_message(message: LLMMessage, context: Dict[str, Any]) -> TrasformerReturnType: assert isinstance(message, FunctionExecutionResultMessage) return [ ChatCompletionToolMessageParam(content=x.content, role="tool", tool_call_id=x.call_id) for x in message.content diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index e250a53967c5..5018e5067777 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -169,16 +169,14 @@ def to_oai_type( } transformers = get_transformer("openai", model_family) - def raise_value_error(message: LLMMessage, context: Dict[str, Any]) -> ChatCompletionMessageParam: + def raise_value_error(message: LLMMessage, context: Dict[str, Any]) -> Sequence[ChatCompletionMessageParam]: raise ValueError(f"Unknown message type: {type(message)}") - transformer: Callable[ - [LLMMessage, Dict[str, Any]], Union[ChatCompletionMessageParam, Sequence[ChatCompletionMessageParam]] - ] = transformers.get(type(message), raise_value_error) + transformer: Callable[[LLMMessage, Dict[str, Any]], Sequence[ChatCompletionMessageParam]] = transformers.get( + type(message), raise_value_error + ) result = transformer(message, context) - if isinstance(result, list): - return cast(List[ChatCompletionMessageParam], result) - return cast(List[ChatCompletionMessageParam], [result]) + return result def calculate_vision_tokens(image: Image, detail: str = "auto") -> int: @@ -276,19 +274,6 @@ def normalize_name(name: str) -> str: return re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:64] -def assert_valid_name(name: str) -> str: - """ - Ensure that configured names are valid, raises ValueError if not. - - For munging LLM responses use _normalize_name to ensure LLM specified names don't break the API. - """ - if not re.match(r"^[a-zA-Z0-9_-]+$", name): - raise ValueError(f"Invalid name: {name}. Only letters, numbers, '_' and '-' are allowed.") - if len(name) > 64: - raise ValueError(f"Invalid name: {name}. Name must be less than 64 characters.") - return name - - def count_tokens_openai( messages: Sequence[LLMMessage], model: str, @@ -308,7 +293,7 @@ def count_tokens_openai( # Message tokens. for message in messages: num_tokens += tokens_per_message - oai_message = to_oai_type(message, prepend_name=add_name_prefixes) + oai_message = to_oai_type(message, prepend_name=add_name_prefixes, model_family=model) for oai_message_part in oai_message: for key, value in oai_message_part.items(): if value is None: diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py index 54e59d8172d7..4259cf75a79f 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py @@ -6,8 +6,10 @@ register_transformer, ) from .types import ( + MessageParam, TransformerFunc, TransformerMap, + TrasformerReturnType, ) __all__ = [ @@ -18,4 +20,6 @@ "MESSAGE_TRANSFORMERS", "TransformerMap", "TransformerFunc", + "MessageParam", + "TrasformerReturnType", ] diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py index 6d463fb1ba90..26fb2cffac30 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py @@ -33,7 +33,7 @@ def transformer_func(message: LLMMessage, context: Any) -> Any: kwargs: Dict[str, Any] = {} for func in funcs: kwargs.update(func(message, context)) - return message_param_func(**kwargs) + return [message_param_func(**kwargs)] return transformer_func @@ -58,11 +58,11 @@ def build_conditional_transformer_func( def transformer(message: LLMMessage, context: Dict[str, Any]) -> Any: condition = condition_func(message, context) - constructor = message_param_func_map[condition] + message_param_func = message_param_func_map[condition] kwargs: Dict[str, Any] = {} for func in funcs_map[condition]: kwargs.update(func(message, context)) - return constructor(**kwargs) + return [message_param_func(**kwargs)] return transformer diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py index 91c9943dfa86..0c1107cf7caa 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py @@ -1,6 +1,9 @@ -from typing import Any, Callable, Dict, Type +from typing import Any, Callable, Dict, Sequence, Type, Union from autogen_core.models import LLMMessage +from openai.types.chat import ChatCompletionMessageParam -TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], Any] +MessageParam = Union[ChatCompletionMessageParam] # If that transformation move to global, add other message params here +TrasformerReturnType = Sequence[MessageParam] +TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], TrasformerReturnType] TransformerMap = Dict[Type[LLMMessage], TransformerFunc] From b3bb4545082d04129ce2c5221a8dbae8d9ce12e9 Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Fri, 28 Mar 2025 22:36:12 +0900 Subject: [PATCH 11/15] FIX: dict[str, Any] to each type --- .../models/openai/_message_transform.py | 27 +++++++++++-------- .../models/openai/_transformation/__init__.py | 2 ++ .../models/openai/_transformation/types.py | 15 ++++++++++- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py index 88fbae27639e..a5c0e5c7d1c4 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -24,6 +24,7 @@ ) from ._transformation import ( + LLMMessageContent, TransformerMap, TrasformerReturnType, build_conditional_transformer_func, @@ -47,30 +48,30 @@ def func_call_to_oai(message: FunctionCall) -> ChatCompletionMessageToolCallPara # ===Mini Transformers=== -def _assert_valid_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _assert_valid_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, None]: assert isinstance(message, (UserMessage, AssistantMessage)) assert_valid_name(message.source) return EMPTY -def _set_role(role: str) -> Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]: - def inner(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_role(role: str) -> Callable[[LLMMessage, Dict[str, Any]], Dict[str, str]]: + def inner(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str]: return {"role": role} return inner -def _set_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_name(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str]: assert isinstance(message, (UserMessage, AssistantMessage)) assert_valid_name(message.source) return {"name": message.source} -def _set_content_direct(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_content_direct(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, LLMMessageContent]: return {"content": message.content} -def _set_prepend_text_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_prepend_text_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str]: assert isinstance(message, (UserMessage, AssistantMessage)) assert isinstance(message.content, str) prepend = context.get("prepend_name", False) @@ -78,7 +79,9 @@ def _set_prepend_text_content(message: LLMMessage, context: Dict[str, Any]) -> D return {"content": prefix + message.content} -def _set_multimodal_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_multimodal_content( + message: LLMMessage, context: Dict[str, Any] +) -> Dict[str, List[ChatCompletionContentPartParam]]: assert isinstance(message, (UserMessage, AssistantMessage)) prepend = context.get("prepend_name", False) parts: List[ChatCompletionContentPartParam] = [] @@ -98,7 +101,9 @@ def _set_multimodal_content(message: LLMMessage, context: Dict[str, Any]) -> Dic return {"content": parts} -def _set_tool_calls(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_tool_calls( + message: LLMMessage, context: Dict[str, Any] +) -> Dict[str, List[ChatCompletionMessageToolCallParam]]: assert isinstance(message.content, list) assert isinstance(message, AssistantMessage) return { @@ -106,17 +111,17 @@ def _set_tool_calls(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, A } -def _set_thought_as_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_thought_as_content(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str | None]: assert isinstance(message, AssistantMessage) return {"content": message.thought} -def _set_thought_as_content_gemini(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_thought_as_content_gemini(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, str | None]: assert isinstance(message, AssistantMessage) return {"content": message.thought or " "} -def _set_empty_to_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, Any]: +def _set_empty_to_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, LLMMessageContent]: return {"content": message.content or " "} diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py index 4259cf75a79f..dc21b9c10815 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/__init__.py @@ -6,6 +6,7 @@ register_transformer, ) from .types import ( + LLMMessageContent, MessageParam, TransformerFunc, TransformerMap, @@ -21,5 +22,6 @@ "TransformerMap", "TransformerFunc", "MessageParam", + "LLMMessageContent", "TrasformerReturnType", ] diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py index 0c1107cf7caa..9cfb28e040cc 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/types.py @@ -1,9 +1,22 @@ -from typing import Any, Callable, Dict, Sequence, Type, Union +from typing import Any, Callable, Dict, List, Sequence, Type, Union +from autogen_core import FunctionCall, Image from autogen_core.models import LLMMessage +from autogen_core.models._types import FunctionExecutionResult from openai.types.chat import ChatCompletionMessageParam MessageParam = Union[ChatCompletionMessageParam] # If that transformation move to global, add other message params here TrasformerReturnType = Sequence[MessageParam] TransformerFunc = Callable[[LLMMessage, Dict[str, Any]], TrasformerReturnType] TransformerMap = Dict[Type[LLMMessage], TransformerFunc] + +LLMMessageContent = Union[ + # SystemMessage.content + str, + # UserMessage.content + List[Union[str, Image]], + # AssistantMessage.content + List[FunctionCall], + # FunctionExecutionResultMessage.content + List[FunctionExecutionResult], +] From d11de3d8e69f0139f5251960a19dde392ef61d7f Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Sun, 30 Mar 2025 17:33:43 +0900 Subject: [PATCH 12/15] FORMAT: linting --- .../autogen-ext/tests/models/test_anthropic_model_client.py | 3 ++- .../autogen-ext/tests/models/test_openai_model_client.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py index 60cfa727c6e0..5fd8e812feaf 100644 --- a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py @@ -560,6 +560,7 @@ def test_merge_system_messages_no_duplicates() -> None: # 중복된 내용도 그대로 병합됨 assert merged_messages[0].content == "Same instruction\nSame instruction" + @pytest.mark.asyncio async def test_empty_assistant_content_string_with_anthropic() -> None: """Test that an empty assistant content string is handled correctly.""" @@ -582,4 +583,4 @@ async def test_empty_assistant_content_string_with_anthropic() -> None: # Verify we got a response assert isinstance(result.content, str) - assert len(result.content) > 0 \ No newline at end of file + assert len(result.content) > 0 diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py index 510ced2a32ae..8d6a3193f554 100644 --- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py @@ -2355,4 +2355,6 @@ async def test_empty_assistant_content_string_with_some_model( result = await openai_client.create(messages=messages) assert isinstance(result.content, str) assert result.content.strip() != "" or result.content == " " + + # TODO: add integration tests for Azure OpenAI using AAD token. From c769ff5ed38a2ef38f0a712b598327bde0f1a122 Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Mon, 31 Mar 2025 10:09:56 +0900 Subject: [PATCH 13/15] FIX: fix my test case and fix code for pass it, and fix model family for it --- .../src/autogen_core/models/_model_client.py | 12 +++++++----- .../models/openai/_message_transform.py | 14 ++++++++++---- .../models/openai/_transformation/registry.py | 2 ++ .../tests/models/test_openai_model_client.py | 18 ++++++++++++++---- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/python/packages/autogen-core/src/autogen_core/models/_model_client.py b/python/packages/autogen-core/src/autogen_core/models/_model_client.py index efe8627b1faf..2a32189583a3 100644 --- a/python/packages/autogen-core/src/autogen_core/models/_model_client.py +++ b/python/packages/autogen-core/src/autogen_core/models/_model_client.py @@ -31,9 +31,9 @@ class ModelFamily: CLAUDE_3_HAIKU = "claude-3-haiku" CLAUDE_3_SONNET = "claude-3-sonnet" CLAUDE_3_OPUS = "claude-3-opus" - CLAUDE_3_5_HAIKU = "claude-3.5-haiku" - CLAUDE_3_5_SONNET = "claude-3.5-sonnet" - CLAUDE_3_7_SONNET = "claude-3.7-sonnet" + CLAUDE_3_5_HAIKU = "claude-3-5-haiku" + CLAUDE_3_5_SONNET = "claude-3-5-sonnet" + CLAUDE_3_7_SONNET = "claude-3-7-sonnet" UNKNOWN = "unknown" ANY: TypeAlias = Literal[ @@ -50,8 +50,9 @@ class ModelFamily: "claude-3-haiku", "claude-3-sonnet", "claude-3-opus", - "claude-3.5-haiku", - "claude-3.5-sonnet", + "claude-3-5-haiku", + "claude-3-5-sonnet", + "claude-3-7-sonnet", "unknown", ] @@ -66,6 +67,7 @@ def is_claude(family: str) -> bool: ModelFamily.CLAUDE_3_OPUS, ModelFamily.CLAUDE_3_5_HAIKU, ModelFamily.CLAUDE_3_5_SONNET, + ModelFamily.CLAUDE_3_7_SONNET, ) @staticmethod diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py index a5c0e5c7d1c4..64dc9e0e0df5 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_message_transform.py @@ -125,6 +125,12 @@ def _set_empty_to_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Di return {"content": message.content or " "} +def _set_pass_message_when_whitespace(message: LLMMessage, context: Dict[str, Any]) -> Dict[str, bool]: + if isinstance(message.content, str) and (message.content.isspace() or not message.content): + return {"pass_message": True} + return {} + + # === Base Transformers list === base_system_message_transformers: List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]] = [ _set_content_direct, @@ -250,13 +256,13 @@ def assistant_condition(message: LLMMessage, context: Dict[str, Any]) -> str: user_transformer_funcs_claude: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { - "text": single_user_transformer_funcs + [_set_empty_to_whitespace], - "multimodal": multimodal_user_transformer_funcs + [_set_empty_to_whitespace], + "text": single_user_transformer_funcs + [_set_pass_message_when_whitespace], + "multimodal": multimodal_user_transformer_funcs + [_set_pass_message_when_whitespace], } assistant_transformer_funcs_claude: Dict[str, List[Callable[[LLMMessage, Dict[str, Any]], Dict[str, Any]]]] = { - "text": single_assistant_transformer_funcs + [_set_empty_to_whitespace], + "text": single_assistant_transformer_funcs + [_set_pass_message_when_whitespace], "tools": tools_assistant_transformer_funcs, # that case, message.content is a list of FunctionCall "thought": thought_assistant_transformer_funcs_gemini, # that case, message.content is a list of FunctionCall } @@ -340,7 +346,7 @@ def function_execution_result_message(message: LLMMessage, context: Dict[str, An register_transformer("openai", model, __BASE_TRANSFORMER_MAP) for model in __claude_models: - register_transformer("openai", model, __BASE_TRANSFORMER_MAP) + register_transformer("openai", model, __CLAUDE_TRANSFORMER_MAP) for model in __gemini_models: register_transformer("openai", model, __GEMINI_TRANSFORMER_MAP) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py index 26fb2cffac30..b2e659158256 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py @@ -62,6 +62,8 @@ def transformer(message: LLMMessage, context: Dict[str, Any]) -> Any: kwargs: Dict[str, Any] = {} for func in funcs_map[condition]: kwargs.update(func(message, context)) + if kwargs.get("pass_message", False): + return [] return [message_param_func(**kwargs)] return transformer diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py index 8d6a3193f554..f59b95c7de55 100644 --- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py @@ -1602,6 +1602,10 @@ def openai_client(request: pytest.FixtureRequest) -> OpenAIChatCompletionClient: api_key = os.getenv("GEMINI_API_KEY") if not api_key: pytest.skip("GEMINI_API_KEY not found in environment variables") + elif model.startswith("claude"): + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + pytest.skip("ANTHROPIC_API_KEY not found in environment variables") else: api_key = os.getenv("OPENAI_API_KEY") if not api_key: @@ -2340,21 +2344,27 @@ async def test_empty_assistant_content_with_gemini(model: str, openai_client: Op @pytest.mark.asyncio -# @pytest.mark.parametrize("model", ["gemini-1.5-flash", "claude-3-5-haiku-latest"]) # -@pytest.mark.parametrize("model", ["gemini-1.5-flash"]) # before add claude-3-5-haiku-latest at openai/_model_info.py +@pytest.mark.parametrize( + "model", + [ + "gpt-4o-mini", + "gemini-1.5-flash", + "claude-3-5-haiku-20241022", + ], +) async def test_empty_assistant_content_string_with_some_model( model: str, openai_client: OpenAIChatCompletionClient ) -> None: # message: assistant is response empty content messages: list[LLMMessage] = [ UserMessage(content="Say something", source="user"), - AssistantMessage(content="", source="assistant"), + AssistantMessage(content="test", source="assistant"), + UserMessage(content="", source="user"), ] # This will crash if _set_empty_to_whitespace is not applied to "content" result = await openai_client.create(messages=messages) assert isinstance(result.content, str) - assert result.content.strip() != "" or result.content == " " # TODO: add integration tests for Azure OpenAI using AAD token. From c8e52a2b29829e490f26b849f7b31ae610010444 Mon Sep 17 00:00:00 2001 From: "chiyoung.song" Date: Mon, 31 Mar 2025 10:13:54 +0900 Subject: [PATCH 14/15] FIX: In the model_info wrong claude model family too --- .../src/autogen_ext/models/openai/_model_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py index c36ea42bf557..ae537fd6c71b 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py @@ -21,9 +21,9 @@ "claude-3-haiku": "claude-3-haiku-20240307", "claude-3-sonnet": "claude-3-sonnet-20240229", "claude-3-opus": "claude-3-opus-20240229", - "claude-3.5-haiku": "claude-3-5-haiku-20241022", - "claude-3.5-sonnet": "claude-3-5-sonnet-20241022", - "claude-3.7-sonnet": "claude-3-7-sonnet-20250219", + "claude-3-5-haiku": "claude-3-5-haiku-20241022", + "claude-3-5-sonnet": "claude-3-5-sonnet-20241022", + "claude-3-7-sonnet": "claude-3-7-sonnet-20250219", } _MODEL_INFO: Dict[str, ModelInfo] = { From 06ab3e6c67ad663f65e101051e90029ce45df5ac Mon Sep 17 00:00:00 2001 From: Eric Zhu Date: Sun, 30 Mar 2025 20:56:54 -0700 Subject: [PATCH 15/15] add claude models to test --- .../autogen-ext/tests/models/test_openai_model_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py index f59b95c7de55..abb270a712ef 100644 --- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py +++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py @@ -1620,7 +1620,7 @@ def openai_client(request: pytest.FixtureRequest) -> OpenAIChatCompletionClient: @pytest.mark.asyncio @pytest.mark.parametrize( "model", - ["gpt-4o-mini", "gemini-1.5-flash"], + ["gpt-4o-mini", "gemini-1.5-flash", "claude-3-5-haiku-20241022"], ) async def test_model_client_basic_completion(model: str, openai_client: OpenAIChatCompletionClient) -> None: # Test basic completion @@ -1637,7 +1637,7 @@ async def test_model_client_basic_completion(model: str, openai_client: OpenAICh @pytest.mark.asyncio @pytest.mark.parametrize( "model", - ["gpt-4o-mini", "gemini-1.5-flash"], + ["gpt-4o-mini", "gemini-1.5-flash", "claude-3-5-haiku-20241022"], ) async def test_model_client_with_function_calling(model: str, openai_client: OpenAIChatCompletionClient) -> None: # Test tool calling @@ -2069,7 +2069,7 @@ async def test_add_name_prefixes(monkeypatch: pytest.MonkeyPatch) -> None: [ "gpt-4o-mini", "gemini-1.5-flash", - # TODO: Add anthropic models when available. + "claude-3-5-haiku-20241022", ], ) async def test_muliple_system_message(model: str, openai_client: OpenAIChatCompletionClient) -> None: