[BugFix][Refactor] Modular Transformer Pipeline and Fix Gemini/Anthropic Empty Content Handling

python/packages/autogen-core/src/autogen_core/models/_model_client.py

@@ -31,9 +31,9 @@ class ModelFamily:
     CLAUDE_3_HAIKU = "claude-3-haiku"
     CLAUDE_3_SONNET = "claude-3-sonnet"
     CLAUDE_3_OPUS = "claude-3-opus"
-    CLAUDE_3_5_HAIKU = "claude-3.5-haiku"
-    CLAUDE_3_5_SONNET = "claude-3.5-sonnet"
-    CLAUDE_3_7_SONNET = "claude-3.7-sonnet"
+    CLAUDE_3_5_HAIKU = "claude-3-5-haiku"
+    CLAUDE_3_5_SONNET = "claude-3-5-sonnet"
+    CLAUDE_3_7_SONNET = "claude-3-7-sonnet"
     UNKNOWN = "unknown"
 
     ANY: TypeAlias = Literal[
@@ -50,8 +50,9 @@ class ModelFamily:
         "claude-3-haiku",
         "claude-3-sonnet",
         "claude-3-opus",
-        "claude-3.5-haiku",
-        "claude-3.5-sonnet",
+        "claude-3-5-haiku",
+        "claude-3-5-sonnet",
+        "claude-3-7-sonnet",
         "unknown",
     ]
 
@@ -66,6 +67,7 @@ def is_claude(family: str) -> bool:
             ModelFamily.CLAUDE_3_OPUS,
             ModelFamily.CLAUDE_3_5_HAIKU,
             ModelFamily.CLAUDE_3_5_SONNET,
+            ModelFamily.CLAUDE_3_7_SONNET,
         )
 
     @staticmethod

python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py

@@ -12,6 +12,7 @@
     AsyncGenerator,
     Coroutine,
     Dict,
+    Iterable,
     List,
     Literal,
     Mapping,
@@ -20,6 +21,7 @@
     Set,
     Union,
     cast,
+    overload,
 )
 
 import tiktoken
@@ -142,20 +144,41 @@ def get_mime_type_from_image(image: Image) -> Literal["image/jpeg", "image/png",
         return "image/jpeg"
 
 
+@overload
+def __empty_content_to_whitespace(content: str) -> str: ...
+
+
+@overload
+def __empty_content_to_whitespace(content: List[Any]) -> Iterable[Any]: ...
+
+
+def __empty_content_to_whitespace(
+    content: Union[str, List[Union[str, Image]]],
+) -> Union[str, Iterable[Any]]:
+    if isinstance(content, str) and not content.strip():
+        return " "
+    elif isinstance(content, list) and not any(isinstance(x, str) and not x.strip() for x in content):
+        for idx, message in enumerate(content):
+            if isinstance(message, str) and not message.strip():
+                content[idx] = " "
+
+    return content
+
+
 def user_message_to_anthropic(message: UserMessage) -> MessageParam:
     assert_valid_name(message.source)
 
     if isinstance(message.content, str):
         return {
             "role": "user",
-            "content": message.content,
+            "content": __empty_content_to_whitespace(message.content),
         }
     else:
         blocks: List[Union[TextBlockParam, ImageBlockParam]] = []
 
         for part in message.content:
             if isinstance(part, str):
-                blocks.append(TextBlockParam(type="text", text=part))
+                blocks.append(TextBlockParam(type="text", text=__empty_content_to_whitespace(part)))
             elif isinstance(part, Image):
                 blocks.append(
                     ImageBlockParam(
@@ -177,7 +200,7 @@ def user_message_to_anthropic(message: UserMessage) -> MessageParam:
 
 
 def system_message_to_anthropic(message: SystemMessage) -> str:
-    return message.content
+    return __empty_content_to_whitespace(message.content)
 
 
 def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam:
@@ -190,6 +213,7 @@ def assistant_message_to_anthropic(message: AssistantMessage) -> MessageParam:
         for func_call in message.content:
             # Parse the arguments and convert to dict if it's a JSON string
             args = func_call.arguments
+            args = __empty_content_to_whitespace(args)
             if isinstance(args, str):
                 try:
                     args_dict = json.loads(args)

python/packages/autogen-ext/src/autogen_ext/models/openai/__init__.py

@@ -1,3 +1,4 @@
+from . import _message_transform
 from ._openai_client import (
     AZURE_OPENAI_USER_AGENT,
     AzureOpenAIChatCompletionClient,
@@ -20,4 +21,5 @@
     "BaseOpenAIClientConfigurationConfigModel",
     "CreateArgumentsConfigModel",
     "AZURE_OPENAI_USER_AGENT",
+    "_message_transform",
 ]