diff --git a/src/any_llm/providers/gemini/base.py b/src/any_llm/providers/gemini/base.py index 436812e10..159f5bec9 100644 --- a/src/any_llm/providers/gemini/base.py +++ b/src/any_llm/providers/gemini/base.py @@ -250,6 +250,7 @@ def _convert_completion_response(response: Any) -> ChatCompletion: content=message_dict.get("content"), tool_calls=cast("list[ChatCompletionMessageToolCallType] | None", tool_calls), reasoning=Reasoning(content=reasoning_content) if reasoning_content else None, + extra_content=message_dict.get("extra_content"), ) from typing import Literal diff --git a/src/any_llm/providers/gemini/utils.py b/src/any_llm/providers/gemini/utils.py index a70ba3b27..67e9233b6 100644 --- a/src/any_llm/providers/gemini/utils.py +++ b/src/any_llm/providers/gemini/utils.py @@ -192,6 +192,46 @@ def _convert_file_to_part(block: dict[str, Any], provider_name: str) -> types.Pa return types.Part.from_uri(file_uri=file_data, mime_type=guessed_type or "application/octet-stream") +def _is_decodable_signature(signature: str) -> bool: + """Return True if *signature* is base64 the google-genai SDK can decode. + + The SDK reads both the standard and the URL-safe alphabet and tolerates missing + padding, so match that leniency rather than being stricter than the layer below. + """ + normalized = signature.replace("-", "+").replace("_", "/") + try: + base64.b64decode(normalized + "=" * (-len(normalized) % 4), validate=True) + except (binascii.Error, ValueError): + return False + return True + + +def _extract_google_thought_signature(container: dict[str, Any], provider_name: str) -> str | bytes | None: + """Extract the thought_signature stored on a message's or tool call's extra_content. + + A container that carries no Google signature (no extra_content, another provider's + payload, an explicit None) yields None. A signature that is present but undecodable is + a caller error, so it is rejected here instead of surfacing from the SDK as a bare + pydantic ValidationError several frames away. + """ + extra_content = container.get("extra_content") + if not isinstance(extra_content, dict) or not isinstance(google_extra := extra_content.get("google"), dict): + return None + + signature = google_extra.get("thought_signature") + if signature is None: + return None + if isinstance(signature, bytes): + return signature + if not isinstance(signature, str) or not signature or not _is_decodable_signature(signature): + msg = ( + "extra_content['google']['thought_signature'] must be a non-empty base64 string " + f"or raw bytes, got {signature!r}" + ) + raise InvalidRequestError(msg, provider_name=provider_name) + return signature + + def _convert_messages( messages: list[dict[str, Any]], provider_name: str = "gemini" ) -> tuple[list[types.Content], str | None]: @@ -232,10 +272,7 @@ def _convert_messages( # Extract thought_signature if present (OpenAI compatibility format) # SDK accepts base64 string or bytes - thought_signature = None - if extra_content := tool_call.get("extra_content"): - if google_extra := extra_content.get("google"): - thought_signature = google_extra.get("thought_signature") + thought_signature = _extract_google_thought_signature(tool_call, provider_name) # For the first function call in parallel calls, if no thought_signature is present, # use the skip validator sentinel per Google's documentation: @@ -246,11 +283,19 @@ def _convert_messages( parts.append( types.Part( function_call=types.FunctionCall(name=function_call["name"], args=args), - thought_signature=thought_signature, + thought_signature=cast("bytes | None", thought_signature), ) ) else: - parts = [types.Part.from_text(text=message["content"])] + parts = [ + types.Part( + text=message["content"], + # The SDK field is typed bytes but its validator decodes a base64 str. + thought_signature=cast( + "bytes | None", _extract_google_thought_signature(message, provider_name) + ), + ) + ] formatted_messages.append(types.Content(role="model", parts=parts)) elif message["role"] == "tool": @@ -348,6 +393,7 @@ def _resolve_finish_reason( def _convert_response_to_response_dict(response: types.GenerateContentResponse) -> dict[str, Any]: + """Convert a Gemini GenerateContentResponse into an OpenAI-shaped completion dict.""" response_dict = { "id": "google_genai_response", "model": "google/genai", @@ -363,6 +409,9 @@ def _convert_response_to_response_dict(response: types.GenerateContentResponse) reasoning = None tool_calls_list: list[dict[str, Any]] = [] text_content = None + # Gemini 3 signs the last non-function-call part of a text answer. It rides message.extra_content, + # the same spelling Google's OpenAI-compatible endpoint uses. + message_extra_content = None parts = candidate.content.parts if candidate.content else None for part in parts or []: @@ -388,8 +437,10 @@ def _convert_response_to_response_dict(response: types.GenerateContentResponse) tool_call_dict["extra_content"] = extra_content tool_calls_list.append(tool_call_dict) - elif part_text := getattr(part, "text", None): - text_content = (text_content or "") + part_text + else: + if part.text: + text_content = (text_content or "") + part.text + message_extra_content = _thought_signature_extra_content(part) or message_extra_content # Truncated or filtered responses produce a choice even without content or tool # calls, e.g. a thinking model that spent the whole max_output_tokens budget on @@ -402,6 +453,7 @@ def _convert_response_to_response_dict(response: types.GenerateContentResponse) "content": None if tool_calls_list else text_content, "reasoning": reasoning or None, "tool_calls": tool_calls_list or None, + "extra_content": message_extra_content, }, "finish_reason": _resolve_finish_reason(mapped_finish_reason, bool(tool_calls_list)) or "stop", "index": 0, @@ -462,6 +514,7 @@ def _create_openai_chunk_from_google_chunk( content = "" reasoning_content = "" tool_calls_list: list[ChoiceDeltaToolCall] = [] + message_extra_content = None # Content can be absent on terminal chunks, e.g. when the response is truncated or # filtered before any part is produced; the finish reason must still be surfaced. @@ -495,8 +548,9 @@ def _create_openai_chunk_from_google_chunk( extra_content=extra_content, ) ) - elif part.text: - content += part.text + else: + content += part.text or "" # the signed final part may carry empty text + message_extra_content = _thought_signature_extra_content(part) or message_extra_content # Unmapped reasons stay None so non-final chunks are not forced to a terminal reason. finish_reason = _resolve_finish_reason(_map_finish_reason(candidate.finish_reason), bool(tool_calls_list)) @@ -506,6 +560,7 @@ def _create_openai_chunk_from_google_chunk( role="assistant", reasoning=Reasoning(content=reasoning_content) if reasoning_content else None, tool_calls=tool_calls_list or None, + extra_content=message_extra_content, ) choice = ChunkChoice( diff --git a/tests/unit/providers/test_gemini_provider.py b/tests/unit/providers/test_gemini_provider.py index ce5eae994..b1154d293 100644 --- a/tests/unit/providers/test_gemini_provider.py +++ b/tests/unit/providers/test_gemini_provider.py @@ -1,4 +1,5 @@ import base64 +import json from collections.abc import AsyncIterator from contextlib import contextmanager from typing import Any, get_args @@ -1248,6 +1249,7 @@ def test_streaming_completion_with_tool_call_preserves_thought_signature() -> No tool_call = chunk.choices[0].delta.tool_calls[0] assert tool_call.extra_content is not None assert tool_call.extra_content["google"]["thought_signature"] == base64.b64encode(original_bytes).decode("utf-8") + assert chunk.choices[0].delta.extra_content is None # signature stays on the tool call def test_streaming_completion_with_tool_call_no_thought_signature() -> None: @@ -1458,6 +1460,7 @@ async def test_streaming_via_acompletion_keeps_tool_call_index_stable_across_chu def test_convert_response_preserves_thought_signature() -> None: + """A signed function call keeps its signature on the tool call and off the message.""" import base64 mock_response = Mock() @@ -1491,6 +1494,7 @@ def test_convert_response_preserves_thought_signature() -> None: assert tool_calls[0]["extra_content"]["google"]["thought_signature"] == base64.b64encode( b"test-signature-bytes" ).decode("utf-8") + assert response_dict["choices"][0]["message"].get("extra_content") is None # signature stays on the tool call def test_convert_response_no_thought_signature() -> None: @@ -1523,6 +1527,159 @@ def test_convert_response_no_thought_signature() -> None: assert "extra_content" not in tool_calls[0] or tool_calls[0].get("extra_content") is None +def test_convert_response_text_part_thought_signature_rides_the_message() -> None: + """Gemini 3 signs the final text part of a text-only answer; it lands on message.extra_content.""" + mock_response = Mock() + mock_response.candidates = [Mock()] + mock_response.candidates[0].content = Mock() + mock_response.candidates[0].finish_reason = types.FinishReason.STOP + mock_response.usage_metadata = None + + mock_thought = Mock() + mock_thought.thought = True + mock_thought.text = "let me think" + mock_thought.function_call = None + mock_thought.thought_signature = None + + mock_text = Mock() + mock_text.thought = None + mock_text.text = "42" + mock_text.function_call = None + mock_text.thought_signature = b"test-signature-bytes" + + mock_response.candidates[0].content.parts = [mock_thought, mock_text] + + message = _convert_response_to_response_dict(mock_response)["choices"][0]["message"] + + assert message["content"] == "42" + assert message["extra_content"] == { + "google": {"thought_signature": base64.b64encode(b"test-signature-bytes").decode("utf-8")} + } + + +def test_streaming_text_part_thought_signature_rides_the_delta() -> None: + """The signed final part arrives with empty text on the last chunk; the delta must still carry it.""" + mock_response = Mock() + mock_response.candidates = [Mock()] + mock_response.candidates[0].content = Mock() + mock_response.candidates[0].finish_reason = types.FinishReason.STOP + mock_response.model_version = "gemini-3-flash-preview" + mock_response.usage_metadata = None + + mock_part = Mock() + mock_part.thought = None + mock_part.text = "" + mock_part.function_call = None + mock_part.thought_signature = b"test-signature-bytes" + + mock_response.candidates[0].content.parts = [mock_part] + + delta = _create_openai_chunk_from_google_chunk(mock_response).choices[0].delta + + assert delta.content is None + assert delta.tool_calls is None + assert delta.extra_content == { + "google": {"thought_signature": base64.b64encode(b"test-signature-bytes").decode("utf-8")} + } + + +def test_convert_messages_text_turn_replays_thought_signature() -> None: + """A text-only assistant turn carrying extra_content replays its signature on the text part.""" + base64_signature = base64.b64encode(b"test-signature-bytes").decode("utf-8") + messages: list[dict[str, Any]] = [ + {"role": "user", "content": "How many sheep?"}, + {"role": "assistant", "content": "42", "extra_content": {"google": {"thought_signature": base64_signature}}}, + ] + + formatted_messages, _ = _convert_messages(messages) + + assert formatted_messages[1].parts is not None + assert formatted_messages[1].parts[0].text == "42" + assert formatted_messages[1].parts[0].thought_signature == b"test-signature-bytes" + + +@pytest.mark.parametrize( + "extra_content", + [ + "not-a-dict", + {"google": "not-a-dict"}, + {"anthropic": {"signature": "abc"}}, + {"google": {"thought_signature": None}}, + ], +) +def test_convert_messages_treats_a_non_google_extra_content_as_unsigned(extra_content: Any) -> None: + """extra_content that carries no Google signature leaves the part unsigned.""" + messages: list[dict[str, Any]] = [{"role": "assistant", "content": "42", "extra_content": extra_content}] + + formatted_messages, _ = _convert_messages(messages) + + assert formatted_messages[0].parts is not None + assert formatted_messages[0].parts[0].thought_signature is None + + +@pytest.mark.parametrize("signature", ["", "not!!base64@@", 123, ["sig"]]) +def test_convert_messages_text_turn_rejects_an_undecodable_signature(signature: Any) -> None: + """An undecodable signature is a caller error, not a bare pydantic ValidationError.""" + messages: list[dict[str, Any]] = [ + {"role": "assistant", "content": "42", "extra_content": {"google": {"thought_signature": signature}}} + ] + + with pytest.raises(InvalidRequestError, match="thought_signature"): + _convert_messages(messages) + + +@pytest.mark.parametrize("signature", ["", "not!!base64@@", 123, ["sig"]]) +def test_convert_messages_tool_call_rejects_an_undecodable_signature(signature: Any) -> None: + """The tool call path rejects the same shapes the text path does.""" + messages: list[dict[str, Any]] = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + "extra_content": {"google": {"thought_signature": signature}}, + } + ], + } + ] + + with pytest.raises(InvalidRequestError, match="thought_signature"): + _convert_messages(messages) + + +@pytest.mark.parametrize("signature", ["dGVzdA==", "dGVzdA", "-_8-P76_", b"test-signature-bytes"]) +def test_convert_messages_accepts_every_signature_spelling_the_sdk_decodes(signature: Any) -> None: + """Unpadded and URL-safe base64, and raw bytes, all reach the part decoded.""" + messages: list[dict[str, Any]] = [ + {"role": "assistant", "content": "42", "extra_content": {"google": {"thought_signature": signature}}} + ] + + formatted_messages, _ = _convert_messages(messages) + + assert formatted_messages[0].parts is not None + assert formatted_messages[0].parts[0].thought_signature is not None + + +def test_convert_messages_tool_call_without_a_signature_keeps_the_skip_sentinel() -> None: + """An unsigned first function call still gets Google's skip sentinel, unvalidated.""" + messages: list[dict[str, Any]] = [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}], + } + ] + + formatted_messages, _ = _convert_messages(messages) + + assert formatted_messages[0].parts is not None + part_json = json.loads(formatted_messages[0].parts[0].model_dump_json(exclude_none=True)) + assert part_json["thought_signature"] == "skip_thought_signature_validator" + + def test_convert_messages_with_thought_signature_in_extra_content() -> None: # Use valid base64 that round-trips correctly original_bytes = b"test-signature-bytes" @@ -2184,6 +2341,28 @@ def test_convert_completion_response_preserves_prompt_tokens_details() -> None: assert result.usage.prompt_tokens_details.cached_tokens == 80 +def test_convert_completion_response_preserves_message_extra_content() -> None: + """The text-part signature placed on the message dict reaches the public ChatCompletion.""" + extra_content = {"google": {"thought_signature": "dGVzdA=="}} + response_dict = { + "id": "google_genai_response", + "model": "google/genai", + "created": 0, + "choices": [ + { + "message": {"role": "assistant", "content": "42", "tool_calls": None, "extra_content": extra_content}, + "finish_reason": "stop", + "index": 0, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + result = GoogleProvider._convert_completion_response((response_dict, "test-model")) + + assert result.choices[0].message.extra_content == extra_content + + def test_convert_completion_response_preserves_completion_tokens_details() -> None: """Test that _convert_completion_response passes completion_tokens_details through to ChatCompletion.""" response_dict = {