From 9b72071c1ce8599376a4b73cc3bd25ab7b3bbb26 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Tue, 11 Aug 2026 18:06:18 +0800 Subject: [PATCH 1/5] feat: preserve provider timing in normalized usage --- src/any_llm/providers/groq/utils.py | 40 +++++++++---- src/any_llm/providers/ollama/utils.py | 36 +++++++++--- tests/unit/providers/test_groq_provider.py | 21 +++++++ tests/unit/providers/test_ollama_provider.py | 60 +++++++++++++++++++- 4 files changed, 136 insertions(+), 21 deletions(-) diff --git a/src/any_llm/providers/groq/utils.py b/src/any_llm/providers/groq/utils.py index 0bf6eb76f..570c8c800 100644 --- a/src/any_llm/providers/groq/utils.py +++ b/src/any_llm/providers/groq/utils.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast from groq.types import ModelListResponse as GroqModelListResponse from groq.types.chat import ChatCompletion as GroqChatCompletion @@ -35,6 +35,16 @@ ) +def _groq_timing_details(usage: Any) -> dict[str, int | float]: + """Return numeric provider timing fields without adding absent fields to usage extras.""" + timing: dict[str, int | float] = {} + for field in ("queue_time", "prompt_time", "completion_time", "total_time"): + value = getattr(usage, field, None) + if isinstance(value, (int, float)): + timing[field] = value + return timing + + def to_chat_completion(response: GroqChatCompletion) -> ChatCompletion: """Convert Groq ChatCompletion into our ChatCompletion type directly.""" @@ -44,11 +54,15 @@ def to_chat_completion(response: GroqChatCompletion) -> ChatCompletion: # Reference: https://console.groq.com/docs/prompt-caching prompt_details = response.usage.prompt_tokens_details cached_tokens = prompt_details.cached_tokens if prompt_details else None - usage = CompletionUsage( - prompt_tokens=response.usage.prompt_tokens, - completion_tokens=response.usage.completion_tokens, - total_tokens=response.usage.total_tokens, - prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens) if cached_tokens else None, + timing_details = _groq_timing_details(response.usage) + usage = CompletionUsage.model_validate( + { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + "prompt_tokens_details": PromptTokensDetails(cached_tokens=cached_tokens) if cached_tokens else None, + **timing_details, + } ) choices: list[Choice] = [] @@ -149,11 +163,15 @@ def _create_openai_chunk_from_groq_chunk(groq_chunk: GroqChatCompletionChunk) -> # Reference: https://console.groq.com/docs/prompt-caching prompt_details = usage_data.prompt_tokens_details cached_tokens = prompt_details.cached_tokens if prompt_details else None - usage = CompletionUsage( - prompt_tokens=usage_data.prompt_tokens, - completion_tokens=usage_data.completion_tokens, - total_tokens=usage_data.total_tokens, - prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens) if cached_tokens else None, + timing_details = _groq_timing_details(usage_data) + usage = CompletionUsage.model_validate( + { + "prompt_tokens": usage_data.prompt_tokens, + "completion_tokens": usage_data.completion_tokens, + "total_tokens": usage_data.total_tokens, + "prompt_tokens_details": PromptTokensDetails(cached_tokens=cached_tokens) if cached_tokens else None, + **timing_details, + } ) return ChatCompletionChunk( diff --git a/src/any_llm/providers/ollama/utils.py b/src/any_llm/providers/ollama/utils.py index 0f21e4cf9..0714f1ec3 100644 --- a/src/any_llm/providers/ollama/utils.py +++ b/src/any_llm/providers/ollama/utils.py @@ -66,6 +66,16 @@ def _create_openai_embedding_response_from_ollama( ) +def _ollama_timing_details(response: OllamaChatResponse) -> dict[str, int | float]: + """Return numeric Ollama timing fields without adding absent fields to usage extras.""" + timing: dict[str, int | float] = {} + for field in ("total_duration", "load_duration", "prompt_eval_duration", "eval_duration"): + value = getattr(response, field, None) + if isinstance(value, (int, float)): + timing[field] = value + return timing + + def _create_openai_chunk_from_ollama_chunk(ollama_chunk: OllamaChatResponse) -> ChatCompletionChunk: """Convert an Ollama streaming chunk to OpenAI ChatCompletionChunk format.""" @@ -125,11 +135,15 @@ def _create_openai_chunk_from_ollama_chunk(ollama_chunk: OllamaChatResponse) -> usage = None prompt_tokens = ollama_chunk.prompt_eval_count completion_tokens = ollama_chunk.eval_count - if prompt_tokens or completion_tokens: - usage = CompletionUsage( - prompt_tokens=prompt_tokens or 0, - completion_tokens=completion_tokens or 0, - total_tokens=(prompt_tokens or 0) + (completion_tokens or 0), + timing_details = _ollama_timing_details(ollama_chunk) + if prompt_tokens or completion_tokens or timing_details: + usage = CompletionUsage.model_validate( + { + "prompt_tokens": prompt_tokens or 0, + "completion_tokens": completion_tokens or 0, + "total_tokens": (prompt_tokens or 0) + (completion_tokens or 0), + **timing_details, + } ) return ChatCompletionChunk( @@ -158,6 +172,7 @@ def _create_chat_completion_from_ollama_response(response: OllamaChatResponse) - prompt_tokens = response.prompt_eval_count or 0 completion_tokens = response.eval_count or 0 + timing_details = _ollama_timing_details(response) response_message: OllamaMessage = response.message if not response_message or not isinstance(response_message, OllamaMessage): @@ -202,10 +217,13 @@ def _create_chat_completion_from_ollama_response(response: OllamaChatResponse) - choice = Choice(index=0, finish_reason=finish_reason, message=message) - usage = CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, + usage = CompletionUsage.model_validate( + { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + **timing_details, + } ) return ChatCompletion( diff --git a/tests/unit/providers/test_groq_provider.py b/tests/unit/providers/test_groq_provider.py index f1554d03b..bde4e5e98 100644 --- a/tests/unit/providers/test_groq_provider.py +++ b/tests/unit/providers/test_groq_provider.py @@ -149,6 +149,10 @@ def test_to_chat_completion_extracts_cached_tokens() -> None: "completion_tokens": 1817, "total_tokens": 6458, "prompt_tokens_details": {"cached_tokens": 4608}, + "queue_time": 0.01, + "prompt_time": 0.02, + "completion_time": 0.03, + "total_time": 0.04, }, } ) @@ -161,6 +165,12 @@ def test_to_chat_completion_extracts_cached_tokens() -> None: assert result.usage.total_tokens == 6458 assert result.usage.prompt_tokens_details is not None assert result.usage.prompt_tokens_details.cached_tokens == 4608 + assert result.usage.model_extra == { + "queue_time": 0.01, + "prompt_time": 0.02, + "completion_time": 0.03, + "total_time": 0.04, + } def test_to_chat_completion_without_cached_tokens() -> None: @@ -197,6 +207,7 @@ def test_to_chat_completion_without_cached_tokens() -> None: assert result.usage.completion_tokens == 50 assert result.usage.total_tokens == 150 assert result.usage.prompt_tokens_details is None + assert result.usage.model_extra == {} @pytest.mark.asyncio @@ -258,6 +269,10 @@ def test_streaming_chunk_extracts_cached_tokens() -> None: "completion_tokens": 1817, "total_tokens": 6458, "prompt_tokens_details": {"cached_tokens": 4608}, + "queue_time": 0, + "prompt_time": 0.02, + "completion_time": 0.03, + "total_time": 0.04, }, } ) @@ -268,6 +283,12 @@ def test_streaming_chunk_extracts_cached_tokens() -> None: assert result.usage.prompt_tokens == 4641 assert result.usage.prompt_tokens_details is not None assert result.usage.prompt_tokens_details.cached_tokens == 4608 + assert result.usage.model_extra == { + "queue_time": 0, + "prompt_time": 0.02, + "completion_time": 0.03, + "total_time": 0.04, + } def _make_openai_response(text: str): # type: ignore[no-untyped-def] diff --git a/tests/unit/providers/test_ollama_provider.py b/tests/unit/providers/test_ollama_provider.py index 663ab3544..ca296a96f 100644 --- a/tests/unit/providers/test_ollama_provider.py +++ b/tests/unit/providers/test_ollama_provider.py @@ -8,7 +8,10 @@ from ollama import Message as OllamaMessage from any_llm.providers.ollama.ollama import OllamaProvider -from any_llm.providers.ollama.utils import _create_chat_completion_from_ollama_response +from any_llm.providers.ollama.utils import ( + _create_chat_completion_from_ollama_response, + _create_openai_chunk_from_ollama_chunk, +) from any_llm.types.completion import CompletionParams @@ -134,6 +137,61 @@ async def test_create_chat_completion_extracts_think_content() -> None: assert result.choices[0].message.reasoning.content == "This is my reasoning process" assert result.choices[0].message.content == "This is the actual response" + assert result.usage is not None + assert result.usage.model_extra == {} + + +def test_create_chat_completion_preserves_timing_details() -> None: + """Provider timing fields survive normalization as extra usage fields.""" + response = OllamaChatResponse( + model="llama3.1", + created_at="2024-01-01T12:00:00.000000Z", + done=True, + done_reason="stop", + total_duration=1_000, + load_duration=2_000, + prompt_eval_count=10, + prompt_eval_duration=3_000, + eval_count=20, + eval_duration=4_000, + message=OllamaMessage(role="assistant", content="Hello"), + ) + + result = _create_chat_completion_from_ollama_response(response) + + assert result.usage is not None + assert result.usage.model_extra == { + "total_duration": 1_000, + "load_duration": 2_000, + "prompt_eval_duration": 3_000, + "eval_duration": 4_000, + } + + +def test_streaming_chunk_preserves_timing_details() -> None: + """Streaming usage keeps timing fields, including reported zero values.""" + chunk = OllamaChatResponse( + model="llama3.1", + done=True, + done_reason="stop", + total_duration=0, + load_duration=1_000, + prompt_eval_count=0, + prompt_eval_duration=2_000, + eval_count=0, + eval_duration=3_000, + message=OllamaMessage(role="assistant", content=""), + ) + + result = _create_openai_chunk_from_ollama_chunk(chunk) + + assert result.usage is not None + assert result.usage.model_extra == { + "total_duration": 0, + "load_duration": 1_000, + "prompt_eval_duration": 2_000, + "eval_duration": 3_000, + } @pytest.mark.asyncio From 714d3da6c05cc350849562c89d76cf7090b1a212 Mon Sep 17 00:00:00 2001 From: njbrake Date: Tue, 11 Aug 2026 14:21:12 +0000 Subject: [PATCH 2/5] refactor(groq,ollama): use typed attribute access for timing helpers The timing fields are declared optionals on both provider SDK types (groq.types.completion_usage.CompletionUsage and ollama.ChatResponse), so dynamic getattr access is unnecessary and hides typos from mypy. AGENTS.md prefers direct attribute access when the field is typed. Two existing Ollama tests relied on the getattr fallback: pydantic v2 field names are absent from dir(ChatResponse), so Mock(spec=...) raises AttributeError for the duration fields. Declare them as None on those mocks, which also makes the no-timing assertion exercise the real filter path instead of the swallowed AttributeError. Co-Authored-By: Claude Opus 5 (1M context) --- src/any_llm/providers/groq/utils.py | 20 +++++++++++--------- src/any_llm/providers/ollama/utils.py | 17 +++++++++-------- tests/unit/providers/test_ollama_provider.py | 8 ++++++++ 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/any_llm/providers/groq/utils.py b/src/any_llm/providers/groq/utils.py index 570c8c800..48431e1c5 100644 --- a/src/any_llm/providers/groq/utils.py +++ b/src/any_llm/providers/groq/utils.py @@ -1,8 +1,9 @@ -from typing import TYPE_CHECKING, Any, Literal, cast +from typing import TYPE_CHECKING, Literal, cast from groq.types import ModelListResponse as GroqModelListResponse from groq.types.chat import ChatCompletion as GroqChatCompletion from groq.types.chat import ChatCompletionChunk as GroqChatCompletionChunk +from groq.types.completion_usage import CompletionUsage as GroqCompletionUsage from any_llm.types.completion import ( ChatCompletion, @@ -35,14 +36,15 @@ ) -def _groq_timing_details(usage: Any) -> dict[str, int | float]: - """Return numeric provider timing fields without adding absent fields to usage extras.""" - timing: dict[str, int | float] = {} - for field in ("queue_time", "prompt_time", "completion_time", "total_time"): - value = getattr(usage, field, None) - if isinstance(value, (int, float)): - timing[field] = value - return timing +def _groq_timing_details(usage: GroqCompletionUsage) -> dict[str, float]: + """Return Groq timing fields in seconds, omitting the ones the provider did not report.""" + timing = { + "queue_time": usage.queue_time, + "prompt_time": usage.prompt_time, + "completion_time": usage.completion_time, + "total_time": usage.total_time, + } + return {field: value for field, value in timing.items() if value is not None} def to_chat_completion(response: GroqChatCompletion) -> ChatCompletion: diff --git a/src/any_llm/providers/ollama/utils.py b/src/any_llm/providers/ollama/utils.py index 0714f1ec3..06f888b20 100644 --- a/src/any_llm/providers/ollama/utils.py +++ b/src/any_llm/providers/ollama/utils.py @@ -66,14 +66,15 @@ def _create_openai_embedding_response_from_ollama( ) -def _ollama_timing_details(response: OllamaChatResponse) -> dict[str, int | float]: - """Return numeric Ollama timing fields without adding absent fields to usage extras.""" - timing: dict[str, int | float] = {} - for field in ("total_duration", "load_duration", "prompt_eval_duration", "eval_duration"): - value = getattr(response, field, None) - if isinstance(value, (int, float)): - timing[field] = value - return timing +def _ollama_timing_details(response: OllamaChatResponse) -> dict[str, int]: + """Return Ollama timing fields in nanoseconds, omitting the ones the provider did not report.""" + timing = { + "total_duration": response.total_duration, + "load_duration": response.load_duration, + "prompt_eval_duration": response.prompt_eval_duration, + "eval_duration": response.eval_duration, + } + return {field: value for field, value in timing.items() if value is not None} def _create_openai_chunk_from_ollama_chunk(ollama_chunk: OllamaChatResponse) -> ChatCompletionChunk: diff --git a/tests/unit/providers/test_ollama_provider.py b/tests/unit/providers/test_ollama_provider.py index ca296a96f..8333c64e8 100644 --- a/tests/unit/providers/test_ollama_provider.py +++ b/tests/unit/providers/test_ollama_provider.py @@ -130,6 +130,10 @@ async def test_create_chat_completion_extracts_think_content() -> None: mock_response.eval_count = 20 mock_response.model = "llama3.1" mock_response.done_reason = "stop" + mock_response.total_duration = None + mock_response.load_duration = None + mock_response.prompt_eval_duration = None + mock_response.eval_duration = None result = _create_chat_completion_from_ollama_response(mock_response) @@ -457,6 +461,10 @@ def _make_chunk(name: str, arguments: dict[str, str]) -> Mock: chunk.done_reason = None chunk.prompt_eval_count = None chunk.eval_count = None + chunk.total_duration = None + chunk.load_duration = None + chunk.prompt_eval_duration = None + chunk.eval_duration = None return chunk chunks = [ From d9b61289092ebe8b965e4d8c2fcf3b8dc36d9895 Mon Sep 17 00:00:00 2001 From: njbrake Date: Tue, 11 Aug 2026 14:34:28 +0000 Subject: [PATCH 3/5] tests(groq): cover streaming usage without timing; rename timing helpers Rename the timing helpers to the repo's verb-first shape (_extract__, as in _extract_anthropic_thinking_signature). The provider-prefixed verb-less form was the only one of its kind in the providers tree. Add the missing Groq branch: a streaming chunk whose usage carries no timing fields leaves usage extras empty. Co-Authored-By: Claude Opus 5 (1M context) --- src/any_llm/providers/groq/utils.py | 6 ++-- src/any_llm/providers/ollama/utils.py | 6 ++-- tests/unit/providers/test_groq_provider.py | 34 ++++++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/any_llm/providers/groq/utils.py b/src/any_llm/providers/groq/utils.py index 48431e1c5..915de7923 100644 --- a/src/any_llm/providers/groq/utils.py +++ b/src/any_llm/providers/groq/utils.py @@ -36,7 +36,7 @@ ) -def _groq_timing_details(usage: GroqCompletionUsage) -> dict[str, float]: +def _extract_groq_timing_details(usage: GroqCompletionUsage) -> dict[str, float]: """Return Groq timing fields in seconds, omitting the ones the provider did not report.""" timing = { "queue_time": usage.queue_time, @@ -56,7 +56,7 @@ def to_chat_completion(response: GroqChatCompletion) -> ChatCompletion: # Reference: https://console.groq.com/docs/prompt-caching prompt_details = response.usage.prompt_tokens_details cached_tokens = prompt_details.cached_tokens if prompt_details else None - timing_details = _groq_timing_details(response.usage) + timing_details = _extract_groq_timing_details(response.usage) usage = CompletionUsage.model_validate( { "prompt_tokens": response.usage.prompt_tokens, @@ -165,7 +165,7 @@ def _create_openai_chunk_from_groq_chunk(groq_chunk: GroqChatCompletionChunk) -> # Reference: https://console.groq.com/docs/prompt-caching prompt_details = usage_data.prompt_tokens_details cached_tokens = prompt_details.cached_tokens if prompt_details else None - timing_details = _groq_timing_details(usage_data) + timing_details = _extract_groq_timing_details(usage_data) usage = CompletionUsage.model_validate( { "prompt_tokens": usage_data.prompt_tokens, diff --git a/src/any_llm/providers/ollama/utils.py b/src/any_llm/providers/ollama/utils.py index 06f888b20..226c89796 100644 --- a/src/any_llm/providers/ollama/utils.py +++ b/src/any_llm/providers/ollama/utils.py @@ -66,7 +66,7 @@ def _create_openai_embedding_response_from_ollama( ) -def _ollama_timing_details(response: OllamaChatResponse) -> dict[str, int]: +def _extract_ollama_timing_details(response: OllamaChatResponse) -> dict[str, int]: """Return Ollama timing fields in nanoseconds, omitting the ones the provider did not report.""" timing = { "total_duration": response.total_duration, @@ -136,7 +136,7 @@ def _create_openai_chunk_from_ollama_chunk(ollama_chunk: OllamaChatResponse) -> usage = None prompt_tokens = ollama_chunk.prompt_eval_count completion_tokens = ollama_chunk.eval_count - timing_details = _ollama_timing_details(ollama_chunk) + timing_details = _extract_ollama_timing_details(ollama_chunk) if prompt_tokens or completion_tokens or timing_details: usage = CompletionUsage.model_validate( { @@ -173,7 +173,7 @@ def _create_chat_completion_from_ollama_response(response: OllamaChatResponse) - prompt_tokens = response.prompt_eval_count or 0 completion_tokens = response.eval_count or 0 - timing_details = _ollama_timing_details(response) + timing_details = _extract_ollama_timing_details(response) response_message: OllamaMessage = response.message if not response_message or not isinstance(response_message, OllamaMessage): diff --git a/tests/unit/providers/test_groq_provider.py b/tests/unit/providers/test_groq_provider.py index bde4e5e98..63428a3b1 100644 --- a/tests/unit/providers/test_groq_provider.py +++ b/tests/unit/providers/test_groq_provider.py @@ -291,6 +291,40 @@ def test_streaming_chunk_extracts_cached_tokens() -> None: } +def test_streaming_chunk_without_timing_details() -> None: + """A streaming chunk whose usage omits timing keeps usage extras empty.""" + from groq.types.chat import ChatCompletionChunk as GroqChatCompletionChunk + + from any_llm.providers.groq.utils import _create_openai_chunk_from_groq_chunk + + chunk = GroqChatCompletionChunk.model_validate( + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "llama-3.3-70b-versatile", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + }, + } + ) + + result = _create_openai_chunk_from_groq_chunk(chunk) + + assert result.usage is not None + assert result.usage.prompt_tokens == 100 + assert result.usage.model_extra == {} + + def _make_openai_response(text: str): # type: ignore[no-untyped-def] from openai.types.responses import ResponseOutputMessage, ResponseOutputText From 4df7f5531352f1f5537bd0dfad1cb09bf42a2815 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 12 Aug 2026 08:15:53 +0800 Subject: [PATCH 4/5] fix(groq): handle usage-only streaming chunks --- src/any_llm/providers/groq/utils.py | 68 ++++++++++++---------- tests/unit/providers/test_groq_provider.py | 45 ++++++++++++++ 2 files changed, 81 insertions(+), 32 deletions(-) diff --git a/src/any_llm/providers/groq/utils.py b/src/any_llm/providers/groq/utils.py index 915de7923..435b68dde 100644 --- a/src/any_llm/providers/groq/utils.py +++ b/src/any_llm/providers/groq/utils.py @@ -124,42 +124,46 @@ def to_chat_completion(response: GroqChatCompletion) -> ChatCompletion: def _create_openai_chunk_from_groq_chunk(groq_chunk: GroqChatCompletionChunk) -> ChatCompletionChunk: """Convert a Groq streaming chunk to OpenAI ChatCompletionChunk format.""" - choice_data = groq_chunk.choices[0] - delta_data = choice_data.delta - - delta = ChoiceDelta( - content=delta_data.content, - reasoning=Reasoning(content=delta_data.reasoning) if delta_data.reasoning else None, - role=cast("Literal['developer', 'system', 'user', 'assistant', 'tool'] | None", delta_data.role), - ) + choices: list[ChunkChoice] = [] + if groq_chunk.choices: + choice_data = groq_chunk.choices[0] + delta_data = choice_data.delta + + delta = ChoiceDelta( + content=delta_data.content, + reasoning=Reasoning(content=delta_data.reasoning) if delta_data.reasoning else None, + role=cast("Literal['developer', 'system', 'user', 'assistant', 'tool'] | None", delta_data.role), + ) - if delta_data.tool_calls: - openai_tool_calls = [] - for tool_call in delta_data.tool_calls: - openai_tool_call = ChoiceDeltaToolCall( - index=tool_call.index if tool_call.index is not None else 0, - id=tool_call.id, - type="function", - function=ChoiceDeltaToolCallFunction( - name=tool_call.function.name if tool_call.function else None, - arguments=tool_call.function.arguments if tool_call.function else None, + if delta_data.tool_calls: + openai_tool_calls = [] + for tool_call in delta_data.tool_calls: + openai_tool_call = ChoiceDeltaToolCall( + index=tool_call.index if tool_call.index is not None else 0, + id=tool_call.id, + type="function", + function=ChoiceDeltaToolCallFunction( + name=tool_call.function.name if tool_call.function else None, + arguments=tool_call.function.arguments if tool_call.function else None, + ) + if tool_call.function + else None, ) - if tool_call.function - else None, + openai_tool_calls.append(openai_tool_call) + delta.tool_calls = openai_tool_calls + else: + delta.tool_calls = None + + choices.append( + ChunkChoice( + index=choice_data.index, + delta=delta, + finish_reason=choice_data.finish_reason, ) - openai_tool_calls.append(openai_tool_call) - delta.tool_calls = openai_tool_calls - else: - delta.tool_calls = None - - choice = ChunkChoice( - index=choice_data.index, - delta=delta, - finish_reason=choice_data.finish_reason, - ) + ) usage = None - usage_data = groq_chunk.usage + usage_data = groq_chunk.usage or (groq_chunk.x_groq.usage if groq_chunk.x_groq else None) if usage_data: # Groq's prompt_tokens already includes cached tokens (cached_tokens is a subset). # Reference: https://console.groq.com/docs/prompt-caching @@ -178,7 +182,7 @@ def _create_openai_chunk_from_groq_chunk(groq_chunk: GroqChatCompletionChunk) -> return ChatCompletionChunk( id=groq_chunk.id, - choices=[choice], + choices=choices, created=groq_chunk.created, model=groq_chunk.model, object="chat.completion.chunk", diff --git a/tests/unit/providers/test_groq_provider.py b/tests/unit/providers/test_groq_provider.py index 63428a3b1..5b3f6132b 100644 --- a/tests/unit/providers/test_groq_provider.py +++ b/tests/unit/providers/test_groq_provider.py @@ -325,6 +325,51 @@ def test_streaming_chunk_without_timing_details() -> None: assert result.usage.model_extra == {} +def test_streaming_chunk_extracts_x_groq_usage_without_choices() -> None: + """Extract usage from Groq's final usage-only chunk.""" + from groq.types.chat import ChatCompletionChunk as GroqChatCompletionChunk + + from any_llm.providers.groq.utils import _create_openai_chunk_from_groq_chunk + + chunk = GroqChatCompletionChunk.model_validate( + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "llama-3.3-70b-versatile", + "choices": [], + "x_groq": { + "usage": { + "prompt_tokens": 4641, + "completion_tokens": 1817, + "total_tokens": 6458, + "prompt_tokens_details": {"cached_tokens": 4608}, + "queue_time": 0, + "prompt_time": 0.02, + "completion_time": 0.03, + "total_time": 0.04, + } + }, + } + ) + + result = _create_openai_chunk_from_groq_chunk(chunk) + + assert result.choices == [] + assert result.usage is not None + assert result.usage.prompt_tokens == 4641 + assert result.usage.completion_tokens == 1817 + assert result.usage.total_tokens == 6458 + assert result.usage.prompt_tokens_details is not None + assert result.usage.prompt_tokens_details.cached_tokens == 4608 + assert result.usage.model_extra == { + "queue_time": 0, + "prompt_time": 0.02, + "completion_time": 0.03, + "total_time": 0.04, + } + + def _make_openai_response(text: str): # type: ignore[no-untyped-def] from openai.types.responses import ResponseOutputMessage, ResponseOutputText From dcb0aa275c29b17817c115986ccbfd223f091ede Mon Sep 17 00:00:00 2001 From: njbrake Date: Wed, 12 Aug 2026 18:40:06 +0000 Subject: [PATCH 5/5] tests(groq,ollama): cover the real Groq streaming shape; add timing integration tests Both new Groq streaming unit tests used shapes Groq does not send on this path: top-level chunk.usage requires stream_options (absent from groq 1.6.0's create()), and an empty choices list only occurs with include_usage. The real final chunk carries a finish_reason choice alongside x_groq.usage, so a regression that keyed the fallback on empty choices would have stayed green. Add integration coverage for the four timing fields on both providers, streaming and non-streaming. The Groq streaming case is the only check that proves the live wire carries x_groq.usage; it needs real credentials, so it skips without them. Co-Authored-By: Claude Opus 5 (1M context) --- tests/integration/test_provider_timing.py | 106 +++++++++++++++++++++ tests/unit/providers/test_groq_provider.py | 47 +++++++++ 2 files changed, 153 insertions(+) create mode 100644 tests/integration/test_provider_timing.py diff --git a/tests/integration/test_provider_timing.py b/tests/integration/test_provider_timing.py new file mode 100644 index 000000000..2f91499f5 --- /dev/null +++ b/tests/integration/test_provider_timing.py @@ -0,0 +1,106 @@ +"""Integration tests for issue #1258: provider-reported timing must survive normalization. + +Groq reports timing in seconds on its usage object, Ollama in nanoseconds on its chat +response. Both land in ``usage.model_extra``. The unit tests hand-build the provider SDK +objects, so only these tests prove the fields are actually on the wire, which matters most +for Groq streaming: groq 1.6.0 takes no ``stream_options``, so streaming usage arrives under +``x_groq.usage`` rather than top-level ``chunk.usage``. + +Requires GROQ_API_KEY for the Groq tests and a reachable Ollama host for the Ollama ones. +""" + +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from any_llm import AnyLLM, LLMProvider +from any_llm.exceptions import MissingApiKeyError +from any_llm.types.completion import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage + +_GROQ_TIMING_FIELDS = ("queue_time", "prompt_time", "completion_time", "total_time") +_OLLAMA_TIMING_FIELDS = ("total_duration", "load_duration", "prompt_eval_duration", "eval_duration") + +_PROMPT: list[dict[str, Any] | ChatCompletionMessage] = [{"role": "user", "content": "Reply with the single word OK."}] + + +@pytest.mark.asyncio +async def test_groq_timing_non_streaming(provider_model_map: dict[LLMProvider, str]) -> None: + """Groq's per-request timing survives on a non-streaming completion.""" + try: + llm = AnyLLM.create(LLMProvider.GROQ) + except MissingApiKeyError: + pytest.skip("Groq API key not provided, skipping") + + result = await llm.acompletion(model=provider_model_map[LLMProvider.GROQ], messages=_PROMPT) + + assert isinstance(result, ChatCompletion) + assert result.usage is not None + extras = result.usage.model_extra or {} + assert set(_GROQ_TIMING_FIELDS) <= extras.keys(), f"missing Groq timing fields in {extras}" + assert extras["total_time"] > 0 + + +@pytest.mark.asyncio +async def test_groq_timing_streaming(provider_model_map: dict[LLMProvider, str]) -> None: + """Groq's streaming usage and timing arrive on the final chunk via x_groq.""" + try: + llm = AnyLLM.create(LLMProvider.GROQ) + except MissingApiKeyError: + pytest.skip("Groq API key not provided, skipping") + + stream = await llm.acompletion(model=provider_model_map[LLMProvider.GROQ], messages=_PROMPT, stream=True) + assert isinstance(stream, AsyncIterator) + + usages = [] + async for chunk in stream: + assert isinstance(chunk, ChatCompletionChunk) + if chunk.usage is not None: + usages.append(chunk.usage) + + assert usages, "no chunk reported usage: streaming usage is not reaching the converter" + extras = usages[-1].model_extra or {} + assert set(_GROQ_TIMING_FIELDS) <= extras.keys(), f"missing Groq timing fields in {extras}" + assert usages[-1].completion_tokens > 0 + + +@pytest.mark.asyncio +async def test_ollama_timing_non_streaming(provider_model_map: dict[LLMProvider, str]) -> None: + """Ollama's duration fields survive on a non-streaming completion.""" + llm = AnyLLM.create(LLMProvider.OLLAMA) + + try: + result = await llm.acompletion(model=provider_model_map[LLMProvider.OLLAMA], messages=_PROMPT) + # An unreachable host surfaces as a builtin ConnectionError from the ollama SDK on the + # non-streaming call and as a raw httpx.ConnectError on the streaming one. + except (ConnectionError, httpx.ConnectError, httpx.HTTPStatusError): + pytest.skip("Local Ollama host is not set up, skipping") + + assert isinstance(result, ChatCompletion) + assert result.usage is not None + extras = result.usage.model_extra or {} + assert set(_OLLAMA_TIMING_FIELDS) <= extras.keys(), f"missing Ollama timing fields in {extras}" + assert extras["total_duration"] > 0 + + +@pytest.mark.asyncio +async def test_ollama_timing_streaming(provider_model_map: dict[LLMProvider, str]) -> None: + """Ollama reports its duration fields on the final streaming chunk.""" + llm = AnyLLM.create(LLMProvider.OLLAMA) + + try: + stream = await llm.acompletion(model=provider_model_map[LLMProvider.OLLAMA], messages=_PROMPT, stream=True) + assert isinstance(stream, AsyncIterator) + + usages = [] + async for chunk in stream: + assert isinstance(chunk, ChatCompletionChunk) + if chunk.usage is not None: + usages.append(chunk.usage) + except (ConnectionError, httpx.ConnectError, httpx.HTTPStatusError): + pytest.skip("Local Ollama host is not set up, skipping") + + assert usages, "no chunk reported usage" + extras = usages[-1].model_extra or {} + assert set(_OLLAMA_TIMING_FIELDS) <= extras.keys(), f"missing Ollama timing fields in {extras}" diff --git a/tests/unit/providers/test_groq_provider.py b/tests/unit/providers/test_groq_provider.py index 5b3f6132b..7dc2b5a50 100644 --- a/tests/unit/providers/test_groq_provider.py +++ b/tests/unit/providers/test_groq_provider.py @@ -370,6 +370,53 @@ def test_streaming_chunk_extracts_x_groq_usage_without_choices() -> None: } +def test_streaming_chunk_extracts_x_groq_usage_alongside_finish_reason() -> None: + """The wire shape any-llm actually sees: usage under x_groq on the finish_reason chunk. + + groq 1.6.0's ``completions.create`` takes no ``stream_options``, so top-level + ``chunk.usage`` is never populated on this path and ``x_groq.usage`` is the only + source of streaming usage. + """ + from groq.types.chat import ChatCompletionChunk as GroqChatCompletionChunk + + from any_llm.providers.groq.utils import _create_openai_chunk_from_groq_chunk + + chunk = GroqChatCompletionChunk.model_validate( + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "llama-3.3-70b-versatile", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "x_groq": { + "id": "req_01hy", + "usage": { + "prompt_tokens": 23, + "completion_tokens": 19, + "total_tokens": 42, + "queue_time": 0.075, + "prompt_time": 0.006, + "completion_time": 0.022, + "total_time": 0.028, + }, + }, + } + ) + + result = _create_openai_chunk_from_groq_chunk(chunk) + + assert result.choices[0].finish_reason == "stop" + assert result.usage is not None + assert result.usage.prompt_tokens == 23 + assert result.usage.completion_tokens == 19 + assert result.usage.model_extra == { + "queue_time": 0.075, + "prompt_time": 0.006, + "completion_time": 0.022, + "total_time": 0.028, + } + + def _make_openai_response(text: str): # type: ignore[no-untyped-def] from openai.types.responses import ResponseOutputMessage, ResponseOutputText