Skip to content
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -615,8 +615,12 @@ async def create(
)
finish_reason = "tool_calls"
else:
# if not tool_calls, then it is a text response and we populate the content and thought fields.
finish_reason = choice.finish_reason
content = choice.message.content or ""
# if there is a reasoning_content field, then we populate the thought field. This is for models such as R1.
if choice.message.model_extra.get('reasoning_content') is not None:
thought = choice.message.model_extra.get('reasoning_content')

logprobs: Optional[List[ChatCompletionTokenLogprob]] = None
if choice.logprobs and choice.logprobs.content:
Expand All @@ -630,8 +634,10 @@ async def create(
for x in choice.logprobs.content
]

if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1:
Comment thread
ZakWork marked this conversation as resolved.
Resolved
# This is for local R1 models.
if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1 and thought is None:
thought, content = parse_r1_content(content)


response = CreateResult(
finish_reason=normalize_stop_reason(finish_reason),
Expand Down Expand Up @@ -725,6 +731,7 @@ async def create_stream(
stop_reason = None
maybe_model = None
content_deltas: List[str] = []
thought_deltas: List[str] = []
full_tool_calls: Dict[int, FunctionCall] = {}
completion_tokens = 0
logprobs: Optional[List[ChatCompletionTokenLogprob]] = None
Expand Down Expand Up @@ -784,7 +791,11 @@ async def create_stream(
# NOTE: for OpenAI, tool_calls and content are mutually exclusive it seems, so we can skip the rest of the loop.
# However, this may not be the case for other APIs -- we should expect this may need to be updated.
continue

# if there is a reasoning_content field, then we populate the thought field. This is for models such as R1.
reasoning_content = choice.delta.model_extra.get('reasoning_content')
if reasoning_content is not None:
thought_deltas.append(reasoning_content)
yield reasoning_content
# Otherwise, get tool calls
if choice.delta.tool_calls is not None:
for tool_call_chunk in choice.delta.tool_calls:
Expand Down Expand Up @@ -837,21 +848,27 @@ async def create_stream(
# Detect whether it is a function call or just text.
content: Union[str, List[FunctionCall]]
thought: str | None = None
# Determine the content and thought based on what was collected
if full_tool_calls:
# This is a tool call.
# This is a tool call response
content = list(full_tool_calls.values())
if len(content_deltas) > 1:
# Put additional text content in the thought field.
if content_deltas:
# Store any text alongside tool calls as thoughts
thought = "".join(content_deltas)
elif len(content_deltas) > 0:
# This is a text-only content.
content = "".join(content_deltas)
else:
warnings.warn("No text content or tool calls are available. Model returned empty result.", stacklevel=2)
content = ""

# Parse R1 content if needed.
if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1:
Comment thread
ZakWork marked this conversation as resolved.
Resolved
# This is a text response (possibly with thoughts)
if content_deltas:
content = "".join(content_deltas)
else:
warnings.warn("No text content or tool calls are available. Model returned empty result.", stacklevel=2)
content = ""

# Always set thoughts if we have any, regardless of other content types
if thought_deltas:
thought = "".join(thought_deltas)

# This is for local R1 models.
if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1 and thought is None:
thought, content = parse_r1_content(content)

# Create the result.
Expand Down