Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 5 additions & 9 deletions bindings/golang/src/proto_parse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,10 @@ fn parse_chunk(json: &Value) -> proto::GenerateStreamChunk {
.and_then(|v| v.as_u64())
.map(|n| n as u32)
.unwrap_or(0),
output_logprobs: None,
hidden_states: vec![],
input_logprobs: None,
index: 0,
// The Go bindings only surface token data over FFI; everything else
// defaults. Using `..Default::default()` keeps new proto fields from
// breaking this builder until they are explicitly wired.
..Default::default()
}
}

Expand Down Expand Up @@ -108,10 +108,6 @@ fn parse_complete(json: &Value) -> proto::GenerateComplete {
.and_then(|v| v.as_u64())
.map(|n| n as u32)
.unwrap_or(0),
output_logprobs: None,
all_hidden_states: vec![],
input_logprobs: None,
matched_stop: None,
index: 0,
..Default::default()
}
}
25 changes: 6 additions & 19 deletions bindings/golang/src/utils.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
//! Utility functions for FFI

use llm_tokenizer::{
chat_template::{ThinkingKeyName, ThinkingToggle},
traits::Tokenizer,
};
use llm_tokenizer::traits::Tokenizer;
use openai_protocol::chat::ChatCompletionRequest;
use smg::routers::grpc::utils::{extract_thinking_from_kwargs, should_mark_reasoning_started};
use uuid::Uuid;

/// Helper function to generate tool call ID (matches router implementation)
Expand Down Expand Up @@ -33,19 +31,8 @@ pub(crate) fn chat_requires_reasoning(
request: &ChatCompletionRequest,
tokenizer: &dyn Tokenizer,
) -> bool {
let user_thinking = request
.chat_template_kwargs
.as_ref()
.and_then(|kwargs| match tokenizer.thinking_key_name() {
Some(ThinkingKeyName::EnableThinking) => kwargs.get("enable_thinking"),
Some(ThinkingKeyName::Thinking) => kwargs.get("thinking"),
None => None,
})
.and_then(|value| value.as_bool());

match tokenizer.thinking_toggle() {
ThinkingToggle::None => false,
ThinkingToggle::DefaultOn => user_thinking != Some(false),
ThinkingToggle::DefaultOff => user_thinking == Some(true),
}
should_mark_reasoning_started(
extract_thinking_from_kwargs(request.chat_template_kwargs.as_ref(), tokenizer),
tokenizer,
)
}
6 changes: 4 additions & 2 deletions model_gateway/src/routers/grpc/utils/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ pub(crate) use logprobs::{
pub(crate) use metrics::{error_type_from_status, route_to_endpoint};
pub(crate) use parsers::{
check_reasoning_parser_availability, check_tool_parser_availability, create_reasoning_parser,
create_tool_parser, extract_thinking_from_kwargs, get_tool_parser,
should_mark_reasoning_started,
create_tool_parser, get_tool_parser,
};
// `pub` (not `pub(crate)`) so the Go bindings can reuse the gateway's reasoning
// detection instead of duplicating it.
pub use parsers::{extract_thinking_from_kwargs, should_mark_reasoning_started};
4 changes: 2 additions & 2 deletions model_gateway/src/routers/grpc/utils/parsers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use tracing::warn;
///
/// `user_thinking`: `Some(true)` = user enabled thinking, `Some(false)` = user
/// disabled it, `None` = not specified (use template default).
pub(crate) fn should_mark_reasoning_started(
pub fn should_mark_reasoning_started(
user_thinking: Option<bool>,
tokenizer: &dyn Tokenizer,
) -> bool {
Expand All @@ -32,7 +32,7 @@ pub(crate) fn should_mark_reasoning_started(
/// Only checks the key that the template actually uses (e.g. `enable_thinking`
/// for Qwen3, `thinking` for Kimi-K2.5). This prevents mismatches where the
/// user passes the wrong key name and the template ignores it.
pub(crate) fn extract_thinking_from_kwargs(
pub fn extract_thinking_from_kwargs(
kwargs: Option<&std::collections::HashMap<String, Value>>,
tokenizer: &dyn Tokenizer,
) -> Option<bool> {
Expand Down
Loading