Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 157 additions & 0 deletions crates/ferrum-server/src/axum_server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6867,6 +6867,69 @@ mod tests {
(router, engine)
}

fn qwen36_chat_template() -> ModelChatTemplate {
ModelChatTemplate::new(
include_str!("../tests/fixtures/chat_template/Qwen__Qwen3.6-35B-A3B/template.jinja"),
"Qwen/Qwen3.6-35B-A3B",
)
}

async fn capture_qwen36_tool_history_request(
reasoning_fields: Value,
stream: bool,
) -> InferenceRequest {
let mut assistant = json!({
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"}
}]
});
assistant
.as_object_mut()
.expect("assistant message object")
.extend(
reasoning_fields
.as_object()
.expect("reasoning fields object")
.clone(),
);
let (router, engine) = router_with_capturing_llm_and_template(qwen36_chat_template());
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [
{"role": "user", "content": "Use the weather tool."},
assistant,
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"}
],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"stream": stream
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
if stream {
assert!(response_text(response).await.contains("[DONE]"));
}
engine.last_request()
}

fn router_with_capturing_lora_llm() -> (Router, Arc<CapturingLlm>) {
let engine = Arc::new(CapturingLlm::new());
let router = AxumServer::from_llm(engine.clone())
Expand Down Expand Up @@ -10001,6 +10064,99 @@ mod tests {
);
}

#[tokio::test]
async fn route_replays_reasoning_content_in_qwen36_tool_history_sync() {
let compatibility = capture_qwen36_tool_history_request(
json!({"reasoning_content": "opencode-reasoning-marker"}),
false,
)
.await;
let canonical = capture_qwen36_tool_history_request(
json!({"reasoning": "opencode-reasoning-marker"}),
false,
)
.await;

assert_eq!(compatibility.prompt, canonical.prompt);
assert!(
compatibility.prompt.contains("opencode-reasoning-marker"),
"Qwen3.6 prompt dropped assistant reasoning history: {}",
compatibility.prompt
);
let message = &compatibility.metadata["openai_messages"][1];
assert_eq!(message["reasoning"], "opencode-reasoning-marker");
assert!(message.get("reasoning_content").is_none());
}

#[tokio::test]
async fn route_replays_reasoning_content_in_qwen36_tool_history_stream() {
let request = capture_qwen36_tool_history_request(
json!({"reasoning_content": "opencode-stream-reasoning-marker"}),
true,
)
.await;
assert!(
request.prompt.contains("opencode-stream-reasoning-marker"),
"Qwen3.6 streaming prompt dropped assistant reasoning history: {}",
request.prompt
);
}

#[tokio::test]
async fn route_prefers_canonical_reasoning_in_qwen36_tool_history() {
for stream in [false, true] {
for reasoning in ["canonical-history-marker", ""] {
let request = capture_qwen36_tool_history_request(
json!({
"reasoning": reasoning,
"reasoning_content": "alias-history-marker"
}),
stream,
)
.await;
let canonical =
capture_qwen36_tool_history_request(json!({"reasoning": reasoning}), stream)
.await;
assert_eq!(request.prompt, canonical.prompt);
assert!(!request.prompt.contains("alias-history-marker"));
let message = &request.metadata["openai_messages"][1];
assert_eq!(message["reasoning"], reasoning);
assert!(message.get("reasoning_content").is_none());
}
}
}

#[tokio::test]
async fn route_does_not_force_reasoning_into_templates_that_ignore_it() {
let template = ModelChatTemplate::new(
"{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}",
"content-only-template",
);
let (router, engine) = router_with_capturing_llm_and_template(template);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [
{"role": "user", "content": "hello"},
{
"role": "assistant",
"content": "visible answer",
"reasoning_content": "hidden-reasoning-marker"
},
{"role": "user", "content": "continue"}
]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);

let request = engine.last_request();
assert!(request.prompt.contains("visible answer"));
assert!(!request.prompt.contains("hidden-reasoning-marker"));
}

#[tokio::test]
async fn route_tool_request_prefers_model_chat_template() {
let template = ModelChatTemplate::new(
Expand Down Expand Up @@ -10504,6 +10660,7 @@ mod tests {
body.contains(r#""reasoning":"\nreason"#),
"stream should emit reasoning delta after full think prefix: {body}"
);
assert!(!body.contains("\"reasoning_content\":"));
assert!(
body.contains(r#""content":"final""#),
"stream should emit visible content after think close: {body}"
Expand Down
150 changes: 150 additions & 0 deletions crates/ferrum-server/src/openai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ pub struct OpenAiJsonSchema {

/// Chat message
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(try_from = "ChatMessageWire")]
pub struct ChatMessage {
/// Message role
pub role: MessageRole,
Expand All @@ -262,6 +263,9 @@ pub struct ChatMessage {
/// vLLM-compatible parsed reasoning text. When Ferrum parses
/// `<think>...</think>`, `content` contains only the final visible
/// answer and this field contains the reasoning block text.
/// Historical input also accepts `reasoning_content`. A string in
/// `reasoning` takes precedence, including an empty string; missing or
/// null `reasoning` falls back to the alias. Output uses only `reasoning`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning: Option<String>,

Expand All @@ -282,6 +286,54 @@ pub struct ChatMessage {
pub function_call: Option<ChatFunctionCall>,
}

/// Deserialization wire shape. Both reasoning field names are parsed separately
/// so receiving both is not a duplicate-field error; conversion below leaves
/// one canonical value for the rest of Ferrum.
#[derive(Deserialize)]
struct ChatMessageWire {
role: MessageRole,
#[serde(default, deserialize_with = "deserialize_message_content")]
content: String,
#[serde(default)]
reasoning: serde_json::Value,
#[serde(default)]
reasoning_content: serde_json::Value,
#[serde(default)]
name: Option<String>,
#[serde(default)]
tool_calls: Option<Vec<ChatToolCall>>,
#[serde(default)]
tool_call_id: Option<String>,
#[serde(default)]
function_call: Option<ChatFunctionCall>,
}

impl TryFrom<ChatMessageWire> for ChatMessage {
type Error = String;

fn try_from(message: ChatMessageWire) -> Result<Self, Self::Error> {
let reasoning = match message.reasoning {
serde_json::Value::String(reasoning) => Some(reasoning),
serde_json::Value::Null => match message.reasoning_content {
serde_json::Value::String(reasoning) => Some(reasoning),
serde_json::Value::Null => None,
_ => return Err("reasoning_content must be a string or null".to_string()),
},
_ => return Err("reasoning must be a string or null".to_string()),
};

Ok(Self {
role: message.role,
content: message.content,
reasoning,
name: message.name,
tool_calls: message.tool_calls,
tool_call_id: message.tool_call_id,
function_call: message.function_call,
})
}
}

/// Assistant tool call in OpenAI responses and historical conversation input.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatToolCall {
Expand Down Expand Up @@ -749,3 +801,101 @@ fn default_audio_format() -> String {
fn default_language() -> String {
"auto".to_string()
}

#[cfg(test)]
mod tests {
use super::*;

fn chat_request_with_assistant_fields(fields: &str) -> String {
format!(r#"{{"model":"test","messages":[{{"role":"assistant","content":null{fields}}}]}}"#)
}

#[test]
fn chat_request_normalizes_reasoning_content_at_the_wire_boundary() {
let cases = [
("missing", "", None),
("compatibility null", r#", "reasoning_content": null"#, None),
(
"compatibility empty",
r#", "reasoning_content": """#,
Some(""),
),
(
"compatibility text",
r#", "reasoning_content": "compatibility""#,
Some("compatibility"),
),
(
"canonical text",
r#", "reasoning": "canonical""#,
Some("canonical"),
),
(
"compatibility then canonical",
r#", "reasoning_content": "compatibility", "reasoning": "canonical""#,
Some("canonical"),
),
(
"canonical then compatibility",
r#", "reasoning": "canonical", "reasoning_content": "compatibility""#,
Some("canonical"),
),
(
"canonical empty wins",
r#", "reasoning": "", "reasoning_content": "compatibility""#,
Some(""),
),
(
"canonical null falls back",
r#", "reasoning": null, "reasoning_content": "compatibility""#,
Some("compatibility"),
),
(
"canonical text ignores invalid compatibility",
r#", "reasoning_content": 7, "reasoning": "canonical""#,
Some("canonical"),
),
(
"canonical empty ignores invalid compatibility",
r#", "reasoning": "", "reasoning_content": {"unexpected": true}"#,
Some(""),
),
];

for (name, fields, expected) in cases {
let request: ChatCompletionsRequest =
serde_json::from_str(&chat_request_with_assistant_fields(fields))
.unwrap_or_else(|error| panic!("{name}: {error}"));
assert_eq!(request.messages[0].reasoning.as_deref(), expected, "{name}");

let normalized = serde_json::to_value(request).expect("normalized request JSON");
let message = &normalized["messages"][0];
assert!(message.get("reasoning_content").is_none(), "{name}");
match expected {
Some(expected) => assert_eq!(message["reasoning"], expected, "{name}"),
None => assert!(message.get("reasoning").is_none(), "{name}"),
}
}
}

#[test]
fn chat_request_rejects_non_string_reasoning_fields() {
for (name, fields) in [
("compatibility", r#", "reasoning_content": 7"#),
(
"canonical is not masked by compatibility",
r#", "reasoning": 7, "reasoning_content": "compatibility""#,
),
(
"canonical null validates compatibility",
r#", "reasoning": null, "reasoning_content": 7"#,
),
] {
let error = serde_json::from_str::<ChatCompletionsRequest>(
&chat_request_with_assistant_fields(fields),
)
.expect_err(name);
assert!(error.to_string().contains("string"), "{name}: {error}");
}
}
}
1 change: 1 addition & 0 deletions docs/openai-api-compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ support matrix and are hidden from the default CLI help.
| `messages` | Supported | `system`, `user`, `assistant`, `tool`, and legacy `function` roles parse into structured request data and are rendered by the chat-template layer. Assistant `tool_calls` / legacy `function_call` history is included in the rendered prompt for caller-owned tool-result loops. |
| string `content` | Supported | Rendered through the model-family chat template layer. |
| text content parts | Supported | `content: [{"type":"text","text":"..."}]` is accepted and concatenated. |
| assistant `reasoning` / `reasoning_content` | Supported extension | Historical reasoning accepts either field. A string in canonical `reasoning` wins, including an empty string; missing or null `reasoning` falls back to `reasoning_content`. The selected value must be a string or null. The model's chat template decides whether to retain it in the prompt. Chat response messages and SSE deltas emit only canonical `reasoning`. |
| multimodal content parts | Rejected | Non-text parts return HTTP 400 instead of being silently dropped. |
| `max_tokens` | Supported | Legacy completion budget. |
| `max_completion_tokens` | Supported | Overrides `max_tokens` when both are supplied. |
Expand Down
Loading