diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 57c6b3acdfff..286c5b5200f6 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -487,7 +487,7 @@ mod tests { use crate::routes::openai::utils::structured_outputs::{JsonSchemaFormat, ResponseFormat}; use crate::routes::openai::utils::types::{ AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio, - MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl, + MessageContent, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl, }; use crate::utils::{ResolvedRequestContext, resolve_request_context}; @@ -968,10 +968,10 @@ mod tests { fn prepare_chat_request_maps_stream_usage_and_token_format_options() { let mut request = base_request(); request.return_tokens_as_token_ids = Some(true); - request.stream_options = Some(StreamOptions { - include_usage: Some(true), - continuous_usage_stats: Some(true), - }); + request.stream_options = Some( + serde_json::from_value(json!({"include_usage": true, "continuous_usage_stats": true})) + .expect("parse stream_options"), + ); let prepared = prepare_chat_request( request, @@ -988,10 +988,10 @@ mod tests { #[test] fn prepare_chat_request_gates_continuous_usage_on_include_usage() { let mut request = base_request(); - request.stream_options = Some(StreamOptions { - include_usage: None, - continuous_usage_stats: Some(true), - }); + request.stream_options = Some( + serde_json::from_value(json!({"continuous_usage_stats": true})) + .expect("parse stream_options"), + ); let prepared = prepare_chat_request( request, diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index fe98bac3753d..6b66677ea372 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -205,6 +205,10 @@ mod tests { let cases = [ json!({"include_usage": false}), json!({"continuous_usage_stats": true}), + // Python checks the raw object, so a key set to null or a key + // Rust does not model still counts as set. + json!({"include_usage": null}), + json!({"include_obfuscation": true}), ]; for stream_options in cases { let request = non_streaming_request(stream_options.clone()); @@ -216,6 +220,25 @@ mod tests { } } + #[test] + fn validate_request_compat_accepts_stream_options_with_streaming() { + for stream_options in [ + json!({"include_usage": null}), + json!({"include_obfuscation": true}), + ] { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hello"}], + "stream": true, + "stream_options": stream_options, + })) + .expect("parse request"); + request.validate().expect("schema validation accepts streaming options"); + validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) + .expect("streaming accepts stream_options"); + } + } + #[test] fn validate_request_compat_accepts_stop() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/completions/validate.rs b/rust/src/server/src/routes/openai/completions/validate.rs index e08e35bbe866..04777b93acd9 100644 --- a/rust/src/server/src/routes/openai/completions/validate.rs +++ b/rust/src/server/src/routes/openai/completions/validate.rs @@ -163,6 +163,10 @@ mod tests { let cases = [ json!({"include_usage": false}), json!({"continuous_usage_stats": true}), + // Python checks the raw object, so a key set to null or a key + // Rust does not model still counts as set. + json!({"include_usage": null}), + json!({"include_obfuscation": true}), ]; for stream_options in cases { let request = non_streaming_request(stream_options.clone()); @@ -174,6 +178,24 @@ mod tests { } } + #[test] + fn validate_request_compat_accepts_stream_options_with_streaming() { + for stream_options in [ + json!({"include_usage": null}), + json!({"include_obfuscation": true}), + ] { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "stream": true, + "stream_options": stream_options, + })) + .expect("parse request"); + validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])) + .expect("streaming accepts stream_options"); + } + } + #[test] fn validate_request_compat_accepts_logprobs() { let request = CompletionRequest { diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index 9ba8963dc30b..30330dee4d28 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -255,23 +255,49 @@ impl StreamResponseEnvelope { } /// Mirrors the Python vLLM `StreamOptions` class. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct StreamOptions { pub include_usage: Option, pub continuous_usage_stats: Option, + /// Whether the request sent an object without keys. + #[serde(skip)] + empty: bool, } impl StreamOptions { - /// Whether no option is set, as in `"stream_options": {}`. + /// Whether the request sent an object without keys, as in + /// `"stream_options": {}`. /// /// Python rejects `stream_options` without `stream=true` only when the - /// object is non-empty, so an empty object must pass that check. + /// raw object is non-empty, so an empty object must pass that check while + /// a key set to `null`, or a key Rust does not know, must not. pub fn is_empty(&self) -> bool { - let Self { + self.empty + } +} + +impl<'de> Deserialize<'de> for StreamOptions { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Options { + include_usage: Option, + continuous_usage_stats: Option, + } + + let object = serde_json::Map::::deserialize(deserializer)?; + let empty = object.is_empty(); + let Options { + include_usage, + continuous_usage_stats, + } = Options::deserialize(Value::Object(object)).map_err(serde::de::Error::custom)?; + Ok(Self { include_usage, continuous_usage_stats, - } = self; - include_usage.is_none() && continuous_usage_stats.is_none() + empty, + }) } }