Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions rust/src/server/src/routes/openai/chat_completions/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ mod tests {
use crate::routes::openai::utils::structured_outputs::{JsonSchemaFormat, ResponseFormat};
use crate::routes::openai::utils::types::{
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
MessageContent, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
};
use crate::utils::{ResolvedRequestContext, resolve_request_context};

Expand Down Expand Up @@ -968,10 +968,10 @@ mod tests {
fn prepare_chat_request_maps_stream_usage_and_token_format_options() {
let mut request = base_request();
request.return_tokens_as_token_ids = Some(true);
request.stream_options = Some(StreamOptions {
include_usage: Some(true),
continuous_usage_stats: Some(true),
});
request.stream_options = Some(
serde_json::from_value(json!({"include_usage": true, "continuous_usage_stats": true}))
.expect("parse stream_options"),
);

let prepared = prepare_chat_request(
request,
Expand All @@ -988,10 +988,10 @@ mod tests {
#[test]
fn prepare_chat_request_gates_continuous_usage_on_include_usage() {
let mut request = base_request();
request.stream_options = Some(StreamOptions {
include_usage: None,
continuous_usage_stats: Some(true),
});
request.stream_options = Some(
serde_json::from_value(json!({"continuous_usage_stats": true}))
.expect("parse stream_options"),
);

let prepared = prepare_chat_request(
request,
Expand Down
23 changes: 23 additions & 0 deletions rust/src/server/src/routes/openai/chat_completions/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,10 @@ mod tests {
let cases = [
json!({"include_usage": false}),
json!({"continuous_usage_stats": true}),
// Python checks the raw object, so a key set to null or a key
// Rust does not model still counts as set.
json!({"include_usage": null}),
json!({"include_obfuscation": true}),
];
for stream_options in cases {
let request = non_streaming_request(stream_options.clone());
Expand All @@ -216,6 +220,25 @@ mod tests {
}
}

#[test]
fn validate_request_compat_accepts_stream_options_with_streaming() {
for stream_options in [
json!({"include_usage": null}),
json!({"include_obfuscation": true}),
] {
let request: ChatCompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"messages": [{"role": "user", "content": "hello"}],
"stream": true,
"stream_options": stream_options,
}))
.expect("parse request");
request.validate().expect("schema validation accepts streaming options");
validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("streaming accepts stream_options");
}
}

#[test]
fn validate_request_compat_accepts_stop() {
let request = ChatCompletionRequest {
Expand Down
22 changes: 22 additions & 0 deletions rust/src/server/src/routes/openai/completions/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ mod tests {
let cases = [
json!({"include_usage": false}),
json!({"continuous_usage_stats": true}),
// Python checks the raw object, so a key set to null or a key
// Rust does not model still counts as set.
json!({"include_usage": null}),
json!({"include_obfuscation": true}),
];
for stream_options in cases {
let request = non_streaming_request(stream_options.clone());
Expand All @@ -174,6 +178,24 @@ mod tests {
}
}

#[test]
fn validate_request_compat_accepts_stream_options_with_streaming() {
for stream_options in [
json!({"include_usage": null}),
json!({"include_obfuscation": true}),
] {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": true,
"stream_options": stream_options,
}))
.expect("parse request");
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"]))
.expect("streaming accepts stream_options");
}
}

#[test]
fn validate_request_compat_accepts_logprobs() {
let request = CompletionRequest {
Expand Down
38 changes: 32 additions & 6 deletions rust/src/server/src/routes/openai/utils/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -255,23 +255,49 @@ impl StreamResponseEnvelope {
}

/// Mirrors the Python vLLM `StreamOptions` class.
#[derive(Debug, Clone, Deserialize, Serialize)]
#[derive(Debug, Clone, Serialize)]
pub struct StreamOptions {
pub include_usage: Option<bool>,
pub continuous_usage_stats: Option<bool>,
/// Whether the request sent an object without keys.
#[serde(skip)]
empty: bool,
}

impl StreamOptions {
/// Whether no option is set, as in `"stream_options": {}`.
/// Whether the request sent an object without keys, as in
/// `"stream_options": {}`.
///
/// Python rejects `stream_options` without `stream=true` only when the
/// object is non-empty, so an empty object must pass that check.
/// raw object is non-empty, so an empty object must pass that check while
/// a key set to `null`, or a key Rust does not know, must not.
pub fn is_empty(&self) -> bool {
let Self {
self.empty
}
}

impl<'de> Deserialize<'de> for StreamOptions {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
struct Options {
include_usage: Option<bool>,
continuous_usage_stats: Option<bool>,
}

let object = serde_json::Map::<String, Value>::deserialize(deserializer)?;
let empty = object.is_empty();
let Options {
include_usage,
continuous_usage_stats,
} = Options::deserialize(Value::Object(object)).map_err(serde::de::Error::custom)?;
Ok(Self {
include_usage,
continuous_usage_stats,
} = self;
include_usage.is_none() && continuous_usage_stats.is_none()
empty,
})
}
}

Expand Down