Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions rust/proto/inference.proto
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ message StoppingCriteria {
bool include_stop_strings = 5;

bool ignore_eos = 6;

// Defaults to true when omitted.
optional bool skip_special_tokens = 7;
Comment thread
biswapanda marked this conversation as resolved.
Outdated
}

message ResponseOptions {
Expand Down
31 changes: 30 additions & 1 deletion rust/src/server/src/grpc/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,9 @@ pub fn to_text_request(
}

let decode_options = TextDecodeOptions {
skip_special_tokens: true,
skip_special_tokens: stopping
.and_then(|criteria| criteria.skip_special_tokens)
.unwrap_or(true),
include_stop_str_in_output: stopping.is_some_and(|s| s.include_stop_strings),
stop_strings: stopping.map(|s| &s.stop_strings).filter(|ss| !ss.is_empty()).cloned(),
min_tokens: stopping.map_or(0, |s| s.min_new_tokens),
Expand Down Expand Up @@ -662,6 +664,33 @@ mod tests {
assert!(matches!(text.prompt, Prompt::Text(s) if s == "hi"));
}

#[test]
fn absent_skip_special_tokens_defaults_to_true() {
let req = pb::GenerateRequest {
stopping: Some(pb::StoppingCriteria::default()),
..base_request()
};

let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");

assert!(text.decode_options.skip_special_tokens);
}

#[test]
fn explicit_false_skip_special_tokens_is_preserved() {
let req = pb::GenerateRequest {
stopping: Some(pb::StoppingCriteria {
skip_special_tokens: Some(false),
..Default::default()
}),
..base_request()
};

let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");

assert!(!text.decode_options.skip_special_tokens);
}

Comment thread
biswapanda marked this conversation as resolved.
Outdated
fn finished(reason: FinishReason) -> Finished {
Finished {
usage: vllm_llm::TokenUsage {
Expand Down
Loading