diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 021d93a1f7fb..897319ca9c07 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -107,6 +107,8 @@ message ResponseOptions { bool output_token_ids = 5; bool output_logprobs = 6; optional CandidateTokens output_candidates = 7; + // Defaults to true when omitted. + optional bool skip_special_tokens = 8; } message KVCacheParameters { diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 6ac1abdcd894..4460f6b22249 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -158,7 +158,9 @@ pub fn to_text_request( } let decode_options = TextDecodeOptions { - skip_special_tokens: true, + skip_special_tokens: response + .and_then(|options| options.skip_special_tokens) + .unwrap_or(true), include_stop_str_in_output: stopping.is_some_and(|s| s.include_stop_strings), stop_strings: stopping.map(|s| &s.stop_strings).filter(|ss| !ss.is_empty()).cloned(), min_tokens: stopping.map_or(0, |s| s.min_new_tokens),