diff --git a/crates/grpc_client/src/trtllm_service.rs b/crates/grpc_client/src/trtllm_service.rs index 2f49bec70a..d7c0b8604c 100644 --- a/crates/grpc_client/src/trtllm_service.rs +++ b/crates/grpc_client/src/trtllm_service.rs @@ -265,6 +265,10 @@ impl TrtllmServiceClient { clippy::unused_self, reason = "method receiver kept for consistent public API across gRPC backends" )] + #[expect( + clippy::too_many_arguments, + reason = "mirrors vLLM backend API; refactoring to a params struct is tracked separately" + )] pub fn build_generate_request_from_chat( &self, request_id: String, @@ -273,6 +277,7 @@ impl TrtllmServiceClient { token_ids: Vec, multimodal_input: Option, tool_call_constraint: Option<(String, String)>, // (constraint_type, constraint_value) + eos_token_ids: &[u32], ) -> Result { // Build sampling config let sampling_config = Self::build_sampling_config_from_chat(body); @@ -287,6 +292,12 @@ impl TrtllmServiceClient { let max_tokens = body.max_completion_tokens.unwrap_or(2048); + let stop_token_ids: Vec = if body.ignore_eos { + vec![] + } else { + eos_token_ids.to_vec() + }; + let grpc_request = proto::GenerateRequest { request_id, tokenized: Some(proto::TokenizedInput { @@ -299,7 +310,7 @@ impl TrtllmServiceClient { max_tokens, streaming: body.stream, stop, - stop_token_ids: vec![], + stop_token_ids, ignore_eos: body.ignore_eos, bad: vec![], bad_token_ids: vec![], @@ -414,6 +425,7 @@ impl TrtllmServiceClient { processed_text: String, token_ids: Vec, constraint: Option<(String, String)>, + eos_token_ids: &[u32], ) -> Result { let sampling_config = Self::build_sampling_config_from_responses(body); let output_config = proto::OutputConfig { @@ -442,7 +454,7 @@ impl TrtllmServiceClient { max_tokens, streaming: body.stream.unwrap_or(false), stop: vec![], - stop_token_ids: vec![], + stop_token_ids: eos_token_ids.to_vec(), ignore_eos: false, bad: vec![], bad_token_ids: vec![], diff --git a/crates/multimodal/src/registry/mod.rs b/crates/multimodal/src/registry/mod.rs index 22ab15cc63..34a48caf58 100644 --- a/crates/multimodal/src/registry/mod.rs +++ b/crates/multimodal/src/registry/mod.rs @@ -131,6 +131,7 @@ pub(super) mod test_helpers { cls_token: None, mask_token: None, additional_special_tokens: vec![], + eos_token_ids: vec![], }); &TOKENS } diff --git a/crates/tokenizer/src/cache/mod.rs b/crates/tokenizer/src/cache/mod.rs index 0f884b104f..845418c4ef 100644 --- a/crates/tokenizer/src/cache/mod.rs +++ b/crates/tokenizer/src/cache/mod.rs @@ -284,6 +284,10 @@ impl Tokenizer for CachedTokenizer { fn think_in_prefill(&self) -> bool { self.inner.think_in_prefill() } + + fn eos_token_ids(&self) -> &[TokenIdType] { + self.inner.eos_token_ids() + } } #[cfg(test)] diff --git a/crates/tokenizer/src/eos.rs b/crates/tokenizer/src/eos.rs new file mode 100644 index 0000000000..6d6d31bed8 --- /dev/null +++ b/crates/tokenizer/src/eos.rs @@ -0,0 +1,91 @@ +use std::{collections::BTreeSet, path::Path}; + +use serde_json::Value; + +use crate::traits::TokenIdType; + +fn collect_eos_ids(cfg: &Value, ids: &mut BTreeSet) { + match cfg.get("eos_token_id") { + Some(Value::Number(n)) => { + if let Some(id) = n.as_u64() { + ids.insert(id as TokenIdType); + } + } + Some(Value::Array(arr)) => { + for v in arr { + if let Some(id) = v.as_u64() { + ids.insert(id as TokenIdType); + } + } + } + _ => {} + } +} + +/// Load and merge EOS token IDs from `config.json` and `generation_config.json`. +/// +/// Models may define different EOS tokens in each file (e.g. Kimi-K2.5 uses +/// `[EOS]` (163585) in config.json and `<|im_end|>` (163586) in +/// generation_config.json). We merge both into a deduplicated list so the +/// engine can stop generation at any of them. +/// +/// SGLang and vLLM handle this internally; this is needed for backends +/// (like TRT-LLM) where SMG must provide explicit `stop_token_ids`. +pub fn load_eos_token_ids(dir: &Path) -> Vec { + let mut ids = BTreeSet::new(); + + for filename in ["config.json", "generation_config.json"] { + if let Ok(content) = std::fs::read_to_string(dir.join(filename)) { + if let Ok(cfg) = serde_json::from_str::(&content) { + collect_eos_ids(&cfg, &mut ids); + } + } + } + + ids.into_iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collect_single_int() { + let cfg: Value = serde_json::from_str(r#"{"eos_token_id": 42}"#).unwrap(); + let mut ids = BTreeSet::new(); + collect_eos_ids(&cfg, &mut ids); + assert_eq!(ids.into_iter().collect::>(), vec![42]); + } + + #[test] + fn test_collect_array() { + let cfg: Value = serde_json::from_str(r#"{"eos_token_id": [10, 20, 30]}"#).unwrap(); + let mut ids = BTreeSet::new(); + collect_eos_ids(&cfg, &mut ids); + assert_eq!(ids.into_iter().collect::>(), vec![10, 20, 30]); + } + + #[test] + fn test_collect_missing_field() { + let cfg: Value = serde_json::from_str(r#"{"model_type": "llama"}"#).unwrap(); + let mut ids = BTreeSet::new(); + collect_eos_ids(&cfg, &mut ids); + assert!(ids.is_empty()); + } + + #[test] + fn test_merge_deduplicates() { + let mut ids = BTreeSet::new(); + let cfg1: Value = serde_json::from_str(r#"{"eos_token_id": 100}"#).unwrap(); + let cfg2: Value = serde_json::from_str(r#"{"eos_token_id": [100, 200]}"#).unwrap(); + collect_eos_ids(&cfg1, &mut ids); + collect_eos_ids(&cfg2, &mut ids); + assert_eq!(ids.into_iter().collect::>(), vec![100, 200]); + } + + #[test] + fn test_load_from_nonexistent_dir() { + let ids = load_eos_token_ids(Path::new("/nonexistent/path")); + assert!(ids.is_empty()); + } +} diff --git a/crates/tokenizer/src/hub.rs b/crates/tokenizer/src/hub.rs index 0879c0545e..d4dc8bf2b6 100644 --- a/crates/tokenizer/src/hub.rs +++ b/crates/tokenizer/src/hub.rs @@ -54,7 +54,9 @@ fn is_tokenizer_file(filename: &str) -> bool { || filename.ends_with("merges.txt") || filename.ends_with(".model") // SentencePiece models || filename.ends_with(".tiktoken") - || is_chat_template_file(filename) // Include chat template files + || filename == "config.json" + || filename == "generation_config.json" + || is_chat_template_file(filename) } /// Checks if a file is a chat template file diff --git a/crates/tokenizer/src/huggingface.rs b/crates/tokenizer/src/huggingface.rs index 87cacb8611..aa7f936fe6 100644 --- a/crates/tokenizer/src/huggingface.rs +++ b/crates/tokenizer/src/huggingface.rs @@ -82,6 +82,12 @@ impl HuggingFaceTokenizer { } } + // Load merged EOS token IDs from config.json + generation_config.json + let mut special_tokens = special_tokens; + if let Some(dir) = std::path::Path::new(file_path).parent() { + special_tokens.eos_token_ids = crate::eos::load_eos_token_ids(dir); + } + Ok(HuggingFaceTokenizer { tokenizer, special_tokens, @@ -208,6 +214,7 @@ impl HuggingFaceTokenizer { cls_token: find_token(&["[CLS]", "", ""]), mask_token: find_token(&["[MASK]", "", ""]), additional_special_tokens, + ..Default::default() } } @@ -384,6 +391,10 @@ impl TokenizerTrait for HuggingFaceTokenizer { self.chat_template.think_in_prefill() } + fn eos_token_ids(&self) -> &[TokenIdType] { + &self.special_tokens.eos_token_ids + } + fn set_chat_template(&mut self, template: String) -> Result<()> { self.chat_template.set(template) } diff --git a/crates/tokenizer/src/lib.rs b/crates/tokenizer/src/lib.rs index b15f831c77..471a246c8c 100644 --- a/crates/tokenizer/src/lib.rs +++ b/crates/tokenizer/src/lib.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use anyhow::Result; pub mod cache; +pub mod eos; pub mod factory; pub mod hub; pub mod mock; diff --git a/crates/tokenizer/src/mock.rs b/crates/tokenizer/src/mock.rs index f057aabe24..ad2a6b6363 100644 --- a/crates/tokenizer/src/mock.rs +++ b/crates/tokenizer/src/mock.rs @@ -51,11 +51,7 @@ impl MockTokenizer { bos_token: Some("".to_string()), eos_token: Some("".to_string()), unk_token: Some("".to_string()), - sep_token: None, - pad_token: None, - cls_token: None, - mask_token: None, - additional_special_tokens: vec![], + ..Default::default() }; Self { diff --git a/crates/tokenizer/src/tiktoken.rs b/crates/tokenizer/src/tiktoken.rs index 1d9cbe5266..7b709e5bcf 100644 --- a/crates/tokenizer/src/tiktoken.rs +++ b/crates/tokenizer/src/tiktoken.rs @@ -119,6 +119,7 @@ fn parse_special_tokens(config: &serde_json::Value) -> SpecialTokens { cls_token: get_str("cls_token"), mask_token: get_str("mask_token"), additional_special_tokens: additional, + ..Default::default() } } @@ -259,9 +260,13 @@ impl TiktokenTokenizer { }) }; + // 6. Load merged EOS token IDs from config.json + generation_config.json + let mut special_tokens = config.special_tokens; + special_tokens.eos_token_ids = crate::eos::load_eos_token_ids(dir); + Ok(TiktokenTokenizer { tokenizer, - special_tokens: config.special_tokens, + special_tokens, vocab, reverse_vocab, vocab_size, @@ -308,27 +313,20 @@ impl TiktokenTokenizer { TiktokenModel::Cl100kBase => SpecialTokens { bos_token: Some("<|endoftext|>".to_string()), eos_token: Some("<|endoftext|>".to_string()), - unk_token: None, - sep_token: None, pad_token: Some("<|endoftext|>".to_string()), - cls_token: None, - mask_token: None, additional_special_tokens: vec![ "<|fim_prefix|>".to_string(), "<|fim_middle|>".to_string(), "<|fim_suffix|>".to_string(), "<|endofprompt|>".to_string(), ], + ..Default::default() }, _ => SpecialTokens { bos_token: Some("<|endoftext|>".to_string()), eos_token: Some("<|endoftext|>".to_string()), - unk_token: None, - sep_token: None, pad_token: Some("<|endoftext|>".to_string()), - cls_token: None, - mask_token: None, - additional_special_tokens: vec![], + ..Default::default() }, } } @@ -519,6 +517,10 @@ impl TokenizerTrait for TiktokenTokenizer { fn thinking_key_name(&self) -> Option { self.chat_template.thinking_key_name() } + fn eos_token_ids(&self) -> &[TokenIdType] { + &self.special_tokens.eos_token_ids + } + fn think_in_prefill(&self) -> bool { self.chat_template.think_in_prefill() } diff --git a/crates/tokenizer/src/traits.rs b/crates/tokenizer/src/traits.rs index 1405e70c45..12ae7ab4d1 100644 --- a/crates/tokenizer/src/traits.rs +++ b/crates/tokenizer/src/traits.rs @@ -114,6 +114,12 @@ pub trait Tokenizer: Encoder + Decoder { false } + /// Merged EOS token IDs from config.json and generation_config.json. + /// Backends should stop generation when any of these tokens is produced. + fn eos_token_ids(&self) -> &[TokenIdType] { + &[] + } + /// Set or override the chat template. /// /// Returns an error if the template fails to parse or the tokenizer @@ -176,4 +182,9 @@ pub struct SpecialTokens { pub cls_token: Option, pub mask_token: Option, pub additional_special_tokens: Vec, + /// Merged EOS token IDs from config.json + generation_config.json. + /// Models like Kimi-K2.5 define different EOS tokens in each file + /// (e.g. `[EOS]` in config.json, `<|im_end|>` in generation_config.json). + /// The engine must stop at any of these tokens. + pub eos_token_ids: Vec, } diff --git a/model_gateway/src/routers/grpc/client.rs b/model_gateway/src/routers/grpc/client.rs index 81cfc8f11f..9e65019776 100644 --- a/model_gateway/src/routers/grpc/client.rs +++ b/model_gateway/src/routers/grpc/client.rs @@ -316,6 +316,10 @@ impl GrpcClient { clippy::unreachable, reason = "assembly stage guarantees matching MultimodalData variant for each backend" )] + #[expect( + clippy::too_many_arguments, + reason = "mirrors per-backend build methods; params struct tracked separately" + )] pub fn build_chat_request( &self, request_id: String, @@ -324,6 +328,7 @@ impl GrpcClient { token_ids: Vec, multimodal_inputs: Option, tool_constraints: Option<(String, String)>, + eos_token_ids: &[u32], ) -> Result { match self { Self::Sglang(client) => { @@ -368,6 +373,7 @@ impl GrpcClient { token_ids, trtllm_mm, tool_constraints, + eos_token_ids, )?; Ok(ProtoGenerateRequest::Trtllm(Box::new(req))) } diff --git a/model_gateway/src/routers/grpc/harmony/stages/request_building.rs b/model_gateway/src/routers/grpc/harmony/stages/request_building.rs index 375adc4a64..d371988432 100644 --- a/model_gateway/src/routers/grpc/harmony/stages/request_building.rs +++ b/model_gateway/src/routers/grpc/harmony/stages/request_building.rs @@ -184,8 +184,9 @@ impl PipelineStage for HarmonyRequestBuildingStage { body, placeholder_processed_text, prep.token_ids.clone(), - None, // No multimodal in Harmony pipeline + None, prep.tool_constraints.clone(), + &[], ) .map_err(|e| { error!(function = "HarmonyRequestBuildingStage::execute", error = %e, "Failed to build TensorRT-LLM generate request"); @@ -199,6 +200,7 @@ impl PipelineStage for HarmonyRequestBuildingStage { placeholder_processed_text, prep.token_ids.clone(), prep.tool_constraints.clone(), + &[], ) .map_err(|e| { error!(function = "HarmonyRequestBuildingStage::execute", error = %e, "Failed to build TensorRT-LLM generate request from responses"); diff --git a/model_gateway/src/routers/grpc/regular/stages/chat/request_building.rs b/model_gateway/src/routers/grpc/regular/stages/chat/request_building.rs index a0c96f0335..a57508d771 100644 --- a/model_gateway/src/routers/grpc/regular/stages/chat/request_building.rs +++ b/model_gateway/src/routers/grpc/regular/stages/chat/request_building.rs @@ -88,6 +88,11 @@ impl PipelineStage for ChatRequestBuildingStage { .multimodal_intermediate .map(|intermediate| assemble_multimodal_data(intermediate, builder_client)); + let eos_token_ids = ctx + .tokenizer_arc() + .map(|t| t.eos_token_ids().to_vec()) + .unwrap_or_default(); + let mut proto_request = builder_client .build_chat_request( request_id, @@ -96,6 +101,7 @@ impl PipelineStage for ChatRequestBuildingStage { prep.token_ids, multimodal_data, prep.tool_constraints, + &eos_token_ids, ) .map_err(|e| { error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build generate request");