Repository navigation
fix(tokenizer): load merged EOS token IDs from config.json + generation_config.json #1074
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| use std::{collections::BTreeSet, path::Path}; | ||
|
|
||
| use serde_json::Value; | ||
|
|
||
| use crate::traits::TokenIdType; | ||
|
|
||
| fn collect_eos_ids(cfg: &Value, ids: &mut BTreeSet<TokenIdType>) { | ||
| match cfg.get("eos_token_id") { | ||
| Some(Value::Number(n)) => { | ||
| if let Some(id) = n.as_u64() { | ||
| ids.insert(id as TokenIdType); | ||
| } | ||
| } | ||
| Some(Value::Array(arr)) => { | ||
| for v in arr { | ||
| if let Some(id) = v.as_u64() { | ||
| ids.insert(id as TokenIdType); | ||
| } | ||
| } | ||
| } | ||
| _ => {} | ||
| } | ||
| } | ||
|
|
||
| /// Load and merge EOS token IDs from `config.json` and `generation_config.json`. | ||
| /// | ||
| /// Models may define different EOS tokens in each file (e.g. Kimi-K2.5 uses | ||
| /// `[EOS]` (163585) in config.json and `<|im_end|>` (163586) in | ||
| /// generation_config.json). We merge both into a deduplicated list so the | ||
| /// engine can stop generation at any of them. | ||
| /// | ||
| /// SGLang and vLLM handle this internally; this is needed for backends | ||
| /// (like TRT-LLM) where SMG must provide explicit `stop_token_ids`. | ||
| pub fn load_eos_token_ids(dir: &Path) -> Vec<TokenIdType> { | ||
| let mut ids = BTreeSet::new(); | ||
|
|
||
| for filename in ["config.json", "generation_config.json"] { | ||
| if let Ok(content) = std::fs::read_to_string(dir.join(filename)) { | ||
| if let Ok(cfg) = serde_json::from_str::<Value>(&content) { | ||
| collect_eos_ids(&cfg, &mut ids); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| ids.into_iter().collect() | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| use super::*; | ||
|
|
||
| #[test] | ||
| fn test_collect_single_int() { | ||
| let cfg: Value = serde_json::from_str(r#"{"eos_token_id": 42}"#).unwrap(); | ||
| let mut ids = BTreeSet::new(); | ||
| collect_eos_ids(&cfg, &mut ids); | ||
| assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![42]); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_collect_array() { | ||
| let cfg: Value = serde_json::from_str(r#"{"eos_token_id": [10, 20, 30]}"#).unwrap(); | ||
| let mut ids = BTreeSet::new(); | ||
| collect_eos_ids(&cfg, &mut ids); | ||
| assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![10, 20, 30]); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_collect_missing_field() { | ||
| let cfg: Value = serde_json::from_str(r#"{"model_type": "llama"}"#).unwrap(); | ||
| let mut ids = BTreeSet::new(); | ||
| collect_eos_ids(&cfg, &mut ids); | ||
| assert!(ids.is_empty()); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_merge_deduplicates() { | ||
| let mut ids = BTreeSet::new(); | ||
| let cfg1: Value = serde_json::from_str(r#"{"eos_token_id": 100}"#).unwrap(); | ||
| let cfg2: Value = serde_json::from_str(r#"{"eos_token_id": [100, 200]}"#).unwrap(); | ||
| collect_eos_ids(&cfg1, &mut ids); | ||
| collect_eos_ids(&cfg2, &mut ids); | ||
| assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![100, 200]); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_load_from_nonexistent_dir() { | ||
| let ids = load_eos_token_ids(Path::new("/nonexistent/path")); | ||
| assert!(ids.is_empty()); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -54,7 +54,9 @@ fn is_tokenizer_file(filename: &str) -> bool { | |||||||||||||||||||||||||||||
| || filename.ends_with("merges.txt") | ||||||||||||||||||||||||||||||
| || filename.ends_with(".model") // SentencePiece models | ||||||||||||||||||||||||||||||
| || filename.ends_with(".tiktoken") | ||||||||||||||||||||||||||||||
| || is_chat_template_file(filename) // Include chat template files | ||||||||||||||||||||||||||||||
| || filename == "config.json" | ||||||||||||||||||||||||||||||
| || filename == "generation_config.json" | ||||||||||||||||||||||||||||||
|
Comment on lines
+57
to
+58
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Useful? React with 👍 / 👎.
Comment on lines
+57
to
+58
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use basename matching for config files instead of full-string equality. Line 57 and Line 58 currently require exact root filenames. HF sibling paths can be nested, so EOS config files may be skipped and the merge path can silently fall back to incomplete EOS IDs. Proposed fix fn is_tokenizer_file(filename: &str) -> bool {
+ let file_name = Path::new(filename).file_name().and_then(|s| s.to_str());
filename.ends_with("tokenizer.json")
|| filename.ends_with("tokenizer_config.json")
|| filename.ends_with("special_tokens_map.json")
|| filename.ends_with("vocab.json")
|| filename.ends_with("merges.txt")
|| filename.ends_with(".model") // SentencePiece models
|| filename.ends_with(".tiktoken")
- || filename == "config.json"
- || filename == "generation_config.json"
+ || matches!(file_name, Some("config.json" | "generation_config.json"))
|| is_chat_template_file(filename)
}📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents
Comment on lines
+57
to
+58
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Including Useful? React with 👍 / 👎. |
||||||||||||||||||||||||||||||
| || is_chat_template_file(filename) | ||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| /// Checks if a file is a chat template file | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -82,6 +82,12 @@ impl HuggingFaceTokenizer { | |
| } | ||
| } | ||
|
|
||
| // Load merged EOS token IDs from config.json + generation_config.json | ||
| let mut special_tokens = special_tokens; | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. when loading hf or tiktokenizer
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Also found and fixed a related issue: when SMG loads the tokenizer via HF Hub (by model ID rather than local path), the hub downloader only fetched tokenizer files, not |
||
| if let Some(dir) = std::path::Path::new(file_path).parent() { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Nit: This calls
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is not a nit
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed, fixed. Extracted |
||
| special_tokens.eos_token_ids = crate::eos::load_eos_token_ids(dir); | ||
| } | ||
|
|
||
| Ok(HuggingFaceTokenizer { | ||
| tokenizer, | ||
| special_tokens, | ||
|
|
@@ -208,6 +214,7 @@ impl HuggingFaceTokenizer { | |
| cls_token: find_token(&["[CLS]", "<cls>", "<CLS>"]), | ||
| mask_token: find_token(&["[MASK]", "<mask>", "<MASK>"]), | ||
| additional_special_tokens, | ||
| ..Default::default() | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -384,6 +391,10 @@ impl TokenizerTrait for HuggingFaceTokenizer { | |
| self.chat_template.think_in_prefill() | ||
| } | ||
|
|
||
| fn eos_token_ids(&self) -> &[TokenIdType] { | ||
| &self.special_tokens.eos_token_ids | ||
| } | ||
|
|
||
| fn set_chat_template(&mut self, template: String) -> Result<()> { | ||
| self.chat_template.set(template) | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -114,6 +114,12 @@ pub trait Tokenizer: Encoder + Decoder { | |||||||||||||||||||||
| false | ||||||||||||||||||||||
| } | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| /// Merged EOS token IDs from config.json and generation_config.json. | ||||||||||||||||||||||
| /// Backends should stop generation when any of these tokens is produced. | ||||||||||||||||||||||
| fn eos_token_ids(&self) -> &[TokenIdType] { | ||||||||||||||||||||||
| &[] | ||||||||||||||||||||||
|
Comment on lines
+119
to
+120
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The new default Useful? React with 👍 / 👎. |
||||||||||||||||||||||
| } | ||||||||||||||||||||||
|
Comment on lines
+117
to
+121
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🛠️ Refactor suggestion | 🟠 Major Make the default Line 119-120 currently hardcodes empty IDs. That creates two independent EOS sources and can silently break stop-token propagation when implementations set Proposed refactor /// Merged EOS token IDs from config.json and generation_config.json.
/// Backends should stop generation when any of these tokens is produced.
fn eos_token_ids(&self) -> &[TokenIdType] {
- &[]
+ self.get_special_tokens().eos_token_ids.as_slice()
}📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents |
||||||||||||||||||||||
|
|
||||||||||||||||||||||
| /// Set or override the chat template. | ||||||||||||||||||||||
| /// | ||||||||||||||||||||||
| /// Returns an error if the template fails to parse or the tokenizer | ||||||||||||||||||||||
|
|
@@ -176,4 +182,9 @@ pub struct SpecialTokens { | |||||||||||||||||||||
| pub cls_token: Option<String>, | ||||||||||||||||||||||
| pub mask_token: Option<String>, | ||||||||||||||||||||||
| pub additional_special_tokens: Vec<String>, | ||||||||||||||||||||||
| /// Merged EOS token IDs from config.json + generation_config.json. | ||||||||||||||||||||||
| /// Models like Kimi-K2.5 define different EOS tokens in each file | ||||||||||||||||||||||
| /// (e.g. `[EOS]` in config.json, `<|im_end|>` in generation_config.json). | ||||||||||||||||||||||
| /// The engine must stop at any of these tokens. | ||||||||||||||||||||||
| pub eos_token_ids: Vec<TokenIdType>, | ||||||||||||||||||||||
| } | ||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -88,6 +88,11 @@ impl PipelineStage for ChatRequestBuildingStage { | |
| .multimodal_intermediate | ||
| .map(|intermediate| assemble_multimodal_data(intermediate, builder_client)); | ||
|
|
||
| let eos_token_ids = ctx | ||
| .tokenizer_arc() | ||
| .map(|t| t.eos_token_ids().to_vec()) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟣 Pre-existing: The TRT-LLM completion path ( |
||
| .unwrap_or_default(); | ||
|
|
||
| let mut proto_request = builder_client | ||
| .build_chat_request( | ||
| request_id, | ||
|
|
@@ -96,6 +101,7 @@ impl PipelineStage for ChatRequestBuildingStage { | |
| prep.token_ids, | ||
| multimodal_data, | ||
| prep.tool_constraints, | ||
| &eos_token_ids, | ||
| ) | ||
| .map_err(|e| { | ||
| error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build generate request"); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why do we only change trt?
and is this specific to kimi-k2?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
since SGLang and vLLM merge EOS tokens internally — they read both config files in their Python startup code. so only TRTLLM.
not Kimi-specific, the fix is generic, just Kimi-K2.5 is the first model we see this issue.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fix this in trt instead. It is not in our scope to pass this to their backend, just because this is a bug in their end.