Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions crates/grpc_client/src/trtllm_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,10 @@ impl TrtllmServiceClient {
clippy::unused_self,
reason = "method receiver kept for consistent public API across gRPC backends"
)]
#[expect(
clippy::too_many_arguments,
reason = "mirrors vLLM backend API; refactoring to a params struct is tracked separately"
)]
pub fn build_generate_request_from_chat(
&self,
request_id: String,
Expand All @@ -273,6 +277,7 @@ impl TrtllmServiceClient {
token_ids: Vec<u32>,
multimodal_input: Option<proto::MultimodalInput>,
tool_call_constraint: Option<(String, String)>, // (constraint_type, constraint_value)
eos_token_ids: &[u32],

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we only change trt?
and is this specific to kimi-k2?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

since SGLang and vLLM merge EOS tokens internally — they read both config files in their Python startup code. so only TRTLLM.

not Kimi-specific, the fix is generic, just Kimi-K2.5 is the first model we see this issue.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fix this in trt instead. It is not in our scope to pass this to their backend, just because this is a bug in their end.

) -> Result<proto::GenerateRequest, String> {
// Build sampling config
let sampling_config = Self::build_sampling_config_from_chat(body);
Expand All @@ -287,6 +292,12 @@ impl TrtllmServiceClient {

let max_tokens = body.max_completion_tokens.unwrap_or(2048);

let stop_token_ids: Vec<u32> = if body.ignore_eos {
vec![]
} else {
eos_token_ids.to_vec()
};

let grpc_request = proto::GenerateRequest {
request_id,
tokenized: Some(proto::TokenizedInput {
Expand All @@ -299,7 +310,7 @@ impl TrtllmServiceClient {
max_tokens,
streaming: body.stream,
stop,
stop_token_ids: vec![],
stop_token_ids,
ignore_eos: body.ignore_eos,
bad: vec![],
bad_token_ids: vec![],
Expand Down Expand Up @@ -414,6 +425,7 @@ impl TrtllmServiceClient {
processed_text: String,
token_ids: Vec<u32>,
constraint: Option<(String, String)>,
eos_token_ids: &[u32],
) -> Result<proto::GenerateRequest, String> {
let sampling_config = Self::build_sampling_config_from_responses(body);
let output_config = proto::OutputConfig {
Expand Down Expand Up @@ -442,7 +454,7 @@ impl TrtllmServiceClient {
max_tokens,
streaming: body.stream.unwrap_or(false),
stop: vec![],
stop_token_ids: vec![],
stop_token_ids: eos_token_ids.to_vec(),
ignore_eos: false,
bad: vec![],
bad_token_ids: vec![],
Expand Down
1 change: 1 addition & 0 deletions crates/multimodal/src/registry/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ pub(super) mod test_helpers {
cls_token: None,
mask_token: None,
additional_special_tokens: vec![],
eos_token_ids: vec![],
});
&TOKENS
}
Expand Down
4 changes: 4 additions & 0 deletions crates/tokenizer/src/cache/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,10 @@ impl Tokenizer for CachedTokenizer {
fn think_in_prefill(&self) -> bool {
self.inner.think_in_prefill()
}

fn eos_token_ids(&self) -> &[TokenIdType] {
self.inner.eos_token_ids()
}
}

#[cfg(test)]
Expand Down
91 changes: 91 additions & 0 deletions crates/tokenizer/src/eos.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
use std::{collections::BTreeSet, path::Path};

use serde_json::Value;

use crate::traits::TokenIdType;

fn collect_eos_ids(cfg: &Value, ids: &mut BTreeSet<TokenIdType>) {
match cfg.get("eos_token_id") {
Some(Value::Number(n)) => {
if let Some(id) = n.as_u64() {
ids.insert(id as TokenIdType);
}
}
Some(Value::Array(arr)) => {
for v in arr {
if let Some(id) = v.as_u64() {
ids.insert(id as TokenIdType);
}
}
}
_ => {}
}
}

/// Load and merge EOS token IDs from `config.json` and `generation_config.json`.
///
/// Models may define different EOS tokens in each file (e.g. Kimi-K2.5 uses
/// `[EOS]` (163585) in config.json and `<|im_end|>` (163586) in
/// generation_config.json). We merge both into a deduplicated list so the
/// engine can stop generation at any of them.
///
/// SGLang and vLLM handle this internally; this is needed for backends
/// (like TRT-LLM) where SMG must provide explicit `stop_token_ids`.
pub fn load_eos_token_ids(dir: &Path) -> Vec<TokenIdType> {
let mut ids = BTreeSet::new();

for filename in ["config.json", "generation_config.json"] {
if let Ok(content) = std::fs::read_to_string(dir.join(filename)) {
if let Ok(cfg) = serde_json::from_str::<Value>(&content) {
collect_eos_ids(&cfg, &mut ids);
}
}
}

ids.into_iter().collect()
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_collect_single_int() {
let cfg: Value = serde_json::from_str(r#"{"eos_token_id": 42}"#).unwrap();
let mut ids = BTreeSet::new();
collect_eos_ids(&cfg, &mut ids);
assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![42]);
}

#[test]
fn test_collect_array() {
let cfg: Value = serde_json::from_str(r#"{"eos_token_id": [10, 20, 30]}"#).unwrap();
let mut ids = BTreeSet::new();
collect_eos_ids(&cfg, &mut ids);
assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![10, 20, 30]);
}

#[test]
fn test_collect_missing_field() {
let cfg: Value = serde_json::from_str(r#"{"model_type": "llama"}"#).unwrap();
let mut ids = BTreeSet::new();
collect_eos_ids(&cfg, &mut ids);
assert!(ids.is_empty());
}

#[test]
fn test_merge_deduplicates() {
let mut ids = BTreeSet::new();
let cfg1: Value = serde_json::from_str(r#"{"eos_token_id": 100}"#).unwrap();
let cfg2: Value = serde_json::from_str(r#"{"eos_token_id": [100, 200]}"#).unwrap();
collect_eos_ids(&cfg1, &mut ids);
collect_eos_ids(&cfg2, &mut ids);
assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![100, 200]);
}

#[test]
fn test_load_from_nonexistent_dir() {
let ids = load_eos_token_ids(Path::new("/nonexistent/path"));
assert!(ids.is_empty());
}
}
4 changes: 3 additions & 1 deletion crates/tokenizer/src/hub.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ fn is_tokenizer_file(filename: &str) -> bool {
|| filename.ends_with("merges.txt")
|| filename.ends_with(".model") // SentencePiece models
|| filename.ends_with(".tiktoken")
|| is_chat_template_file(filename) // Include chat template files
|| filename == "config.json"
|| filename == "generation_config.json"
Comment on lines +57 to +58

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Match config files by basename when downloading tokenizer assets

download_tokenizer_from_hf filters using sib.rfilename, which is a repo-relative path, but is_tokenizer_file now only accepts exact matches for "config.json" and "generation_config.json". For models that store files under subdirectories (for example original/config.json), these EOS config files are skipped, so load_eos_token_ids sees an empty set and TRT-LLM requests go out without EOS stop_token_ids, undoing the stopping fix for that packaging layout.

Useful? React with 👍 / 👎.

Comment on lines +57 to +58

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Use basename matching for config files instead of full-string equality.

Line 57 and Line 58 currently require exact root filenames. HF sibling paths can be nested, so EOS config files may be skipped and the merge path can silently fall back to incomplete EOS IDs.

Proposed fix
 fn is_tokenizer_file(filename: &str) -> bool {
+    let file_name = Path::new(filename).file_name().and_then(|s| s.to_str());
     filename.ends_with("tokenizer.json")
         || filename.ends_with("tokenizer_config.json")
         || filename.ends_with("special_tokens_map.json")
         || filename.ends_with("vocab.json")
         || filename.ends_with("merges.txt")
         || filename.ends_with(".model")  // SentencePiece models
         || filename.ends_with(".tiktoken")
-        || filename == "config.json"
-        || filename == "generation_config.json"
+        || matches!(file_name, Some("config.json" | "generation_config.json"))
         || is_chat_template_file(filename)
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
|| filename == "config.json"
|| filename == "generation_config.json"
fn is_tokenizer_file(filename: &str) -> bool {
let file_name = Path::new(filename).file_name().and_then(|s| s.to_str());
filename.ends_with("tokenizer.json")
|| filename.ends_with("tokenizer_config.json")
|| filename.ends_with("special_tokens_map.json")
|| filename.ends_with("vocab.json")
|| filename.ends_with("merges.txt")
|| filename.ends_with(".model") // SentencePiece models
|| filename.ends_with(".tiktoken")
|| matches!(file_name, Some("config.json" | "generation_config.json"))
|| is_chat_template_file(filename)
}
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@crates/tokenizer/src/hub.rs` around lines 57 - 58, Replace the current
full-string equality checks against filename with a basename check so nested HF
sibling paths are handled; specifically, where the code currently does filename
== "config.json" || filename == "generation_config.json", use
Path::new(&filename).file_name().and_then(|s| s.to_str()).map_or(false, |b| b ==
"config.json" || b == "generation_config.json") (or equivalent) so only the
final path component is compared; update the surrounding boolean expression to
use this basename result.

Comment on lines +57 to +58

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Keep cache dir anchored to actual tokenizer assets

Including config.json/generation_config.json in is_tokenizer_file() can make download_tokenizer_from_hf() set cache_dir from a root config file before any tokenizer file is seen. For models that keep tokenizer files in a subdirectory (for example tokenizer/tokenizer.json), create_tokenizer_async_with_chat_template() then probes only the returned root directory and fails to find a tokenizer, so loading regresses from working to failing for that layout. Please ensure cache-dir selection is based on tokenizer artifacts (or add recursive tokenizer discovery) rather than config-file matches.

Useful? React with 👍 / 👎.

|| is_chat_template_file(filename)
}

/// Checks if a file is a chat template file
Expand Down
11 changes: 11 additions & 0 deletions crates/tokenizer/src/huggingface.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@ impl HuggingFaceTokenizer {
}
}

// Load merged EOS token IDs from config.json + generation_config.json
let mut special_tokens = special_tokens;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when loading hf or tiktokenizer
do we not have those special tokens already?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tokenizer_config.json only provides eos_token as a string (e.g. "[EOS]"), but not the token ID. The actual EOS token IDs come from config.json and generation_config.json. Again, SGLang and vLLM handle this merge internally in their Python code. TRT-LLM's gRPC path doesn't, so SMG needs to read and merge them explicitly.

Also found and fixed a related issue: when SMG loads the tokenizer via HF Hub (by model ID rather than local path), the hub downloader only fetched tokenizer files, not config.json/generation_config.json. Fixed in hub.rs so these are now included in the download.

if let Some(dir) = std::path::Path::new(file_path).parent() {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Nit: This calls load_eos_token_ids from crate::tiktoken — a function that's semantically not tiktoken-specific (it reads config.json / generation_config.json). Consider either moving it to a shared module (e.g. crate::utils or crate::config) or at least re-exporting from the crate root, so the cross-module dependency reads more naturally.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is not a nit
this is a critical piece that should be fixed

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, fixed. Extracted collect_eos_ids() helper and replaced the duplication with a loop over both config filenames. Also moved the function out of tiktoken.rs into a shared crates/tokenizer/src/eos.rs module.

special_tokens.eos_token_ids = crate::eos::load_eos_token_ids(dir);
}

Ok(HuggingFaceTokenizer {
tokenizer,
special_tokens,
Expand Down Expand Up @@ -208,6 +214,7 @@ impl HuggingFaceTokenizer {
cls_token: find_token(&["[CLS]", "<cls>", "<CLS>"]),
mask_token: find_token(&["[MASK]", "<mask>", "<MASK>"]),
additional_special_tokens,
..Default::default()
}
}

Expand Down Expand Up @@ -384,6 +391,10 @@ impl TokenizerTrait for HuggingFaceTokenizer {
self.chat_template.think_in_prefill()
}

fn eos_token_ids(&self) -> &[TokenIdType] {
&self.special_tokens.eos_token_ids
}

fn set_chat_template(&mut self, template: String) -> Result<()> {
self.chat_template.set(template)
}
Expand Down
1 change: 1 addition & 0 deletions crates/tokenizer/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use std::sync::Arc;
use anyhow::Result;

pub mod cache;
pub mod eos;
pub mod factory;
pub mod hub;
pub mod mock;
Expand Down
6 changes: 1 addition & 5 deletions crates/tokenizer/src/mock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,7 @@ impl MockTokenizer {
bos_token: Some("<bos>".to_string()),
eos_token: Some("<eos>".to_string()),
unk_token: Some("<unk>".to_string()),
sep_token: None,
pad_token: None,
cls_token: None,
mask_token: None,
additional_special_tokens: vec![],
..Default::default()
};

Self {
Expand Down
22 changes: 12 additions & 10 deletions crates/tokenizer/src/tiktoken.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ fn parse_special_tokens(config: &serde_json::Value) -> SpecialTokens {
cls_token: get_str("cls_token"),
mask_token: get_str("mask_token"),
additional_special_tokens: additional,
..Default::default()
}
}

Expand Down Expand Up @@ -259,9 +260,13 @@ impl TiktokenTokenizer {
})
};

// 6. Load merged EOS token IDs from config.json + generation_config.json
let mut special_tokens = config.special_tokens;
special_tokens.eos_token_ids = crate::eos::load_eos_token_ids(dir);

Ok(TiktokenTokenizer {
tokenizer,
special_tokens: config.special_tokens,
special_tokens,
vocab,
reverse_vocab,
vocab_size,
Expand Down Expand Up @@ -308,27 +313,20 @@ impl TiktokenTokenizer {
TiktokenModel::Cl100kBase => SpecialTokens {
bos_token: Some("<|endoftext|>".to_string()),
eos_token: Some("<|endoftext|>".to_string()),
unk_token: None,
sep_token: None,
pad_token: Some("<|endoftext|>".to_string()),
cls_token: None,
mask_token: None,
additional_special_tokens: vec![
"<|fim_prefix|>".to_string(),
"<|fim_middle|>".to_string(),
"<|fim_suffix|>".to_string(),
"<|endofprompt|>".to_string(),
],
..Default::default()
},
_ => SpecialTokens {
bos_token: Some("<|endoftext|>".to_string()),
eos_token: Some("<|endoftext|>".to_string()),
unk_token: None,
sep_token: None,
pad_token: Some("<|endoftext|>".to_string()),
cls_token: None,
mask_token: None,
additional_special_tokens: vec![],
..Default::default()
},
}
}
Expand Down Expand Up @@ -519,6 +517,10 @@ impl TokenizerTrait for TiktokenTokenizer {
fn thinking_key_name(&self) -> Option<ThinkingKeyName> {
self.chat_template.thinking_key_name()
}
fn eos_token_ids(&self) -> &[TokenIdType] {
&self.special_tokens.eos_token_ids
}

fn think_in_prefill(&self) -> bool {
self.chat_template.think_in_prefill()
}
Expand Down
11 changes: 11 additions & 0 deletions crates/tokenizer/src/traits.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,12 @@ pub trait Tokenizer: Encoder + Decoder {
false
}

/// Merged EOS token IDs from config.json and generation_config.json.
/// Backends should stop generation when any of these tokens is produced.
fn eos_token_ids(&self) -> &[TokenIdType] {
&[]
Comment on lines +119 to +120

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Forward eos_token_ids through wrapped tokenizers

The new default eos_token_ids() implementation returns an empty slice, but CachedTokenizer (in crates/tokenizer/src/cache/mod.rs, impl Tokenizer for CachedTokenizer) does not override this method to delegate to inner. In deployments with tokenizer cache enabled, calls like ctx.tokenizer_arc().eos_token_ids() now resolve to [], so TRT-LLM chat requests still send empty stop_token_ids and this EOS-merge fix is effectively bypassed. Please delegate eos_token_ids() in wrapper tokenizers (or source it from get_special_tokens) so cached and uncached paths behave the same.

Useful? React with 👍 / 👎.

}
Comment on lines +117 to +121

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion | 🟠 Major

Make the default eos_token_ids() derive from SpecialTokens to avoid silent divergence.

Line 119-120 currently hardcodes empty IDs. That creates two independent EOS sources and can silently break stop-token propagation when implementations set SpecialTokens.eos_token_ids but forget to override this method.

Proposed refactor
     /// Merged EOS token IDs from config.json and generation_config.json.
     /// Backends should stop generation when any of these tokens is produced.
     fn eos_token_ids(&self) -> &[TokenIdType] {
-        &[]
+        self.get_special_tokens().eos_token_ids.as_slice()
     }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
/// Merged EOS token IDs from config.json and generation_config.json.
/// Backends should stop generation when any of these tokens is produced.
fn eos_token_ids(&self) -> &[TokenIdType] {
&[]
}
/// Merged EOS token IDs from config.json and generation_config.json.
/// Backends should stop generation when any of these tokens is produced.
fn eos_token_ids(&self) -> &[TokenIdType] {
self.get_special_tokens().eos_token_ids.as_slice()
}
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@crates/tokenizer/src/traits.rs` around lines 117 - 121, The default
eos_token_ids() currently returns an empty slice causing divergence from
configured SpecialTokens; change the default implementation of fn
eos_token_ids(&self) -> &[TokenIdType] to return the EOS IDs from the type's
SpecialTokens (i.e. derive the default by reading self.special_tokens() ->
SpecialTokens and returning that SpecialTokens' eos_token_ids slice) instead of
a hardcoded &[] so implementors who only set SpecialTokens get correct
stop-token behavior.


/// Set or override the chat template.
///
/// Returns an error if the template fails to parse or the tokenizer
Expand Down Expand Up @@ -176,4 +182,9 @@ pub struct SpecialTokens {
pub cls_token: Option<String>,
pub mask_token: Option<String>,
pub additional_special_tokens: Vec<String>,
/// Merged EOS token IDs from config.json + generation_config.json.
/// Models like Kimi-K2.5 define different EOS tokens in each file
/// (e.g. `[EOS]` in config.json, `<|im_end|>` in generation_config.json).
/// The engine must stop at any of these tokens.
pub eos_token_ids: Vec<TokenIdType>,
}
6 changes: 6 additions & 0 deletions model_gateway/src/routers/grpc/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,10 @@ impl GrpcClient {
clippy::unreachable,
reason = "assembly stage guarantees matching MultimodalData variant for each backend"
)]
#[expect(
clippy::too_many_arguments,
reason = "mirrors per-backend build methods; params struct tracked separately"
)]
pub fn build_chat_request(
&self,
request_id: String,
Expand All @@ -324,6 +328,7 @@ impl GrpcClient {
token_ids: Vec<u32>,
multimodal_inputs: Option<MultimodalData>,
tool_constraints: Option<(String, String)>,
eos_token_ids: &[u32],
) -> Result<ProtoGenerateRequest, String> {
match self {
Self::Sglang(client) => {
Expand Down Expand Up @@ -368,6 +373,7 @@ impl GrpcClient {
token_ids,
trtllm_mm,
tool_constraints,
eos_token_ids,
)?;
Ok(ProtoGenerateRequest::Trtllm(Box::new(req)))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,9 @@ impl PipelineStage for HarmonyRequestBuildingStage {
body,
placeholder_processed_text,
prep.token_ids.clone(),
None, // No multimodal in Harmony pipeline
None,
prep.tool_constraints.clone(),
&[],
)
.map_err(|e| {
error!(function = "HarmonyRequestBuildingStage::execute", error = %e, "Failed to build TensorRT-LLM generate request");
Expand All @@ -199,6 +200,7 @@ impl PipelineStage for HarmonyRequestBuildingStage {
placeholder_processed_text,
prep.token_ids.clone(),
prep.tool_constraints.clone(),
&[],
)
.map_err(|e| {
error!(function = "HarmonyRequestBuildingStage::execute", error = %e, "Failed to build TensorRT-LLM generate request from responses");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ impl PipelineStage for ChatRequestBuildingStage {
.multimodal_intermediate
.map(|intermediate| assemble_multimodal_data(intermediate, builder_client));

let eos_token_ids = ctx
.tokenizer_arc()
.map(|t| t.eos_token_ids().to_vec())

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟣 Pre-existing: The TRT-LLM completion path (regular/stages/completion/request_building.rs) and the plain generate path (regular/stages/generate/request_building.rs) also build TRT-LLM requests with stop_token_ids: vec![]. If models like Kimi-K2.5 are served on those paths, they'll have the same missing-EOS-stop bug. Not introduced by this PR, but worth tracking as a follow-up.

.unwrap_or_default();

let mut proto_request = builder_client
.build_chat_request(
request_id,
Expand All @@ -96,6 +101,7 @@ impl PipelineStage for ChatRequestBuildingStage {
prep.token_ids,
multimodal_data,
prep.tool_constraints,
&eos_token_ids,
)
.map_err(|e| {
error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build generate request");
Expand Down
Loading