Skip to content

[None][feat] add chat template kwargs support to longbench-v2 #9544

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lfr-0531 merged 4 commits into from
Dec 1, 2025
Merged

[None][feat] add chat template kwargs support to longbench-v2 #9544

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
82ecd97
add chat template kwargs to longbench-v2.
lfr-0531 Nov 28, 2025
00056fa
update.
lfr-0531 Nov 28, 2025
a454a05
update.
lfr-0531 Nov 30, 2025
e66196e
Merge branch 'main' into user/fanrongl/add_tmp_longbench
lfr-0531 Nov 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions tensorrt_llm/evaluate/longbench_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ def __init__(self,
output_dir: Optional[str] = None,
random_seed: int = 0,
apply_chat_template: bool = False,
system_prompt: Optional[str] = None):
system_prompt: Optional[str] = None,
chat_template_kwargs: Optional[dict[str, Any]] = None):
"""Initialize LongBench v2 evaluator.

Args:
Expand All @@ -85,10 +86,12 @@ def __init__(self,
random_seed: Random seed for reproducibility
apply_chat_template: Whether to apply model's chat template
system_prompt: System prompt to prepend
chat_template_kwargs: Chat template kwargs as JSON string
"""
super().__init__(random_seed=random_seed,
apply_chat_template=apply_chat_template,
system_prompt=system_prompt)
system_prompt=system_prompt,
chat_template_kwargs=chat_template_kwargs)

self.dataset_path = dataset_path
self.num_samples = num_samples
Expand Down Expand Up @@ -813,6 +816,15 @@ def _save_results(self, results: List[Dict], metrics: Dict[str, float]):
type=int,
default=32000,
help="Maximum generation length in sampling parameters.")
@click.option(
"--chat_template_kwargs",
type=str,
default=None,
callback=lambda ctx, param, value: json.loads(value) if value else None,
help=
'A JSON string specifying chat template arguments, used to enable features like thinking mode. Examples: '
'\'{"enable_thinking": true}\' for Qwen3, or \'{"thinking": true}\' for DeepSeek-V3.2.'
)
@click.pass_context
@staticmethod
def command(ctx, dataset_path: str, prompts_dir: Optional[str],
Expand All @@ -821,7 +833,8 @@ def command(ctx, dataset_path: str, prompts_dir: Optional[str],
cot: bool, no_context: bool, rag: int, max_len: int,
output_dir: Optional[str], random_seed: int,
apply_chat_template: bool, system_prompt: Optional[str],
max_input_length: int, max_output_length: int) -> None:
max_input_length: int, max_output_length: int,
chat_template_kwargs: Optional[dict[str, Any]]) -> None:
llm: Union[LLM, PyTorchLLM] = ctx.obj

sampling_params = SamplingParams(
Expand All @@ -844,7 +857,8 @@ def command(ctx, dataset_path: str, prompts_dir: Optional[str],
output_dir=output_dir,
random_seed=random_seed,
apply_chat_template=apply_chat_template,
system_prompt=system_prompt)
system_prompt=system_prompt,
chat_template_kwargs=chat_template_kwargs)

evaluator.evaluate(llm, sampling_params)
llm.shutdown()