From e74f133a97cec90f886dd03c44fd86a5afa51f29 Mon Sep 17 00:00:00 2001 From: "cliu1004@amd.com" Date: Wed, 27 May 2026 15:47:46 +0000 Subject: [PATCH] bench_serving: fallback when vLLM get_tokenizer hits removed API vLLM's tokenizer wrapper can raise AttributeError on all_special_tokens_extended with newer transformers (e.g. Qwen3.5). Use backend_request_func.get_tokenizer on fallback so client tokenization keeps the sglang v5 fix (#1381); bare AutoTokenizer only if backend is unavailable. Co-authored-by: Cursor --- utils/bench_serving/benchmark_serving.py | 44 +++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/utils/bench_serving/benchmark_serving.py b/utils/bench_serving/benchmark_serving.py index c88a7308f1..741e442368 100644 --- a/utils/bench_serving/benchmark_serving.py +++ b/utils/bench_serving/benchmark_serving.py @@ -95,10 +95,44 @@ class BenchmarkMetrics: _worker_tokenizer = None +def _load_tokenizer(tokenizer_id, tokenizer_mode, trust_remote_code): + """Load tokenizer for random-prompt generation. + + vLLM's get_tokenizer can raise AttributeError when transformers removes + LlamaTokenizer.all_special_tokens_extended (e.g. Qwen3.5 with newer + transformers). Prefer backend_request_func.get_tokenizer on fallback so + client tokenization stays aligned with the sglang server (#1381, #1428). + """ + try: + return get_tokenizer( + tokenizer_id, + tokenizer_mode=tokenizer_mode, + trust_remote_code=trust_remote_code, + ) + except AttributeError as exc: + if "all_special_tokens_extended" not in str(exc): + raise + try: + from backend_request_func import get_tokenizer as _backend_get_tokenizer + return _backend_get_tokenizer( + tokenizer_id, + tokenizer_mode=tokenizer_mode, + trust_remote_code=trust_remote_code, + ) + except ImportError: + from transformers import AutoTokenizer + use_fast = tokenizer_mode != "slow" + return AutoTokenizer.from_pretrained( + tokenizer_id, + trust_remote_code=trust_remote_code, + use_fast=use_fast, + ) + + def _init_tokenizer_worker(tokenizer_id, tokenizer_mode, trust_remote_code): """Initialize tokenizer once per worker process.""" global _worker_tokenizer - _worker_tokenizer = get_tokenizer( + _worker_tokenizer = _load_tokenizer( tokenizer_id, tokenizer_mode=tokenizer_mode, trust_remote_code=trust_remote_code, @@ -787,9 +821,11 @@ def main(args: argparse.Namespace): api_url = f"http://{args.host}:{args.port}{args.endpoint}" base_url = f"http://{args.host}:{args.port}" - tokenizer = get_tokenizer(tokenizer_id, - tokenizer_mode=tokenizer_mode, - trust_remote_code=args.trust_remote_code) + tokenizer = _load_tokenizer( + tokenizer_id, + tokenizer_mode=tokenizer_mode, + trust_remote_code=args.trust_remote_code, + ) if args.dataset_name == "random":