diff --git a/miles/utils/mask_utils.py b/miles/utils/mask_utils.py index 0ddb3a1410b..cd2d72eb4c1 100644 --- a/miles/utils/mask_utils.py +++ b/miles/utils/mask_utils.py @@ -38,7 +38,7 @@ def get_system_message_length(self) -> tuple[int, int]: end_interval = len(chat_template_token_ids) - len(raw_token_ids) - idx_2 gen_token_length = len( self.tokenizer.apply_chat_template( - test_messages, add_special_tokens=False, tokenize=True, add_generation_prompt=True + test_messages, add_special_tokens=False, tokenize=True, return_dict=False, add_generation_prompt=True ) ) - len(chat_template_token_ids) @@ -53,9 +53,11 @@ def gen_multi_turn_loss_mask_qwen( for i, message in enumerate(messages): if i == 0: - message_ids = self.tokenizer.apply_chat_template([message], tokenize=True, tools=tools) + message_ids = self.tokenizer.apply_chat_template( + [message], tokenize=True, return_dict=False, tools=tools + ) else: - message_ids = self.tokenizer.apply_chat_template([message], tokenize=True) + message_ids = self.tokenizer.apply_chat_template([message], tokenize=True, return_dict=False) if message["role"] != "system" and i > 0: message_ids = message_ids[self.system_message_length :] @@ -80,16 +82,18 @@ def gen_multi_turn_loss_mask_qwen3( all_token_ids = [] prefix_message = {"role": "user", "content": "FOR CALCULATING LOSS MASK ONLY"} - prefix_token_ids = self.tokenizer.apply_chat_template([prefix_message], tokenize=True) + prefix_token_ids = self.tokenizer.apply_chat_template([prefix_message], tokenize=True, return_dict=False) for i, message in enumerate(messages): if i == 0: tailed_message_ids = self.tokenizer.apply_chat_template( - [message, prefix_message], tokenize=True, tools=tools + [message, prefix_message], tokenize=True, return_dict=False, tools=tools ) message_ids = tailed_message_ids[: -len(prefix_token_ids)] else: - prefixed_message_ids = self.tokenizer.apply_chat_template([prefix_message, message], tokenize=True) + prefixed_message_ids = self.tokenizer.apply_chat_template( + [prefix_message, message], tokenize=True, return_dict=False + ) message_ids = prefixed_message_ids[len(prefix_token_ids) :] if message["role"] != "system" and i > 0: diff --git a/miles/utils/processing_utils.py b/miles/utils/processing_utils.py index 75fd2fb75e5..3291a7ffe4f 100644 --- a/miles/utils/processing_utils.py +++ b/miles/utils/processing_utils.py @@ -28,17 +28,13 @@ def load_tokenizer(name_or_path: str, chat_template_path: str = None, **kwargs): def build_processor_kwargs(multimodal_inputs: dict | None = None) -> dict: - forced = { - # force return_tensors to None for input_ids - "return_tensors": None, - } modality_forced = {"return_tensors": "pt"} result = dict(multimodal_inputs) if multimodal_inputs else {} - result.update(forced) - - # set return_tensors="pt" for modality-specific outputs + # return_tensors=None for text (input_ids), "pt" for modality-specific outputs. + # Use per-modality dicts to avoid transformers >=5.0 duplicate kwarg error. + result["text_kwargs"] = {**result.get("text_kwargs", {}), "return_tensors": None} for key in ("audio_kwargs", "images_kwargs", "videos_kwargs"): if key in result: result[key] = {**result[key], **modality_forced} diff --git a/tests/fast/rollout/generate_hub/test_tool_call_utils.py b/tests/fast/rollout/generate_hub/test_tool_call_utils.py index 0f2305e753f..b4f684dd803 100644 --- a/tests/fast/rollout/generate_hub/test_tool_call_utils.py +++ b/tests/fast/rollout/generate_hub/test_tool_call_utils.py @@ -7,24 +7,39 @@ "Qwen/Qwen3-0.6B", "Qwen/Qwen3-4B-Instruct-2507", "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "Qwen/Qwen3.5-0.8B", + "Qwen/Qwen3-Coder-Next", # "meta-llama/Llama-3.2-1B-Instruct", # Skipped: gated repo, requires HF_TOKEN in CI "mistralai/Mistral-7B-Instruct-v0.3", - "deepseek-ai/DeepSeek-V3", - "stepfun-ai/step3", "MiniMaxAI/MiniMax-M2", + "MiniMaxAI/MiniMax-M2.5", "internlm/internlm3-8b-instruct", - "THUDM/glm-4-9b-chat", + "zai-org/GLM-4.7-Flash", + "stepfun-ai/Step-3.5-Flash", "moonshotai/Kimi-K2-Instruct", + "moonshotai/Kimi-K2.5", "XiaomiMiMo/MiMo-7B-RL", + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", ] -SINGLE_TOOL_CALL_ONLY_MODELS = [ - # "meta-llama/Llama-3.2-1B-Instruct", # Skipped: gated repo +# Models that fail decode round-trip under transformers>=5.x due to upstream tokenizer issues. +# These are excluded from TOOL_CALL_TEST_MODELS but listed here for tracking. +# - DeepSeek-V3, step3: transformers v5 unified LlamaTokenizer overwrites their ByteLevel +# pre_tokenizer/decoder with Metaspace, causing decode(encode(text)) != text. +# See https://github.com/huggingface/transformers/issues/43066 +# - DeepSeek-V3.1: its tool-call chat template concatenates function.arguments as a string, +# but our dummy tool-call shape provides a dict, raising TypeError before the round-trip check. +# - glm-4-9b-chat: v5 removed the legacy _decode special-token segmentation, exposing a bug in +# the model's custom convert_tokens_to_string (doesn't handle str-type special tokens). +TOOL_CALL_KNOWN_FAILURES = [ + "deepseek-ai/DeepSeek-V3", + "deepseek-ai/DeepSeek-V3.1", + "stepfun-ai/step3", + "THUDM/glm-4-9b-chat", ] -# Models where tokenize->decode produces extra whitespace vs direct string diff -TOKENIZE_DECODE_WHITESPACE_DIFF_MODELS = [ - "THUDM/glm-4-9b-chat", +SINGLE_TOOL_CALL_ONLY_MODELS = [ + # "meta-llama/Llama-3.2-1B-Instruct", # Skipped: gated repo ] SAMPLE_TOOL_RESPONSES = [ @@ -83,11 +98,6 @@ def test_tokenize_tool_responses(self, model_name, num_tools): base_messages = [_DUMMY_USER, dummy_assistant] expected_str = self._compute_chat_template_diff(base_messages, tool_responses, tokenizer) - if model_name in TOKENIZE_DECODE_WHITESPACE_DIFF_MODELS: - # Some models produce whitespace differences between tokenize->decode and direct string diff - actual_str = actual_str.replace(" ", "") - expected_str = expected_str.replace(" ", "") - assert actual_str == expected_str, f"{model_name=}" @staticmethod