flash attention addition

examples/tool_chat_template_gemma4.jinja

@@ -356,8 +356,5 @@
 {%- if add_generation_prompt -%}
     {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}
         {{- '<|turn>model\n' -}}
-        {%- if not enable_thinking | default(false) -%}
-            {{- '<|channel>thought\n<channel|>' -}}
-        {%- endif -%}
     {%- endif -%}
 {%- endif -%}

tests/models/registry.py

@@ -290,6 +290,11 @@ def check_available_online(
         "google/gemma-4-E2B-it",
         min_transformers_version="5.0.0",
     ),
+    "Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
+        "google/gemma-4-12B-it",
+        min_transformers_version="5.8.0",
+        is_available_online=False,
+    ),
     "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
     "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
     "Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),

tests/reasoning/test_gemma4_reasoning_parser.py

@@ -22,26 +22,26 @@ def generic_tokenizer():
 
 INVALID_SIMPLE_NONSTREAMING = {
     "output": "This is a reasoning section<channel|>This is the rest",
-    "reasoning": "This is a reasoning section",
-    "content": "This is the rest",
+    "reasoning": None,
+    "content": "This is a reasoning section<channel|>This is the rest",
     "is_reasoning_end": True,
 }
 INVALID_SIMPLE_STREAMING = {
     "output": "This is a reasoning section<channel|>This is the rest",
     "reasoning": None,
-    "content": "This is a reasoning sectionThis is the rest",
+    "content": "This is a reasoning section<channel|>This is the rest",
     "is_reasoning_end": True,
 }
 INVALID_COMPLETE_NONSTREAMING = {
     "output": "This is a reasoning section<channel|>",
-    "reasoning": "This is a reasoning section",
-    "content": None,
+    "reasoning": None,
+    "content": "This is a reasoning section<channel|>",
     "is_reasoning_end": True,
 }
 INVALID_COMPLETE_STREAMING = {
     "output": "This is a reasoning section<channel|>",
     "reasoning": None,
-    "content": "This is a reasoning section",
+    "content": "This is a reasoning section<channel|>",
     "is_reasoning_end": True,
 }
 NO_CONTENT = {

tests/renderers/test_gemma4_chat_template.py

@@ -32,8 +32,8 @@ def _render(template, messages, **kwargs):
 
 class TestGemma4ChatTemplate:
     def test_basic_multiturn_thinking_disabled(self, gemma4_template):
-        """With enable_thinking=False (default), generation prompt ends with
-        an empty thought channel to suppress thinking."""
+        """With enable_thinking=False (default), HF template ends generation
+        prompt at <|turn>model (no empty thought-channel injection)."""
         messages = [
             {"role": "user", "content": "Hello"},
             {"role": "assistant", "content": "Hi there!"},
@@ -45,7 +45,8 @@ def test_basic_multiturn_thinking_disabled(self, gemma4_template):
         assert "Hello" in result
         assert "Hi there!" in result
         assert "How are you?" in result
-        assert result.rstrip("\n").endswith("<|channel>thought\n<channel|>")
+        assert "<|think|>" not in result
+        assert result.rstrip("\n").endswith("<|turn>model")
 
     def test_basic_multiturn_thinking_enabled(self, gemma4_template):
         """With enable_thinking=True, generation prompt ends with model
@@ -172,6 +173,39 @@ def test_tool_responses_openai_style(self, gemma4_template):
         assert "<tool_response|>" in result
         assert '"temperature": 15' in result
 
+    def test_tool_response_multimodal_content_parts(self, gemma4_template):
+        """role='tool' with content parts emits modality markers after response."""
+        messages = [
+            {"role": "user", "content": "Describe image"},
+            {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "function": {
+                            "name": "analyze",
+                            "arguments": {},
+                        },
+                    }
+                ],
+            },
+            {
+                "role": "tool",
+                "tool_call_id": "call_1",
+                "content": [
+                    {"type": "text", "text": "red circle"},
+                    {"type": "image"},
+                    {"type": "audio"},
+                ],
+            },
+        ]
+        result = _render(gemma4_template, messages)
+        assert "<|tool_response>" in result
+        assert "<tool_response|>" in result
+        assert "<|image|>" in result
+        assert "<|audio|>" in result
+
     def test_tool_responses_legacy_style(self, gemma4_template):
         """tool_responses embedded on the assistant message."""
         messages = [

tests/tool_parsers/test_gemma4_format_thought_strip.py

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Standalone tests for Gemma ``thought`` glued-shard stripping.
+
+Loads ``gemma4_format.py`` directly (no ``import vllm``).
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+def _load_format():
+    path = Path(__file__).resolve().parents[2] / "vllm" / "tool_parsers" / "gemma4_format.py"
+    spec = importlib.util.spec_from_file_location("gemma4_format_under_test", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader
+    spec.loader.exec_module(module)
+    return module
+
+
+_mod = None
+
+
+def _fmt():
+    global _mod  # noqa: PLW0603
+    if _mod is None:
+        _mod = _load_format()
+    return _mod
+
+
+def strip(s: str) -> str:
+    return _fmt().strip_leaked_empty_thinking(s)
+
+
+def test_many_glued_odd_and_truncated_suffix():
+    core = "".join(("thought",) * 11) + "tho"
+    assert strip(core).strip() == ""
+
+
+def test_pairs_then_partial_only():
+    assert strip("thoughtthoughtthoughttho").strip() == ""
+
+
+def test_glued_then_real_text():
+    assert strip("thoughtthoughtHello") == "thoughtthoughtHello"
+
+
+def test_thoughtthoughtful_untouched():
+    assert strip("thoughtthoughtful") == "thoughtthoughtful"
+
+
+def test_mid_sentence_thoughtful():
+    s = "That was thoughtful of you."
+    assert strip(s) == s
+
+
+def test_multiline_only_garbage_lines():
+    lines = ("\n".join(("thoughtthought",) * 3)) + "\n"
+    assert strip(lines).strip() == ""

tests/tool_parsers/test_gemma4_tool_parser.py

@@ -321,6 +321,20 @@ def test_dotted_function_name(self, parser, mock_request):
         assert result.tools_called is True
         assert result.tool_calls[0].function.name == "weather.get"
 
+    def test_nested_details_in_arguments(self, parser, mock_request):
+        """Nested objects inside braces (SGLang parity)."""
+        model_output = (
+            '<|tool_call>call:get_weather{location:<|"|>Tokyo<|"|>,'
+            'details:{temp:25,unit:<|"|>celsius<|"|>}'
+            '}<tool_call|>'
+        )
+        result = parser.extract_tool_calls(model_output, mock_request)
+
+        assert result.tools_called is True
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args["location"] == "Tokyo"
+        assert args["details"] == {"temp": 25, "unit": "celsius"}
+
     def test_no_arguments(self, parser, mock_request):
         """Tool calls with empty arguments."""
         model_output = "<|tool_call>call:get_status{}<tool_call|>"
@@ -539,6 +553,28 @@ def test_streaming_text_before_tool_call(self, parser, mock_request):
 
         assert "".join(content_parts).strip().startswith("Let me check")
 
+    def test_streaming_suppressed_thinking_fragments_not_in_content(
+        self, parser, mock_request
+    ):
+        """suppress-CoT channel markers split across deltas must never reach clients."""
+        chunks = [
+            "<|channel>",
+            "thought\n",
+            "<channel|>",
+            "<|channel>",
+            "Hi. ",
+            "<|tool_call>",
+            "call:get_status{}",
+            "<tool_call|>",
+        ]
+        results = self._simulate_streaming(parser, mock_request, chunks)
+        joined = "".join(
+            delta.content for delta, _ in results if delta and delta.content
+        )
+        assert "<|channel>" not in joined
+        assert "<channel|>" not in joined
+        assert joined.strip().startswith("Hi.")
+
     def test_streaming_numeric_args(self, parser, mock_request):
         """Streaming with numeric and boolean argument values."""
         chunks = [
@@ -601,6 +637,21 @@ def test_streaming_number_split_across_chunks(self, parser, mock_request):
         parsed_args = json.loads(args_text)
         assert parsed_args["count"] == 42
 
+    def test_nested_array_streaming_sglang_parity(self, parser, mock_request):
+        """Nested array + object inside tool args across chunks (SGLang parity)."""
+        chunks = [
+            '<|tool_call>call:get_weather{location:<|"',
+            '|>New York<|"|>,nested:[1, 2, {inner:<|"|>',
+            'val<|"|>}]}<tool_call|>',
+        ]
+
+        results = self._simulate_streaming(parser, mock_request, chunks)
+        args_text = self._collect_arguments(results)
+        assert args_text, "No arguments were streamed"
+        parsed_args = json.loads(args_text)
+        assert parsed_args["location"] == "New York"
+        assert parsed_args["nested"] == [1, 2, {"inner": "val"}]
+
     def test_streaming_empty_args(self, parser, mock_request):
         """Tool call with no arguments."""
         chunks = [