[CI][Bugfix] Use token-id prefix in batch-invariant decode/prefill consistency test

tests/v1/determinism/test_batch_invariance.py

@@ -716,39 +716,27 @@ def test_decode_logprobs_match_prefill_logprobs(
         print(f"[Prompt {prompt_idx}] Generated {len(token_ids)} tokens: {token_ids}")
         print(f"[Prompt {prompt_idx}] Decode logprobs: {decode_logprobs.tolist()}")
 
-        # Step 2: For each token position, run prefill and compare
+        # Step 2: For each token position, run prefill and compare.
+        #
+        # Build the prefix directly from token ids rather than reconstructing
+        # it from text. A `decode -> encode` round trip is not lossless for
+        # many BPE tokenizers (e.g. Qwen2.5-Coder merges adjacent tokens like
+        # `'.' + '#'` into `'.#'` after detokenization), which would silently
+        # change the token-level prefix and produce false mismatches in the
+        # comparisons below. Using `prompt_token_ids + decode_token_ids[:i]`
+        # guarantees the prefill request sees exactly the same token prefix
+        # as the decode path. As a side effect, this also avoids an O(N^2)
+        # extra `llm.generate` call per token that the previous text-based
+        # reconstruction needed.
+        prompt_token_ids = list(decode_output.prompt_token_ids or [])
+
         print(f"\n[Prompt {prompt_idx}] Verifying each token via prefill...")
 
         for token_idx in range(len(token_ids)):
-            # Construct the prefix up to (but not including) this token
+            # Construct the prefix up to (but not including) this token using
+            # token ids directly.
             current_token = token_ids[token_idx]
-
-            # We need to detokenize to get the text prefix
-            # For this, we'll use the tokenizer from the LLM
-            # However, the LLM API doesn't expose tokenizer easily, so we'll
-            # construct the prefix by decoding from the original prompt
-
-            # Get text up to this point by using the output text
-            # This is approximate but should work for verification
-            if token_idx == 0:
-                prefix_prompt = prompt
-            else:
-                # Use the partial output text up to this token
-                # We'll need to construct this from the full output
-                prefix_output = decode_output.outputs[0]
-                # Get the text for tokens 0 to token_idx-1
-                # Unfortunately, we don't have per-token text, so we'll use
-                # a different approach: run prefill with prompt + tokens[0:token_idx]
-
-                # Actually, we need to get the actual text. Let's use a workaround:
-                # Run a generation with max_tokens = token_idx to get that prefix
-                prefix_sp = SamplingParams(
-                    temperature=0.0,
-                    max_tokens=token_idx,
-                    logprobs=1,
-                )
-                prefix_output = llm.generate([prompt], prefix_sp, use_tqdm=False)[0]
-                prefix_prompt = prompt + prefix_output.outputs[0].text
+            prefix_token_ids = prompt_token_ids + list(token_ids[:token_idx])
 
             # Now run prefill with max_tokens=1 to get the logprob of the next token
             prefill_sp = SamplingParams(
@@ -759,19 +747,23 @@ def test_decode_logprobs_match_prefill_logprobs(
 
             print(
                 f"  [Token {token_idx}] Running prefill for prefix "
-                f"(len={len(prefix_prompt)})..."
+                f"(num_tokens={len(prefix_token_ids)})..."
+            )
+            prefill_output = llm.generate(
+                [{"prompt_token_ids": prefix_token_ids}],
+                prefill_sp,
+                use_tqdm=False,
+            )[0]
+            prefill_logprobs, prefill_token_ids_out = _extract_step_logprobs(
+                prefill_output
             )
-            prefill_output = llm.generate([prefix_prompt], prefill_sp, use_tqdm=False)[
-                0
-            ]
-            prefill_logprobs, prefill_token_ids = _extract_step_logprobs(prefill_output)
 
             if prefill_logprobs is None:
                 print(f"  [Token {token_idx}] Warning: No prefill logprobs available")
                 continue
 
             # The first token from prefill should match the current token
-            prefill_token = prefill_token_ids[0]
+            prefill_token = prefill_token_ids_out[0]
             prefill_logprob = prefill_logprobs[0].item()
             decode_logprob = decode_logprobs[token_idx].item()
 
@@ -796,7 +788,7 @@ def test_decode_logprobs_match_prefill_logprobs(
                         "decode_logprob": decode_logprob,
                         "prefill_logprob": prefill_logprob,
                         "prompt_text": prompt[:100],
-                        "prefix_text": prefix_prompt[:100],
+                        "prefix_token_ids": prefix_token_ids[:50],
                     }
                 )
                 print(f"  [Token {token_idx}] ✗ TOKEN MISMATCH!")
@@ -816,7 +808,7 @@ def test_decode_logprobs_match_prefill_logprobs(
                         "prefill_logprob": prefill_logprob,
                         "diff": diff,
                         "prompt_text": prompt[:100],
-                        "prefix_text": prefix_prompt[:100],
+                        "prefix_token_ids": prefix_token_ids[:50],
                         "decode_all_tokens": token_ids,
                         "decode_all_logprobs": decode_logprobs.tolist(),
                     }
@@ -853,7 +845,7 @@ def test_decode_logprobs_match_prefill_logprobs(
             for i, fail in enumerate(failures[:5]):  # Show first 5 failures per prompt
                 print(f"\n  [Failure {i + 1}] Token position {fail['token_idx']}:")
                 print(f"    Reason: {fail['reason']}")
-                print(f"    Prefix text: '{fail['prefix_text']}...'")
+                print(f"    Prefix token ids: {fail['prefix_token_ids']}...")
                 print(
                     f"    Decode:  token={fail['decode_token']}, "
                     f"logprob={fail['decode_logprob']:.10f}"

tests/v1/determinism/test_prefix_token_ids_regression.py

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Regression tests for the prefill-prefix construction used by
+``test_decode_logprobs_match_prefill_logprobs`` in ``test_batch_invariance.py``.
+
+The decode/prefill consistency test verifies, for each decoded token at
+position ``i``, that running prefill on ``prompt + decode_tokens[:i]``
+reproduces the same logprob for ``decode_tokens[i]`` that decode produced.
+
+A previous implementation reconstructed the prefill prefix from text by
+running ``llm.generate`` to detokenize the partial output and then passing
+``prompt + partial_text`` back as a string prompt. vLLM internally re-encoded
+that string through the model's tokenizer, which silently changes the token
+sequence for many BPE tokenizers because ``tokenizer.encode(tokenizer.decode(ids))``
+is not lossless. That broke the underlying invariant of the test (the
+prefill prefix was no longer the same token sequence as the decode prefix)
+and produced sporadic false mismatches.
+
+These tests pin down two properties so the issue does not silently regress:
+
+1. There exist real-world inputs/tokenizers where ``encode(decode(ids)) != ids``.
+   Future "simplification" PRs that reintroduce the text-based round trip must
+   contend with this evidence.
+2. Building the prefix directly with ``prompt_token_ids + decode_tokens[:i]``
+   trivially preserves the token sequence and is the implementation that
+   ``test_decode_logprobs_match_prefill_logprobs`` relies on.
+
+The tests are intentionally tokenizer-only so they run quickly on CPU in CI
+and do not require a GPU, model weights, or batch-invariant kernels.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytestmark = pytest.mark.cpu_test
+
+
+@pytest.fixture(scope="module")
+def qwen_coder_tokenizer():
+    """A tokenizer that exhibits BPE merges across the decode/encode boundary.
+
+    Skips if ``transformers`` (or the cached tokenizer files) are unavailable
+    in the current environment.
+    """
+    transformers = pytest.importorskip("transformers")
+    try:
+        return transformers.AutoTokenizer.from_pretrained(
+            "Qwen/Qwen2.5-Coder-0.5B-Instruct",
+        )
+    except Exception as exc:  # pragma: no cover - depends on local cache/network
+        pytest.skip(f"Tokenizer not available offline: {exc}")
+
+
+def _encode(tok, text: str) -> list[int]:
+    return tok.encode(text, add_special_tokens=False)
+
+
+def test_decode_then_encode_can_change_token_ids(qwen_coder_tokenizer):
+    """The decode -> encode round trip is not lossless for some BPE tokenizers.
+
+    With Qwen2.5-Coder, the adjacent tokens ``'.'`` and ``'#'`` get merged into
+    a single ``'.#'`` token after detokenization. Reconstructing a prefill
+    prefix via text therefore produces a *different* token sequence than the
+    decode path used.
+    """
+    tok = qwen_coder_tokenizer
+
+    prompt = "Yesterday I went to the store and bought a new toy."
+    continuation = "#1."
+
+    prompt_ids = _encode(tok, prompt)
+    continuation_ids = _encode(tok, continuation)
+    original_ids = prompt_ids + continuation_ids
+
+    text = tok.decode(original_ids)
+    re_ids = _encode(tok, text)
+
+    # The round trip must change the token sequence on this input; otherwise
+    # the regression scenario this test is guarding against would not exist.
+    assert original_ids != re_ids, (
+        "Expected `encode(decode(ids)) != ids` for this BPE tokenizer/input. "
+        f"original={original_ids}, round_trip={re_ids}"
+    )
+
+
+def test_token_id_prefix_is_preserved_without_round_trip(qwen_coder_tokenizer):
+    """Building a prefix directly from token ids preserves the sequence.
+
+    This mirrors the construction used by
+    ``test_decode_logprobs_match_prefill_logprobs``:
+    ``prefix_token_ids = prompt_token_ids + decode_tokens[:i]``.
+    """
+    tok = qwen_coder_tokenizer
+
+    prompt = "Yesterday I went to the store and bought a new toy."
+    continuation = "#1."
+
+    prompt_ids = _encode(tok, prompt)
+    decode_tokens = _encode(tok, continuation)
+
+    for i in range(len(decode_tokens) + 1):
+        prefix_token_ids = prompt_ids + decode_tokens[:i]
+
+        # The first ``len(prompt_ids)`` ids are exactly the original prompt.
+        assert prefix_token_ids[: len(prompt_ids)] == prompt_ids
+        # The remaining ids are the first ``i`` decode tokens, unchanged.
+        assert prefix_token_ids[len(prompt_ids) :] == decode_tokens[:i]