From f34bf837c3d98f50fda2a4c7407cd4a6c4f454d7 Mon Sep 17 00:00:00 2001 From: Yifan Da <1363818765@qq.com> Date: Thu, 21 May 2026 20:46:49 +0800 Subject: [PATCH] [CI][Bugfix] Use token-id prefix in batch-invariant decode/prefill consistency test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test_decode_logprobs_match_prefill_logprobs` previously rebuilt the prefill prefix by running an extra `llm.generate(max_tokens=token_idx)` to detokenize the partial output and then submitting `prompt + partial_text` as a string prompt. vLLM re-encodes that string with the model's tokenizer, which is not lossless for many BPE tokenizers — for example Qwen2.5-Coder merges `'.' + '#'` into `'.#'` after detokenization. This silently changed the token-level prefix the test was supposed to verify, producing sporadic false mismatches on certain models/inputs. Build the prefix directly from token ids using `decode_output.prompt_token_ids` plus `decode_token_ids[:i]` and pass it to `llm.generate` as `{"prompt_token_ids": ...}`. This guarantees the prefill request sees the exact same token prefix as the decode path. As a side effect it also avoids the previous O(N^2) extra `llm.generate` call per decoded token. Add a tokenizer-only regression test (`test_prefix_token_ids_regression.py`) that pins down both properties: `encode(decode(ids)) != ids` is reachable on a real tokenizer/input, and the new token-id construction preserves the sequence by construction. Reported by @xRay2016 in https://github.com/vllm-project/vllm/issues/27433 Test plan: # CPU-only regression tests .venv/bin/python -m pytest tests/v1/determinism/test_prefix_token_ids_regression.py -v # 2 passed in 5.42s # GPU smoke (Qwen3-0.6B on H20, 4 prompts x 8 tokens) VLLM_TEST_MODEL=Qwen/Qwen3-0.6B \ VLLM_DECODE_PREFILL_NUM_PROMPTS=4 \ VLLM_DECODE_PREFILL_MAX_TOKENS=8 \ .venv/bin/python -m pytest \ tests/v1/determinism/test_batch_invariance.py::test_decode_logprobs_match_prefill_logprobs -v # 1 passed in 22.69s Lint: pre-commit run --files tests/v1/determinism/test_batch_invariance.py \ tests/v1/determinism/test_prefix_token_ids_regression.py # all hooks pass This change was prepared with AI assistance (CodeBuddy). Every changed line was reviewed and the commands above were run by the human submitter. Co-authored-by: CodeBuddy Signed-off-by: Yifan Da <1363818765@qq.com> --- tests/v1/determinism/test_batch_invariance.py | 68 +++++------ .../test_prefix_token_ids_regression.py | 108 ++++++++++++++++++ 2 files changed, 138 insertions(+), 38 deletions(-) create mode 100644 tests/v1/determinism/test_prefix_token_ids_regression.py diff --git a/tests/v1/determinism/test_batch_invariance.py b/tests/v1/determinism/test_batch_invariance.py index 415c7d5f3f26..18c4c77778b4 100644 --- a/tests/v1/determinism/test_batch_invariance.py +++ b/tests/v1/determinism/test_batch_invariance.py @@ -716,39 +716,27 @@ def test_decode_logprobs_match_prefill_logprobs( print(f"[Prompt {prompt_idx}] Generated {len(token_ids)} tokens: {token_ids}") print(f"[Prompt {prompt_idx}] Decode logprobs: {decode_logprobs.tolist()}") - # Step 2: For each token position, run prefill and compare + # Step 2: For each token position, run prefill and compare. + # + # Build the prefix directly from token ids rather than reconstructing + # it from text. A `decode -> encode` round trip is not lossless for + # many BPE tokenizers (e.g. Qwen2.5-Coder merges adjacent tokens like + # `'.' + '#'` into `'.#'` after detokenization), which would silently + # change the token-level prefix and produce false mismatches in the + # comparisons below. Using `prompt_token_ids + decode_token_ids[:i]` + # guarantees the prefill request sees exactly the same token prefix + # as the decode path. As a side effect, this also avoids an O(N^2) + # extra `llm.generate` call per token that the previous text-based + # reconstruction needed. + prompt_token_ids = list(decode_output.prompt_token_ids or []) + print(f"\n[Prompt {prompt_idx}] Verifying each token via prefill...") for token_idx in range(len(token_ids)): - # Construct the prefix up to (but not including) this token + # Construct the prefix up to (but not including) this token using + # token ids directly. current_token = token_ids[token_idx] - - # We need to detokenize to get the text prefix - # For this, we'll use the tokenizer from the LLM - # However, the LLM API doesn't expose tokenizer easily, so we'll - # construct the prefix by decoding from the original prompt - - # Get text up to this point by using the output text - # This is approximate but should work for verification - if token_idx == 0: - prefix_prompt = prompt - else: - # Use the partial output text up to this token - # We'll need to construct this from the full output - prefix_output = decode_output.outputs[0] - # Get the text for tokens 0 to token_idx-1 - # Unfortunately, we don't have per-token text, so we'll use - # a different approach: run prefill with prompt + tokens[0:token_idx] - - # Actually, we need to get the actual text. Let's use a workaround: - # Run a generation with max_tokens = token_idx to get that prefix - prefix_sp = SamplingParams( - temperature=0.0, - max_tokens=token_idx, - logprobs=1, - ) - prefix_output = llm.generate([prompt], prefix_sp, use_tqdm=False)[0] - prefix_prompt = prompt + prefix_output.outputs[0].text + prefix_token_ids = prompt_token_ids + list(token_ids[:token_idx]) # Now run prefill with max_tokens=1 to get the logprob of the next token prefill_sp = SamplingParams( @@ -759,19 +747,23 @@ def test_decode_logprobs_match_prefill_logprobs( print( f" [Token {token_idx}] Running prefill for prefix " - f"(len={len(prefix_prompt)})..." + f"(num_tokens={len(prefix_token_ids)})..." + ) + prefill_output = llm.generate( + [{"prompt_token_ids": prefix_token_ids}], + prefill_sp, + use_tqdm=False, + )[0] + prefill_logprobs, prefill_token_ids_out = _extract_step_logprobs( + prefill_output ) - prefill_output = llm.generate([prefix_prompt], prefill_sp, use_tqdm=False)[ - 0 - ] - prefill_logprobs, prefill_token_ids = _extract_step_logprobs(prefill_output) if prefill_logprobs is None: print(f" [Token {token_idx}] Warning: No prefill logprobs available") continue # The first token from prefill should match the current token - prefill_token = prefill_token_ids[0] + prefill_token = prefill_token_ids_out[0] prefill_logprob = prefill_logprobs[0].item() decode_logprob = decode_logprobs[token_idx].item() @@ -796,7 +788,7 @@ def test_decode_logprobs_match_prefill_logprobs( "decode_logprob": decode_logprob, "prefill_logprob": prefill_logprob, "prompt_text": prompt[:100], - "prefix_text": prefix_prompt[:100], + "prefix_token_ids": prefix_token_ids[:50], } ) print(f" [Token {token_idx}] ✗ TOKEN MISMATCH!") @@ -816,7 +808,7 @@ def test_decode_logprobs_match_prefill_logprobs( "prefill_logprob": prefill_logprob, "diff": diff, "prompt_text": prompt[:100], - "prefix_text": prefix_prompt[:100], + "prefix_token_ids": prefix_token_ids[:50], "decode_all_tokens": token_ids, "decode_all_logprobs": decode_logprobs.tolist(), } @@ -853,7 +845,7 @@ def test_decode_logprobs_match_prefill_logprobs( for i, fail in enumerate(failures[:5]): # Show first 5 failures per prompt print(f"\n [Failure {i + 1}] Token position {fail['token_idx']}:") print(f" Reason: {fail['reason']}") - print(f" Prefix text: '{fail['prefix_text']}...'") + print(f" Prefix token ids: {fail['prefix_token_ids']}...") print( f" Decode: token={fail['decode_token']}, " f"logprob={fail['decode_logprob']:.10f}" diff --git a/tests/v1/determinism/test_prefix_token_ids_regression.py b/tests/v1/determinism/test_prefix_token_ids_regression.py new file mode 100644 index 000000000000..65bcdd660863 --- /dev/null +++ b/tests/v1/determinism/test_prefix_token_ids_regression.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for the prefill-prefix construction used by +``test_decode_logprobs_match_prefill_logprobs`` in ``test_batch_invariance.py``. + +The decode/prefill consistency test verifies, for each decoded token at +position ``i``, that running prefill on ``prompt + decode_tokens[:i]`` +reproduces the same logprob for ``decode_tokens[i]`` that decode produced. + +A previous implementation reconstructed the prefill prefix from text by +running ``llm.generate`` to detokenize the partial output and then passing +``prompt + partial_text`` back as a string prompt. vLLM internally re-encoded +that string through the model's tokenizer, which silently changes the token +sequence for many BPE tokenizers because ``tokenizer.encode(tokenizer.decode(ids))`` +is not lossless. That broke the underlying invariant of the test (the +prefill prefix was no longer the same token sequence as the decode prefix) +and produced sporadic false mismatches. + +These tests pin down two properties so the issue does not silently regress: + +1. There exist real-world inputs/tokenizers where ``encode(decode(ids)) != ids``. + Future "simplification" PRs that reintroduce the text-based round trip must + contend with this evidence. +2. Building the prefix directly with ``prompt_token_ids + decode_tokens[:i]`` + trivially preserves the token sequence and is the implementation that + ``test_decode_logprobs_match_prefill_logprobs`` relies on. + +The tests are intentionally tokenizer-only so they run quickly on CPU in CI +and do not require a GPU, model weights, or batch-invariant kernels. +""" + +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.cpu_test + + +@pytest.fixture(scope="module") +def qwen_coder_tokenizer(): + """A tokenizer that exhibits BPE merges across the decode/encode boundary. + + Skips if ``transformers`` (or the cached tokenizer files) are unavailable + in the current environment. + """ + transformers = pytest.importorskip("transformers") + try: + return transformers.AutoTokenizer.from_pretrained( + "Qwen/Qwen2.5-Coder-0.5B-Instruct", + ) + except Exception as exc: # pragma: no cover - depends on local cache/network + pytest.skip(f"Tokenizer not available offline: {exc}") + + +def _encode(tok, text: str) -> list[int]: + return tok.encode(text, add_special_tokens=False) + + +def test_decode_then_encode_can_change_token_ids(qwen_coder_tokenizer): + """The decode -> encode round trip is not lossless for some BPE tokenizers. + + With Qwen2.5-Coder, the adjacent tokens ``'.'`` and ``'#'`` get merged into + a single ``'.#'`` token after detokenization. Reconstructing a prefill + prefix via text therefore produces a *different* token sequence than the + decode path used. + """ + tok = qwen_coder_tokenizer + + prompt = "Yesterday I went to the store and bought a new toy." + continuation = "#1." + + prompt_ids = _encode(tok, prompt) + continuation_ids = _encode(tok, continuation) + original_ids = prompt_ids + continuation_ids + + text = tok.decode(original_ids) + re_ids = _encode(tok, text) + + # The round trip must change the token sequence on this input; otherwise + # the regression scenario this test is guarding against would not exist. + assert original_ids != re_ids, ( + "Expected `encode(decode(ids)) != ids` for this BPE tokenizer/input. " + f"original={original_ids}, round_trip={re_ids}" + ) + + +def test_token_id_prefix_is_preserved_without_round_trip(qwen_coder_tokenizer): + """Building a prefix directly from token ids preserves the sequence. + + This mirrors the construction used by + ``test_decode_logprobs_match_prefill_logprobs``: + ``prefix_token_ids = prompt_token_ids + decode_tokens[:i]``. + """ + tok = qwen_coder_tokenizer + + prompt = "Yesterday I went to the store and bought a new toy." + continuation = "#1." + + prompt_ids = _encode(tok, prompt) + decode_tokens = _encode(tok, continuation) + + for i in range(len(decode_tokens) + 1): + prefix_token_ids = prompt_ids + decode_tokens[:i] + + # The first ``len(prompt_ids)`` ids are exactly the original prompt. + assert prefix_token_ids[: len(prompt_ids)] == prompt_ids + # The remaining ids are the first ``i`` decode tokens, unchanged. + assert prefix_token_ids[len(prompt_ids) :] == decode_tokens[:i]