Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions tests/v1/core/test_kv_cache_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2313,6 +2313,55 @@ def test_group_dcp_replicated_dflash_with_hybrid_mla_target():
assert groups[-1].kv_cache_spec.dcp_replicated is True


def test_glm5next_split_cache_preserves_physical_pages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Split GLM-5.3 cache groups retain their independent page geometry."""
target = MLAAttentionSpec(
block_size=512,
num_kv_heads=1,
head_size=528,
dtype=torch.uint8,
cache_dtype_str="fp8",
model_version="glm5_next",
page_tail_bytes_per_token=33,
)
recurrent = MambaSpec(
block_size=512,
shapes=((585728,),),
dtypes=(torch.bfloat16,),
mamba_cache_mode="align",
)
assert target.page_size_bytes == 287232
assert recurrent.page_size_bytes == 1171456

monkeypatch.setenv("VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE", "512")
groups = get_kv_cache_groups(
_grouping_config(),
{"model.target": target, "model.recurrent": recurrent},
)

pages = {
layer_name: group.kv_cache_spec.page_size_bytes
for group in groups
for layer_name in group.layer_names
}
assert pages == {
"model.target": target.page_size_bytes,
"model.recurrent": recurrent.page_size_bytes,
}

c4_index_page_bytes = 64 * 132
expected_pool_stride = (
(recurrent.page_size_bytes + c4_index_page_bytes - 1)
// c4_index_page_bytes
* c4_index_page_bytes
)
assert kv_cache_utils._get_kv_cache_bytes_per_block(groups) == (
expected_pool_stride
)


def new_indexer_mla_spec(block_size=16):
# Sparse-attention indexer k_cache: an MLAAttentionSpec with a much smaller
# page size than the main MLA attention (uint8, small head), so their pages
Expand Down
47 changes: 47 additions & 0 deletions vllm/platforms/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,53 @@ def _align_hybrid_block_size(
model_config = vllm_config.model_config
parallel_config = vllm_config.parallel_config

split_target_block_size = os.getenv("VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE")
if split_target_block_size is not None:
if model_config.architecture != "Glm5NextForConditionalGeneration":
raise ValueError(
"VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE is supported only for "
"Glm5NextForConditionalGeneration."
)
target_block_size = int(split_target_block_size)
split_mamba_block_size = int(
os.getenv(
"VLLM_GLM53_SPLIT_MAMBA_BLOCK_SIZE",
split_target_block_size,
)
)
if target_block_size <= 0 or target_block_size % 64 != 0:
raise ValueError(
"VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE must be a positive "
"multiple of 64."
)
if (
split_mamba_block_size <= 0
or split_mamba_block_size % target_block_size != 0
):
raise ValueError(
"VLLM_GLM53_SPLIT_MAMBA_BLOCK_SIZE must be a positive "
"multiple of VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE."
)
if cache_config.mamba_cache_mode != "align":
raise ValueError(
"Split GLM-5.3 target and recurrent-state pages require "
"mamba_cache_mode='align'."
)

# The target MLA cache and recurrent-state cache use independent
# physical pages. Their token block sizes remain scheduler-visible,
# so the recurrent block must be a multiple of the target block.
cache_config.block_size = target_block_size
cache_config.mamba_block_size = split_mamba_block_size
cache_config.mamba_page_size_padded = None
logger.warning(
"Using split GLM-5.3 cache pages: target block size %d tokens, "
"recurrent-state block size %d tokens.",
target_block_size,
split_mamba_block_size,
)
return

if cache_config.cache_dtype == "auto":
kv_cache_dtype = model_config.dtype
else:
Expand Down
37 changes: 37 additions & 0 deletions vllm/v1/core/kv_cache_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1294,6 +1294,13 @@ def _get_kv_cache_bytes_per_block(
for group in kv_cache_groups
)
assert bytes_per_block > 0
if os.getenv("VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE") is not None:
# GLM-5.3 stores two 64-row by 132-byte FP8 C4 index pages in the
# target MLA page tail when the target block contains 512 tokens.
# The block-outermost pool stride must preserve the index-page unit
# even when a larger recurrent-state group determines pool capacity.
glm_c4_index_page_bytes = 64 * 132
bytes_per_block = round_up(bytes_per_block, glm_c4_index_page_bytes)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return bytes_per_block


Expand Down Expand Up @@ -1903,6 +1910,36 @@ def get_kv_cache_groups(
if not isinstance(v, HiddenStateCacheSpec)
}

split_target_block_size = os.getenv("VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE")
is_glm5_next_split_cache = split_target_block_size is not None and any(
isinstance(spec, MLAAttentionSpec) and spec.model_version == "glm5_next"
for spec in filtered_spec.values()
)
if is_glm5_next_split_cache:
if hidden_specs:
raise ValueError(
"Split GLM-5.3 cache pages do not support hidden-state cache layers."
)
if not any(isinstance(spec, MambaSpec) for spec in filtered_spec.values()):
raise ValueError(
"Split GLM-5.3 cache pages require recurrent-state cache layers."
)
# Preserve the target MLA and recurrent-state physical page sizes.
# The block-outermost layout packs each group's pages independently,
# while the hybrid coordinator handles their token block sizes.
groups = _get_kv_cache_groups_uniform_page_size(filtered_spec)
logger.warning(
"Keeping split GLM-5.3 cache groups with physical page sizes %s.",
sorted(
{
_get_per_layer_spec(group, layer_name).page_size_bytes
for group in groups
for layer_name in group.layer_names
}
),
)
return groups

# Prefer preserving each layer's cache semantics. If physical pages cannot
# be unified, try a supported allocation-only fallback before failing.
try:
Expand Down
Loading