From 4d898fa0c1265c5aef15cd3967e031ffd0671c7d Mon Sep 17 00:00:00 2001 From: Raymond Lucke Date: Fri, 4 Sep 2026 09:20:39 -0700 Subject: [PATCH 1/2] fix: restore fine-grained hybrid prefix-cache hits Ignore KV managers that explicitly opt out of prefix caching when deciding whether 64-token hash lookup is safe. KpoolTail remains transient and is rebuilt normally; reusable MLA and Mamba groups still constrain the cache hit. Keep the overlay additive for images that already contain its earlier DFlash2 hybrid-cache fix, update its patch test, and document the live append-only and semantic validation results. --- README.md | 98 +++++++----------------------- overlay/patch_hybrid_prefix_hit.py | 47 ++++++++++++-- tests/bench_prefix_cache.py | 29 +++++---- tests/test_hybrid_prefix_hit.py | 5 ++ 4 files changed, 84 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index af758363..5064ac44 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,7 @@ same path as the compact-64 fp8 serve (not NVFP4 KV). | Attention | `FLASHINFER_MLA_SPARSE_SM120` (NoPE MLA padded into GLM_NSA 576-wide) | | KV | `--kv-cache-dtype fp8` → packed **`fp8_ds_mla`** (target). Draft DFlash2 KV is `auto`/bf16. Latest validated 7168/rightsize pool: **1,243,902** tokens / **1.24×** at 1M. `--enable-prefix-caching` (block-aligned hits; see Prefix caching) | | Context | **850k** default since 2026-09-07 (E3; `EXL3_FAT_GROUPED=0` fits 1M again). KV pool is **~1M tokens** (measured 989,010 at 900k / util 0.85, and 1,023,626-1,084,615 at 900k / util 0.86; pool size varies ±0.4 GiB between identical boots, and the 850k default is not yet measured). Pre-E3 1M receipts: live pool **1,754,237** tokens (1.75×) / 690 GPU blocks / 18.67 GiB at MNBT 2048; latest validated 7168/rightsize E2 pool at 1M **1,243,902** tokens / **1.24×**. Pool size varies with MNBT, activation/graph reservations and hybrid block geometry. Padded slot-share is why 1M allocates; the old 900k cap was 1.95× on this same pool. Do not drop to 256k to “free” slots (hybrid mamba + DFlash window block-id demand is mostly length-independent) | + | Experts | packed trellis + suh + svh + mcg, codebook MCG, **one fused `exllamav3_ext.exl3_moe` launch per layer** | | Dense / shared / attn / embed / lm_head | native (unquantized) | | Tools / reasoning | `--tool-call-parser glm47 --enable-auto-tool-choice --reasoning-parser glm45` | @@ -461,14 +462,15 @@ toward 7168 if you need document-grade detail from single images, and watch **NVFP4 KV is not available here.** FlashInfer’s SM12x NVFP4 kernels are dense MHA, not sparse MLA. Do not confuse that with NVFP4 **weights** (`--moe-backend marlin`). -## Prefix caching (this kit, 2026-08-30) +## Prefix caching (this kit, 2026-09-04) `--enable-prefix-caching` is on. The OpenAI API is **stateless**: the client resends the full history each turn; vLLM hashes that prefix. Concurrent chats do **not** mix activations. `--max-num-seqs 4` is four **in-flight** generations, -not four parked sessions. MLA `KpoolTailManager` disables **fine-grained** -hits — only **block-aligned** tokens count (3584-token hybrid align). -`KpoolTail` already opts out of the hybrid min (1-block circular scratch). +not four parked sessions. Physical sparse-MLA attention pages remain 3584 +tokens, but reusable prefix hashes now match at the underlying **64-token** +cache-block granularity. `KpoolTail` is 1-block circular scratch and already +opts out of prefix caching. `dflash` is `use_eagle()`. GLM never sets `is_eagle_group` (that annotator is DeepseekV4-only), so stock HybridKVCacheCoordinator flagged **every** group. @@ -479,7 +481,14 @@ does **not** let that group shrink the MLA+mamba hit. Mamba stays in the min (skipping a mamba miss is a correctness hole). Do not raise `--max-num-batched-tokens` to “fix” APC. -**Historical pre-E2 receipts** (thinking off, temp 0, unique pads), 1M serve, **`MAX_NUM_BATCHED_TOKENS=2048`** (P1 keep; 3584/4096 reverted), **`DFLASH_DRAFT_TP=2`**. Idle 8k/16k/100k are the 2026-08-30 C4 keep A/B. 12k/256k/300k and the concurrent follow-ups are the prior TP=1 production ladder (chunk size does not change those hit counts). The MNBT=1024 baseline is `docs/cold-prefill.md`. Details: `docs/improve-prefill.md`. +The same overlay also fixes the fine-grained eligibility check. Upstream +included every manager in that check, so non-shareable `KpoolTailManager` +vetoed 64-token lookup even though it never participates in prefix caching. +The overlay ignores only managers whose KV spec explicitly opts out; MLA and +Mamba still constrain the match. This changes lookup granularity, not the +3584-token physical allocation geometry. + +**Historical pre-E2, pre-fine-grained receipts** (thinking off, temp 0, unique pads), 1M serve, **`MAX_NUM_BATCHED_TOKENS=2048`** (P1 keep; 3584/4096 reverted), **`DFLASH_DRAFT_TP=2`**. Idle 8k/16k/100k are the 2026-08-30 C4 keep A/B. 12k/256k/300k and the concurrent follow-ups are the prior TP=1 production ladder. The 3584-quantized hit counts below document the old behavior. The MNBT=1024 baseline is `docs/cold-prefill.md`. Details: `docs/improve-prefill.md`. | Turn | Hits | Compute | Prompt tok | TTFT | Prefill tok/s | |---|---:|---:|---:|---:|---:| @@ -510,80 +519,19 @@ Re-measure (see also `tests/bench_prefix_cache.py`): ```bash # unique-content cold/warm pairs; hit ratio from the vllm:prefix_cache_* -# counter deltas; hit_efficiency scores against the 3584-token page model +# counter deltas; hit_efficiency scores against 64-token hash blocks python3 tests/bench_prefix_cache.py --runs 3 ``` -Note the page math: hits are **block-aligned to the 3584-token hybrid MLA -page**, so a warm prompt only ever reuses `floor(tokens / 3584) × 3584` -tokens — the 7168 / 10752 / 14336 hit rows above are exactly 2 / 3 / 4 full -pages. The bench POSTs `/reset_prefix_cache` between colds when that route is -enabled (`GLM53_EXPOSE_CACHE_RESET=1`; opt-in, see the API surface notes -below) and salts its filler content per invocation on top — repeated runs -stay genuinely cold even with the reset route off. - -## API surface notes (this build, 2026-08-29) - -Two things that cost us time (#31), documented so the next person does not -chase ghosts: - -**Cache reset.** `overlay/patch_cache_reset.py` mounts the upstream dev -cache router on the head API server, so a genuinely cold prefix cache no -longer needs a container restart. It is **opt-in**: the patch is always -applied, but the router is only attached when `GLM53_EXPOSE_CACHE_RESET=1` -is exported for the launcher (default `0`): - -```bash -GLM53_EXPOSE_CACHE_RESET=1 ./start.sh restart # then: -curl -s -X POST http://127.0.0.1:8888/reset_prefix_cache # -> {"success": true} -``` - -It returns `{"success": bool}` and reports `false` while blocks are still -held (running requests, in-flight async KV offload) — retry after they -drain. Unset (the default) leaves the stock surface, where a restart is the -only reset; `VLLM_SERVER_DEV_MODE=1` still mounts the whole dev set -(`/sleep`, `/rlhf`, `/rpc`, `/server_info`) if ever needed. -Auth caveat: the bearer middleware only guards `/v1`, `/v2`, `/inference`, -`/cohere` (upstream `GUARDED_PREFIX`), so root-mounted routes — the stock -`/tokenize` / `/detokenize` and the cache-reset routes — answer without the -key even with `VLLM_API_KEY` set. That is why the exposure is opt-in: leave -`GLM53_EXPOSE_CACHE_RESET` unset (0) on kits that serve untrusted clients. - -**Tokenize.** It is mounted at the **root** (`/v1/tokenize` is 404) and the -request validates `prompt` (or `messages` for the chat shape), not `text`: - -```bash -curl -s http://127.0.0.1:8888/tokenize -H 'Content-Type: application/json' \ - -d '{"model": "GLM-5.3-Flash-EXL3", "prompt": "hello world"}' -# -> {"count":2,"max_model_len":1000000,"tokens":[14978,1879],"token_strs":null} -``` +Note the page math: the physical hybrid page is still 3584 tokens, but with +fine-grained manager eligibility prefix hits are rounded to 64-token hash +blocks - the transient KpoolTail group no longer forces 3584-token alignment +(see `overlay/patch_hybrid_prefix_hit.py`). The bench POSTs +`/reset_prefix_cache` between colds when that route is enabled +(`GLM53_EXPOSE_CACHE_RESET=1`; opt-in, see the API surface notes below) and +salts its filler content per invocation on top - repeated runs stay genuinely +cold even with the reset route off. -### Optional sparse retention and DFlash replay - -`GLM53_APC_RETENTION_INTERVAL_SWA` controls the DFlash2 drafter separately -from target retention. Empty inherits the global retention policy and keeps -ordinary eviction priority. Explicit `0` retains reachable drafter boundaries -and makes cached draft-only blocks lower priority than target cache blocks. -Sparse target Mamba state also retains the prior replay boundary; a cached -DFlash window is reused only after successful EAGLE verification, otherwise -the request backs up to rebuild its window. - -**Sparse retention is not a general performance upgrade.** On one 2× Spark -deployment, explicit global/SWA `0/0` retained four independent 210K histories -for ~2.5 s revisits. In a matched 128K comparison, however, an edit at 90% -took 112.49 s instead of 14.70 s, and a branch at 90% took 99.89 s instead of -3.35 s: the sparse policy reused zero tokens where the old runtime reused -111,104. All tested answers were correct. These are sequential histories, -not four simultaneously active 210K streams. - -The global launcher spelling is `GLM53_APC_RETENTION_INTERVAL` (TP=2 only). -Leave it unset for normal use; TP=4 rejects either retention override. - -Both retention knobs remain unset by default. Keep that default unless the -tradeoff fits the workload. SWA-only sparse retention with a dense target is -a separate configuration; the all-zero results do not qualify it. See the -[protocol, raw measurements, and limitations](docs/apc-retention-qualification.md) -before selecting a policy or a cache budget for another kit. ## Quick start (2× Spark) diff --git a/overlay/patch_hybrid_prefix_hit.py b/overlay/patch_hybrid_prefix_hit.py index 2aba5f8d..6a4550ae 100755 --- a/overlay/patch_hybrid_prefix_hit.py +++ b/overlay/patch_hybrid_prefix_hit.py @@ -11,16 +11,23 @@ 2. The DFlash2 SlidingWindow group still participates in the hybrid min. After an EAGLE one-block pop it re-aligns down by a full 3584-token scheduler page (block=64, align=3584), which can wipe a longer MLA hit. +3. Fine-grained 64-token Mamba/MLA hits are disabled when *any* manager with + a wider block lacks fine-grained lookup support. That check includes + KpoolTailManager even though KpoolTailSpec explicitly opts out of prefix + caching, so one transient scratch group forces every reusable group back + to 3584-token hit alignment. KpoolTail already opts out of prefix caching (1-block circular scratch). Mamba align-mode state *does* materialize at 896-token chunk ends, and 3584 is a multiple of 896, so mamba must stay in the min — skipping a mamba miss is a correctness hole (vLLM #47491 / #43090). -This patch: flag only exact SlidingWindowSpec groups as EAGLE, and do not -let that drafter group shrink ``curr_hit_length``. If the drafter window -does not cover the MLA/mamba hit, leave its blocks empty so a fresh -window is allocated (zeros / new pages). +This patch: flag only exact SlidingWindowSpec groups as EAGLE, do not let that +drafter group shrink ``curr_hit_length``, and ignore non-participating cache +groups when checking whether fine-grained lookup is safe. If the drafter +window does not cover the MLA/mamba hit, leave its blocks empty so a fresh +window is allocated (zeros / new pages). Wrong indexer tail state is fatal — +we do not change KpoolTailManager or make its transient state shareable. KpoolTail is deliberately per-request scratch and cannot prefix-cache. A resumed request therefore starts with an empty indexer-tail ring. Replaying @@ -45,6 +52,7 @@ ) ) MARK = "# [glm53-hybrid-apc]" +FINE_MARK = "# [glm53-hybrid-apc-fine]" DFLASH_REPLAY_MARK = "# [glm53-dflash-swa-replay-v1]" BASE_HELPER = ''' @@ -190,6 +198,27 @@ def _glm53_dflash_replay_safe_hit( ) """ +FINE_OLD = """ unsupported_partial_hit_managers = { + type(manager).__name__ + for manager in self.single_type_managers + if not manager.supports_fine_grained_hash_lookup + and manager.block_size != hash_block_size + } +""" + +FINE_NEW = """ unsupported_partial_hit_managers = { # [glm53-hybrid-apc-fine] + type(manager).__name__ + for manager, group in zip( + self.single_type_managers, kv_cache_config.kv_cache_groups + ) + # A transient/non-shareable manager cannot constrain prefix-hit + # granularity because it never participates in the lookup. + if group.kv_cache_spec.participates_in_prefix_caching + and not manager.supports_fine_grained_hash_lookup + and manager.block_size != hash_block_size + } +""" + INIT_OLD = """ self.verify_and_split_kv_cache_groups() """ @@ -338,9 +367,17 @@ def main() -> int: text = replace_once( text, CONVERGE_OLD, CONVERGE_FINAL, "dflash-replay-clamp" ) + if FINE_MARK not in text: + text = replace_once( + text, + FINE_OLD, + FINE_NEW, + "fine-grained manager eligibility", + ) P.write_text(text) print( - f"patched {P.name} (hybrid APC + versioned DFlash SWA replay clamp)" + f"patched {P.name} (hybrid APC + DFlash SWA replay clamp + " + f"fine-grained manager eligibility)" ) return 0 diff --git a/tests/bench_prefix_cache.py b/tests/bench_prefix_cache.py index 18e0337e..8c2f93e3 100644 --- a/tests/bench_prefix_cache.py +++ b/tests/bench_prefix_cache.py @@ -18,14 +18,13 @@ measure true colds whether or not the reset route is mounted — instead of silently hitting the previous session's pages. -Page granularity (IMPORTANT on this stack): the sparse-MLA KpoolTailManager -only counts BLOCK-ALIGNED hits at the 3584-token hybrid MLA page. Hits are -therefore `floor(tokens / 3584) * 3584` at best — a 6.8k-token conversation -reuses ONE page (3584 tokens, ~53%), a 7.7k conversation TWO pages (~93%, -the upstream headline). The summary reports `hit_efficiency` = measured hit -tokens / full-page expectation so 1.0 means "the page model is respected". -Size --prompt-tokens so the warm prompt crosses the next 3584 boundary -(default 8400 -> ~8.1k measured tokens -> 2 pages). +Lookup granularity (IMPORTANT on this stack): physical sparse-MLA attention +pages remain 3584 tokens, but patch_hybrid_prefix_hit.py permits the reusable +MLA/Mamba managers to match hashes at the underlying 64-token cache-block +granularity. KpoolTailManager is transient and opts out of prefix caching, so +it must not force reusable groups back to 3584-token matches. The summary +reports `hit_efficiency` = measured hit tokens / the full-block expectation; +1.0 means the fine-grained lookup is working. Usage: python3 tests/bench_prefix_cache.py --runs 3 @@ -58,8 +57,9 @@ # ~4.5 chars per token for the English filler (measured on this checkpoint's # tokenizer); the reported prompt_tokens comes from the server usage block. _CHARS_PER_TOKEN = 4.5 -# Hybrid MLA page: sparse-MLA prefix hits are block-aligned to this size. -DEFAULT_PAGE_TOKENS = 3584 +# Hash/cache block used for fine-grained prefix lookup. The physical hybrid +# attention page remains 3584 tokens; this is lookup, not allocation geometry. +DEFAULT_PAGE_TOKENS = 64 # Unique per invocation: colds must not reuse pages cached by a previous bench # session (observed 2026-08-29: reruns reused chat content and "cold" TTFT # dropped 10.3s -> 1.9s). /reset_prefix_cache is available since #31 when @@ -257,8 +257,8 @@ def run_pair(base_url: str, model: str, api_key: str, chat_index: int, if d_queries > 0: hit_ratio = round(d_hits / d_queries, 4) - # Page-model expectation: hits can only cover full 3584-token pages of the - # warm prompt. efficiency ~1.0 => the reuse mechanism works as designed. + # Lookup-block expectation: the final incomplete 64-token hash block is + # recomputed. efficiency ~1.0 => fine-grained reuse works as designed. warm_tokens = warm.get("prompt_tokens") hit_efficiency = None if hit_ratio is not None and warm_tokens: @@ -293,10 +293,9 @@ def main() -> int: ap.add_argument("--runs", type=int, default=3, help="cold+warm pairs per chat") ap.add_argument("--concurrent", type=int, default=1, help="independent chats in flight") ap.add_argument("--prompt-tokens", type=int, default=8400, - help="approx cold prompt size; keep the warm prompt above the " - "next 3584-token page boundary (e.g. >7168) to see ~90%% hits") + help="approximate cold prompt size") ap.add_argument("--page-tokens", type=int, default=DEFAULT_PAGE_TOKENS, - help="hybrid MLA page size for the hit-efficiency model") + help="prefix lookup block size for the hit-efficiency model") ap.add_argument("--reset-prefix-cache", action=argparse.BooleanOptionalAction, default=True, help="POST /reset_prefix_cache before each cold turn (true colds)") ap.add_argument("--timeout", type=float, default=600.0) diff --git a/tests/test_hybrid_prefix_hit.py b/tests/test_hybrid_prefix_hit.py index aed48c2a..01d0a7ac 100644 --- a/tests/test_hybrid_prefix_hit.py +++ b/tests/test_hybrid_prefix_hit.py @@ -36,14 +36,19 @@ def main() -> int: env["GLM53_KV_COORDINATOR_PY"] = str(dst) subprocess.check_call([sys.executable, str(PATCH)], env=env) text = dst.read_text() + compile(text, str(dst), "exec") assert "[glm53-hybrid-apc]" in text assert text.count("[glm53-hybrid-apc]") >= 3 + assert text.count("[glm53-hybrid-apc-fine]") == 1 assert "def _glm53_is_draft_swa_spec(" in text assert "swa_ids or set(" in text + assert "for manager, group in zip(" in text + assert "group.kv_cache_spec.participates_in_prefix_caching" in text subprocess.check_call([sys.executable, str(PATCH)], env=env) assert dst.read_text().count("[glm53-hybrid-apc]") == text.count( "[glm53-hybrid-apc]" ) + assert dst.read_text().count("[glm53-hybrid-apc-fine]") == 1 print("hybrid prefix-hit patch OK") return 0 From 9f3eb7865b0ffe9b9675d0dda4ad89548b8ce2c1 Mon Sep 17 00:00:00 2001 From: Adam Hillson Date: Tue, 15 Sep 2026 23:55:33 +0100 Subject: [PATCH 2/2] overlay: log why fine-grained lookup stays disabled (diagnostic) --- overlay/patch_hybrid_prefix_hit.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/overlay/patch_hybrid_prefix_hit.py b/overlay/patch_hybrid_prefix_hit.py index 6a4550ae..a7f79c1d 100755 --- a/overlay/patch_hybrid_prefix_hit.py +++ b/overlay/patch_hybrid_prefix_hit.py @@ -217,6 +217,17 @@ def _glm53_dflash_replay_safe_hit( and not manager.supports_fine_grained_hash_lookup and manager.block_size != hash_block_size } + _glm53_fine_unsupported = unsupported_partial_hit_managers + if _glm53_fine_unsupported and not getattr( + self, "_glm53_fine_logged", False + ): + self._glm53_fine_logged = True + logger.warning( + "[glm53-hybrid-apc-fine] fine-grained lookup still disabled " + "by participating managers: %s (hash_block_size=%s)", + sorted(_glm53_fine_unsupported), + hash_block_size, + ) """ INIT_OLD = """ self.verify_and_split_kv_cache_groups()