From 03c694493be972c085e42309103582e64476bc84 Mon Sep 17 00:00:00 2001 From: perzhang Date: Thu, 17 Sep 2026 04:36:17 -0500 Subject: [PATCH] fix(offload): read PP/TP from parallel_config for LMCache world size lmcache_replica_world_size() read pipeline_parallel_size and tensor_parallel_size off the top level of the config object. That is correct for ATOM's own config, which carries them directly, but on the vLLM plugin path `config` is a VllmConfig and both live under config.parallel_config. Both getattr calls fell back to 1, so a TP4 replica reported world=1. world=1 then puts the all-rank lookup scope out of range. With LMCACHE_LOOKUP_SERVER_WORKER_IDS=0,1,2,3 the four per-rank lookup servers start normally, but the scheduler-side client fails to build and dense/connector.py swallows it with a bare `except Exception`, continuing with lookup_client=None. The server comes up healthy and nothing in the run looks wrong, yet ChunkedOffloadSchedulerBase can never learn lmcache_cached_tokens, never builds a LoadSpec, and _reqs_need_recv stays empty: the offload tier is written and never read. Measured on MiniMax-M3 TP4, plugin backend, concurrency 40, 1 h window: Retrieved 0 (native, same config: 1760) Double unpin 0 (native: 878384) Stored 3508 -- the save path is fine vllm:external_prefix_cache_hits_total 0.0 vllm:external_prefix_cache_queries_total 1.3e+07 i.e. an "LMCache on" arm paying the full store cost for zero reuse. The same build on the ATOM-native backend logs (world=4) and retrieves normally, which is the direct proof that the config shape is what differs. This stayed hidden because the default lookup scope is [0], which is in range even when world is wrongly 1 -- so arms that left LMCACHE_LOOKUP_SERVER_WORKER_IDS unset built a rank-0 client and did retrieve. It only fires in the "plugin backend + all-rank lookup" cell, which is what the MiniMax-M3 agentic recipe prescribes at 48c. The fix keeps reading the top level first and only falls back to parallel_config, so ATOM's native path is bit-for-bit unchanged. Three call sites are affected: dense, hybrid/dsv4 and mp/backend. After the fix, the same arm logs (world=4) and the three discriminators go non-zero: Retrieved 4 / Double unpin 476 / external_prefix_cache_hits_total 30464.0, the last matching the first "Retrieved 30464 out of 30464 required tokens" log line exactly. Independently confirmed on a second model and a second connector. GLM-5.2-MXFP4 TP4, vLLM plugin backend with ATOM's own AtomLMCacheOffloadConnector, AgentX replay at concurrency 32, 64 GiB/rank tier, LMCACHE_LOOKUP_SERVER_WORKER_IDS=0,1,2,3: [atom] LMCache offload scheduler: lookup client on atom-offload-dp0 (world=4) which is the line the old code printed as (world=1). Over the run the tier was read on all four ranks -- 50,048 tokens / 2.22 GiB per rank at 7.0-12.1 GB/s, and vllm:external_prefix_cache_hits_total ended at 50,048, matching the Retrieved log lines exactly. That arm was run with the fix already cherry-picked, so it is confirmation that world=4 is reached on a second config shape, not a second before/after comparison. Co-Authored-By: Claude Opus 5 --- atom/kv_transfer/offload/config.py | 22 +++++++++++++++++-- tests/test_lmcache_offload_connector.py | 28 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/atom/kv_transfer/offload/config.py b/atom/kv_transfer/offload/config.py index 2ccdc80197..86112ee341 100644 --- a/atom/kv_transfer/offload/config.py +++ b/atom/kv_transfer/offload/config.py @@ -436,9 +436,27 @@ def lmcache_replica_world_size(config) -> int: "shard i of the model", which holds the same bytes in every replica, so replicas sharing a disk/remote backend share entries instead of duplicating them. + + Read the sizes from the top level first and fall back to + ``config.parallel_config``: ATOM's own config carries them directly, while + the vLLM plugin path hands us a ``VllmConfig`` that keeps both under + ``parallel_config``. Looking only at the top level made both lookups fall + back to 1 there, so a TP4 replica reported world=1 -- which puts the + recipe's all-rank ``lookup_server_worker_ids=[0,1,2,3]`` out of range, and + the lookup client then fails to build. """ - pp_size = int(getattr(config, "pipeline_parallel_size", 1) or 1) - tp_size = int(getattr(config, "tensor_parallel_size", 1) or 1) + + def _dim(name: str) -> int: + for src in (config, getattr(config, "parallel_config", None)): + if src is None: + continue + val = getattr(src, name, None) + if val: + return int(val) + return 1 + + pp_size = _dim("pipeline_parallel_size") + tp_size = _dim("tensor_parallel_size") return max(1, pp_size * tp_size) diff --git a/tests/test_lmcache_offload_connector.py b/tests/test_lmcache_offload_connector.py index b051b99403..4a46881f31 100644 --- a/tests/test_lmcache_offload_connector.py +++ b/tests/test_lmcache_offload_connector.py @@ -5240,6 +5240,34 @@ def test_replica_world_size_counts_pp_and_tp_but_not_dp(): assert offcfg.lmcache_replica_world_size(SimpleNamespace()) == 1 +def test_replica_world_size_reads_vllm_parallel_config(): + # On the plugin path `config` is a VllmConfig, which keeps the parallel + # sizes under `parallel_config` and carries nothing at the top level. + # Reading only the top level returned world=1 for a TP4 replica, so the + # all-rank lookup_server_worker_ids=[0,1,2,3] went out of range and the + # lookup client silently failed to build -- leaving the offload tier + # written but never read. + vllm_shaped = SimpleNamespace( + parallel_config=SimpleNamespace( + pipeline_parallel_size=1, + tensor_parallel_size=4, + ) + ) + assert offcfg.lmcache_replica_world_size(vllm_shaped) == 4 + + vllm_shaped_pp = SimpleNamespace( + parallel_config=SimpleNamespace( + pipeline_parallel_size=2, + tensor_parallel_size=4, + ) + ) + assert offcfg.lmcache_replica_world_size(vllm_shaped_pp) == 8 + + # A parallel_config that only carries ranks (ATOM's own shape) must not + # shadow the top-level sizes. + assert offcfg.lmcache_replica_world_size(_dp_config(5, pp_size=4, tp_size=8)) == 32 + + class _RecordingRunnerMgr: def __init__(self) -> None: self.calls: list[tuple[str, object]] = []