diff --git a/atom/kv_transfer/offload/config.py b/atom/kv_transfer/offload/config.py index 2ccdc80197..86112ee341 100644 --- a/atom/kv_transfer/offload/config.py +++ b/atom/kv_transfer/offload/config.py @@ -436,9 +436,27 @@ def lmcache_replica_world_size(config) -> int: "shard i of the model", which holds the same bytes in every replica, so replicas sharing a disk/remote backend share entries instead of duplicating them. + + Read the sizes from the top level first and fall back to + ``config.parallel_config``: ATOM's own config carries them directly, while + the vLLM plugin path hands us a ``VllmConfig`` that keeps both under + ``parallel_config``. Looking only at the top level made both lookups fall + back to 1 there, so a TP4 replica reported world=1 -- which puts the + recipe's all-rank ``lookup_server_worker_ids=[0,1,2,3]`` out of range, and + the lookup client then fails to build. """ - pp_size = int(getattr(config, "pipeline_parallel_size", 1) or 1) - tp_size = int(getattr(config, "tensor_parallel_size", 1) or 1) + + def _dim(name: str) -> int: + for src in (config, getattr(config, "parallel_config", None)): + if src is None: + continue + val = getattr(src, name, None) + if val: + return int(val) + return 1 + + pp_size = _dim("pipeline_parallel_size") + tp_size = _dim("tensor_parallel_size") return max(1, pp_size * tp_size) diff --git a/tests/test_lmcache_offload_connector.py b/tests/test_lmcache_offload_connector.py index b051b99403..4a46881f31 100644 --- a/tests/test_lmcache_offload_connector.py +++ b/tests/test_lmcache_offload_connector.py @@ -5240,6 +5240,34 @@ def test_replica_world_size_counts_pp_and_tp_but_not_dp(): assert offcfg.lmcache_replica_world_size(SimpleNamespace()) == 1 +def test_replica_world_size_reads_vllm_parallel_config(): + # On the plugin path `config` is a VllmConfig, which keeps the parallel + # sizes under `parallel_config` and carries nothing at the top level. + # Reading only the top level returned world=1 for a TP4 replica, so the + # all-rank lookup_server_worker_ids=[0,1,2,3] went out of range and the + # lookup client silently failed to build -- leaving the offload tier + # written but never read. + vllm_shaped = SimpleNamespace( + parallel_config=SimpleNamespace( + pipeline_parallel_size=1, + tensor_parallel_size=4, + ) + ) + assert offcfg.lmcache_replica_world_size(vllm_shaped) == 4 + + vllm_shaped_pp = SimpleNamespace( + parallel_config=SimpleNamespace( + pipeline_parallel_size=2, + tensor_parallel_size=4, + ) + ) + assert offcfg.lmcache_replica_world_size(vllm_shaped_pp) == 8 + + # A parallel_config that only carries ranks (ATOM's own shape) must not + # shadow the top-level sizes. + assert offcfg.lmcache_replica_world_size(_dp_config(5, pp_size=4, tp_size=8)) == 32 + + class _RecordingRunnerMgr: def __init__(self) -> None: self.calls: list[tuple[str, object]] = []