Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions atom/kv_transfer/offload/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,9 +436,27 @@ def lmcache_replica_world_size(config) -> int:
"shard i of the model", which holds the same bytes in every replica, so
replicas sharing a disk/remote backend share entries instead of
duplicating them.

Read the sizes from the top level first and fall back to
``config.parallel_config``: ATOM's own config carries them directly, while
the vLLM plugin path hands us a ``VllmConfig`` that keeps both under
``parallel_config``. Looking only at the top level made both lookups fall
back to 1 there, so a TP4 replica reported world=1 -- which puts the
recipe's all-rank ``lookup_server_worker_ids=[0,1,2,3]`` out of range, and
the lookup client then fails to build.
"""
pp_size = int(getattr(config, "pipeline_parallel_size", 1) or 1)
tp_size = int(getattr(config, "tensor_parallel_size", 1) or 1)

def _dim(name: str) -> int:
for src in (config, getattr(config, "parallel_config", None)):
if src is None:
continue
val = getattr(src, name, None)
if val:
return int(val)
return 1

pp_size = _dim("pipeline_parallel_size")
tp_size = _dim("tensor_parallel_size")
return max(1, pp_size * tp_size)


Expand Down
28 changes: 28 additions & 0 deletions tests/test_lmcache_offload_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -5240,6 +5240,34 @@ def test_replica_world_size_counts_pp_and_tp_but_not_dp():
assert offcfg.lmcache_replica_world_size(SimpleNamespace()) == 1


def test_replica_world_size_reads_vllm_parallel_config():
# On the plugin path `config` is a VllmConfig, which keeps the parallel
# sizes under `parallel_config` and carries nothing at the top level.
# Reading only the top level returned world=1 for a TP4 replica, so the
# all-rank lookup_server_worker_ids=[0,1,2,3] went out of range and the
# lookup client silently failed to build -- leaving the offload tier
# written but never read.
vllm_shaped = SimpleNamespace(
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=4,
)
)
assert offcfg.lmcache_replica_world_size(vllm_shaped) == 4

vllm_shaped_pp = SimpleNamespace(
parallel_config=SimpleNamespace(
pipeline_parallel_size=2,
tensor_parallel_size=4,
)
)
assert offcfg.lmcache_replica_world_size(vllm_shaped_pp) == 8

# A parallel_config that only carries ranks (ATOM's own shape) must not
# shadow the top-level sizes.
assert offcfg.lmcache_replica_world_size(_dp_config(5, pp_size=4, tp_size=8)) == 32


class _RecordingRunnerMgr:
def __init__(self) -> None:
self.calls: list[tuple[str, object]] = []
Expand Down
Loading