From e49bff7bea3f40710c29429d10f13469a4b0fce2 Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Mon, 28 Sep 2026 05:04:05 +0000 Subject: [PATCH 1/7] Fix native K3 PD transfer lifetimes and stream finalization Co-authored-by: Yichao Zhu Co-authored-by: Codex Signed-off-by: whx-sjtu --- .../test_kimi_k3_reasoning_parser.py | 140 ++++++++++ tests/v1/core/test_contiguous_kv_packing.py | 34 +++ tests/v1/core/test_scheduler.py | 38 +++ .../unit/test_moriio_proxy_routing.py | 96 +++++++ .../kv_connector/unit/test_moriio_routing.py | 33 +++ .../kv_connector/unit/test_moriio_tp_ack.py | 243 ++++++++++++++++++ .../kv_connector/unit/test_multi_connector.py | 203 ++++++++++++++- tests/v1/worker/test_gpu_model_runner_v2.py | 116 +++++++++ .../kv_transfer/kv_connector/v1/base.py | 18 ++ .../kv_connector/v1/moriio/moriio_common.py | 4 +- .../v1/moriio/moriio_connector.py | 220 +++++++++------- .../v1/moriio/moriio_heartbeat.py | 86 +++++++ .../kv_connector/v1/multi_connector.py | 147 ++++++++--- vllm/parser/kimi_k3.py | 34 ++- vllm/reasoning/kimi_k3_reasoning_parser.py | 18 +- vllm/tool_parsers/kimi_k3_tool_parser.py | 6 +- vllm/v1/core/sched/scheduler.py | 7 +- vllm/v1/worker/gpu/kv_connector.py | 8 + vllm/v1/worker/gpu/model_runner.py | 18 +- vllm/v1/worker/utils.py | 15 +- 20 files changed, 1332 insertions(+), 152 deletions(-) create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py diff --git a/tests/reasoning/test_kimi_k3_reasoning_parser.py b/tests/reasoning/test_kimi_k3_reasoning_parser.py index 8dcac4a83aa9..b6c2570ad715 100644 --- a/tests/reasoning/test_kimi_k3_reasoning_parser.py +++ b/tests/reasoning/test_kimi_k3_reasoning_parser.py @@ -13,6 +13,7 @@ from vllm.parser.parser_manager import ParserManager from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser pytestmark = pytest.mark.skip_global_cleanup @@ -50,6 +51,10 @@ class ReasoningOnlyParser(KimiK3Parser): reasoning_parser_cls = KimiK3ReasoningParser +class ReasoningAndToolParser(ReasoningOnlyParser): + tool_parser_cls = KimiK3ToolParser + + def test_parser_manager_selects_kimi_k3_parser_for_reasoning_only(): parser_cls = ParserManager.get_parser(reasoning_parser_name="kimi_k3") @@ -301,6 +306,141 @@ def test_delegating_parser_thinking_false_streams_response_content(): assert closed is None +@pytest.mark.parametrize("suffix", ["<", OPEN, f"{CLOSE}think"]) +@pytest.mark.parametrize("prefix", ["", "value "]) +@pytest.mark.parametrize("empty_final", [False, True]) +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +def test_streaming_eof_preserves_unfinished_reasoning_marker( + suffix, prefix, empty_final, parser_cls +): + parser = parser_cls(_dummy_tokenizer()) + request = ChatCompletionRequest(model="test-model", messages=[]) + text = prefix + suffix + chunks = [text, ""] if empty_final else [text] + deltas = [ + parser.parse_delta( + chunk, + parser.model_tokenizer.encode(chunk), + request, + prompt_token_ids=OPEN_IDS, + finished=index == len(chunks) - 1, + ) + for index, chunk in enumerate(chunks) + ] + + assert "".join(delta.reasoning or "" for delta in deltas if delta) == text + assert not any(delta.content or delta.tool_calls for delta in deltas if delta) + + +@pytest.mark.parametrize("thinking", [False, True]) +@pytest.mark.parametrize("suffix", ["<", f"{CLOSE}response"]) +@pytest.mark.parametrize("prefix", ["", "value "]) +@pytest.mark.parametrize("empty_final", [False, True]) +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +def test_streaming_eof_preserves_unfinished_content_marker( + thinking, suffix, prefix, empty_final, parser_cls +): + parser = parser_cls(_dummy_tokenizer(), chat_template_kwargs={"thinking": thinking}) + request = ChatCompletionRequest(model="test-model", messages=[]) + chunks = ["step", THINK_CLOSE] if thinking else [] + chunks += [RESPONSE_OPEN, prefix + suffix] + if empty_final: + chunks.append("") + deltas = [ + parser.parse_delta( + chunk, + parser.model_tokenizer.encode(chunk), + request, + prompt_token_ids=OPEN_IDS, + finished=index == len(chunks) - 1, + ) + for index, chunk in enumerate(chunks) + ] + + assert "".join(delta.content or "" for delta in deltas if delta) == prefix + suffix + assert "".join(delta.reasoning or "" for delta in deltas if delta) == ( + "step" if thinking else "" + ) + assert not any(delta.tool_calls for delta in deltas if delta) + + +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +def test_streaming_eof_does_not_expose_suppressed_reasoning(parser_cls): + parser = parser_cls(_dummy_tokenizer()) + request = ChatCompletionRequest( + model="test-model", messages=[], include_reasoning=False + ) + + assert ( + parser.parse_delta( + "value <", [9], request, prompt_token_ids=OPEN_IDS, finished=True + ) + is None + ) + + +def test_streaming_eof_does_not_expose_other_parser_reasoning(): + class Tokenizer(DummyTokenizer): + def get_vocab(self): + return {"": 1000, "": 1001} + + def encode(self, text, add_special_tokens=False): + if text in self.get_vocab(): + return [self.get_vocab()[text]] + return super().encode(text, add_special_tokens=add_special_tokens) + + parser_cls = ParserManager.get_parser( + reasoning_parser_name="deepseek_r1", + tool_parser_name="kimi_k3", + enable_auto_tools=True, + ) + parser = parser_cls(Tokenizer()) + request = ChatCompletionRequest( + model="test-model", messages=[], include_reasoning=False + ) + + assert ( + parser.parse_delta( + "internal thought <", [9], request, prompt_token_ids=[1000], finished=True + ) + is None + ) + + +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +@pytest.mark.parametrize("empty_final", [False, True]) +def test_streaming_eof_strips_complete_markers_after_reasoning(parser_cls, empty_final): + parser = parser_cls(_dummy_tokenizer()) + request = ChatCompletionRequest(model="test-model", messages=[]) + # A speculative step can close reasoning and complete the response together. + chunks = [ + (THINK_OPEN + "step", [*OPEN_IDS, 9]), + ( + THINK_CLOSE + + RESPONSE_OPEN + + "answer" + + f"{CLOSE}response{SEP}{CLOSE}message{SEP}", + [*CLOSE_IDS, 10], + ), + ] + if empty_final: + chunks.append(("", [])) + deltas = [ + parser.parse_delta( + chunk, + token_ids, + request, + prompt_token_ids=OPEN_IDS, + finished=index == len(chunks) - 1, + ) + for index, (chunk, token_ids) in enumerate(chunks) + ] + + assert "".join(delta.reasoning or "" for delta in deltas if delta) == "step" + assert "".join(delta.content or "" for delta in deltas if delta) == "answer" + assert not any(delta.tool_calls for delta in deltas if delta) + + def test_adjust_request_keeps_xtml_markers_contiguous(): parser = KimiK3ReasoningParser(DummyTokenizer()) request = ChatCompletionRequest(model="test-model", messages=[]) diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 638deeedc579..373ca5de266b 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -16,6 +16,7 @@ import pytest import torch +import vllm.v1.worker.utils as worker_utils from vllm.config import CacheConfig from vllm.v1.core.kv_cache_manager import KVCacheManager from vllm.v1.core.kv_cache_utils import ( @@ -125,6 +126,39 @@ def _bind(config, layout: str): return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None) +@pytest.mark.parametrize("is_rocm", [False, True]) +def test_shared_allocation_alignment_preserves_layer_views(monkeypatch, is_rocm): + """ROCm RDMA needs an aligned view even with a misaligned allocator base.""" + groups = [KVCacheGroupSpec(["a", "b"], _mla(128))] + config = get_kv_cache_config_from_groups(_mock_vllm_config("LBNHC"), groups, MEMORY) + monkeypatch.setattr(worker_utils.current_platform, "is_rocm", lambda: is_rocm) + monkeypatch.setattr( + worker_utils, "warmup_rocm_skinny_gemm_workspaces", lambda _: None + ) + zeros = torch.zeros + + def misaligned_zeros(size, **kwargs): + allocation = zeros(size + 4096, **kwargs) + offset = (256 - allocation.data_ptr()) % 4096 + return allocation.narrow(0, offset, size) + + if is_rocm: + monkeypatch.setattr(worker_utils.torch, "zeros", misaligned_zeros) + caches = _bind(config, "LBNHC") + a, b = caches["a"], caches["b"] + assert a.untyped_storage().data_ptr() == b.untyped_storage().data_ptr() + assert b.data_ptr() - a.data_ptr() == config.kv_cache_tensors[0].layer_stride + a.fill_(1) + b.fill_(2) + assert torch.all(a == 1) and torch.all(b == 2) + if is_rocm: + assert a.data_ptr() % 4096 == 0 + assert a.storage_offset() > 0 + else: + assert a.storage_offset() == 0 + assert a.untyped_storage().nbytes() == config.kv_cache_tensors[0].size + + MAIN_KV_PAGE_BYTES = 2_048 COMPRESSED_PAGE_BYTES = 128 NUM_CACHE_TUPLES = 3 diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 722cbfb79e00..d9e27783a2b0 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -2566,6 +2566,44 @@ def test_has_sync_kv_loads( assert output.has_sync_kv_loads is expected_has_sync_loads +@pytest.mark.parametrize("overwrite_full_blocks", [False, True]) +@pytest.mark.skip_global_cleanup +def test_sync_kv_load_only_skips_zeroing_overwritten_blocks( + overwrite_full_blocks: bool, tmp_path +): + """A synchronous RDMA load must not race zeroing of its destinations.""" + (tmp_path / "config.json").write_text( + '{"architectures": ["OPTForCausalLM"], "model_type": "opt"}' + ) + scheduler = create_scheduler( + model=str(tmp_path), + skip_tokenizer_init=True, + use_kv_connector=mock_kv(matched_tokens=16, is_async=False), + block_size=16, + ) + scheduler.needs_kv_cache_zeroing = True + # Full-attention pages also require zeroing in a mixed-precision pool. + manager = scheduler.kv_cache_manager.coordinator.single_type_managers[0] + manager._record_new_block_ids = True + request = create_requests(num_requests=1, num_tokens=48, block_size=16)[0] + scheduler.add_request(request) + + def loaded_blocks(request): + blocks = scheduler.kv_cache_manager.get_blocks(request.request_id) + return list(blocks.get_block_ids()[0][:1]) + + if overwrite_full_blocks: + scheduler.connector.get_sync_load_block_ids = loaded_blocks + + output = scheduler.schedule() + blocks = list( + scheduler.kv_cache_manager.get_blocks(request.request_id).get_block_ids()[0] + ) + expected = blocks[1:] if overwrite_full_blocks else blocks + assert output.new_block_ids_to_zero == expected + assert not scheduler._skip_zero_block_ids + + def test_kv_connector_honors_skip_reading_prefix_cache(): """A request that must score every prompt row takes no external hit.""" BLOCK_SIZE = 16 diff --git a/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py b/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py index 71ee0ef5d799..68066ede1c3d 100644 --- a/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py +++ b/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py @@ -22,14 +22,20 @@ """ import contextlib +import ctypes import importlib.util +import os +import subprocess import sys +import time import types from collections import Counter from pathlib import Path from typing import cast +import msgpack import pytest +import zmq _MISSING = object() @@ -203,3 +209,93 @@ def test_consecutive_requests_alternate_instances(route): # than filling one instance's ranks before moving on. insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)] assert insts == [0, 1, 2] + + +@pytest.mark.skipif(sys.platform != "linux", reason="GIL probe uses libc.usleep") +def test_discovery_heartbeat_progresses_while_worker_holds_gil(): + from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( + MoRIIOHeartbeat, + ) + + payload = { + "type": "P", + "http_address": "127.0.0.1:8000", + "zmq_address": "host:127.0.0.1,handshake:6301,notify:61005", + "dp_size": 1, + "tp_size": 8, + "transfer_mode": "READ", + } + with zmq.Context() as context, context.socket(zmq.ROUTER) as socket: + port = socket.bind_to_random_port("tcp://127.0.0.1") + heartbeat = MoRIIOHeartbeat(f"tcp://127.0.0.1:{port}", payload, 0.02) + try: + assert socket.poll(5000), "Discovery did not register" + assert msgpack.unpackb(socket.recv_multipart()[1]) == payload + # PyDLL retains the GIL throughout the native call. + ctypes.PyDLL(None).usleep(300_000) + received = 0 + while socket.poll(0): + assert msgpack.unpackb(socket.recv_multipart()[1]) == payload + received += 1 + assert received >= 3, "Discovery stalled while the worker held the GIL" + finally: + heartbeat.shutdown() + heartbeat.shutdown() + while socket.poll(0): + socket.recv_multipart() + assert not socket.poll(100), "Heartbeats continued after shutdown" + + +@pytest.mark.skipif(sys.platform != "linux", reason="Checks child exit via procfs") +def test_discovery_heartbeat_exits_when_worker_dies(): + from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_heartbeat + + owner_code = """ +import runpy +import sys +Heartbeat = runpy.run_path(sys.argv[1])["MoRIIOHeartbeat"] +heartbeat = Heartbeat(sys.argv[2], {"type": "D"}, 0.02) +print(heartbeat._process.pid, flush=True) +sys.stdin.read() +""" + with zmq.Context() as context, context.socket(zmq.ROUTER) as socket: + port = socket.bind_to_random_port("tcp://127.0.0.1") + owner = subprocess.Popen( + [ + sys.executable, + "-c", + owner_code, + moriio_heartbeat.__file__, + f"tcp://127.0.0.1:{port}", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + child_pid = None + child_exited = False + try: + assert socket.poll(5000), "Discovery did not register" + assert owner.stdout is not None + child_pid = int(owner.stdout.readline()) + owner.kill() + owner.wait(timeout=5) + stat = Path(f"/proc/{child_pid}/stat") + deadline = time.monotonic() + 5 + while stat.exists(): + # A terminated child may await reaping by the container's init. + try: + if stat.read_text().split(") ")[1].startswith("Z "): + break + except FileNotFoundError: + break + assert time.monotonic() < deadline, "Orphan heartbeat survived worker" + time.sleep(0.01) + child_exited = True + finally: + if owner.poll() is None: + owner.kill() + owner.communicate(timeout=5) + if child_pid is not None and not child_exited: + with contextlib.suppress(ProcessLookupError): + os.kill(child_pid, 9) diff --git a/tests/v1/kv_connector/unit/test_moriio_routing.py b/tests/v1/kv_connector/unit/test_moriio_routing.py index b0ac6df3cba3..eb9d70d48989 100644 --- a/tests/v1/kv_connector/unit/test_moriio_routing.py +++ b/tests/v1/kv_connector/unit/test_moriio_routing.py @@ -14,6 +14,10 @@ from types import SimpleNamespace +import pytest + +from vllm.config import KVTransferConfig +from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_common from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( MoRIIOConnectorScheduler, ) @@ -23,6 +27,35 @@ _clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks +@pytest.mark.parametrize("backend, expected_port", [("rdma", 0), ("xgmi", 12345)]) +def test_rdma_listener_reserves_port_during_bind(monkeypatch, backend, expected_port): + monkeypatch.setattr(moriio_common, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + moriio_common, "get_tensor_model_parallel_world_size", lambda: 8 + ) + monkeypatch.setattr(moriio_common, "get_open_port", lambda: 12345) + monkeypatch.setattr(moriio_common, "resolve_host_ip", lambda _: "127.0.0.1") + config = SimpleNamespace( + kv_transfer_config=KVTransferConfig( + kv_connector="MoRIIOConnector", + kv_role="kv_producer", + kv_connector_extra_config={ + "backend": backend, + "http_port": 8000, + "handshake_port": 6300, + "notify_port": 7300, + }, + ), + parallel_config=SimpleNamespace( + data_parallel_rank=0, data_parallel_size=1, data_parallel_size_local=1 + ), + ) + assert ( + moriio_common.MoRIIOConfig.from_vllm_config(config).local_kv_port + == expected_port + ) + + def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int): """Minimal prefill-side (producer) scheduler stand-in for request_finished.""" s = SimpleNamespace( diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py index a35f0a1169b6..efe1fe9e12f3 100644 --- a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py +++ b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import pytest +import torch from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOMode, @@ -13,6 +14,7 @@ ) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( MoRIIOConnector, + MoRIIOConnectorScheduler, MoRIIOConnectorWorker, get_moriio_expected_ack_count, get_moriio_remote_tp_rank, @@ -22,6 +24,173 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( MoRIIOWrapper, ) +from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( + MambaConvSplitInfo, +) +from vllm.v1.kv_cache_interface import MambaSpec + + +@pytest.mark.parametrize( + "params,expected", + [ + (None, (0, False)), + ({}, (0, False)), + ({"do_remote_prefill": False}, (0, False)), + ({"do_remote_prefill": True}, (31, False)), + ], +) +def test_read_matches_only_requests_with_pending_remote_prefill(params, expected): + scheduler = MoRIIOConnectorScheduler.__new__(MoRIIOConnectorScheduler) + scheduler.is_producer = False + scheduler.mode = MoRIIOMode.READ + request = SimpleNamespace(num_prompt_tokens=32, kv_transfer_params=params) + + assert scheduler.get_num_new_matched_tokens(request, 0) == expected + + +@pytest.mark.parametrize( + "mode,consumer,has_mamba,layout,pending,expected", + [ + (MoRIIOMode.READ, True, True, "LBNHC", [[7, 8], [90]], [7, 8]), + (MoRIIOMode.READ, True, True, "LBNHC", [[], [90]], []), + (MoRIIOMode.READ, True, True, "LBNHC", None, []), + (MoRIIOMode.READ, False, True, "LBNHC", [[7], [90]], []), + (MoRIIOMode.READ, True, False, "LBNHC", [[7]], []), + (MoRIIOMode.READ, True, True, "NBLHC", [[7], [90]], []), + (MoRIIOMode.WRITE, True, True, "LBNHC", [[7], [90]], []), + ], +) +def test_sync_read_initializes_only_supported_attention_destinations( + mode, consumer, has_mamba, layout, pending, expected +): + connector = MoRIIOConnector.__new__(MoRIIOConnector) + connector.mode = mode + connector.kv_transfer_config = SimpleNamespace(is_kv_consumer=consumer) + connector._vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + get_resolved_kv_cache_layout=lambda: SimpleNamespace(name=layout) + ) + ) + request = SimpleNamespace(request_id="req") + connector.connector_scheduler = SimpleNamespace( + _has_mamba=has_mamba, + _reqs_need_recv={} if pending is None else {"req": (request, pending)}, + ) + + assert connector.get_sync_load_block_ids(request) == expected + + +@pytest.mark.parametrize("mode", [MoRIIOMode.READ, MoRIIOMode.WRITE]) +def test_read_requires_completion_of_draft_kv_writes(mode): + connector = MoRIIOConnector.__new__(MoRIIOConnector) + connector.mode = mode + + assert connector.requires_full_step_completion is (mode == MoRIIOMode.READ) + + +def _unaligned_cpu_backing(): + raw = bytearray(12288) + original = torch.frombuffer(raw, dtype=torch.uint8) + offset = (128 - original.data_ptr()) % 4096 + return torch.frombuffer(raw, dtype=torch.uint8, offset=offset, count=8192) + + +def _mamba_registration_worker(caches): + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.kv_caches = caches + spec = MambaSpec( + block_size=16, + shapes=((4, 1), (2, 2)), + dtypes=(torch.float32, torch.float32), + ) + worker.layer_to_spec = dict.fromkeys(caches, spec) + return worker + + +def test_shared_registration_covers_final_ssm_with_unaligned_first_enabled_layer(): + backing = _unaligned_cpu_backing() + # Earlier layers can be transfer-disabled. The first enabled layer need + # not begin at the aligned start of the common allocation. + cache = backing[-64:].view(2, 1, 1, 32) + worker = _mamba_registration_worker({"kda": cache}) + + registration, offsets = worker._build_shared_kv_mr(worker.kv_caches) + + expected_base = cache.data_ptr() // 4096 * 4096 + assert registration.data_ptr() == expected_base + assert registration.data_ptr() >= backing.data_ptr() + assert registration.data_ptr() + registration.numel() == ( + backing.data_ptr() + backing.numel() + ) + assert registration.untyped_storage().data_ptr() == backing.data_ptr() + assert offsets == {"kda": cache.data_ptr() - expected_base} + + +def test_shared_registration_refuses_alignment_before_storage(): + cache = _unaligned_cpu_backing()[:64].view(2, 1, 1, 32) + worker = _mamba_registration_worker({"kda": cache}) + + with pytest.raises(ValueError, match="outside.*storage"): + worker._build_shared_kv_mr(worker.kv_caches) + + +def test_shared_registration_refuses_regions_from_other_storage(): + backing = _unaligned_cpu_backing() + other = _unaligned_cpu_backing() + worker = _mamba_registration_worker( + { + "first": backing[-64:].view(2, 1, 1, 32), + "other": other[-64:].view(2, 1, 1, 32), + } + ) + + with pytest.raises(ValueError, match="same storage"): + worker._build_shared_kv_mr(worker.kv_caches) + + +def test_mamba_reads_address_each_region_relative_to_shared_registration(monkeypatch): + cache = torch.zeros((2, 1, 1, 32), dtype=torch.uint8) + worker = _mamba_registration_worker({"kda": cache}) + worker.world_size = 1 + worker._conv_decomp = MambaConvSplitInfo(1, (1, 1, 2), 4, (16, 16)) + worker._mamba_offset_templates = {} + worker.kv_region_mr_offsets = {"kda": [128, 144]} + worker.layer_base_addr_index = {"kda": 0} + worker.layer_name_to_remote_kv_cache_metadata = {"peer": {"kda": [4096, 4096]}} + worker.moriio_wrapper = MoRIIOWrapper( + moriio_engine=SimpleNamespace(allocate_transfer_uid=lambda: 1) + ) + worker.moriio_wrapper.local_memory_registered = True + monkeypatch.setattr( + worker.moriio_wrapper, + "get_unpack_memory_metadata", + lambda address: SimpleNamespace(data=address), + ) + posted = [] + + class Session: + def batch_read(self, local, remote, sizes, uid): + posted.append((local, remote, sizes)) + return SimpleNamespace(Failed=lambda: False) + + statuses = worker._post_mamba_reads( + "kda", + [Session(), Session()], + [0, 1], + [1], + [0], + 1, + SimpleNamespace(kv_caches_base_addr=[4352, 4368]), + "peer", + "req", + float("inf"), + ) + + assert len(statuses) == 2 + assert posted == [ + ([160, 164, 168], [256, 260, 264], [4, 4, 8]), + ([176], [272], [16]), + ] def test_remote_tp_rank_same_tp_maps_to_self(): @@ -432,6 +601,80 @@ def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid assert worker.get_block_ids_with_load_errors() == expected_invalid +@pytest.mark.parametrize( + "states,elapsed,has_mamba,error", + [ + (["failed", "pending"], 1, True, "failed"), + (["pending", "failed"], 1, True, "failed"), + (["pending"], 121, True, "timed out"), + (["done", "done"], 121, True, None), + (["pending"], 120, True, None), + (["failed", "pending"], 1, False, None), + ], +) +def test_read_completion_preserves_hybrid_blocks_on_error( + monkeypatch, states, elapsed, has_mamba, error +): + class Status: + def __init__(self, state): + self.state = state + + def Succeeded(self): + return self.state == "done" + + def Failed(self): + return self.state == "failed" + + def Message(self): + return self.state + + def Code(self): + return self.state + + monkeypatch.setattr( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.time.monotonic", + lambda: 1000.0, + ) + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.is_producer = False + worker.mode = MoRIIOMode.READ + worker.world_size = 8 + worker._has_mamba = has_mamba + worker.moriio_config = SimpleNamespace(recv_abort_timeout=120.0) + worker.moriio_wrapper = MoRIIOWrapper() + notifications = [] + monkeypatch.setattr( + worker.moriio_wrapper, + "send_notify", + lambda *args, **kwargs: notifications.append((args, kwargs)), + ) + worker._recving_transfers = {"req": {"layer": [Status(state) for state in states]}} + worker._recving_transfers_callback_addr = {"req": ("host", "7000", "tx")} + worker._recving_transfers_start = {"req": 1000.0 - elapsed} + worker._recving_local_blocks = {"req": [7, 8]} + worker._invalid_block_ids = set() + + if error is not None: + with pytest.raises(TransferError, match=error): + worker.get_finished() + assert notifications == [] + assert "req" in worker._recving_transfers + assert worker._recving_transfers_callback_addr == { + "req": ("host", "7000", "tx") + } + assert worker._recving_transfers_start == {"req": 1000.0 - elapsed} + assert worker._recving_local_blocks == {"req": [7, 8]} + assert worker.get_block_ids_with_load_errors() == set() + else: + assert worker.get_finished() == (set(), set()) + completed = states == ["done", "done"] or not has_mamba + assert len(notifications) == int(completed) + assert bool(worker._recving_transfers) is not completed + assert worker.get_block_ids_with_load_errors() == ( + set() if has_mamba else {7, 8} + ) + + def test_hybrid_step_barrier_fails_closed(monkeypatch): class FailingWrapper: def waiting_for_transfer_complete(self, _statuses): diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 9d04a2ea537a..2602ac07421b 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -4,6 +4,7 @@ import shutil import tempfile from pathlib import Path +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock @@ -14,9 +15,11 @@ from vllm import LLM, SamplingParams from vllm.config import KVTransferConfig from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory +from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, + KVConnectorTransferResults, SupportsHMA, supports_hma, ) @@ -34,7 +37,11 @@ NixlKVConnectorStats, ) from vllm.v1.kv_cache_interface import KVCacheConfig -from vllm.v1.outputs import KVConnectorOutput, KVConnectorWorkerMetadata +from vllm.v1.outputs import ( + KVConnectorOutput, + KVConnectorWorkerMetadata, + ModelRunnerOutput, +) MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct" @@ -202,6 +209,200 @@ def mc() -> MultiConnector: return mc +@pytest.fixture +def async_saves(mc): + """Two save owners whose two worker ranks may finish in different steps.""" + for child in mc.sub_connectors: + child.get_finished_count.return_value = None + child.request_finished.return_value = True, None + child.build_connector_worker_meta.return_value = None + mc._world_size = 2 + return mc + + +def _send_step(connector, notifications=(), *, receiving=None): + outputs = [] + for rank in range(2): + metadata = MultiKVConnectorWorkerMetadata( + metadata=(None, None), + finished_sending=tuple( + {"r": {rank}} if (child, rank) in notifications else {} + for child in range(2) + ), + ) + outputs.append( + ModelRunnerOutput.with_kv_conn_output_only( + KVConnectorOutput(kv_connector_worker_meta=metadata) + ) + ) + output = KVOutputAggregator(2).aggregate(outputs).kv_connector_output + assert output is not None + output.finished_recving = receiving + connector.update_connector_output(output) + return output + + +def _finish_with_async_saves(connector): + request = SimpleNamespace(request_id="r") + connector.on_new_request(request) + assert connector.request_finished(request, [1])[0] + + +def _assert_save_state_cleared(connector): + assert not connector._async_save_owners + assert not connector._completed_async_saves + assert not any(connector._send_workers) + + +def test_send_completion_requires_each_child_and_distinct_rank(async_saves): + """A different child or a duplicate rank cannot satisfy a missing ACK.""" + _finish_with_async_saves(async_saves) + assert _send_step(async_saves, [(0, 0), (1, 1)]).finished_sending is None + assert _send_step(async_saves, [(0, 0), (1, 0)]).finished_sending is None + assert _send_step(async_saves, [(0, 1)]).finished_sending == {"r"} + _assert_save_state_cleared(async_saves) + assert ( + _send_step(async_saves, [(0, 0), (0, 1), (1, 0), (1, 1)]).finished_sending + is None + ) + _assert_save_state_cleared(async_saves) + + +def test_non_owner_cannot_filter_other_connector_send_completion(async_saves): + first = async_saves.sub_connectors[0] + first.request_finished.return_value = False, None + seen = [] + + def filter_completion(output): + seen.append(output.finished_sending) + output.finished_sending = None + + first.update_connector_output.side_effect = filter_completion + _finish_with_async_saves(async_saves) + assert _send_step(async_saves, [(1, 0), (1, 1)]).finished_sending == {"r"} + assert seen == [None] + _assert_save_state_cleared(async_saves) + + +def test_scheduler_child_completion_waits_for_other_save_owner(async_saves): + """A child may finish in update_connector_output, without a worker ACK.""" + _finish_with_async_saves(async_saves) + + def finish_first(output): + output.finished_sending = {"r"} + + async_saves.sub_connectors[0].update_connector_output.side_effect = finish_first + assert _send_step(async_saves).finished_sending is None + assert _send_step(async_saves, [(1, 0), (1, 1)]).finished_sending == {"r"} + _assert_save_state_cleared(async_saves) + + +@pytest.mark.parametrize("early_children", [(0,), (0, 1)]) +def test_send_completion_before_request_finished_is_reconciled( + async_saves, early_children +): + request = SimpleNamespace(request_id="r") + async_saves.on_new_request(request) + early = [(child, rank) for child in early_children for rank in range(2)] + assert _send_step(async_saves, early).finished_sending is None + pending, _ = async_saves.request_finished(request, [1]) + assert pending == (early_children == (0,)) + if pending: + assert _send_step(async_saves, [(1, 0), (1, 1)]).finished_sending == {"r"} + _assert_save_state_cleared(async_saves) + + +def test_send_completion_uses_child_finished_count(async_saves): + async_saves.sub_connectors[0].get_finished_count.return_value = 1 + _finish_with_async_saves(async_saves) + assert _send_step(async_saves, [(0, 0), (1, 0)]).finished_sending is None + assert _send_step(async_saves, [(1, 1)]).finished_sending == {"r"} + + +def test_unexpected_send_and_aborted_receive_release_tracking(async_saves): + # Unregistered ACKs must not create persistent request state. + assert _send_step(async_saves, [(0, 0), (1, 1)]).finished_sending is None + _assert_save_state_cleared(async_saves) + _finish_with_async_saves(async_saves) + assert _send_step(async_saves, [(0, 0)]).finished_sending is None + output = _send_step(async_saves, receiving={"r"}) + assert output.finished_recving == {"r"} + assert output.finished_sending is None + _assert_save_state_cleared(async_saves) + assert _send_step(async_saves, [(0, 1), (1, 0), (1, 1)]).finished_sending is None + _assert_save_state_cleared(async_saves) + + +def test_normal_receive_keeps_tracking_for_later_saves(async_saves): + request = SimpleNamespace(request_id="r") + async_saves.on_new_request(request) + assert _send_step(async_saves, receiving={"r"}).finished_recving == {"r"} + assert async_saves.request_finished(request, [1])[0] + assert _send_step( + async_saves, [(0, 0), (0, 1), (1, 0), (1, 1)] + ).finished_sending == {"r"} + _assert_save_state_cleared(async_saves) + + +def test_no_async_saves_release_tracking(async_saves): + request = SimpleNamespace(request_id="r") + async_saves.on_new_request(request) + for child in async_saves.sub_connectors: + child.request_finished.return_value = False, None + assert not async_saves.request_finished(request, [1])[0] + _assert_save_state_cleared(async_saves) + + +def test_sync_load_zeroing_exclusion_uses_only_selected_connector(mc): + """An unselected child cannot suppress initialization of fresh pages.""" + request = SimpleNamespace(request_id="r") + first, second = mc.sub_connectors + first.get_sync_load_block_ids.return_value = [1, 2] + second.get_sync_load_block_ids.return_value = [3] + assert mc.get_sync_load_block_ids(request) == [] + mc._requests_to_connector["r"] = 1 + assert mc.get_sync_load_block_ids(request) == [3] + first.get_sync_load_block_ids.assert_not_called() + + +def test_worker_preserves_send_identity_and_receive_failures(async_saves, monkeypatch): + from vllm.distributed.kv_transfer.kv_connector.v1 import multi_connector + + monkeypatch.setattr( + multi_connector, "get_world_group", lambda: SimpleNamespace(rank=1) + ) + async_saves.sub_connectors[ + 0 + ].get_transfer_results.return_value = KVConnectorTransferResults( + finished_sending={"r"} + ) + async_saves.sub_connectors[ + 1 + ].get_transfer_results.return_value = KVConnectorTransferResults( + finished_recving={"recv"}, failed_recving={"recv"} + ) + result = async_saves.get_transfer_results(set()) + assert not result.finished_sending + assert result.finished_recving == result.failed_recving == {"recv"} + metadata = async_saves.build_connector_worker_meta() + assert isinstance(metadata, MultiKVConnectorWorkerMetadata) + assert metadata.finished_sending == ({"r": {1}}, {}) + assert async_saves.build_connector_worker_meta() is None + + +def test_worker_send_metadata_merge_preserves_inputs_and_deduplicates_ranks(): + first = MultiKVConnectorWorkerMetadata( + metadata=(None, None), finished_sending=({"r": {0}}, {"r": {1}}) + ) + second = MultiKVConnectorWorkerMetadata( + metadata=(None, None), finished_sending=({"r": {0, 1}}, {}) + ) + result = first.aggregate(second) + assert result.finished_sending == ({"r": {0, 1}}, {"r": {1}}) + assert first.finished_sending == ({"r": {0}}, {"r": {1}}) + assert second.finished_sending == ({"r": {0, 1}}, {}) + + def test_multi_connector_mem_pool_context_none(mc: MultiConnector): assert mc.get_mem_pool_context() is None diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 2e0c7271ce62..d5612fbb3dfb 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -8,7 +8,10 @@ import pytest import torch +import vllm.forward_context as forward_context_module import vllm.v1.worker.gpu.model_runner as model_runner_module +from vllm.config import CUDAGraphMode, VllmConfig +from vllm.forward_context import get_forward_context from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry from vllm.v1.kv_cache_interface import ( CircularBufferSpec, @@ -58,6 +61,119 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback(): runner.pcp_manager.restore_for_sampling.assert_not_called() +@pytest.mark.parametrize("data_parallel_size", [1, 2]) +def test_full_graph_sync_load_observes_replay_mode(monkeypatch, data_parallel_size): + """Connectors must wait for READs before replay bypasses layer hooks.""" + runner = Mock() + runner.vllm_config = VllmConfig() + runner.vllm_config.parallel_config.data_parallel_size = data_parallel_size + runner.vllm_config.parallel_config.is_moe_model = True + monkeypatch.setattr( + forward_context_module, + "coordinate_batch_across_dp", + Mock(side_effect=AssertionError("connector context must not synchronize DP")), + ) + runner.lora_config = None + runner.is_encoder_decoder = False + runner.uses_inputs_embeds = False + runner.is_first_pp_rank = True + runner.dcp_size = 1 + runner.pcp_manager = None + runner.aux_output_connector = None + runner.observability_config.cudagraph_metrics = False + runner.gather_batch_req_state.return_value = (Mock(has_prefill=False), 1) + runner.prepare_attn.return_value = ({}, {}) + runner.model_state.prepare_inputs.return_value = {} + batch_desc = SimpleNamespace( + num_tokens=1, cg_mode=CUDAGraphMode.FULL, num_ubatches=1 + ) + monkeypatch.setattr( + model_runner_module, + "dispatch_cg_and_sync_dp", + lambda *a, **kw: (batch_desc, None), + ) + monkeypatch.setattr( + model_runner_module, "build_slot_mappings_by_layer", lambda *a: {} + ) + + class BeforeReplay(Exception): + pass + + def load(**kwargs): + assert get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL + + runner.kv_connector.pre_forward.side_effect = load + runner.cudagraph_manager.run_fullgraph.side_effect = BeforeReplay + scheduler_output = SimpleNamespace( + num_scheduled_tokens={"request": 1}, total_num_scheduled_tokens=1 + ) + with pytest.raises(BeforeReplay): + GPUModelRunner.execute_model(runner, scheduler_output) + runner.kv_connector.pre_forward.assert_called_once() + + +@pytest.mark.parametrize("has_speculator", [False, True]) +@pytest.mark.parametrize("requires_completion", [False, True]) +def test_output_readiness_covers_draft_kv_writes( + monkeypatch, has_speculator: bool, requires_completion: bool +): + """Publishing output must cover draft writes before RDMA can reuse KV.""" + writes: set[str] = set() + visible: set[str] = set() + ready: set[str] = set() + copy_stream = Mock() + copy_stream.wait_stream.side_effect = lambda _: visible.update(writes) + event = Mock() + event.record.side_effect = lambda _: ready.update(visible) + + def async_output(**kwargs): + copy_stream.wait_stream(kwargs["main_stream"]) + event.record(copy_stream) + return SimpleNamespace(copy_event=event) + + monkeypatch.setattr(model_runner_module, "AsyncOutput", async_output) + runner = Mock() + runner.output_copy_stream = copy_stream + runner.is_last_pp_rank = True + runner.pcp_manager = None + runner.pp_handler = None + runner.aux_output_connector = None + runner.adaptive_verification = None + runner.num_speculative_steps = 0 + runner._draft_workspace_lane = 0 + runner.model = Mock(spec=["compute_logits"]) + runner.kv_connector.requires_full_step_completion = requires_completion + runner.req_states.draft_tokens = torch.zeros((1, 1), dtype=torch.int64) + runner.prompt_logprobs_worker.compute_prompt_logprobs.return_value = {} + input_batch = SimpleNamespace( + req_ids=["request"], idx_mapping=torch.tensor([0]), query_start_loc=None + ) + runner.execute_model_state.input_batch = input_batch + runner.sample.return_value = (Mock(), torch.tensor([1]), torch.tensor([0])) + runner.postprocess_sampled.side_effect = lambda *a: writes.add("postprocess") + if has_speculator: + runner.speculator.supports_mm_inputs = False + + def propose(*args, **kwargs): + assert "draft" not in ready + writes.add("draft") + return torch.zeros((1, 1), dtype=torch.int64) + + runner.speculator.propose.side_effect = propose + else: + runner.speculator = None + monkeypatch.setattr( + model_runner_module, "use_workspace_lane", lambda _: contextlib.nullcontext() + ) + GPUModelRunner.sample_tokens(runner, None) + assert ("draft" in ready) == (has_speculator and requires_completion) + assert ("postprocess" in ready) == (has_speculator and requires_completion) + assert event.record.call_count == ( + 2 if has_speculator and requires_completion else 1 + ) + event.synchronize.assert_not_called() + + def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch): runner = GPUModelRunner.__new__(GPUModelRunner) runner.max_model_len = 262144 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index 6df9d1cfbe8b..363fe8d653c3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -188,6 +188,24 @@ def supports_divergent_local_hybrid_hits(self) -> bool: """ return False + def get_sync_load_block_ids(self, request: "Request") -> list[int]: + """Return blocks whose synchronous load replaces worker zeroing. + + Each block must be fully initialized before forward consumes it. A + failed load must abort forward instead of recomputing with uninitialized + blocks. Defaults to no blocks. + """ + return [] + + @property + def requires_full_step_completion(self) -> bool: + """Whether output readiness must cover post-sampling KV writes. + + Out-of-band transfers cannot rely on the GPU stream to order loads + after earlier writes to a reused block, or before source publication. + """ + return False + @property def requires_kv_delivery(self) -> bool: """Whether this connector hands off KV that must be reliably delivered. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index ae1eecf6e55d..37e8a05769b0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -369,7 +369,9 @@ def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig": return cls( local_ip=resolve_host_ip(extra_config), - local_kv_port=get_open_port(), + # MoRI publishes the bound port after RDMA backend initialization. + # Let bind(0) reserve it atomically across concurrent TP workers. + local_kv_port=0 if backend == "rdma" else get_open_port(), proxy_ip=extra_config.get("proxy_ip", ""), local_ping_port=get_open_port(), proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)), diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index b482e6a8c4a3..4fd131be444d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -59,6 +59,9 @@ MoRIIOWrapper, MoRIIOWriter, ) +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( + MoRIIOHeartbeat, +) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout import ( LayerTransferGeometry, MambaOffsetTemplate, @@ -225,6 +228,28 @@ def supports_divergent_local_hybrid_hits(self) -> bool: # when the attention prefix is already present locally. return self.mode == MoRIIOMode.READ and self._transfers_mamba_state + @property + def requires_full_step_completion(self) -> bool: + return self.mode == MoRIIOMode.READ + + def get_sync_load_block_ids(self, request: "Request") -> list[int]: + scheduler = self.connector_scheduler + if ( + self.mode != MoRIIOMode.READ + or not self.kv_transfer_config.is_kv_consumer + or scheduler is None + or not scheduler._has_mamba + or self._vllm_config.cache_config.get_resolved_kv_cache_layout().name + != "LBNHC" + ): + return [] + pending = scheduler._reqs_need_recv.get(request.request_id) + if pending is None: + return [] + # Hybrid READ fills these entire attention pages and aborts on failure. + # The destination list already excludes local hits and lookahead blocks. + return pending[1][0] + def __init__( self, vllm_config: VllmConfig, @@ -761,6 +786,10 @@ def get_num_new_matched_tokens( # no recurrent-state accounting to do. return num_external_tokens, True + params = request.kv_transfer_params + if not params or not params.get("do_remote_prefill"): + return 0, False + # READ mode always recomputes the last token locally on the decoder. # # The second element declares who waits for the KV, not whether it has @@ -1613,7 +1642,7 @@ def __init__( self.moriio_engine = None self._handle_request_thread = None - self._ping_thread = None + self._heartbeat: MoRIIOHeartbeat | None = None self._writer = MoRIIOWriter(self) # Completions that arrived before transfer_id_to_request_id was populated. # Retried each step until the mapping is established. @@ -1643,10 +1672,7 @@ def __init__( ) if self._rank == 0 and self.moriio_config.proxy_ip: - self._ping_thread = threading.Thread( - target=self._ping, args=(self.zmq_context,), daemon=True - ) - self._ping_thread.start() + self._heartbeat = self._start_heartbeat() logger.info( "Initializing MoRIIO Engine, engine = %s, role = %s", @@ -1708,6 +1734,7 @@ def __init__( # KV Caches and moriio tracking data. self.kv_caches: dict[str, torch.Tensor] = {} self.kv_layer_mr_offset: dict[str, int] = {} + self.kv_region_mr_offsets: dict[str, list[int]] = {} self.layer_base_addr_index: dict[str, int] = {} # Map of engine_id -> kv_caches_base_addr. For TP case, each local @@ -1894,75 +1921,31 @@ def _get_built_session(self, remote_engine_id): remote_engine_id ] - def _ping(self, zmq_context): - # Use host:port format for http_address (compatible with official router) - http_address = f"{self.request_address}" - # Include host so the router embeds it in the request_id; the connector - # on the other side parses host/ports from there. - zmq_address = ( - f"host:{self.local_ip}," - f"handshake:{self.handshake_port}," - f"notify:{self.notify_port}" + def _start_heartbeat(self) -> MoRIIOHeartbeat: + payload = { + "type": "P" if self.is_producer else "D", + "http_address": self.request_address, + "zmq_address": ( + f"host:{self.local_ip},handshake:{self.handshake_port}," + f"notify:{self.notify_port}" + ), + "dp_size": self.moriio_config.dp_size, + "tp_size": self.moriio_config.tp_size, + "transfer_mode": self.mode.name, + } + return MoRIIOHeartbeat( + f"tcp://{self.proxy_ip}:{self.proxy_ping_port}", + payload, + MoRIIOConstants.PING_INTERVAL, + MoRIIOConstants.MAX_PING_RETRIES, ) - role = "P" if self.is_producer else "D" - - retry_count = 0 - index = 1 - with zmq_context.socket(zmq.DEALER) as sock: - sock.connect(f"tcp://{self.proxy_ip}:{self.proxy_ping_port}") - - while True: - try: - data = { - "type": role, # "P" or "D" - "http_address": http_address, - "zmq_address": zmq_address, - # dp_size/tp_size are not used by the official vLLM router - # (routing operates at the http_address level); they are - # consumed only by the toy proxy server. - "dp_size": self.moriio_config.dp_size, - "tp_size": self.moriio_config.tp_size, - # transfer_mode is included so the router can distinguish - # READ (prefill-then-decode, sequential) from WRITE (concurrent) - # scheduling. - "transfer_mode": self.mode.name, - } - - sock.send(msgpack.dumps(data)) - # logger.debug(f"Successfully sent ping message #{index}") - retry_count = 0 - - except ConnectionRefusedError: - logger.info( - "Connection refused: %s:%s -> %s:%s", - self.local_ip, - self.local_ping_port, - self.proxy_ip, - self.proxy_ping_port, - ) - retry_count += 1 - - except OSError as e: - logger.info("OS error when sending ping: %s", e) - retry_count += 1 - - except Exception as e: - logger.info("Unexpected error when sending ping: %s", e) - retry_count += 1 - if retry_count >= MoRIIOConstants.MAX_PING_RETRIES: - logger.error( - "Max retries (%s) exceeded. Stopping ping loop.", - MoRIIOConstants.MAX_PING_RETRIES, - ) - raise RuntimeError( - f"Ping failed after {retry_count} retries" - ) from e - - finally: - time.sleep(MoRIIOConstants.PING_INTERVAL) - index += 1 def shutdown(self): + heartbeat = getattr(self, "_heartbeat", None) + if heartbeat is not None: + heartbeat.shutdown() + self._heartbeat = None + if hasattr(self, "moriio_wrapper") and self.moriio_wrapper: self.moriio_wrapper.shutdown() @@ -2269,24 +2252,28 @@ def _build_shared_kv_mr( ) -> tuple[torch.Tensor, dict[str, int]]: page_size = 4096 backing = next(iter(kv_caches.values())).view(torch.uint8) - nbytes = backing.untyped_storage().nbytes() - base = torch.as_strided(backing, (nbytes,), (1,), 0) - ptr = base.data_ptr() - slack = ptr % page_size - end = 0 + storage = backing.untyped_storage() + storage_ptr = storage.data_ptr() + regions = [] for layer_name in kv_caches: - _, region_len = next( - iter(self._iter_layer_registration_regions(layer_name)) - ) - end = max(end, kv_caches[layer_name].data_ptr() - ptr + region_len) - reg_nbytes = ((slack + end + page_size - 1) // page_size) * page_size - if reg_nbytes > nbytes: + for region, region_len in self._iter_layer_registration_regions(layer_name): + if ( + region.device != backing.device + or region.untyped_storage().data_ptr() != storage_ptr + ): + raise ValueError("Shared KV regions must use the same storage") + regions.append((region.data_ptr(), region_len)) + # The first transferable layer need not be the allocation's first view. + # Align its registration within storage, never before the allocation. + mr_ptr = min(ptr for ptr, _ in regions) // page_size * page_size + end = max(ptr + length for ptr, length in regions) + if mr_ptr < storage_ptr or end > storage_ptr + storage.nbytes(): raise ValueError( - f"shared KV backing too small for page-aligned MR: " - f"need {reg_nbytes}, have {nbytes}" + "Page-aligned shared KV registration is outside its storage" ) - reg = torch.as_strided(base, (reg_nbytes,), (1,), 0 if slack == 0 else -slack) - mr_ptr = reg.data_ptr() + # RDMA accepts an arbitrary extent; rounding the final SSM region up + # could extend the registration past the last allocated byte. + reg = torch.as_strided(backing, (end - mr_ptr,), (1,), mr_ptr - storage_ptr) return reg, { name: cache.data_ptr() - mr_ptr for name, cache in kv_caches.items() } @@ -2463,9 +2450,11 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): and len({id(t.untyped_storage()) for t in kv_caches.values()}) == 1 ) shared_mr = None + shared_mr_base = None if shared_backing: reg_tensor, self.kv_layer_mr_offset = self._build_shared_kv_mr(kv_caches) shared_mr = self.moriio_wrapper.register_local_tensor(reg_tensor) + shared_mr_base = reg_tensor.data_ptr() # Say it out loud when the groups disagree. This is expected on a hybrid # model and used to be fatal, so a run that silently had one page size @@ -2511,11 +2500,21 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): ssm.is_contiguous(), ssm.untyped_storage().nbytes(), ) + self.kv_region_mr_offsets[layer_name] = [] for tensor in (conv, ssm): alias = self._contiguous_byte_alias(tensor) - meta = self.moriio_wrapper.register_local_tensor(alias) + meta = ( + shared_mr + if shared_mr is not None + else self.moriio_wrapper.register_local_tensor(alias) + ) self.layer_name_to_local_kv_cache_metadata[layer_name].append(meta) self.local_kv_cache_size.append(alias.numel()) + self.kv_region_mr_offsets[layer_name].append( + 0 + if shared_mr_base is None + else alias.data_ptr() - shared_mr_base + ) else: moriio_mem_metadata = shared_mr or ( self.moriio_wrapper.register_local_tensor(kv_cache) @@ -2728,6 +2727,11 @@ def _pop_done_transfers(self) -> set[str]: ) to_remove.append(req_id) elif state is TransferBatchState.FAILED: + if self._has_mamba: + raise TransferError( + f"MoRIIO hybrid READ failed for request {req_id}; " + "refusing to release or reuse in-flight KV blocks" + ) failed_status = next( (status for status in statuses if status.Failed()), None ) @@ -2765,6 +2769,11 @@ def _pop_done_transfers(self) -> set[str]: # indefinitely on this request. _age = time.monotonic() - self._recving_transfers_start[req_id] if _age > _xfer_timeout: + if self._has_mamba: + raise TransferError( + f"MoRIIO hybrid READ timed out for request {req_id}; " + "refusing to release or reuse in-flight KV blocks" + ) logger.error( "RDMA read TIMED OUT for req %s after %.1fs " "(kv_connector_extra_config.recv_abort_timeout=%.0f)", @@ -3445,6 +3454,8 @@ def _post_mamba_reads( local_slots: list[int], remote_slots: list[int], remote_tp_size: int, + remote_moriio_meta: MoRIIOAgentMetadata, + remote_engine_id: EngineId, request_id: str, deadline: float, ) -> list: @@ -3460,10 +3471,22 @@ def _post_mamba_reads( remote_slots, remote_tp_size=remote_tp_size, ) + local_region_offsets = self.kv_region_mr_offsets[layer_name] + remote_metas = self.layer_name_to_remote_kv_cache_metadata[remote_engine_id][ + layer_name + ] + base_idx = self.layer_base_addr_index[layer_name] + remote_region_offsets = [ + remote_moriio_meta.kv_caches_base_addr[base_idx + i] + - self.moriio_wrapper.get_unpack_memory_metadata(meta).data + for i, meta in enumerate(remote_metas) + ] statuses: list = [] - for sess_idx, sl in ( - (region_sessions[0], slice(0, n_conv)), - (region_sessions[1], slice(n_conv, None)), + for region_idx, (sess_idx, sl) in enumerate( + ( + (region_sessions[0], slice(0, n_conv)), + (region_sessions[1], slice(n_conv, None)), + ) ): region_sizes = sizes[sl] if not region_sizes: @@ -3472,8 +3495,11 @@ def _post_mamba_reads( self._post_read_with_backoff( sessions[sess_idx], region_sizes, - local[sl], - remote[sl], + [offset + local_region_offsets[region_idx] for offset in local[sl]], + [ + offset + remote_region_offsets[region_idx] + for offset in remote[sl] + ], request_id, layer_name, deadline, @@ -3496,8 +3522,8 @@ def _post_read_with_backoff( read_remote_data posts synchronously, so a send-queue-full rejection is a Failed() status on return; a separate CQ-poll thread drains completions and frees SQ depth, so we back off and re-post until - transfer_timeout, then store the failed status (get_finished notifies - prefill and drops the request non-fatally). + transfer_timeout, then store the failed status for completion/error + handling. Hybrid failures abort before releasing source KV. """ _backoff = _SQ_FULL_BACKOFF_INITIAL_S while True: @@ -3510,8 +3536,8 @@ def _post_read_with_backoff( logger.warning( "MoRIIO READ send queue stayed full past " "transfer_timeout for req %s layer %s; storing failed " - "status (get_finished notifies prefill and drops the " - "request). Raise qp_per_transfer if frequent.", + "status for completion/error handling. " + "Raise qp_per_transfer if frequent.", request_id, layer_name, ) @@ -3652,6 +3678,8 @@ def _read_blocks( local_mamba, remote_mamba, remote_tp_size, + remote_moriio_meta, + remote_dp_engine_id, request_id, _sq_deadline, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py new file mode 100644 index 000000000000..01a7f38c59f2 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Keep discovery heartbeats independent of the model worker's GIL.""" + +import json +import logging +import select +import subprocess +import sys +from typing import Any + +import msgpack +import zmq + +logger = logging.getLogger(__name__) + + +class MoRIIOHeartbeat: + def __init__( + self, + endpoint: str, + payload: dict[str, Any], + interval: float, + max_retries: int = 100, + ): + self._process = subprocess.Popen( + [sys.executable, __file__, endpoint, str(interval), str(max_retries)], + stdin=subprocess.PIPE, + text=True, + ) + assert self._process.stdin is not None + try: + self._process.stdin.write(json.dumps(payload) + "\n") + self._process.stdin.flush() + except BaseException: + self.shutdown() + raise + + def shutdown(self) -> None: + assert self._process.stdin is not None + try: + if not self._process.stdin.closed: + self._process.stdin.close() + except BrokenPipeError: + pass + finally: + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.terminate() + try: + self._process.wait(timeout=1) + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait(timeout=1) + + +def main() -> None: + endpoint, interval_arg, max_retries_arg = sys.argv[1:] + initial = sys.stdin.readline() + if not initial: + return + payload = msgpack.dumps(json.loads(initial)) + interval = float(interval_arg) + max_retries = int(max_retries_arg) + failures = 0 + with zmq.Context() as context, context.socket(zmq.DEALER) as sock: + sock.setsockopt(zmq.LINGER, 0) + sock.setsockopt(zmq.SNDTIMEO, 1000) + sock.connect(endpoint) + while not select.select([sys.stdin], [], [], 0)[0]: + try: + sock.send(payload) + failures = 0 + except zmq.ZMQError: + failures += 1 + logger.exception("MoRIIO discovery heartbeat send failed") + if failures >= max_retries: + raise + # EOF also stops the child when the worker exits without cleanup. + if select.select([sys.stdin], [], [], interval)[0]: + break + + +if __name__ == "__main__": + main() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index 73b6b0ccf643..e140b69fff5e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -10,6 +10,7 @@ from vllm.config import VllmConfig from vllm.config.kv_transfer import KVTransferConfig +from vllm.distributed import get_world_group from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.distributed.kv_transfer.kv_connector.v1.base import ( @@ -47,12 +48,12 @@ @dataclass class MultiKVConnectorMetadata(KVConnectorMetadata): metadata: tuple[KVConnectorMetadata, ...] - extra_async_saves: dict[str, int] | None = None @dataclass class MultiKVConnectorWorkerMetadata(KVConnectorWorkerMetadata): metadata: tuple[KVConnectorWorkerMetadata | None, ...] + finished_sending: tuple[dict[str, set[int]], ...] = () def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata: assert isinstance(other, MultiKVConnectorWorkerMetadata) @@ -67,7 +68,17 @@ def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetada else: metadata_list.append(metadata1.aggregate(metadata2)) - return MultiKVConnectorWorkerMetadata(metadata=tuple(metadata_list)) + finished_sending = [] + for i in range(len(self.metadata)): + merged: dict[str, set[int]] = {} + for source in (self.finished_sending, other.finished_sending): + if source: + for req_id, ranks in source[i].items(): + merged.setdefault(req_id, set()).update(ranks) + finished_sending.append(merged) + return MultiKVConnectorWorkerMetadata( + metadata=tuple(metadata_list), finished_sending=tuple(finished_sending) + ) @dataclass @@ -201,11 +212,17 @@ def __init__( # load the request from (if any). self._requests_to_connector: dict[str, int] = {} - # Keeps track of *additional* remaining async saves (beyond 1) to be - # finished per request. Not needed for async loads since we only allow - # a single connector to load. - # Propagated from scheduler to worker side via the connector metadata. - self._extra_async_saves: dict[str, int] = {} + # Scheduler-side owners and completions, including ACKs before finish. + self._async_save_owners: dict[str, set[int]] = {} + self._completed_async_saves: dict[str, set[int]] = {} + self._send_workers: tuple[dict[str, set[int]], ...] = tuple( + {} for _ in self._connectors + ) + self._world_size = vllm_config.parallel_config.world_size + # Worker-side ACKs retain child and rank identity across scheduler steps. + self._worker_finished_sending: tuple[dict[str, set[int]], ...] = tuple( + {} for _ in self._connectors + ) @property def sub_connectors(self) -> list[KVConnectorBase_V1]: @@ -221,6 +238,10 @@ def supports_divergent_local_hybrid_hits(self) -> bool: def requires_kv_delivery(self) -> bool: return any(c.requires_kv_delivery for c in self._connectors) + @property + def requires_full_step_completion(self) -> bool: + return any(c.requires_full_step_completion for c in self._connectors) + @classmethod def _get_connector_classes_and_configs( cls, vllm_config: "VllmConfig" @@ -268,8 +289,6 @@ def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None: # always return False. def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None: assert isinstance(connector_metadata, MultiKVConnectorMetadata) - if connector_metadata.extra_async_saves: - self._extra_async_saves.update(connector_metadata.extra_async_saves) for c, cm in zip(self._connectors, connector_metadata.metadata): c.bind_connector_metadata(cm) super().bind_connector_metadata(connector_metadata) @@ -358,20 +377,14 @@ def get_transfer_results( self, finished_req_ids: set[str] ) -> KVConnectorTransferResults: results = KVConnectorTransferResults() - for connector in self._connectors: + for i, connector in enumerate(self._connectors): child_results = connector.get_transfer_results(finished_req_ids) results.finished_recving.update(child_results.finished_recving) results.failed_recving.update(child_results.failed_recving) for req_id in child_results.finished_sending: - extra_pending = self._extra_async_saves.get(req_id) - if extra_pending is None: - results.finished_sending.add(req_id) - else: - assert extra_pending > 0 - if extra_pending == 1: - del self._extra_async_saves[req_id] - else: - self._extra_async_saves[req_id] = extra_pending - 1 + self._worker_finished_sending[i].setdefault(req_id, set()).add( + get_world_group().rank + ) return results def get_block_ids_with_load_errors(self) -> set[int]: @@ -406,16 +419,14 @@ def get_finished_count(self) -> int | None: return next(iter(counts), None) def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None: - metadata_list: list[KVConnectorWorkerMetadata | None] | None = None - for i, c in enumerate(self._connectors): - kv_connector_worker_meta = c.build_connector_worker_meta() - if metadata_list is None and kv_connector_worker_meta is not None: - metadata_list = [None] * i - if metadata_list is not None: - metadata_list.append(kv_connector_worker_meta) - if metadata_list is None: + metadata = tuple(c.build_connector_worker_meta() for c in self._connectors) + finished_sending = self._worker_finished_sending + self._worker_finished_sending = tuple({} for _ in self._connectors) + if all(item is None for item in metadata) and not any(finished_sending): return None - return MultiKVConnectorWorkerMetadata(metadata=tuple(metadata_list)) + return MultiKVConnectorWorkerMetadata( + metadata=metadata, finished_sending=finished_sending + ) # TODO: Add a generic implementation of 'get_kv_connector_kv_cache_events' # method for the MultiConnector. It should be able to get events from @@ -447,6 +458,12 @@ def get_num_new_matched_tokens( to_return = (toks, load_async) return to_return + def get_sync_load_block_ids(self, request: "Request") -> list[int]: + chosen = self._requests_to_connector.get(request.request_id) + if chosen is None: + return [] + return self._connectors[chosen].get_sync_load_block_ids(request) + def update_state_after_alloc( self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int ): @@ -460,21 +477,24 @@ def update_state_after_alloc( c.update_state_after_alloc(request, blocks, 0) def on_new_request(self, request: "Request") -> None: + self._completed_async_saves[request.request_id] = set() for c in self._connectors: c.on_new_request(request) def build_connector_meta( self, scheduler_output: SchedulerOutput ) -> MultiKVConnectorMetadata: - metadata = MultiKVConnectorMetadata( + return MultiKVConnectorMetadata( metadata=tuple( c.build_connector_meta(scheduler_output) for c in self._connectors ) ) - if self._extra_async_saves: - metadata.extra_async_saves = self._extra_async_saves - self._extra_async_saves = {} - return metadata + + def _forget_async_saves(self, req_id: str) -> None: + self._async_save_owners.pop(req_id, None) + self._completed_async_saves.pop(req_id, None) + for workers in self._send_workers: + workers.pop(req_id, None) def update_connector_output(self, connector_output: KVConnectorOutput): multi_connector_worker_meta: MultiKVConnectorWorkerMetadata | None = None @@ -485,6 +505,7 @@ def update_connector_output(self, connector_output: KVConnectorOutput): ) multi_connector_worker_meta = connector_output.kv_connector_worker_meta + finished_sending: set[str] = set() try: for i, c in enumerate(self._connectors): if multi_connector_worker_meta is not None: @@ -492,10 +513,54 @@ def update_connector_output(self, connector_output: KVConnectorOutput): connector_output.kv_connector_worker_meta = ( multi_connector_worker_meta.metadata[i] ) + child_sending: set[str] = set() + if ( + multi_connector_worker_meta is not None + and multi_connector_worker_meta.finished_sending + ): + expected = ( + c.get_finished_count() + or connector_output.expected_finished_count + or self._world_size + ) + for req_id, ranks in multi_connector_worker_meta.finished_sending[ + i + ].items(): + completed = self._completed_async_saves.get(req_id) + owners = self._async_save_owners.get(req_id) + if ( + completed is None + or i in completed + or (owners is not None and i not in owners) + ): + continue + workers = self._send_workers[i].setdefault(req_id, set()) + workers.update(ranks) + if len(workers) >= expected: + child_sending.add(req_id) + connector_output.finished_sending = child_sending or None c.update_connector_output(connector_output) + for req_id in connector_output.finished_sending or (): + completed = self._completed_async_saves.get(req_id) + owners = self._async_save_owners.get(req_id) + if completed is None or (owners is not None and i not in owners): + continue + completed.add(i) + self._send_workers[i].pop(req_id, None) + if owners is not None: + owners.discard(i) + if not owners: + finished_sending.add(req_id) finally: # restore kv_connector_worker_meta connector_output.kv_connector_worker_meta = multi_connector_worker_meta + connector_output.finished_sending = finished_sending or None + aborted_recvs = ( + set(connector_output.finished_recving or ()) + & self._async_save_owners.keys() + ) + for req_id in finished_sending | aborted_recvs: + self._forget_async_saves(req_id) def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None: """Get the KVConnector handshake metadata from sub-connectors. @@ -529,12 +594,12 @@ def _aggregate_request_finished( [KVConnectorBase_V1], tuple[bool, dict[str, Any] | None] ], ) -> tuple[bool, dict[str, Any] | None]: - async_saves = 0 + async_save_owners: set[int] = set() kv_txfer_params = None - for c in self._connectors: + for i, c in enumerate(self._connectors): async_save, txfer_params = per_connector_fn(c) if async_save: - async_saves += 1 + async_save_owners.add(i) if txfer_params is not None: if kv_txfer_params is not None: clashes = set(kv_txfer_params) & set(txfer_params) @@ -546,12 +611,16 @@ def _aggregate_request_finished( kv_txfer_params.update(txfer_params) else: kv_txfer_params = txfer_params - if async_saves > 1: - self._extra_async_saves[request.request_id] = async_saves - 1 + completed = self._completed_async_saves.setdefault(request.request_id, set()) + pending = async_save_owners - completed + if pending: + self._async_save_owners[request.request_id] = pending + else: + self._forget_async_saves(request.request_id) self._requests_to_connector.pop(request.request_id, None) - return async_saves > 0, kv_txfer_params + return bool(pending), kv_txfer_params def request_finished( self, diff --git a/vllm/parser/kimi_k3.py b/vllm/parser/kimi_k3.py index 587e1d5ad6b5..e1342aad0e02 100644 --- a/vllm/parser/kimi_k3.py +++ b/vllm/parser/kimi_k3.py @@ -10,8 +10,9 @@ DeltaMessage, FunctionCall, ) -from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.abstract_parser import DelegatingParser, StreamState from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser +from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser if TYPE_CHECKING: from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -93,6 +94,33 @@ def _extract_tool_calls_streaming( delta_message.tool_calls = [] return delta_message, False + def finalize_generation( + self, + delta_message: DeltaMessage | None, + request: ChatCompletionRequest | ResponsesRequest, + state: StreamState, + ) -> DeltaMessage | None: + delta_message = super().finalize_generation(delta_message, request, state) + if not state.reasoning_ended and isinstance( + self._reasoning_parser, KimiK3ReasoningParser + ): + tail = self._reasoning_parser.finish_reasoning_streaming( + state.previous_text + ) + if tail: + if delta_message is None: + delta_message = DeltaMessage() + delta_message.reasoning = (delta_message.reasoning or "") + tail + elif state.reasoning_ended and isinstance(self._tool_parser, KimiK3ToolParser): + content_tail = self._tool_parser._extract_response_content( + state.previous_text, finished=True + ) + if content_tail: + if delta_message is None: + delta_message = DeltaMessage() + delta_message.content = (delta_message.content or "") + content_tail + return delta_message + def parse_delta( self, delta_text: str, @@ -116,14 +144,16 @@ def parse_delta( self._tool_parser is not None or not isinstance(self._reasoning_parser, KimiK3ReasoningParser) or not state.reasoning_ended - or delta_message is None ): return delta_message stripped = self._reasoning_parser.strip_content_streaming( previous_text=previous_content, current_text=state.previous_text, + finished=finished, ) + if delta_message is None: + return stripped delta_message.content = stripped.content if stripped is not None else None if ( delta_message.role is None diff --git a/vllm/reasoning/kimi_k3_reasoning_parser.py b/vllm/reasoning/kimi_k3_reasoning_parser.py index 1a1c62ba6def..5ab9a139e635 100644 --- a/vllm/reasoning/kimi_k3_reasoning_parser.py +++ b/vllm/reasoning/kimi_k3_reasoning_parser.py @@ -359,7 +359,16 @@ def _reasoning_text_ready_to_emit(self, text: str) -> str: break return text[:-overlap] if overlap else text - def _content_ready_to_emit(self, text: str) -> str: + def finish_reasoning_streaming(self, text: str) -> str: + """Release an unfinished marker as literal text at end of stream.""" + if self._think_close_re.search(text): + return "" + safe = self._reasoning_text_ready_to_emit(text) + opened = self._think_open_re.search(text) + body = text[opened.end() :] if opened is not None else text + return body[len(safe) :] + + def _content_ready_to_emit(self, text: str, *, finished: bool = False) -> str: """Return the content prefix that is safe to stream now. Mirrors ``_reasoning_text_ready_to_emit`` but for the post-reasoning @@ -376,6 +385,9 @@ def _content_ready_to_emit(self, text: str) -> str: text = self._response_close_re.sub("", text) text = self._message_close_re.sub("", text) + if finished: + return text + # Hold back partial markers at the end overlap = 0 for marker in ( @@ -394,6 +406,8 @@ def strip_content_streaming( self, previous_text: str, current_text: str, + *, + finished: bool = False, ) -> DeltaMessage | None: """Strip XTML content wrappers from streaming deltas after reasoning. @@ -404,7 +418,7 @@ def strip_content_streaming( Works from accumulated text (``previous_text`` / ``current_text`` already contain only post-reasoning content). """ - current_safe = self._content_ready_to_emit(current_text) + current_safe = self._content_ready_to_emit(current_text, finished=finished) previous_safe = self._content_ready_to_emit(previous_text) if current_safe.startswith(previous_safe): delta = current_safe[len(previous_safe) :] diff --git a/vllm/tool_parsers/kimi_k3_tool_parser.py b/vllm/tool_parsers/kimi_k3_tool_parser.py index 76c6b0aa657c..8fcc2fdaa505 100644 --- a/vllm/tool_parsers/kimi_k3_tool_parser.py +++ b/vllm/tool_parsers/kimi_k3_tool_parser.py @@ -268,7 +268,9 @@ def _content(self, model_output: str, before: str) -> str | None: return m["c"] or None return self._strip_response_content(before) - def _extract_response_content(self, current_text: str) -> str | None: + def _extract_response_content( + self, current_text: str, *, finished: bool = False + ) -> str | None: # Streaming response text is computed from the accumulated text. This is # what keeps split markers from leaking: # <|open|> / response / <|sep|>Hi -> emit only "Hi" after open closes @@ -289,6 +291,8 @@ def _extract_response_content(self, current_text: str) -> str | None: candidates = [i for i in (tools_start, response_end) if i != -1] if candidates: sendable_idx = min(candidates) + elif finished: + sendable_idx = len(current_text) else: overlap = max( _partial_tag_overlap(current_text, self.response_open), diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index fcb1421b1046..9d2c341d4dca 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -349,7 +349,7 @@ def __init__( self.has_mamba_layers = kv_cache_config.has_mamba_layers self.needs_kv_cache_zeroing = kv_cache_config.needs_kv_cache_zeroing - # Blocks that async KV loads will overwrite this step, skipped from + # Blocks that external KV loads will overwrite this step, skipped from # zeroing since the zeroing could race the out-of-band write. self._skip_zero_block_ids: set[int] = set() self.need_mamba_block_aligned_split = ( @@ -1295,6 +1295,11 @@ def skip_request(from_queue: RequestQueue) -> None: if num_external_computed_tokens > 0: # load_kv_async is False here has_sync_kv_loads = True + if self.needs_kv_cache_zeroing: + assert self.connector is not None + self._skip_zero_block_ids.update( + self.connector.get_sync_load_block_ids(request) + ) if self.log_stats: request.record_event( EngineCoreEventType.SCHEDULED, scheduled_timestamp diff --git a/vllm/v1/worker/gpu/kv_connector.py b/vllm/v1/worker/gpu/kv_connector.py index 1ba75beae271..5a1a1ce0db04 100644 --- a/vllm/v1/worker/gpu/kv_connector.py +++ b/vllm/v1/worker/gpu/kv_connector.py @@ -29,6 +29,10 @@ class KVConnector: """KVConnector interface used by GPUModelRunner.""" + @property + def requires_full_step_completion(self) -> bool: + return False + def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None: pass @@ -61,6 +65,10 @@ def __init__( self._pending_load_kwargs: dict[str, Any] | None = None self._disabled = False + @property + def requires_full_step_completion(self) -> bool: + return not self._disabled and self.kv_connector.requires_full_step_completion + def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None: if self._disabled: return diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index b40fb4eac138..e4bf91bf3f8e 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1981,9 +1981,12 @@ def execute_model( # NOTE(woosuk): Here, we don't need to pass the input tensors, # because they are already copied to the CUDA graph input buffers. assert self.cudagraph_manager is not None - self.kv_connector.pre_forward( - **connector_kwargs, attn_metadata=attn_metadata - ) + with set_forward_context( + None, self.vllm_config, cudagraph_runtime_mode=CUDAGraphMode.FULL + ): + self.kv_connector.pre_forward( + **connector_kwargs, attn_metadata=attn_metadata + ) model_output = self.cudagraph_manager.run_fullgraph(batch_desc) else: # For piecewise and eager mode, just call model(). @@ -2259,6 +2262,15 @@ def sample_tokens( model_runner_output.kv_connector_output = kv_connector_output model_runner_output.ec_connector_output = ec_connector_output + if ( + self.speculator is not None + and self.kv_connector.requires_full_step_completion + ): + # Keep early D2H copies, but publish output only after draft KV + # writes finish, before a READ can export or reuse these blocks. + self.output_copy_stream.wait_stream(self.main_stream) + async_output.copy_event.record(self.output_copy_stream) + return async_output def take_draft_token_ids(self) -> DraftTokenIds | None: diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 422ba8885fd7..6b1194c43e57 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -416,15 +416,18 @@ def allocate_kv_cache( # rounding tail it would pin the whole segment at engine shutdown. if current_platform.is_rocm(): warmup_rocm_skinny_gemm_workspaces(device) - # Pad to the page granularity MoRIIO needs to register the shared - # backing as a single RDMA memory region. Other platforms keep the - # exact-size allocation (see #53974), so anything reading - # storage.nbytes() has to tolerate the tail on ROCm alone. + # Align the view as well as its length for MoRIIO's shared RDMA region: + # the ROCm caching allocator need not return a page-aligned base. + # Other platforms keep exact-size storage (see #53974). page_size = 4096 buf_size = ((raw_size + page_size - 1) // page_size) * page_size + allocation = torch.zeros( + buf_size + page_size - 1, dtype=torch.int8, device=device + ) + alignment_offset = (-allocation.data_ptr()) % page_size + buf = allocation.narrow(0, alignment_offset, buf_size) else: - buf_size = raw_size - buf = torch.zeros(buf_size, dtype=torch.int8, device=device) + buf = torch.zeros(raw_size, dtype=torch.int8, device=device) kv_caches: dict[str, torch.Tensor] = {} for tensor in kv_cache_config.kv_cache_tensors: From c093809a079773248bc2cb40508f3be8bf9c15ea Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Mon, 28 Sep 2026 05:31:44 +0000 Subject: [PATCH 2/7] Resolve native PD review findings without changing the shared allocator Co-authored-by: Codex Signed-off-by: whx-sjtu --- tests/v1/core/test_contiguous_kv_packing.py | 34 --- .../kv_connector/unit/test_multi_connector.py | 240 ++++++++++++++++-- .../kv_connector/v1/multi_connector.py | 74 ++++-- vllm/v1/worker/utils.py | 15 +- 4 files changed, 283 insertions(+), 80 deletions(-) diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 373ca5de266b..638deeedc579 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -16,7 +16,6 @@ import pytest import torch -import vllm.v1.worker.utils as worker_utils from vllm.config import CacheConfig from vllm.v1.core.kv_cache_manager import KVCacheManager from vllm.v1.core.kv_cache_utils import ( @@ -126,39 +125,6 @@ def _bind(config, layout: str): return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None) -@pytest.mark.parametrize("is_rocm", [False, True]) -def test_shared_allocation_alignment_preserves_layer_views(monkeypatch, is_rocm): - """ROCm RDMA needs an aligned view even with a misaligned allocator base.""" - groups = [KVCacheGroupSpec(["a", "b"], _mla(128))] - config = get_kv_cache_config_from_groups(_mock_vllm_config("LBNHC"), groups, MEMORY) - monkeypatch.setattr(worker_utils.current_platform, "is_rocm", lambda: is_rocm) - monkeypatch.setattr( - worker_utils, "warmup_rocm_skinny_gemm_workspaces", lambda _: None - ) - zeros = torch.zeros - - def misaligned_zeros(size, **kwargs): - allocation = zeros(size + 4096, **kwargs) - offset = (256 - allocation.data_ptr()) % 4096 - return allocation.narrow(0, offset, size) - - if is_rocm: - monkeypatch.setattr(worker_utils.torch, "zeros", misaligned_zeros) - caches = _bind(config, "LBNHC") - a, b = caches["a"], caches["b"] - assert a.untyped_storage().data_ptr() == b.untyped_storage().data_ptr() - assert b.data_ptr() - a.data_ptr() == config.kv_cache_tensors[0].layer_stride - a.fill_(1) - b.fill_(2) - assert torch.all(a == 1) and torch.all(b == 2) - if is_rocm: - assert a.data_ptr() % 4096 == 0 - assert a.storage_offset() > 0 - else: - assert a.storage_offset() == 0 - assert a.untyped_storage().nbytes() == config.kv_cache_tensors[0].size - - MAIN_KV_PAGE_BYTES = 2_048 COMPRESSED_PAGE_BYTES = 128 NUM_CACHE_TUPLES = 3 diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 2602ac07421b..a4166014f423 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -36,12 +36,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( NixlKVConnectorStats, ) +from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.outputs import ( KVConnectorOutput, KVConnectorWorkerMetadata, ModelRunnerOutput, ) +from vllm.v1.request import RequestStatus MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct" @@ -160,13 +162,11 @@ class MockDivergentHMAConnector(MockHMAConnector): ) -def test_register_finished_partial_tail_notifies_every_connector(): - connector = object.__new__(MultiConnector) - first = MagicMock(spec_set=KVConnectorBase_V1) - second = MagicMock(spec_set=KVConnectorBase_V1) +def test_register_finished_partial_tail_notifies_every_connector(mc): + connector = mc + first, second = connector.sub_connectors first.register_finished_partial_tail.return_value = True second.register_finished_partial_tail.return_value = False - connector._connectors = [first, second] request = MagicMock() block_ids = ([1], [2]) offloads = [(1, 2, 12)] @@ -220,7 +220,9 @@ def async_saves(mc): return mc -def _send_step(connector, notifications=(), *, receiving=None): +def _send_step( + connector, notifications=(), *, receiving=None, failed_receiving=(), scheduler=None +): outputs = [] for rank in range(2): metadata = MultiKVConnectorWorkerMetadata( @@ -238,10 +240,34 @@ def _send_step(connector, notifications=(), *, receiving=None): output = KVOutputAggregator(2).aggregate(outputs).kv_connector_output assert output is not None output.finished_recving = receiving - connector.update_connector_output(output) + output.failed_recving = set(failed_receiving) + if scheduler is None: + connector.update_connector_output(output) + else: + scheduler._update_from_kv_xfer_finished(output) return output +def _scheduler_with_request(connector, status): + request = SimpleNamespace(request_id="r", status=status) + request.is_finished = lambda: RequestStatus.is_finished(request.status) + scheduler = object.__new__(Scheduler) + scheduler.connector = connector + scheduler.requests = {request.request_id: request} + scheduler.finished_recving_kv_req_ids = set() + scheduler._free_request_blocks = MagicMock() + connector.on_new_request(request) + return scheduler, request + + +def _start_async_receive(connector, request): + first, second = connector.sub_connectors + first.get_num_new_matched_tokens.return_value = 16, True + second.get_num_new_matched_tokens.return_value = 0, False + assert connector.get_num_new_matched_tokens(request, 0) == (16, True) + connector.update_state_after_alloc(request, MagicMock(), 16) + + def _finish_with_async_saves(connector): request = SimpleNamespace(request_id="r") connector.on_new_request(request) @@ -249,6 +275,10 @@ def _finish_with_async_saves(connector): def _assert_save_state_cleared(connector): + assert not connector._partial_tail_owners + assert not connector._async_load_candidates + assert not connector._async_load_owners + assert not connector._requests_to_connector assert not connector._async_save_owners assert not connector._completed_async_saves assert not any(connector._send_workers) @@ -319,28 +349,198 @@ def test_send_completion_uses_child_finished_count(async_saves): assert _send_step(async_saves, [(1, 1)]).finished_sending == {"r"} -def test_unexpected_send_and_aborted_receive_release_tracking(async_saves): +def test_unexpected_send_does_not_create_tracking(async_saves): # Unregistered ACKs must not create persistent request state. assert _send_step(async_saves, [(0, 0), (1, 1)]).finished_sending is None _assert_save_state_cleared(async_saves) - _finish_with_async_saves(async_saves) - assert _send_step(async_saves, [(0, 0)]).finished_sending is None - output = _send_step(async_saves, receiving={"r"}) + + +@pytest.mark.parametrize("receiver_saves", [False, True]) +@pytest.mark.parametrize("first_completion", ["receive", "send", "both"]) +def test_aborted_receive_waits_for_another_send_owner( + async_saves, receiver_saves, first_completion +): + """Finishing a cancelled load must not free another child's save source.""" + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS + ) + _start_async_receive(async_saves, request) + async_saves.sub_connectors[0].request_finished.return_value = ( + receiver_saves, + None, + ) + request.status = RequestStatus.FINISHED_ABORTED + assert async_saves.request_finished(request, [1])[0] + + send_acks = [(1, 0), (1, 1)] + if receiver_saves: + send_acks += [(0, 0), (0, 1)] + completions: dict[str, dict[str, set[str] | list[tuple[int, int]]]] = { + "receive": {"receiving": {"r"}}, + "send": {"notifications": send_acks}, + "both": {"receiving": {"r"}, "notifications": send_acks}, + } + output = _send_step( + async_saves, scheduler=scheduler, **completions[first_completion] + ) + if first_completion != "both": + scheduler._free_request_blocks.assert_not_called() + assert output.finished_recving is None + assert output.finished_sending is None + assert "r" in scheduler.requests + remaining = "send" if first_completion == "receive" else "receive" + _send_step(async_saves, scheduler=scheduler, **completions[remaining]) + scheduler._free_request_blocks.assert_called_once_with(request) + assert not scheduler.requests + _assert_save_state_cleared(async_saves) + # Duplicate terminal notifications cannot trigger a second block release. + _send_step(async_saves, scheduler=scheduler, **completions["both"]) + scheduler._free_request_blocks.assert_called_once_with(request) + + +def test_aborted_receive_only_releases_once_and_allows_request_id_reuse( + async_saves, +): + async_saves.sub_connectors[0].request_finished.return_value = False, None + async_saves.sub_connectors[1].request_finished.return_value = False, None + for _ in range(2): + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS + ) + _start_async_receive(async_saves, request) + request.status = RequestStatus.FINISHED_ABORTED + assert async_saves.request_finished(request, [1])[0] + _send_step(async_saves, receiving={"r"}, scheduler=scheduler) + scheduler._free_request_blocks.assert_called_once_with(request) + assert not scheduler.requests + _assert_save_state_cleared(async_saves) + + +def test_receiving_child_save_is_independent_of_its_receive(async_saves): + """A receiver may still be saving blocks from an earlier prefill.""" + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS + ) + _start_async_receive(async_saves, request) + request.status = RequestStatus.FINISHED_ABORTED + assert async_saves.request_finished(request, [1])[0] + _send_step(async_saves, [(1, 0), (1, 1)], receiving={"r"}, scheduler=scheduler) + scheduler._free_request_blocks.assert_not_called() + _send_step(async_saves, [(0, 0), (0, 1)], scheduler=scheduler) + scheduler._free_request_blocks.assert_called_once_with(request) + _assert_save_state_cleared(async_saves) + + +@pytest.mark.parametrize("failed", [False, True]) +def test_normal_receive_keeps_tracking_for_later_saves(async_saves, failed): + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS + ) + _start_async_receive(async_saves, request) + output = _send_step( + async_saves, + receiving={"r"}, + failed_receiving={"r"} if failed else (), + scheduler=scheduler, + ) assert output.finished_recving == {"r"} - assert output.finished_sending is None + assert output.failed_recving == ({"r"} if failed else set()) + assert scheduler.finished_recving_kv_req_ids == {"r"} + scheduler._free_request_blocks.assert_not_called() + request.status = RequestStatus.FINISHED_STOPPED + assert async_saves.request_finished(request, [1])[0] + _send_step(async_saves, [(1, 0), (1, 1)], scheduler=scheduler) + scheduler._free_request_blocks.assert_not_called() + _send_step(async_saves, [(0, 0), (0, 1)], scheduler=scheduler) + scheduler._free_request_blocks.assert_called_once_with(request) _assert_save_state_cleared(async_saves) - assert _send_step(async_saves, [(0, 1), (1, 0), (1, 1)]).finished_sending is None + + +@pytest.mark.parametrize("allocate_zero_tokens", [False, True]) +def test_async_lookup_without_external_allocation_does_not_create_a_receive_wait( + async_saves, allocate_zero_tokens +): + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.FINISHED_ABORTED + ) + first, second = async_saves.sub_connectors + first.get_num_new_matched_tokens.return_value = 16, True + second.get_num_new_matched_tokens.return_value = 0, False + assert async_saves.get_num_new_matched_tokens(request, 0) == (16, True) + if allocate_zero_tokens: + async_saves.update_state_after_alloc(request, MagicMock(), 0) + assert async_saves.request_finished(request, [1])[0] + _send_step(async_saves, [(0, 0), (0, 1), (1, 0), (1, 1)], scheduler=scheduler) + scheduler._free_request_blocks.assert_called_once_with(request) _assert_save_state_cleared(async_saves) -def test_normal_receive_keeps_tracking_for_later_saves(async_saves): - request = SimpleNamespace(request_id="r") - async_saves.on_new_request(request) - assert _send_step(async_saves, receiving={"r"}).finished_recving == {"r"} +def test_partial_tail_owner_releases_after_its_send_ack(async_saves): + """Scheduler must hear completion even if only the tail hook delays free.""" + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.FINISHED_STOPPED + ) + first, second = async_saves.sub_connectors + first.register_finished_partial_tail.return_value = True + second.register_finished_partial_tail.return_value = False + for child in async_saves.sub_connectors: + child.request_finished.return_value = False, None + partial_tail_delay = async_saves.register_finished_partial_tail( + request, ([1],), [(0, 1, 12)] + ) + delay_free, _ = async_saves.request_finished(request, [1]) + assert partial_tail_delay or delay_free + _send_step(async_saves, [(0, 0)], scheduler=scheduler) + scheduler._free_request_blocks.assert_not_called() + _send_step(async_saves, [(0, 1)], scheduler=scheduler) + scheduler._free_request_blocks.assert_called_once_with(request) + assert not scheduler.requests + _assert_save_state_cleared(async_saves) + + +@pytest.mark.parametrize("early_ranks", [(0,), (0, 1)]) +def test_partial_tail_requires_new_acks_after_earlier_saves(async_saves, early_ranks): + """A newly accepted tail must not reuse earlier full-block send ACKs.""" + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.FINISHED_STOPPED + ) + first, second = async_saves.sub_connectors + first.register_finished_partial_tail.return_value = True + second.register_finished_partial_tail.return_value = False + first.request_finished.return_value = False, None + _send_step(async_saves, [(0, rank) for rank in early_ranks], scheduler=scheduler) + assert async_saves.register_finished_partial_tail(request, ([1],), [(0, 1, 12)]) + assert async_saves.request_finished(request, [1])[0] + _send_step(async_saves, [(0, 1), (1, 0), (1, 1)], scheduler=scheduler) + scheduler._free_request_blocks.assert_not_called() + _send_step(async_saves, [(0, 0)], scheduler=scheduler) + scheduler._free_request_blocks.assert_called_once_with(request) + _assert_save_state_cleared(async_saves) + + +@pytest.mark.parametrize("receive_first", [False, True]) +def test_partial_tail_and_receive_from_same_child_both_delay_free( + async_saves, receive_first +): + scheduler, request = _scheduler_with_request( + async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS + ) + _start_async_receive(async_saves, request) + first, second = async_saves.sub_connectors + first.register_finished_partial_tail.return_value = True + second.register_finished_partial_tail.return_value = False + for child in async_saves.sub_connectors: + child.request_finished.return_value = False, None + request.status = RequestStatus.FINISHED_ABORTED + assert async_saves.register_finished_partial_tail(request, ([1],), [(0, 1, 12)]) assert async_saves.request_finished(request, [1])[0] - assert _send_step( - async_saves, [(0, 0), (0, 1), (1, 0), (1, 1)] - ).finished_sending == {"r"} + receive = {"receiving": {"r"}} + send = {"notifications": [(0, 0), (0, 1)]} + _send_step(async_saves, scheduler=scheduler, **(receive if receive_first else send)) + scheduler._free_request_blocks.assert_not_called() + _send_step(async_saves, scheduler=scheduler, **(send if receive_first else receive)) + scheduler._free_request_blocks.assert_called_once_with(request) + assert not scheduler.requests _assert_save_state_cleared(async_saves) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index e140b69fff5e..29d61cae20a2 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -211,8 +211,11 @@ def __init__( # A mapping from request id to the index of the connector chosen to # load the request from (if any). self._requests_to_connector: dict[str, int] = {} + self._async_load_candidates: set[str] = set() + self._async_load_owners: dict[str, int] = {} # Scheduler-side owners and completions, including ACKs before finish. + self._partial_tail_owners: dict[str, set[int]] = {} self._async_save_owners: dict[str, set[int]] = {} self._completed_async_saves: dict[str, set[int]] = {} self._send_workers: tuple[dict[str, set[int]], ...] = tuple( @@ -442,6 +445,8 @@ def get_num_new_matched_tokens( request: "Request", num_computed_tokens: int, ) -> tuple[int | None, bool]: + self._requests_to_connector.pop(request.request_id, None) + self._async_load_candidates.discard(request.request_id) to_return = (0, False) for i, c in enumerate(self._connectors): toks, load_async = c.get_num_new_matched_tokens( @@ -455,6 +460,8 @@ def get_num_new_matched_tokens( # to this request. if to_return[0] == 0 and toks > 0: self._requests_to_connector[request.request_id] = i + if load_async: + self._async_load_candidates.add(request.request_id) to_return = (toks, load_async) return to_return @@ -475,6 +482,12 @@ def update_state_after_alloc( else: # Other connectors still receive the request's real blocks c.update_state_after_alloc(request, blocks, 0) + if ( + num_external_tokens > 0 + and request.request_id in self._async_load_candidates + ): + self._async_load_owners[request.request_id] = chosen_connector + self._async_load_candidates.discard(request.request_id) def on_new_request(self, request: "Request") -> None: self._completed_async_saves[request.request_id] = set() @@ -491,8 +504,12 @@ def build_connector_meta( ) def _forget_async_saves(self, req_id: str) -> None: + self._partial_tail_owners.pop(req_id, None) self._async_save_owners.pop(req_id, None) self._completed_async_saves.pop(req_id, None) + self._async_load_owners.pop(req_id, None) + self._async_load_candidates.discard(req_id) + self._requests_to_connector.pop(req_id, None) for workers in self._send_workers: workers.pop(req_id, None) @@ -549,17 +566,25 @@ def update_connector_output(self, connector_output: KVConnectorOutput): self._send_workers[i].pop(req_id, None) if owners is not None: owners.discard(i) - if not owners: + if not owners and req_id not in self._async_load_owners: finished_sending.add(req_id) finally: # restore kv_connector_worker_meta connector_output.kv_connector_worker_meta = multi_connector_worker_meta - connector_output.finished_sending = finished_sending or None - aborted_recvs = ( - set(connector_output.finished_recving or ()) - & self._async_save_owners.keys() - ) - for req_id in finished_sending | aborted_recvs: + finished_recving: set[str] = set() + for req_id in connector_output.finished_recving or (): + if self._async_load_owners.pop(req_id, None) is None: + continue + owners = self._async_save_owners.get(req_id) + if owners is None: + # A live request can resume as soon as its selected load ends. + finished_recving.add(req_id) + elif not owners: + # A finished request releases once both load and save are done. + finished_sending.add(req_id) + connector_output.finished_recving = finished_recving or None + connector_output.finished_sending = finished_sending or None + for req_id in finished_sending: self._forget_async_saves(req_id) def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None: @@ -594,11 +619,12 @@ def _aggregate_request_finished( [KVConnectorBase_V1], tuple[bool, dict[str, Any] | None] ], ) -> tuple[bool, dict[str, Any] | None]: - async_save_owners: set[int] = set() + async_save_owners = self._partial_tail_owners.pop(request.request_id, set()) + load_owner = self._async_load_owners.get(request.request_id) kv_txfer_params = None for i, c in enumerate(self._connectors): - async_save, txfer_params = per_connector_fn(c) - if async_save: + delay_free, txfer_params = per_connector_fn(c) + if delay_free: async_save_owners.add(i) if txfer_params is not None: if kv_txfer_params is not None: @@ -613,14 +639,15 @@ def _aggregate_request_finished( kv_txfer_params = txfer_params completed = self._completed_async_saves.setdefault(request.request_id, set()) pending = async_save_owners - completed - if pending: + delay_free = bool(pending) or load_owner is not None + if delay_free: self._async_save_owners[request.request_id] = pending else: self._forget_async_saves(request.request_id) self._requests_to_connector.pop(request.request_id, None) - return bool(pending), kv_txfer_params + return delay_free, kv_txfer_params def request_finished( self, @@ -638,11 +665,24 @@ def register_finished_partial_tail( block_ids: tuple[list[int], ...], partial_tail_offloads: list[tuple[int, int, int]], ) -> bool: - accepted = [ - c.register_finished_partial_tail(request, block_ids, partial_tail_offloads) - for c in self._connectors - ] - return any(accepted) + owners = { + i + for i, c in enumerate(self._connectors) + if c.register_finished_partial_tail( + request, block_ids, partial_tail_offloads + ) + } + if owners: + self._partial_tail_owners.setdefault(request.request_id, set()).update( + owners + ) + # A finish-time tail is new work, independent of earlier save ACKs. + self._completed_async_saves.setdefault( + request.request_id, set() + ).difference_update(owners) + for i in owners: + self._send_workers[i].pop(request.request_id, None) + return bool(owners) def request_finished_all_groups( self, diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 6b1194c43e57..422ba8885fd7 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -416,18 +416,15 @@ def allocate_kv_cache( # rounding tail it would pin the whole segment at engine shutdown. if current_platform.is_rocm(): warmup_rocm_skinny_gemm_workspaces(device) - # Align the view as well as its length for MoRIIO's shared RDMA region: - # the ROCm caching allocator need not return a page-aligned base. - # Other platforms keep exact-size storage (see #53974). + # Pad to the page granularity MoRIIO needs to register the shared + # backing as a single RDMA memory region. Other platforms keep the + # exact-size allocation (see #53974), so anything reading + # storage.nbytes() has to tolerate the tail on ROCm alone. page_size = 4096 buf_size = ((raw_size + page_size - 1) // page_size) * page_size - allocation = torch.zeros( - buf_size + page_size - 1, dtype=torch.int8, device=device - ) - alignment_offset = (-allocation.data_ptr()) % page_size - buf = allocation.narrow(0, alignment_offset, buf_size) else: - buf = torch.zeros(raw_size, dtype=torch.int8, device=device) + buf_size = raw_size + buf = torch.zeros(buf_size, dtype=torch.int8, device=device) kv_caches: dict[str, torch.Tensor] = {} for tensor in kv_cache_config.kv_cache_tensors: From 0f7d0c3a1c9d9e1d648e6197c10e063104d14f84 Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Tue, 29 Sep 2026 07:30:28 +0000 Subject: [PATCH 3/7] Extract sync-load zeroing into a standalone main-based fix Remove the connector hook, scheduler exclusions, and their tests from this broader PD branch. The independent MoRIIO zeroing fix includes both LBHNC and LBNHC and remains part of the complete hybrid READ validation stack. Co-authored-by: Codex Signed-off-by: whx-sjtu --- tests/v1/core/test_scheduler.py | 38 ------------------- .../kv_connector/unit/test_moriio_tp_ack.py | 32 ---------------- .../kv_connector/unit/test_multi_connector.py | 12 ------ .../kv_transfer/kv_connector/v1/base.py | 9 ----- .../v1/moriio/moriio_connector.py | 18 --------- .../kv_connector/v1/multi_connector.py | 6 --- vllm/v1/core/sched/scheduler.py | 7 +--- 7 files changed, 1 insertion(+), 121 deletions(-) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index d9e27783a2b0..722cbfb79e00 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -2566,44 +2566,6 @@ def test_has_sync_kv_loads( assert output.has_sync_kv_loads is expected_has_sync_loads -@pytest.mark.parametrize("overwrite_full_blocks", [False, True]) -@pytest.mark.skip_global_cleanup -def test_sync_kv_load_only_skips_zeroing_overwritten_blocks( - overwrite_full_blocks: bool, tmp_path -): - """A synchronous RDMA load must not race zeroing of its destinations.""" - (tmp_path / "config.json").write_text( - '{"architectures": ["OPTForCausalLM"], "model_type": "opt"}' - ) - scheduler = create_scheduler( - model=str(tmp_path), - skip_tokenizer_init=True, - use_kv_connector=mock_kv(matched_tokens=16, is_async=False), - block_size=16, - ) - scheduler.needs_kv_cache_zeroing = True - # Full-attention pages also require zeroing in a mixed-precision pool. - manager = scheduler.kv_cache_manager.coordinator.single_type_managers[0] - manager._record_new_block_ids = True - request = create_requests(num_requests=1, num_tokens=48, block_size=16)[0] - scheduler.add_request(request) - - def loaded_blocks(request): - blocks = scheduler.kv_cache_manager.get_blocks(request.request_id) - return list(blocks.get_block_ids()[0][:1]) - - if overwrite_full_blocks: - scheduler.connector.get_sync_load_block_ids = loaded_blocks - - output = scheduler.schedule() - blocks = list( - scheduler.kv_cache_manager.get_blocks(request.request_id).get_block_ids()[0] - ) - expected = blocks[1:] if overwrite_full_blocks else blocks - assert output.new_block_ids_to_zero == expected - assert not scheduler._skip_zero_block_ids - - def test_kv_connector_honors_skip_reading_prefix_cache(): """A request that must score every prompt row takes no external hit.""" BLOCK_SIZE = 16 diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py index efe1fe9e12f3..bcf630b9e4a3 100644 --- a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py +++ b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py @@ -48,38 +48,6 @@ def test_read_matches_only_requests_with_pending_remote_prefill(params, expected assert scheduler.get_num_new_matched_tokens(request, 0) == expected -@pytest.mark.parametrize( - "mode,consumer,has_mamba,layout,pending,expected", - [ - (MoRIIOMode.READ, True, True, "LBNHC", [[7, 8], [90]], [7, 8]), - (MoRIIOMode.READ, True, True, "LBNHC", [[], [90]], []), - (MoRIIOMode.READ, True, True, "LBNHC", None, []), - (MoRIIOMode.READ, False, True, "LBNHC", [[7], [90]], []), - (MoRIIOMode.READ, True, False, "LBNHC", [[7]], []), - (MoRIIOMode.READ, True, True, "NBLHC", [[7], [90]], []), - (MoRIIOMode.WRITE, True, True, "LBNHC", [[7], [90]], []), - ], -) -def test_sync_read_initializes_only_supported_attention_destinations( - mode, consumer, has_mamba, layout, pending, expected -): - connector = MoRIIOConnector.__new__(MoRIIOConnector) - connector.mode = mode - connector.kv_transfer_config = SimpleNamespace(is_kv_consumer=consumer) - connector._vllm_config = SimpleNamespace( - cache_config=SimpleNamespace( - get_resolved_kv_cache_layout=lambda: SimpleNamespace(name=layout) - ) - ) - request = SimpleNamespace(request_id="req") - connector.connector_scheduler = SimpleNamespace( - _has_mamba=has_mamba, - _reqs_need_recv={} if pending is None else {"req": (request, pending)}, - ) - - assert connector.get_sync_load_block_ids(request) == expected - - @pytest.mark.parametrize("mode", [MoRIIOMode.READ, MoRIIOMode.WRITE]) def test_read_requires_completion_of_draft_kv_writes(mode): connector = MoRIIOConnector.__new__(MoRIIOConnector) diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index a4166014f423..483857ba1109 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -553,18 +553,6 @@ def test_no_async_saves_release_tracking(async_saves): _assert_save_state_cleared(async_saves) -def test_sync_load_zeroing_exclusion_uses_only_selected_connector(mc): - """An unselected child cannot suppress initialization of fresh pages.""" - request = SimpleNamespace(request_id="r") - first, second = mc.sub_connectors - first.get_sync_load_block_ids.return_value = [1, 2] - second.get_sync_load_block_ids.return_value = [3] - assert mc.get_sync_load_block_ids(request) == [] - mc._requests_to_connector["r"] = 1 - assert mc.get_sync_load_block_ids(request) == [3] - first.get_sync_load_block_ids.assert_not_called() - - def test_worker_preserves_send_identity_and_receive_failures(async_saves, monkeypatch): from vllm.distributed.kv_transfer.kv_connector.v1 import multi_connector diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index 363fe8d653c3..397d1a3f7c6e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -188,15 +188,6 @@ def supports_divergent_local_hybrid_hits(self) -> bool: """ return False - def get_sync_load_block_ids(self, request: "Request") -> list[int]: - """Return blocks whose synchronous load replaces worker zeroing. - - Each block must be fully initialized before forward consumes it. A - failed load must abort forward instead of recomputing with uninitialized - blocks. Defaults to no blocks. - """ - return [] - @property def requires_full_step_completion(self) -> bool: """Whether output readiness must cover post-sampling KV writes. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 4fd131be444d..0ad0f29f1033 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -232,24 +232,6 @@ def supports_divergent_local_hybrid_hits(self) -> bool: def requires_full_step_completion(self) -> bool: return self.mode == MoRIIOMode.READ - def get_sync_load_block_ids(self, request: "Request") -> list[int]: - scheduler = self.connector_scheduler - if ( - self.mode != MoRIIOMode.READ - or not self.kv_transfer_config.is_kv_consumer - or scheduler is None - or not scheduler._has_mamba - or self._vllm_config.cache_config.get_resolved_kv_cache_layout().name - != "LBNHC" - ): - return [] - pending = scheduler._reqs_need_recv.get(request.request_id) - if pending is None: - return [] - # Hybrid READ fills these entire attention pages and aborts on failure. - # The destination list already excludes local hits and lookahead blocks. - return pending[1][0] - def __init__( self, vllm_config: VllmConfig, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index 29d61cae20a2..15bc88299a3f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -465,12 +465,6 @@ def get_num_new_matched_tokens( to_return = (toks, load_async) return to_return - def get_sync_load_block_ids(self, request: "Request") -> list[int]: - chosen = self._requests_to_connector.get(request.request_id) - if chosen is None: - return [] - return self._connectors[chosen].get_sync_load_block_ids(request) - def update_state_after_alloc( self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int ): diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 9d2c341d4dca..fcb1421b1046 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -349,7 +349,7 @@ def __init__( self.has_mamba_layers = kv_cache_config.has_mamba_layers self.needs_kv_cache_zeroing = kv_cache_config.needs_kv_cache_zeroing - # Blocks that external KV loads will overwrite this step, skipped from + # Blocks that async KV loads will overwrite this step, skipped from # zeroing since the zeroing could race the out-of-band write. self._skip_zero_block_ids: set[int] = set() self.need_mamba_block_aligned_split = ( @@ -1295,11 +1295,6 @@ def skip_request(from_queue: RequestQueue) -> None: if num_external_computed_tokens > 0: # load_kv_async is False here has_sync_kv_loads = True - if self.needs_kv_cache_zeroing: - assert self.connector is not None - self._skip_zero_block_ids.update( - self.connector.get_sync_load_block_ids(request) - ) if self.log_stats: request.record_event( EngineCoreEventType.SCHEDULED, scheduled_timestamp From 6f25f231d403bc7f9e089c6eeb94e0cf8a3563e5 Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Wed, 30 Sep 2026 14:05:31 +0000 Subject: [PATCH 4/7] Extract discovery heartbeats into a standalone main-based fix Move MoRIIOHeartbeat, the worker startup/shutdown hooks, and their two process lifecycle regressions to whx/moriio-discovery-heartbeat. Restore only the discovery thread behavior from this branch base; preserve all remaining KV ownership, ordering, registration and parser fixes. Co-authored-by: Codex Signed-off-by: whx-sjtu --- .../unit/test_moriio_proxy_routing.py | 96 ------------------ .../v1/moriio/moriio_connector.py | 98 ++++++++++++++----- .../v1/moriio/moriio_heartbeat.py | 86 ---------------- 3 files changed, 71 insertions(+), 209 deletions(-) delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py diff --git a/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py b/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py index 68066ede1c3d..71ee0ef5d799 100644 --- a/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py +++ b/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py @@ -22,20 +22,14 @@ """ import contextlib -import ctypes import importlib.util -import os -import subprocess import sys -import time import types from collections import Counter from pathlib import Path from typing import cast -import msgpack import pytest -import zmq _MISSING = object() @@ -209,93 +203,3 @@ def test_consecutive_requests_alternate_instances(route): # than filling one instance's ranks before moving on. insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)] assert insts == [0, 1, 2] - - -@pytest.mark.skipif(sys.platform != "linux", reason="GIL probe uses libc.usleep") -def test_discovery_heartbeat_progresses_while_worker_holds_gil(): - from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( - MoRIIOHeartbeat, - ) - - payload = { - "type": "P", - "http_address": "127.0.0.1:8000", - "zmq_address": "host:127.0.0.1,handshake:6301,notify:61005", - "dp_size": 1, - "tp_size": 8, - "transfer_mode": "READ", - } - with zmq.Context() as context, context.socket(zmq.ROUTER) as socket: - port = socket.bind_to_random_port("tcp://127.0.0.1") - heartbeat = MoRIIOHeartbeat(f"tcp://127.0.0.1:{port}", payload, 0.02) - try: - assert socket.poll(5000), "Discovery did not register" - assert msgpack.unpackb(socket.recv_multipart()[1]) == payload - # PyDLL retains the GIL throughout the native call. - ctypes.PyDLL(None).usleep(300_000) - received = 0 - while socket.poll(0): - assert msgpack.unpackb(socket.recv_multipart()[1]) == payload - received += 1 - assert received >= 3, "Discovery stalled while the worker held the GIL" - finally: - heartbeat.shutdown() - heartbeat.shutdown() - while socket.poll(0): - socket.recv_multipart() - assert not socket.poll(100), "Heartbeats continued after shutdown" - - -@pytest.mark.skipif(sys.platform != "linux", reason="Checks child exit via procfs") -def test_discovery_heartbeat_exits_when_worker_dies(): - from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_heartbeat - - owner_code = """ -import runpy -import sys -Heartbeat = runpy.run_path(sys.argv[1])["MoRIIOHeartbeat"] -heartbeat = Heartbeat(sys.argv[2], {"type": "D"}, 0.02) -print(heartbeat._process.pid, flush=True) -sys.stdin.read() -""" - with zmq.Context() as context, context.socket(zmq.ROUTER) as socket: - port = socket.bind_to_random_port("tcp://127.0.0.1") - owner = subprocess.Popen( - [ - sys.executable, - "-c", - owner_code, - moriio_heartbeat.__file__, - f"tcp://127.0.0.1:{port}", - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - text=True, - ) - child_pid = None - child_exited = False - try: - assert socket.poll(5000), "Discovery did not register" - assert owner.stdout is not None - child_pid = int(owner.stdout.readline()) - owner.kill() - owner.wait(timeout=5) - stat = Path(f"/proc/{child_pid}/stat") - deadline = time.monotonic() + 5 - while stat.exists(): - # A terminated child may await reaping by the container's init. - try: - if stat.read_text().split(") ")[1].startswith("Z "): - break - except FileNotFoundError: - break - assert time.monotonic() < deadline, "Orphan heartbeat survived worker" - time.sleep(0.01) - child_exited = True - finally: - if owner.poll() is None: - owner.kill() - owner.communicate(timeout=5) - if child_pid is not None and not child_exited: - with contextlib.suppress(ProcessLookupError): - os.kill(child_pid, 9) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 0ad0f29f1033..a8c94e470f72 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -59,9 +59,6 @@ MoRIIOWrapper, MoRIIOWriter, ) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( - MoRIIOHeartbeat, -) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout import ( LayerTransferGeometry, MambaOffsetTemplate, @@ -1624,7 +1621,7 @@ def __init__( self.moriio_engine = None self._handle_request_thread = None - self._heartbeat: MoRIIOHeartbeat | None = None + self._ping_thread = None self._writer = MoRIIOWriter(self) # Completions that arrived before transfer_id_to_request_id was populated. # Retried each step until the mapping is established. @@ -1654,7 +1651,10 @@ def __init__( ) if self._rank == 0 and self.moriio_config.proxy_ip: - self._heartbeat = self._start_heartbeat() + self._ping_thread = threading.Thread( + target=self._ping, args=(self.zmq_context,), daemon=True + ) + self._ping_thread.start() logger.info( "Initializing MoRIIO Engine, engine = %s, role = %s", @@ -1903,31 +1903,75 @@ def _get_built_session(self, remote_engine_id): remote_engine_id ] - def _start_heartbeat(self) -> MoRIIOHeartbeat: - payload = { - "type": "P" if self.is_producer else "D", - "http_address": self.request_address, - "zmq_address": ( - f"host:{self.local_ip},handshake:{self.handshake_port}," - f"notify:{self.notify_port}" - ), - "dp_size": self.moriio_config.dp_size, - "tp_size": self.moriio_config.tp_size, - "transfer_mode": self.mode.name, - } - return MoRIIOHeartbeat( - f"tcp://{self.proxy_ip}:{self.proxy_ping_port}", - payload, - MoRIIOConstants.PING_INTERVAL, - MoRIIOConstants.MAX_PING_RETRIES, + def _ping(self, zmq_context): + # Use host:port format for http_address (compatible with official router) + http_address = f"{self.request_address}" + # Include host so the router embeds it in the request_id; the connector + # on the other side parses host/ports from there. + zmq_address = ( + f"host:{self.local_ip}," + f"handshake:{self.handshake_port}," + f"notify:{self.notify_port}" ) + role = "P" if self.is_producer else "D" - def shutdown(self): - heartbeat = getattr(self, "_heartbeat", None) - if heartbeat is not None: - heartbeat.shutdown() - self._heartbeat = None + retry_count = 0 + index = 1 + with zmq_context.socket(zmq.DEALER) as sock: + sock.connect(f"tcp://{self.proxy_ip}:{self.proxy_ping_port}") + while True: + try: + data = { + "type": role, # "P" or "D" + "http_address": http_address, + "zmq_address": zmq_address, + # dp_size/tp_size are not used by the official vLLM router + # (routing operates at the http_address level); they are + # consumed only by the toy proxy server. + "dp_size": self.moriio_config.dp_size, + "tp_size": self.moriio_config.tp_size, + # transfer_mode is included so the router can distinguish + # READ (prefill-then-decode, sequential) from WRITE (concurrent) + # scheduling. + "transfer_mode": self.mode.name, + } + + sock.send(msgpack.dumps(data)) + # logger.debug(f"Successfully sent ping message #{index}") + retry_count = 0 + + except ConnectionRefusedError: + logger.info( + "Connection refused: %s:%s -> %s:%s", + self.local_ip, + self.local_ping_port, + self.proxy_ip, + self.proxy_ping_port, + ) + retry_count += 1 + + except OSError as e: + logger.info("OS error when sending ping: %s", e) + retry_count += 1 + + except Exception as e: + logger.info("Unexpected error when sending ping: %s", e) + retry_count += 1 + if retry_count >= MoRIIOConstants.MAX_PING_RETRIES: + logger.error( + "Max retries (%s) exceeded. Stopping ping loop.", + MoRIIOConstants.MAX_PING_RETRIES, + ) + raise RuntimeError( + f"Ping failed after {retry_count} retries" + ) from e + + finally: + time.sleep(MoRIIOConstants.PING_INTERVAL) + index += 1 + + def shutdown(self): if hasattr(self, "moriio_wrapper") and self.moriio_wrapper: self.moriio_wrapper.shutdown() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py deleted file mode 100644 index 01a7f38c59f2..000000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Keep discovery heartbeats independent of the model worker's GIL.""" - -import json -import logging -import select -import subprocess -import sys -from typing import Any - -import msgpack -import zmq - -logger = logging.getLogger(__name__) - - -class MoRIIOHeartbeat: - def __init__( - self, - endpoint: str, - payload: dict[str, Any], - interval: float, - max_retries: int = 100, - ): - self._process = subprocess.Popen( - [sys.executable, __file__, endpoint, str(interval), str(max_retries)], - stdin=subprocess.PIPE, - text=True, - ) - assert self._process.stdin is not None - try: - self._process.stdin.write(json.dumps(payload) + "\n") - self._process.stdin.flush() - except BaseException: - self.shutdown() - raise - - def shutdown(self) -> None: - assert self._process.stdin is not None - try: - if not self._process.stdin.closed: - self._process.stdin.close() - except BrokenPipeError: - pass - finally: - try: - self._process.wait(timeout=5) - except subprocess.TimeoutExpired: - self._process.terminate() - try: - self._process.wait(timeout=1) - except subprocess.TimeoutExpired: - self._process.kill() - self._process.wait(timeout=1) - - -def main() -> None: - endpoint, interval_arg, max_retries_arg = sys.argv[1:] - initial = sys.stdin.readline() - if not initial: - return - payload = msgpack.dumps(json.loads(initial)) - interval = float(interval_arg) - max_retries = int(max_retries_arg) - failures = 0 - with zmq.Context() as context, context.socket(zmq.DEALER) as sock: - sock.setsockopt(zmq.LINGER, 0) - sock.setsockopt(zmq.SNDTIMEO, 1000) - sock.connect(endpoint) - while not select.select([sys.stdin], [], [], 0)[0]: - try: - sock.send(payload) - failures = 0 - except zmq.ZMQError: - failures += 1 - logger.exception("MoRIIO discovery heartbeat send failed") - if failures >= max_retries: - raise - # EOF also stops the child when the worker exits without cleanup. - if select.select([sys.stdin], [], [], interval)[0]: - break - - -if __name__ == "__main__": - main() From 1bb6e2b97edaba91f6b94def5f8149985c46cdd3 Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Thu, 1 Oct 2026 08:31:09 +0000 Subject: [PATCH 5/7] [MoRIIO] Align READ regression fixtures with explicit prefill eligibility Keep hybrid N-1 and embeddings-only positive cases eligible for remote READ without weakening the ordinary-request guard. Adapt the EOF parser regression to the current optional parser factory and tokenizer protocol. Signed-off-by: whx-sjtu Co-Authored-By: Claude Code --- tests/reasoning/test_kimi_k3_reasoning_parser.py | 3 ++- tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/reasoning/test_kimi_k3_reasoning_parser.py b/tests/reasoning/test_kimi_k3_reasoning_parser.py index b6c2570ad715..67329988ab2d 100644 --- a/tests/reasoning/test_kimi_k3_reasoning_parser.py +++ b/tests/reasoning/test_kimi_k3_reasoning_parser.py @@ -394,7 +394,8 @@ def encode(self, text, add_special_tokens=False): tool_parser_name="kimi_k3", enable_auto_tools=True, ) - parser = parser_cls(Tokenizer()) + assert parser_cls is not None + parser = parser_cls(cast(TokenizerLike, Tokenizer())) request = ChatCompletionRequest( model="test-model", messages=[], include_reasoning=False ) diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py index 87ea53225a5f..7523724de015 100644 --- a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py +++ b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py @@ -828,7 +828,7 @@ def test_get_num_new_matched_tokens_read_recomputes_last_token(): req = SimpleNamespace( num_prompt_tokens=10, prompt_token_ids=list(range(10)), - kv_transfer_params=None, + kv_transfer_params={"do_remote_prefill": True}, ) n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0) # READ always recomputes the final token locally: N-1 - computed. @@ -865,7 +865,9 @@ def test_get_num_new_matched_tokens_supports_embeds_only_prompts( num_prompt_tokens=10, prompt_token_ids=None, prompt_embeds=object(), - kv_transfer_params=None, + kv_transfer_params={"do_remote_prefill": True} + if mode == MoRIIOMode.READ + else None, ) assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == ( From b5626f70f21579165f535da84fd036c131223700 Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Thu, 1 Oct 2026 08:50:43 +0000 Subject: [PATCH 6/7] [KVConnector] Isolate nested MultiConnector completion updates Keep original receive notifications for outer ownership reconciliation while forwarding independent child completion sets and metadata. Preserve auxiliary/error output updates. Cover both child orders and cancelled nested loads with pending sibling saves. Co-Authored-By: Claude Code Signed-off-by: whx-sjtu --- .../kv_connector/unit/test_multi_connector.py | 133 ++++++++++++++++++ .../kv_connector/v1/multi_connector.py | 107 ++++++++------ 2 files changed, 194 insertions(+), 46 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 483857ba1109..281e8a5efb97 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -314,6 +314,139 @@ def filter_completion(output): _assert_save_state_cleared(async_saves) +@pytest.fixture(params=[False, True], ids=["direct-first", "nested-first"]) +def nested_mc(request): + child = { + "kv_connector": "MockConnector", + "kv_role": "kv_both", + "kv_connector_module_path": __name__, + } + children = [ + child, + { + "kv_connector": "MultiConnector", + "kv_role": "kv_both", + "kv_connector_extra_config": {"connectors": [child]}, + }, + ] + if request.param: + children.reverse() + config = create_vllm_config( + kv_connector="MultiConnector", + kv_connector_extra_config={"connectors": children}, + ) + connector = MultiConnector( + config, + KVConnectorRole.SCHEDULER, + KVCacheConfig(num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]), + ) + nested_first = request.param + direct = connector.sub_connectors[int(nested_first)] + nested = connector.sub_connectors[int(not nested_first)] + for leaf in [direct, *nested.sub_connectors]: + leaf.get_num_new_matched_tokens.return_value = 0, False + leaf.get_finished_count.return_value = 1 + leaf.request_finished.return_value = False, None + return connector, direct, nested, int(nested_first) + + +def test_nested_nonowner_does_not_swallow_receive_completion(nested_mc): + """A nested non-loader must not strand the selected child's request.""" + nested_mc, direct, nested, direct_index = nested_mc + direct.get_num_new_matched_tokens.return_value = 16, True + request = SimpleNamespace(request_id="r") + nested_mc.on_new_request(request) + assert nested_mc.get_num_new_matched_tokens(request, 0) == (16, True) + nested_mc.update_state_after_alloc(request, MagicMock(), 16) + + leaf_meta = MagicMock(spec=KVConnectorWorkerMetadata) + nested_meta = MultiKVConnectorWorkerMetadata(metadata=(leaf_meta,)) + child_metadata: list[KVConnectorWorkerMetadata | None] = [nested_meta, nested_meta] + child_metadata[direct_index] = None + metadata = MultiKVConnectorWorkerMetadata(metadata=tuple(child_metadata)) + output = KVConnectorOutput( + finished_recving={"r"}, kv_connector_worker_meta=metadata + ) + nested_mc.update_connector_output(output) + + assert output.finished_recving == {"r"} + assert output.finished_sending is None + assert output.kv_connector_worker_meta is metadata + assert metadata.metadata[1 - direct_index] is nested_meta + leaf_output = nested.sub_connectors[0].update_connector_output.call_args.args[0] + assert leaf_output.kv_connector_worker_meta is leaf_meta + assert not nested_mc.request_finished(request, [1])[0] + + +def test_child_output_updates_survive_receive_isolation(nested_mc): + """Copying completions must not discard child error or auxiliary updates.""" + nested_mc, direct, nested, direct_index = nested_mc + direct.get_num_new_matched_tokens.return_value = 16, True + request = SimpleNamespace(request_id="r") + nested_mc.on_new_request(request) + assert nested_mc.get_num_new_matched_tokens(request, 0) == (16, True) + nested_mc.update_state_after_alloc(request, MagicMock(), 16) + stats, events = MagicMock(), MagicMock() + + def report_errors(output): + output.finished_recving.clear() + output.invalid_block_ids = output.invalid_block_ids | {2} + output.failed_recving = output.failed_recving | {"r"} + output.kv_connector_stats = stats + output.kv_cache_events = events + output.expected_finished_count = 3 + + def report_more_errors(output): + output.invalid_block_ids.add(3) + output.failed_recving.add("other") + + direct.update_connector_output.side_effect = report_errors + nested.sub_connectors[0].update_connector_output.side_effect = report_more_errors + output = KVConnectorOutput(finished_recving={"r"}, invalid_block_ids={1}) + nested_mc.update_connector_output(output) + + assert output.finished_recving == {"r"} + assert output.invalid_block_ids == {1, 2, 3} + assert output.failed_recving == {"r", "other"} + assert output.kv_connector_stats is stats + assert output.kv_cache_events is events + assert output.expected_finished_count == 3 + + +@pytest.mark.parametrize("receive_first", [True, False]) +def test_cancelled_nested_loader_waits_for_other_save_owner(nested_mc, receive_first): + """Cancellation releases once, after both the nested load and sibling save.""" + nested_mc, direct, nested, direct_index = nested_mc + direct.request_finished.return_value = True, None + nested.sub_connectors[0].get_num_new_matched_tokens.return_value = 16, True + scheduler, request = _scheduler_with_request( + nested_mc, RequestStatus.FINISHED_ABORTED + ) + assert nested_mc.get_num_new_matched_tokens(request, 0) == (16, True) + nested_mc.update_state_after_alloc(request, MagicMock(), 16) + assert nested_mc.request_finished(request, [1])[0] + + steps = [{"receiving": {"r"}}, {"notifications": [(direct_index, 0)]}] + if not receive_first: + steps.reverse() + first = _send_step(nested_mc, scheduler=scheduler, **steps[0]) + assert first.finished_recving is None + assert first.finished_sending is None + scheduler._free_request_blocks.assert_not_called() + + last = _send_step(nested_mc, scheduler=scheduler, **steps[1]) + assert last.finished_recving is None + assert last.finished_sending == {"r"} + scheduler._free_request_blocks.assert_called_once_with(request) + + duplicate = _send_step( + nested_mc, [(direct_index, 0)], receiving={"r"}, scheduler=scheduler + ) + assert duplicate.finished_recving is None + assert duplicate.finished_sending is None + scheduler._free_request_blocks.assert_called_once_with(request) + + def test_scheduler_child_completion_waits_for_other_save_owner(async_saves): """A child may finish in update_connector_output, without a worker ACK.""" _finish_with_async_saves(async_saves) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index 15bc88299a3f..4ebf49f6ad27 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -3,7 +3,7 @@ import copy from collections.abc import Callable, Iterable from contextlib import AbstractContextManager -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any, cast import torch @@ -517,54 +517,69 @@ def update_connector_output(self, connector_output: KVConnectorOutput): multi_connector_worker_meta = connector_output.kv_connector_worker_meta finished_sending: set[str] = set() - try: - for i, c in enumerate(self._connectors): - if multi_connector_worker_meta is not None: - # set the connector-specific worker metadata - connector_output.kv_connector_worker_meta = ( - multi_connector_worker_meta.metadata[i] - ) - child_sending: set[str] = set() - if ( - multi_connector_worker_meta is not None - and multi_connector_worker_meta.finished_sending - ): - expected = ( - c.get_finished_count() - or connector_output.expected_finished_count - or self._world_size - ) - for req_id, ranks in multi_connector_worker_meta.finished_sending[ - i - ].items(): - completed = self._completed_async_saves.get(req_id) - owners = self._async_save_owners.get(req_id) - if ( - completed is None - or i in completed - or (owners is not None and i not in owners) - ): - continue - workers = self._send_workers[i].setdefault(req_id, set()) - workers.update(ranks) - if len(workers) >= expected: - child_sending.add(req_id) - connector_output.finished_sending = child_sending or None - c.update_connector_output(connector_output) - for req_id in connector_output.finished_sending or (): + for i, c in enumerate(self._connectors): + child_sending: set[str] = set() + if ( + multi_connector_worker_meta is not None + and multi_connector_worker_meta.finished_sending + ): + expected = ( + c.get_finished_count() + or connector_output.expected_finished_count + or self._world_size + ) + for req_id, ranks in multi_connector_worker_meta.finished_sending[ + i + ].items(): completed = self._completed_async_saves.get(req_id) owners = self._async_save_owners.get(req_id) - if completed is None or (owners is not None and i not in owners): + if ( + completed is None + or i in completed + or (owners is not None and i not in owners) + ): continue - completed.add(i) - self._send_workers[i].pop(req_id, None) - if owners is not None: - owners.discard(i) - if not owners and req_id not in self._async_load_owners: - finished_sending.add(req_id) - finally: - # restore kv_connector_worker_meta - connector_output.kv_connector_worker_meta = multi_connector_worker_meta + workers = self._send_workers[i].setdefault(req_id, set()) + workers.update(ranks) + if len(workers) >= expected: + child_sending.add(req_id) + # Nested connectors reconcile their own completions. Neither their + # filtering nor in-place mutations may erase a sibling's receives. + child_output = replace( + connector_output, + kv_connector_worker_meta=( + multi_connector_worker_meta.metadata[i] + if multi_connector_worker_meta is not None + else None + ), + finished_sending=child_sending or None, + finished_recving=( + set(connector_output.finished_recving) + if connector_output.finished_recving is not None + else None + ), + ) + c.update_connector_output(child_output) + # Only completion and metadata reconciliation is wrapper-owned. + # Preserve all other child updates, including replacement payloads. + connector_output.invalid_block_ids = child_output.invalid_block_ids + connector_output.failed_recving = child_output.failed_recving + connector_output.kv_connector_stats = child_output.kv_connector_stats + connector_output.kv_cache_events = child_output.kv_cache_events + connector_output.expected_finished_count = ( + child_output.expected_finished_count + ) + for req_id in child_output.finished_sending or (): + completed = self._completed_async_saves.get(req_id) + owners = self._async_save_owners.get(req_id) + if completed is None or (owners is not None and i not in owners): + continue + completed.add(i) + self._send_workers[i].pop(req_id, None) + if owners is not None: + owners.discard(i) + if not owners and req_id not in self._async_load_owners: + finished_sending.add(req_id) finished_recving: set[str] = set() for req_id in connector_output.finished_recving or (): if self._async_load_owners.pop(req_id, None) is None: From fc9f0ca96d3990cc1c82600a68b8e92f74e5eecc Mon Sep 17 00:00:00 2001 From: whx-sjtu Date: Thu, 8 Oct 2026 13:25:41 +0000 Subject: [PATCH 7/7] [KVConnector] Extract completion ownership and MRV2 ordering fixes Separate the three independently reviewable lifecycle fixes from #58968. Co-authored-by: Codex Signed-off-by: whx-sjtu --- .../kv_connector/unit/test_moriio_tp_ack.py | 8 - .../kv_connector/unit/test_multi_connector.py | 534 +----------------- tests/v1/worker/test_gpu_model_runner_v2.py | 116 ---- .../kv_transfer/kv_connector/v1/base.py | 9 - .../v1/moriio/moriio_connector.py | 4 - .../kv_connector/v1/multi_connector.py | 236 ++------ vllm/v1/worker/gpu/kv_connector.py | 8 - vllm/v1/worker/gpu/model_runner.py | 18 +- 8 files changed, 68 insertions(+), 865 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py index bcf630b9e4a3..f8dbe8e067d4 100644 --- a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py +++ b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py @@ -48,14 +48,6 @@ def test_read_matches_only_requests_with_pending_remote_prefill(params, expected assert scheduler.get_num_new_matched_tokens(request, 0) == expected -@pytest.mark.parametrize("mode", [MoRIIOMode.READ, MoRIIOMode.WRITE]) -def test_read_requires_completion_of_draft_kv_writes(mode): - connector = MoRIIOConnector.__new__(MoRIIOConnector) - connector.mode = mode - - assert connector.requires_full_step_completion is (mode == MoRIIOMode.READ) - - def _unaligned_cpu_backing(): raw = bytearray(12288) original = torch.frombuffer(raw, dtype=torch.uint8) diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 281e8a5efb97..9d04a2ea537a 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -4,7 +4,6 @@ import shutil import tempfile from pathlib import Path -from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock @@ -15,11 +14,9 @@ from vllm import LLM, SamplingParams from vllm.config import KVTransferConfig from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory -from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, - KVConnectorTransferResults, SupportsHMA, supports_hma, ) @@ -36,14 +33,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( NixlKVConnectorStats, ) -from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.kv_cache_interface import KVCacheConfig -from vllm.v1.outputs import ( - KVConnectorOutput, - KVConnectorWorkerMetadata, - ModelRunnerOutput, -) -from vllm.v1.request import RequestStatus +from vllm.v1.outputs import KVConnectorOutput, KVConnectorWorkerMetadata MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct" @@ -162,11 +153,13 @@ class MockDivergentHMAConnector(MockHMAConnector): ) -def test_register_finished_partial_tail_notifies_every_connector(mc): - connector = mc - first, second = connector.sub_connectors +def test_register_finished_partial_tail_notifies_every_connector(): + connector = object.__new__(MultiConnector) + first = MagicMock(spec_set=KVConnectorBase_V1) + second = MagicMock(spec_set=KVConnectorBase_V1) first.register_finished_partial_tail.return_value = True second.register_finished_partial_tail.return_value = False + connector._connectors = [first, second] request = MagicMock() block_ids = ([1], [2]) offloads = [(1, 2, 12)] @@ -209,521 +202,6 @@ def mc() -> MultiConnector: return mc -@pytest.fixture -def async_saves(mc): - """Two save owners whose two worker ranks may finish in different steps.""" - for child in mc.sub_connectors: - child.get_finished_count.return_value = None - child.request_finished.return_value = True, None - child.build_connector_worker_meta.return_value = None - mc._world_size = 2 - return mc - - -def _send_step( - connector, notifications=(), *, receiving=None, failed_receiving=(), scheduler=None -): - outputs = [] - for rank in range(2): - metadata = MultiKVConnectorWorkerMetadata( - metadata=(None, None), - finished_sending=tuple( - {"r": {rank}} if (child, rank) in notifications else {} - for child in range(2) - ), - ) - outputs.append( - ModelRunnerOutput.with_kv_conn_output_only( - KVConnectorOutput(kv_connector_worker_meta=metadata) - ) - ) - output = KVOutputAggregator(2).aggregate(outputs).kv_connector_output - assert output is not None - output.finished_recving = receiving - output.failed_recving = set(failed_receiving) - if scheduler is None: - connector.update_connector_output(output) - else: - scheduler._update_from_kv_xfer_finished(output) - return output - - -def _scheduler_with_request(connector, status): - request = SimpleNamespace(request_id="r", status=status) - request.is_finished = lambda: RequestStatus.is_finished(request.status) - scheduler = object.__new__(Scheduler) - scheduler.connector = connector - scheduler.requests = {request.request_id: request} - scheduler.finished_recving_kv_req_ids = set() - scheduler._free_request_blocks = MagicMock() - connector.on_new_request(request) - return scheduler, request - - -def _start_async_receive(connector, request): - first, second = connector.sub_connectors - first.get_num_new_matched_tokens.return_value = 16, True - second.get_num_new_matched_tokens.return_value = 0, False - assert connector.get_num_new_matched_tokens(request, 0) == (16, True) - connector.update_state_after_alloc(request, MagicMock(), 16) - - -def _finish_with_async_saves(connector): - request = SimpleNamespace(request_id="r") - connector.on_new_request(request) - assert connector.request_finished(request, [1])[0] - - -def _assert_save_state_cleared(connector): - assert not connector._partial_tail_owners - assert not connector._async_load_candidates - assert not connector._async_load_owners - assert not connector._requests_to_connector - assert not connector._async_save_owners - assert not connector._completed_async_saves - assert not any(connector._send_workers) - - -def test_send_completion_requires_each_child_and_distinct_rank(async_saves): - """A different child or a duplicate rank cannot satisfy a missing ACK.""" - _finish_with_async_saves(async_saves) - assert _send_step(async_saves, [(0, 0), (1, 1)]).finished_sending is None - assert _send_step(async_saves, [(0, 0), (1, 0)]).finished_sending is None - assert _send_step(async_saves, [(0, 1)]).finished_sending == {"r"} - _assert_save_state_cleared(async_saves) - assert ( - _send_step(async_saves, [(0, 0), (0, 1), (1, 0), (1, 1)]).finished_sending - is None - ) - _assert_save_state_cleared(async_saves) - - -def test_non_owner_cannot_filter_other_connector_send_completion(async_saves): - first = async_saves.sub_connectors[0] - first.request_finished.return_value = False, None - seen = [] - - def filter_completion(output): - seen.append(output.finished_sending) - output.finished_sending = None - - first.update_connector_output.side_effect = filter_completion - _finish_with_async_saves(async_saves) - assert _send_step(async_saves, [(1, 0), (1, 1)]).finished_sending == {"r"} - assert seen == [None] - _assert_save_state_cleared(async_saves) - - -@pytest.fixture(params=[False, True], ids=["direct-first", "nested-first"]) -def nested_mc(request): - child = { - "kv_connector": "MockConnector", - "kv_role": "kv_both", - "kv_connector_module_path": __name__, - } - children = [ - child, - { - "kv_connector": "MultiConnector", - "kv_role": "kv_both", - "kv_connector_extra_config": {"connectors": [child]}, - }, - ] - if request.param: - children.reverse() - config = create_vllm_config( - kv_connector="MultiConnector", - kv_connector_extra_config={"connectors": children}, - ) - connector = MultiConnector( - config, - KVConnectorRole.SCHEDULER, - KVCacheConfig(num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]), - ) - nested_first = request.param - direct = connector.sub_connectors[int(nested_first)] - nested = connector.sub_connectors[int(not nested_first)] - for leaf in [direct, *nested.sub_connectors]: - leaf.get_num_new_matched_tokens.return_value = 0, False - leaf.get_finished_count.return_value = 1 - leaf.request_finished.return_value = False, None - return connector, direct, nested, int(nested_first) - - -def test_nested_nonowner_does_not_swallow_receive_completion(nested_mc): - """A nested non-loader must not strand the selected child's request.""" - nested_mc, direct, nested, direct_index = nested_mc - direct.get_num_new_matched_tokens.return_value = 16, True - request = SimpleNamespace(request_id="r") - nested_mc.on_new_request(request) - assert nested_mc.get_num_new_matched_tokens(request, 0) == (16, True) - nested_mc.update_state_after_alloc(request, MagicMock(), 16) - - leaf_meta = MagicMock(spec=KVConnectorWorkerMetadata) - nested_meta = MultiKVConnectorWorkerMetadata(metadata=(leaf_meta,)) - child_metadata: list[KVConnectorWorkerMetadata | None] = [nested_meta, nested_meta] - child_metadata[direct_index] = None - metadata = MultiKVConnectorWorkerMetadata(metadata=tuple(child_metadata)) - output = KVConnectorOutput( - finished_recving={"r"}, kv_connector_worker_meta=metadata - ) - nested_mc.update_connector_output(output) - - assert output.finished_recving == {"r"} - assert output.finished_sending is None - assert output.kv_connector_worker_meta is metadata - assert metadata.metadata[1 - direct_index] is nested_meta - leaf_output = nested.sub_connectors[0].update_connector_output.call_args.args[0] - assert leaf_output.kv_connector_worker_meta is leaf_meta - assert not nested_mc.request_finished(request, [1])[0] - - -def test_child_output_updates_survive_receive_isolation(nested_mc): - """Copying completions must not discard child error or auxiliary updates.""" - nested_mc, direct, nested, direct_index = nested_mc - direct.get_num_new_matched_tokens.return_value = 16, True - request = SimpleNamespace(request_id="r") - nested_mc.on_new_request(request) - assert nested_mc.get_num_new_matched_tokens(request, 0) == (16, True) - nested_mc.update_state_after_alloc(request, MagicMock(), 16) - stats, events = MagicMock(), MagicMock() - - def report_errors(output): - output.finished_recving.clear() - output.invalid_block_ids = output.invalid_block_ids | {2} - output.failed_recving = output.failed_recving | {"r"} - output.kv_connector_stats = stats - output.kv_cache_events = events - output.expected_finished_count = 3 - - def report_more_errors(output): - output.invalid_block_ids.add(3) - output.failed_recving.add("other") - - direct.update_connector_output.side_effect = report_errors - nested.sub_connectors[0].update_connector_output.side_effect = report_more_errors - output = KVConnectorOutput(finished_recving={"r"}, invalid_block_ids={1}) - nested_mc.update_connector_output(output) - - assert output.finished_recving == {"r"} - assert output.invalid_block_ids == {1, 2, 3} - assert output.failed_recving == {"r", "other"} - assert output.kv_connector_stats is stats - assert output.kv_cache_events is events - assert output.expected_finished_count == 3 - - -@pytest.mark.parametrize("receive_first", [True, False]) -def test_cancelled_nested_loader_waits_for_other_save_owner(nested_mc, receive_first): - """Cancellation releases once, after both the nested load and sibling save.""" - nested_mc, direct, nested, direct_index = nested_mc - direct.request_finished.return_value = True, None - nested.sub_connectors[0].get_num_new_matched_tokens.return_value = 16, True - scheduler, request = _scheduler_with_request( - nested_mc, RequestStatus.FINISHED_ABORTED - ) - assert nested_mc.get_num_new_matched_tokens(request, 0) == (16, True) - nested_mc.update_state_after_alloc(request, MagicMock(), 16) - assert nested_mc.request_finished(request, [1])[0] - - steps = [{"receiving": {"r"}}, {"notifications": [(direct_index, 0)]}] - if not receive_first: - steps.reverse() - first = _send_step(nested_mc, scheduler=scheduler, **steps[0]) - assert first.finished_recving is None - assert first.finished_sending is None - scheduler._free_request_blocks.assert_not_called() - - last = _send_step(nested_mc, scheduler=scheduler, **steps[1]) - assert last.finished_recving is None - assert last.finished_sending == {"r"} - scheduler._free_request_blocks.assert_called_once_with(request) - - duplicate = _send_step( - nested_mc, [(direct_index, 0)], receiving={"r"}, scheduler=scheduler - ) - assert duplicate.finished_recving is None - assert duplicate.finished_sending is None - scheduler._free_request_blocks.assert_called_once_with(request) - - -def test_scheduler_child_completion_waits_for_other_save_owner(async_saves): - """A child may finish in update_connector_output, without a worker ACK.""" - _finish_with_async_saves(async_saves) - - def finish_first(output): - output.finished_sending = {"r"} - - async_saves.sub_connectors[0].update_connector_output.side_effect = finish_first - assert _send_step(async_saves).finished_sending is None - assert _send_step(async_saves, [(1, 0), (1, 1)]).finished_sending == {"r"} - _assert_save_state_cleared(async_saves) - - -@pytest.mark.parametrize("early_children", [(0,), (0, 1)]) -def test_send_completion_before_request_finished_is_reconciled( - async_saves, early_children -): - request = SimpleNamespace(request_id="r") - async_saves.on_new_request(request) - early = [(child, rank) for child in early_children for rank in range(2)] - assert _send_step(async_saves, early).finished_sending is None - pending, _ = async_saves.request_finished(request, [1]) - assert pending == (early_children == (0,)) - if pending: - assert _send_step(async_saves, [(1, 0), (1, 1)]).finished_sending == {"r"} - _assert_save_state_cleared(async_saves) - - -def test_send_completion_uses_child_finished_count(async_saves): - async_saves.sub_connectors[0].get_finished_count.return_value = 1 - _finish_with_async_saves(async_saves) - assert _send_step(async_saves, [(0, 0), (1, 0)]).finished_sending is None - assert _send_step(async_saves, [(1, 1)]).finished_sending == {"r"} - - -def test_unexpected_send_does_not_create_tracking(async_saves): - # Unregistered ACKs must not create persistent request state. - assert _send_step(async_saves, [(0, 0), (1, 1)]).finished_sending is None - _assert_save_state_cleared(async_saves) - - -@pytest.mark.parametrize("receiver_saves", [False, True]) -@pytest.mark.parametrize("first_completion", ["receive", "send", "both"]) -def test_aborted_receive_waits_for_another_send_owner( - async_saves, receiver_saves, first_completion -): - """Finishing a cancelled load must not free another child's save source.""" - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS - ) - _start_async_receive(async_saves, request) - async_saves.sub_connectors[0].request_finished.return_value = ( - receiver_saves, - None, - ) - request.status = RequestStatus.FINISHED_ABORTED - assert async_saves.request_finished(request, [1])[0] - - send_acks = [(1, 0), (1, 1)] - if receiver_saves: - send_acks += [(0, 0), (0, 1)] - completions: dict[str, dict[str, set[str] | list[tuple[int, int]]]] = { - "receive": {"receiving": {"r"}}, - "send": {"notifications": send_acks}, - "both": {"receiving": {"r"}, "notifications": send_acks}, - } - output = _send_step( - async_saves, scheduler=scheduler, **completions[first_completion] - ) - if first_completion != "both": - scheduler._free_request_blocks.assert_not_called() - assert output.finished_recving is None - assert output.finished_sending is None - assert "r" in scheduler.requests - remaining = "send" if first_completion == "receive" else "receive" - _send_step(async_saves, scheduler=scheduler, **completions[remaining]) - scheduler._free_request_blocks.assert_called_once_with(request) - assert not scheduler.requests - _assert_save_state_cleared(async_saves) - # Duplicate terminal notifications cannot trigger a second block release. - _send_step(async_saves, scheduler=scheduler, **completions["both"]) - scheduler._free_request_blocks.assert_called_once_with(request) - - -def test_aborted_receive_only_releases_once_and_allows_request_id_reuse( - async_saves, -): - async_saves.sub_connectors[0].request_finished.return_value = False, None - async_saves.sub_connectors[1].request_finished.return_value = False, None - for _ in range(2): - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS - ) - _start_async_receive(async_saves, request) - request.status = RequestStatus.FINISHED_ABORTED - assert async_saves.request_finished(request, [1])[0] - _send_step(async_saves, receiving={"r"}, scheduler=scheduler) - scheduler._free_request_blocks.assert_called_once_with(request) - assert not scheduler.requests - _assert_save_state_cleared(async_saves) - - -def test_receiving_child_save_is_independent_of_its_receive(async_saves): - """A receiver may still be saving blocks from an earlier prefill.""" - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS - ) - _start_async_receive(async_saves, request) - request.status = RequestStatus.FINISHED_ABORTED - assert async_saves.request_finished(request, [1])[0] - _send_step(async_saves, [(1, 0), (1, 1)], receiving={"r"}, scheduler=scheduler) - scheduler._free_request_blocks.assert_not_called() - _send_step(async_saves, [(0, 0), (0, 1)], scheduler=scheduler) - scheduler._free_request_blocks.assert_called_once_with(request) - _assert_save_state_cleared(async_saves) - - -@pytest.mark.parametrize("failed", [False, True]) -def test_normal_receive_keeps_tracking_for_later_saves(async_saves, failed): - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS - ) - _start_async_receive(async_saves, request) - output = _send_step( - async_saves, - receiving={"r"}, - failed_receiving={"r"} if failed else (), - scheduler=scheduler, - ) - assert output.finished_recving == {"r"} - assert output.failed_recving == ({"r"} if failed else set()) - assert scheduler.finished_recving_kv_req_ids == {"r"} - scheduler._free_request_blocks.assert_not_called() - request.status = RequestStatus.FINISHED_STOPPED - assert async_saves.request_finished(request, [1])[0] - _send_step(async_saves, [(1, 0), (1, 1)], scheduler=scheduler) - scheduler._free_request_blocks.assert_not_called() - _send_step(async_saves, [(0, 0), (0, 1)], scheduler=scheduler) - scheduler._free_request_blocks.assert_called_once_with(request) - _assert_save_state_cleared(async_saves) - - -@pytest.mark.parametrize("allocate_zero_tokens", [False, True]) -def test_async_lookup_without_external_allocation_does_not_create_a_receive_wait( - async_saves, allocate_zero_tokens -): - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.FINISHED_ABORTED - ) - first, second = async_saves.sub_connectors - first.get_num_new_matched_tokens.return_value = 16, True - second.get_num_new_matched_tokens.return_value = 0, False - assert async_saves.get_num_new_matched_tokens(request, 0) == (16, True) - if allocate_zero_tokens: - async_saves.update_state_after_alloc(request, MagicMock(), 0) - assert async_saves.request_finished(request, [1])[0] - _send_step(async_saves, [(0, 0), (0, 1), (1, 0), (1, 1)], scheduler=scheduler) - scheduler._free_request_blocks.assert_called_once_with(request) - _assert_save_state_cleared(async_saves) - - -def test_partial_tail_owner_releases_after_its_send_ack(async_saves): - """Scheduler must hear completion even if only the tail hook delays free.""" - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.FINISHED_STOPPED - ) - first, second = async_saves.sub_connectors - first.register_finished_partial_tail.return_value = True - second.register_finished_partial_tail.return_value = False - for child in async_saves.sub_connectors: - child.request_finished.return_value = False, None - partial_tail_delay = async_saves.register_finished_partial_tail( - request, ([1],), [(0, 1, 12)] - ) - delay_free, _ = async_saves.request_finished(request, [1]) - assert partial_tail_delay or delay_free - _send_step(async_saves, [(0, 0)], scheduler=scheduler) - scheduler._free_request_blocks.assert_not_called() - _send_step(async_saves, [(0, 1)], scheduler=scheduler) - scheduler._free_request_blocks.assert_called_once_with(request) - assert not scheduler.requests - _assert_save_state_cleared(async_saves) - - -@pytest.mark.parametrize("early_ranks", [(0,), (0, 1)]) -def test_partial_tail_requires_new_acks_after_earlier_saves(async_saves, early_ranks): - """A newly accepted tail must not reuse earlier full-block send ACKs.""" - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.FINISHED_STOPPED - ) - first, second = async_saves.sub_connectors - first.register_finished_partial_tail.return_value = True - second.register_finished_partial_tail.return_value = False - first.request_finished.return_value = False, None - _send_step(async_saves, [(0, rank) for rank in early_ranks], scheduler=scheduler) - assert async_saves.register_finished_partial_tail(request, ([1],), [(0, 1, 12)]) - assert async_saves.request_finished(request, [1])[0] - _send_step(async_saves, [(0, 1), (1, 0), (1, 1)], scheduler=scheduler) - scheduler._free_request_blocks.assert_not_called() - _send_step(async_saves, [(0, 0)], scheduler=scheduler) - scheduler._free_request_blocks.assert_called_once_with(request) - _assert_save_state_cleared(async_saves) - - -@pytest.mark.parametrize("receive_first", [False, True]) -def test_partial_tail_and_receive_from_same_child_both_delay_free( - async_saves, receive_first -): - scheduler, request = _scheduler_with_request( - async_saves, RequestStatus.WAITING_FOR_REMOTE_KVS - ) - _start_async_receive(async_saves, request) - first, second = async_saves.sub_connectors - first.register_finished_partial_tail.return_value = True - second.register_finished_partial_tail.return_value = False - for child in async_saves.sub_connectors: - child.request_finished.return_value = False, None - request.status = RequestStatus.FINISHED_ABORTED - assert async_saves.register_finished_partial_tail(request, ([1],), [(0, 1, 12)]) - assert async_saves.request_finished(request, [1])[0] - receive = {"receiving": {"r"}} - send = {"notifications": [(0, 0), (0, 1)]} - _send_step(async_saves, scheduler=scheduler, **(receive if receive_first else send)) - scheduler._free_request_blocks.assert_not_called() - _send_step(async_saves, scheduler=scheduler, **(send if receive_first else receive)) - scheduler._free_request_blocks.assert_called_once_with(request) - assert not scheduler.requests - _assert_save_state_cleared(async_saves) - - -def test_no_async_saves_release_tracking(async_saves): - request = SimpleNamespace(request_id="r") - async_saves.on_new_request(request) - for child in async_saves.sub_connectors: - child.request_finished.return_value = False, None - assert not async_saves.request_finished(request, [1])[0] - _assert_save_state_cleared(async_saves) - - -def test_worker_preserves_send_identity_and_receive_failures(async_saves, monkeypatch): - from vllm.distributed.kv_transfer.kv_connector.v1 import multi_connector - - monkeypatch.setattr( - multi_connector, "get_world_group", lambda: SimpleNamespace(rank=1) - ) - async_saves.sub_connectors[ - 0 - ].get_transfer_results.return_value = KVConnectorTransferResults( - finished_sending={"r"} - ) - async_saves.sub_connectors[ - 1 - ].get_transfer_results.return_value = KVConnectorTransferResults( - finished_recving={"recv"}, failed_recving={"recv"} - ) - result = async_saves.get_transfer_results(set()) - assert not result.finished_sending - assert result.finished_recving == result.failed_recving == {"recv"} - metadata = async_saves.build_connector_worker_meta() - assert isinstance(metadata, MultiKVConnectorWorkerMetadata) - assert metadata.finished_sending == ({"r": {1}}, {}) - assert async_saves.build_connector_worker_meta() is None - - -def test_worker_send_metadata_merge_preserves_inputs_and_deduplicates_ranks(): - first = MultiKVConnectorWorkerMetadata( - metadata=(None, None), finished_sending=({"r": {0}}, {"r": {1}}) - ) - second = MultiKVConnectorWorkerMetadata( - metadata=(None, None), finished_sending=({"r": {0, 1}}, {}) - ) - result = first.aggregate(second) - assert result.finished_sending == ({"r": {0, 1}}, {"r": {1}}) - assert first.finished_sending == ({"r": {0}}, {"r": {1}}) - assert second.finished_sending == ({"r": {0, 1}}, {}) - - def test_multi_connector_mem_pool_context_none(mc: MultiConnector): assert mc.get_mem_pool_context() is None diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index d5612fbb3dfb..2e0c7271ce62 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -8,10 +8,7 @@ import pytest import torch -import vllm.forward_context as forward_context_module import vllm.v1.worker.gpu.model_runner as model_runner_module -from vllm.config import CUDAGraphMode, VllmConfig -from vllm.forward_context import get_forward_context from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry from vllm.v1.kv_cache_interface import ( CircularBufferSpec, @@ -61,119 +58,6 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback(): runner.pcp_manager.restore_for_sampling.assert_not_called() -@pytest.mark.parametrize("data_parallel_size", [1, 2]) -def test_full_graph_sync_load_observes_replay_mode(monkeypatch, data_parallel_size): - """Connectors must wait for READs before replay bypasses layer hooks.""" - runner = Mock() - runner.vllm_config = VllmConfig() - runner.vllm_config.parallel_config.data_parallel_size = data_parallel_size - runner.vllm_config.parallel_config.is_moe_model = True - monkeypatch.setattr( - forward_context_module, - "coordinate_batch_across_dp", - Mock(side_effect=AssertionError("connector context must not synchronize DP")), - ) - runner.lora_config = None - runner.is_encoder_decoder = False - runner.uses_inputs_embeds = False - runner.is_first_pp_rank = True - runner.dcp_size = 1 - runner.pcp_manager = None - runner.aux_output_connector = None - runner.observability_config.cudagraph_metrics = False - runner.gather_batch_req_state.return_value = (Mock(has_prefill=False), 1) - runner.prepare_attn.return_value = ({}, {}) - runner.model_state.prepare_inputs.return_value = {} - batch_desc = SimpleNamespace( - num_tokens=1, cg_mode=CUDAGraphMode.FULL, num_ubatches=1 - ) - monkeypatch.setattr( - model_runner_module, - "dispatch_cg_and_sync_dp", - lambda *a, **kw: (batch_desc, None), - ) - monkeypatch.setattr( - model_runner_module, "build_slot_mappings_by_layer", lambda *a: {} - ) - - class BeforeReplay(Exception): - pass - - def load(**kwargs): - assert get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL - - runner.kv_connector.pre_forward.side_effect = load - runner.cudagraph_manager.run_fullgraph.side_effect = BeforeReplay - scheduler_output = SimpleNamespace( - num_scheduled_tokens={"request": 1}, total_num_scheduled_tokens=1 - ) - with pytest.raises(BeforeReplay): - GPUModelRunner.execute_model(runner, scheduler_output) - runner.kv_connector.pre_forward.assert_called_once() - - -@pytest.mark.parametrize("has_speculator", [False, True]) -@pytest.mark.parametrize("requires_completion", [False, True]) -def test_output_readiness_covers_draft_kv_writes( - monkeypatch, has_speculator: bool, requires_completion: bool -): - """Publishing output must cover draft writes before RDMA can reuse KV.""" - writes: set[str] = set() - visible: set[str] = set() - ready: set[str] = set() - copy_stream = Mock() - copy_stream.wait_stream.side_effect = lambda _: visible.update(writes) - event = Mock() - event.record.side_effect = lambda _: ready.update(visible) - - def async_output(**kwargs): - copy_stream.wait_stream(kwargs["main_stream"]) - event.record(copy_stream) - return SimpleNamespace(copy_event=event) - - monkeypatch.setattr(model_runner_module, "AsyncOutput", async_output) - runner = Mock() - runner.output_copy_stream = copy_stream - runner.is_last_pp_rank = True - runner.pcp_manager = None - runner.pp_handler = None - runner.aux_output_connector = None - runner.adaptive_verification = None - runner.num_speculative_steps = 0 - runner._draft_workspace_lane = 0 - runner.model = Mock(spec=["compute_logits"]) - runner.kv_connector.requires_full_step_completion = requires_completion - runner.req_states.draft_tokens = torch.zeros((1, 1), dtype=torch.int64) - runner.prompt_logprobs_worker.compute_prompt_logprobs.return_value = {} - input_batch = SimpleNamespace( - req_ids=["request"], idx_mapping=torch.tensor([0]), query_start_loc=None - ) - runner.execute_model_state.input_batch = input_batch - runner.sample.return_value = (Mock(), torch.tensor([1]), torch.tensor([0])) - runner.postprocess_sampled.side_effect = lambda *a: writes.add("postprocess") - if has_speculator: - runner.speculator.supports_mm_inputs = False - - def propose(*args, **kwargs): - assert "draft" not in ready - writes.add("draft") - return torch.zeros((1, 1), dtype=torch.int64) - - runner.speculator.propose.side_effect = propose - else: - runner.speculator = None - monkeypatch.setattr( - model_runner_module, "use_workspace_lane", lambda _: contextlib.nullcontext() - ) - GPUModelRunner.sample_tokens(runner, None) - assert ("draft" in ready) == (has_speculator and requires_completion) - assert ("postprocess" in ready) == (has_speculator and requires_completion) - assert event.record.call_count == ( - 2 if has_speculator and requires_completion else 1 - ) - event.synchronize.assert_not_called() - - def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch): runner = GPUModelRunner.__new__(GPUModelRunner) runner.max_model_len = 262144 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index 397d1a3f7c6e..6df9d1cfbe8b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -188,15 +188,6 @@ def supports_divergent_local_hybrid_hits(self) -> bool: """ return False - @property - def requires_full_step_completion(self) -> bool: - """Whether output readiness must cover post-sampling KV writes. - - Out-of-band transfers cannot rely on the GPU stream to order loads - after earlier writes to a reused block, or before source publication. - """ - return False - @property def requires_kv_delivery(self) -> bool: """Whether this connector hands off KV that must be reliably delivered. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index a8c94e470f72..ee4643636fba 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -225,10 +225,6 @@ def supports_divergent_local_hybrid_hits(self) -> bool: # when the attention prefix is already present locally. return self.mode == MoRIIOMode.READ and self._transfers_mamba_state - @property - def requires_full_step_completion(self) -> bool: - return self.mode == MoRIIOMode.READ - def __init__( self, vllm_config: VllmConfig, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index 4ebf49f6ad27..73b6b0ccf643 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -3,14 +3,13 @@ import copy from collections.abc import Callable, Iterable from contextlib import AbstractContextManager -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast import torch from vllm.config import VllmConfig from vllm.config.kv_transfer import KVTransferConfig -from vllm.distributed import get_world_group from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.distributed.kv_transfer.kv_connector.v1.base import ( @@ -48,12 +47,12 @@ @dataclass class MultiKVConnectorMetadata(KVConnectorMetadata): metadata: tuple[KVConnectorMetadata, ...] + extra_async_saves: dict[str, int] | None = None @dataclass class MultiKVConnectorWorkerMetadata(KVConnectorWorkerMetadata): metadata: tuple[KVConnectorWorkerMetadata | None, ...] - finished_sending: tuple[dict[str, set[int]], ...] = () def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata: assert isinstance(other, MultiKVConnectorWorkerMetadata) @@ -68,17 +67,7 @@ def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetada else: metadata_list.append(metadata1.aggregate(metadata2)) - finished_sending = [] - for i in range(len(self.metadata)): - merged: dict[str, set[int]] = {} - for source in (self.finished_sending, other.finished_sending): - if source: - for req_id, ranks in source[i].items(): - merged.setdefault(req_id, set()).update(ranks) - finished_sending.append(merged) - return MultiKVConnectorWorkerMetadata( - metadata=tuple(metadata_list), finished_sending=tuple(finished_sending) - ) + return MultiKVConnectorWorkerMetadata(metadata=tuple(metadata_list)) @dataclass @@ -211,21 +200,12 @@ def __init__( # A mapping from request id to the index of the connector chosen to # load the request from (if any). self._requests_to_connector: dict[str, int] = {} - self._async_load_candidates: set[str] = set() - self._async_load_owners: dict[str, int] = {} - - # Scheduler-side owners and completions, including ACKs before finish. - self._partial_tail_owners: dict[str, set[int]] = {} - self._async_save_owners: dict[str, set[int]] = {} - self._completed_async_saves: dict[str, set[int]] = {} - self._send_workers: tuple[dict[str, set[int]], ...] = tuple( - {} for _ in self._connectors - ) - self._world_size = vllm_config.parallel_config.world_size - # Worker-side ACKs retain child and rank identity across scheduler steps. - self._worker_finished_sending: tuple[dict[str, set[int]], ...] = tuple( - {} for _ in self._connectors - ) + + # Keeps track of *additional* remaining async saves (beyond 1) to be + # finished per request. Not needed for async loads since we only allow + # a single connector to load. + # Propagated from scheduler to worker side via the connector metadata. + self._extra_async_saves: dict[str, int] = {} @property def sub_connectors(self) -> list[KVConnectorBase_V1]: @@ -241,10 +221,6 @@ def supports_divergent_local_hybrid_hits(self) -> bool: def requires_kv_delivery(self) -> bool: return any(c.requires_kv_delivery for c in self._connectors) - @property - def requires_full_step_completion(self) -> bool: - return any(c.requires_full_step_completion for c in self._connectors) - @classmethod def _get_connector_classes_and_configs( cls, vllm_config: "VllmConfig" @@ -292,6 +268,8 @@ def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None: # always return False. def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None: assert isinstance(connector_metadata, MultiKVConnectorMetadata) + if connector_metadata.extra_async_saves: + self._extra_async_saves.update(connector_metadata.extra_async_saves) for c, cm in zip(self._connectors, connector_metadata.metadata): c.bind_connector_metadata(cm) super().bind_connector_metadata(connector_metadata) @@ -380,14 +358,20 @@ def get_transfer_results( self, finished_req_ids: set[str] ) -> KVConnectorTransferResults: results = KVConnectorTransferResults() - for i, connector in enumerate(self._connectors): + for connector in self._connectors: child_results = connector.get_transfer_results(finished_req_ids) results.finished_recving.update(child_results.finished_recving) results.failed_recving.update(child_results.failed_recving) for req_id in child_results.finished_sending: - self._worker_finished_sending[i].setdefault(req_id, set()).add( - get_world_group().rank - ) + extra_pending = self._extra_async_saves.get(req_id) + if extra_pending is None: + results.finished_sending.add(req_id) + else: + assert extra_pending > 0 + if extra_pending == 1: + del self._extra_async_saves[req_id] + else: + self._extra_async_saves[req_id] = extra_pending - 1 return results def get_block_ids_with_load_errors(self) -> set[int]: @@ -422,14 +406,16 @@ def get_finished_count(self) -> int | None: return next(iter(counts), None) def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None: - metadata = tuple(c.build_connector_worker_meta() for c in self._connectors) - finished_sending = self._worker_finished_sending - self._worker_finished_sending = tuple({} for _ in self._connectors) - if all(item is None for item in metadata) and not any(finished_sending): + metadata_list: list[KVConnectorWorkerMetadata | None] | None = None + for i, c in enumerate(self._connectors): + kv_connector_worker_meta = c.build_connector_worker_meta() + if metadata_list is None and kv_connector_worker_meta is not None: + metadata_list = [None] * i + if metadata_list is not None: + metadata_list.append(kv_connector_worker_meta) + if metadata_list is None: return None - return MultiKVConnectorWorkerMetadata( - metadata=metadata, finished_sending=finished_sending - ) + return MultiKVConnectorWorkerMetadata(metadata=tuple(metadata_list)) # TODO: Add a generic implementation of 'get_kv_connector_kv_cache_events' # method for the MultiConnector. It should be able to get events from @@ -445,8 +431,6 @@ def get_num_new_matched_tokens( request: "Request", num_computed_tokens: int, ) -> tuple[int | None, bool]: - self._requests_to_connector.pop(request.request_id, None) - self._async_load_candidates.discard(request.request_id) to_return = (0, False) for i, c in enumerate(self._connectors): toks, load_async = c.get_num_new_matched_tokens( @@ -460,8 +444,6 @@ def get_num_new_matched_tokens( # to this request. if to_return[0] == 0 and toks > 0: self._requests_to_connector[request.request_id] = i - if load_async: - self._async_load_candidates.add(request.request_id) to_return = (toks, load_async) return to_return @@ -476,36 +458,23 @@ def update_state_after_alloc( else: # Other connectors still receive the request's real blocks c.update_state_after_alloc(request, blocks, 0) - if ( - num_external_tokens > 0 - and request.request_id in self._async_load_candidates - ): - self._async_load_owners[request.request_id] = chosen_connector - self._async_load_candidates.discard(request.request_id) def on_new_request(self, request: "Request") -> None: - self._completed_async_saves[request.request_id] = set() for c in self._connectors: c.on_new_request(request) def build_connector_meta( self, scheduler_output: SchedulerOutput ) -> MultiKVConnectorMetadata: - return MultiKVConnectorMetadata( + metadata = MultiKVConnectorMetadata( metadata=tuple( c.build_connector_meta(scheduler_output) for c in self._connectors ) ) - - def _forget_async_saves(self, req_id: str) -> None: - self._partial_tail_owners.pop(req_id, None) - self._async_save_owners.pop(req_id, None) - self._completed_async_saves.pop(req_id, None) - self._async_load_owners.pop(req_id, None) - self._async_load_candidates.discard(req_id) - self._requests_to_connector.pop(req_id, None) - for workers in self._send_workers: - workers.pop(req_id, None) + if self._extra_async_saves: + metadata.extra_async_saves = self._extra_async_saves + self._extra_async_saves = {} + return metadata def update_connector_output(self, connector_output: KVConnectorOutput): multi_connector_worker_meta: MultiKVConnectorWorkerMetadata | None = None @@ -516,85 +485,17 @@ def update_connector_output(self, connector_output: KVConnectorOutput): ) multi_connector_worker_meta = connector_output.kv_connector_worker_meta - finished_sending: set[str] = set() - for i, c in enumerate(self._connectors): - child_sending: set[str] = set() - if ( - multi_connector_worker_meta is not None - and multi_connector_worker_meta.finished_sending - ): - expected = ( - c.get_finished_count() - or connector_output.expected_finished_count - or self._world_size - ) - for req_id, ranks in multi_connector_worker_meta.finished_sending[ - i - ].items(): - completed = self._completed_async_saves.get(req_id) - owners = self._async_save_owners.get(req_id) - if ( - completed is None - or i in completed - or (owners is not None and i not in owners) - ): - continue - workers = self._send_workers[i].setdefault(req_id, set()) - workers.update(ranks) - if len(workers) >= expected: - child_sending.add(req_id) - # Nested connectors reconcile their own completions. Neither their - # filtering nor in-place mutations may erase a sibling's receives. - child_output = replace( - connector_output, - kv_connector_worker_meta=( - multi_connector_worker_meta.metadata[i] - if multi_connector_worker_meta is not None - else None - ), - finished_sending=child_sending or None, - finished_recving=( - set(connector_output.finished_recving) - if connector_output.finished_recving is not None - else None - ), - ) - c.update_connector_output(child_output) - # Only completion and metadata reconciliation is wrapper-owned. - # Preserve all other child updates, including replacement payloads. - connector_output.invalid_block_ids = child_output.invalid_block_ids - connector_output.failed_recving = child_output.failed_recving - connector_output.kv_connector_stats = child_output.kv_connector_stats - connector_output.kv_cache_events = child_output.kv_cache_events - connector_output.expected_finished_count = ( - child_output.expected_finished_count - ) - for req_id in child_output.finished_sending or (): - completed = self._completed_async_saves.get(req_id) - owners = self._async_save_owners.get(req_id) - if completed is None or (owners is not None and i not in owners): - continue - completed.add(i) - self._send_workers[i].pop(req_id, None) - if owners is not None: - owners.discard(i) - if not owners and req_id not in self._async_load_owners: - finished_sending.add(req_id) - finished_recving: set[str] = set() - for req_id in connector_output.finished_recving or (): - if self._async_load_owners.pop(req_id, None) is None: - continue - owners = self._async_save_owners.get(req_id) - if owners is None: - # A live request can resume as soon as its selected load ends. - finished_recving.add(req_id) - elif not owners: - # A finished request releases once both load and save are done. - finished_sending.add(req_id) - connector_output.finished_recving = finished_recving or None - connector_output.finished_sending = finished_sending or None - for req_id in finished_sending: - self._forget_async_saves(req_id) + try: + for i, c in enumerate(self._connectors): + if multi_connector_worker_meta is not None: + # set the connector-specific worker metadata + connector_output.kv_connector_worker_meta = ( + multi_connector_worker_meta.metadata[i] + ) + c.update_connector_output(connector_output) + finally: + # restore kv_connector_worker_meta + connector_output.kv_connector_worker_meta = multi_connector_worker_meta def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None: """Get the KVConnector handshake metadata from sub-connectors. @@ -628,13 +529,12 @@ def _aggregate_request_finished( [KVConnectorBase_V1], tuple[bool, dict[str, Any] | None] ], ) -> tuple[bool, dict[str, Any] | None]: - async_save_owners = self._partial_tail_owners.pop(request.request_id, set()) - load_owner = self._async_load_owners.get(request.request_id) + async_saves = 0 kv_txfer_params = None - for i, c in enumerate(self._connectors): - delay_free, txfer_params = per_connector_fn(c) - if delay_free: - async_save_owners.add(i) + for c in self._connectors: + async_save, txfer_params = per_connector_fn(c) + if async_save: + async_saves += 1 if txfer_params is not None: if kv_txfer_params is not None: clashes = set(kv_txfer_params) & set(txfer_params) @@ -646,17 +546,12 @@ def _aggregate_request_finished( kv_txfer_params.update(txfer_params) else: kv_txfer_params = txfer_params - completed = self._completed_async_saves.setdefault(request.request_id, set()) - pending = async_save_owners - completed - delay_free = bool(pending) or load_owner is not None - if delay_free: - self._async_save_owners[request.request_id] = pending - else: - self._forget_async_saves(request.request_id) + if async_saves > 1: + self._extra_async_saves[request.request_id] = async_saves - 1 self._requests_to_connector.pop(request.request_id, None) - return delay_free, kv_txfer_params + return async_saves > 0, kv_txfer_params def request_finished( self, @@ -674,24 +569,11 @@ def register_finished_partial_tail( block_ids: tuple[list[int], ...], partial_tail_offloads: list[tuple[int, int, int]], ) -> bool: - owners = { - i - for i, c in enumerate(self._connectors) - if c.register_finished_partial_tail( - request, block_ids, partial_tail_offloads - ) - } - if owners: - self._partial_tail_owners.setdefault(request.request_id, set()).update( - owners - ) - # A finish-time tail is new work, independent of earlier save ACKs. - self._completed_async_saves.setdefault( - request.request_id, set() - ).difference_update(owners) - for i in owners: - self._send_workers[i].pop(request.request_id, None) - return bool(owners) + accepted = [ + c.register_finished_partial_tail(request, block_ids, partial_tail_offloads) + for c in self._connectors + ] + return any(accepted) def request_finished_all_groups( self, diff --git a/vllm/v1/worker/gpu/kv_connector.py b/vllm/v1/worker/gpu/kv_connector.py index 5a1a1ce0db04..1ba75beae271 100644 --- a/vllm/v1/worker/gpu/kv_connector.py +++ b/vllm/v1/worker/gpu/kv_connector.py @@ -29,10 +29,6 @@ class KVConnector: """KVConnector interface used by GPUModelRunner.""" - @property - def requires_full_step_completion(self) -> bool: - return False - def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None: pass @@ -65,10 +61,6 @@ def __init__( self._pending_load_kwargs: dict[str, Any] | None = None self._disabled = False - @property - def requires_full_step_completion(self) -> bool: - return not self._disabled and self.kv_connector.requires_full_step_completion - def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None: if self._disabled: return diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index e4bf91bf3f8e..b40fb4eac138 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1981,12 +1981,9 @@ def execute_model( # NOTE(woosuk): Here, we don't need to pass the input tensors, # because they are already copied to the CUDA graph input buffers. assert self.cudagraph_manager is not None - with set_forward_context( - None, self.vllm_config, cudagraph_runtime_mode=CUDAGraphMode.FULL - ): - self.kv_connector.pre_forward( - **connector_kwargs, attn_metadata=attn_metadata - ) + self.kv_connector.pre_forward( + **connector_kwargs, attn_metadata=attn_metadata + ) model_output = self.cudagraph_manager.run_fullgraph(batch_desc) else: # For piecewise and eager mode, just call model(). @@ -2262,15 +2259,6 @@ def sample_tokens( model_runner_output.kv_connector_output = kv_connector_output model_runner_output.ec_connector_output = ec_connector_output - if ( - self.speculator is not None - and self.kv_connector.requires_full_step_completion - ): - # Keep early D2H copies, but publish output only after draft KV - # writes finish, before a READ can export or reuse these blocks. - self.output_copy_stream.wait_stream(self.main_stream) - async_output.copy_event.record(self.output_copy_stream) - return async_output def take_draft_token_ids(self) -> DraftTokenIds | None: