diff --git a/tests/reasoning/test_kimi_k3_reasoning_parser.py b/tests/reasoning/test_kimi_k3_reasoning_parser.py index 8dcac4a83aa9..67329988ab2d 100644 --- a/tests/reasoning/test_kimi_k3_reasoning_parser.py +++ b/tests/reasoning/test_kimi_k3_reasoning_parser.py @@ -13,6 +13,7 @@ from vllm.parser.parser_manager import ParserManager from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser pytestmark = pytest.mark.skip_global_cleanup @@ -50,6 +51,10 @@ class ReasoningOnlyParser(KimiK3Parser): reasoning_parser_cls = KimiK3ReasoningParser +class ReasoningAndToolParser(ReasoningOnlyParser): + tool_parser_cls = KimiK3ToolParser + + def test_parser_manager_selects_kimi_k3_parser_for_reasoning_only(): parser_cls = ParserManager.get_parser(reasoning_parser_name="kimi_k3") @@ -301,6 +306,142 @@ def test_delegating_parser_thinking_false_streams_response_content(): assert closed is None +@pytest.mark.parametrize("suffix", ["<", OPEN, f"{CLOSE}think"]) +@pytest.mark.parametrize("prefix", ["", "value "]) +@pytest.mark.parametrize("empty_final", [False, True]) +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +def test_streaming_eof_preserves_unfinished_reasoning_marker( + suffix, prefix, empty_final, parser_cls +): + parser = parser_cls(_dummy_tokenizer()) + request = ChatCompletionRequest(model="test-model", messages=[]) + text = prefix + suffix + chunks = [text, ""] if empty_final else [text] + deltas = [ + parser.parse_delta( + chunk, + parser.model_tokenizer.encode(chunk), + request, + prompt_token_ids=OPEN_IDS, + finished=index == len(chunks) - 1, + ) + for index, chunk in enumerate(chunks) + ] + + assert "".join(delta.reasoning or "" for delta in deltas if delta) == text + assert not any(delta.content or delta.tool_calls for delta in deltas if delta) + + +@pytest.mark.parametrize("thinking", [False, True]) +@pytest.mark.parametrize("suffix", ["<", f"{CLOSE}response"]) +@pytest.mark.parametrize("prefix", ["", "value "]) +@pytest.mark.parametrize("empty_final", [False, True]) +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +def test_streaming_eof_preserves_unfinished_content_marker( + thinking, suffix, prefix, empty_final, parser_cls +): + parser = parser_cls(_dummy_tokenizer(), chat_template_kwargs={"thinking": thinking}) + request = ChatCompletionRequest(model="test-model", messages=[]) + chunks = ["step", THINK_CLOSE] if thinking else [] + chunks += [RESPONSE_OPEN, prefix + suffix] + if empty_final: + chunks.append("") + deltas = [ + parser.parse_delta( + chunk, + parser.model_tokenizer.encode(chunk), + request, + prompt_token_ids=OPEN_IDS, + finished=index == len(chunks) - 1, + ) + for index, chunk in enumerate(chunks) + ] + + assert "".join(delta.content or "" for delta in deltas if delta) == prefix + suffix + assert "".join(delta.reasoning or "" for delta in deltas if delta) == ( + "step" if thinking else "" + ) + assert not any(delta.tool_calls for delta in deltas if delta) + + +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +def test_streaming_eof_does_not_expose_suppressed_reasoning(parser_cls): + parser = parser_cls(_dummy_tokenizer()) + request = ChatCompletionRequest( + model="test-model", messages=[], include_reasoning=False + ) + + assert ( + parser.parse_delta( + "value <", [9], request, prompt_token_ids=OPEN_IDS, finished=True + ) + is None + ) + + +def test_streaming_eof_does_not_expose_other_parser_reasoning(): + class Tokenizer(DummyTokenizer): + def get_vocab(self): + return {"": 1000, "": 1001} + + def encode(self, text, add_special_tokens=False): + if text in self.get_vocab(): + return [self.get_vocab()[text]] + return super().encode(text, add_special_tokens=add_special_tokens) + + parser_cls = ParserManager.get_parser( + reasoning_parser_name="deepseek_r1", + tool_parser_name="kimi_k3", + enable_auto_tools=True, + ) + assert parser_cls is not None + parser = parser_cls(cast(TokenizerLike, Tokenizer())) + request = ChatCompletionRequest( + model="test-model", messages=[], include_reasoning=False + ) + + assert ( + parser.parse_delta( + "internal thought <", [9], request, prompt_token_ids=[1000], finished=True + ) + is None + ) + + +@pytest.mark.parametrize("parser_cls", [ReasoningOnlyParser, ReasoningAndToolParser]) +@pytest.mark.parametrize("empty_final", [False, True]) +def test_streaming_eof_strips_complete_markers_after_reasoning(parser_cls, empty_final): + parser = parser_cls(_dummy_tokenizer()) + request = ChatCompletionRequest(model="test-model", messages=[]) + # A speculative step can close reasoning and complete the response together. + chunks = [ + (THINK_OPEN + "step", [*OPEN_IDS, 9]), + ( + THINK_CLOSE + + RESPONSE_OPEN + + "answer" + + f"{CLOSE}response{SEP}{CLOSE}message{SEP}", + [*CLOSE_IDS, 10], + ), + ] + if empty_final: + chunks.append(("", [])) + deltas = [ + parser.parse_delta( + chunk, + token_ids, + request, + prompt_token_ids=OPEN_IDS, + finished=index == len(chunks) - 1, + ) + for index, (chunk, token_ids) in enumerate(chunks) + ] + + assert "".join(delta.reasoning or "" for delta in deltas if delta) == "step" + assert "".join(delta.content or "" for delta in deltas if delta) == "answer" + assert not any(delta.tool_calls for delta in deltas if delta) + + def test_adjust_request_keeps_xtml_markers_contiguous(): parser = KimiK3ReasoningParser(DummyTokenizer()) request = ChatCompletionRequest(model="test-model", messages=[]) diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py index 87ea53225a5f..7523724de015 100644 --- a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py +++ b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py @@ -828,7 +828,7 @@ def test_get_num_new_matched_tokens_read_recomputes_last_token(): req = SimpleNamespace( num_prompt_tokens=10, prompt_token_ids=list(range(10)), - kv_transfer_params=None, + kv_transfer_params={"do_remote_prefill": True}, ) n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0) # READ always recomputes the final token locally: N-1 - computed. @@ -865,7 +865,9 @@ def test_get_num_new_matched_tokens_supports_embeds_only_prompts( num_prompt_tokens=10, prompt_token_ids=None, prompt_embeds=object(), - kv_transfer_params=None, + kv_transfer_params={"do_remote_prefill": True} + if mode == MoRIIOMode.READ + else None, ) assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == ( diff --git a/tests/v1/kv_connector/unit/test_moriio_routing.py b/tests/v1/kv_connector/unit/test_moriio_routing.py index b0ac6df3cba3..eb9d70d48989 100644 --- a/tests/v1/kv_connector/unit/test_moriio_routing.py +++ b/tests/v1/kv_connector/unit/test_moriio_routing.py @@ -14,6 +14,10 @@ from types import SimpleNamespace +import pytest + +from vllm.config import KVTransferConfig +from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_common from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( MoRIIOConnectorScheduler, ) @@ -23,6 +27,35 @@ _clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks +@pytest.mark.parametrize("backend, expected_port", [("rdma", 0), ("xgmi", 12345)]) +def test_rdma_listener_reserves_port_during_bind(monkeypatch, backend, expected_port): + monkeypatch.setattr(moriio_common, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + moriio_common, "get_tensor_model_parallel_world_size", lambda: 8 + ) + monkeypatch.setattr(moriio_common, "get_open_port", lambda: 12345) + monkeypatch.setattr(moriio_common, "resolve_host_ip", lambda _: "127.0.0.1") + config = SimpleNamespace( + kv_transfer_config=KVTransferConfig( + kv_connector="MoRIIOConnector", + kv_role="kv_producer", + kv_connector_extra_config={ + "backend": backend, + "http_port": 8000, + "handshake_port": 6300, + "notify_port": 7300, + }, + ), + parallel_config=SimpleNamespace( + data_parallel_rank=0, data_parallel_size=1, data_parallel_size_local=1 + ), + ) + assert ( + moriio_common.MoRIIOConfig.from_vllm_config(config).local_kv_port + == expected_port + ) + + def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int): """Minimal prefill-side (producer) scheduler stand-in for request_finished.""" s = SimpleNamespace( diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py index a35f0a1169b6..f8dbe8e067d4 100644 --- a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py +++ b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import pytest +import torch from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOMode, @@ -13,6 +14,7 @@ ) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( MoRIIOConnector, + MoRIIOConnectorScheduler, MoRIIOConnectorWorker, get_moriio_expected_ack_count, get_moriio_remote_tp_rank, @@ -22,6 +24,133 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( MoRIIOWrapper, ) +from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( + MambaConvSplitInfo, +) +from vllm.v1.kv_cache_interface import MambaSpec + + +@pytest.mark.parametrize( + "params,expected", + [ + (None, (0, False)), + ({}, (0, False)), + ({"do_remote_prefill": False}, (0, False)), + ({"do_remote_prefill": True}, (31, False)), + ], +) +def test_read_matches_only_requests_with_pending_remote_prefill(params, expected): + scheduler = MoRIIOConnectorScheduler.__new__(MoRIIOConnectorScheduler) + scheduler.is_producer = False + scheduler.mode = MoRIIOMode.READ + request = SimpleNamespace(num_prompt_tokens=32, kv_transfer_params=params) + + assert scheduler.get_num_new_matched_tokens(request, 0) == expected + + +def _unaligned_cpu_backing(): + raw = bytearray(12288) + original = torch.frombuffer(raw, dtype=torch.uint8) + offset = (128 - original.data_ptr()) % 4096 + return torch.frombuffer(raw, dtype=torch.uint8, offset=offset, count=8192) + + +def _mamba_registration_worker(caches): + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.kv_caches = caches + spec = MambaSpec( + block_size=16, + shapes=((4, 1), (2, 2)), + dtypes=(torch.float32, torch.float32), + ) + worker.layer_to_spec = dict.fromkeys(caches, spec) + return worker + + +def test_shared_registration_covers_final_ssm_with_unaligned_first_enabled_layer(): + backing = _unaligned_cpu_backing() + # Earlier layers can be transfer-disabled. The first enabled layer need + # not begin at the aligned start of the common allocation. + cache = backing[-64:].view(2, 1, 1, 32) + worker = _mamba_registration_worker({"kda": cache}) + + registration, offsets = worker._build_shared_kv_mr(worker.kv_caches) + + expected_base = cache.data_ptr() // 4096 * 4096 + assert registration.data_ptr() == expected_base + assert registration.data_ptr() >= backing.data_ptr() + assert registration.data_ptr() + registration.numel() == ( + backing.data_ptr() + backing.numel() + ) + assert registration.untyped_storage().data_ptr() == backing.data_ptr() + assert offsets == {"kda": cache.data_ptr() - expected_base} + + +def test_shared_registration_refuses_alignment_before_storage(): + cache = _unaligned_cpu_backing()[:64].view(2, 1, 1, 32) + worker = _mamba_registration_worker({"kda": cache}) + + with pytest.raises(ValueError, match="outside.*storage"): + worker._build_shared_kv_mr(worker.kv_caches) + + +def test_shared_registration_refuses_regions_from_other_storage(): + backing = _unaligned_cpu_backing() + other = _unaligned_cpu_backing() + worker = _mamba_registration_worker( + { + "first": backing[-64:].view(2, 1, 1, 32), + "other": other[-64:].view(2, 1, 1, 32), + } + ) + + with pytest.raises(ValueError, match="same storage"): + worker._build_shared_kv_mr(worker.kv_caches) + + +def test_mamba_reads_address_each_region_relative_to_shared_registration(monkeypatch): + cache = torch.zeros((2, 1, 1, 32), dtype=torch.uint8) + worker = _mamba_registration_worker({"kda": cache}) + worker.world_size = 1 + worker._conv_decomp = MambaConvSplitInfo(1, (1, 1, 2), 4, (16, 16)) + worker._mamba_offset_templates = {} + worker.kv_region_mr_offsets = {"kda": [128, 144]} + worker.layer_base_addr_index = {"kda": 0} + worker.layer_name_to_remote_kv_cache_metadata = {"peer": {"kda": [4096, 4096]}} + worker.moriio_wrapper = MoRIIOWrapper( + moriio_engine=SimpleNamespace(allocate_transfer_uid=lambda: 1) + ) + worker.moriio_wrapper.local_memory_registered = True + monkeypatch.setattr( + worker.moriio_wrapper, + "get_unpack_memory_metadata", + lambda address: SimpleNamespace(data=address), + ) + posted = [] + + class Session: + def batch_read(self, local, remote, sizes, uid): + posted.append((local, remote, sizes)) + return SimpleNamespace(Failed=lambda: False) + + statuses = worker._post_mamba_reads( + "kda", + [Session(), Session()], + [0, 1], + [1], + [0], + 1, + SimpleNamespace(kv_caches_base_addr=[4352, 4368]), + "peer", + "req", + float("inf"), + ) + + assert len(statuses) == 2 + assert posted == [ + ([160, 164, 168], [256, 260, 264], [4, 4, 8]), + ([176], [272], [16]), + ] def test_remote_tp_rank_same_tp_maps_to_self(): @@ -432,6 +561,80 @@ def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid assert worker.get_block_ids_with_load_errors() == expected_invalid +@pytest.mark.parametrize( + "states,elapsed,has_mamba,error", + [ + (["failed", "pending"], 1, True, "failed"), + (["pending", "failed"], 1, True, "failed"), + (["pending"], 121, True, "timed out"), + (["done", "done"], 121, True, None), + (["pending"], 120, True, None), + (["failed", "pending"], 1, False, None), + ], +) +def test_read_completion_preserves_hybrid_blocks_on_error( + monkeypatch, states, elapsed, has_mamba, error +): + class Status: + def __init__(self, state): + self.state = state + + def Succeeded(self): + return self.state == "done" + + def Failed(self): + return self.state == "failed" + + def Message(self): + return self.state + + def Code(self): + return self.state + + monkeypatch.setattr( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.time.monotonic", + lambda: 1000.0, + ) + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.is_producer = False + worker.mode = MoRIIOMode.READ + worker.world_size = 8 + worker._has_mamba = has_mamba + worker.moriio_config = SimpleNamespace(recv_abort_timeout=120.0) + worker.moriio_wrapper = MoRIIOWrapper() + notifications = [] + monkeypatch.setattr( + worker.moriio_wrapper, + "send_notify", + lambda *args, **kwargs: notifications.append((args, kwargs)), + ) + worker._recving_transfers = {"req": {"layer": [Status(state) for state in states]}} + worker._recving_transfers_callback_addr = {"req": ("host", "7000", "tx")} + worker._recving_transfers_start = {"req": 1000.0 - elapsed} + worker._recving_local_blocks = {"req": [7, 8]} + worker._invalid_block_ids = set() + + if error is not None: + with pytest.raises(TransferError, match=error): + worker.get_finished() + assert notifications == [] + assert "req" in worker._recving_transfers + assert worker._recving_transfers_callback_addr == { + "req": ("host", "7000", "tx") + } + assert worker._recving_transfers_start == {"req": 1000.0 - elapsed} + assert worker._recving_local_blocks == {"req": [7, 8]} + assert worker.get_block_ids_with_load_errors() == set() + else: + assert worker.get_finished() == (set(), set()) + completed = states == ["done", "done"] or not has_mamba + assert len(notifications) == int(completed) + assert bool(worker._recving_transfers) is not completed + assert worker.get_block_ids_with_load_errors() == ( + set() if has_mamba else {7, 8} + ) + + def test_hybrid_step_barrier_fails_closed(monkeypatch): class FailingWrapper: def waiting_for_transfer_complete(self, _statuses): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index ae1eecf6e55d..37e8a05769b0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -369,7 +369,9 @@ def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig": return cls( local_ip=resolve_host_ip(extra_config), - local_kv_port=get_open_port(), + # MoRI publishes the bound port after RDMA backend initialization. + # Let bind(0) reserve it atomically across concurrent TP workers. + local_kv_port=0 if backend == "rdma" else get_open_port(), proxy_ip=extra_config.get("proxy_ip", ""), local_ping_port=get_open_port(), proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)), diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index b482e6a8c4a3..ee4643636fba 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -761,6 +761,10 @@ def get_num_new_matched_tokens( # no recurrent-state accounting to do. return num_external_tokens, True + params = request.kv_transfer_params + if not params or not params.get("do_remote_prefill"): + return 0, False + # READ mode always recomputes the last token locally on the decoder. # # The second element declares who waits for the KV, not whether it has @@ -1708,6 +1712,7 @@ def __init__( # KV Caches and moriio tracking data. self.kv_caches: dict[str, torch.Tensor] = {} self.kv_layer_mr_offset: dict[str, int] = {} + self.kv_region_mr_offsets: dict[str, list[int]] = {} self.layer_base_addr_index: dict[str, int] = {} # Map of engine_id -> kv_caches_base_addr. For TP case, each local @@ -2269,24 +2274,28 @@ def _build_shared_kv_mr( ) -> tuple[torch.Tensor, dict[str, int]]: page_size = 4096 backing = next(iter(kv_caches.values())).view(torch.uint8) - nbytes = backing.untyped_storage().nbytes() - base = torch.as_strided(backing, (nbytes,), (1,), 0) - ptr = base.data_ptr() - slack = ptr % page_size - end = 0 + storage = backing.untyped_storage() + storage_ptr = storage.data_ptr() + regions = [] for layer_name in kv_caches: - _, region_len = next( - iter(self._iter_layer_registration_regions(layer_name)) - ) - end = max(end, kv_caches[layer_name].data_ptr() - ptr + region_len) - reg_nbytes = ((slack + end + page_size - 1) // page_size) * page_size - if reg_nbytes > nbytes: + for region, region_len in self._iter_layer_registration_regions(layer_name): + if ( + region.device != backing.device + or region.untyped_storage().data_ptr() != storage_ptr + ): + raise ValueError("Shared KV regions must use the same storage") + regions.append((region.data_ptr(), region_len)) + # The first transferable layer need not be the allocation's first view. + # Align its registration within storage, never before the allocation. + mr_ptr = min(ptr for ptr, _ in regions) // page_size * page_size + end = max(ptr + length for ptr, length in regions) + if mr_ptr < storage_ptr or end > storage_ptr + storage.nbytes(): raise ValueError( - f"shared KV backing too small for page-aligned MR: " - f"need {reg_nbytes}, have {nbytes}" + "Page-aligned shared KV registration is outside its storage" ) - reg = torch.as_strided(base, (reg_nbytes,), (1,), 0 if slack == 0 else -slack) - mr_ptr = reg.data_ptr() + # RDMA accepts an arbitrary extent; rounding the final SSM region up + # could extend the registration past the last allocated byte. + reg = torch.as_strided(backing, (end - mr_ptr,), (1,), mr_ptr - storage_ptr) return reg, { name: cache.data_ptr() - mr_ptr for name, cache in kv_caches.items() } @@ -2463,9 +2472,11 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): and len({id(t.untyped_storage()) for t in kv_caches.values()}) == 1 ) shared_mr = None + shared_mr_base = None if shared_backing: reg_tensor, self.kv_layer_mr_offset = self._build_shared_kv_mr(kv_caches) shared_mr = self.moriio_wrapper.register_local_tensor(reg_tensor) + shared_mr_base = reg_tensor.data_ptr() # Say it out loud when the groups disagree. This is expected on a hybrid # model and used to be fatal, so a run that silently had one page size @@ -2511,11 +2522,21 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): ssm.is_contiguous(), ssm.untyped_storage().nbytes(), ) + self.kv_region_mr_offsets[layer_name] = [] for tensor in (conv, ssm): alias = self._contiguous_byte_alias(tensor) - meta = self.moriio_wrapper.register_local_tensor(alias) + meta = ( + shared_mr + if shared_mr is not None + else self.moriio_wrapper.register_local_tensor(alias) + ) self.layer_name_to_local_kv_cache_metadata[layer_name].append(meta) self.local_kv_cache_size.append(alias.numel()) + self.kv_region_mr_offsets[layer_name].append( + 0 + if shared_mr_base is None + else alias.data_ptr() - shared_mr_base + ) else: moriio_mem_metadata = shared_mr or ( self.moriio_wrapper.register_local_tensor(kv_cache) @@ -2728,6 +2749,11 @@ def _pop_done_transfers(self) -> set[str]: ) to_remove.append(req_id) elif state is TransferBatchState.FAILED: + if self._has_mamba: + raise TransferError( + f"MoRIIO hybrid READ failed for request {req_id}; " + "refusing to release or reuse in-flight KV blocks" + ) failed_status = next( (status for status in statuses if status.Failed()), None ) @@ -2765,6 +2791,11 @@ def _pop_done_transfers(self) -> set[str]: # indefinitely on this request. _age = time.monotonic() - self._recving_transfers_start[req_id] if _age > _xfer_timeout: + if self._has_mamba: + raise TransferError( + f"MoRIIO hybrid READ timed out for request {req_id}; " + "refusing to release or reuse in-flight KV blocks" + ) logger.error( "RDMA read TIMED OUT for req %s after %.1fs " "(kv_connector_extra_config.recv_abort_timeout=%.0f)", @@ -3445,6 +3476,8 @@ def _post_mamba_reads( local_slots: list[int], remote_slots: list[int], remote_tp_size: int, + remote_moriio_meta: MoRIIOAgentMetadata, + remote_engine_id: EngineId, request_id: str, deadline: float, ) -> list: @@ -3460,10 +3493,22 @@ def _post_mamba_reads( remote_slots, remote_tp_size=remote_tp_size, ) + local_region_offsets = self.kv_region_mr_offsets[layer_name] + remote_metas = self.layer_name_to_remote_kv_cache_metadata[remote_engine_id][ + layer_name + ] + base_idx = self.layer_base_addr_index[layer_name] + remote_region_offsets = [ + remote_moriio_meta.kv_caches_base_addr[base_idx + i] + - self.moriio_wrapper.get_unpack_memory_metadata(meta).data + for i, meta in enumerate(remote_metas) + ] statuses: list = [] - for sess_idx, sl in ( - (region_sessions[0], slice(0, n_conv)), - (region_sessions[1], slice(n_conv, None)), + for region_idx, (sess_idx, sl) in enumerate( + ( + (region_sessions[0], slice(0, n_conv)), + (region_sessions[1], slice(n_conv, None)), + ) ): region_sizes = sizes[sl] if not region_sizes: @@ -3472,8 +3517,11 @@ def _post_mamba_reads( self._post_read_with_backoff( sessions[sess_idx], region_sizes, - local[sl], - remote[sl], + [offset + local_region_offsets[region_idx] for offset in local[sl]], + [ + offset + remote_region_offsets[region_idx] + for offset in remote[sl] + ], request_id, layer_name, deadline, @@ -3496,8 +3544,8 @@ def _post_read_with_backoff( read_remote_data posts synchronously, so a send-queue-full rejection is a Failed() status on return; a separate CQ-poll thread drains completions and frees SQ depth, so we back off and re-post until - transfer_timeout, then store the failed status (get_finished notifies - prefill and drops the request non-fatally). + transfer_timeout, then store the failed status for completion/error + handling. Hybrid failures abort before releasing source KV. """ _backoff = _SQ_FULL_BACKOFF_INITIAL_S while True: @@ -3510,8 +3558,8 @@ def _post_read_with_backoff( logger.warning( "MoRIIO READ send queue stayed full past " "transfer_timeout for req %s layer %s; storing failed " - "status (get_finished notifies prefill and drops the " - "request). Raise qp_per_transfer if frequent.", + "status for completion/error handling. " + "Raise qp_per_transfer if frequent.", request_id, layer_name, ) @@ -3652,6 +3700,8 @@ def _read_blocks( local_mamba, remote_mamba, remote_tp_size, + remote_moriio_meta, + remote_dp_engine_id, request_id, _sq_deadline, ) diff --git a/vllm/parser/kimi_k3.py b/vllm/parser/kimi_k3.py index 587e1d5ad6b5..e1342aad0e02 100644 --- a/vllm/parser/kimi_k3.py +++ b/vllm/parser/kimi_k3.py @@ -10,8 +10,9 @@ DeltaMessage, FunctionCall, ) -from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.abstract_parser import DelegatingParser, StreamState from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser +from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser if TYPE_CHECKING: from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -93,6 +94,33 @@ def _extract_tool_calls_streaming( delta_message.tool_calls = [] return delta_message, False + def finalize_generation( + self, + delta_message: DeltaMessage | None, + request: ChatCompletionRequest | ResponsesRequest, + state: StreamState, + ) -> DeltaMessage | None: + delta_message = super().finalize_generation(delta_message, request, state) + if not state.reasoning_ended and isinstance( + self._reasoning_parser, KimiK3ReasoningParser + ): + tail = self._reasoning_parser.finish_reasoning_streaming( + state.previous_text + ) + if tail: + if delta_message is None: + delta_message = DeltaMessage() + delta_message.reasoning = (delta_message.reasoning or "") + tail + elif state.reasoning_ended and isinstance(self._tool_parser, KimiK3ToolParser): + content_tail = self._tool_parser._extract_response_content( + state.previous_text, finished=True + ) + if content_tail: + if delta_message is None: + delta_message = DeltaMessage() + delta_message.content = (delta_message.content or "") + content_tail + return delta_message + def parse_delta( self, delta_text: str, @@ -116,14 +144,16 @@ def parse_delta( self._tool_parser is not None or not isinstance(self._reasoning_parser, KimiK3ReasoningParser) or not state.reasoning_ended - or delta_message is None ): return delta_message stripped = self._reasoning_parser.strip_content_streaming( previous_text=previous_content, current_text=state.previous_text, + finished=finished, ) + if delta_message is None: + return stripped delta_message.content = stripped.content if stripped is not None else None if ( delta_message.role is None diff --git a/vllm/reasoning/kimi_k3_reasoning_parser.py b/vllm/reasoning/kimi_k3_reasoning_parser.py index 1a1c62ba6def..5ab9a139e635 100644 --- a/vllm/reasoning/kimi_k3_reasoning_parser.py +++ b/vllm/reasoning/kimi_k3_reasoning_parser.py @@ -359,7 +359,16 @@ def _reasoning_text_ready_to_emit(self, text: str) -> str: break return text[:-overlap] if overlap else text - def _content_ready_to_emit(self, text: str) -> str: + def finish_reasoning_streaming(self, text: str) -> str: + """Release an unfinished marker as literal text at end of stream.""" + if self._think_close_re.search(text): + return "" + safe = self._reasoning_text_ready_to_emit(text) + opened = self._think_open_re.search(text) + body = text[opened.end() :] if opened is not None else text + return body[len(safe) :] + + def _content_ready_to_emit(self, text: str, *, finished: bool = False) -> str: """Return the content prefix that is safe to stream now. Mirrors ``_reasoning_text_ready_to_emit`` but for the post-reasoning @@ -376,6 +385,9 @@ def _content_ready_to_emit(self, text: str) -> str: text = self._response_close_re.sub("", text) text = self._message_close_re.sub("", text) + if finished: + return text + # Hold back partial markers at the end overlap = 0 for marker in ( @@ -394,6 +406,8 @@ def strip_content_streaming( self, previous_text: str, current_text: str, + *, + finished: bool = False, ) -> DeltaMessage | None: """Strip XTML content wrappers from streaming deltas after reasoning. @@ -404,7 +418,7 @@ def strip_content_streaming( Works from accumulated text (``previous_text`` / ``current_text`` already contain only post-reasoning content). """ - current_safe = self._content_ready_to_emit(current_text) + current_safe = self._content_ready_to_emit(current_text, finished=finished) previous_safe = self._content_ready_to_emit(previous_text) if current_safe.startswith(previous_safe): delta = current_safe[len(previous_safe) :] diff --git a/vllm/tool_parsers/kimi_k3_tool_parser.py b/vllm/tool_parsers/kimi_k3_tool_parser.py index 76c6b0aa657c..8fcc2fdaa505 100644 --- a/vllm/tool_parsers/kimi_k3_tool_parser.py +++ b/vllm/tool_parsers/kimi_k3_tool_parser.py @@ -268,7 +268,9 @@ def _content(self, model_output: str, before: str) -> str | None: return m["c"] or None return self._strip_response_content(before) - def _extract_response_content(self, current_text: str) -> str | None: + def _extract_response_content( + self, current_text: str, *, finished: bool = False + ) -> str | None: # Streaming response text is computed from the accumulated text. This is # what keeps split markers from leaking: # <|open|> / response / <|sep|>Hi -> emit only "Hi" after open closes @@ -289,6 +291,8 @@ def _extract_response_content(self, current_text: str) -> str | None: candidates = [i for i in (tools_start, response_end) if i != -1] if candidates: sendable_idx = min(candidates) + elif finished: + sendable_idx = len(current_text) else: overlap = max( _partial_tag_overlap(current_text, self.response_open),