diff --git a/tests/unit_tests/kv_offload/offloading_connector/test_scheduler.py b/tests/unit_tests/kv_offload/offloading_connector/test_scheduler.py index 29e75b14de..130405498c 100644 --- a/tests/unit_tests/kv_offload/offloading_connector/test_scheduler.py +++ b/tests/unit_tests/kv_offload/offloading_connector/test_scheduler.py @@ -47,15 +47,11 @@ def test_offloading_connector(request_runner, async_scheduling: bool): runner.run(decoded_tokens=[0] * (offloaded_block_size + 1)) # 1 more block (+ token for kicking off offloading) - # now check touch was called with all 6 blocks runner.manager.prepare_store.side_effect = (lambda block_hashes, req_context: generate_store_output(block_hashes)) runner.run( decoded_tokens=[0] * (offloaded_block_size + 1), expected_stored_gpu_block_indexes=(15, 16, 17), ) - runner.manager.touch.assert_called() - block_hashes1 = list(runner.manager.touch.call_args.args[0]) - assert len(block_hashes1) == 6 # terminate request runner.run(decoded_tokens=[EOS_TOKEN_ID]) @@ -63,13 +59,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool): # create a new request differing only on the last token runner.new_request(token_ids=[0] * (offloaded_block_size * 6 - 1) + [1]) runner.run(decoded_tokens=[0]) - runner.manager.touch.assert_called() - block_hashes2 = list(runner.manager.touch.call_args.args[0]) - assert len(block_hashes2) == 6 - - # verify hashes are the same, except for the last block - assert block_hashes1[:5] == block_hashes2[:5] - assert block_hashes1[5] != block_hashes2[5] + runner.manager.touch.assert_not_called() # terminate request runner.run( diff --git a/vllm_gaudi/entrypoints/openai/multi_model_api_server.py b/vllm_gaudi/entrypoints/openai/multi_model_api_server.py index 4534fa34e8..a9207f9e33 100644 --- a/vllm_gaudi/entrypoints/openai/multi_model_api_server.py +++ b/vllm_gaudi/entrypoints/openai/multi_model_api_server.py @@ -149,6 +149,9 @@ async def reset_prefix_cache(self, reset_running_requests: bool = False, reset_c async def sleep(self, level: int = 1, mode: str = "abort") -> None: await self._engine.sleep(level=level, mode=mode) + async def release_kv_cache_memory(self) -> None: + await self._engine.release_kv_cache_memory() + async def wake_up(self, tags: list[str] | None = None) -> None: await self._engine.wake_up(tags=tags) diff --git a/vllm_gaudi/lora/punica_wrapper/punica_hpu.py b/vllm_gaudi/lora/punica_wrapper/punica_hpu.py index ca031a3b42..b54842a368 100644 --- a/vllm_gaudi/lora/punica_wrapper/punica_hpu.py +++ b/vllm_gaudi/lora/punica_wrapper/punica_hpu.py @@ -69,6 +69,36 @@ def add_lora_logits( dispatch_bgmv_linear(y, x, lora_a_stacked, lora_b_stacked, 0, scale) y = y.view_as(y_org) + def apply_lora_full_linear( + self, + y: torch.Tensor, + x: torch.Tensor, + weight_stacked: torch.Tensor, + bias_stacked: torch.Tensor, + module_enabled: torch.Tensor, + ) -> None: + """Apply request-routed full linear weights to selected rows. + + Used by `modules_to_save` adapters (e.g. a LoRA-replaced classification + head), where the adapter ships whole weights instead of an A/B pair. + + Args: + y: Base layer output, shape (num_rows, out_features), updated in place. + x: Layer input, shape (num_rows, in_features). + weight_stacked: Replacement weights, shape (max_loras, 1, out_features, in_features). + bias_stacked: Replacement bias, shape (max_loras, out_features). + module_enabled: Per-adapter flag marking which adapters saved the module. + """ + x = x.view(-1, x.shape[-1]) + num_loras, _, out_features, _ = weight_stacked.shape + # Multiply against every adapter and gather the routed row afterwards, so the + # weight operand keeps a static shape for HPU graph compilation. + weights = weight_stacked.view(num_loras * out_features, -1).t().to(x.dtype) + all_y = (x @ weights).view(-1, num_loras, out_features) + indices = self.sampler_indices.clamp_min(0) + adapter_y = all_y.gather(1, indices.view(-1, 1, 1).expand(-1, 1, out_features)).squeeze(1) + y.copy_(self._select_full_linear_output(y, adapter_y, bias_stacked, module_enabled)) + def add_shrink( self, y: Union[tuple[torch.Tensor, ...], torch.Tensor], diff --git a/vllm_gaudi/ops/hpu_fused_moe.py b/vllm_gaudi/ops/hpu_fused_moe.py index 7e102f6ee3..e6a81f6fb5 100644 --- a/vllm_gaudi/ops/hpu_fused_moe.py +++ b/vllm_gaudi/ops/hpu_fused_moe.py @@ -836,6 +836,8 @@ def create_fused_moe_router( # Deepseek V4 vision routing bias parameters bias_vl: torch.Tensor | None = None, image_sentinel_lo: int = 0, + # sequence-parallel padding parameters (upstream vLLM PR 56079+) + skip_padding: bool = False, ) -> FusedMoERouter: """ Factory function to create the appropriate FusedMoERouter subclass based on @@ -884,6 +886,12 @@ def create_fused_moe_router( hash_indices_table: Used to map input_ids to experts, needed for Deepseek V4 + Sequence-parallel padding arguments (upstream vLLM PR 56079+): + skip_padding: Whether grouped routing should invalidate padding rows. + Forwarded to GroupedTopKRouter for signature parity; the guard it + enables (VLLM_MOE_SKIP_PADDING + a DeepEP-v2-kernel forward + context) is not exercised on HPU, so this is a no-op here. + Vision routing bias arguments (upstream vLLM PR 54566+): bias_vl: Vision routing bias for image tokens (Deepseek V4). image_sentinel_lo: First of five consecutive in-vocab image sentinel @@ -932,6 +940,7 @@ def create_fused_moe_router( routed_scaling_factor=routed_scaling_factor, e_score_correction_bias=e_score_correction_bias, num_fused_shared_experts=num_fused_shared_experts, + skip_padding=skip_padding, ) return grouped_topk_router diff --git a/vllm_gaudi/v1/engine/core_patch.py b/vllm_gaudi/v1/engine/core_patch.py index a7a01c8263..9c7e98c878 100644 --- a/vllm_gaudi/v1/engine/core_patch.py +++ b/vllm_gaudi/v1/engine/core_patch.py @@ -88,7 +88,14 @@ def _reset_executor_sleep_state(model_executor: Any) -> None: else: model_executor.sleeping_tags = set() - model_executor.is_sleeping = False + # Upstream vllm#44890 turned `is_sleeping` into a read-only property + # computed from `sleeping_tags` (Executor.is_sleeping -> bool(self.sleeping_tags)), + # so assigning it raises AttributeError on executors built against that + # vllm. Clearing sleeping_tags above already makes the property read + # False there; only assign the plain attribute on older executors where + # is_sleeping is still a writable instance attribute. + if not isinstance(getattr(type(model_executor), "is_sleeping", None), property): + model_executor.is_sleeping = False def _require_reconfigure_attr(config: Any, path: tuple[str, ...]) -> None: