From ed66a60518786d0141c4c8733d9908226d9b0156 Mon Sep 17 00:00:00 2001 From: yuzhou Date: Wed, 15 Jan 2025 16:34:32 +0200 Subject: [PATCH 01/23] add block groups Signed-off-by: yuzhou Signed-off-by: zhouyu5 --- vllm/attention/backends/hpu_attn.py | 2 +- vllm/attention/ops/hpu_paged_attn.py | 1 + vllm/worker/hpu_model_runner.py | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vllm/attention/backends/hpu_attn.py b/vllm/attention/backends/hpu_attn.py index 94a461e0c8c2..d05629ec9c0c 100644 --- a/vllm/attention/backends/hpu_attn.py +++ b/vllm/attention/backends/hpu_attn.py @@ -237,7 +237,7 @@ def forward( block_mapping=attn_metadata.block_mapping, block_bias=attn_metadata.attn_bias, block_scales=attn_metadata.block_scales, - block_groups=None, + block_groups=attn_metadata.block_groups, scale=self.scale, matmul_qk_op=self.matmul_qk, matmul_av_op=self.matmul_av, diff --git a/vllm/attention/ops/hpu_paged_attn.py b/vllm/attention/ops/hpu_paged_attn.py index 4c0fb2a62836..603d3959377c 100644 --- a/vllm/attention/ops/hpu_paged_attn.py +++ b/vllm/attention/ops/hpu_paged_attn.py @@ -21,6 +21,7 @@ class HPUPagedAttentionMetadata: block_indices: Optional[torch.Tensor] block_offsets: Optional[torch.Tensor] block_scales: Optional[torch.Tensor] + block_groups: Optional[torch.Tensor] class HPUPagedAttention: diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index 260ffaf27f9a..43239efe2041 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -892,6 +892,7 @@ def _prepare_prompt( block_indices=block_indices, block_offsets=block_offsets, block_scales=None, + block_groups=None, attn_bias=None, seq_lens_tensor=seq_lens_tensor, num_prefills=real_num_seqs, @@ -1047,6 +1048,7 @@ def _prepare_decode( block_indices=block_indices, block_offsets=block_offsets, block_scales=block_scales, + block_groups=None, attn_bias=None, seq_lens_tensor=None, num_prefills=0, @@ -1258,7 +1260,7 @@ def trim_attn_metadata(self, metadata: AttentionMetadata) -> object: attention_metadata = subtuple(metadata, 'TrimmedAttentionMetadata', [ 'attn_bias', 'seq_lens_tensor', 'block_list', 'block_mapping', 'block_usage', 'slot_mapping', 'is_prompt', 'block_indices', - 'block_offsets', 'block_scales' + 'block_offsets', 'block_scales', 'block_groups' ]) return attention_metadata From 4ea16015bfcc6149543b99aa605cd5657865d36f Mon Sep 17 00:00:00 2001 From: yuzhou Date: Wed, 15 Jan 2025 16:38:37 +0200 Subject: [PATCH 02/23] add env Signed-off-by: yuzhou Signed-off-by: zhouyu5 --- vllm/attention/backends/hpu_attn.py | 4 ---- vllm/worker/hpu_model_runner.py | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/vllm/attention/backends/hpu_attn.py b/vllm/attention/backends/hpu_attn.py index d05629ec9c0c..93c28bcf88fa 100644 --- a/vllm/attention/backends/hpu_attn.py +++ b/vllm/attention/backends/hpu_attn.py @@ -114,12 +114,8 @@ def __init__( self.matmul_av = Matmul() self.batch2block_matmul = Matmul() self.block2batch_matmul = Matmul() - # NOTE(kzawora): Contiguous PA is off until model runner supports it self.k_cache = VLLMKVCache() - self.k_cache.use_contiguous_pa = False self.v_cache = VLLMKVCache() - self.v_cache.use_contiguous_pa = False - # NOTE(kzawora): Pipelined PA is off until model runner supports it ops.pa_impl = ops.pa self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index 43239efe2041..15b148de1c78 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -257,10 +257,19 @@ def setup_profiler(): return profiler -def pad_list(list, k, v): - target_len = round_up(len(list), k) - padding = target_len - len(list) - return list + [v] * padding +def pad_list(input, k, v): + input_len = len(input) + target_len = round_up(input_len, k) + padding = target_len - input_len + return input + [v] * padding + + +def gather_list(input, indices, v): + return [input[i] if i is not None else v for i in indices] + + +def flatten(in_list): + return list(itertools.chain(*in_list)) def precompute_indices_and_offsets(block_size, slot_mapping, is_prompt): @@ -583,6 +592,8 @@ def __init__( self.bucketing_global_state = HPUBucketingGlobalState() self._setup_buckets() self._set_gc_threshold() + self.use_contiguous_pa = os.environ.get('VLLM_CONTIGUOUS_PA', + 'true').lower() == 'true' def _set_gc_threshold(self) -> None: # Read https://docs.python.org/3/library/gc.html#gc.set_threshold From 98047e1c013f371ca199877e24aae5261cec87ad Mon Sep 17 00:00:00 2001 From: yuzhou Date: Wed, 15 Jan 2025 17:03:36 +0200 Subject: [PATCH 03/23] make cont pa functional Signed-off-by: yuzhou Signed-off-by: zhouyu5 --- vllm/worker/hpu_model_runner.py | 92 +++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index 15b148de1c78..5eb7f7382e53 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -22,6 +22,7 @@ import habana_frameworks.torch.internal.bridge_config as bc import torch from vllm_hpu_extension.ops import LoraMask as LoraMask +from vllm_hpu_extension.ops import batch2block, block2batch from vllm_hpu_extension.profiler import (HabanaHighLevelProfiler, HabanaMemoryProfiler, format_bytes) @@ -340,12 +341,22 @@ def _set_block_mapping(self, metadata, batch_size, device, dtype): mask = mask >= metadata.block_usage.unsqueeze(-1) attn_bias = (torch.zeros_like(mask, dtype=dtype).masked_fill_( mask, -math.inf)) - block_mapping = torch.nn.functional.one_hot(metadata.block_mapping, + block_mapping = torch.nn.functional.one_hot(metadata.block_groups, num_classes=batch_size) block_mapping = block_mapping.to(dtype) metadata = metadata._replace(block_mapping=block_mapping, attn_bias=attn_bias) return metadata + + def _set_block_scales(self, metadata, device): + block_mapping = metadata.block_mapping + ones = torch.ones((block_mapping.size(0), ), + device=device, + dtype=block_mapping.dtype) + sums = batch2block(block2batch(ones, block_mapping), block_mapping) + block_scales = torch.reciprocal(torch.maximum(ones, sums)) + metadata = metadata._replace(block_scales=block_scales) + return metadata def _update_metadata(self, attn_metadata, batch_size, seq_len, device, dtype): @@ -357,6 +368,7 @@ def _update_metadata(self, attn_metadata, batch_size, seq_len, device, meta = attn_metadata attn_metadata = self._set_block_mapping(meta, batch_size, device, dtype) + attn_metadata = self._set_block_scales(attn_metadata, device) return attn_metadata def forward(self, *args, **kwargs): @@ -1000,66 +1012,68 @@ def _prepare_decode( num_decode_tokens = sum(seq_lens) - blocks_used = [len(bt) for bt in block_tables if bt] - block_list = [] - block_scales = [] - for i, bt in enumerate(block_tables): - block_list.extend(bt) - blocks_in_group = len(bt) - if blocks_in_group > 0: - scale = 1.0 / blocks_in_group - block_scales.extend([scale] * blocks_in_group) - - block_mapping_nested: List[List[int]] = [ - [i] * b_u for i, b_u in enumerate(blocks_used) + last_block_usage = [ + slot[0] % self.block_size + 1 for slot in slot_mapping ] - block_mapping: List[int] = list( - itertools.chain.from_iterable(block_mapping_nested)) + block_groups = [[i] * len(bt) for i, bt in enumerate(block_tables)] + block_usage = [[self.block_size] * (len(bt) - 1) + [lbu] + for bt, lbu in zip(block_tables, last_block_usage) + if bt] + + block_list = flatten(block_tables) + block_groups = flatten(block_groups) + block_usage = flatten(block_usage) + + assert len(block_list) == len(block_groups) + assert len(block_list) == len(block_usage) + + padding_fn = None + if self.use_contiguous_pa: + block_bucket_size = find_bucket( + block_bucket_size, + self.bucketing_global_state.decode_block_bucket_cfg) + indices: List[Any] + indices = [None] * block_bucket_size + for i, bid in enumerate(block_list): + indices[bid] = i + padding_fn = lambda tensor, pad_value: gather_list( + tensor, indices, pad_value) + else: + block_bucket_size = find_bucket( + len(block_list), + self.bucketing_global_state.decode_block_bucket_cfg) + padding_fn = lambda tensor, pad_value: pad_list( + tensor, block_bucket_size, pad_value) - last_block = [ - sl % self.block_size + 1 for sl in itertools.chain(*slot_mapping) - ] - block_usage = [[self.block_size] * (b_u - 1) + [lb] - for b_u, lb in zip(blocks_used, last_block)] - block_usage = list(itertools.chain(*block_usage)) - - block_bucket_size = find_bucket( - len(block_list), - self.bucketing_global_state.decode_block_bucket_cfg) - block_list = pad_list(block_list, block_bucket_size, _PAD_BLOCK_ID) - block_mapping = pad_list(block_mapping, block_bucket_size, -1) - block_usage = pad_list(block_usage, block_bucket_size, 1) - block_scales = pad_list(block_scales, block_bucket_size, 0.0) + block_list = padding_fn(block_list, _PAD_BLOCK_ID) + block_groups = padding_fn(block_groups, -1) + block_usage = padding_fn(block_usage, 1) block_list = torch.tensor(block_list, dtype=torch.int, device=self.device) - block_mapping = torch.tensor(block_mapping, - dtype=torch.long, - device=self.device) + block_groups = torch.tensor(block_groups, + dtype=torch.int, + device=self.device) block_usage = torch.tensor(block_usage, dtype=self.model_config.dtype, device=self.device) - slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=self.device) block_indices, block_offsets = precompute_indices_and_offsets( self.block_size, slot_mapping, False) - block_scales = torch.tensor(block_scales, - dtype=self.model_config.dtype, - device=self.device) attn_metadata = self.attn_backend.make_metadata( is_prompt=False, block_list=block_list, - block_mapping=block_mapping, + block_mapping=None, block_usage=block_usage, block_indices=block_indices, block_offsets=block_offsets, - block_scales=block_scales, - block_groups=None, + block_scales=None, + block_groups=block_groups, attn_bias=None, seq_lens_tensor=None, num_prefills=0, From eede3ce14213cbe38eb900fff84ae4ea8eedc653 Mon Sep 17 00:00:00 2001 From: yuzhou Date: Wed, 15 Jan 2025 17:13:19 +0200 Subject: [PATCH 04/23] bug fix Signed-off-by: yuzhou Signed-off-by: zhouyu5 --- vllm/worker/hpu_model_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index 5eb7f7382e53..d00fabc815a1 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -1029,6 +1029,7 @@ def _prepare_decode( padding_fn = None if self.use_contiguous_pa: + block_bucket_size = max(max(block_list) + 1, len(block_list)) block_bucket_size = find_bucket( block_bucket_size, self.bucketing_global_state.decode_block_bucket_cfg) From 7ea419d86c5bd3214642e7d409bde0337b3c1fa4 Mon Sep 17 00:00:00 2001 From: yuzhou Date: Wed, 15 Jan 2025 17:07:56 +0200 Subject: [PATCH 05/23] Reduce block fragmentation Signed-off-by: yuzhou Signed-off-by: zhouyu5 --- vllm/core/block/naive_block.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py index 9b94918ab38e..e886e8bb1fce 100644 --- a/vllm/core/block/naive_block.py +++ b/vllm/core/block/naive_block.py @@ -1,4 +1,4 @@ -from collections import deque +import heapq from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter, @@ -36,7 +36,8 @@ def __init__( if block_ids is None: block_ids = range(num_blocks) - self._free_block_indices: Deque[BlockId] = deque(block_ids) + self._free_block_indices: Deque[BlockId] = block_ids[:] + heapq.heapify(self._free_block_indices) self._all_block_indices = frozenset(block_ids) assert len(self._all_block_indices) == num_blocks @@ -132,7 +133,7 @@ def _allocate_block_id(self) -> BlockId: if not self._free_block_indices: raise BlockAllocator.NoFreeBlocksError() - block_id = self._free_block_indices.popleft() + block_id = heapq.heappop(self._free_block_indices) self._refcounter.incr(block_id) return block_id @@ -142,7 +143,7 @@ def _free_block_id(self, block: Block) -> None: refcount = self._refcounter.decr(block_id) if refcount == 0: - self._free_block_indices.appendleft(block_id) + heapq.heappush(self._free_block_indices, block_id) block.block_id = None From ba1020d6dd85b24279ddd5fc6b373104d2b19a03 Mon Sep 17 00:00:00 2001 From: yuzhou Date: Fri, 17 Jan 2025 10:23:41 +0200 Subject: [PATCH 06/23] fix type Signed-off-by: yuzhou Signed-off-by: zhouyu5 --- vllm/core/block/naive_block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py index e886e8bb1fce..ed9120ba25a1 100644 --- a/vllm/core/block/naive_block.py +++ b/vllm/core/block/naive_block.py @@ -36,7 +36,7 @@ def __init__( if block_ids is None: block_ids = range(num_blocks) - self._free_block_indices: Deque[BlockId] = block_ids[:] + self._free_block_indices: List[BlockId] = block_ids[:] heapq.heapify(self._free_block_indices) self._all_block_indices = frozenset(block_ids) assert len(self._all_block_indices) == num_blocks From 62fd8ed835420482c4b3c457c54fb4018abb70ad Mon Sep 17 00:00:00 2001 From: yuzhou Date: Fri, 17 Jan 2025 10:28:11 +0200 Subject: [PATCH 07/23] fix format Signed-off-by: yuzhou Signed-off-by: zhouyu5 --- vllm/core/block/naive_block.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py index ed9120ba25a1..8d2a60881d74 100644 --- a/vllm/core/block/naive_block.py +++ b/vllm/core/block/naive_block.py @@ -1,5 +1,5 @@ import heapq -from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple +from typing import FrozenSet, Iterable, List, Optional, Tuple from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter, get_all_blocks_recursively) @@ -36,7 +36,7 @@ def __init__( if block_ids is None: block_ids = range(num_blocks) - self._free_block_indices: List[BlockId] = block_ids[:] + self._free_block_indices: List[BlockId] = list(block_ids) heapq.heapify(self._free_block_indices) self._all_block_indices = frozenset(block_ids) assert len(self._all_block_indices) == num_blocks From bb0a1cfd69d2e5e5595d1aaa389a017a47d7515a Mon Sep 17 00:00:00 2001 From: yuzhou Date: Fri, 17 Jan 2025 11:07:55 +0200 Subject: [PATCH 08/23] fix format Signed-off-by: zhouyu5 --- vllm/worker/hpu_model_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index d00fabc815a1..1874c428c668 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -347,7 +347,7 @@ def _set_block_mapping(self, metadata, batch_size, device, dtype): metadata = metadata._replace(block_mapping=block_mapping, attn_bias=attn_bias) return metadata - + def _set_block_scales(self, metadata, device): block_mapping = metadata.block_mapping ones = torch.ones((block_mapping.size(0), ), @@ -1540,7 +1540,7 @@ def warmup_model(self, kv_caches: List[torch.Tensor]) -> None: msg = (f"Generated {len(self.bucketing_global_state.prompt_buckets)} " f"prompt buckets [bs, seq]: \ - {list(sorted(self.bucketing_global_state.prompt_buckets))}") + {list(sorted(self.bucketing_global_state.prompt_buckets))}" ) logger.info(msg) msg = (f"Omitted {len(prompt_omitted_buckets)} " From 81c362daa5cf01bcee9cc758bc5834dcc27b84c7 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Fri, 17 Jan 2025 11:18:17 +0200 Subject: [PATCH 09/23] fix format Signed-off-by: zhouyu5 --- vllm/worker/hpu_model_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index 1874c428c668..203b6dc416d6 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -1540,7 +1540,7 @@ def warmup_model(self, kv_caches: List[torch.Tensor]) -> None: msg = (f"Generated {len(self.bucketing_global_state.prompt_buckets)} " f"prompt buckets [bs, seq]: \ - {list(sorted(self.bucketing_global_state.prompt_buckets))}" ) + {list(sorted(self.bucketing_global_state.prompt_buckets))}") logger.info(msg) msg = (f"Omitted {len(prompt_omitted_buckets)} " From 0987c2f6a631077f72b343eabfcb681e3d8deeb7 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Sun, 26 Jan 2025 09:11:52 +0200 Subject: [PATCH 10/23] minor fix Signed-off-by: zhouyu5 --- vllm/core/block/naive_block.py | 6 ++++++ vllm/worker/hpu_model_runner.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py index d7842ee3243d..c676c044e938 100644 --- a/vllm/core/block/naive_block.py +++ b/vllm/core/block/naive_block.py @@ -36,6 +36,12 @@ def __init__( if block_ids is None: block_ids = range(num_blocks) + # use heap to manage free block indices instead of deque + # similar to deque, heap removes the first element in O(1) + # it ensure that the smallest block id is always at the top + # which is the next block to be allocated + # it can improve the performance of block allocation + # in some cases while do not affect the overall performance self._free_block_indices: List[BlockId] = list(block_ids) heapq.heapify(self._free_block_indices) self._all_block_indices = frozenset(block_ids) diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index 1c39e0d87540..b4f7894e65ab 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -606,7 +606,7 @@ def __init__( self._setup_buckets() self._set_gc_threshold() self.use_contiguous_pa = os.environ.get('VLLM_CONTIGUOUS_PA', - 'true').lower() == 'true' + 'true').lower() in ['1', 'true'] def _set_gc_threshold(self) -> None: # Read https://docs.python.org/3/library/gc.html#gc.set_threshold From 35ab680eb877bd086df366cf74c498a984ec1f58 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Mon, 10 Feb 2025 03:19:00 +0200 Subject: [PATCH 11/23] fix format Signed-off-by: zhouyu5 --- vllm/worker/hpu_model_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index d342f0c3b429..65747e64099a 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -607,8 +607,8 @@ def __init__( self.bucketing_global_state = HPUBucketingGlobalState() self._setup_buckets() self._set_gc_threshold() - self.use_contiguous_pa = os.environ.get('VLLM_CONTIGUOUS_PA', - 'true').lower() in ['1', 'true'] + self.use_contiguous_pa = os.environ.get( + 'VLLM_CONTIGUOUS_PA', 'true').lower() in ['1', 'true'] def _set_gc_threshold(self) -> None: # Read https://docs.python.org/3/library/gc.html#gc.set_threshold From 2d837f335a5fabd9f414b3a4b22f0b254f014703 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Mon, 10 Feb 2025 03:27:30 +0200 Subject: [PATCH 12/23] fix format Signed-off-by: zhouyu5 --- vllm/core/block/naive_block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py index df0eae18fc30..0580b14e45ff 100644 --- a/vllm/core/block/naive_block.py +++ b/vllm/core/block/naive_block.py @@ -42,7 +42,7 @@ def __init__( # similar to deque, heap removes the first element in O(1) # it ensure that the smallest block id is always at the top # which is the next block to be allocated - # it can improve the performance of block allocation + # it can improve the performance of block allocation # in some cases while do not affect the overall performance self._free_block_indices: List[BlockId] = list(block_ids) heapq.heapify(self._free_block_indices) From 349548daee0fc8596ec82968b8e0ef918abe7276 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Thu, 13 Feb 2025 05:24:48 +0200 Subject: [PATCH 13/23] fix env Signed-off-by: zhouyu5 --- vllm/envs.py | 8 ++++++++ vllm/worker/hpu_model_runner.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 745b068b7a45..1e0fb5340aea 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -87,6 +87,7 @@ VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False VLLM_RAY_PER_WORKER_GPUS: float = 1.0 VLLM_RAY_BUNDLE_INDICES: str = "" + VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH: bool = True def get_default_cache_root(): @@ -572,6 +573,13 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]: # models the alignment is already naturally aligned to 256 bytes. "VLLM_CUDA_MEM_ALIGN_KV_CACHE": lambda: bool(int(os.getenv("VLLM_CUDA_MEM_ALIGN_KV_CACHE", "1"))), + + # Contiguous cache fetching to avoid using costly gather operation on + # Gaudi3. This is only applicable to HPU contiguous cache. If set to true, + # contiguous cache fetch will be used. + "VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH": + lambda: os.environ.get( + 'VLLM_CONTIGUOUS_PA', 'true').lower() in ['1', 'true'], } # end-env-vars-definition diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py index 65747e64099a..fe7c776d0a23 100644 --- a/vllm/worker/hpu_model_runner.py +++ b/vllm/worker/hpu_model_runner.py @@ -29,6 +29,7 @@ from vllm_hpu_extension.profiler import (HabanaHighLevelProfiler, HabanaMemoryProfiler, format_bytes) +import vllm.envs as envs from vllm.attention import AttentionMetadata, get_attn_backend from vllm.config import DeviceConfig, VllmConfig from vllm.distributed.parallel_state import get_world_group @@ -607,8 +608,7 @@ def __init__( self.bucketing_global_state = HPUBucketingGlobalState() self._setup_buckets() self._set_gc_threshold() - self.use_contiguous_pa = os.environ.get( - 'VLLM_CONTIGUOUS_PA', 'true').lower() in ['1', 'true'] + self.use_contiguous_pa = envs.VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH def _set_gc_threshold(self) -> None: # Read https://docs.python.org/3/library/gc.html#gc.set_threshold From edf207e8b3fd8af579a287a0d019e68d215a9fbf Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Thu, 13 Feb 2025 07:23:18 +0200 Subject: [PATCH 14/23] fix format Signed-off-by: zhouyu5 --- vllm/envs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index dfc27373fad1..733a80b1f1ab 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -584,8 +584,8 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]: # Gaudi3. This is only applicable to HPU contiguous cache. If set to true, # contiguous cache fetch will be used. "VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH": - lambda: os.environ.get( - 'VLLM_CONTIGUOUS_PA', 'true').lower() in ['1', 'true'], + lambda: os.environ.get('VLLM_CONTIGUOUS_PA', 'true').lower( + ) in ['1', 'true'], } # end-env-vars-definition From f755ce8a3aa6c1b1b97c34caffe82403c06b7e72 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Thu, 13 Feb 2025 07:42:57 +0200 Subject: [PATCH 15/23] fix format Signed-off-by: zhouyu5 --- vllm/envs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/envs.py b/vllm/envs.py index 733a80b1f1ab..cff84e2d458d 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -579,7 +579,7 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]: # specify the path through environment variable VLLM_CUDART_SO_PATH. "VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None), - + # Contiguous cache fetching to avoid using costly gather operation on # Gaudi3. This is only applicable to HPU contiguous cache. If set to true, # contiguous cache fetch will be used. From 1965878393c7b3f04f7df007f8fbf92d642479ee Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Thu, 13 Feb 2025 07:51:05 +0200 Subject: [PATCH 16/23] fix format Signed-off-by: zhouyu5 --- vllm/envs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/envs.py b/vllm/envs.py index cff84e2d458d..24e31d1b7fd1 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -579,7 +579,7 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]: # specify the path through environment variable VLLM_CUDART_SO_PATH. "VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None), - + # Contiguous cache fetching to avoid using costly gather operation on # Gaudi3. This is only applicable to HPU contiguous cache. If set to true, # contiguous cache fetch will be used. From 1b74f81cb56a396338915874af4bc8e29ee35671 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Mon, 17 Feb 2025 10:20:20 +0200 Subject: [PATCH 17/23] revert change for _free_block_indices Signed-off-by: zhouyu5 --- vllm/core/block/naive_block.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py index 0580b14e45ff..c388366b825f 100644 --- a/vllm/core/block/naive_block.py +++ b/vllm/core/block/naive_block.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -import heapq -from typing import FrozenSet, Iterable, List, Optional, Tuple, Union +from collections import deque +from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple, Union from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter, get_all_blocks_recursively) @@ -38,14 +38,7 @@ def __init__( if block_ids is None: block_ids = range(num_blocks) - # use heap to manage free block indices instead of deque - # similar to deque, heap removes the first element in O(1) - # it ensure that the smallest block id is always at the top - # which is the next block to be allocated - # it can improve the performance of block allocation - # in some cases while do not affect the overall performance - self._free_block_indices: List[BlockId] = list(block_ids) - heapq.heapify(self._free_block_indices) + self._free_block_indices: Deque[BlockId] = deque(block_ids) self._all_block_indices = frozenset(block_ids) assert len(self._all_block_indices) == num_blocks @@ -141,7 +134,7 @@ def _allocate_block_id(self) -> BlockId: if not self._free_block_indices: raise BlockAllocator.NoFreeBlocksError() - block_id = heapq.heappop(self._free_block_indices) + block_id = self._free_block_indices.popleft() self._refcounter.incr(block_id) return block_id @@ -155,7 +148,7 @@ def _free_block_id(self, block: Union[Block, BlockId]) -> None: refcount = self._refcounter.decr(block_id) if refcount == 0: - heapq.heappush(self._free_block_indices, block_id) + self._free_block_indices.appendleft(block_id) def free(self, block: Block, keep_block_object: bool = False) -> None: # Release the physical block id From 912d08b9a749fe1ade3ddaa31cd4c9877f918e59 Mon Sep 17 00:00:00 2001 From: Yu-Zhou Date: Tue, 18 Feb 2025 09:58:41 +0800 Subject: [PATCH 18/23] Update vllm/envs.py Use tuple with double quotes Co-authored-by: Cody Yu Signed-off-by: zhouyu5 --- vllm/envs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/envs.py b/vllm/envs.py index 7576e88d470e..47cab38ed981 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -592,7 +592,7 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]: # contiguous cache fetch will be used. "VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH": lambda: os.environ.get('VLLM_CONTIGUOUS_PA', 'true').lower( - ) in ['1', 'true'], + ) in ("1", "true"), } # end-env-vars-definition From 91ca7d9f289778a9e2895616372c67716452b4e9 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Tue, 18 Feb 2025 04:14:55 +0200 Subject: [PATCH 19/23] fix format Signed-off-by: zhouyu5 --- vllm/envs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 47cab38ed981..548be15eb401 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -591,8 +591,8 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]: # Gaudi3. This is only applicable to HPU contiguous cache. If set to true, # contiguous cache fetch will be used. "VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH": - lambda: os.environ.get('VLLM_CONTIGUOUS_PA', 'true').lower( - ) in ("1", "true"), + lambda: os.environ.get("VLLM_CONTIGUOUS_PA", "true").lower() in + ("1", "true"), } # end-env-vars-definition From 097b34e6f1e9f06da0d9266c6b5f1041b59ebdd3 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Tue, 18 Feb 2025 04:33:30 +0200 Subject: [PATCH 20/23] fix format Signed-off-by: zhouyu5 --- vllm/envs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/envs.py b/vllm/envs.py index 548be15eb401..45547416314f 100644 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -592,7 +592,7 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]: # contiguous cache fetch will be used. "VLLM_USE_HPU_CONTIGUOUS_CACHE_FETCH": lambda: os.environ.get("VLLM_CONTIGUOUS_PA", "true").lower() in - ("1", "true"), + ("1", "true"), } # end-env-vars-definition From b0fdbaaeb5525c488b843877bf7a4d202991f751 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Tue, 18 Feb 2025 08:33:21 +0200 Subject: [PATCH 21/23] Dummy commit for triggering CI Signed-off-by: zhouyu5 From 2678b2c239782ab3e4efe52cc9c3fd5167c8a214 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Tue, 18 Feb 2025 10:13:38 +0200 Subject: [PATCH 22/23] Dummy commit for triggering CI Signed-off-by: zhouyu5 From 23dc975ddf887ee8d70e039dd52a756cfad38e3a Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Wed, 19 Feb 2025 01:56:15 +0200 Subject: [PATCH 23/23] Dummy commit for triggering CI Signed-off-by: zhouyu5