From f284ed09f4c85d0d58c5864251298fb56ba24414 Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Thu, 6 Nov 2025 12:31:31 -0800 Subject: [PATCH 1/5] [Perf] Use np.ndarray instead of list[list[int]] to reduce GC overhead Signed-off-by: Jialin Ouyang --- vllm/v1/outputs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index b5cba96e1026..947593b3c901 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, NamedTuple +import numpy as np import torch if TYPE_CHECKING: @@ -15,11 +16,11 @@ class LogprobsLists(NamedTuple): # [num_reqs x num_generated_tokens, max_num_logprobs + 1] - logprob_token_ids: list[list[int]] + logprob_token_ids: np.ndarray # [num_reqs x num_generated_tokens, max_num_logprobs + 1] - logprobs: list[list[float]] + logprobs: np.ndarray # [num_reqs x num_generated_tokens] - sampled_token_ranks: list[int] + sampled_token_ranks: np.ndarray # [num_reqs] # Used for slicing the logprobs in cases like speculative # decoding where the number of generated tokens may be @@ -60,9 +61,9 @@ class LogprobsTensors(NamedTuple): def tolists(self, cu_num_generated_tokens: list[int] | None = None): return LogprobsLists( - self.logprob_token_ids.tolist(), - self.logprobs.tolist(), - self.selected_token_ranks.tolist(), + self.logprob_token_ids.numpy(), + self.logprobs.numpy(), + self.selected_token_ranks.numpy(), cu_num_generated_tokens, ) From d9ba4cbae4fb5e08af339cf679c1d8ba1f1c0315 Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Thu, 6 Nov 2025 12:46:14 -0800 Subject: [PATCH 2/5] Call .cpu() before .numpy() to ensure the tensor is on CPU Signed-off-by: Jialin Ouyang --- vllm/v1/outputs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 947593b3c901..d6e4e252bfac 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -61,9 +61,9 @@ class LogprobsTensors(NamedTuple): def tolists(self, cu_num_generated_tokens: list[int] | None = None): return LogprobsLists( - self.logprob_token_ids.numpy(), - self.logprobs.numpy(), - self.selected_token_ranks.numpy(), + self.logprob_token_ids.to("cpu").numpy(), + self.logprobs.to("cpu").numpy(), + self.selected_token_ranks.to("cpu").numpy(), cu_num_generated_tokens, ) From afe7186086f9963c965bf38b8343bf0e33a7e1bb Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Mon, 10 Nov 2025 15:27:48 -0800 Subject: [PATCH 3/5] Convert numpy to list before appending Signed-off-by: Jialin Ouyang --- vllm/v1/engine/logprobs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vllm/v1/engine/logprobs.py b/vllm/v1/engine/logprobs.py index 4c5955d7ee2e..b618d2347265 100644 --- a/vllm/v1/engine/logprobs.py +++ b/vllm/v1/engine/logprobs.py @@ -74,7 +74,12 @@ def _update_sample_logprobs(self, logprobs_lists: LogprobsLists) -> None: token_ids_lst, logprobs_lst, ranks_lst, _ = logprobs_lists - for rank, logprobs, token_ids in zip(ranks_lst, logprobs_lst, token_ids_lst): + for rank_np, logprobs_np, token_ids_np in zip( + ranks_lst, logprobs_lst, token_ids_lst + ): + rank = rank_np.tolist() + logprobs = logprobs_np.tolist() + token_ids = token_ids_np.tolist() # Detokenize (non-incrementally). decoded_tokens = ( NONES From 9793b9094669736628b37bf81808629262af4d8d Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Mon, 10 Nov 2025 17:42:29 -0800 Subject: [PATCH 4/5] Fix MockEngineCore signature Signed-off-by: Jialin Ouyang --- tests/v1/engine/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/v1/engine/utils.py b/tests/v1/engine/utils.py index 23684a2c55ce..3541ef89bfc1 100644 --- a/tests/v1/engine/utils.py +++ b/tests/v1/engine/utils.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import TypeAlias +import numpy as np import torch from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast @@ -369,9 +370,9 @@ def get_outputs(self) -> list[EngineCoreOutput]: self.generated_logprobs_raw[req_idx][token_idx] ) logprobs = LogprobsLists( - [logprobs_token_ids_], - [logprobs_], - [sampled_token_ranks_], + np.array([logprobs_token_ids_]), + np.array([logprobs_]), + np.array([sampled_token_ranks_]), ) else: logprobs = None From cb327f8c844a953b049b81195a361af2681dde94 Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Mon, 10 Nov 2025 17:43:44 -0800 Subject: [PATCH 5/5] to('cpu') -> cpu() Signed-off-by: Jialin Ouyang --- vllm/v1/outputs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index d6e4e252bfac..5f65e4ee0d1f 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -61,9 +61,9 @@ class LogprobsTensors(NamedTuple): def tolists(self, cu_num_generated_tokens: list[int] | None = None): return LogprobsLists( - self.logprob_token_ids.to("cpu").numpy(), - self.logprobs.to("cpu").numpy(), - self.selected_token_ranks.to("cpu").numpy(), + self.logprob_token_ids.cpu().numpy(), + self.logprobs.cpu().numpy(), + self.selected_token_ranks.cpu().numpy(), cu_num_generated_tokens, )