From f7f6bb6994e4fb84ce3709d9828486e94d756d46 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Tue, 1 Jul 2025 10:34:29 -0700 Subject: [PATCH 01/29] =?UTF-8?q?=F0=9F=90=9B=20req=5Fids=20is=20now=20a?= =?UTF-8?q?=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index 48bfe0d58..c1493a5ab 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -516,7 +516,7 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens, # Set up dummy cached_requests for decode steps cached_requests = [ CachedRequestData( - req_id=req.req_id, + req_ids=[req.req_id], resumed_from_preemption=False, new_token_ids=[ valid_token_ids_tensor[torch.randint( From 8f7acaf8a664520aa1e5208e3914cf835b540cc5 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Tue, 1 Jul 2025 10:40:15 -0700 Subject: [PATCH 02/29] =?UTF-8?q?=F0=9F=90=9B=20req=5Fids=20is=20now=20a?= =?UTF-8?q?=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index c1493a5ab..41f25aafd 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -361,7 +361,7 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): # one decode iteration across both sequences cached_requests = [ CachedRequestData( - req_id=req.req_id, + req_ids=[req.req_id], resumed_from_preemption=False, new_token_ids=[ valid_token_ids_tensor[torch.randint( From 06b8d7d85b72205c009cd8f347a702e052257357 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Tue, 1 Jul 2025 12:35:07 -0700 Subject: [PATCH 03/29] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20using=20cached=20req?= =?UTF-8?q?uests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 28 +++++++------ vllm_spyre/v1/worker/spyre_worker.py | 48 ++++++++++++++-------- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 17abe4129..fd593b0bc 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -241,32 +241,35 @@ def update_states(self, scheduler_output: SchedulerOutput): # # NOTE: req_state.output_token_ids is being mutated. - for req_data in scheduler_output.scheduled_cached_reqs: - req_id = req_data.req_id + req_data = scheduler_output.scheduled_cached_reqs + for i, req_id in enumerate(req_data.req_ids): req_state = self.requests[req_id] + # for req_data in scheduler_output.scheduled_cached_reqs: + # req_id = req_data.req_ids[0] + # req_state = self.requests[req_id] + # Update the cached states. - num_computed_tokens = req_data.num_computed_tokens + num_computed_tokens = req_data.num_computed_tokens[i] + new_token_ids = req_data.new_token_ids[i] # Add the sampled token(s) from the previous step (if any). # This doesn't include "unverified" tokens like spec decode tokens. - num_new_tokens = (num_computed_tokens + - len(req_data.new_token_ids) - + num_new_tokens = (num_computed_tokens + len(new_token_ids) - req_state.num_tokens) if num_new_tokens == 1: # Avoid slicing list in most common case. - req_state.output_token_ids.append(req_data.new_token_ids[-1]) + req_state.output_token_ids.append(new_token_ids[-1]) elif num_new_tokens > 0: req_state.output_token_ids.extend( - req_data.new_token_ids[-num_new_tokens:]) + new_token_ids[-num_new_tokens:]) req_index = self.input_batch.get_req_index(req_id) # Add new_token_ids to token_ids_cpu. # TODO: Update for spec decoding in the future start_token_index = num_computed_tokens - end_token_index = num_computed_tokens + len(req_data.new_token_ids) + end_token_index = num_computed_tokens + len(new_token_ids) self.input_batch.token_ids_cpu[ - req_index, - start_token_index:end_token_index] = req_data.new_token_ids + req_index, start_token_index:end_token_index] = new_token_ids if scheduler_output.finished_req_ids: for req_id in scheduler_output.finished_req_ids: @@ -277,8 +280,7 @@ def update_states(self, scheduler_output: SchedulerOutput): def _prepare_prompt(self, _: list[NewRequestData]) -> ModelForwardInputs: raise NotImplementedError - def _prepare_decode(self, - _: list[CachedRequestData]) -> ModelForwardInputs: + def _prepare_decode(self, _: CachedRequestData) -> ModelForwardInputs: raise NotImplementedError def prepare_model_input( @@ -291,7 +293,7 @@ def prepare_model_input( # Prepare input tensors. if is_prompt: # Assert no running requests - assert len(scheduler_output.scheduled_cached_reqs) == 0 + assert len(scheduler_output.scheduled_cached_reqs.req_ids) == 0 return self._prepare_prompt(scheduler_output.scheduled_new_reqs) else: diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index 41f25aafd..2ef0698b1 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -344,7 +344,7 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): for i, req in enumerate(dummy_requests): scheduler_output = SchedulerOutput( scheduled_new_reqs=[req], - scheduled_cached_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), num_scheduled_tokens={req.req_id: prompt_len}, total_num_scheduled_tokens=prompt_len, scheduled_spec_decode_tokens={}, @@ -359,22 +359,38 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): self.execute_model(scheduler_output) # one decode iteration across both sequences - cached_requests = [ - CachedRequestData( - req_ids=[req.req_id], - resumed_from_preemption=False, - new_token_ids=[ - valid_token_ids_tensor[torch.randint( - 0, len(valid_token_ids_tensor), (1, )).item()] - ], # placeholder token - new_block_ids=req.block_ids, - num_computed_tokens=prompt_len, - ) for req in dummy_requests - ] + # cached_requests = [ + # CachedRequestData( + # req_ids=[req.req_id], + # resumed_from_preemption=False, + # new_token_ids=[ + # valid_token_ids_tensor[torch.randint( + # 0, len(valid_token_ids_tensor), (1, )).item()] + # ], # placeholder token + # new_block_ids=req.block_ids, + # num_computed_tokens=prompt_len, + # ) for req in dummy_requests + # ] + req_ids = [] + new_token_ids = [] + new_block_ids = [] + for req in dummy_requests: + req_ids.append(req.req_id) + new_token_ids.append(valid_token_ids_tensor[torch.randint( + 0, len(valid_token_ids_tensor), + (1, )).item()]), # placeholder token + new_block_ids.append(req.block_ids), + cached_request_data = CachedRequestData( + req_ids=req_ids, + resumed_from_preemption=False, + new_token_ids=new_token_ids, + new_block_ids=new_block_ids, + num_computed_tokens=[prompt_len], + ) scheduler_output = SchedulerOutput( scheduled_new_reqs=[], - scheduled_cached_reqs=cached_requests, + scheduled_cached_reqs=cached_request_data, num_scheduled_tokens={f"warmup-{i}": 1 for i in range(batch_size)}, total_num_scheduled_tokens=batch_size, @@ -393,7 +409,7 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): # Needed to clean up the data of model runner scheduler_output = SchedulerOutput( scheduled_new_reqs=[], - scheduled_cached_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), num_scheduled_tokens={}, # NOTE: this means no work to do total_num_scheduled_tokens=0, @@ -530,7 +546,7 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens, # Set up scheduler_output for execute_model scheduler_output = SchedulerOutput( scheduled_new_reqs=dummy_requests, - scheduled_cached_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), num_scheduled_tokens={i: prompt_len for i in range(batch_size)}, total_num_scheduled_tokens=sum(prompt_len From a064d3b9284664d2076135bcd16ca0f5bb9558c1 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Wed, 2 Jul 2025 11:55:03 -0700 Subject: [PATCH 04/29] =?UTF-8?q?=F0=9F=90=9B=20first=20pass=20for=20sb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 24 ++++---- vllm_spyre/v1/worker/spyre_worker.py | 68 ++++++++++++++-------- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index fd593b0bc..063d4133d 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -457,19 +457,22 @@ def _prepare_prompt( def _prepare_decode( self, - cached_requests: list[CachedRequestData], + cached_requests: CachedRequestData, ) -> ModelForwardInputs: - assert len(cached_requests) > 0 + assert len(cached_requests.req_ids) > 0 input_tokens: list[list[int]] = [ [0] for _ in range(self._position_ids.shape[0]) ] - for cached_request in cached_requests: + for i, req_id in enumerate(cached_requests.req_ids): + # for cached_request in cached_requests: # TODO: Will this always just be one token ID if there's no spec # or jump decoding? - generation_token = cached_request.new_token_ids[-1] - input_tokens[self.input_batch.req_id_to_index[ - cached_request.req_id]] = [generation_token] + new_token_ids = cached_requests.new_token_ids[i] + generation_token = new_token_ids[-1] + input_tokens[self.input_batch.req_id_to_index[req_id]] = [ + generation_token + ] # update position ids and attention mask self._update_position_ids() @@ -754,20 +757,21 @@ def _prepare_prompt( def _prepare_decode( self, - cached_requests: list[CachedRequestData], + cached_requests: CachedRequestData, ) -> ModelForwardInputs: - assert len(cached_requests) > 0 + assert len(cached_requests.req_ids) > 0 input_tokens = [] input_positions = [] block_table = [] slot_mapping = [] left_padded_prompt_mask = [] - self.model.indices = torch.ones(len(cached_requests), + self.model.indices = torch.ones(len(cached_requests.req_ids), dtype=torch.bool, device="cpu") - assert len(self.input_batch.req_id_to_index) == len(cached_requests) + assert len(self.input_batch.req_id_to_index) == len( + cached_requests.req_ids) # TODO(wallas): I think we can do better here, without sorting or # creating an intermediary dictionary cached_reqs_map = {c.req_id: c for c in cached_requests} diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index 2ef0698b1..e0d210a37 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -372,14 +372,15 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): # ) for req in dummy_requests # ] req_ids = [] - new_token_ids = [] - new_block_ids = [] + new_token_ids: list[list[int]] = [] + new_block_ids: list[tuple[list[int], ...]] = [] for req in dummy_requests: req_ids.append(req.req_id) - new_token_ids.append(valid_token_ids_tensor[torch.randint( - 0, len(valid_token_ids_tensor), - (1, )).item()]), # placeholder token - new_block_ids.append(req.block_ids), + new_token_ids.append([ + valid_token_ids_tensor[torch.randint( + 0, len(valid_token_ids_tensor), (1, )).item()] + ]), # placeholder token + new_block_ids.append([req.block_ids]), cached_request_data = CachedRequestData( req_ids=req_ids, resumed_from_preemption=False, @@ -530,23 +531,43 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens, ] # Set up dummy cached_requests for decode steps - cached_requests = [ - CachedRequestData( - req_ids=[req.req_id], - resumed_from_preemption=False, - new_token_ids=[ - valid_token_ids_tensor[torch.randint( - 0, len(valid_token_ids_tensor), (1, )).item()] - ], # placeholder token - new_block_ids=req.block_ids, - num_computed_tokens=req.num_computed_tokens, - ) for req in dummy_requests - ] + # cached_requests = [ + # CachedRequestData( + # req_ids=[req.req_id], + # resumed_from_preemption=False, + # new_token_ids=[ + # valid_token_ids_tensor[torch.randint( + # 0, len(valid_token_ids_tensor), (1, )).item()] + # ], # placeholder token + # new_block_ids=req.block_ids, + # num_computed_tokens=req.num_computed_tokens, + # ) for req in dummy_requests + # ] + req_ids = [] + new_token_ids: list[list[int]] = [] + new_block_ids: list[tuple[list[int], ...]] = [] + num_computed_tokens = [] + for req in dummy_requests: + req_ids.append(req.req_id) + new_token_ids.append([ + valid_token_ids_tensor[torch.randint( + 0, len(valid_token_ids_tensor), (1, )).item()] + ]), # placeholder token + new_block_ids.append([req.block_ids]), + num_computed_tokens.append(req.num_computed_tokens) + + cached_request_data = CachedRequestData( + req_ids=req_ids, + resumed_from_preemption=False, + new_token_ids=new_token_ids, + new_block_ids=new_block_ids, + num_computed_tokens=num_computed_tokens, + ) # Set up scheduler_output for execute_model scheduler_output = SchedulerOutput( scheduled_new_reqs=dummy_requests, - scheduled_cached_reqs=CachedRequestData.make_empty(), + scheduled_cached_reqs=cached_request_data, num_scheduled_tokens={i: prompt_len for i in range(batch_size)}, total_num_scheduled_tokens=sum(prompt_len @@ -565,7 +586,8 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens, # The fixed size warmup needs to happen only in here with _maybe_warmup_context(): self._warmup_model_forward_pass(scheduler_output, dummy_requests, - cached_requests, num_decode_tokens) + cached_request_data, + num_decode_tokens) self.perf_metrics.log("warmup 1 time", time.time() - warmup_start_t, batch_size=batch_size, @@ -576,7 +598,7 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens, logger.info("Warmup forward pass 2/2...") warmup2_start_t = time.time() self._warmup_model_forward_pass(scheduler_output, dummy_requests, - cached_requests, num_decode_tokens) + cached_request_data, num_decode_tokens) warmup_end_t = time.time() warmup_total_t = warmup_end_t - warmup_start_t @@ -595,12 +617,12 @@ def _warmup_model_forward_pass( self, scheduler_output: SchedulerOutput, requests: list[NewRequestData], - cached_requests: list[CachedRequestData], + cached_requests: CachedRequestData, num_decode_tokens, ): """Handle a complete forward pass""" scheduler_output.scheduled_new_reqs = requests - scheduler_output.scheduled_cached_reqs = [] + scheduler_output.scheduled_cached_reqs = CachedRequestData.make_empty() self.execute_model(scheduler_output) # Prefill # Switch to cached requests to trigger decoding steps From 989f6d20deca46053ccbbc15389da2502c10c6c7 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Wed, 2 Jul 2025 14:16:11 -0700 Subject: [PATCH 05/29] =?UTF-8?q?=F0=9F=90=9B=20first=20pass=20for=20cb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 40 ++++++++++++++-------- vllm_spyre/v1/worker/spyre_worker.py | 4 ++- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 063d4133d..53e8922cc 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -757,49 +757,59 @@ def _prepare_prompt( def _prepare_decode( self, - cached_requests: CachedRequestData, + cached_request_data: CachedRequestData, ) -> ModelForwardInputs: - assert len(cached_requests.req_ids) > 0 + assert len(cached_request_data.req_ids) > 0 input_tokens = [] input_positions = [] block_table = [] slot_mapping = [] left_padded_prompt_mask = [] - self.model.indices = torch.ones(len(cached_requests.req_ids), + self.model.indices = torch.ones(len(cached_request_data.req_ids), dtype=torch.bool, device="cpu") assert len(self.input_batch.req_id_to_index) == len( - cached_requests.req_ids) + cached_request_data.req_ids) # TODO(wallas): I think we can do better here, without sorting or # creating an intermediary dictionary - cached_reqs_map = {c.req_id: c for c in cached_requests} + # for req in cached_request_data: + # cached_reqs_map = {c.req_id: c for c in cached_requests} + + cached_reqs_map = { + req_id: i + for i, req_id in enumerate(cached_request_data.req_ids) + } + req_ids = self.input_batch.sorted_requests_ids + # for _, req_id in enumerate(cached_request_data.req_ids): for req_id in req_ids: + # TODO: Will this always just be one token ID if there's no spec # or jump decoding? - cached_request = cached_reqs_map[req_id] + # cached_request = cached_reqs_map[req_id] # adding new blocks if needed if self.tkv // self.block_size + 1 > len( - self.req_ids2blocks[cached_request.req_id]): - self.req_ids2blocks[cached_request.req_id].append( - self.free_blocks.popleft()) - block_table.append(self.req_ids2blocks[cached_request.req_id]) + self.req_ids2blocks[req_id]): + self.req_ids2blocks[req_id].append(self.free_blocks.popleft()) + block_table.append(self.req_ids2blocks[req_id]) # slot_mapping for all blocks of sequence start_slot = block_table[-1][-1] * self.block_size offset = self.tkv % self.block_size slot = [start_slot + offset] slot_mapping.append(slot) - - generation_token = cached_request.new_token_ids[-1] + new_token_ids = cached_request_data.new_token_ids[ + cached_reqs_map[req_id]] + generation_token = new_token_ids[-1] input_tokens.append([generation_token]) - seq_len = cached_request.num_computed_tokens + seq_len = cached_request_data.num_computed_tokens[ + cached_reqs_map[req_id]] input_positions.append([seq_len]) - req_state = self.requests[cached_request.req_id] + req_state = self.requests[req_id] left_padded_prompt_mask.append(req_state.left_padding) input_tokens = torch.tensor(input_tokens, @@ -825,7 +835,7 @@ def _prepare_decode( dtype=torch.int64) # add pads for min decode batch size of 2 (Spyre compiler constraint) - if len(cached_requests) == 1: + if len(cached_request_data.req_ids) == 1: padd_seq_indices = torch.zeros(1, dtype=torch.bool, device="cpu") self.model.indices = torch.cat( (self.model.indices, padd_seq_indices), -1) diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index e0d210a37..daf8d06e6 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -374,6 +374,7 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): req_ids = [] new_token_ids: list[list[int]] = [] new_block_ids: list[tuple[list[int], ...]] = [] + num_computed_tokens = [] for req in dummy_requests: req_ids.append(req.req_id) new_token_ids.append([ @@ -381,12 +382,13 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): 0, len(valid_token_ids_tensor), (1, )).item()] ]), # placeholder token new_block_ids.append([req.block_ids]), + num_computed_tokens.append(prompt_len), cached_request_data = CachedRequestData( req_ids=req_ids, resumed_from_preemption=False, new_token_ids=new_token_ids, new_block_ids=new_block_ids, - num_computed_tokens=[prompt_len], + num_computed_tokens=num_computed_tokens, ) scheduler_output = SchedulerOutput( From bd7e008e94140ebaf6ebe0c0bcfaf710425c22af Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Wed, 2 Jul 2025 14:53:46 -0700 Subject: [PATCH 06/29] =?UTF-8?q?=F0=9F=90=9B=20fix=20merge=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_worker.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index 1cb6e37c8..136135a92 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -395,10 +395,8 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): scheduler_output = SchedulerOutput( scheduled_new_reqs=[], scheduled_cached_reqs=cached_request_data, - num_scheduled_tokens={ - f"warmup-{i}": 1 - for i in range(batch_size) - }, + num_scheduled_tokens={f"warmup-{i}": 1 + for i in range(batch_size)}, total_num_scheduled_tokens=batch_size, scheduled_spec_decode_tokens={}, scheduled_encoder_inputs={}, @@ -449,7 +447,7 @@ def _cleanup_model_runner(self, request) -> None: # Needed to clean up the data of model runner scheduler_output = SchedulerOutput( scheduled_new_reqs=[], - scheduled_cached_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), num_scheduled_tokens={}, # NOTE: this means no work to do total_num_scheduled_tokens=0, From fe8e64c8a679a3f0fa79130ecf805347fec142f2 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 09:30:06 -0700 Subject: [PATCH 07/29] =?UTF-8?q?=F0=9F=8E=A8=20renaming=20vars?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 10 +++++----- vllm_spyre/v1/worker/spyre_worker.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 53e8922cc..1776f9068 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -457,18 +457,18 @@ def _prepare_prompt( def _prepare_decode( self, - cached_requests: CachedRequestData, + cached_request_data: CachedRequestData, ) -> ModelForwardInputs: - assert len(cached_requests.req_ids) > 0 + assert len(cached_request_data.req_ids) > 0 input_tokens: list[list[int]] = [ [0] for _ in range(self._position_ids.shape[0]) ] - for i, req_id in enumerate(cached_requests.req_ids): - # for cached_request in cached_requests: + for i, req_id in enumerate(cached_request_data.req_ids): + # for cached_request in cached_request_data: # TODO: Will this always just be one token ID if there's no spec # or jump decoding? - new_token_ids = cached_requests.new_token_ids[i] + new_token_ids = cached_request_data.new_token_ids[i] generation_token = new_token_ids[-1] input_tokens[self.input_batch.req_id_to_index[req_id]] = [ generation_token diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index 136135a92..94371e130 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -642,7 +642,7 @@ def _warmup_model_forward_pass( self, scheduler_output: SchedulerOutput, requests: list[NewRequestData], - cached_requests: CachedRequestData, + cached_request_data: CachedRequestData, num_decode_tokens, ): """Handle a complete forward pass""" @@ -652,7 +652,7 @@ def _warmup_model_forward_pass( # Switch to cached requests to trigger decoding steps scheduler_output.scheduled_new_reqs = [] - scheduler_output.scheduled_cached_reqs = cached_requests + scheduler_output.scheduled_cached_reqs = cached_request_data for _ in range(num_decode_tokens - 1): self.execute_model(scheduler_output) From df9214b3b257d5872cd5609cdad5d20ccd191b51 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 09:36:39 -0700 Subject: [PATCH 08/29] =?UTF-8?q?=F0=9F=94=A5=20remove=20commented=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 12 -------- vllm_spyre/v1/worker/spyre_worker.py | 32 +++------------------- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 1776f9068..d3847f00f 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -245,10 +245,6 @@ def update_states(self, scheduler_output: SchedulerOutput): for i, req_id in enumerate(req_data.req_ids): req_state = self.requests[req_id] - # for req_data in scheduler_output.scheduled_cached_reqs: - # req_id = req_data.req_ids[0] - # req_state = self.requests[req_id] - # Update the cached states. num_computed_tokens = req_data.num_computed_tokens[i] new_token_ids = req_data.new_token_ids[i] @@ -465,7 +461,6 @@ def _prepare_decode( ] for i, req_id in enumerate(cached_request_data.req_ids): - # for cached_request in cached_request_data: # TODO: Will this always just be one token ID if there's no spec # or jump decoding? new_token_ids = cached_request_data.new_token_ids[i] @@ -774,22 +769,15 @@ def _prepare_decode( cached_request_data.req_ids) # TODO(wallas): I think we can do better here, without sorting or # creating an intermediary dictionary - # for req in cached_request_data: - # cached_reqs_map = {c.req_id: c for c in cached_requests} - cached_reqs_map = { req_id: i for i, req_id in enumerate(cached_request_data.req_ids) } - req_ids = self.input_batch.sorted_requests_ids - # for _, req_id in enumerate(cached_request_data.req_ids): for req_id in req_ids: - # TODO: Will this always just be one token ID if there's no spec # or jump decoding? - # cached_request = cached_reqs_map[req_id] # adding new blocks if needed if self.tkv // self.block_size + 1 > len( diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index 94371e130..846dfb315 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -360,21 +360,9 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): self.execute_model(scheduler_output) # one decode iteration across both sequences - # cached_requests = [ - # CachedRequestData( - # req_ids=[req.req_id], - # resumed_from_preemption=False, - # new_token_ids=[ - # valid_token_ids_tensor[torch.randint( - # 0, len(valid_token_ids_tensor), (1, )).item()] - # ], # placeholder token - # new_block_ids=req.block_ids, - # num_computed_tokens=prompt_len, - # ) for req in dummy_requests - # ] req_ids = [] - new_token_ids: list[list[int]] = [] - new_block_ids: list[tuple[list[int], ...]] = [] + new_token_ids = [] + new_block_ids = [] num_computed_tokens = [] for req in dummy_requests: req_ids.append(req.req_id) @@ -556,21 +544,9 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens, ] # Set up dummy cached_requests for decode steps - # cached_requests = [ - # CachedRequestData( - # req_ids=[req.req_id], - # resumed_from_preemption=False, - # new_token_ids=[ - # valid_token_ids_tensor[torch.randint( - # 0, len(valid_token_ids_tensor), (1, )).item()] - # ], # placeholder token - # new_block_ids=req.block_ids, - # num_computed_tokens=req.num_computed_tokens, - # ) for req in dummy_requests - # ] req_ids = [] - new_token_ids: list[list[int]] = [] - new_block_ids: list[tuple[list[int], ...]] = [] + new_token_ids = [] + new_block_ids = [] num_computed_tokens = [] for req in dummy_requests: req_ids.append(req.req_id) From c23f4c5e914b09e055fd5bad827701186cb44e8d Mon Sep 17 00:00:00 2001 From: Max de Bayser Date: Thu, 3 Jul 2025 14:08:48 -0300 Subject: [PATCH 09/29] Duplicate the SamplingMetadata class We were previously reusing the GPU SamplingMetadata class but there have been incompatible changes upstream (PR https://github.com/vllm-project/vllm/pull/16728) Since it's not clear for now whether we want, should or can reuse the LogitsProcessor implementation as is, I'm making a copy of the old version of the class for the spyre backend. This won't affect any features for now since the vllm change was an internal refactoring without UX impact. Signed-off-by: Max de Bayser --- vllm_spyre/v1/sample/metadata.py | 51 +++++++++++++++++++++++ vllm_spyre/v1/worker/spyre_input_batch.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 vllm_spyre/v1/sample/metadata.py diff --git a/vllm_spyre/v1/sample/metadata.py b/vllm_spyre/v1/sample/metadata.py new file mode 100644 index 000000000..47807cad6 --- /dev/null +++ b/vllm_spyre/v1/sample/metadata.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass +from typing import Optional + +import torch + +# This is a copy of the vLLM Sampling Metadata prior to PR +# https://github.com/vllm-project/vllm/pull/16728 +# TODO: Figure out if we want to apply the LogitsProcessor +# approach here and whether we want to reuse the code. That +# would require some refactoring since processors like the +# MinPLogitsProcessor have a device tensor and a CPU tensor, +# which we don't need + +@dataclass +class SamplingMetadata: + + temperature: Optional[torch.Tensor] + all_greedy: bool + all_random: bool + + top_p: Optional[torch.Tensor] + top_k: Optional[torch.Tensor] + min_p: Optional[torch.Tensor] + + generators: dict[int, torch.Generator] + + # None means no logprobs, 0 means sampled token logprobs only + max_num_logprobs: Optional[int] + + no_penalties: bool + prompt_token_ids: Optional[torch.Tensor] + frequency_penalties: torch.Tensor + presence_penalties: torch.Tensor + repetition_penalties: torch.Tensor + + output_token_ids: list[list[int]] + + # req_index -> (min_tokens, stop_token_ids) + min_tokens: dict[int, tuple[int, set[int]]] + + logit_bias: list[Optional[dict[int, float]]] + + # `allowed_token_ids_mask` is a 2D bool tensor of shape (max batch size, + # vocab size). + allowed_token_ids_mask: Optional[torch.Tensor] + + # req_index -> bad_words_token_ids + bad_words_token_ids: dict[int, list[list[int]]] diff --git a/vllm_spyre/v1/worker/spyre_input_batch.py b/vllm_spyre/v1/worker/spyre_input_batch.py index 9778fa3f5..b4b0ae565 100644 --- a/vllm_spyre/v1/worker/spyre_input_batch.py +++ b/vllm_spyre/v1/worker/spyre_input_batch.py @@ -9,7 +9,7 @@ import numpy as np import torch from vllm.sampling_params import SamplingParams, SamplingType -from vllm.v1.sample.metadata import SamplingMetadata +from vllm_spyre.v1.sample.metadata import SamplingMetadata _SAMPLING_EPS = 1e-5 From f06578535519b759e1d580e6698efc9112b6d14b Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 10:16:43 -0700 Subject: [PATCH 10/29] =?UTF-8?q?=F0=9F=94=A5=20remove=20extra=20commas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_worker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_worker.py b/vllm_spyre/v1/worker/spyre_worker.py index 846dfb315..a84596abf 100644 --- a/vllm_spyre/v1/worker/spyre_worker.py +++ b/vllm_spyre/v1/worker/spyre_worker.py @@ -369,9 +369,9 @@ def _warmup_spyre_dynamic_size(self, special_token_ids): new_token_ids.append([ valid_token_ids_tensor[torch.randint( 0, len(valid_token_ids_tensor), (1, )).item()] - ]), # placeholder token - new_block_ids.append([req.block_ids]), - num_computed_tokens.append(prompt_len), + ]) # placeholder token + new_block_ids.append([req.block_ids]) + num_computed_tokens.append(prompt_len) cached_request_data = CachedRequestData( req_ids=req_ids, resumed_from_preemption=False, @@ -553,8 +553,8 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens, new_token_ids.append([ valid_token_ids_tensor[torch.randint( 0, len(valid_token_ids_tensor), (1, )).item()] - ]), # placeholder token - new_block_ids.append([req.block_ids]), + ]) # placeholder token + new_block_ids.append([req.block_ids]) num_computed_tokens.append(req.num_computed_tokens) cached_request_data = CachedRequestData( From 425c3d2e646c6784845b9ab1bdbd9d2b4a1586c7 Mon Sep 17 00:00:00 2001 From: Max de Bayser Date: Thu, 3 Jul 2025 14:24:12 -0300 Subject: [PATCH 11/29] fix linting Signed-off-by: Max de Bayser --- vllm_spyre/v1/sample/metadata.py | 1 + vllm_spyre/v1/worker/spyre_input_batch.py | 1 + 2 files changed, 2 insertions(+) diff --git a/vllm_spyre/v1/sample/metadata.py b/vllm_spyre/v1/sample/metadata.py index 47807cad6..0957391a7 100644 --- a/vllm_spyre/v1/sample/metadata.py +++ b/vllm_spyre/v1/sample/metadata.py @@ -14,6 +14,7 @@ # MinPLogitsProcessor have a device tensor and a CPU tensor, # which we don't need + @dataclass class SamplingMetadata: diff --git a/vllm_spyre/v1/worker/spyre_input_batch.py b/vllm_spyre/v1/worker/spyre_input_batch.py index b4b0ae565..f6601e624 100644 --- a/vllm_spyre/v1/worker/spyre_input_batch.py +++ b/vllm_spyre/v1/worker/spyre_input_batch.py @@ -9,6 +9,7 @@ import numpy as np import torch from vllm.sampling_params import SamplingParams, SamplingType + from vllm_spyre.v1.sample.metadata import SamplingMetadata _SAMPLING_EPS = 1e-5 From 05ea42339407ef68ec141526a99eaa9783767326 Mon Sep 17 00:00:00 2001 From: Max de Bayser Date: Thu, 3 Jul 2025 14:38:26 -0300 Subject: [PATCH 12/29] Actually more classes need to be duplicated Signed-off-by: Max de Bayser --- vllm_spyre/v1/sample/__init__.py | 0 vllm_spyre/v1/sample/metadata.py | 5 +- vllm_spyre/v1/sample/ops/__init__.py | 0 vllm_spyre/v1/sample/ops/bad_words.py | 40 +++ vllm_spyre/v1/sample/ops/penalties.py | 59 ++++ vllm_spyre/v1/sample/ops/topk_topp_sampler.py | 293 ++++++++++++++++++ vllm_spyre/v1/sample/sampler.py | 288 +++++++++++++++++ 7 files changed, 683 insertions(+), 2 deletions(-) create mode 100644 vllm_spyre/v1/sample/__init__.py create mode 100644 vllm_spyre/v1/sample/ops/__init__.py create mode 100644 vllm_spyre/v1/sample/ops/bad_words.py create mode 100644 vllm_spyre/v1/sample/ops/penalties.py create mode 100644 vllm_spyre/v1/sample/ops/topk_topp_sampler.py create mode 100644 vllm_spyre/v1/sample/sampler.py diff --git a/vllm_spyre/v1/sample/__init__.py b/vllm_spyre/v1/sample/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_spyre/v1/sample/metadata.py b/vllm_spyre/v1/sample/metadata.py index 0957391a7..5f1636779 100644 --- a/vllm_spyre/v1/sample/metadata.py +++ b/vllm_spyre/v1/sample/metadata.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project - +# This is a copy of the vLLM vllm file prior to PR +# https://github.com/vllm-project/vllm/pull/16728 from dataclasses import dataclass from typing import Optional import torch -# This is a copy of the vLLM Sampling Metadata prior to PR +# This is a copy of the vLLM vllm file prior to PR # https://github.com/vllm-project/vllm/pull/16728 # TODO: Figure out if we want to apply the LogitsProcessor # approach here and whether we want to reuse the code. That diff --git a/vllm_spyre/v1/sample/ops/__init__.py b/vllm_spyre/v1/sample/ops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_spyre/v1/sample/ops/bad_words.py b/vllm_spyre/v1/sample/ops/bad_words.py new file mode 100644 index 000000000..b235d4384 --- /dev/null +++ b/vllm_spyre/v1/sample/ops/bad_words.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# This is a copy of the vLLM vllm file prior to PR +# https://github.com/vllm-project/vllm/pull/16728 +import torch + +_SMALLEST_LOGIT = float("-inf") + + +def _apply_bad_words_single_batch( + logits: torch.Tensor, + bad_words_token_ids: list[list[int]], + past_tokens_ids: list[int], +) -> None: + for bad_word_ids in bad_words_token_ids: + if len(bad_word_ids) > len(past_tokens_ids) + 1: + continue + + prefix_length = len(bad_word_ids) - 1 + last_token_id = bad_word_ids[-1] + if prefix_length > 0: + actual_prefix = past_tokens_ids[-prefix_length:] + else: + actual_prefix = [] + expected_prefix = bad_word_ids[:prefix_length] + + assert len(actual_prefix) == len(expected_prefix) + + if actual_prefix == expected_prefix: + logits[last_token_id] = _SMALLEST_LOGIT + + +def apply_bad_words( + logits: torch.Tensor, + bad_words_token_ids: dict[int, list[list[int]]], + past_tokens_ids: list[list[int]], +) -> None: + for i, bad_words_ids in bad_words_token_ids.items(): + _apply_bad_words_single_batch(logits[i], bad_words_ids, + past_tokens_ids[i]) diff --git a/vllm_spyre/v1/sample/ops/penalties.py b/vllm_spyre/v1/sample/ops/penalties.py new file mode 100644 index 000000000..6bf488796 --- /dev/null +++ b/vllm_spyre/v1/sample/ops/penalties.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# This is a copy of the vLLM vllm file prior to PR +# https://github.com/vllm-project/vllm/pull/16728 +import torch +from vllm.model_executor.layers.utils import apply_penalties +from vllm.utils import is_pin_memory_available, make_tensor_with_pad + + +def apply_min_token_penalties( + logits: torch.Tensor, output_token_ids: list[list[int]], + min_tokens: dict[int, tuple[int, set[int]]]) -> None: + """ + Applies minimum token penalty by setting the logits of the stop tokens + to -inf. + """ + min_tokens_logits_to_penalize: list[tuple[int, int]] = [] + for index, (min_token, stop_token_ids) in min_tokens.items(): + if len(output_token_ids[index]) < min_token: + for stop_token_id in stop_token_ids: + min_tokens_logits_to_penalize.append((index, stop_token_id)) + if min_tokens_logits_to_penalize: + logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf") + + +def apply_all_penalties( + logits: torch.Tensor, + prompt_token_ids: torch.Tensor, + presence_penalties: torch.Tensor, + frequency_penalties: torch.Tensor, + repetition_penalties: torch.Tensor, + output_token_ids: list[list[int]], +) -> torch.Tensor: + """ + Applies presence, frequency and repetition penalties to the logits. + """ + _, vocab_size = logits.shape + output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size, + logits.device) + return apply_penalties(logits, prompt_token_ids, output_tokens_t, + presence_penalties, frequency_penalties, + repetition_penalties) + + +def _convert_to_tensors(output_token_ids: list[list[int]], vocab_size: int, + device: torch.device) -> torch.Tensor: + """ + Convert the different list data structures to tensors. + """ + output_tokens_tensor = make_tensor_with_pad( + output_token_ids, + # Use the value of vocab_size as a pad since we don't have a + # token_id of this value. + pad=vocab_size, + device="cpu", + dtype=torch.int64, + pin_memory=is_pin_memory_available(), + ) + return output_tokens_tensor.to(device, non_blocking=True) diff --git a/vllm_spyre/v1/sample/ops/topk_topp_sampler.py b/vllm_spyre/v1/sample/ops/topk_topp_sampler.py new file mode 100644 index 000000000..c0cfbd436 --- /dev/null +++ b/vllm_spyre/v1/sample/ops/topk_topp_sampler.py @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# This is a copy of the vLLM vllm file prior to PR +# https://github.com/vllm-project/vllm/pull/16728 +from typing import Optional + +import torch +import torch.nn as nn +from vllm import envs +from vllm.logger import init_logger +from vllm.platforms import current_platform + +logger = init_logger(__name__) + +try: + import flashinfer.sampling + is_flashinfer_available = True +except ImportError: + is_flashinfer_available = False + + +class TopKTopPSampler(nn.Module): + """ + Module that performs optional top-k and top-p filtering followed by + weighted random sampling of logits. + + Implementations may update the logits tensor in-place. + """ + + def __init__(self): + super().__init__() + if current_platform.is_cuda(): + if is_flashinfer_available: + flashinfer_version = flashinfer.__version__ + if flashinfer_version < "0.2.3": + logger.warning( + "FlashInfer version >= 0.2.3 required. " + "Falling back to default sampling implementation.") + self.forward = self.forward_native + elif envs.VLLM_USE_FLASHINFER_SAMPLER is not False: + # NOTE(woosuk): The V0 sampler doesn't use FlashInfer for + # sampling unless VLLM_USE_FLASHINFER_SAMPLER=1 (i.e., by + # default it is unused). For backward compatibility, we set + # `VLLM_USE_FLASHINFER_SAMPLER` as None by default and + # interpret it differently in V0 and V1 samplers: In V0, + # None means False, while in V1, None means True. This is + # why we use the condition + # `envs.VLLM_USE_FLASHINFER_SAMPLER is not False` here. + logger.info("Using FlashInfer for top-p & top-k sampling.") + self.forward = self.forward_cuda + else: + logger.warning( + "FlashInfer is available, but it is not enabled. " + "Falling back to the PyTorch-native implementation of " + "top-p & top-k sampling. For the best performance, " + "please set VLLM_USE_FLASHINFER_SAMPLER=1.") + self.forward = self.forward_native + else: + logger.warning( + "FlashInfer is not available. Falling back to the PyTorch-" + "native implementation of top-p & top-k sampling. For the " + "best performance, please install FlashInfer.") + self.forward = self.forward_native + elif current_platform.is_tpu(): + self.forward = self.forward_tpu + else: + self.forward = self.forward_native + + def forward_native( + self, + logits: torch.Tensor, + generators: dict[int, torch.Generator], + k: Optional[torch.Tensor], + p: Optional[torch.Tensor], + ) -> torch.Tensor: + """ + PyTorch-native implementation of top-k and top-p sampling. + + The logits tensor may be updated in-place. + """ + logits = apply_top_k_top_p(logits, k, p) + probs = logits.softmax(dim=-1, dtype=torch.float32) + return random_sample(probs, generators) + + def forward_cuda( + self, + logits: torch.Tensor, + generators: dict[int, torch.Generator], + k: Optional[torch.Tensor], + p: Optional[torch.Tensor], + ) -> torch.Tensor: + """More optimized implementation for top-k and top-p sampling.""" + if k is None and p is None: + # We prefer `random_sample` over `flashinfer_sample` when sorting is + # not needed. This is because `random_sample` does not require + # CPU-GPU synchronization while `flashinfer_sample` does. + probs = logits.softmax(dim=-1, dtype=torch.float32) + return random_sample(probs, generators) + if generators: + logger.warning("FlashInfer 0.2.3+ does not support " + "per-request generators. Falling back to " + "PyTorch-native implementation.") + return self.forward_native(logits, generators, k, p) + return flashinfer_sample(logits, k, p, generators) + + def forward_tpu( + self, + logits: torch.Tensor, + generators: dict[int, torch.Generator], + k: Optional[torch.Tensor], + p: Optional[torch.Tensor], + ) -> torch.Tensor: + logits = apply_top_k_top_p_tpu(logits, k, p) + probs = logits.softmax(dim=-1, dtype=torch.float32) + return random_sample(probs, generators) + + +def apply_top_k_top_p_tpu( + logits: torch.Tensor, + k: torch.Tensor, + p: torch.Tensor, +) -> torch.Tensor: + """ + Apply top-k and top-p optimized for TPU. + + This algorithm avoids using torch.scatter which is extremely slow on TPU. + This is achieved by finding a "cut-off" element in the original logit, and + after thresholding the logit using this cut-off, the remaining elements + shall constitute the top-p set. + + Note: in the case of tie (i.e. multipple cut-off elements present in the + logit), all tie elements are included in the top-p set. In other words, + this function does not break ties. Instead, these tie tokens have equal + chance of being chosen during final sampling, so we can consider the tie + being broken then. + """ + probs = logits.softmax(dim=-1) + probs_sort, _ = probs.sort(dim=-1, descending=False) + + if k is not None: + top_k_count = probs_sort.size(1) - k.to(torch.long) # shape: (batch, ) + top_k_count = top_k_count.unsqueeze(dim=1) + top_k_cutoff = probs_sort.gather(-1, top_k_count) + + # Make sure the no top-k rows are no-op. + no_top_k_mask = (k == logits.shape[1]).unsqueeze(dim=1) + top_k_cutoff.masked_fill_(no_top_k_mask, -float("inf")) + + elements_to_discard = probs < top_k_cutoff + logits.masked_fill_(elements_to_discard, -float("inf")) + + if p is not None: + cumprob = torch.cumsum(probs_sort, dim=-1) + top_p_mask = cumprob <= 1 - p.unsqueeze(dim=1) + top_p_mask[:, -1] = False # at least one + + top_p_count = top_p_mask.sum(dim=-1).unsqueeze(1) + top_p_cutoff = probs_sort.gather(-1, top_p_count) + elements_to_discard = probs < top_p_cutoff + logits.masked_fill_(elements_to_discard, -float("inf")) + + return logits + + +def apply_top_k_top_p( + logits: torch.Tensor, + k: Optional[torch.Tensor], + p: Optional[torch.Tensor], +) -> torch.Tensor: + """Apply top-k and top-p masks to the logits. + + If a top-p is used, this function will sort the logits tensor, + which can be slow for large batches. + + The logits tensor may be updated in-place. + """ + if p is None: + if k is None: + return logits + + # Avoid sorting vocab for top-k only case. + return apply_top_k_only(logits, k) + + logits_sort, logits_idx = logits.sort(dim=-1, descending=False) + + if k is not None: + # Apply top-k. + top_k_mask = logits_sort.size(1) - k.to(torch.long) # shape: B + # Get all the top_k values. + top_k_mask = logits_sort.gather(1, top_k_mask.unsqueeze(dim=1)) + top_k_mask = logits_sort < top_k_mask + logits_sort.masked_fill_(top_k_mask, -float("inf")) + + if p is not None: + # Apply top-p. + probs_sort = logits_sort.softmax(dim=-1) + probs_sum = torch.cumsum(probs_sort, dim=-1, out=probs_sort) + top_p_mask = probs_sum <= 1 - p.unsqueeze(dim=1) + # at least one + top_p_mask[:, -1] = False + logits_sort.masked_fill_(top_p_mask, -float("inf")) + + # Re-sort the probabilities. + logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort) + return logits + + +def apply_top_k_only( + logits: torch.Tensor, + k: torch.Tensor, +) -> torch.Tensor: + """ + Apply top-k mask to the logits. + + This implementation doesn't involve sorting the entire vocab. + + The logits tensor may be updated in-place. + """ + no_top_k_mask = k == logits.shape[1] + # Set non-top-k rows to 1 so that we can gather. + k = k.masked_fill(no_top_k_mask, 1) + max_top_k = k.max() + # topk.values tensor has shape [batch_size, max_top_k]. + # Convert top k to 0-based index in range [0, max_top_k). + k_index = k.sub_(1).unsqueeze(1) + top_k_mask = logits.topk(max_top_k, dim=1).values.gather(1, k_index.long()) + # Handle non-topk rows. + top_k_mask.masked_fill_(no_top_k_mask.unsqueeze(1), -float("inf")) + logits.masked_fill_(logits < top_k_mask, -float("inf")) + return logits + + +def random_sample( + probs: torch.Tensor, + generators: dict[int, torch.Generator], +) -> torch.Tensor: + """Randomly sample from the probabilities. + + We use this function instead of torch.multinomial because torch.multinomial + causes CPU-GPU synchronization. + """ + q = torch.empty_like(probs) + # NOTE(woosuk): To batch-process the requests without their own seeds, + # which is the common case, we first assume that every request does + # not have its own seed. Then, we overwrite the values for the requests + # that have their own seeds. + if len(generators) != probs.shape[0]: + q.exponential_() + if generators: + # TODO(woosuk): This can be slow because we handle each request + # one by one. Optimize this. + for i, generator in generators.items(): + q[i].exponential_(generator=generator) + return probs.div_(q).argmax(dim=-1).view(-1) + + +def flashinfer_sample( + logits: torch.Tensor, + k: Optional[torch.Tensor], + p: Optional[torch.Tensor], + generators: dict[int, torch.Generator], +) -> torch.Tensor: + """Sample from the logits using FlashInfer. + + Statistically, this function is equivalent to the `random_sample` function. + However, this function is faster because it avoids sorting the logits tensor + via rejection sampling. + + NOTE: The outputs of this function do not necessarily match the outputs of + the `random_sample` function. It only guarantees that the outputs are + statistically equivalent. + + NOTE: This function includes CPU-GPU synchronization, while `random_sample` + does not. Call this function at the end of the forward pass to minimize + the synchronization overhead. + """ + assert not (k is None and p is None) + if k is None: + # Top-p only. + probs = logits.softmax(dim=-1, dtype=torch.float32) + next_token_ids = flashinfer.sampling.top_p_sampling_from_probs( + probs, p, deterministic=True) + elif p is None: + # Top-k only. + probs = logits.softmax(dim=-1, dtype=torch.float32) + next_token_ids = flashinfer.sampling.top_k_sampling_from_probs( + probs, k, deterministic=True) + else: + # Both top-k and top-p. + next_token_ids = flashinfer.sampling.top_k_top_p_sampling_from_logits( + logits, k, p, deterministic=True) + + return next_token_ids.view(-1) diff --git a/vllm_spyre/v1/sample/sampler.py b/vllm_spyre/v1/sample/sampler.py new file mode 100644 index 000000000..4f3de9583 --- /dev/null +++ b/vllm_spyre/v1/sample/sampler.py @@ -0,0 +1,288 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# This is a copy of the vLLM vllm file prior to PR +# https://github.com/vllm-project/vllm/pull/16728 +"""A layer that samples the next tokens from the model's outputs.""" + +import torch +import torch.nn as nn +from vllm.utils import async_tensor_h2d, is_pin_memory_available +from vllm.v1.outputs import LogprobsTensors, SamplerOutput + +from vllm_spyre.v1.sample.metadata import SamplingMetadata +from vllm_spyre.v1.sample.ops.bad_words import apply_bad_words +from vllm_spyre.v1.sample.ops.penalties import (apply_all_penalties, + apply_min_token_penalties) +from vllm_spyre.v1.sample.ops.topk_topp_sampler import TopKTopPSampler + +_SAMPLING_EPS = 1e-5 + + +class Sampler(nn.Module): + + def __init__(self): + super().__init__() + self.topk_topp_sampler = TopKTopPSampler() + self.pin_memory = is_pin_memory_available() + + def forward( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> SamplerOutput: + # NOTE(woosuk): Use the original logits (before any penalties or + # temperature scaling) for the top-k logprobs. + # This is different from the V0 sampler, which uses the logits that + # is used for sampling (after penalties and temperature scaling). + # TODO(rob): provide option for logprobs post sampling. + # See https://vllm-dev.slack.com/archives/C07UUL8E61Z/p1735907856007919 # noqa: E501 + num_logprobs = sampling_metadata.max_num_logprobs + if num_logprobs is not None: + raw_logprobs = self.compute_logprobs(logits) + + # Use float32 for the logits. + logits = logits.to(torch.float32) + # Apply allowed token ids. + logits = self.apply_allowed_token_ids(logits, sampling_metadata) + # Apply bad words exclusion. + logits = self.apply_bad_words(logits, sampling_metadata) + # Apply logits bias. + logits = self.apply_logits_bias(logits, sampling_metadata) + # Apply penalties (e.g., min_tokens, freq_penalties). + logits = self.apply_penalties(logits, sampling_metadata) + # Sample the next token. + sampled = self.sample(logits, sampling_metadata) + # Convert sampled token ids to int64 (long) type to ensure compatibility + # with subsequent operations that may use these values as indices. + # This conversion is necessary because FlashInfer sampling operations + # return int32 (while PyTorch argmax and topk return int64). + sampled = sampled.long() + + # Gather the logprobs of the topk and sampled token (if requested). + # Get logprobs and rank tensors (if requested) + logprobs_tensors = None if num_logprobs is None else \ + self.gather_logprobs(raw_logprobs, num_logprobs, token_ids=sampled) + + # Use int32 to reduce the tensor size. + sampled = sampled.to(torch.int32) + + # These are GPU tensors. + sampler_output = SamplerOutput( + # The sampled tokens are expanded to 2D tensor with shape + # [num_requests, 1], where each row represents one generated + # token per request. + sampled_token_ids=sampled.unsqueeze(-1), + logprobs_tensors=logprobs_tensors, + ) + return sampler_output + + def apply_temperature( + self, + logits: torch.Tensor, + temp: torch.Tensor, + ) -> torch.Tensor: + # Use in-place division to avoid creating a new tensor. + return logits.div_(temp.unsqueeze(dim=1)) + + def greedy_sample(self, logits: torch.Tensor) -> torch.Tensor: + return logits.argmax(dim=-1).view(-1) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> torch.Tensor: + """Sample logits based on sampling metadata. + + The various logits processing functions called in this method + may update the logits tensor in-place. + """ + + assert not (sampling_metadata.all_greedy + and sampling_metadata.all_random) + if sampling_metadata.all_random: + greedy_sampled = None + else: + greedy_sampled = self.greedy_sample(logits) + if sampling_metadata.all_greedy: + return greedy_sampled + + assert sampling_metadata.temperature is not None + + # Apply temperature. + logits = self.apply_temperature(logits, sampling_metadata.temperature) + + # Apply min_p. + if sampling_metadata.min_p is not None: + logits = self.apply_min_p(logits, sampling_metadata.min_p) + + # Apply top_k and/or top_p. + random_sampled = self.topk_topp_sampler( + logits, + sampling_metadata.generators, + sampling_metadata.top_k, + sampling_metadata.top_p, + ) + + if greedy_sampled is None: + return random_sampled + + sampled = torch.where( + sampling_metadata.temperature < _SAMPLING_EPS, + greedy_sampled, + random_sampled, + out=greedy_sampled, # Reuse tensor + ) + return sampled + + def compute_logprobs(self, logits: torch.Tensor) -> torch.Tensor: + return logits.log_softmax(dim=-1, dtype=torch.float32) + + def gather_logprobs( + self, + logprobs: torch.Tensor, + num_logprobs: int, + token_ids: torch.Tensor, + ) -> LogprobsTensors: + """ + Gather logprobs for topk and sampled/prompt token. + + Args: + logprobs: (num tokens) x (vocab) tensor + num_logprobs: minimum number of logprobs to + retain per token + token_ids: prompt tokens (if prompt logprobs) + or sampled tokens (if sampled + logprobs); 1D token ID tensor + with (num tokens) elements + Must be int64. + + Returns: + Top-k int indices tensor, (num tokens) x (num_logprobs + 1) + Top-k float logprobs tensor, (num tokens) x (num_logprobs + 1) + Sampled token rank tensor, (num tokens) + """ + assert token_ids.dtype == torch.int64 + # Find the topK values. + topk_logprobs, topk_indices = torch.topk(logprobs, + num_logprobs, + dim=-1) + + # Get with the logprob of the prompt or sampled token. + token_ids = token_ids.unsqueeze(-1) + token_logprobs = logprobs.gather(-1, token_ids) + + # Compute the ranks of the actual token. + token_ranks = (logprobs >= token_logprobs).sum(-1) + + # Concatenate together with the topk. + indices = torch.cat((token_ids, topk_indices), dim=1) + logprobs = torch.cat((token_logprobs, topk_logprobs), dim=1) + + # Use int32 to reduce the tensor size. + indices = indices.to(torch.int32) + + return LogprobsTensors(indices, logprobs, token_ranks) + + def apply_penalties( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> torch.Tensor: + if sampling_metadata.min_tokens: + apply_min_token_penalties(logits, + sampling_metadata.output_token_ids, + sampling_metadata.min_tokens) + if not sampling_metadata.no_penalties: + assert sampling_metadata.prompt_token_ids is not None + logits = apply_all_penalties( + logits, + sampling_metadata.prompt_token_ids, + sampling_metadata.presence_penalties, + sampling_metadata.frequency_penalties, + sampling_metadata.repetition_penalties, + sampling_metadata.output_token_ids, + ) + return logits + + def apply_min_p( + self, + logits: torch.Tensor, + min_p: torch.Tensor, + ) -> torch.Tensor: + """ + Filters logits using adaptive probability thresholding. + """ + # Convert logits to probability distribution + probability_values = torch.nn.functional.softmax(logits, dim=-1) + # Calculate maximum probabilities per sequence + max_probabilities = torch.amax(probability_values, + dim=-1, + keepdim=True) + # Reshape min_p for broadcasting + adjusted_min_p = min_p.unsqueeze(1) * max_probabilities + # Identify valid tokens using threshold comparison + valid_token_mask = probability_values >= adjusted_min_p + # Apply mask using boolean indexing + logits[~valid_token_mask] = -float('inf') + return logits + + def apply_logits_bias( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> torch.Tensor: + # TODO(houseroad): this implementation is extremely inefficient. + # One idea is implement this as a PyTorch C++ op, and we may + # even optimize the logit_bias layout. + + rows: list[int] = [] + cols: list[int] = [] + vals: list[float] = [] + + # Get vocabulary size from logits + vocab_size = logits.shape[-1] + + for i, logit_bias in enumerate(sampling_metadata.logit_bias): + if logit_bias: + for token_id, bias in logit_bias.items(): + # Check token_id bounds to ensure within vocabulary + if token_id < 0 or token_id >= vocab_size: + raise ValueError( + f"token_id {token_id} in logit_bias contains " + f"out-of-vocab token id. Vocabulary size: " + f"{vocab_size}") + rows.append(i) + cols.append(token_id) + vals.append(bias) + + if rows: + indices = async_tensor_h2d([rows, cols], torch.int64, + logits.device, self.pin_memory) + values = async_tensor_h2d(vals, torch.float, logits.device, + self.pin_memory) + logits.index_put_(tuple(indices), values=values, accumulate=True) + return logits + + def apply_allowed_token_ids( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> torch.Tensor: + if sampling_metadata.allowed_token_ids_mask is not None: + logits.masked_fill_(sampling_metadata.allowed_token_ids_mask, + float("-inf")) + return logits + + def apply_bad_words( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> torch.Tensor: + if sampling_metadata.bad_words_token_ids: + apply_bad_words( + logits, + sampling_metadata.bad_words_token_ids, + sampling_metadata.output_token_ids, + ) + return logits From 6e1f7123841b145cc6a6b39e32266d0e069e789e Mon Sep 17 00:00:00 2001 From: Max de Bayser Date: Thu, 3 Jul 2025 14:47:27 -0300 Subject: [PATCH 13/29] import the right sampler Signed-off-by: Max de Bayser --- vllm_spyre/model_executor/model_loader/spyre.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/vllm_spyre/model_executor/model_loader/spyre.py b/vllm_spyre/model_executor/model_loader/spyre.py index 75d7553f1..2a2183410 100644 --- a/vllm_spyre/model_executor/model_loader/spyre.py +++ b/vllm_spyre/model_executor/model_loader/spyre.py @@ -7,13 +7,14 @@ import torch._inductor.config import torch.distributed as dist import torch.nn as nn +import vllm.envs as envs from fms.models import get_model from transformers import PretrainedConfig from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput from vllm.model_executor.model_loader.weight_utils import ( download_weights_from_hf) from vllm.model_executor.sampling_metadata import SamplingMetadata @@ -31,6 +32,14 @@ logger = init_logger(__name__) +def get_sampler() -> torch.nn.Module: + if envs.VLLM_USE_V1: + # Lazy import: the v1 package isn't distributed + from vllm_spyre.v1.sample.sampler import Sampler as V1Sampler + return V1Sampler() + return Sampler() + + @dataclass class SpyreAttentionMetadata: slot_mapping: torch.Tensor = None From c614eb1b88c1b79f3160cfe968c5fe448422e1d2 Mon Sep 17 00:00:00 2001 From: Max de Bayser Date: Thu, 3 Jul 2025 14:52:44 -0300 Subject: [PATCH 14/29] fix tests Signed-off-by: Max de Bayser --- tests/v1/worker/test_spyre_input_batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/v1/worker/test_spyre_input_batch.py b/tests/v1/worker/test_spyre_input_batch.py index 9788b9dc4..430ceee64 100644 --- a/tests/v1/worker/test_spyre_input_batch.py +++ b/tests/v1/worker/test_spyre_input_batch.py @@ -7,7 +7,7 @@ import torch from vllm.sampling_params import SamplingParams from vllm.utils import is_pin_memory_available, make_tensor_with_pad -from vllm.v1.sample.metadata import SamplingMetadata +from vllm_spyre.v1.sample.metadata import SamplingMetadata from vllm_spyre.v1.worker.spyre_input_batch import (CachedRequestState, InputBatch) From a3b37c42610cff38c74722094af56412916b02aa Mon Sep 17 00:00:00 2001 From: Max de Bayser Date: Thu, 3 Jul 2025 14:53:06 -0300 Subject: [PATCH 15/29] fix tests Signed-off-by: Max de Bayser --- tests/v1/worker/test_spyre_input_batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/v1/worker/test_spyre_input_batch.py b/tests/v1/worker/test_spyre_input_batch.py index 430ceee64..077b42897 100644 --- a/tests/v1/worker/test_spyre_input_batch.py +++ b/tests/v1/worker/test_spyre_input_batch.py @@ -7,8 +7,8 @@ import torch from vllm.sampling_params import SamplingParams from vllm.utils import is_pin_memory_available, make_tensor_with_pad -from vllm_spyre.v1.sample.metadata import SamplingMetadata +from vllm_spyre.v1.sample.metadata import SamplingMetadata from vllm_spyre.v1.worker.spyre_input_batch import (CachedRequestState, InputBatch) From d96533859c1aea201524860e91f05b0e9f670082 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 11:33:07 -0700 Subject: [PATCH 16/29] =?UTF-8?q?=F0=9F=9A=A7=20wip=20to=20see=20if=20test?= =?UTF-8?q?s=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 760c79d21..1b97a6587 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: - name: "default" repo: "" - name: "vLLM:main" - repo: "git+https://github.com/vllm-project/vllm --branch main" + repo: "git+https://github.com/vllm-project/vllm@02cabff207ca68094a73ba21296c82cdbcb1d1a5" test_suite: - name: "static batching" markers: "cpu and decoder and not cb" From 7c823588f9c5bde370582e03374ef9ad1bcd3ab0 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 17:03:42 -0700 Subject: [PATCH 17/29] =?UTF-8?q?=F0=9F=9A=A7=20cache=20new=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 37 +++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index d3847f00f..a5037ce78 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -247,7 +247,8 @@ def update_states(self, scheduler_output: SchedulerOutput): # Update the cached states. num_computed_tokens = req_data.num_computed_tokens[i] - new_token_ids = req_data.new_token_ids[i] + new_token_ids = req_data.new_token_ids[i] if len( + req_data.new_token_ids) > 0 else [] # Add the sampled token(s) from the previous step (if any). # This doesn't include "unverified" tokens like spec decode tokens. num_new_tokens = (num_computed_tokens + len(new_token_ids) - @@ -339,6 +340,29 @@ def execute_model( # Get mapping between requests ids to the index within the batch req_id_to_index = self.get_req_id_to_index(is_prefill) + reqs = scheduler_output.scheduled_new_reqs \ + if is_prefill else scheduler_output.scheduled_cached_reqs.req_ids + sampled_ids = output.sampled_token_ids.tolist() + for i, req in enumerate(reqs): + req_state = self.requests[req.req_id] \ + if not isinstance( + req, str) else self.requests[req] + # for sampled_ids in output.sampled_token_ids.tolist(): + # if is_prefill: + # req_id = self.input_batch.req_ids[next(iter(req_id_to_index.values()))] + # req_state = self.requests[ + # req_id_to_index(scheduler_output.scheduled_new_reqs[0].req_id) + # ] + # req_state = self.requests[scheduler_output.scheduled_new_reqs[0].req_id] + # else: + + # num_new_tokens = (num_computed_tokens + len(new_token_ids) - + # req_state.num_tokens) + # if num_new_tokens == 1: + # # Avoid slicing list in most common case. + # req_state.output_token_ids.append(new_token_ids[-1]) + # elif num_new_tokens > 0: + req_state.output_token_ids.extend(sampled_ids[i]) extra_kwargs: dict[str, Any] = {} if "pooler_output" in ModelRunnerOutput.__dataclass_fields__: @@ -775,10 +799,11 @@ def _prepare_decode( } req_ids = self.input_batch.sorted_requests_ids - for req_id in req_ids: + for i, req_id in enumerate(req_ids): # TODO: Will this always just be one token ID if there's no spec # or jump decoding? + req_cache = self.requests[req_id] # adding new blocks if needed if self.tkv // self.block_size + 1 > len( self.req_ids2blocks[req_id]): @@ -789,9 +814,13 @@ def _prepare_decode( offset = self.tkv % self.block_size slot = [start_slot + offset] slot_mapping.append(slot) - new_token_ids = cached_request_data.new_token_ids[ - cached_reqs_map[req_id]] + print(f"{req_id}:{req_cache.output_token_ids}") + new_token_ids = req_cache.output_token_ids + # print(new_token_ids) generation_token = new_token_ids[-1] + # generation_token = ( + # new_token_ids[-1] if not isinstance(new_token_ids, int) else new_token_ids + # ) input_tokens.append([generation_token]) seq_len = cached_request_data.num_computed_tokens[ cached_reqs_map[req_id]] From 6f4c5dea66e1dae2929c5ab42c345862b465cba0 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 17:19:52 -0700 Subject: [PATCH 18/29] =?UTF-8?q?=F0=9F=9A=A7=20cache=20new=20tokens=20wor?= =?UTF-8?q?ks=3F=3F=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index a5037ce78..de87c4e06 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -342,8 +342,14 @@ def execute_model( req_id_to_index = self.get_req_id_to_index(is_prefill) reqs = scheduler_output.scheduled_new_reqs \ if is_prefill else scheduler_output.scheduled_cached_reqs.req_ids + req_ids = ( + self.input_batch.sorted_requests_ids + if not is_prefill + else scheduler_output.scheduled_new_reqs + ) sampled_ids = output.sampled_token_ids.tolist() - for i, req in enumerate(reqs): + for i, req in enumerate(req_ids): + # for i, req in enumerate(reqs): req_state = self.requests[req.req_id] \ if not isinstance( req, str) else self.requests[req] @@ -814,10 +820,10 @@ def _prepare_decode( offset = self.tkv % self.block_size slot = [start_slot + offset] slot_mapping.append(slot) - print(f"{req_id}:{req_cache.output_token_ids}") new_token_ids = req_cache.output_token_ids # print(new_token_ids) generation_token = new_token_ids[-1] + print(f"{req_id}:{generation_token}") # generation_token = ( # new_token_ids[-1] if not isinstance(new_token_ids, int) else new_token_ids # ) From a4d610b4e63b7e1b38dc230b2e38ec78389f4e47 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 17:21:27 -0700 Subject: [PATCH 19/29] test till logits processor commit Signed-off-by: Prashant Gupta --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b97a6587..9acd98fa2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: - name: "default" repo: "" - name: "vLLM:main" - repo: "git+https://github.com/vllm-project/vllm@02cabff207ca68094a73ba21296c82cdbcb1d1a5" + repo: "git+https://github.com/vllm-project/vllm@c1909e7e8ccd2037e76536a8e726120c85d3754e" test_suite: - name: "static batching" markers: "cpu and decoder and not cb" From 09c7c03fba41618ceb80ed185221c1a1a0cb9329 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 17:27:55 -0700 Subject: [PATCH 20/29] revert to test from main Signed-off-by: Prashant Gupta --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9acd98fa2..760c79d21 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: - name: "default" repo: "" - name: "vLLM:main" - repo: "git+https://github.com/vllm-project/vllm@c1909e7e8ccd2037e76536a8e726120c85d3754e" + repo: "git+https://github.com/vllm-project/vllm --branch main" test_suite: - name: "static batching" markers: "cpu and decoder and not cb" From c9596489f82fbe640eb4003df6ee9695c1d25453 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Thu, 3 Jul 2025 17:37:19 -0700 Subject: [PATCH 21/29] =?UTF-8?q?=F0=9F=90=9B=20fix=20for=20sb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index de87c4e06..318639b76 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -493,7 +493,9 @@ def _prepare_decode( for i, req_id in enumerate(cached_request_data.req_ids): # TODO: Will this always just be one token ID if there's no spec # or jump decoding? - new_token_ids = cached_request_data.new_token_ids[i] + req_cache = self.requests[req_id] + # new_token_ids = cached_request_data.new_token_ids[i] + new_token_ids = req_cache.output_token_ids generation_token = new_token_ids[-1] input_tokens[self.input_batch.req_id_to_index[req_id]] = [ generation_token From efc467fc03ab38fcf2136aa8f2f7f2b06cb2f4da Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Sat, 5 Jul 2025 11:41:40 -0700 Subject: [PATCH 22/29] =?UTF-8?q?=F0=9F=8E=A8=20fmt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 39 ++++------------------ 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 318639b76..976fc85ae 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -340,34 +340,14 @@ def execute_model( # Get mapping between requests ids to the index within the batch req_id_to_index = self.get_req_id_to_index(is_prefill) - reqs = scheduler_output.scheduled_new_reqs \ - if is_prefill else scheduler_output.scheduled_cached_reqs.req_ids - req_ids = ( - self.input_batch.sorted_requests_ids - if not is_prefill - else scheduler_output.scheduled_new_reqs - ) + + req_ids = (self.input_batch.sorted_requests_ids + if not is_prefill else scheduler_output.scheduled_new_reqs) sampled_ids = output.sampled_token_ids.tolist() for i, req in enumerate(req_ids): - # for i, req in enumerate(reqs): req_state = self.requests[req.req_id] \ if not isinstance( req, str) else self.requests[req] - # for sampled_ids in output.sampled_token_ids.tolist(): - # if is_prefill: - # req_id = self.input_batch.req_ids[next(iter(req_id_to_index.values()))] - # req_state = self.requests[ - # req_id_to_index(scheduler_output.scheduled_new_reqs[0].req_id) - # ] - # req_state = self.requests[scheduler_output.scheduled_new_reqs[0].req_id] - # else: - - # num_new_tokens = (num_computed_tokens + len(new_token_ids) - - # req_state.num_tokens) - # if num_new_tokens == 1: - # # Avoid slicing list in most common case. - # req_state.output_token_ids.append(new_token_ids[-1]) - # elif num_new_tokens > 0: req_state.output_token_ids.extend(sampled_ids[i]) extra_kwargs: dict[str, Any] = {} @@ -495,8 +475,8 @@ def _prepare_decode( # or jump decoding? req_cache = self.requests[req_id] # new_token_ids = cached_request_data.new_token_ids[i] - new_token_ids = req_cache.output_token_ids - generation_token = new_token_ids[-1] + output_token_ids = req_cache.output_token_ids + generation_token = output_token_ids[-1] input_tokens[self.input_batch.req_id_to_index[req_id]] = [ generation_token ] @@ -822,13 +802,8 @@ def _prepare_decode( offset = self.tkv % self.block_size slot = [start_slot + offset] slot_mapping.append(slot) - new_token_ids = req_cache.output_token_ids - # print(new_token_ids) - generation_token = new_token_ids[-1] - print(f"{req_id}:{generation_token}") - # generation_token = ( - # new_token_ids[-1] if not isinstance(new_token_ids, int) else new_token_ids - # ) + output_token_ids = req_cache.output_token_ids + generation_token = output_token_ids[-1] input_tokens.append([generation_token]) seq_len = cached_request_data.num_computed_tokens[ cached_reqs_map[req_id]] From 24a765314fc1a81211088101fa42c4575ca147cf Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Sat, 5 Jul 2025 12:16:37 -0700 Subject: [PATCH 23/29] =?UTF-8?q?=F0=9F=8E=A8=20fmt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 976fc85ae..9f2244daa 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -238,12 +238,9 @@ def update_states(self, scheduler_output: SchedulerOutput): # Update input_batch's `token_ids_cpu`, # `num_tokens`. For continuous batching it cleans # finished requests from the batch - # - # NOTE: req_state.output_token_ids is being mutated. - req_data = scheduler_output.scheduled_cached_reqs for i, req_id in enumerate(req_data.req_ids): - req_state = self.requests[req_id] + req_state: CachedRequestState = self.requests[req_id] # Update the cached states. num_computed_tokens = req_data.num_computed_tokens[i] @@ -341,6 +338,7 @@ def execute_model( # Get mapping between requests ids to the index within the batch req_id_to_index = self.get_req_id_to_index(is_prefill) + # Add the sampled token(s) to the request cache req_ids = (self.input_batch.sorted_requests_ids if not is_prefill else scheduler_output.scheduled_new_reqs) sampled_ids = output.sampled_token_ids.tolist() @@ -473,9 +471,8 @@ def _prepare_decode( for i, req_id in enumerate(cached_request_data.req_ids): # TODO: Will this always just be one token ID if there's no spec # or jump decoding? - req_cache = self.requests[req_id] - # new_token_ids = cached_request_data.new_token_ids[i] - output_token_ids = req_cache.output_token_ids + req_state: CachedRequestState = self.requests[req_id] + output_token_ids = req_state.output_token_ids generation_token = output_token_ids[-1] input_tokens[self.input_batch.req_id_to_index[req_id]] = [ generation_token @@ -791,7 +788,7 @@ def _prepare_decode( # TODO: Will this always just be one token ID if there's no spec # or jump decoding? - req_cache = self.requests[req_id] + req_state: CachedRequestState = self.requests[req_id] # adding new blocks if needed if self.tkv // self.block_size + 1 > len( self.req_ids2blocks[req_id]): @@ -802,14 +799,13 @@ def _prepare_decode( offset = self.tkv % self.block_size slot = [start_slot + offset] slot_mapping.append(slot) - output_token_ids = req_cache.output_token_ids + output_token_ids = req_state.output_token_ids generation_token = output_token_ids[-1] input_tokens.append([generation_token]) seq_len = cached_request_data.num_computed_tokens[ cached_reqs_map[req_id]] input_positions.append([seq_len]) - req_state = self.requests[req_id] left_padded_prompt_mask.append(req_state.left_padding) input_tokens = torch.tensor(input_tokens, From dfcbba4f7892aca451f024993a2f2a79cecf664f Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Mon, 7 Jul 2025 13:47:57 -0700 Subject: [PATCH 24/29] =?UTF-8?q?=F0=9F=94=A5=20remove=20unused=20var?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index ccca6f092..7e9bb730a 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -858,7 +858,7 @@ def _prepare_decode( } req_ids = self.input_batch.sorted_requests_ids - for i, req_id in enumerate(req_ids): + for req_id in req_ids: # TODO: Will this always just be one token ID if there's no spec # or jump decoding? From 6d6178480fb39c55d8bb355e89151b04e4c0e6f1 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Mon, 7 Jul 2025 13:58:38 -0700 Subject: [PATCH 25/29] =?UTF-8?q?=F0=9F=8E=A8=20add=20comment,=20remove=20?= =?UTF-8?q?unused=20var?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 7e9bb730a..9be8d0ced 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -249,6 +249,8 @@ def update_states(self, scheduler_output: SchedulerOutput): # Update the cached states. num_computed_tokens = req_data.num_computed_tokens[i] + # In the future, when using PP, the scheduler will send the sampled + # tokens back new_token_ids = req_data.new_token_ids[i] if len( req_data.new_token_ids) > 0 else [] # Add the sampled token(s) from the previous step (if any). @@ -542,7 +544,7 @@ def _prepare_decode( [0] for _ in range(self._position_ids.shape[0]) ] - for i, req_id in enumerate(cached_request_data.req_ids): + for req_id in cached_request_data.req_ids: # TODO: Will this always just be one token ID if there's no spec # or jump decoding? req_state: CachedRequestState = self.requests[req_id] From 9a5769ff8a5cc49eefe3bbfc483f5cfc22505017 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Mon, 7 Jul 2025 14:08:56 -0700 Subject: [PATCH 26/29] =?UTF-8?q?=F0=9F=8E=A8=20improve=20condition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 9be8d0ced..72fe28998 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -415,8 +415,8 @@ def execute_model( req_id_to_index = self.get_req_id_to_index(is_prefill) # Add the sampled token(s) to the request cache - req_ids = (self.input_batch.sorted_requests_ids - if not is_prefill else scheduler_output.scheduled_new_reqs) + req_ids = (scheduler_output.scheduled_new_reqs + if is_prefill else self.input_batch.sorted_requests_ids) sampled_ids = output.sampled_token_ids.tolist() for i, req in enumerate(req_ids): req_state = self.requests[req.req_id] \ From b1a7c85e1a45b3ca9789e07e5a2ad530c06a29c4 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Mon, 7 Jul 2025 14:21:12 -0700 Subject: [PATCH 27/29] =?UTF-8?q?=F0=9F=8E=A8=20add=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 72fe28998..596840298 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -243,6 +243,9 @@ def update_states(self, scheduler_output: SchedulerOutput): # Update input_batch's `token_ids_cpu`, # `num_tokens`. For continuous batching it cleans # finished requests from the batch + # + # NOTE: req_state.output_token_ids will be mutated when + # using PP req_data = scheduler_output.scheduled_cached_reqs for i, req_id in enumerate(req_data.req_ids): req_state: CachedRequestState = self.requests[req_id] From 6a5d3a6587a4236522948cddb830b6aee69c5e55 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Tue, 8 Jul 2025 07:29:42 -0700 Subject: [PATCH 28/29] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20bump=20to=20vllm>=3D?= =?UTF-8?q?0.9.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- pyproject.toml | 2 +- uv.lock | 449 +++++++++++++++++++++---------------------------- 2 files changed, 190 insertions(+), 261 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4ba84770f..b9c9fbfb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ license = {text = "Apache 2"} dependencies = [ "fms-model-optimizer>=0.2.0", "ibm-fms==1.1.0", - "vllm>=0.9.0,!=0.9.1", + "vllm>=0.9.2", ] requires-python = ">=3.9" dynamic = ["version"] diff --git a/uv.lock b/uv.lock index cfa9cf01c..df4670601 100644 --- a/uv.lock +++ b/uv.lock @@ -693,16 +693,16 @@ wheels = [ [[package]] name = "compressed-tensors" -version = "0.9.4" +version = "0.10.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic", marker = "python_full_version >= '3.10'" }, { name = "torch", marker = "python_full_version >= '3.10' and sys_platform == 'never'" }, { name = "transformers", marker = "python_full_version >= '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/af/cb/1de205e8018cb21a4dc995324652115cf209357de5aaed8d6af101d26b42/compressed_tensors-0.9.4.tar.gz", hash = "sha256:34779417ffa31a207adb0cc4fd2a86cb75e239e504fb2068e494092f4b5703b7", size = 111223, upload-time = "2025-04-24T19:19:07.826Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/86/d43d369abc81ec63ec7b8f6f27fc8b113ea0fd18a4116ae12063387b8b34/compressed_tensors-0.10.2.tar.gz", hash = "sha256:6de13ac535d7ffdd8890fad3d229444c33076170acaa8fab6bab8ecfa96c1d8f", size = 173459, upload-time = "2025-06-23T13:19:06.135Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/98/bf09fd8196e0b658e7b48404ed1b60544b5111f80731d76b378e3d8765bb/compressed_tensors-0.9.4-py3-none-any.whl", hash = "sha256:b12e3616f06243a074f61b736596882c6549cdc3669ac48434102a4a88e8002a", size = 100345, upload-time = "2025-04-24T19:19:05.415Z" }, + { url = "https://files.pythonhosted.org/packages/43/ac/56bb4b6b3150783119479e2f05e32ebfc39ca6ff8e6fcd45eb178743b39e/compressed_tensors-0.10.2-py3-none-any.whl", hash = "sha256:e1b4d9bc2006e3fd3a938e59085f318fdb280c5af64688a4792bf1bc263e579d", size = 169030, upload-time = "2025-06-23T13:19:03.487Z" }, ] [[package]] @@ -803,18 +803,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] -[[package]] -name = "deprecated" -version = "1.2.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, -] - [[package]] name = "depyf" version = "0.18.0" @@ -1175,18 +1163,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/ee/0301c4a2bdb43da8d059d67382ec7ca554677366ebb73db82b690a10c98a/gguf-0.14.0-py3-none-any.whl", hash = "sha256:d279b33cd743d6211c09d96f0797eb36652c0d9d90844f8986a7c25e445906c4", size = 76159, upload-time = "2025-01-08T19:19:00.497Z" }, ] -[[package]] -name = "googleapis-common-protos" -version = "1.70.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, -] - [[package]] name = "grpcio" version = "1.71.0" @@ -1342,7 +1318,7 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "0.32.4" +version = "0.33.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock", marker = "python_full_version >= '3.10'" }, @@ -1354,9 +1330,9 @@ dependencies = [ { name = "tqdm", marker = "python_full_version >= '3.10'" }, { name = "typing-extensions", marker = "python_full_version >= '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/60/c8/4f7d270285c46324fd66f62159eb16739aa5696f422dba57678a8c6b78e9/huggingface_hub-0.32.4.tar.gz", hash = "sha256:f61d45cd338736f59fb0e97550b74c24ee771bcc92c05ae0766b9116abe720be", size = 424494, upload-time = "2025-06-03T09:59:46.105Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/42/8a95c5632080ae312c0498744b2b852195e10b05a20b1be11c5141092f4c/huggingface_hub-0.33.2.tar.gz", hash = "sha256:84221defaec8fa09c090390cd68c78b88e3c4c2b7befba68d3dc5aacbc3c2c5f", size = 426637, upload-time = "2025-07-02T06:26:05.156Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/8b/222140f3cfb6f17b0dd8c4b9a0b36bd4ebefe9fb0098ba35d6960abcda0f/huggingface_hub-0.32.4-py3-none-any.whl", hash = "sha256:37abf8826b38d971f60d3625229221c36e53fe58060286db9baf619cfbf39767", size = 512101, upload-time = "2025-06-03T09:59:44.099Z" }, + { url = "https://files.pythonhosted.org/packages/44/f4/5f3f22e762ad1965f01122b42dae5bf0e009286e2dba601ce1d0dba72424/huggingface_hub-0.33.2-py3-none-any.whl", hash = "sha256:3749498bfa91e8cde2ddc2c1db92c79981f40e66434c20133b39e5928ac9bcc5", size = 515373, upload-time = "2025-07-02T06:26:03.072Z" }, ] [package.optional-dependencies] @@ -1384,18 +1360,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] -[[package]] -name = "importlib-metadata" -version = "8.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/20/ff/bd28f70283b9cca0cbf0c2a6082acbecd822d1962ae7b2a904861b9965f8/importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812", size = 52667, upload-time = "2024-06-25T18:38:04.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/ef/38766b2edb096260d9b1b6ad35adaa0bce3b0567abb452b21eb074af88c4/importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f", size = 24769, upload-time = "2024-06-25T18:38:02.324Z" }, -] - [[package]] name = "iniconfig" version = "2.1.0" @@ -2023,7 +1987,7 @@ wheels = [ [[package]] name = "mistral-common" -version = "1.5.4" +version = "1.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonschema", marker = "python_full_version >= '3.10'" }, @@ -2035,9 +1999,9 @@ dependencies = [ { name = "tiktoken", marker = "python_full_version >= '3.10'" }, { name = "typing-extensions", marker = "python_full_version >= '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/31/0453db671b61d2716bd263aecd95c53fcc0aa4d32ab404fa7070909dc005/mistral_common-1.5.4.tar.gz", hash = "sha256:0af4124ab09d1409761e91ec61681476882d46f9418eea8908d39c01222e0f6b", size = 6269751, upload-time = "2025-03-16T21:05:45.102Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/1e/61ca75d5918e9e3fe401ba373582e9fe8be46a5ba1ea213c7603879b9ade/mistral_common-1.6.3.tar.gz", hash = "sha256:a574807f79a639db672bede258b79ad7570e137adff9f2bf811764365449eb13", size = 6304308, upload-time = "2025-07-01T12:22:18.053Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/80/7a/421819257cd642b33d71819e2ff259fb019a49ea48e830e5a32558c52cb7/mistral_common-1.5.4-py3-none-any.whl", hash = "sha256:acef3367a4386d5dd3d9e23330348bbebe90a5cbd2fc5587d8a8d13d9893e537", size = 6477779, upload-time = "2025-03-16T21:05:41.366Z" }, + { url = "https://files.pythonhosted.org/packages/6a/17/6b8a864726d6d8ad433aa9c94d67b2887a3025d15ee5378c629b68fa5f89/mistral_common-1.6.3-py3-none-any.whl", hash = "sha256:28ab533118e472cabaceb3d3a17ad43a9656dc3cd5bcb9cf28c61d8c7018a663", size = 6493602, upload-time = "2025-07-01T12:22:16.246Z" }, ] [package.optional-dependencies] @@ -2057,6 +2021,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/4d/23c4e4f09da849e127e9f123241946c23c1e30f45a88366879e064211815/mistune-3.1.3-py3-none-any.whl", hash = "sha256:1a32314113cff28aa6432e99e522677c8587fd83e3d51c29b82a52409c842bd9", size = 53410, upload-time = "2025-03-19T14:27:23.451Z" }, ] +[[package]] +name = "mlx" +version = "0.26.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/aa/8f21ff5a6f17216146f277a66b5b5a6a92277aa7cc28cc432716c93170d9/mlx-0.26.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:547297872a84c768b176885c7995666c77eceaeb58ae907cafc1999dba959a75", size = 33063249, upload-time = "2025-07-01T22:13:52.954Z" }, + { url = "https://files.pythonhosted.org/packages/48/28/ded06fe4a5fb6ab7648925e66088342eb7dd2950299b5dd277f4b67f7604/mlx-0.26.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:cc2b1faf65e97e1b4b62018b76888772ea2daac11d4b7c3ee31af7a2fb8c6c9a", size = 32456637, upload-time = "2025-07-01T22:14:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/7e/c6/5ada90f493f48b9897f5ae0abe4ddac855fe9f721ec2c72c2cf2b14b7ec2/mlx-0.26.2-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:d63d2023cb4958938ed75e976444f2a212039a2facf04a3020ced8544485b662", size = 32457123, upload-time = "2025-07-01T22:14:20.22Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/895838604ccb8c68b34b284d94a9b0d0cef9d08d67f7525fe8d86f43e238/mlx-0.26.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:d9d2e886482dc86847ea25ef71bd103c5772dc69ea4c3516621626a009bf5b76", size = 33063991, upload-time = "2025-07-01T22:13:55.974Z" }, + { url = "https://files.pythonhosted.org/packages/27/da/6aad0f95182ddfea1979a077a4c4c6c9e56a7fc9b7a04b9aa16673850e5d/mlx-0.26.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bc006c9db31fb8962f522fc45af2258fe093eb15f38b62937810fe2e11a06ead", size = 32456745, upload-time = "2025-07-01T22:13:18.355Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3c/2075296d53285538407afca1f2d65ee002895b9b99f76bd99867d4de6c61/mlx-0.26.2-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:c1dea89c317023b1e419c337bb9b7af32e4a29ecd881df14a2429eb001afd72c", size = 32457367, upload-time = "2025-07-01T22:13:47.436Z" }, + { url = "https://files.pythonhosted.org/packages/31/b8/57f16154ae3b859e5388e4ca0c816510f3137ea61c0da61db35188372944/mlx-0.26.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:ee14eebe6625f1344c13246a5b67ea40015706a7ff9affd9a4c943811ecb5b53", size = 33057841, upload-time = "2025-07-01T22:13:57.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/db/ccde1938bc7d65e918e7b062fd5128d9c0e00d699c1ef28a57068a539ffc/mlx-0.26.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:0decf09ea33dc58c3365af2f437f519ba0765da9b0199b5eaf6e09f90f2f1d6f", size = 32457702, upload-time = "2025-07-01T22:14:12.305Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/25729ead37a6f196838aaf48e467dfc2aa0ff7f8008f79421cf7684f8c6a/mlx-0.26.2-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:7e18fc05e85e4bd1519b55fb62b58a93760d927e12acc7b1bfe2cb531b284e04", size = 32458397, upload-time = "2025-07-01T22:14:22.124Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/367c5f3ae199db0abfd6e8945b8996efa86fda20524de322de3c34e0b379/mlx-0.26.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:ee59e1ebf1f3a6fb90bfff187b1a71dd11de29c0c79f7e3d04abdf3d2c2df88b", size = 33057778, upload-time = "2025-07-01T22:13:54.656Z" }, + { url = "https://files.pythonhosted.org/packages/22/f0/f6fd97514e8556a0611289aca555977d3561289ce4ef9b7a691052c15d20/mlx-0.26.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:80be08dcdc4da45311fd01a05e1a40e2ad47e4128ffa79abc52ab0725ec0dbdf", size = 32457625, upload-time = "2025-07-01T22:14:28.575Z" }, + { url = "https://files.pythonhosted.org/packages/65/ce/0df9ac206dfd20998a2838b09f7566b495b560a6e1b987af712707da690d/mlx-0.26.2-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:36e34dd119d77a000a0d788268d20f9e926bdc0a706045fa30de8a26d8a9e059", size = 32458371, upload-time = "2025-07-01T22:14:51.532Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3e/fbbd3364fd49381837b5532f5d39f5cd2bf499137e3bee8c31b0f6621729/mlx-0.26.2-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:cb1cad0f810550b5902596ad59501e60d8d3016b755dc45b7b1fbf09d956dc2b", size = 33063313, upload-time = "2025-07-01T22:14:04.922Z" }, + { url = "https://files.pythonhosted.org/packages/ed/58/014c489891a8530333868673da3b4c1ed070fe785a6bdd78e1403e575d98/mlx-0.26.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:f6471769f5f2b7006b729fd02cb61ad808833ccc61f50b501da795e3302f600d", size = 32456801, upload-time = "2025-07-01T22:14:39.855Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e6/9548b49d6558c089fefc0f87f43dce20992b38e2175bb122b05e52f7dfe4/mlx-0.26.2-cp39-cp39-macosx_15_0_arm64.whl", hash = "sha256:cc407d8c039e9c2bc0adfdb9ecd4f6d8ae49a3c05797d60d51b9ae5ae05b5185", size = 32457585, upload-time = "2025-07-01T22:13:47.924Z" }, +] + +[[package]] +name = "mlx-lm" +version = "0.25.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, + { name = "mlx", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, + { name = "numpy", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, + { name = "protobuf", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, + { name = "pyyaml", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, + { name = "transformers", extra = ["sentencepiece"], marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ec/bc/0c3f69a8ff78fc8152985be99b2f83dc7e902b9b96ff5260c6a4958c10f1/mlx_lm-0.25.3.tar.gz", hash = "sha256:40ea0a2849abd804a40a3e388627ae5327918a8656287022610150fd453a2242", size = 154221, upload-time = "2025-07-01T03:04:07.056Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/ce/3484a973943572461765977231e3b9b68876a8d7e16c3e6110b81c180a89/mlx_lm-0.25.3-py3-none-any.whl", hash = "sha256:56a84f1ae4a3581b13c84c4d8edaa6704b971b40090b725dfc3b719b522ccc2b", size = 203913, upload-time = "2025-07-01T03:04:05.928Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -2573,128 +2576,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/8a/69176a64335aed183529207ba8bc3d329c2999d852b4f3818027203f50e6/opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca", size = 39402386, upload-time = "2025-01-16T13:52:56.418Z" }, ] -[[package]] -name = "opentelemetry-api" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated", marker = "python_full_version >= '3.10'" }, - { name = "importlib-metadata", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/d4/e9a0ddef6eed086c96e8265d864a46da099611b7be153b0cfb63fd47e1b4/opentelemetry_api-1.26.0.tar.gz", hash = "sha256:2bd639e4bed5b18486fef0b5a520aaffde5a18fc225e808a1ac4df363f43a1ce", size = 60904, upload-time = "2024-07-25T04:02:03.937Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/a7/6322d1d7a1fb926e8b99208c27730f21217da2f1e0e11dab48a78a0427a4/opentelemetry_api-1.26.0-py3-none-any.whl", hash = "sha256:7d7ea33adf2ceda2dd680b18b1677e4152000b37ca76e679da71ff103b943064", size = 61533, upload-time = "2024-07-25T04:01:38.504Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-exporter-otlp-proto-http", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/be/99/80edf6286f9040fadf065f9a11869fda34449a61e62a5372cb84d5a6f53b/opentelemetry_exporter_otlp-1.26.0.tar.gz", hash = "sha256:cf0e093f080011951d9f97431a83869761e4d4ebe83a4195ee92d7806223299c", size = 6168, upload-time = "2024-07-25T04:02:05.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/71/b9221af6af61213c522401b5f46a5eaa41d8dd7daeb0740dc5604f5c3980/opentelemetry_exporter_otlp-1.26.0-py3-none-any.whl", hash = "sha256:f839989f54bda85ee33c5dae033c44dcec9ccbb0dafc6a43d585df44da1d2036", size = 7001, upload-time = "2024-07-25T04:01:41.651Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-proto", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/84/cd/ed9eaa1d80facb6609d02af6c393b02ce3797a15742361be4859db6fdc17/opentelemetry_exporter_otlp_proto_common-1.26.0.tar.gz", hash = "sha256:bdbe50e2e22a1c71acaa0c8ba6efaadd58882e5a5978737a44a4c4b10d304c92", size = 17815, upload-time = "2024-07-25T04:02:06.537Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/25/2f/0f7e0a73fd901c9abc6ea680d7f19a803dac830c450f21e1123d3a3ec488/opentelemetry_exporter_otlp_proto_common-1.26.0-py3-none-any.whl", hash = "sha256:ee4d8f8891a1b9c372abf8d109409e5b81947cf66423fd998e56880057afbc71", size = 17837, upload-time = "2024-07-25T04:01:42.942Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated", marker = "python_full_version >= '3.10'" }, - { name = "googleapis-common-protos", marker = "python_full_version >= '3.10'" }, - { name = "grpcio", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-api", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-exporter-otlp-proto-common", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-proto", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-sdk", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a0/23/cac89aca97ecb8f7498a875dc2ac89224b4f3345bcb8ffff643b59886196/opentelemetry_exporter_otlp_proto_grpc-1.26.0.tar.gz", hash = "sha256:a65b67a9a6b06ba1ec406114568e21afe88c1cdb29c464f2507d529eb906d8ae", size = 25239, upload-time = "2024-07-25T04:02:07.242Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/0c/e4473692fec8076008c7926dfcef7223fc6d2785f04ad9d8402347a4eba9/opentelemetry_exporter_otlp_proto_grpc-1.26.0-py3-none-any.whl", hash = "sha256:e2be5eff72ebcb010675b818e8d7c2e7d61ec451755b8de67a140bc49b9b0280", size = 18228, upload-time = "2024-07-25T04:01:44.308Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated", marker = "python_full_version >= '3.10'" }, - { name = "googleapis-common-protos", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-api", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-exporter-otlp-proto-common", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-proto", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-sdk", marker = "python_full_version >= '3.10'" }, - { name = "requests", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/d2/4e6e2066b87626966f99f8fc7fcb9414e7548779d751def7db54c9d25b1c/opentelemetry_exporter_otlp_proto_http-1.26.0.tar.gz", hash = "sha256:5801ebbcf7b527377883e6cbbdda35ee712dc55114fff1e93dfee210be56c908", size = 14451, upload-time = "2024-07-25T04:02:08.192Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/d3/0b7217b61903249035d219fbe93a8558287f86aead340c7b2dc1226b8ad4/opentelemetry_exporter_otlp_proto_http-1.26.0-py3-none-any.whl", hash = "sha256:ee72a87c48ec977421b02f16c52ea8d884122470e0be573905237b540f4ee562", size = 16795, upload-time = "2024-07-25T04:01:45.645Z" }, -] - -[[package]] -name = "opentelemetry-proto" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a9/06/9505ef04e527fa711ebffb47f3f56cac6015405953ff688fc349d170fb9c/opentelemetry_proto-1.26.0.tar.gz", hash = "sha256:c5c18796c0cab3751fc3b98dee53855835e90c0422924b484432ac852d93dc1e", size = 34749, upload-time = "2024-07-25T04:02:16.651Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/f4/66a3892eea913cded9bac0fdd3fb1a412fa2da8eb50014ec87a52648444a/opentelemetry_proto-1.26.0-py3-none-any.whl", hash = "sha256:6c4d7b4d4d9c88543bcf8c28ae3f8f0448a753dc291c18c5390444c90b76a725", size = 52466, upload-time = "2024-07-25T04:01:58.287Z" }, -] - -[[package]] -name = "opentelemetry-sdk" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-semantic-conventions", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/85/8ca0d5ebfe708287b091dffcd15553b74bbfe4532f8dd42662b78b2e0cab/opentelemetry_sdk-1.26.0.tar.gz", hash = "sha256:c90d2868f8805619535c05562d699e2f4fb1f00dbd55a86dcefca4da6fa02f85", size = 143139, upload-time = "2024-07-25T04:02:17.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/f1/a9b550d0f9c049653dd2eab45cecf8fe4baa9795ed143d87834056ffabaf/opentelemetry_sdk-1.26.0-py3-none-any.whl", hash = "sha256:feb5056a84a88670c041ea0ded9921fca559efec03905dddeb3885525e0af897", size = 109475, upload-time = "2024-07-25T04:01:59.997Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.47b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-api", marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/85/edef14d10ad00ddd9fffb20e4d3d938f4c5c1247e11a175066fe2b4a72f8/opentelemetry_semantic_conventions-0.47b0.tar.gz", hash = "sha256:a8d57999bbe3495ffd4d510de26a97dadc1dace53e0275001b2c1b2f67992a7e", size = 83994, upload-time = "2024-07-25T04:02:19.064Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/c2/ca5cef8e4cd8eec5a95deed95ec3f6005e499fd9d17ca08731ced03a6921/opentelemetry_semantic_conventions-0.47b0-py3-none-any.whl", hash = "sha256:4ff9d595b85a59c1c1413f02bba320ce7ea6bf9e2ead2b0913c4395c7bbc1063", size = 138027, upload-time = "2024-07-25T04:02:01.7Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions-ai" -version = "0.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/8f/7fb173fd1928398b81d0952f7a9f30381ce3215817e3ac6e92f180434874/opentelemetry_semantic_conventions_ai-0.4.3.tar.gz", hash = "sha256:761a68a7e99436dfc53cfe1f99507316aa0114ac480f0c42743b9320b7c94831", size = 4540, upload-time = "2025-03-04T16:33:13.893Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/56/b178de82b650526ff5d5e67037786008ea0acd043051d535c483dabd3cc4/opentelemetry_semantic_conventions_ai-0.4.3-py3-none-any.whl", hash = "sha256:9ff60bbf38c8a891c20a355b4ca1948380361e27412c3ead264de0d050fa2570", size = 5384, upload-time = "2025-03-04T16:33:11.784Z" }, -] - [[package]] name = "outlines" version = "0.1.11" @@ -3222,6 +3103,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/ee/c110d8da8bdde8e832ccf1ff90be747cb684874e2dc8acf26840058b0c32/pyarrow-19.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429", size = 25465593, upload-time = "2025-02-18T18:55:54.191Z" }, ] +[[package]] +name = "pybase64" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/32/5d25a15256d2e80d1e92be821f19fc49190e65a90ea86733cb5af2285449/pybase64-1.4.1.tar.gz", hash = "sha256:03fc365c601671add4f9e0713c2bc2485fa4ab2b32f0d3bb060bd7e069cdaa43", size = 136836, upload-time = "2025-03-02T11:13:57.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/68/32b6446f679a0236735bf55f7b6595a5398d614f4c29e022d205d3359858/pybase64-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7628c86c431e04ae192ffeff0f8ae96b70ff4c053ad666625e7d6335196ea8a", size = 38066, upload-time = "2025-03-02T11:10:09.239Z" }, + { url = "https://files.pythonhosted.org/packages/73/10/73637b81b54d785bc5873ba6a28d5b5062493a3801c37afb7734fa78ed09/pybase64-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5202939f188cf150e1bc56f8b0da54a2cae2dcb9b27f4f7d313b358f707e1f7f", size = 31487, upload-time = "2025-03-02T11:10:11.285Z" }, + { url = "https://files.pythonhosted.org/packages/ed/5c/64ffd0c251fbd672c1306ddc792762eec09d39d7748d2656592b5e24cd39/pybase64-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e15e0eaf665bcc5427c1f32f604ed02d599b7777e8b7f8391e943a8d7bc443f", size = 57334, upload-time = "2025-03-02T11:10:13.656Z" }, + { url = "https://files.pythonhosted.org/packages/f6/69/d5b5f2a0d036bd0cadd17b0e581c11863074a3aab2090b07209c5fc1e18a/pybase64-1.4.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a0206b4b65f7cc0e0b6c26428765d3f0bae1312cb9d0fcebfad7cc24dfae4788", size = 54342, upload-time = "2025-03-02T11:10:16.003Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bf/521c75786f519745de80b50eed22d73f16df201a954fbd613de0fa8e96b7/pybase64-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:732c5a4f7b389e6655375e75bde6fbab15508c8ae819bf41bda2c0202a59ff19", size = 56996, upload-time = "2025-03-02T11:10:18.491Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f7/a510a06bea28ce17caec42a31d6587e196c288a9604a09af39191b410e76/pybase64-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecc374ea70bcef1884d3745480e07d1502bfbb41ac138cc38445c58c685dee32", size = 57544, upload-time = "2025-03-02T11:10:21.395Z" }, + { url = "https://files.pythonhosted.org/packages/3f/68/e592b7641932a54a8255253865a646cfad4921471407263c33af47976023/pybase64-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a0433a4e76f10862817f303c2bf74371e118cb24124836bfb0d95ebc182dc97", size = 66115, upload-time = "2025-03-02T11:10:23.279Z" }, + { url = "https://files.pythonhosted.org/packages/4c/46/24f97d76fec6532a7a60133fd9691a8afab6c7eab791368d14353dac5488/pybase64-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25b8405f632cce8b2e2f991ec2e4074b6a98ea44273cd218ffc3f88524ed162a", size = 68719, upload-time = "2025-03-02T11:10:24.868Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/5d8f1b530c4bc22c943ce4879f4e66aa879fe23ff411c8725b81a03bdf95/pybase64-1.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ab02c31afe58b03d55a66fd9bd2cc4a04698b6bb2c33f68955aaec151542d838", size = 56136, upload-time = "2025-03-02T11:10:27.206Z" }, + { url = "https://files.pythonhosted.org/packages/f7/34/f40fea3fb306857d8e86473b1b5c2bc8d401c58ac424f59f8ec8fd7e55be/pybase64-1.4.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:8030ad8fe74c034cfad9a9a037c7b6ee85094b522c8b94c05e81df46e9a0eb5c", size = 49929, upload-time = "2025-03-02T11:10:29.069Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/7cd961e5cfb6fee5f3838586b0036876d0c58566f65d5973b78d4c090cc7/pybase64-1.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fb18c6a4defe85d23b16b1e6d6c7c3038cc402adfd8af14acc774dc585e814c4", size = 66380, upload-time = "2025-03-02T11:10:30.95Z" }, + { url = "https://files.pythonhosted.org/packages/85/a3/384601da9e09907d7509ec448afbce4be75a366db9ac36692c924dae7519/pybase64-1.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:3f645629fae78e337faaa2ad7d35ced3f65b66f66629542d374641e30b218d1f", size = 55508, upload-time = "2025-03-02T11:10:32.13Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f7/74ae590bafed894c634bd3684ea0c86d4878c5ccd31e3a10ae1e5391bdf3/pybase64-1.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02ff55724616a11eebceac6c8445dadac79289ae8d1e40eed1b24aa7517fa225", size = 53781, upload-time = "2025-03-02T11:10:33.946Z" }, + { url = "https://files.pythonhosted.org/packages/2b/74/26c2d3f1893cc6904822fb8966dd722f432438273cce9e14f45ddfb454d0/pybase64-1.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:426e1ab673c744012d4b072fa6dc0642ca900b5c341f5e0c3a1c30b5dac332d1", size = 68233, upload-time = "2025-03-02T11:10:35.133Z" }, + { url = "https://files.pythonhosted.org/packages/09/10/f6a2bb04e11f7e639e7b59a41fd4597f68d9f3dde1014184ddaa480e3eac/pybase64-1.4.1-cp310-cp310-win32.whl", hash = "sha256:9101ee786648fc45b4765626eaf71114dd021b73543d8a3ab975df3dfdcca667", size = 34219, upload-time = "2025-03-02T11:10:36.298Z" }, + { url = "https://files.pythonhosted.org/packages/46/61/efc03bf48590681839f7391696c51d6d304f4d5df7f47828c373dc657c3c/pybase64-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9117f9be7f9a190e245dd7045b760b775d0b11ccc4414925cf725cdee807d5f6", size = 36414, upload-time = "2025-03-02T11:10:38.046Z" }, + { url = "https://files.pythonhosted.org/packages/55/b1/c6edc2630e4e574f681f60e2b00e7b852e7127f37603e440d28d21a2ea67/pybase64-1.4.1-cp310-cp310-win_arm64.whl", hash = "sha256:aa4232a7082cca16db5de64f30056702d2d4ee4a5da1e2bbf9fd59bd3a67baed", size = 29637, upload-time = "2025-03-02T11:10:39.9Z" }, + { url = "https://files.pythonhosted.org/packages/ff/74/6f60bddbc6badd9a821e590f960fcf55b2008842b724552e062273d2f3a2/pybase64-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a230b64474f02075608d81fc19073c86cb4e63111d5c94f8bf77a3f2c0569956", size = 38068, upload-time = "2025-03-02T11:10:41.74Z" }, + { url = "https://files.pythonhosted.org/packages/0e/ce/1e56414745cb92ed0b22fd640af1d559d8161c28d26e288da7bcd2836f93/pybase64-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26ebcd7ccadde46ab35b16fee6f3b9478142833a164e10040b942ad5ccc8c4c0", size = 31485, upload-time = "2025-03-02T11:10:42.943Z" }, + { url = "https://files.pythonhosted.org/packages/96/38/f561708ec3740ac7f0395122672d663cc525295a1021a0b9c16aba19115b/pybase64-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f033501b08bbfc89a725f9a283b485348df2cb7acb8c41ca52ccfa76785d9343", size = 59642, upload-time = "2025-03-02T11:10:44.016Z" }, + { url = "https://files.pythonhosted.org/packages/43/70/71ed3d6d8905079668e75c6eeaa2e5c6fd4c33b0f8d4672e9ec99bb4925a/pybase64-1.4.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6634d77e2f4b559daf30234f2dc679de9de3ba88effbdc0354a68b3aa2d29d3", size = 56464, upload-time = "2025-03-02T11:10:45.116Z" }, + { url = "https://files.pythonhosted.org/packages/60/53/1558b2d756896f15ea6396e2791bb710a9f289a3e2a24db5bfcf203d54e6/pybase64-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e1837488c7aa9bc7ba7bb0449908e57ecfe444e3c7347a905a87450c7e523e00", size = 59197, upload-time = "2025-03-02T11:10:47.009Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ae/300cb522d7f7eb543165843d28db4046909a8aabe110afa50cdab0947c9d/pybase64-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80e85e5ca298d3a9916c47e6fb0c47ebe5bf7996eac6983c887027b378e9bcae", size = 59803, upload-time = "2025-03-02T11:10:48.163Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b4/355f03c656bb331e623466bc6be4307efd2c41cfe58fdbf869cfb126a70c/pybase64-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:480c0c444eb07e4855d2eeab3f91a70331b75862d7a3dce0e6d4caddbfb4c09b", size = 68444, upload-time = "2025-03-02T11:10:49.32Z" }, + { url = "https://files.pythonhosted.org/packages/6f/4b/8d0730e9507026e05a7e34daddcac3d548cf8ce51cda858d033b142fed4d/pybase64-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97e25723ecf7c439f650192d43699aab0a22850dca9cc6d60377c42bb4df7812", size = 71184, upload-time = "2025-03-02T11:10:51.147Z" }, + { url = "https://files.pythonhosted.org/packages/53/95/4e7cda0cd38e5e38697fcb62ede30c42ed8f5a2427adc73296d2746ec12c/pybase64-1.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:82efee94d6bd93f7787afc42f260fa0b60e24c8dc7f172bd45cfe99fa39567ff", size = 58479, upload-time = "2025-03-02T11:10:52.908Z" }, + { url = "https://files.pythonhosted.org/packages/26/ed/cac0892746795de07b2e71f48e651af597ccb8b52ba36ac2afaa07e7da55/pybase64-1.4.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c15765be7921914d0dad0a2fb57c35a1811e1cbe2d1e47c39e0c66ed7db52898", size = 52148, upload-time = "2025-03-02T11:10:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ca/8eaae3ee3c0e7b8a827c00ca5d850a9188e0cab9575764ae3638cce6ff78/pybase64-1.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d1dcddfa521fb6cbab0385032d43f0ca13212459abd6efc381b6e9847e9fbd79", size = 68801, upload-time = "2025-03-02T11:10:55.416Z" }, + { url = "https://files.pythonhosted.org/packages/c7/55/a847b02b2c17a6353e7156f995a44bdd26b326332851fb35ee3a5dfedf82/pybase64-1.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd1de051b9b032d84e799af498b44499e90122a095da7dad89c2873518473c67", size = 57857, upload-time = "2025-03-02T11:10:56.607Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6d/7562e73ab1dbf7d735e1a2da6be06a4bdb3bb8ddfecf3c29f25288528bb7/pybase64-1.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bf8213e6b8c658df2971c5a56df42202d7f89d5d6312d066d49923cc98a39299", size = 56075, upload-time = "2025-03-02T11:10:57.796Z" }, + { url = "https://files.pythonhosted.org/packages/99/a4/795935ad7ef2d066c082a9c852b8dd658f2c61a2de1742b46c576665edd5/pybase64-1.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7d83ab7822da5740f1d17c72fb451e9468e72976b89cfb9eb4f6a5b66491b5dc", size = 70710, upload-time = "2025-03-02T11:10:58.947Z" }, + { url = "https://files.pythonhosted.org/packages/13/16/b487ba1382fca5451cb18552333999a52c47d5e561d41b1ba17bf3bbf407/pybase64-1.4.1-cp311-cp311-win32.whl", hash = "sha256:7726e655134132dde59bddabcd74d140f818eeecc70d149267267d5e29335193", size = 34200, upload-time = "2025-03-02T11:11:00.841Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a4/354cfd978a145cbeacba73f70266687f3dd34e1df1cdeb882c23153697a3/pybase64-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:9d5202cd4a8a0cd1b28c11730cf5da3c014450ad03732b5da03fac89b7693ec2", size = 36417, upload-time = "2025-03-02T11:11:02.006Z" }, + { url = "https://files.pythonhosted.org/packages/19/6c/5a576f95c79aa28a4b476ec84afe751ac0cab23572d9fd000b93adab6c76/pybase64-1.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:72808de9aab43112deb04003e5e0d060c7cb1a60c3dcf74bbf61a9d7c596c5af", size = 29638, upload-time = "2025-03-02T11:11:03.635Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a9/43bac4f39401f7241d233ddaf9e6561860b2466798cfb83b9e7dbf89bc1b/pybase64-1.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bbdcf77e424c91389f22bf10158851ce05c602c50a74ccf5943ee3f5ef4ba489", size = 38152, upload-time = "2025-03-02T11:11:07.576Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/d0ae801e31a5052dbb1744a45318f822078dd4ce4cc7f49bfe97e7768f7e/pybase64-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af41e2e6015f980d15eae0df0c365df94c7587790aea236ba0bf48c65a9fa04e", size = 31488, upload-time = "2025-03-02T11:11:09.758Z" }, + { url = "https://files.pythonhosted.org/packages/be/34/bf4119a88b2ad0536a8ed9d66ce4d70ff8152eac00ef8a27e5ae35da4328/pybase64-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ac21c1943a15552347305943b1d0d6298fb64a98b67c750cb8fb2c190cdefd4", size = 59734, upload-time = "2025-03-02T11:11:11.493Z" }, + { url = "https://files.pythonhosted.org/packages/99/1c/1901547adc7d4f24bdcb2f75cb7dcd3975bff42f39da37d4bd218c608c60/pybase64-1.4.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:65567e8f4f31cf6e1a8cc570723cc6b18adda79b4387a18f8d93c157ff5f1979", size = 56529, upload-time = "2025-03-02T11:11:12.657Z" }, + { url = "https://files.pythonhosted.org/packages/c5/1e/1993e4b9a03e94fc53552285e3998079d864fff332798bf30c25afdac8f3/pybase64-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:988e987f8cfe2dfde7475baf5f12f82b2f454841aef3a174b694a57a92d5dfb0", size = 59114, upload-time = "2025-03-02T11:11:13.972Z" }, + { url = "https://files.pythonhosted.org/packages/c5/f6/061fee5b7ba38b8824dd95752ab7115cf183ffbd3330d5fc1734a47b0f9e/pybase64-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:92b2305ac2442b451e19d42c4650c3bb090d6aa9abd87c0c4d700267d8fa96b1", size = 60095, upload-time = "2025-03-02T11:11:15.182Z" }, + { url = "https://files.pythonhosted.org/packages/37/da/ccfe5d1a9f1188cd703390522e96a31045c5b93af84df04a98e69ada5c8b/pybase64-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1ff80e03357b09dab016f41b4c75cf06e9b19cda7f898e4f3681028a3dff29b", size = 68431, upload-time = "2025-03-02T11:11:17.059Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d3/8ca4b0695876b52c0073a3557a65850b6d5c723333b5a271ab10a1085852/pybase64-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cdda297e668e118f6b9ba804e858ff49e3dd945d01fdd147de90445fd08927d", size = 71417, upload-time = "2025-03-02T11:11:19.178Z" }, + { url = "https://files.pythonhosted.org/packages/94/34/5f8f72d1b7b4ddb64c48d60160f3f4f03cfd0bfd2e7068d4558499d948ed/pybase64-1.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51a24d21a21a959eb8884f24346a6480c4bd624aa7976c9761504d847a2f9364", size = 58429, upload-time = "2025-03-02T11:11:20.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/b7/edf53af308c6e8aada1e6d6a0a3789176af8cbae37a2ce084eb9da87bf33/pybase64-1.4.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b19e169ea1b8a15a03d3a379116eb7b17740803e89bc6eb3efcc74f532323cf7", size = 52228, upload-time = "2025-03-02T11:11:21.632Z" }, + { url = "https://files.pythonhosted.org/packages/0c/bf/c9df141e24a259f38a38bdda5a3b63206f13e612ecbd3880fa10625e0294/pybase64-1.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8a9f1b614efd41240c9bb2cf66031aa7a2c3c092c928f9d429511fe18d4a3fd1", size = 68632, upload-time = "2025-03-02T11:11:23.56Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ae/1aec72325a3c48f7776cc55a3bab8b168eb77aea821253da8b9f09713734/pybase64-1.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d9947b5e289e2c5b018ddc2aee2b9ed137b8aaaba7edfcb73623e576a2407740", size = 57682, upload-time = "2025-03-02T11:11:25.656Z" }, + { url = "https://files.pythonhosted.org/packages/4d/7a/7ad2799c0b3c4e2f7b993e1636468445c30870ca5485110b589b8921808d/pybase64-1.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ba4184ea43aa88a5ab8d6d15db284689765c7487ff3810764d8d823b545158e6", size = 56308, upload-time = "2025-03-02T11:11:26.803Z" }, + { url = "https://files.pythonhosted.org/packages/be/01/6008a4fbda0c4308dab00b95aedde8748032d7620bd95b686619c66917fe/pybase64-1.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4471257628785296efb2d50077fb9dfdbd4d2732c3487795224dd2644216fb07", size = 70784, upload-time = "2025-03-02T11:11:28.427Z" }, + { url = "https://files.pythonhosted.org/packages/27/31/913365a4f0e2922ec369ddaa3a1d6c11059acbe54531b003653efa007a48/pybase64-1.4.1-cp312-cp312-win32.whl", hash = "sha256:614561297ad14de315dd27381fd6ec3ea4de0d8206ba4c7678449afaff8a2009", size = 34271, upload-time = "2025-03-02T11:11:30.585Z" }, + { url = "https://files.pythonhosted.org/packages/d9/98/4d514d3e4c04819d80bccf9ea7b30d1cfc701832fa5ffca168f585004488/pybase64-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:35635db0d64fcbe9b3fad265314c052c47dc9bcef8dea17493ea8e3c15b2b972", size = 36496, upload-time = "2025-03-02T11:11:32.552Z" }, + { url = "https://files.pythonhosted.org/packages/c4/61/01353bc9c461e7b36d692daca3eee9616d8936ea6d8a64255ef7ec9ac307/pybase64-1.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:b4ccb438c4208ff41a260b70994c30a8631051f3b025cdca48be586b068b8f49", size = 29692, upload-time = "2025-03-02T11:11:33.735Z" }, + { url = "https://files.pythonhosted.org/packages/4b/1a/4e243ba702c07df3df3ba1795cfb02cf7a4242c53fc574b06a2bfa4f8478/pybase64-1.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1c38d9c4a7c132d45859af8d5364d3ce90975a42bd5995d18d174fb57621973", size = 38149, upload-time = "2025-03-02T11:11:35.537Z" }, + { url = "https://files.pythonhosted.org/packages/9c/35/3eae81bc8688a83f8b5bb84979d88e2cc3c3279a3b870a506f277d746c56/pybase64-1.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ab0b93ea93cf1f56ca4727d678a9c0144c2653e9de4e93e789a92b4e098c07d9", size = 31485, upload-time = "2025-03-02T11:11:36.656Z" }, + { url = "https://files.pythonhosted.org/packages/48/55/d99b9ff8083573bbf97fc433bbc20e2efb612792025f3bad0868c96c37ce/pybase64-1.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:644f393e9bb7f3bacc5cbd3534d02e1b660b258fc8315ecae74d2e23265e5c1f", size = 59738, upload-time = "2025-03-02T11:11:38.468Z" }, + { url = "https://files.pythonhosted.org/packages/63/3c/051512b9e139a11585447b286ede5ac3b284ce5df85de37eb8cff57d90f8/pybase64-1.4.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff172a4dacbd964e5edcf1c2152dae157aabf856508aed15276f46d04a22128e", size = 56239, upload-time = "2025-03-02T11:11:39.718Z" }, + { url = "https://files.pythonhosted.org/packages/af/11/f40c5cca587274d50baee88540a7839576204cb425fe2f73a752ea48ae74/pybase64-1.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2ab7b4535abc72d40114540cae32c9e07d76ffba132bdd5d4fff5fe340c5801", size = 59137, upload-time = "2025-03-02T11:11:41.524Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a9/ace9f6d0926962c083671d7df247de442ef63cd06bd134f7c8251aab5c51/pybase64-1.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da66eb7cfb641486944fb0b95ab138e691ab78503115022caf992b6c89b10396", size = 60109, upload-time = "2025-03-02T11:11:42.699Z" }, + { url = "https://files.pythonhosted.org/packages/88/9c/d4e308b4b4e3b513bc084fc71b4e2dd00d21d4cd245a9a28144d2f6b03c9/pybase64-1.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:678f573ea1d06183b32d0336044fb5db60396333599dffcce28ffa3b68319fc0", size = 68391, upload-time = "2025-03-02T11:11:43.898Z" }, + { url = "https://files.pythonhosted.org/packages/53/87/e184bf982a3272f1021f417e5a18fac406e042c606950e9082fc3b0cec30/pybase64-1.4.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bccdf340c2a1d3dd1f41528f192265ddce7f8df1ee4f7b5b9163cdba0fe0ccb", size = 71438, upload-time = "2025-03-02T11:11:45.112Z" }, + { url = "https://files.pythonhosted.org/packages/2f/7f/d6e6a72db055eb2dc01ab877d8ee39d05cb665403433ff922fb95d1003ad/pybase64-1.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1ddf6366c34eb78931fd8a47c00cb886ba187a5ff8e6dbffe1d9dae4754b6c28", size = 58437, upload-time = "2025-03-02T11:11:47.034Z" }, + { url = "https://files.pythonhosted.org/packages/71/ef/c9051f2c0128194b861f3cd3b2d211b8d4d21ed2be354aa669fe29a059d8/pybase64-1.4.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:500afcb717a84e262c68f0baf9c56abaf97e2f058ba80c5546a9ed21ff4b705f", size = 52267, upload-time = "2025-03-02T11:11:48.448Z" }, + { url = "https://files.pythonhosted.org/packages/12/92/ae30a54eaa437989839c4f2404c1f004d7383c0f46d6ebb83546d587d2a7/pybase64-1.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d2de043312a1e7f15ee6d2b7d9e39ee6afe24f144e2248cce942b6be357b70d8", size = 68659, upload-time = "2025-03-02T11:11:49.615Z" }, + { url = "https://files.pythonhosted.org/packages/2b/65/d94788a35904f21694c4c581bcee2e165bec2408cc6fbed85a7fef5959ae/pybase64-1.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c36e214c25fb8dd4f3ecdaa0ff90073b793056e0065cc0a1e1e5525a6866a1ad", size = 57727, upload-time = "2025-03-02T11:11:50.843Z" }, + { url = "https://files.pythonhosted.org/packages/d0/97/8db416066b7917909c38346c03a8f3e6d4fc8a1dc98636408156514269ad/pybase64-1.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8ec003224f6e36e8e607a1bb8df182b367c87ca7135788ffe89173c7d5085005", size = 56302, upload-time = "2025-03-02T11:11:52.547Z" }, + { url = "https://files.pythonhosted.org/packages/70/0b/98f0601391befe0f19aa8cbda821c62d95056a94cc41d452fe893d205523/pybase64-1.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c536c6ed161e6fb19f6acd6074f29a4c78cb41c9155c841d56aec1a4d20d5894", size = 70779, upload-time = "2025-03-02T11:11:53.735Z" }, + { url = "https://files.pythonhosted.org/packages/cc/07/116119c5b20688c052697f677cf56f05aa766535ff7691aba38447d4a0d8/pybase64-1.4.1-cp313-cp313-win32.whl", hash = "sha256:1d34872e5aa2eff9dc54cedaf36038bbfbd5a3440fdf0bdc5b3c81c54ef151ea", size = 34266, upload-time = "2025-03-02T11:11:54.892Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f5/a7eed9f3692209a9869a28bdd92deddf8cbffb06b40954f89f4577e5c96e/pybase64-1.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8b7765515d7e0a48ddfde914dc2b1782234ac188ce3fab173b078a6e82ec7017", size = 36488, upload-time = "2025-03-02T11:11:56.063Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8a/0d65c4dcda06487305035f24888ffed219897c03fb7834635d5d5e27dae1/pybase64-1.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:7fb782f3ceb30e24dc4d8d99c1221a381917bffaf85d29542f0f25b51829987c", size = 29690, upload-time = "2025-03-02T11:11:57.702Z" }, + { url = "https://files.pythonhosted.org/packages/a3/83/646d65fafe5e6edbdaf4c9548efb2e1dd7784caddbde3ff8a843dd942b0f/pybase64-1.4.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2a98d323e97444a38db38e022ccaf1d3e053b1942455790a93f29086c687855f", size = 38506, upload-time = "2025-03-02T11:11:58.936Z" }, + { url = "https://files.pythonhosted.org/packages/87/14/dbf7fbbe91d71c8044fefe20d22480ad64097e2ba424944de512550e12a4/pybase64-1.4.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:19ef58d36b9b32024768fcedb024f32c05eb464128c75c07cac2b50c9ed47f4a", size = 31894, upload-time = "2025-03-02T11:12:00.762Z" }, + { url = "https://files.pythonhosted.org/packages/bd/5d/f8a47da2a5f8b599297b307d3bd0293adedc4e135be310620f061906070f/pybase64-1.4.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04fee0f5c174212868fde97b109db8fac8249b306a00ea323531ee61c7b0f398", size = 65212, upload-time = "2025-03-02T11:12:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/90/95/ad9869c7cdcce3e8ada619dab5f9f2eff315ffb001704a3718c1597a2119/pybase64-1.4.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47737ff9eabc14b7553de6bc6395d67c5be80afcdbd25180285d13e089e40888", size = 60300, upload-time = "2025-03-02T11:12:03.071Z" }, + { url = "https://files.pythonhosted.org/packages/c2/91/4d8268b2488ae10c485cba04ecc23a5a7bdfb47ce9b876017b11ea0249a2/pybase64-1.4.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0d8b5888cc239654fe68a0db196a18575ffc8b1c8c8f670c2971a44e3b7fe682", size = 63773, upload-time = "2025-03-02T11:12:04.231Z" }, + { url = "https://files.pythonhosted.org/packages/ae/1a/8afd27facc0723b1d69231da8c59a2343feb255f5db16f8b8765ddf1600b/pybase64-1.4.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a1af8d387dbce05944b65a618639918804b2d4438fed32bb7f06d9c90dbed01", size = 64684, upload-time = "2025-03-02T11:12:05.409Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cd/422c74397210051125419fc8e425506ff27c04665459e18c8f7b037a754b/pybase64-1.4.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b0093c52bd099b80e422ad8cddf6f2c1ac1b09cb0922cca04891d736c2ad647", size = 72880, upload-time = "2025-03-02T11:12:06.652Z" }, + { url = "https://files.pythonhosted.org/packages/04/c1/c4f02f1d5f8e8a3d75715a3dd04196dde9e263e471470d099a26e91ebe2f/pybase64-1.4.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15e54f9b2a1686f5bbdc4ac8440b6f6145d9699fd53aa30f347931f3063b0915", size = 75344, upload-time = "2025-03-02T11:12:07.816Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0b/013006ca984f0472476cf7c0540db2e2b1f997d52977b15842a7681ab79c/pybase64-1.4.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:3a0fdcf13f986c82f7ef04a1cd1163c70f39662d6f02aa4e7b448dacb966b39f", size = 63439, upload-time = "2025-03-02T11:12:09.669Z" }, + { url = "https://files.pythonhosted.org/packages/8a/d5/7848543b3c8dcc5396be574109acbe16706e6a9b4dbd9fc4e22f211668a9/pybase64-1.4.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:ac03f8eba72dd6da15dc25bb3e1b440ad21f5cb7ee2e6ffbbae4bd1b206bb503", size = 56004, upload-time = "2025-03-02T11:12:10.981Z" }, + { url = "https://files.pythonhosted.org/packages/63/58/70de1efb1b6f21d7aaea33578868214f82925d969e2091f7de3175a10092/pybase64-1.4.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:ea835272570aa811e08ae17612632b057623a9b27265d44288db666c02b438dc", size = 72460, upload-time = "2025-03-02T11:12:13.122Z" }, + { url = "https://files.pythonhosted.org/packages/90/0d/aa52dd1b1f25b98b1d94cc0522f864b03de55aa115de67cb6dbbddec4f46/pybase64-1.4.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:8f52c4c29a35381f3ae06d520144a0707132f2cbfb53bc907b74811734bc4ef3", size = 62295, upload-time = "2025-03-02T11:12:15.004Z" }, + { url = "https://files.pythonhosted.org/packages/39/cf/4d378a330249c937676ee8eab7992ec700ade362f35db36c15922b33b1c8/pybase64-1.4.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:fa5cdabcb4d21b7e56d0b2edd7ed6fa933ac3535be30c2a9cf0a2e270c5369c8", size = 60604, upload-time = "2025-03-02T11:12:16.23Z" }, + { url = "https://files.pythonhosted.org/packages/15/45/e3f23929018d0aada84246ddd398843050971af614da67450bb20f45f880/pybase64-1.4.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8db9acf239bb71a888748bc9ffc12c97c1079393a38bc180c0548330746ece94", size = 74500, upload-time = "2025-03-02T11:12:17.48Z" }, + { url = "https://files.pythonhosted.org/packages/8d/98/6d2adaec318cae6ee968a10df0a7e870f17ee385ef623bcb2ab63fa11b59/pybase64-1.4.1-cp313-cp313t-win32.whl", hash = "sha256:bc06186cfa9a43e871fdca47c1379bdf1cfe964bd94a47f0919a1ffab195b39e", size = 34543, upload-time = "2025-03-02T11:12:18.625Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e7/1823de02d2c23324cf1142e9dce53b032085cee06c3f982806040f975ce7/pybase64-1.4.1-cp313-cp313t-win_amd64.whl", hash = "sha256:02c3647d270af1a3edd35e485bb7ccfe82180b8347c49e09973466165c03d7aa", size = 36909, upload-time = "2025-03-02T11:12:20.122Z" }, + { url = "https://files.pythonhosted.org/packages/43/6a/8ec0e4461bf89ef0499ef6c746b081f3520a1e710aeb58730bae693e0681/pybase64-1.4.1-cp313-cp313t-win_arm64.whl", hash = "sha256:4b3635e5873707906e72963c447a67969cfc6bac055432a57a91d7a4d5164fdf", size = 29961, upload-time = "2025-03-02T11:12:21.908Z" }, + { url = "https://files.pythonhosted.org/packages/e9/e0/e24db103af867f957a11a7c05cf40a14c1936f0dca71c2bc24fe2b458d05/pybase64-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c226a24e4ab8eb351b1e979aca91590742515a7069347a9fe7deae31cab9442", size = 38068, upload-time = "2025-03-02T11:12:55.576Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8b/6681990f548530225b84188c9c7db96f92568513be5f071085ae90791197/pybase64-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e0ea46295faf5951e0bcc0859be015e9630cdc854c40dc3c5d8401da1eeb6e84", size = 31480, upload-time = "2025-03-02T11:12:56.703Z" }, + { url = "https://files.pythonhosted.org/packages/a1/af/955755800fc26331c66e42d0d5b816152efca57545e79c751c49236193d5/pybase64-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78165489e1026b80d3914488de51d28b247d9c75dbf8f2d0bf81c88d1636eb81", size = 57155, upload-time = "2025-03-02T11:12:57.994Z" }, + { url = "https://files.pythonhosted.org/packages/24/1d/c6d29b729d0231b291223946913254c72b9bb73f8039c05f097066bc0ef1/pybase64-1.4.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77339b232fbaf7f6ecbfb8a31aec25f3eeca8bc938188180c730d2084e4a246a", size = 54113, upload-time = "2025-03-02T11:12:59.877Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f4/461a8a686f8835bcf7982bf7c02f27998a24ff619a223cda883ee4b73d9a/pybase64-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1cef7bb7f0a84f3ffa97f431e65924bdaa95bf1696006fd7a391aaa8aa67753", size = 56779, upload-time = "2025-03-02T11:13:02.416Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a1/762b034beac0909a82edcde132dd5e04c1190d772f8bc4fc69c05d35a70d/pybase64-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbce0df09d627ec35971aa02b14adef739be59b4c7816418d1c06c92e580d4c3", size = 57267, upload-time = "2025-03-02T11:13:04.437Z" }, + { url = "https://files.pythonhosted.org/packages/a9/64/039164164dd24c6595ff221f2311c60f0e9d80fe20a3ea29eb203af1e3fd/pybase64-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:734e3dea40a30225b53d8d341ee4308f7b0182f1a8ce3f4309575c0af07b9902", size = 65901, upload-time = "2025-03-02T11:13:06.35Z" }, + { url = "https://files.pythonhosted.org/packages/2c/da/8c0266ffeb301ff70179ef684fcd63144a1292713d124e5dfc6647e121f0/pybase64-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12987975c58f6547eff106454c252ad19b59e5a2de3c47a9efecee1a2a15aba5", size = 68490, upload-time = "2025-03-02T11:13:07.529Z" }, + { url = "https://files.pythonhosted.org/packages/02/6a/f9957ec9fa53e650452d121c192a5bc82888cfb4bd50f281a239817a117b/pybase64-1.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e45d3b174f20563878b7d745940d3a80a5c10ba556d39a5d7b9a7ed0d82c672e", size = 55884, upload-time = "2025-03-02T11:13:08.662Z" }, + { url = "https://files.pythonhosted.org/packages/b3/07/b72a532ccbde4e81bdb14fda0e3a3a554c757e287d0d22c6fce8e5871c98/pybase64-1.4.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:5dac8d885342d49f6306e666688288c50515d0743e36a4405b1413feb43f39cc", size = 49711, upload-time = "2025-03-02T11:13:10.632Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c5/7ec20dcc27d3acbd80701137824c9fd938f8ab03d6588b51c61242617883/pybase64-1.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c1b16691be4b63be973804de22b4b79e40c439e54ad9587f86f31f958b518625", size = 66179, upload-time = "2025-03-02T11:13:11.946Z" }, + { url = "https://files.pythonhosted.org/packages/4d/93/d940dccceb64e0fc7b9b8844d1a1b27c1375bd24b56f076d15ebbe634328/pybase64-1.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:4c87f0149c2c6b0c19746c72e146067275f632a495e7f2de9bbd38b2e48630ee", size = 55287, upload-time = "2025-03-02T11:13:13.137Z" }, + { url = "https://files.pythonhosted.org/packages/c6/99/4a03cf45a950131df1857ec4f19b6fb85107789dc6d6a9dc2ec95aa76fe2/pybase64-1.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:bceafd1450436dfca597958bd77cc619ed79311310b2a9271ce7a8069bdcb139", size = 53562, upload-time = "2025-03-02T11:13:15.074Z" }, + { url = "https://files.pythonhosted.org/packages/6b/26/f0c4662362c31f89bab162dc92e8a51085395f4cf000a689482505935d2e/pybase64-1.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:290adeb7844a5889decdf2424862179205dc4239f38cd0f87c5b56f87b87db99", size = 67982, upload-time = "2025-03-02T11:13:16.273Z" }, + { url = "https://files.pythonhosted.org/packages/e6/03/d0d764c8bf15429cb5651a5a567119b6e39456d9ed8a6dfae767a58a54c9/pybase64-1.4.1-cp39-cp39-win32.whl", hash = "sha256:1d8370f7930b3a8e9c8da341830898f1391a050d703f42bd2b95120664844368", size = 34216, upload-time = "2025-03-02T11:13:17.512Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7a/47332521930d17a8d411172d6aa38d89185322f8a1fc198a3361ef3a4161/pybase64-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:20e575310b2ddc8f303f9a41987dc8b4c8dc6b992567bca5eda7f1ab6cf4289b", size = 36403, upload-time = "2025-03-02T11:13:18.707Z" }, + { url = "https://files.pythonhosted.org/packages/5b/36/35b409ba9b264247401a035af822fccc2e7f358076bf9b162055d75d60ac/pybase64-1.4.1-cp39-cp39-win_arm64.whl", hash = "sha256:e6b22cbc8ec3dd26791293113b9102f9887f41865e442fb228f661a8340f9461", size = 29639, upload-time = "2025-03-02T11:13:19.858Z" }, + { url = "https://files.pythonhosted.org/packages/34/22/4fcbd6b8dcbcabe30fdcd4d5145445cffc6724a90425dda0043c1cbd4919/pybase64-1.4.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b0bdb646f859132c68230efabc09fd8828ca20c59de7d53082f372c4b8af7aaa", size = 38055, upload-time = "2025-03-02T11:13:21.751Z" }, + { url = "https://files.pythonhosted.org/packages/16/d8/9a6c325c31c81897349c83bd4857f09f78d342bb03f0107df5ab9de0de1a/pybase64-1.4.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8d4bf9c94bc948cb3c3b0e38074d0de04f23d35765a306059417751e982da384", size = 31354, upload-time = "2025-03-02T11:13:23.016Z" }, + { url = "https://files.pythonhosted.org/packages/72/30/4212a953d3fc4affa5ffa652096440daf1093ad6db734b17231f1f82a79a/pybase64-1.4.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b31da1466faf3cfa775027d161d07640f3d1c6bbc8edf3725f8833ed0b25a2f", size = 35265, upload-time = "2025-03-02T11:13:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/12/b4/a54e9e3eb7f11f80a659eed05b0bfa6bc68ad8e7ec075e40236c7987d18e/pybase64-1.4.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc9a3f56630e707dbe7a34383943a1daefa699bc99c3250f8af9f8245056fccd", size = 40968, upload-time = "2025-03-02T11:13:26.165Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f1/d6bc1a548edc806ce8d25b6d761d2aed68abc3162f072f984940f59ae15b/pybase64-1.4.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdabd0d7fda2517ff36559189f7c00b376feafbd5d23bf5914e256246d29d7e", size = 41117, upload-time = "2025-03-02T11:13:27.433Z" }, + { url = "https://files.pythonhosted.org/packages/9a/6c/5952201a062ac4746fc767c8556a7b933cb59295068b9dba0bcba8bde378/pybase64-1.4.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:62e42807bde3a7d18a0a7d35bd7fb1fe68f99c897eea8d3ea3aa0791b91358eb", size = 36804, upload-time = "2025-03-02T11:13:29.414Z" }, + { url = "https://files.pythonhosted.org/packages/a8/3e/90633da698742bfd11a1d6301295e9974c2f9e0e510aaae8cdd26cd10880/pybase64-1.4.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e8c28700ccf55348a7a4ad3554e6b4c5b83c640bfaa272fee6b4d0030566fe05", size = 38056, upload-time = "2025-03-02T11:13:30.6Z" }, + { url = "https://files.pythonhosted.org/packages/b1/02/79bdf96a780c3d1f4e9f1b583525247f3a33afebbba1e12e57fb28c395e7/pybase64-1.4.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eb09bd829d4fef567505212b6bb87cd7a42b5aa2a3b83fc2bd61a188db7793e0", size = 31352, upload-time = "2025-03-02T11:13:32.395Z" }, + { url = "https://files.pythonhosted.org/packages/ce/d0/4f8135c2459724a834a70481f6bb8af3e89ff527c9b5cff0b799321e29d6/pybase64-1.4.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc9504c4c2e893e0a6c1cc80bce51907e3461288289f630eab22b5735eba1104", size = 35262, upload-time = "2025-03-02T11:13:33.55Z" }, + { url = "https://files.pythonhosted.org/packages/21/c6/45ace9c84ccc9d51002c5bcfe8c50e7660f064e2bc272a30c7802036f1f3/pybase64-1.4.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45a785a3d29faf0309910d96e13c34870adb4ae43ea262868c6cf6a311936f37", size = 40968, upload-time = "2025-03-02T11:13:34.748Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d5/1bf0b5354ca404ba096e99e2634c27836c212affe722bd2ade7103fd3c48/pybase64-1.4.1-pp311-pypy311_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10e2cb40869fe703484ba89ae50e05d63a169f7c42db59e29f8af0890c50515d", size = 41107, upload-time = "2025-03-02T11:13:35.996Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d7/0987f3d1c8196ad9affea9102c135a45342e1fa5affb849bf31bd633d000/pybase64-1.4.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1a18644fb3e940ed622738f2ee14d9a2811bb542ffd3f85c3fb661130675ac4f", size = 36817, upload-time = "2025-03-02T11:13:37.624Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a3/a163cde0fd600c7c96bfd7b5133e1d6bd368d512951a5986870e21b6f8de/pybase64-1.4.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:eda1a04db3c3a5f9a8f902a3d537bac4bbc91f2f93a7e5cb4396ec50e16899d5", size = 38044, upload-time = "2025-03-02T11:13:47.921Z" }, + { url = "https://files.pythonhosted.org/packages/ff/e8/ea79c77c0b43d587bf8f5d2c41bbc413f0537c1eff9e7955503308900340/pybase64-1.4.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a306cb9ae5a6361e094e5617454dd26d19c896ccfc67d0357d96b96c5197547a", size = 31344, upload-time = "2025-03-02T11:13:49.124Z" }, + { url = "https://files.pythonhosted.org/packages/0e/aa/0aa7b788ba5fcf66c49c895c5adae11896c5e37caaf45637203504e70bf1/pybase64-1.4.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06d4d29312746e56a89ffc7cf797e8d1c3dfc4d0ab9cf883bb3f7267a7c74b25", size = 35260, upload-time = "2025-03-02T11:13:50.302Z" }, + { url = "https://files.pythonhosted.org/packages/c4/06/867e6680cb246e2631a73c543134f5aae7d2922a29c615511f88fa951271/pybase64-1.4.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f73a1ac604accfff484f88786197822b4b8b9c727d10854d9475704707c267f8", size = 40960, upload-time = "2025-03-02T11:13:52.83Z" }, + { url = "https://files.pythonhosted.org/packages/c3/01/3a90d4ea2da403a0f247423bc13e1140db5a3dd850f423b0e94b8e2c397b/pybase64-1.4.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:011a54ff6ca44c5d03746aec3f1f492fce3155bd3f943fb2ceaea92416d40eeb", size = 41113, upload-time = "2025-03-02T11:13:54.628Z" }, + { url = "https://files.pythonhosted.org/packages/b1/bd/51902af191ed28bc0f5eb29a630bd40f68640ed7a24b3eb7aa1630d78cf0/pybase64-1.4.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a20cff09b13cb8b72b35a9dd12173a7e3bd8e54efb9a708680014562ba47c648", size = 36784, upload-time = "2025-03-02T11:13:55.821Z" }, +] + [[package]] name = "pycountry" version = "24.6.1" @@ -4545,6 +4554,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/af/a3eb4449c8fdde24413555a66e9c100b669f4428fc829bad4ceb73472f4f/transformers-4.51.2-py3-none-any.whl", hash = "sha256:5cb8259098b75ff4b5dd04533a318f7c4750d5307d9617e6d0593526432c404d", size = 10366692, upload-time = "2025-04-10T16:00:10.287Z" }, ] +[package.optional-dependencies] +sentencepiece = [ + { name = "protobuf", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, + { name = "sentencepiece", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, +] + [[package]] name = "triton" version = "3.1.0" @@ -4729,7 +4744,7 @@ wheels = [ [[package]] name = "vllm" -version = "0.9.0.1" +version = "0.9.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp", marker = "python_full_version >= '3.10'" }, @@ -4753,10 +4768,6 @@ dependencies = [ { name = "numpy", marker = "python_full_version >= '3.10'" }, { name = "openai", marker = "python_full_version >= '3.10'" }, { name = "opencv-python-headless", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-api", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-exporter-otlp", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-sdk", marker = "python_full_version >= '3.10'" }, - { name = "opentelemetry-semantic-conventions-ai", marker = "python_full_version >= '3.10'" }, { name = "outlines", marker = "python_full_version >= '3.10'" }, { name = "partial-json-parser", marker = "python_full_version >= '3.10'" }, { name = "pillow", marker = "python_full_version >= '3.10'" }, @@ -4765,6 +4776,7 @@ dependencies = [ { name = "protobuf", marker = "python_full_version >= '3.10'" }, { name = "psutil", marker = "python_full_version >= '3.10'" }, { name = "py-cpuinfo", marker = "python_full_version >= '3.10'" }, + { name = "pybase64", marker = "python_full_version >= '3.10'" }, { name = "pydantic", marker = "python_full_version >= '3.10'" }, { name = "python-json-logger", marker = "python_full_version >= '3.10'" }, { name = "pyyaml", marker = "python_full_version >= '3.10'" }, @@ -4786,11 +4798,11 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version >= '3.10'" }, { name = "watchfiles", marker = "python_full_version >= '3.10'" }, { name = "xformers", marker = "python_full_version >= '3.10' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "xgrammar", marker = "(python_full_version >= '3.10' and platform_machine == 'aarch64') or (python_full_version >= '3.10' and platform_machine == 'x86_64')" }, + { name = "xgrammar", marker = "(python_full_version >= '3.10' and platform_machine == 'aarch64') or (python_full_version >= '3.10' and platform_machine == 'arm64') or (python_full_version >= '3.10' and platform_machine == 'x86_64')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/e8/abe874ca849a6bb06b4b8290b47784d6c93fa101c0ca5e9d96ed4ac47d2c/vllm-0.9.0.1.tar.gz", hash = "sha256:a1b4e9a832241f981c0b2cbdc1daca71d3ade32f083ec6dcb0ead58a882e9fca", size = 8552044, upload-time = "2025-05-30T20:16:56.49Z" } +sdist = { url = "https://files.pythonhosted.org/packages/35/89/2fbf95d398b5751b44c7256bd80e57c589142f1bfcc15f5dc76438b8853a/vllm-0.9.2.tar.gz", hash = "sha256:6b0d855ea8ba18d76364c9b82ea94bfcaa9c9e724055438b5733e4716ed104e1", size = 8997087, upload-time = "2025-07-08T04:49:01.722Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/ca/6ecb087415b9e6251ff3694e59c23c77880270a5c545184b192f35f73484/vllm-0.9.0.1-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:b581df16f68f871773cf57fe8cc7737808a8745f94971e691b4113ba3b76c304", size = 377193539, upload-time = "2025-05-30T20:16:47.698Z" }, + { url = "https://files.pythonhosted.org/packages/f4/72/c14ff1acac64294f45782769b9c8144a1c3e8d4f2228d4648197511b015a/vllm-0.9.2-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:f3c5da29a286f4933b480a5b4749fab226564f35c96928eeef547f88d385cd34", size = 383350132, upload-time = "2025-07-08T04:48:54.133Z" }, ] [[package]] @@ -4830,7 +4842,7 @@ lint = [ requires-dist = [ { name = "fms-model-optimizer", specifier = ">=0.2.0" }, { name = "ibm-fms", specifier = "==1.1.0" }, - { name = "vllm", specifier = ">=0.9.0,!=0.9.1" }, + { name = "vllm", specifier = ">=0.9.2" }, ] [package.metadata.requires-dev] @@ -5062,81 +5074,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" }, ] -[[package]] -name = "wrapt" -version = "1.17.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/fc/e91cc220803d7bc4db93fb02facd8461c37364151b8494762cc88b0fbcef/wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3", size = 55531, upload-time = "2025-01-14T10:35:45.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/d1/1daec934997e8b160040c78d7b31789f19b122110a75eca3d4e8da0049e1/wrapt-1.17.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d57c572081fed831ad2d26fd430d565b76aa277ed1d30ff4d40670b1c0dd984", size = 53307, upload-time = "2025-01-14T10:33:13.616Z" }, - { url = "https://files.pythonhosted.org/packages/1b/7b/13369d42651b809389c1a7153baa01d9700430576c81a2f5c5e460df0ed9/wrapt-1.17.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5e251054542ae57ac7f3fba5d10bfff615b6c2fb09abeb37d2f1463f841ae22", size = 38486, upload-time = "2025-01-14T10:33:15.947Z" }, - { url = "https://files.pythonhosted.org/packages/62/bf/e0105016f907c30b4bd9e377867c48c34dc9c6c0c104556c9c9126bd89ed/wrapt-1.17.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80dd7db6a7cb57ffbc279c4394246414ec99537ae81ffd702443335a61dbf3a7", size = 38777, upload-time = "2025-01-14T10:33:17.462Z" }, - { url = "https://files.pythonhosted.org/packages/27/70/0f6e0679845cbf8b165e027d43402a55494779295c4b08414097b258ac87/wrapt-1.17.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a6e821770cf99cc586d33833b2ff32faebdbe886bd6322395606cf55153246c", size = 83314, upload-time = "2025-01-14T10:33:21.282Z" }, - { url = "https://files.pythonhosted.org/packages/0f/77/0576d841bf84af8579124a93d216f55d6f74374e4445264cb378a6ed33eb/wrapt-1.17.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b60fb58b90c6d63779cb0c0c54eeb38941bae3ecf7a73c764c52c88c2dcb9d72", size = 74947, upload-time = "2025-01-14T10:33:24.414Z" }, - { url = "https://files.pythonhosted.org/packages/90/ec/00759565518f268ed707dcc40f7eeec38637d46b098a1f5143bff488fe97/wrapt-1.17.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b870b5df5b71d8c3359d21be8f0d6c485fa0ebdb6477dda51a1ea54a9b558061", size = 82778, upload-time = "2025-01-14T10:33:26.152Z" }, - { url = "https://files.pythonhosted.org/packages/f8/5a/7cffd26b1c607b0b0c8a9ca9d75757ad7620c9c0a9b4a25d3f8a1480fafc/wrapt-1.17.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4011d137b9955791f9084749cba9a367c68d50ab8d11d64c50ba1688c9b457f2", size = 81716, upload-time = "2025-01-14T10:33:27.372Z" }, - { url = "https://files.pythonhosted.org/packages/7e/09/dccf68fa98e862df7e6a60a61d43d644b7d095a5fc36dbb591bbd4a1c7b2/wrapt-1.17.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1473400e5b2733e58b396a04eb7f35f541e1fb976d0c0724d0223dd607e0f74c", size = 74548, upload-time = "2025-01-14T10:33:28.52Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8e/067021fa3c8814952c5e228d916963c1115b983e21393289de15128e867e/wrapt-1.17.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3cedbfa9c940fdad3e6e941db7138e26ce8aad38ab5fe9dcfadfed9db7a54e62", size = 81334, upload-time = "2025-01-14T10:33:29.643Z" }, - { url = "https://files.pythonhosted.org/packages/4b/0d/9d4b5219ae4393f718699ca1c05f5ebc0c40d076f7e65fd48f5f693294fb/wrapt-1.17.2-cp310-cp310-win32.whl", hash = "sha256:582530701bff1dec6779efa00c516496968edd851fba224fbd86e46cc6b73563", size = 36427, upload-time = "2025-01-14T10:33:30.832Z" }, - { url = "https://files.pythonhosted.org/packages/72/6a/c5a83e8f61aec1e1aeef939807602fb880e5872371e95df2137142f5c58e/wrapt-1.17.2-cp310-cp310-win_amd64.whl", hash = "sha256:58705da316756681ad3c9c73fd15499aa4d8c69f9fd38dc8a35e06c12468582f", size = 38774, upload-time = "2025-01-14T10:33:32.897Z" }, - { url = "https://files.pythonhosted.org/packages/cd/f7/a2aab2cbc7a665efab072344a8949a71081eed1d2f451f7f7d2b966594a2/wrapt-1.17.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ff04ef6eec3eee8a5efef2401495967a916feaa353643defcc03fc74fe213b58", size = 53308, upload-time = "2025-01-14T10:33:33.992Z" }, - { url = "https://files.pythonhosted.org/packages/50/ff/149aba8365fdacef52b31a258c4dc1c57c79759c335eff0b3316a2664a64/wrapt-1.17.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4db983e7bca53819efdbd64590ee96c9213894272c776966ca6306b73e4affda", size = 38488, upload-time = "2025-01-14T10:33:35.264Z" }, - { url = "https://files.pythonhosted.org/packages/65/46/5a917ce85b5c3b490d35c02bf71aedaa9f2f63f2d15d9949cc4ba56e8ba9/wrapt-1.17.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9abc77a4ce4c6f2a3168ff34b1da9b0f311a8f1cfd694ec96b0603dff1c79438", size = 38776, upload-time = "2025-01-14T10:33:38.28Z" }, - { url = "https://files.pythonhosted.org/packages/ca/74/336c918d2915a4943501c77566db41d1bd6e9f4dbc317f356b9a244dfe83/wrapt-1.17.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b929ac182f5ace000d459c59c2c9c33047e20e935f8e39371fa6e3b85d56f4a", size = 83776, upload-time = "2025-01-14T10:33:40.678Z" }, - { url = "https://files.pythonhosted.org/packages/09/99/c0c844a5ccde0fe5761d4305485297f91d67cf2a1a824c5f282e661ec7ff/wrapt-1.17.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f09b286faeff3c750a879d336fb6d8713206fc97af3adc14def0cdd349df6000", size = 75420, upload-time = "2025-01-14T10:33:41.868Z" }, - { url = "https://files.pythonhosted.org/packages/b4/b0/9fc566b0fe08b282c850063591a756057c3247b2362b9286429ec5bf1721/wrapt-1.17.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7ed2d9d039bd41e889f6fb9364554052ca21ce823580f6a07c4ec245c1f5d6", size = 83199, upload-time = "2025-01-14T10:33:43.598Z" }, - { url = "https://files.pythonhosted.org/packages/9d/4b/71996e62d543b0a0bd95dda485219856def3347e3e9380cc0d6cf10cfb2f/wrapt-1.17.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:129a150f5c445165ff941fc02ee27df65940fcb8a22a61828b1853c98763a64b", size = 82307, upload-time = "2025-01-14T10:33:48.499Z" }, - { url = "https://files.pythonhosted.org/packages/39/35/0282c0d8789c0dc9bcc738911776c762a701f95cfe113fb8f0b40e45c2b9/wrapt-1.17.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1fb5699e4464afe5c7e65fa51d4f99e0b2eadcc176e4aa33600a3df7801d6662", size = 75025, upload-time = "2025-01-14T10:33:51.191Z" }, - { url = "https://files.pythonhosted.org/packages/4f/6d/90c9fd2c3c6fee181feecb620d95105370198b6b98a0770cba090441a828/wrapt-1.17.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9a2bce789a5ea90e51a02dfcc39e31b7f1e662bc3317979aa7e5538e3a034f72", size = 81879, upload-time = "2025-01-14T10:33:52.328Z" }, - { url = "https://files.pythonhosted.org/packages/8f/fa/9fb6e594f2ce03ef03eddbdb5f4f90acb1452221a5351116c7c4708ac865/wrapt-1.17.2-cp311-cp311-win32.whl", hash = "sha256:4afd5814270fdf6380616b321fd31435a462019d834f83c8611a0ce7484c7317", size = 36419, upload-time = "2025-01-14T10:33:53.551Z" }, - { url = "https://files.pythonhosted.org/packages/47/f8/fb1773491a253cbc123c5d5dc15c86041f746ed30416535f2a8df1f4a392/wrapt-1.17.2-cp311-cp311-win_amd64.whl", hash = "sha256:acc130bc0375999da18e3d19e5a86403667ac0c4042a094fefb7eec8ebac7cf3", size = 38773, upload-time = "2025-01-14T10:33:56.323Z" }, - { url = "https://files.pythonhosted.org/packages/a1/bd/ab55f849fd1f9a58ed7ea47f5559ff09741b25f00c191231f9f059c83949/wrapt-1.17.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d5e2439eecc762cd85e7bd37161d4714aa03a33c5ba884e26c81559817ca0925", size = 53799, upload-time = "2025-01-14T10:33:57.4Z" }, - { url = "https://files.pythonhosted.org/packages/53/18/75ddc64c3f63988f5a1d7e10fb204ffe5762bc663f8023f18ecaf31a332e/wrapt-1.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fc7cb4c1c744f8c05cd5f9438a3caa6ab94ce8344e952d7c45a8ed59dd88392", size = 38821, upload-time = "2025-01-14T10:33:59.334Z" }, - { url = "https://files.pythonhosted.org/packages/48/2a/97928387d6ed1c1ebbfd4efc4133a0633546bec8481a2dd5ec961313a1c7/wrapt-1.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8fdbdb757d5390f7c675e558fd3186d590973244fab0c5fe63d373ade3e99d40", size = 38919, upload-time = "2025-01-14T10:34:04.093Z" }, - { url = "https://files.pythonhosted.org/packages/73/54/3bfe5a1febbbccb7a2f77de47b989c0b85ed3a6a41614b104204a788c20e/wrapt-1.17.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bb1d0dbf99411f3d871deb6faa9aabb9d4e744d67dcaaa05399af89d847a91d", size = 88721, upload-time = "2025-01-14T10:34:07.163Z" }, - { url = "https://files.pythonhosted.org/packages/25/cb/7262bc1b0300b4b64af50c2720ef958c2c1917525238d661c3e9a2b71b7b/wrapt-1.17.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d18a4865f46b8579d44e4fe1e2bcbc6472ad83d98e22a26c963d46e4c125ef0b", size = 80899, upload-time = "2025-01-14T10:34:09.82Z" }, - { url = "https://files.pythonhosted.org/packages/2a/5a/04cde32b07a7431d4ed0553a76fdb7a61270e78c5fd5a603e190ac389f14/wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc570b5f14a79734437cb7b0500376b6b791153314986074486e0b0fa8d71d98", size = 89222, upload-time = "2025-01-14T10:34:11.258Z" }, - { url = "https://files.pythonhosted.org/packages/09/28/2e45a4f4771fcfb109e244d5dbe54259e970362a311b67a965555ba65026/wrapt-1.17.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6d9187b01bebc3875bac9b087948a2bccefe464a7d8f627cf6e48b1bbae30f82", size = 86707, upload-time = "2025-01-14T10:34:12.49Z" }, - { url = "https://files.pythonhosted.org/packages/c6/d2/dcb56bf5f32fcd4bd9aacc77b50a539abdd5b6536872413fd3f428b21bed/wrapt-1.17.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9e8659775f1adf02eb1e6f109751268e493c73716ca5761f8acb695e52a756ae", size = 79685, upload-time = "2025-01-14T10:34:15.043Z" }, - { url = "https://files.pythonhosted.org/packages/80/4e/eb8b353e36711347893f502ce91c770b0b0929f8f0bed2670a6856e667a9/wrapt-1.17.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8b2816ebef96d83657b56306152a93909a83f23994f4b30ad4573b00bd11bb9", size = 87567, upload-time = "2025-01-14T10:34:16.563Z" }, - { url = "https://files.pythonhosted.org/packages/17/27/4fe749a54e7fae6e7146f1c7d914d28ef599dacd4416566c055564080fe2/wrapt-1.17.2-cp312-cp312-win32.whl", hash = "sha256:468090021f391fe0056ad3e807e3d9034e0fd01adcd3bdfba977b6fdf4213ea9", size = 36672, upload-time = "2025-01-14T10:34:17.727Z" }, - { url = "https://files.pythonhosted.org/packages/15/06/1dbf478ea45c03e78a6a8c4be4fdc3c3bddea5c8de8a93bc971415e47f0f/wrapt-1.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:ec89ed91f2fa8e3f52ae53cd3cf640d6feff92ba90d62236a81e4e563ac0e991", size = 38865, upload-time = "2025-01-14T10:34:19.577Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b9/0ffd557a92f3b11d4c5d5e0c5e4ad057bd9eb8586615cdaf901409920b14/wrapt-1.17.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6ed6ffac43aecfe6d86ec5b74b06a5be33d5bb9243d055141e8cabb12aa08125", size = 53800, upload-time = "2025-01-14T10:34:21.571Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ef/8be90a0b7e73c32e550c73cfb2fa09db62234227ece47b0e80a05073b375/wrapt-1.17.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:35621ae4c00e056adb0009f8e86e28eb4a41a4bfa8f9bfa9fca7d343fe94f998", size = 38824, upload-time = "2025-01-14T10:34:22.999Z" }, - { url = "https://files.pythonhosted.org/packages/36/89/0aae34c10fe524cce30fe5fc433210376bce94cf74d05b0d68344c8ba46e/wrapt-1.17.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a604bf7a053f8362d27eb9fefd2097f82600b856d5abe996d623babd067b1ab5", size = 38920, upload-time = "2025-01-14T10:34:25.386Z" }, - { url = "https://files.pythonhosted.org/packages/3b/24/11c4510de906d77e0cfb5197f1b1445d4fec42c9a39ea853d482698ac681/wrapt-1.17.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cbabee4f083b6b4cd282f5b817a867cf0b1028c54d445b7ec7cfe6505057cf8", size = 88690, upload-time = "2025-01-14T10:34:28.058Z" }, - { url = "https://files.pythonhosted.org/packages/71/d7/cfcf842291267bf455b3e266c0c29dcb675b5540ee8b50ba1699abf3af45/wrapt-1.17.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49703ce2ddc220df165bd2962f8e03b84c89fee2d65e1c24a7defff6f988f4d6", size = 80861, upload-time = "2025-01-14T10:34:29.167Z" }, - { url = "https://files.pythonhosted.org/packages/d5/66/5d973e9f3e7370fd686fb47a9af3319418ed925c27d72ce16b791231576d/wrapt-1.17.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8112e52c5822fc4253f3901b676c55ddf288614dc7011634e2719718eaa187dc", size = 89174, upload-time = "2025-01-14T10:34:31.702Z" }, - { url = "https://files.pythonhosted.org/packages/a7/d3/8e17bb70f6ae25dabc1aaf990f86824e4fd98ee9cadf197054e068500d27/wrapt-1.17.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fee687dce376205d9a494e9c121e27183b2a3df18037f89d69bd7b35bcf59e2", size = 86721, upload-time = "2025-01-14T10:34:32.91Z" }, - { url = "https://files.pythonhosted.org/packages/6f/54/f170dfb278fe1c30d0ff864513cff526d624ab8de3254b20abb9cffedc24/wrapt-1.17.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:18983c537e04d11cf027fbb60a1e8dfd5190e2b60cc27bc0808e653e7b218d1b", size = 79763, upload-time = "2025-01-14T10:34:34.903Z" }, - { url = "https://files.pythonhosted.org/packages/4a/98/de07243751f1c4a9b15c76019250210dd3486ce098c3d80d5f729cba029c/wrapt-1.17.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:703919b1633412ab54bcf920ab388735832fdcb9f9a00ae49387f0fe67dad504", size = 87585, upload-time = "2025-01-14T10:34:36.13Z" }, - { url = "https://files.pythonhosted.org/packages/f9/f0/13925f4bd6548013038cdeb11ee2cbd4e37c30f8bfd5db9e5a2a370d6e20/wrapt-1.17.2-cp313-cp313-win32.whl", hash = "sha256:abbb9e76177c35d4e8568e58650aa6926040d6a9f6f03435b7a522bf1c487f9a", size = 36676, upload-time = "2025-01-14T10:34:37.962Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ae/743f16ef8c2e3628df3ddfd652b7d4c555d12c84b53f3d8218498f4ade9b/wrapt-1.17.2-cp313-cp313-win_amd64.whl", hash = "sha256:69606d7bb691b50a4240ce6b22ebb319c1cfb164e5f6569835058196e0f3a845", size = 38871, upload-time = "2025-01-14T10:34:39.13Z" }, - { url = "https://files.pythonhosted.org/packages/3d/bc/30f903f891a82d402ffb5fda27ec1d621cc97cb74c16fea0b6141f1d4e87/wrapt-1.17.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:4a721d3c943dae44f8e243b380cb645a709ba5bd35d3ad27bc2ed947e9c68192", size = 56312, upload-time = "2025-01-14T10:34:40.604Z" }, - { url = "https://files.pythonhosted.org/packages/8a/04/c97273eb491b5f1c918857cd26f314b74fc9b29224521f5b83f872253725/wrapt-1.17.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:766d8bbefcb9e00c3ac3b000d9acc51f1b399513f44d77dfe0eb026ad7c9a19b", size = 40062, upload-time = "2025-01-14T10:34:45.011Z" }, - { url = "https://files.pythonhosted.org/packages/4e/ca/3b7afa1eae3a9e7fefe499db9b96813f41828b9fdb016ee836c4c379dadb/wrapt-1.17.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e496a8ce2c256da1eb98bd15803a79bee00fc351f5dfb9ea82594a3f058309e0", size = 40155, upload-time = "2025-01-14T10:34:47.25Z" }, - { url = "https://files.pythonhosted.org/packages/89/be/7c1baed43290775cb9030c774bc53c860db140397047cc49aedaf0a15477/wrapt-1.17.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d615e4fe22f4ad3528448c193b218e077656ca9ccb22ce2cb20db730f8d306", size = 113471, upload-time = "2025-01-14T10:34:50.934Z" }, - { url = "https://files.pythonhosted.org/packages/32/98/4ed894cf012b6d6aae5f5cc974006bdeb92f0241775addad3f8cd6ab71c8/wrapt-1.17.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5aaeff38654462bc4b09023918b7f21790efb807f54c000a39d41d69cf552cb", size = 101208, upload-time = "2025-01-14T10:34:52.297Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fd/0c30f2301ca94e655e5e057012e83284ce8c545df7661a78d8bfca2fac7a/wrapt-1.17.2-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a7d15bbd2bc99e92e39f49a04653062ee6085c0e18b3b7512a4f2fe91f2d681", size = 109339, upload-time = "2025-01-14T10:34:53.489Z" }, - { url = "https://files.pythonhosted.org/packages/75/56/05d000de894c4cfcb84bcd6b1df6214297b8089a7bd324c21a4765e49b14/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e3890b508a23299083e065f435a492b5435eba6e304a7114d2f919d400888cc6", size = 110232, upload-time = "2025-01-14T10:34:55.327Z" }, - { url = "https://files.pythonhosted.org/packages/53/f8/c3f6b2cf9b9277fb0813418e1503e68414cd036b3b099c823379c9575e6d/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8c8b293cd65ad716d13d8dd3624e42e5a19cc2a2f1acc74b30c2c13f15cb61a6", size = 100476, upload-time = "2025-01-14T10:34:58.055Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b1/0bb11e29aa5139d90b770ebbfa167267b1fc548d2302c30c8f7572851738/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c82b8785d98cdd9fed4cac84d765d234ed3251bd6afe34cb7ac523cb93e8b4f", size = 106377, upload-time = "2025-01-14T10:34:59.3Z" }, - { url = "https://files.pythonhosted.org/packages/6a/e1/0122853035b40b3f333bbb25f1939fc1045e21dd518f7f0922b60c156f7c/wrapt-1.17.2-cp313-cp313t-win32.whl", hash = "sha256:13e6afb7fe71fe7485a4550a8844cc9ffbe263c0f1a1eea569bc7091d4898555", size = 37986, upload-time = "2025-01-14T10:35:00.498Z" }, - { url = "https://files.pythonhosted.org/packages/09/5e/1655cf481e079c1f22d0cabdd4e51733679932718dc23bf2db175f329b76/wrapt-1.17.2-cp313-cp313t-win_amd64.whl", hash = "sha256:eaf675418ed6b3b31c7a989fd007fa7c3be66ce14e5c3b27336383604c9da85c", size = 40750, upload-time = "2025-01-14T10:35:03.378Z" }, - { url = "https://files.pythonhosted.org/packages/8a/f4/6ed2b8f6f1c832933283974839b88ec7c983fd12905e01e97889dadf7559/wrapt-1.17.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99039fa9e6306880572915728d7f6c24a86ec57b0a83f6b2491e1d8ab0235b9a", size = 53308, upload-time = "2025-01-14T10:35:24.413Z" }, - { url = "https://files.pythonhosted.org/packages/a2/a9/712a53f8f4f4545768ac532619f6e56d5d0364a87b2212531685e89aeef8/wrapt-1.17.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2696993ee1eebd20b8e4ee4356483c4cb696066ddc24bd70bcbb80fa56ff9061", size = 38489, upload-time = "2025-01-14T10:35:26.913Z" }, - { url = "https://files.pythonhosted.org/packages/fa/9b/e172c8f28a489a2888df18f953e2f6cb8d33b1a2e78c9dfc52d8bf6a5ead/wrapt-1.17.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:612dff5db80beef9e649c6d803a8d50c409082f1fedc9dbcdfde2983b2025b82", size = 38776, upload-time = "2025-01-14T10:35:28.183Z" }, - { url = "https://files.pythonhosted.org/packages/cf/cb/7a07b51762dcd59bdbe07aa97f87b3169766cadf240f48d1cbe70a1be9db/wrapt-1.17.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62c2caa1585c82b3f7a7ab56afef7b3602021d6da34fbc1cf234ff139fed3cd9", size = 83050, upload-time = "2025-01-14T10:35:30.645Z" }, - { url = "https://files.pythonhosted.org/packages/a5/51/a42757dd41032afd6d8037617aa3bc6803ba971850733b24dfb7d5c627c4/wrapt-1.17.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c958bcfd59bacc2d0249dcfe575e71da54f9dcf4a8bdf89c4cb9a68a1170d73f", size = 74718, upload-time = "2025-01-14T10:35:32.047Z" }, - { url = "https://files.pythonhosted.org/packages/bf/bb/d552bfe47db02fcfc950fc563073a33500f8108efa5f7b41db2f83a59028/wrapt-1.17.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc78a84e2dfbc27afe4b2bd7c80c8db9bca75cc5b85df52bfe634596a1da846b", size = 82590, upload-time = "2025-01-14T10:35:33.329Z" }, - { url = "https://files.pythonhosted.org/packages/77/99/77b06b3c3c410dbae411105bf22496facf03a5496bfaca8fbcf9da381889/wrapt-1.17.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ba0f0eb61ef00ea10e00eb53a9129501f52385c44853dbd6c4ad3f403603083f", size = 81462, upload-time = "2025-01-14T10:35:34.933Z" }, - { url = "https://files.pythonhosted.org/packages/2d/21/cf0bd85ae66f92600829ea1de8e1da778e5e9f6e574ccbe74b66db0d95db/wrapt-1.17.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1e1fe0e6ab7775fd842bc39e86f6dcfc4507ab0ffe206093e76d61cde37225c8", size = 74309, upload-time = "2025-01-14T10:35:37.542Z" }, - { url = "https://files.pythonhosted.org/packages/6d/16/112d25e9092398a0dd6fec50ab7ac1b775a0c19b428f049785096067ada9/wrapt-1.17.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c86563182421896d73858e08e1db93afdd2b947a70064b813d515d66549e15f9", size = 81081, upload-time = "2025-01-14T10:35:38.9Z" }, - { url = "https://files.pythonhosted.org/packages/2b/49/364a615a0cc0872685646c495c7172e4fc7bf1959e3b12a1807a03014e05/wrapt-1.17.2-cp39-cp39-win32.whl", hash = "sha256:f393cda562f79828f38a819f4788641ac7c4085f30f1ce1a68672baa686482bb", size = 36423, upload-time = "2025-01-14T10:35:40.177Z" }, - { url = "https://files.pythonhosted.org/packages/00/ad/5d2c1b34ba3202cd833d9221833e74d6500ce66730974993a8dc9a94fb8c/wrapt-1.17.2-cp39-cp39-win_amd64.whl", hash = "sha256:36ccae62f64235cf8ddb682073a60519426fdd4725524ae38874adf72b5f2aeb", size = 38772, upload-time = "2025-01-14T10:35:42.763Z" }, - { url = "https://files.pythonhosted.org/packages/2d/82/f56956041adef78f849db6b289b282e72b55ab8045a75abad81898c28d19/wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", size = 23594, upload-time = "2025-01-14T10:35:44.018Z" }, -] - [[package]] name = "xformers" version = "0.0.30" @@ -5158,6 +5095,7 @@ name = "xgrammar" version = "0.1.19" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "mlx-lm", marker = "python_full_version >= '3.10' and platform_machine == 'arm64' and sys_platform == 'darwin'" }, { name = "ninja", marker = "python_full_version >= '3.10'" }, { name = "pydantic", marker = "python_full_version >= '3.10'" }, { name = "sentencepiece", marker = "python_full_version >= '3.10'" }, @@ -5392,12 +5330,3 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/fc/a8aef69156ad5508165d8ae956736d55c3a68890610834bd985540966008/yarl-1.18.3-cp39-cp39-win_amd64.whl", hash = "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9", size = 90968, upload-time = "2024-12-01T20:35:18.962Z" }, { url = "https://files.pythonhosted.org/packages/f5/4b/a06e0ec3d155924f77835ed2d167ebd3b211a7b0853da1cf8d8414d784ef/yarl-1.18.3-py3-none-any.whl", hash = "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b", size = 45109, upload-time = "2024-12-01T20:35:20.834Z" }, ] - -[[package]] -name = "zipp" -version = "3.21.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e9468ebd0bcd6505de3b275e06f202c2cb016e3ff56f/zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4", size = 24545, upload-time = "2024-11-10T15:05:20.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, -] From daadc0e0e2ce551ffeb0ec80474c40cdb4f046d7 Mon Sep 17 00:00:00 2001 From: Prashant Gupta Date: Tue, 8 Jul 2025 10:41:50 -0700 Subject: [PATCH 29/29] =?UTF-8?q?=F0=9F=8E=A8=20fix=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Gupta --- vllm_spyre/v1/worker/spyre_model_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm_spyre/v1/worker/spyre_model_runner.py b/vllm_spyre/v1/worker/spyre_model_runner.py index 596840298..f63c1824d 100644 --- a/vllm_spyre/v1/worker/spyre_model_runner.py +++ b/vllm_spyre/v1/worker/spyre_model_runner.py @@ -245,15 +245,15 @@ def update_states(self, scheduler_output: SchedulerOutput): # finished requests from the batch # # NOTE: req_state.output_token_ids will be mutated when - # using PP + # PP will be enabled in the future req_data = scheduler_output.scheduled_cached_reqs for i, req_id in enumerate(req_data.req_ids): req_state: CachedRequestState = self.requests[req_id] # Update the cached states. num_computed_tokens = req_data.num_computed_tokens[i] - # In the future, when using PP, the scheduler will send the sampled - # tokens back + # The scheduler will send the sampled tokens back + # when PP will be enabled in the future new_token_ids = req_data.new_token_ids[i] if len( req_data.new_token_ids) > 0 else [] # Add the sampled token(s) from the previous step (if any).