diff --git a/python/sglang/bench_latency.py b/python/sglang/bench_latency.py index 130f2b82ffdf..338ab8d41637 100644 --- a/python/sglang/bench_latency.py +++ b/python/sglang/bench_latency.py @@ -54,7 +54,7 @@ from sglang.srt.model_config import ModelConfig from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.model_runner import ModelRunner -from sglang.srt.sampling_params import SamplingParams +from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ServerArgs from sglang.srt.utils import suppress_other_loggers diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 75208874ffc8..a0534e7d784d 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -25,7 +25,7 @@ import torch from sglang.srt.managers.schedule_batch import BaseFinishReason -from sglang.srt.sampling_params import SamplingParams +from sglang.srt.sampling.sampling_params import SamplingParams @dataclass diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 278ed006ef5a..c499a18056d9 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -31,6 +31,7 @@ from sglang.srt.mem_cache.chunk_cache import ChunkCache from sglang.srt.mem_cache.memory_pool import BaseTokenToKVPool, ReqToTokenPool from sglang.srt.mem_cache.radix_cache import RadixCache +from sglang.srt.model_config import ModelConfig INIT_INCREMENTAL_DETOKENIZATION_OFFSET = 5 @@ -314,6 +315,10 @@ class ScheduleBatch: token_to_kv_pool: BaseTokenToKVPool tree_cache: RadixCache + # Some auxiliary fields + model_config: ModelConfig = None + int_token_logit_bias: torch.Tensor = None + # Batched arguments to model runner input_ids: torch.Tensor = None req_pool_indices: torch.Tensor = None @@ -334,7 +339,15 @@ class ScheduleBatch: logit_bias: torch.Tensor = None @classmethod - def init_new(cls, reqs, req_to_token_pool, token_to_kv_pool, tree_cache): + def init_new( + cls, + reqs, + req_to_token_pool, + token_to_kv_pool, + tree_cache, + model_config, + int_token_logit_bias, + ): return_logprob = any(req.return_logprob for req in reqs) return cls( @@ -343,6 +356,8 @@ def init_new(cls, reqs, req_to_token_pool, token_to_kv_pool, tree_cache): token_to_kv_pool=token_to_kv_pool, tree_cache=tree_cache, return_logprob=return_logprob, + model_config=model_config, + int_token_logit_bias=int_token_logit_bias, ) def batch_size(self): @@ -380,7 +395,7 @@ def alloc_token_slots(self, num_tokens: int): return out_cache_loc - def batch_sampling_params(self, vocab_size, int_token_logit_bias): + def batch_sampling_params(self): device = "cuda" bs, reqs = self.batch_size(), self.reqs self.temperatures = torch.tensor( @@ -403,7 +418,7 @@ def batch_sampling_params(self, vocab_size, int_token_logit_bias): # could add additional complexity to the {ScheduleBatch} class, especially we need to # handle {filter_batch()} and {merge()} cases as well. self.penalizer_orchestrator = penaltylib.BatchedPenalizerOrchestrator( - vocab_size=vocab_size, + vocab_size=self.model_config.vocab_size, batch=self, device=device, Penalizers={ @@ -420,11 +435,15 @@ def batch_sampling_params(self, vocab_size, int_token_logit_bias): if reqs[i].sampling_params.dtype == "int": if self.logit_bias is None: self.logit_bias = torch.zeros( - (bs, vocab_size), dtype=torch.float32, device=device + (bs, self.model_config.vocab_size), + dtype=torch.float32, + device=device, ) - self.logit_bias[i][: len(int_token_logit_bias)] = int_token_logit_bias + self.logit_bias[i][ + : len(self.int_token_logit_bias) + ] = self.int_token_logit_bias - def prepare_for_extend(self, vocab_size: int, int_token_logit_bias: torch.Tensor): + def prepare_for_extend(self): bs = self.batch_size() reqs = self.reqs input_ids = [r.input_ids[len(r.prefix_indices) :] for r in reqs] @@ -463,7 +482,7 @@ def prepare_for_extend(self, vocab_size: int, int_token_logit_bias: torch.Tensor self.out_cache_loc = out_cache_loc self.top_logprobs_nums = [r.top_logprobs_num for r in reqs] - self.batch_sampling_params(vocab_size, int_token_logit_bias) + self.batch_sampling_params() def check_decode_mem(self): bs = self.batch_size() @@ -710,18 +729,17 @@ def merge(self, other: "ScheduleBatch"): # logit_bias can be None if self.logit_bias is not None or other.logit_bias is not None: - vocab_size = ( - self.logit_bias.shape[1] - if self.logit_bias is not None - else other.logit_bias.shape[1] - ) if self.logit_bias is None: self.logit_bias = torch.zeros( - (len(self.reqs), vocab_size), dtype=torch.float32, device="cuda" + (len(self.reqs), self.model_config.vocab_size), + dtype=torch.float32, + device="cuda", ) if other.logit_bias is None: other.logit_bias = torch.zeros( - (len(other.reqs), vocab_size), dtype=torch.float32, device="cuda" + (len(other.reqs), self.model_config.vocab_size), + dtype=torch.float32, + device="cuda", ) self.logit_bias = torch.concat([self.logit_bias, other.logit_bias]) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index e2c8259737c3..4a20369586c2 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -48,7 +48,7 @@ TokenizedGenerateReqInput, ) from sglang.srt.mm_utils import expand2square, process_anyres_image -from sglang.srt.sampling_params import SamplingParams +from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import PortArgs, ServerArgs from sglang.srt.utils import is_generation_model, is_multimodal_model, load_image from sglang.utils import get_exception_traceback diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index c668977106cf..e1bee67aefe2 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -431,15 +431,15 @@ def get_new_prefill_batch(self) -> Optional[ScheduleBatch]: self.req_to_token_pool, self.token_to_kv_pool, self.tree_cache, + self.model_config, + self.int_token_logit_bias, ) self.waiting_queue = [x for x in self.waiting_queue if x not in can_run_list] return new_batch def forward_prefill_batch(self, batch: ScheduleBatch): # Build batch tensors - batch.prepare_for_extend( - self.model_config.vocab_size, self.int_token_logit_bias - ) + batch.prepare_for_extend() if self.model_runner.is_generation: # Forward and sample the next tokens diff --git a/python/sglang/srt/sampling_params.py b/python/sglang/srt/sampling/sampling_params.py similarity index 100% rename from python/sglang/srt/sampling_params.py rename to python/sglang/srt/sampling/sampling_params.py