Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion python/sglang/bench_latency.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
from sglang.srt.model_config import ModelConfig
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.sampling_params import SamplingParams
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import suppress_other_loggers

Expand Down
2 changes: 1 addition & 1 deletion python/sglang/srt/managers/io_struct.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import torch

from sglang.srt.managers.schedule_batch import BaseFinishReason
from sglang.srt.sampling_params import SamplingParams
from sglang.srt.sampling.sampling_params import SamplingParams


@dataclass
Expand Down
46 changes: 32 additions & 14 deletions python/sglang/srt/managers/schedule_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from sglang.srt.mem_cache.chunk_cache import ChunkCache
from sglang.srt.mem_cache.memory_pool import BaseTokenToKVPool, ReqToTokenPool
from sglang.srt.mem_cache.radix_cache import RadixCache
from sglang.srt.model_config import ModelConfig

INIT_INCREMENTAL_DETOKENIZATION_OFFSET = 5

Expand Down Expand Up @@ -314,6 +315,10 @@ class ScheduleBatch:
token_to_kv_pool: BaseTokenToKVPool
tree_cache: RadixCache

# Some auxiliary fields
model_config: ModelConfig = None
int_token_logit_bias: torch.Tensor = None

# Batched arguments to model runner
input_ids: torch.Tensor = None
req_pool_indices: torch.Tensor = None
Expand All @@ -334,7 +339,15 @@ class ScheduleBatch:
logit_bias: torch.Tensor = None

@classmethod
def init_new(cls, reqs, req_to_token_pool, token_to_kv_pool, tree_cache):
def init_new(
cls,
reqs,
req_to_token_pool,
token_to_kv_pool,
tree_cache,
model_config,
int_token_logit_bias,
):
return_logprob = any(req.return_logprob for req in reqs)

return cls(
Expand All @@ -343,6 +356,8 @@ def init_new(cls, reqs, req_to_token_pool, token_to_kv_pool, tree_cache):
token_to_kv_pool=token_to_kv_pool,
tree_cache=tree_cache,
return_logprob=return_logprob,
model_config=model_config,
int_token_logit_bias=int_token_logit_bias,
)

def batch_size(self):
Expand Down Expand Up @@ -380,7 +395,7 @@ def alloc_token_slots(self, num_tokens: int):

return out_cache_loc

def batch_sampling_params(self, vocab_size, int_token_logit_bias):
def batch_sampling_params(self):
device = "cuda"
bs, reqs = self.batch_size(), self.reqs
self.temperatures = torch.tensor(
Expand All @@ -403,7 +418,7 @@ def batch_sampling_params(self, vocab_size, int_token_logit_bias):
# could add additional complexity to the {ScheduleBatch} class, especially we need to
# handle {filter_batch()} and {merge()} cases as well.
self.penalizer_orchestrator = penaltylib.BatchedPenalizerOrchestrator(
vocab_size=vocab_size,
vocab_size=self.model_config.vocab_size,
batch=self,
device=device,
Penalizers={
Expand All @@ -420,11 +435,15 @@ def batch_sampling_params(self, vocab_size, int_token_logit_bias):
if reqs[i].sampling_params.dtype == "int":
if self.logit_bias is None:
self.logit_bias = torch.zeros(
(bs, vocab_size), dtype=torch.float32, device=device
(bs, self.model_config.vocab_size),
dtype=torch.float32,
device=device,
)
self.logit_bias[i][: len(int_token_logit_bias)] = int_token_logit_bias
self.logit_bias[i][
: len(self.int_token_logit_bias)
] = self.int_token_logit_bias

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can remove int_token_logit_bias now. It was used before we had regex decoding. But please keep logit_bias because it is still useful and it is in OpenAI API (although we haven't supported it yet)


def prepare_for_extend(self, vocab_size: int, int_token_logit_bias: torch.Tensor):
def prepare_for_extend(self):
bs = self.batch_size()
reqs = self.reqs
input_ids = [r.input_ids[len(r.prefix_indices) :] for r in reqs]
Expand Down Expand Up @@ -463,7 +482,7 @@ def prepare_for_extend(self, vocab_size: int, int_token_logit_bias: torch.Tensor
self.out_cache_loc = out_cache_loc
self.top_logprobs_nums = [r.top_logprobs_num for r in reqs]

self.batch_sampling_params(vocab_size, int_token_logit_bias)
self.batch_sampling_params()

def check_decode_mem(self):
bs = self.batch_size()
Expand Down Expand Up @@ -710,18 +729,17 @@ def merge(self, other: "ScheduleBatch"):

# logit_bias can be None
if self.logit_bias is not None or other.logit_bias is not None:
vocab_size = (
self.logit_bias.shape[1]
if self.logit_bias is not None
else other.logit_bias.shape[1]
)
if self.logit_bias is None:
self.logit_bias = torch.zeros(
(len(self.reqs), vocab_size), dtype=torch.float32, device="cuda"
(len(self.reqs), self.model_config.vocab_size),
dtype=torch.float32,
device="cuda",
)
if other.logit_bias is None:
other.logit_bias = torch.zeros(
(len(other.reqs), vocab_size), dtype=torch.float32, device="cuda"
(len(other.reqs), self.model_config.vocab_size),
dtype=torch.float32,
device="cuda",
)
self.logit_bias = torch.concat([self.logit_bias, other.logit_bias])

Expand Down
2 changes: 1 addition & 1 deletion python/sglang/srt/managers/tokenizer_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
TokenizedGenerateReqInput,
)
from sglang.srt.mm_utils import expand2square, process_anyres_image
from sglang.srt.sampling_params import SamplingParams
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.server_args import PortArgs, ServerArgs
from sglang.srt.utils import is_generation_model, is_multimodal_model, load_image
from sglang.utils import get_exception_traceback
Expand Down
6 changes: 3 additions & 3 deletions python/sglang/srt/managers/tp_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -431,15 +431,15 @@ def get_new_prefill_batch(self) -> Optional[ScheduleBatch]:
self.req_to_token_pool,
self.token_to_kv_pool,
self.tree_cache,
self.model_config,
self.int_token_logit_bias,
)
self.waiting_queue = [x for x in self.waiting_queue if x not in can_run_list]
return new_batch

def forward_prefill_batch(self, batch: ScheduleBatch):
# Build batch tensors
batch.prepare_for_extend(
self.model_config.vocab_size, self.int_token_logit_bias
)
batch.prepare_for_extend()

if self.model_runner.is_generation:
# Forward and sample the next tokens
Expand Down