Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions python/sglang/srt/constrained/fsm_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,16 @@


class FSMCache(BaseToolCache):
def __init__(self, tokenizer_path, tokenizer_args_dict, enable=True):
def __init__(self, tokenizer_path, tokenizer_args_dict, enable=True, skip_tokenizer_init=False):
super().__init__(enable=enable)

if tokenizer_path.endswith(".json") or tokenizer_path.endswith(".model"):
# Do not support TiktokenTokenizer or SentencePieceTokenizer
return
if (
skip_tokenizer_init
or tokenizer_path.endswith(".json")
or tokenizer_path.endswith(".model")
):
# Do not support TiktokenTokenizer or SentencePieceTokenizer
return

from importlib.metadata import version

Expand Down
18 changes: 13 additions & 5 deletions python/sglang/srt/managers/detokenizer_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,14 @@ def __init__(
self.send_to_tokenizer = context.socket(zmq.PUSH)
self.send_to_tokenizer.connect(f"tcp://127.0.0.1:{port_args.tokenizer_port}")

self.tokenizer = get_tokenizer(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
if server_args.skip_tokenizer_init:
self.tokenizer = None
else:
self.tokenizer = get_tokenizer(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)

self.decode_status = {}

Expand All @@ -69,6 +72,11 @@ async def handle_loop(self):
assert isinstance(recv_obj, BatchTokenIDOut)
bs = len(recv_obj.rids)

if self.tokenizer is None:
# Send BatchTokenIDOut if no tokenizer init'ed.
self.send_to_tokenizer.send_pyobj(recv_obj)
continue

# Initialize decode status
read_ids, surr_ids = [], []
for i in range(bs):
Expand Down
11 changes: 7 additions & 4 deletions python/sglang/srt/managers/schedule_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ def init_incremental_detokenize(self):
return all_ids[self.surr_offset :], self.read_offset - self.surr_offset

def get_next_inc_detokenization(self):
if self.tokenizer is None:
return False, ""
read_ids, read_offset = self.init_incremental_detokenize()
surr_ids = read_ids[:read_offset]

Expand Down Expand Up @@ -220,10 +222,11 @@ def check_finished(self):
self.finished_reason = FINISH_LENGTH(len(self.output_ids))
return

if (
self.output_ids[-1] == self.tokenizer.eos_token_id
and not self.sampling_params.ignore_eos
):
if self.tokenizer is None:
matched_eos = self.output_ids[-1] in self.sampling_params.stop_token_ids
else:
matched_eos = self.output_ids[-1] == self.tokenizer.eos_token_id
if matched_eos and not self.sampling_params.ignore_eos:
self.finished_reason = FINISH_MATCHED_TOKEN(
matched=self.tokenizer.eos_token_id
)
Expand Down
79 changes: 55 additions & 24 deletions python/sglang/srt/managers/tokenizer_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,24 +92,31 @@ def __init__(
self.context_len = get_context_length(self.hf_config)

if is_multimodal_model(self.model_path):
Comment thread
merrymercy marked this conversation as resolved.
Resolved
self.processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
self.tokenizer = self.processor.tokenizer
if server_args.skip_tokenizer_init:
self.tokenizer = None
else:
self.processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
self.tokenizer = self.processor.tokenizer

os.environ["TOKENIZERS_PARALLELISM"] = "false"
self.executor = concurrent.futures.ProcessPoolExecutor(
initializer=init_global_processor,
mp_context=mp.get_context("fork"),
initargs=(server_args,),
)
else:
self.tokenizer = get_tokenizer(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
if server_args.skip_tokenizer_init:
self.tokenizer = None
else:
self.tokenizer = get_tokenizer(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)

self.to_create_loop = True
self.rid_to_state: Dict[str, ReqState] = {}
Expand Down Expand Up @@ -158,11 +165,11 @@ async def _handle_single_request(

rid = obj.rid if not_use_index else obj.rid[index]
input_text = obj.text if not_use_index else obj.text[index]
input_ids = (
self.tokenizer.encode(input_text)
if obj.input_ids is None
else obj.input_ids
)
if obj.input_ids is None:
assert self.tokenizer is not None
input_ids = self.tokenizer.encode(input_text)
else:
input_ids = obj.input_ids
if not not_use_index and obj.input_ids:
input_ids = obj.input_ids[index]

Expand Down Expand Up @@ -191,7 +198,20 @@ async def _handle_single_request(
else:
input_text = obj.text
rid = obj.rid[0]
input_ids = self.tokenizer.encode(input_text)
if self.tokenizer is not None:
input_ids = self.tokenizer.encode(input_text)
else:
assert obj.input_ids is not None
input_ids = obj.input_ids
if isinstance(obj.input_ids, list) and isinstance(
obj.input_ids[0], list
):
# when obj["input_ids"] is List[List[int]]
input_ids = obj.input_ids[index]
rid = obj.rid[index]
else:
input_ids = obj.input_ids
rid = obj.rid[0]
else:
input_text = None
if isinstance(obj.input_ids, list) and isinstance(
Expand Down Expand Up @@ -391,7 +411,7 @@ async def _wait_for_response(
# Log requests
if self.server_args.log_requests and state.finished:
if obj.text is None:
in_obj = {"text": self.tokenizer.decode(obj.input_ids)}
in_obj = {"input_ids": obj.input_ids}
else:
in_obj = {"text": obj.text}
logger.info(f"in={in_obj}, out={out}")
Expand Down Expand Up @@ -459,19 +479,29 @@ def create_handle_loop(self):

async def handle_loop(self):
while True:
recv_obj: BatchStrOut = await self.recv_from_detokenizer.recv_pyobj()
assert isinstance(recv_obj, BatchStrOut)
recv_obj = await self.recv_from_detokenizer.recv_pyobj()
is_str_output = isinstance(recv_obj, BatchStrOut)
is_token_id_output = isinstance(recv_obj, BatchTokenIDOut)
assert is_str_output or is_token_id_output, f"Unexpected obj received: {type(recv_obj)}"

for i, rid in enumerate(recv_obj.rids):
state = self.rid_to_state.get(rid, None)
if state is None:
continue

recv_obj.meta_info[i]["id"] = rid
out_dict = {
"text": recv_obj.output_strs[i],
"meta_info": recv_obj.meta_info[i],
}
if is_str_output:
out_dict = {
"text": recv_obj.output_strs[i],
"meta_info": recv_obj.meta_info[i],
}
elif is_token_id_output:
read_start = 0 if i==0 else recv_obj.read_offsets[i-1]
out_dict = {
"token_ids": recv_obj.decode_ids[read_start:recv_obj.read_offsets[i]],
"meta_info": recv_obj.meta_info[i],
}

state.out_list.append(out_dict)
state.finished = recv_obj.finished_reason[i] is not None
state.event.set()
Expand Down Expand Up @@ -511,6 +541,7 @@ def detokenize_logprob_tokens(
if not decode_to_text:
return [(logprob, token_id, None) for logprob, token_id in token_logprobs]

assert self.tokenizer is not None
token_ids = [tid for _, tid in token_logprobs]
token_texts = self.tokenizer.batch_decode(token_ids)
return [
Expand Down
51 changes: 32 additions & 19 deletions python/sglang/srt/managers/tp_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,18 +100,24 @@ def __init__(
)

if is_multimodal_model(server_args.model_path):
Comment thread
merrymercy marked this conversation as resolved.
Resolved
self.processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
self.tokenizer = self.processor.tokenizer
if server_args.skip_tokenizer_init:
self.tokenizer = None
else:
self.processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
self.tokenizer = self.processor.tokenizer
else:
self.tokenizer = get_tokenizer(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
if server_args.skip_tokenizer_init:
self.tokenizer = None
else:
self.tokenizer = get_tokenizer(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
)
self.max_total_num_tokens = self.model_runner.max_total_num_tokens
self.max_prefill_tokens = (
16384
Expand Down Expand Up @@ -180,13 +186,15 @@ def __init__(
self.last_stats_tic = time.time()

# Init the FSM cache for constrained generation
self.regex_fsm_cache = FSMCache(
server_args.tokenizer_path,
{
"tokenizer_mode": server_args.tokenizer_mode,
"trust_remote_code": server_args.trust_remote_code,
},
)
if not server_args.skip_tokenizer_init:
self.regex_fsm_cache = FSMCache(
server_args.tokenizer_path,
{
"tokenizer_mode": server_args.tokenizer_mode,
"trust_remote_code": server_args.trust_remote_code,
},
skip_tokenizer_init=server_args.skip_tokenizer_init,
)
self.jump_forward_cache = JumpForwardCache()

# Init new token estimation
Expand Down Expand Up @@ -440,6 +448,7 @@ def forward_prefill_batch(self, batch: ScheduleBatch):
)

# Forward and sample the next tokens
next_token_ids = []
if batch.extend_num_tokens != 0:
output = self.model_runner.forward(batch, ForwardMode.EXTEND)
next_token_ids = batch.sample(output.next_token_logits)
Expand All @@ -457,7 +466,11 @@ def forward_prefill_batch(self, batch: ScheduleBatch):

next_token_ids = next_token_ids.tolist()
else:
next_token_ids = [self.tokenizer.eos_token_id] * len(batch.reqs)
if self.tokenizer is None:
for i, req in enumerate(batch.reqs):
next_token_ids.extend(req.sampling_params.stop_token_ids)
else:
next_token_ids = [self.tokenizer.eos_token_id] * len(batch.reqs)

# Check finish conditions
pt = 0
Expand Down
14 changes: 11 additions & 3 deletions python/sglang/srt/sampling_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ def __init__(
self,
max_new_tokens: int = 128,
stop: Optional[Union[str, List[str]]] = None,
stop_token_ids: Optional[List[int]] = None,
temperature: float = 1.0,
top_p: float = 1.0,
top_k: int = -1,
Expand All @@ -43,6 +44,7 @@ def __init__(
self.frequency_penalty = frequency_penalty
self.presence_penalty = presence_penalty
self.stop_strs = stop
self.stop_token_ids = stop_token_ids
self.max_new_tokens = max_new_tokens
self.ignore_eos = ignore_eos
self.skip_special_tokens = skip_special_tokens
Expand Down Expand Up @@ -90,13 +92,19 @@ def normalize(self, tokenizer):
# Process stop strings
if self.stop_strs is None:
self.stop_strs = []
self.stop_str_max_len = 0
if self.stop_token_ids is None:
self.stop_str_max_len = 0
else:
self.stop_str_max_len = 1
else:
if isinstance(self.stop_strs, str):
self.stop_strs = [self.stop_strs]

stop_str_max_len = 0
for stop_str in self.stop_strs:
stop_str_ids = tokenizer.encode(stop_str, add_special_tokens=False)
stop_str_max_len = max(stop_str_max_len, len(stop_str_ids))
if tokenizer is not None:
stop_str_ids = tokenizer.encode(stop_str, add_special_tokens=False)
stop_str_max_len = max(stop_str_max_len, len(stop_str_ids))
else:
stop_str_max_len = max(stop_str_max_len, len(stop_str))
self.stop_str_max_len = stop_str_max_len
43 changes: 22 additions & 21 deletions python/sglang/srt/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -392,28 +392,29 @@ def _wait_and_warmup(server_args, pipe_finish_writer):
print(f"Initialization failed. warmup error: {last_traceback}", flush=True)
sys.exit(1)

# Send a warmup request
try:
for _ in range(server_args.dp_size):
res = requests.post(
url + "/generate",
json={
"text": "The capital city of France is",
"sampling_params": {
"temperature": 0,
"max_new_tokens": 8,
if not server_args.skip_tokenizer_init:
Comment thread
gryffindor-rr marked this conversation as resolved.
Resolved
# Send a warmup request
try:
for _ in range(server_args.dp_size):
res = requests.post(
url + "/generate",
json={
"text": "The capital city of France is",
"sampling_params": {
"temperature": 0,
"max_new_tokens": 8,
},
},
},
headers=headers,
timeout=600,
)
assert res.status_code == 200, f"{res}"
except Exception as e:
last_traceback = get_exception_traceback()
if pipe_finish_writer is not None:
pipe_finish_writer.send(last_traceback)
print(f"Initialization failed. warmup error: {last_traceback}", flush=True)
sys.exit(1)
headers=headers,
timeout=600,
)
assert res.status_code == 200, f"{res}"
except Exception as e:
last_traceback = get_exception_traceback()
if pipe_finish_writer is not None:
pipe_finish_writer.send(last_traceback)
print(f"Initialization failed. warmup error: {last_traceback}", flush=True)
sys.exit(1)

logger.info("The server is fired up and ready to roll!")
if pipe_finish_writer is not None:
Expand Down
Loading