diff --git a/docs/en/sampling_params.md b/docs/en/sampling_params.md index 5f1cdece6a25..e1748d77ab78 100644 --- a/docs/en/sampling_params.md +++ b/docs/en/sampling_params.md @@ -35,7 +35,7 @@ The `sampling_params` follows this format # The maximum number of output tokens max_new_tokens: int = 128, # Stop when hitting any of the strings in this list. -stop: Optional[Union[str, List[str]]] = None, +stop_strs: Optional[Union[str, List[str]]] = None, # Stop when hitting any of the token_ids in this list. Could be useful when mixed with # `min_new_tokens`. stop_token_ids: Optional[List[int]] = [], @@ -72,8 +72,8 @@ presence_penalty: float = 0.0, repetition_penalty: float = 1.0, # Guides inference to generate at least this number of tokens by penalizing logits of tokenizer's # EOS token and `stop_token_ids` to -inf, until the output token reaches given length. -# Note that any of the `stop` string can be generated before reaching `min_new_tokens`, as it is -# difficult to infer the correct token ID by given `stop` strings. +# Note that any of the `stop_strs` string can be generated before reaching `min_new_tokens`, as it is +# difficult to infer the correct token ID by given `stop_strs` strings. # Must be 0 <= value < max_new_tokens. Setting to 0 (default) will disable this penalty. min_new_tokens: int = 0, ``` diff --git a/python/sglang/lang/backend/openai.py b/python/sglang/lang/backend/openai.py index 6fa93d9b2eb8..0f68c4e9c9f6 100644 --- a/python/sglang/lang/backend/openai.py +++ b/python/sglang/lang/backend/openai.py @@ -119,7 +119,7 @@ def _prepare_spec_execution( params = sampling_params.to_openai_kwargs() for key, value in params.items(): - if key in ["stop"]: + if key in ["stop_strs"]: continue if key in ["max_tokens"]: warnings.warn( @@ -133,7 +133,7 @@ def _prepare_spec_execution( value == self.spec_kwargs[key] ), "sampling parameters should be consistent if turn on api speculative execution." self.spec_format.append( - {"text": "", "stop": params["stop"], "name": spec_var_name} + {"text": "", "stop": params["stop_strs"], "name": spec_var_name} ) return "", {} diff --git a/python/sglang/lang/backend/runtime_endpoint.py b/python/sglang/lang/backend/runtime_endpoint.py index 5012f646ea19..7cdeb78f82dd 100644 --- a/python/sglang/lang/backend/runtime_endpoint.py +++ b/python/sglang/lang/backend/runtime_endpoint.py @@ -101,18 +101,18 @@ def _handle_dtype_to_regex(self, sampling_params: SglSamplingParams): if sampling_params.dtype is None: return - if sampling_params.stop == (): - sampling_params.stop = [] + if sampling_params.stop_strs == (): + sampling_params.stop_strs = [] dtype_regex = None if sampling_params.dtype in ["int", int]: dtype_regex = REGEX_INT - sampling_params.stop.extend([" ", "\n"]) + sampling_params.stop_strs.extend([" ", "\n"]) elif sampling_params.dtype in ["float", float]: dtype_regex = REGEX_FLOAT - sampling_params.stop.extend([" ", "\n"]) + sampling_params.stop_strs.extend([" ", "\n"]) elif sampling_params.dtype in ["str", str]: dtype_regex = REGEX_STR diff --git a/python/sglang/lang/compiler.py b/python/sglang/lang/compiler.py index 95af04adb0a9..5afab4a9710a 100644 --- a/python/sglang/lang/compiler.py +++ b/python/sglang/lang/compiler.py @@ -141,7 +141,7 @@ def run( default_sampling_para = SglSamplingParams( max_new_tokens=max_new_tokens, - stop=stop, + stop_strs=stop, temperature=temperature, top_p=top_p, top_k=top_k, @@ -174,7 +174,7 @@ def run_batch( default_sampling_para = SglSamplingParams( max_new_tokens=max_new_tokens, - stop=stop, + stop_strs=stop, temperature=temperature, top_p=top_p, top_k=top_k, diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py index 844c9d062b87..d28c52f51bfb 100644 --- a/python/sglang/lang/interpreter.py +++ b/python/sglang/lang/interpreter.py @@ -438,7 +438,7 @@ def _execute_video(self, expr: SglVideo): # self.backend.fill_image(self) def _spec_gen(self, sampling_params): - stop = sampling_params.stop + stop = sampling_params.stop_strs max_new_tokens = sampling_params.max_new_tokens meta_info = {} @@ -448,7 +448,7 @@ def regen(): sampling_params.max_new_tokens = max( sampling_params.max_new_tokens, self.num_api_spec_tokens ) - sampling_params.stop = None + sampling_params.stop_strs = None self.speculated_text, meta_info = self.backend.generate( self, sampling_params=sampling_params ) @@ -658,7 +658,7 @@ def _resolve_sampling_params(self, sampling_params): clone = None for item in [ "max_new_tokens", - "stop", + "stop_strs", "stop_token_ids", "temperature", "top_p", @@ -682,11 +682,11 @@ def _resolve_sampling_params(self, sampling_params): if self.chat_template.stop_str: if not clone: clone = self.default_sampling_para.clone() - if clone.stop == (): - clone.stop = [] - elif isinstance(clone.stop, str): - clone.stop = [clone.stop] - clone.stop += self.chat_template.stop_str + if clone.stop_strs == (): + clone.stop_strs = [] + elif isinstance(clone.stop_strs, str): + clone.stop_strs = [clone.stop_strs] + clone.stop_strs += self.chat_template.stop_str return clone or self.default_sampling_para diff --git a/python/sglang/lang/ir.py b/python/sglang/lang/ir.py index 9db5f2719ea1..d78580c73204 100644 --- a/python/sglang/lang/ir.py +++ b/python/sglang/lang/ir.py @@ -17,7 +17,7 @@ @dataclasses.dataclass class SglSamplingParams: max_new_tokens: int = 128 - stop: Union[str, List[str]] = () + stop_strs: Union[str, List[str]] = () stop_token_ids: Optional[List[int]] = () temperature: float = 1.0 top_p: float = 1.0 @@ -37,7 +37,7 @@ class SglSamplingParams: def clone(self): return SglSamplingParams( self.max_new_tokens, - self.stop, + self.stop_strs, self.stop_token_ids, self.temperature, self.top_p, @@ -57,7 +57,7 @@ def to_openai_kwargs(self): warnings.warn("Regular expression is not supported in the OpenAI backend.") return { "max_tokens": self.max_new_tokens, - "stop": self.stop or None, + "stop": self.stop_strs or None, "temperature": self.temperature, "top_p": self.top_p, "frequency_penalty": self.frequency_penalty, @@ -72,7 +72,7 @@ def to_vertexai_kwargs(self): return { "candidate_count": 1, "max_output_tokens": self.max_new_tokens, - "stop_sequences": self.stop, + "stop_sequences": self.stop_strs, "temperature": self.temperature, "top_p": self.top_p, "top_k": self.top_k if self.top_k > 0 else None, @@ -87,7 +87,9 @@ def to_anthropic_kwargs(self): return { "max_tokens": self.max_new_tokens, "stop_sequences": ( - self.stop if isinstance(self.stop, (list, tuple)) else [self.stop] + self.stop_strs + if isinstance(self.stop_strs, (list, tuple)) + else [self.stop_strs] ), "temperature": self.temperature, "top_p": self.top_p, @@ -99,7 +101,7 @@ def to_litellm_kwargs(self): warnings.warn("Regular expression is not supported in the LiteLLM backend.") return { "max_tokens": self.max_new_tokens, - "stop": self.stop or None, + "stop": self.stop_strs or None, "temperature": self.temperature, "top_p": self.top_p, "frequency_penalty": self.frequency_penalty, @@ -109,7 +111,7 @@ def to_litellm_kwargs(self): def to_srt_kwargs(self): return { "max_new_tokens": self.max_new_tokens, - "stop": self.stop, + "stop_strs": self.stop_strs, "stop_token_ids": self.stop_token_ids, "temperature": self.temperature, "top_p": self.top_p, @@ -164,7 +166,7 @@ def run( default_sampling_para = SglSamplingParams( max_new_tokens=max_new_tokens, - stop=stop, + stop_strs=stop, stop_token_ids=stop_token_ids, temperature=temperature, top_p=top_p, @@ -223,7 +225,7 @@ def run_batch( default_sampling_para = SglSamplingParams( max_new_tokens=max_new_tokens, - stop=stop, + stop_strs=stop, stop_token_ids=stop_token_ids, temperature=temperature, top_p=top_p, @@ -423,7 +425,7 @@ def __init__( self.name = name self.sampling_params = SglSamplingParams( max_new_tokens=max_new_tokens, - stop=stop, + stop_strs=stop, stop_token_ids=stop_token_ids, temperature=temperature, top_p=top_p, diff --git a/python/sglang/srt/openai_api/adapter.py b/python/sglang/srt/openai_api/adapter.py index 5d7bb7af7d93..555294bc0244 100644 --- a/python/sglang/srt/openai_api/adapter.py +++ b/python/sglang/srt/openai_api/adapter.py @@ -419,7 +419,7 @@ def v1_generate_request(all_requests: List[CompletionRequest]): "temperature": request.temperature, "max_new_tokens": request.max_tokens, "min_new_tokens": request.min_tokens, - "stop": request.stop, + "stop_strs": request.stop, "stop_token_ids": request.stop_token_ids, "top_p": request.top_p, "presence_penalty": request.presence_penalty, @@ -757,7 +757,7 @@ def v1_chat_generate_request( "temperature": request.temperature, "max_new_tokens": request.max_tokens, "min_new_tokens": request.min_tokens, - "stop": stop, + "stop_strs": stop, "stop_token_ids": request.stop_token_ids, "top_p": request.top_p, "presence_penalty": request.presence_penalty, diff --git a/python/sglang/srt/sampling_params.py b/python/sglang/srt/sampling_params.py index 6a8823cc4de1..b377221f46c1 100644 --- a/python/sglang/srt/sampling_params.py +++ b/python/sglang/srt/sampling_params.py @@ -25,7 +25,7 @@ def __init__( self, max_new_tokens: int = 128, min_new_tokens: int = 0, - stop: Optional[Union[str, List[str]]] = None, + stop_strs: Optional[Union[str, List[str]]] = None, stop_token_ids: Optional[List[int]] = [], temperature: float = 1.0, top_p: float = 1.0, @@ -45,7 +45,7 @@ def __init__( self.frequency_penalty = frequency_penalty self.presence_penalty = presence_penalty self.repetition_penalty = repetition_penalty - self.stop_strs = stop + self.stop_strs = stop_strs self.stop_token_ids = {*stop_token_ids} self.max_new_tokens = max_new_tokens self.min_new_tokens = min_new_tokens diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py index 997b805cc8ce..4a63d0113b82 100644 --- a/python/sglang/srt/server.py +++ b/python/sglang/srt/server.py @@ -521,17 +521,22 @@ async def async_generate( prompt: str, sampling_params: Optional[Dict] = None, ): + if isinstance(sampling_params, dict) and sampling_params.get("n", 1) > 1: + stream = False + else: + stream = True + if self.server_args.skip_tokenizer_init: json_data = { "input_ids": prompt, "sampling_params": sampling_params, - "stream": True, + "stream": stream, } else: json_data = { "text": prompt, "sampling_params": sampling_params, - "stream": True, + "stream": stream, } pos = 0 diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 9f6aa68ab128..8bab69a5d9ea 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -60,7 +60,7 @@ def call_generate_vllm(prompt, temperature, max_tokens, stop=None, n=1, url=None "prompt": prompt, "temperature": temperature, "max_tokens": max_tokens, - "stop": stop, + "stop_strs": stop, "n": n, } res = requests.post(url, json=data) @@ -81,7 +81,7 @@ def call_generate_outlines( "prompt": prompt, "temperature": temperature, "max_tokens": max_tokens, - "stop": stop, + "stop_strs": stop, "regex": regex, "n": n, } @@ -102,7 +102,7 @@ def call_generate_srt_raw(prompt, temperature, max_tokens, stop=None, url=None): "sampling_params": { "temperature": temperature, "max_new_tokens": max_tokens, - "stop": stop, + "stop_strs": stop, }, } res = requests.post(url, json=data)