[Fix] sampling_params and Runtime bugs for some cases.

docs/en/sampling_params.md

@@ -35,7 +35,7 @@ The `sampling_params` follows this format
 # The maximum number of output tokens
 max_new_tokens: int = 128,
 # Stop when hitting any of the strings in this list.
-stop: Optional[Union[str, List[str]]] = None,
+stop_strs: Optional[Union[str, List[str]]] = None,
 # Stop when hitting any of the token_ids in this list. Could be useful when mixed with
 # `min_new_tokens`.
 stop_token_ids: Optional[List[int]] = [],
@@ -72,8 +72,8 @@ presence_penalty: float = 0.0,
 repetition_penalty: float = 1.0,
 # Guides inference to generate at least this number of tokens by penalizing logits of tokenizer's
 # EOS token and `stop_token_ids` to -inf, until the output token reaches given length.
-# Note that any of the `stop` string can be generated before reaching `min_new_tokens`, as it is
-# difficult to infer the correct token ID by given `stop` strings.
+# Note that any of the `stop_strs` string can be generated before reaching `min_new_tokens`, as it is
+# difficult to infer the correct token ID by given `stop_strs` strings.
 # Must be 0 <= value < max_new_tokens. Setting to 0 (default) will disable this penalty.
 min_new_tokens: int = 0,
 ```

python/sglang/lang/backend/openai.py

@@ -119,7 +119,7 @@ def _prepare_spec_execution(
 
         params = sampling_params.to_openai_kwargs()
         for key, value in params.items():
-            if key in ["stop"]:
+            if key in ["stop_strs"]:
                 continue
             if key in ["max_tokens"]:
                 warnings.warn(
@@ -133,7 +133,7 @@ def _prepare_spec_execution(
                     value == self.spec_kwargs[key]
                 ), "sampling parameters should be consistent if turn on api speculative execution."
         self.spec_format.append(
-            {"text": "", "stop": params["stop"], "name": spec_var_name}
+            {"text": "", "stop": params["stop_strs"], "name": spec_var_name}
         )
         return "", {}
 

python/sglang/lang/backend/runtime_endpoint.py

@@ -101,18 +101,18 @@ def _handle_dtype_to_regex(self, sampling_params: SglSamplingParams):
         if sampling_params.dtype is None:
             return
 
-        if sampling_params.stop == ():
-            sampling_params.stop = []
+        if sampling_params.stop_strs == ():
+            sampling_params.stop_strs = []
 
         dtype_regex = None
         if sampling_params.dtype in ["int", int]:
 
             dtype_regex = REGEX_INT
-            sampling_params.stop.extend([" ", "\n"])
+            sampling_params.stop_strs.extend([" ", "\n"])
         elif sampling_params.dtype in ["float", float]:
 
             dtype_regex = REGEX_FLOAT
-            sampling_params.stop.extend([" ", "\n"])
+            sampling_params.stop_strs.extend([" ", "\n"])
         elif sampling_params.dtype in ["str", str]:
 
             dtype_regex = REGEX_STR

python/sglang/lang/compiler.py

@@ -141,7 +141,7 @@ def run(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             temperature=temperature,
             top_p=top_p,
             top_k=top_k,
@@ -174,7 +174,7 @@ def run_batch(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             temperature=temperature,
             top_p=top_p,
             top_k=top_k,

python/sglang/lang/interpreter.py

@@ -438,7 +438,7 @@ def _execute_video(self, expr: SglVideo):
         #     self.backend.fill_image(self)
 
     def _spec_gen(self, sampling_params):
-        stop = sampling_params.stop
+        stop = sampling_params.stop_strs
         max_new_tokens = sampling_params.max_new_tokens
         meta_info = {}
 
@@ -448,7 +448,7 @@ def regen():
             sampling_params.max_new_tokens = max(
                 sampling_params.max_new_tokens, self.num_api_spec_tokens
             )
-            sampling_params.stop = None
+            sampling_params.stop_strs = None
             self.speculated_text, meta_info = self.backend.generate(
                 self, sampling_params=sampling_params
             )
@@ -658,7 +658,7 @@ def _resolve_sampling_params(self, sampling_params):
         clone = None
         for item in [
             "max_new_tokens",
-            "stop",
+            "stop_strs",
             "stop_token_ids",
             "temperature",
             "top_p",
@@ -682,11 +682,11 @@ def _resolve_sampling_params(self, sampling_params):
         if self.chat_template.stop_str:
             if not clone:
                 clone = self.default_sampling_para.clone()
-            if clone.stop == ():
-                clone.stop = []
-            elif isinstance(clone.stop, str):
-                clone.stop = [clone.stop]
-            clone.stop += self.chat_template.stop_str
+            if clone.stop_strs == ():
+                clone.stop_strs = []
+            elif isinstance(clone.stop_strs, str):
+                clone.stop_strs = [clone.stop_strs]
+            clone.stop_strs += self.chat_template.stop_str
 
         return clone or self.default_sampling_para
 

python/sglang/lang/ir.py

@@ -17,7 +17,7 @@
 @dataclasses.dataclass
 class SglSamplingParams:
     max_new_tokens: int = 128
-    stop: Union[str, List[str]] = ()
+    stop_strs: Union[str, List[str]] = ()
     stop_token_ids: Optional[List[int]] = ()
     temperature: float = 1.0
     top_p: float = 1.0
@@ -37,7 +37,7 @@ class SglSamplingParams:
     def clone(self):
         return SglSamplingParams(
             self.max_new_tokens,
-            self.stop,
+            self.stop_strs,
             self.stop_token_ids,
             self.temperature,
             self.top_p,
@@ -57,7 +57,7 @@ def to_openai_kwargs(self):
             warnings.warn("Regular expression is not supported in the OpenAI backend.")
         return {
             "max_tokens": self.max_new_tokens,
-            "stop": self.stop or None,
+            "stop": self.stop_strs or None,
             "temperature": self.temperature,
             "top_p": self.top_p,
             "frequency_penalty": self.frequency_penalty,
@@ -72,7 +72,7 @@ def to_vertexai_kwargs(self):
         return {
             "candidate_count": 1,
             "max_output_tokens": self.max_new_tokens,
-            "stop_sequences": self.stop,
+            "stop_sequences": self.stop_strs,
             "temperature": self.temperature,
             "top_p": self.top_p,
             "top_k": self.top_k if self.top_k > 0 else None,
@@ -87,7 +87,9 @@ def to_anthropic_kwargs(self):
         return {
             "max_tokens": self.max_new_tokens,
             "stop_sequences": (
-                self.stop if isinstance(self.stop, (list, tuple)) else [self.stop]
+                self.stop_strs
+                if isinstance(self.stop_strs, (list, tuple))
+                else [self.stop_strs]
             ),
             "temperature": self.temperature,
             "top_p": self.top_p,
@@ -99,7 +101,7 @@ def to_litellm_kwargs(self):
             warnings.warn("Regular expression is not supported in the LiteLLM backend.")
         return {
             "max_tokens": self.max_new_tokens,
-            "stop": self.stop or None,
+            "stop": self.stop_strs or None,
             "temperature": self.temperature,
             "top_p": self.top_p,
             "frequency_penalty": self.frequency_penalty,
@@ -109,7 +111,7 @@ def to_litellm_kwargs(self):
     def to_srt_kwargs(self):
         return {
             "max_new_tokens": self.max_new_tokens,
-            "stop": self.stop,
+            "stop_strs": self.stop_strs,
             "stop_token_ids": self.stop_token_ids,
             "temperature": self.temperature,
             "top_p": self.top_p,
@@ -164,7 +166,7 @@ def run(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             stop_token_ids=stop_token_ids,
             temperature=temperature,
             top_p=top_p,
@@ -223,7 +225,7 @@ def run_batch(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             stop_token_ids=stop_token_ids,
             temperature=temperature,
             top_p=top_p,
@@ -423,7 +425,7 @@ def __init__(
         self.name = name
         self.sampling_params = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             stop_token_ids=stop_token_ids,
             temperature=temperature,
             top_p=top_p,

python/sglang/srt/openai_api/adapter.py

@@ -419,7 +419,7 @@ def v1_generate_request(all_requests: List[CompletionRequest]):
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
                 "min_new_tokens": request.min_tokens,
-                "stop": request.stop,
+                "stop_strs": request.stop,
                 "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,
@@ -757,7 +757,7 @@ def v1_chat_generate_request(
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
                 "min_new_tokens": request.min_tokens,
-                "stop": stop,
+                "stop_strs": stop,
                 "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,

python/sglang/srt/sampling_params.py

@@ -25,7 +25,7 @@ def __init__(
         self,
         max_new_tokens: int = 128,
         min_new_tokens: int = 0,
-        stop: Optional[Union[str, List[str]]] = None,
+        stop_strs: Optional[Union[str, List[str]]] = None,
         stop_token_ids: Optional[List[int]] = [],
         temperature: float = 1.0,
         top_p: float = 1.0,
@@ -45,7 +45,7 @@ def __init__(
         self.frequency_penalty = frequency_penalty
         self.presence_penalty = presence_penalty
         self.repetition_penalty = repetition_penalty
-        self.stop_strs = stop
+        self.stop_strs = stop_strs
         self.stop_token_ids = {*stop_token_ids}
         self.max_new_tokens = max_new_tokens
         self.min_new_tokens = min_new_tokens

python/sglang/srt/server.py

@@ -521,17 +521,22 @@ async def async_generate(
         prompt: str,
         sampling_params: Optional[Dict] = None,
     ):
+        if isinstance(sampling_params, dict) and sampling_params.get("n", 1) > 1:
+            stream = False
+        else:
+            stream = True
+
         if self.server_args.skip_tokenizer_init:
             json_data = {
                 "input_ids": prompt,
                 "sampling_params": sampling_params,
-                "stream": True,
+                "stream": stream,
             }
         else:
             json_data = {
                 "text": prompt,
                 "sampling_params": sampling_params,
-                "stream": True,
+                "stream": stream,
             }
         pos = 0
 

python/sglang/test/test_utils.py

@@ -60,7 +60,7 @@ def call_generate_vllm(prompt, temperature, max_tokens, stop=None, n=1, url=None
         "prompt": prompt,
         "temperature": temperature,
         "max_tokens": max_tokens,
-        "stop": stop,
+        "stop_strs": stop,
         "n": n,
     }
     res = requests.post(url, json=data)
@@ -81,7 +81,7 @@ def call_generate_outlines(
         "prompt": prompt,
         "temperature": temperature,
         "max_tokens": max_tokens,
-        "stop": stop,
+        "stop_strs": stop,
         "regex": regex,
         "n": n,
     }
@@ -102,7 +102,7 @@ def call_generate_srt_raw(prompt, temperature, max_tokens, stop=None, url=None):
         "sampling_params": {
             "temperature": temperature,
             "max_new_tokens": max_tokens,
-            "stop": stop,
+            "stop_strs": stop,
         },
     }
     res = requests.post(url, json=data)