From 5977e9d16d450d1d2c6653721d5e1ab251657863 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 13 Aug 2024 20:21:43 +0800
Subject: [PATCH 01/15] Fix create_abort_task, GenerateReqInput does not have
 rids.

---
 python/sglang/srt/managers/tokenizer_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py
index e1bfbc7e6708..d5fbfe05d3b5 100644
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -507,7 +507,7 @@ async def abort_request():
             if obj.is_single:
                 self.abort_request(obj.rid)
             else:
-                for rid in obj.rids:
+                for rid in obj.rid:
                     self.abort_request(rid)
 
         background_tasks = BackgroundTasks()

From 04fae70563dd3ef444e8e6e6262b4bfc11859d61 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Wed, 14 Aug 2024 12:01:17 +0800
Subject: [PATCH 02/15] set input_ids instead of text if skip_tokenizer_init is
 set.

---
 python/sglang/srt/server.py | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py
index 7331425fae9a..350830790465 100644
--- a/python/sglang/srt/server.py
+++ b/python/sglang/srt/server.py
@@ -533,13 +533,22 @@ async def async_generate(
         prompt: str,
         sampling_params: Optional[Dict] = None,
     ):
-        json_data = {
-            "text": prompt,
-            "sampling_params": sampling_params,
-            "stream": True,
-        }
+        if self.server_args.skip_tokenizer_init:
+            json_data = {
+                "input_ids": prompt,
+                "sampling_params": sampling_params,
+                "stream": True,
+            }
+        else:
+            json_data = {
+                "text": prompt,
+                "sampling_params": sampling_params,
+                "stream": True,
+            }
         pos = 0
 
+        print(json_data)
+        print(self.generate_url)
         timeout = aiohttp.ClientTimeout(total=3 * 3600)
         async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
             async with session.post(self.generate_url, json=json_data) as response:
@@ -549,10 +558,13 @@ async def async_generate(
                         if chunk == "data: [DONE]\n\n":
                             break
                         data = json.loads(chunk[5:].strip("\n"))
-                        cur = data["text"][pos:]
-                        if cur:
-                            yield cur
-                        pos += len(cur)
+                        if hasattr(data, 'text'):
+                            cur = data["text"][pos:]
+                            if cur:
+                                yield cur
+                            pos += len(cur)
+                        else:
+                            yield data
 
     add_request = async_generate
 

From 3e97ba69cae724e1f6e8749e590c9386f26d7f67 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Wed, 14 Aug 2024 12:48:03 +0800
Subject: [PATCH 03/15] Remove unnecessary print.

---
 python/sglang/srt/server.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py
index 350830790465..7a7bd25a3f8c 100644
--- a/python/sglang/srt/server.py
+++ b/python/sglang/srt/server.py
@@ -547,8 +547,6 @@ async def async_generate(
             }
         pos = 0
 
-        print(json_data)
-        print(self.generate_url)
         timeout = aiohttp.ClientTimeout(total=3 * 3600)
         async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
             async with session.post(self.generate_url, json=json_data) as response:

From 5cd2f79856c83ccaf25b935a6d9cd76dc688376e Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Wed, 14 Aug 2024 12:49:28 +0800
Subject: [PATCH 04/15] Fix lint.

---
 python/sglang/srt/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py
index 7a7bd25a3f8c..8f735ac0c749 100644
--- a/python/sglang/srt/server.py
+++ b/python/sglang/srt/server.py
@@ -556,7 +556,7 @@ async def async_generate(
                         if chunk == "data: [DONE]\n\n":
                             break
                         data = json.loads(chunk[5:].strip("\n"))
-                        if hasattr(data, 'text'):
+                        if hasattr(data, "text"):
                             cur = data["text"][pos:]
                             if cur:
                                 yield cur

From 55365790f56193058f0bde7505313307e3ba07ed Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 11:41:18 +0800
Subject: [PATCH 05/15] Modify sampling params to better support json.loads and
 json.dumps

---
 python/sglang/srt/sampling_params.py | 4 ++--
 python/sglang/srt/server.py          | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/python/sglang/srt/sampling_params.py b/python/sglang/srt/sampling_params.py
index 6a8823cc4de1..b377221f46c1 100644
--- a/python/sglang/srt/sampling_params.py
+++ b/python/sglang/srt/sampling_params.py
@@ -25,7 +25,7 @@ def __init__(
         self,
         max_new_tokens: int = 128,
         min_new_tokens: int = 0,
-        stop: Optional[Union[str, List[str]]] = None,
+        stop_strs: Optional[Union[str, List[str]]] = None,
         stop_token_ids: Optional[List[int]] = [],
         temperature: float = 1.0,
         top_p: float = 1.0,
@@ -45,7 +45,7 @@ def __init__(
         self.frequency_penalty = frequency_penalty
         self.presence_penalty = presence_penalty
         self.repetition_penalty = repetition_penalty
-        self.stop_strs = stop
+        self.stop_strs = stop_strs
         self.stop_token_ids = {*stop_token_ids}
         self.max_new_tokens = max_new_tokens
         self.min_new_tokens = min_new_tokens
diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py
index 997b805cc8ce..44be29eada95 100644
--- a/python/sglang/srt/server.py
+++ b/python/sglang/srt/server.py
@@ -521,17 +521,21 @@ async def async_generate(
         prompt: str,
         sampling_params: Optional[Dict] = None,
     ):
+        if isinstance(sampling_params, dict) and sampling_params.get("n", 1) > 1:
+            stream = False
+        else:
+            stream = True
         if self.server_args.skip_tokenizer_init:
             json_data = {
                 "input_ids": prompt,
                 "sampling_params": sampling_params,
-                "stream": True,
+                "stream": stream,
             }
         else:
             json_data = {
                 "text": prompt,
                 "sampling_params": sampling_params,
-                "stream": True,
+                "stream": stream,
             }
         pos = 0
 

From 67cf7e564b38c39889f1b07d8443d9e2dcba30e3 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 11:42:57 +0800
Subject: [PATCH 06/15] Revert "Modify sampling params to better support
 json.loads and json.dumps"

This reverts commit 55365790f56193058f0bde7505313307e3ba07ed.
---
 python/sglang/srt/sampling_params.py | 4 ++--
 python/sglang/srt/server.py          | 8 ++------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/python/sglang/srt/sampling_params.py b/python/sglang/srt/sampling_params.py
index b377221f46c1..6a8823cc4de1 100644
--- a/python/sglang/srt/sampling_params.py
+++ b/python/sglang/srt/sampling_params.py
@@ -25,7 +25,7 @@ def __init__(
         self,
         max_new_tokens: int = 128,
         min_new_tokens: int = 0,
-        stop_strs: Optional[Union[str, List[str]]] = None,
+        stop: Optional[Union[str, List[str]]] = None,
         stop_token_ids: Optional[List[int]] = [],
         temperature: float = 1.0,
         top_p: float = 1.0,
@@ -45,7 +45,7 @@ def __init__(
         self.frequency_penalty = frequency_penalty
         self.presence_penalty = presence_penalty
         self.repetition_penalty = repetition_penalty
-        self.stop_strs = stop_strs
+        self.stop_strs = stop
         self.stop_token_ids = {*stop_token_ids}
         self.max_new_tokens = max_new_tokens
         self.min_new_tokens = min_new_tokens
diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py
index 44be29eada95..997b805cc8ce 100644
--- a/python/sglang/srt/server.py
+++ b/python/sglang/srt/server.py
@@ -521,21 +521,17 @@ async def async_generate(
         prompt: str,
         sampling_params: Optional[Dict] = None,
     ):
-        if isinstance(sampling_params, dict) and sampling_params.get("n", 1) > 1:
-            stream = False
-        else:
-            stream = True
         if self.server_args.skip_tokenizer_init:
             json_data = {
                 "input_ids": prompt,
                 "sampling_params": sampling_params,
-                "stream": stream,
+                "stream": True,
             }
         else:
             json_data = {
                 "text": prompt,
                 "sampling_params": sampling_params,
-                "stream": stream,
+                "stream": True,
             }
         pos = 0
 

From 8461cadfbb628842d24d962181ccbf217438416a Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 11:44:13 +0800
Subject: [PATCH 07/15] Modify sampling params to better support json.loads and
 json.dumps

---
 python/sglang/srt/sampling_params.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/sglang/srt/sampling_params.py b/python/sglang/srt/sampling_params.py
index 6a8823cc4de1..b377221f46c1 100644
--- a/python/sglang/srt/sampling_params.py
+++ b/python/sglang/srt/sampling_params.py
@@ -25,7 +25,7 @@ def __init__(
         self,
         max_new_tokens: int = 128,
         min_new_tokens: int = 0,
-        stop: Optional[Union[str, List[str]]] = None,
+        stop_strs: Optional[Union[str, List[str]]] = None,
         stop_token_ids: Optional[List[int]] = [],
         temperature: float = 1.0,
         top_p: float = 1.0,
@@ -45,7 +45,7 @@ def __init__(
         self.frequency_penalty = frequency_penalty
         self.presence_penalty = presence_penalty
         self.repetition_penalty = repetition_penalty
-        self.stop_strs = stop
+        self.stop_strs = stop_strs
         self.stop_token_ids = {*stop_token_ids}
         self.max_new_tokens = max_new_tokens
         self.min_new_tokens = min_new_tokens

From fd84464aadc2b56c5bc43f19cfb6b13cfe44fe53 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 11:46:11 +0800
Subject: [PATCH 08/15] when n in sampling_params is greater than 1 will hit
 batch mode, should set stream to False.

---
 python/sglang/srt/server.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py
index 997b805cc8ce..4a63d0113b82 100644
--- a/python/sglang/srt/server.py
+++ b/python/sglang/srt/server.py
@@ -521,17 +521,22 @@ async def async_generate(
         prompt: str,
         sampling_params: Optional[Dict] = None,
     ):
+        if isinstance(sampling_params, dict) and sampling_params.get("n", 1) > 1:
+            stream = False
+        else:
+            stream = True
+
         if self.server_args.skip_tokenizer_init:
             json_data = {
                 "input_ids": prompt,
                 "sampling_params": sampling_params,
-                "stream": True,
+                "stream": stream,
             }
         else:
             json_data = {
                 "text": prompt,
                 "sampling_params": sampling_params,
-                "stream": True,
+                "stream": stream,
             }
         pos = 0
 

From 1491962e731e72df83335bd551fcc481cca1f602 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 11:53:55 +0800
Subject: [PATCH 09/15] Update sampling_params.md accordingly.

---
 docs/en/sampling_params.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/sampling_params.md b/docs/en/sampling_params.md
index 5f1cdece6a25..e1748d77ab78 100644
--- a/docs/en/sampling_params.md
+++ b/docs/en/sampling_params.md
@@ -35,7 +35,7 @@ The `sampling_params` follows this format
 # The maximum number of output tokens
 max_new_tokens: int = 128,
 # Stop when hitting any of the strings in this list.
-stop: Optional[Union[str, List[str]]] = None,
+stop_strs: Optional[Union[str, List[str]]] = None,
 # Stop when hitting any of the token_ids in this list. Could be useful when mixed with
 # `min_new_tokens`.
 stop_token_ids: Optional[List[int]] = [],
@@ -72,8 +72,8 @@ presence_penalty: float = 0.0,
 repetition_penalty: float = 1.0,
 # Guides inference to generate at least this number of tokens by penalizing logits of tokenizer's
 # EOS token and `stop_token_ids` to -inf, until the output token reaches given length.
-# Note that any of the `stop` string can be generated before reaching `min_new_tokens`, as it is
-# difficult to infer the correct token ID by given `stop` strings.
+# Note that any of the `stop_strs` string can be generated before reaching `min_new_tokens`, as it is
+# difficult to infer the correct token ID by given `stop_strs` strings.
 # Must be 0 <= value < max_new_tokens. Setting to 0 (default) will disable this penalty.
 min_new_tokens: int = 0,
 ```

From 17e6b5b938d07079c0ac83ff1c47f1f46618d8f6 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 14:38:28 +0800
Subject: [PATCH 10/15] Modify test_utils to set stop_strs in sampling_params.

---
 python/sglang/test/test_utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py
index 9f6aa68ab128..8bab69a5d9ea 100644
--- a/python/sglang/test/test_utils.py
+++ b/python/sglang/test/test_utils.py
@@ -60,7 +60,7 @@ def call_generate_vllm(prompt, temperature, max_tokens, stop=None, n=1, url=None
         "prompt": prompt,
         "temperature": temperature,
         "max_tokens": max_tokens,
-        "stop": stop,
+        "stop_strs": stop,
         "n": n,
     }
     res = requests.post(url, json=data)
@@ -81,7 +81,7 @@ def call_generate_outlines(
         "prompt": prompt,
         "temperature": temperature,
         "max_tokens": max_tokens,
-        "stop": stop,
+        "stop_strs": stop,
         "regex": regex,
         "n": n,
     }
@@ -102,7 +102,7 @@ def call_generate_srt_raw(prompt, temperature, max_tokens, stop=None, url=None):
         "sampling_params": {
             "temperature": temperature,
             "max_new_tokens": max_tokens,
-            "stop": stop,
+            "stop_strs": stop,
         },
     }
     res = requests.post(url, json=data)

From 46d8574de96f3336dcf1eb025dc2283e848aa1ff Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 14:46:48 +0800
Subject: [PATCH 11/15] Update openapi adapter params.

---
 python/sglang/srt/openai_api/adapter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/sglang/srt/openai_api/adapter.py b/python/sglang/srt/openai_api/adapter.py
index 5d7bb7af7d93..555294bc0244 100644
--- a/python/sglang/srt/openai_api/adapter.py
+++ b/python/sglang/srt/openai_api/adapter.py
@@ -419,7 +419,7 @@ def v1_generate_request(all_requests: List[CompletionRequest]):
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
                 "min_new_tokens": request.min_tokens,
-                "stop": request.stop,
+                "stop_strs": request.stop,
                 "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,
@@ -757,7 +757,7 @@ def v1_chat_generate_request(
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
                 "min_new_tokens": request.min_tokens,
-                "stop": stop,
+                "stop_strs": stop,
                 "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,

From 12b36c1454048b8c68c9ceb041bb3107cbe9a616 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 15:57:30 +0800
Subject: [PATCH 12/15] Update SglSamplingParams.

---
 python/sglang/lang/ir.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/python/sglang/lang/ir.py b/python/sglang/lang/ir.py
index 9db5f2719ea1..1d1c02a59485 100644
--- a/python/sglang/lang/ir.py
+++ b/python/sglang/lang/ir.py
@@ -17,7 +17,7 @@
 @dataclasses.dataclass
 class SglSamplingParams:
     max_new_tokens: int = 128
-    stop: Union[str, List[str]] = ()
+    stop_strs: Union[str, List[str]] = ()
     stop_token_ids: Optional[List[int]] = ()
     temperature: float = 1.0
     top_p: float = 1.0
@@ -37,7 +37,7 @@ class SglSamplingParams:
     def clone(self):
         return SglSamplingParams(
             self.max_new_tokens,
-            self.stop,
+            self.stop_strs,
             self.stop_token_ids,
             self.temperature,
             self.top_p,
@@ -57,7 +57,7 @@ def to_openai_kwargs(self):
             warnings.warn("Regular expression is not supported in the OpenAI backend.")
         return {
             "max_tokens": self.max_new_tokens,
-            "stop": self.stop or None,
+            "stop": self.stop_strs or None,
             "temperature": self.temperature,
             "top_p": self.top_p,
             "frequency_penalty": self.frequency_penalty,
@@ -72,7 +72,7 @@ def to_vertexai_kwargs(self):
         return {
             "candidate_count": 1,
             "max_output_tokens": self.max_new_tokens,
-            "stop_sequences": self.stop,
+            "stop_sequences": self.stop_strs,
             "temperature": self.temperature,
             "top_p": self.top_p,
             "top_k": self.top_k if self.top_k > 0 else None,
@@ -87,7 +87,7 @@ def to_anthropic_kwargs(self):
         return {
             "max_tokens": self.max_new_tokens,
             "stop_sequences": (
-                self.stop if isinstance(self.stop, (list, tuple)) else [self.stop]
+                self.stop_strs if isinstance(self.stop_strs, (list, tuple)) else [self.stop_strs]
             ),
             "temperature": self.temperature,
             "top_p": self.top_p,
@@ -99,7 +99,7 @@ def to_litellm_kwargs(self):
             warnings.warn("Regular expression is not supported in the LiteLLM backend.")
         return {
             "max_tokens": self.max_new_tokens,
-            "stop": self.stop or None,
+            "stop": self.stop_strs or None,
             "temperature": self.temperature,
             "top_p": self.top_p,
             "frequency_penalty": self.frequency_penalty,
@@ -109,7 +109,7 @@ def to_litellm_kwargs(self):
     def to_srt_kwargs(self):
         return {
             "max_new_tokens": self.max_new_tokens,
-            "stop": self.stop,
+            "stop_strs": self.stop_strs,
             "stop_token_ids": self.stop_token_ids,
             "temperature": self.temperature,
             "top_p": self.top_p,
@@ -164,7 +164,7 @@ def run(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             stop_token_ids=stop_token_ids,
             temperature=temperature,
             top_p=top_p,
@@ -223,7 +223,7 @@ def run_batch(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             stop_token_ids=stop_token_ids,
             temperature=temperature,
             top_p=top_p,
@@ -423,7 +423,7 @@ def __init__(
         self.name = name
         self.sampling_params = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             stop_token_ids=stop_token_ids,
             temperature=temperature,
             top_p=top_p,

From 25c207ddd3fb8e71482b6c8b9a491bb5e0dfea0e Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 15:59:00 +0800
Subject: [PATCH 13/15] Fix lint.

---
 python/sglang/lang/ir.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/sglang/lang/ir.py b/python/sglang/lang/ir.py
index 1d1c02a59485..d78580c73204 100644
--- a/python/sglang/lang/ir.py
+++ b/python/sglang/lang/ir.py
@@ -87,7 +87,9 @@ def to_anthropic_kwargs(self):
         return {
             "max_tokens": self.max_new_tokens,
             "stop_sequences": (
-                self.stop_strs if isinstance(self.stop_strs, (list, tuple)) else [self.stop_strs]
+                self.stop_strs
+                if isinstance(self.stop_strs, (list, tuple))
+                else [self.stop_strs]
             ),
             "temperature": self.temperature,
             "top_p": self.top_p,

From e74f68aa9a52a6afa1b7d7ff1431dd3fc3f55843 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 16:36:59 +0800
Subject: [PATCH 14/15] update interpreter params.

---
 python/sglang/lang/interpreter.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py
index 844c9d062b87..d28c52f51bfb 100644
--- a/python/sglang/lang/interpreter.py
+++ b/python/sglang/lang/interpreter.py
@@ -438,7 +438,7 @@ def _execute_video(self, expr: SglVideo):
         #     self.backend.fill_image(self)
 
     def _spec_gen(self, sampling_params):
-        stop = sampling_params.stop
+        stop = sampling_params.stop_strs
         max_new_tokens = sampling_params.max_new_tokens
         meta_info = {}
 
@@ -448,7 +448,7 @@ def regen():
             sampling_params.max_new_tokens = max(
                 sampling_params.max_new_tokens, self.num_api_spec_tokens
             )
-            sampling_params.stop = None
+            sampling_params.stop_strs = None
             self.speculated_text, meta_info = self.backend.generate(
                 self, sampling_params=sampling_params
             )
@@ -658,7 +658,7 @@ def _resolve_sampling_params(self, sampling_params):
         clone = None
         for item in [
             "max_new_tokens",
-            "stop",
+            "stop_strs",
             "stop_token_ids",
             "temperature",
             "top_p",
@@ -682,11 +682,11 @@ def _resolve_sampling_params(self, sampling_params):
         if self.chat_template.stop_str:
             if not clone:
                 clone = self.default_sampling_para.clone()
-            if clone.stop == ():
-                clone.stop = []
-            elif isinstance(clone.stop, str):
-                clone.stop = [clone.stop]
-            clone.stop += self.chat_template.stop_str
+            if clone.stop_strs == ():
+                clone.stop_strs = []
+            elif isinstance(clone.stop_strs, str):
+                clone.stop_strs = [clone.stop_strs]
+            clone.stop_strs += self.chat_template.stop_str
 
         return clone or self.default_sampling_para
 

From bc753e26991909be7845b72b5888b7158f24c053 Mon Sep 17 00:00:00 2001
From: lzhang <zhanglei@modelbest.cn>
Date: Tue, 20 Aug 2024 17:17:23 +0800
Subject: [PATCH 15/15] Update more params.stop references.

---
 python/sglang/lang/backend/openai.py           | 4 ++--
 python/sglang/lang/backend/runtime_endpoint.py | 8 ++++----
 python/sglang/lang/compiler.py                 | 4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/python/sglang/lang/backend/openai.py b/python/sglang/lang/backend/openai.py
index 6fa93d9b2eb8..0f68c4e9c9f6 100644
--- a/python/sglang/lang/backend/openai.py
+++ b/python/sglang/lang/backend/openai.py
@@ -119,7 +119,7 @@ def _prepare_spec_execution(
 
         params = sampling_params.to_openai_kwargs()
         for key, value in params.items():
-            if key in ["stop"]:
+            if key in ["stop_strs"]:
                 continue
             if key in ["max_tokens"]:
                 warnings.warn(
@@ -133,7 +133,7 @@ def _prepare_spec_execution(
                     value == self.spec_kwargs[key]
                 ), "sampling parameters should be consistent if turn on api speculative execution."
         self.spec_format.append(
-            {"text": "", "stop": params["stop"], "name": spec_var_name}
+            {"text": "", "stop": params["stop_strs"], "name": spec_var_name}
         )
         return "", {}
 
diff --git a/python/sglang/lang/backend/runtime_endpoint.py b/python/sglang/lang/backend/runtime_endpoint.py
index 5012f646ea19..7cdeb78f82dd 100644
--- a/python/sglang/lang/backend/runtime_endpoint.py
+++ b/python/sglang/lang/backend/runtime_endpoint.py
@@ -101,18 +101,18 @@ def _handle_dtype_to_regex(self, sampling_params: SglSamplingParams):
         if sampling_params.dtype is None:
             return
 
-        if sampling_params.stop == ():
-            sampling_params.stop = []
+        if sampling_params.stop_strs == ():
+            sampling_params.stop_strs = []
 
         dtype_regex = None
         if sampling_params.dtype in ["int", int]:
 
             dtype_regex = REGEX_INT
-            sampling_params.stop.extend([" ", "\n"])
+            sampling_params.stop_strs.extend([" ", "\n"])
         elif sampling_params.dtype in ["float", float]:
 
             dtype_regex = REGEX_FLOAT
-            sampling_params.stop.extend([" ", "\n"])
+            sampling_params.stop_strs.extend([" ", "\n"])
         elif sampling_params.dtype in ["str", str]:
 
             dtype_regex = REGEX_STR
diff --git a/python/sglang/lang/compiler.py b/python/sglang/lang/compiler.py
index 95af04adb0a9..5afab4a9710a 100644
--- a/python/sglang/lang/compiler.py
+++ b/python/sglang/lang/compiler.py
@@ -141,7 +141,7 @@ def run(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             temperature=temperature,
             top_p=top_p,
             top_k=top_k,
@@ -174,7 +174,7 @@ def run_batch(
 
         default_sampling_para = SglSamplingParams(
             max_new_tokens=max_new_tokens,
-            stop=stop,
+            stop_strs=stop,
             temperature=temperature,
             top_p=top_p,
             top_k=top_k,