From 1ea24996f96dc90478bfedef62bd23b68da2c9a8 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Mon, 15 Jan 2024 08:57:32 +0000 Subject: [PATCH 1/5] fix versionc --- python/pyproject.toml | 6 +++++- python/sglang/__init__.py | 2 +- python/sglang/api.py | 8 +++++++- python/upload_pypi.sh | 3 +++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 84079db35879..c3b87ba29255 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sglang" -version = "0.1.0" +version = "0.1.2" description = "A structured generation langauge for LLMs." readme = "README.md" requires-python = ">=3.8" @@ -24,6 +24,10 @@ openai = ["openai>=1.0"] anthropic = ["anthropic"] all = ["sglang[srt]", "sglang[openai]", "sglang[anthropic]"] +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + [tool.setuptools.packages.find] exclude = ["assets*", "benchmark*", "docs*", "dist*", "playground*", "scripts*", "tests*"] diff --git a/python/sglang/__init__.py b/python/sglang/__init__.py index 0efeb4be08ba..baa8db3a6aec 100644 --- a/python/sglang/__init__.py +++ b/python/sglang/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.0" +__version__ = "0.1.2" from sglang.api import * from sglang.global_config import global_config diff --git a/python/sglang/api.py b/python/sglang/api.py index 6dde54c592fb..b81c41983dbf 100644 --- a/python/sglang/api.py +++ b/python/sglang/api.py @@ -17,13 +17,19 @@ SglRoleEnd, SglSelect, ) -from sglang.srt.server import Runtime def function(func: Callable): return SglFunction(func) +def Runtime(*args, **kwargs): + # Avoid importing unnecessary dependency + from sglang.srt.server import Runtime + + return Runtime(*args, **kwargs) + + def set_default_backend(backend: BaseBackend): global_config.default_backend = backend diff --git a/python/upload_pypi.sh b/python/upload_pypi.sh index b0da77ef2e37..35616e1dad80 100644 --- a/python/upload_pypi.sh +++ b/python/upload_pypi.sh @@ -1,3 +1,6 @@ +cp ../README.md ../LICENSE . rm -rf dist python3 -m build python3 -m twine upload dist/* + +rm -rf README.md LICENSE From 09226c5efb5df55d22e4623f9fd3f63c9958d960 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Mon, 15 Jan 2024 09:03:13 +0000 Subject: [PATCH 2/5] fix test programs --- python/sglang/test/test_programs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sglang/test/test_programs.py b/python/sglang/test/test_programs.py index 38e1029026e9..04d9c12235d1 100644 --- a/python/sglang/test/test_programs.py +++ b/python/sglang/test/test_programs.py @@ -174,7 +174,7 @@ def calculate(expression): def tool_use(s, lhs, rhs): s += "Please perform computations using a calculator. You can use calculate(expression) to get the results.\n" s += "For example,\ncalculate(1+2)=3\ncalculate(3*4)=12\n" - s += "Question: What is the product of " + lhs + " and " + rhs + "?\n" + s += "Question: What is the product of " + str(lhs) + " and " + str(rhs) + "?\n" s += ( "Answer: The answer is calculate(" + sgl.gen("expression", stop=")") From 6bd2c93af8b2d60e1f2fc78f961d69e0dd682149 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Mon, 15 Jan 2024 09:06:25 +0000 Subject: [PATCH 3/5] fix all --- python/sglang/lang/compiler.py | 4 ---- python/sglang/lang/interpreter.py | 1 - test/lang/test_tracing.py | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/python/sglang/lang/compiler.py b/python/sglang/lang/compiler.py index 8f9259096d5b..e4eff89b5304 100644 --- a/python/sglang/lang/compiler.py +++ b/python/sglang/lang/compiler.py @@ -137,7 +137,6 @@ def run( ): backend = backend or global_config.default_backend - kwargs = {k: SglArgument(k, v) for k, v in kwargs.items()} kwargs.update(self.function.bind_arguments) default_sampling_para = SglSamplingParams( @@ -182,9 +181,6 @@ def run_batch( frequency_penalty=frequency_penalty, presence_penalty=presence_penalty, ) - batch_kwargs = [ - {k: SglArgument(k, v) for k, v in kwargs.items()} for kwargs in batch_kwargs - ] # Extract prefix by tracing and cache it if len(batch_kwargs) > 1: diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py index 9486d4406621..de92daafc4fe 100644 --- a/python/sglang/lang/interpreter.py +++ b/python/sglang/lang/interpreter.py @@ -12,7 +12,6 @@ import tqdm from sglang.global_config import global_config from sglang.lang.ir import ( - SglArgument, SglCommitLazy, SglConcateAndAppend, SglConstantText, diff --git a/test/lang/test_tracing.py b/test/lang/test_tracing.py index 4b162983a543..cdc9000d8982 100644 --- a/test/lang/test_tracing.py +++ b/test/lang/test_tracing.py @@ -129,4 +129,4 @@ def tip_suggestion(s): unittest.main(warnings="ignore") # t = TestTracing() - # t.test_fork() + # t.test_multi_function() From b5682417b851b42917c62660997a8449f4a895f8 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Mon, 15 Jan 2024 09:11:12 +0000 Subject: [PATCH 4/5] fix --- benchmark/llava_bench/questions.jsonl | 60 +++++++++++++++++++++++++++ python/sglang/lang/interpreter.py | 4 +- python/sglang/srt/server_args.py | 9 ++-- 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 benchmark/llava_bench/questions.jsonl diff --git a/benchmark/llava_bench/questions.jsonl b/benchmark/llava_bench/questions.jsonl new file mode 100644 index 000000000000..157725502a82 --- /dev/null +++ b/benchmark/llava_bench/questions.jsonl @@ -0,0 +1,60 @@ +{"image": "001.jpg", "text": "What is the name of this famous sight in the photo?", "category": "conv", "question_id": 0} +{"image": "001.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 1} +{"image": "001.jpg", "text": "What are the possible reasons of the formation of this sight?", "category": "complex", "question_id": 2} +{"image": "001.jpg", "text": "Compose an engaging travel blog post about a recent trip to this place, highlighting cultural experiences and must-see attractions, including both the attraction seen in the photo and other must-see attractions as well.", "category": "complex", "question_id": 3} +{"image": "002.jpg", "text": "What type of fruit is this?", "category": "conv", "question_id": 4} +{"image": "002.jpg", "text": "How many uncut fruits are in the image?", "category": "conv", "question_id": 5} +{"image": "002.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 6} +{"image": "002.jpg", "text": "Imagine the fragrance of the fruits in the image. How would you describe this to someone who has never had this fruit before?", "category": "complex", "question_id": 7} +{"image": "003.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 8} +{"image": "003.jpg", "text": "What might be the intended effect of this painting?", "category": "complex", "question_id": 9} +{"image": "003.jpg", "text": "Discuss how this creative twist on a classic work of art might be interpreted differently by various audiences.", "category": "complex", "question_id": 10} +{"image": "004.jpg", "text": "What is the name of the man in the photo?", "category": "conv", "question_id": 11} +{"image": "004.jpg", "text": "Which iconic movie scene is being parodied in the meme?", "category": "conv", "question_id": 12} +{"image": "004.jpg", "text": "How does this meme reflect or comment on Elon Musk's public image, personality, or actions?", "category": "complex", "question_id": 13} +{"image": "005.jpg", "text": "Please explain the meme in detail.", "category": "detail", "question_id": 14} +{"image": "005.jpg", "text": "In what other ways might someone express the same sentiment that this meme is expressing?", "category": "complex", "question_id": 15} +{"image": "006.jpg", "text": "Do you know who paint this?", "category": "conv", "question_id": 16} +{"image": "006.jpg", "text": "Describe this painting in detail.", "category": "detail", "question_id": 17} +{"image": "006.jpg", "text": "Discuss the historical impact and the significance of this painting in the art world.", "category": "complex", "question_id": 18} +{"image": "007.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 19} +{"image": "007.jpg", "text": "What's the best weather, season, time of the day of visiting this place? Is the time when this photo was taken a good time to visit this place?", "category": "complex", "question_id": 20} +{"image": "008.jpg", "text": "What is the name of the character in the image?", "category": "conv", "question_id": 21} +{"image": "008.jpg", "text": "What's the personality of this character? Explain what elements or aspects of the character's design may have contributed to its popularity.", "category": "complex", "question_id": 22} +{"image": "009.jpg", "text": "What are the things I should be cautious about when I visit here?", "category": "complex", "question_id": 23} +{"image": "009.jpg", "text": "If you were a photographer looking to capture this location's essence, what time of day and weather conditions would you choose? Describe the reasons behind your choice.", "category": "complex", "question_id": 24} +{"image": "010.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 25} +{"image": "010.jpg", "text": "What is unusual about this image?", "category": "complex", "question_id": 26} +{"image": "011.jpg", "text": "What fruit is in the left part of the fridge?", "category": "conv", "question_id": 27} +{"image": "011.jpg", "text": "What is the brand of the yogurt flavored with blueberry?", "category": "conv", "question_id": 28} +{"image": "011.jpg", "text": "Is there any strawberry-flavored yogurt in the fridge?", "category": "conv", "question_id": 29} +{"image": "011.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 30} +{"image": "011.jpg", "text": "What are the meals that I can cook with these?", "category": "complex", "question_id": 31} +{"image": "012.jpg", "text": "How many coffee mugs are in the set?", "category": "conv", "question_id": 32} +{"image": "012.jpg", "text": "Write an attractive product description for this.", "category": "complex", "question_id": 33} +{"image": "013.jpg", "text": "Show the detailed recipe for this dish.", "category": "complex", "question_id": 34} +{"image": "014.jpg", "text": "Can you explain this meme in detail?", "category": "complex", "question_id": 35} +{"image": "015.jpg", "text": "What are the two machine learning concepts mentioned in the meme?", "category": "conv", "question_id": 36} +{"image": "015.jpg", "text": "Give a detailed description of this meme.", "category": "detail", "question_id": 37} +{"image": "015.jpg", "text": "Can you explain why this is funny. Think about it step-by-step.", "category": "complex", "question_id": 38} +{"image": "016.jpg", "text": "Give a detailed description of this image. Describe it panel by panel.", "category": "detail", "question_id": 39} +{"image": "016.jpg", "text": "What is funny about this image? Describe it panel by panel.", "category": "complex", "question_id": 40} +{"image": "017.jpg", "text": "What material appears to make up the creature?", "category": "conv", "question_id": 41} +{"image": "017.jpg", "text": "This is the logo of LLaVA, Large Language and Vision Assistant, based on the LLaMA architecture. Please explain this logo in detail, and how do you think of its design.", "category": "complex", "question_id": 42} +{"image": "018.jpg", "text": "What are the animals in the painting and what are they doing?", "category": "conv", "question_id": 43} +{"image": "018.jpg", "text": "Write a fairy tale based on this painting.", "category": "complex", "question_id": 44} +{"image": "019.jpg", "text": "Describe this sketch in detail.", "category": "detail", "question_id": 45} +{"image": "019.jpg", "text": "Write brief HTML/JS to turn this mock-up into a colorful website, where the jokes are replaced by two real jokes.", "category": "complex", "question_id": 46} +{"image": "020.jpg", "text": "Describe this sketch in detail.", "category": "detail", "question_id": 47} +{"image": "020.jpg", "text": "Write brief HTML/JS to turn this mock-up into a colorful and interactive website, where the joke is replaced by a real joke.", "category": "complex", "question_id": 48} +{"image": "021.jpg", "text": "What's the ending of this movie?", "category": "conv", "question_id": 49} +{"image": "021.jpg", "text": "What is the significance of this scene in the context of the movie?", "category": "complex", "question_id": 50} +{"image": "022.jpg", "text": "What's the name of the restaurant serving these dishes?", "category": "conv", "question_id": 51} +{"image": "022.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 52} +{"image": "022.jpg", "text": "If someone were to recommend a new flavor or topping to the dish, describe the reason for this change and how it might alter the overall taste.", "category": "complex", "question_id": 53} +{"image": "023.jpg", "text": "What brand is featured in this advertisement?", "category": "conv", "question_id": 54} +{"image": "023.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 55} +{"image": "023.jpg", "text": "Show me a detailed recipe for cooking this at home.", "category": "complex", "question_id": 56} +{"image": "024.jpg", "text": "Describe this photo in detail.", "category": "detail", "question_id": 57} +{"image": "024.jpg", "text": "What is the problem this city might be facing? What are some possible solutions?", "category": "complex", "question_id": 58} +{"image": "024.jpg", "text": "Explain all the cues that indicate the current traffic conditions.", "category": "complex", "question_id": 59} diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py index de92daafc4fe..342f4e0fb021 100644 --- a/python/sglang/lang/interpreter.py +++ b/python/sglang/lang/interpreter.py @@ -88,7 +88,7 @@ def run_program_batch( for arguments in batch_arguments: rets.append( run_program( - program, backend, (), arguments, default_sampling_para, False, False + program, backend, (), arguments, default_sampling_para, False, True ) ) else: @@ -107,7 +107,7 @@ def run_program_batch( arguments, default_sampling_para, False, - False, + True, ) ) if progress_bar: diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 51d35069c36c..db9d317f73de 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -12,7 +12,7 @@ class ServerArgs: load_format: str = "auto" tokenizer_mode: str = "auto" trust_remote_code: bool = True - mem_fraction_static: float = 0.91 + mem_fraction_static: Optional[float] = None tp_size: int = 1 model_mode: List[str] = () schedule_heuristic: str = "lpm" @@ -24,8 +24,11 @@ class ServerArgs: def __post_init__(self): if self.tokenizer_path is None: self.tokenizer_path = self.model_path - if self.tp_size > 1: - self.mem_fraction_static = 0.8 + if self.mem_fraction_static is None: + if self.tp_size > 1: + self.mem_fraction_static = 0.8 + else: + self.mem_fraction_static = 0.9 @staticmethod def add_cli_args(parser: argparse.ArgumentParser): From 6ae61c8fa2fe9452dbfe9dba17355ffa28e08052 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Mon, 15 Jan 2024 09:14:05 +0000 Subject: [PATCH 5/5] fix --- python/sglang/backend/runtime_endpoint.py | 2 +- python/sglang/lang/compiler.py | 2 +- python/sglang/lang/interpreter.py | 2 +- python/sglang/srt/managers/router/manager.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/sglang/backend/runtime_endpoint.py b/python/sglang/backend/runtime_endpoint.py index 9773d4b390f2..1a58cf75da07 100644 --- a/python/sglang/backend/runtime_endpoint.py +++ b/python/sglang/backend/runtime_endpoint.py @@ -7,7 +7,7 @@ from sglang.global_config import global_config from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.lang.interpreter import StreamExecutor -from sglang.lang.ir import SglSamplingParams, SglArgument +from sglang.lang.ir import SglArgument, SglSamplingParams from sglang.utils import encode_image_base64, find_printable_text, http_request diff --git a/python/sglang/lang/compiler.py b/python/sglang/lang/compiler.py index e4eff89b5304..2c071e407e6b 100644 --- a/python/sglang/lang/compiler.py +++ b/python/sglang/lang/compiler.py @@ -6,10 +6,10 @@ from sglang.global_config import global_config from sglang.lang.interpreter import ProgramState, StreamExecutor, pin_program from sglang.lang.ir import ( - SglSamplingParams, SglArgument, SglConstantText, SglExpr, + SglSamplingParams, SglVariable, ) diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py index 342f4e0fb021..b9a4c184ba5d 100644 --- a/python/sglang/lang/interpreter.py +++ b/python/sglang/lang/interpreter.py @@ -477,7 +477,7 @@ def _resolve_sampling_params(self, sampling_params): "top_k", "frequency_penalty", "presence_penalty", - "ignore_eos", + "ignore_eos", "dtype", "regex", ]: diff --git a/python/sglang/srt/managers/router/manager.py b/python/sglang/srt/managers/router/manager.py index b6abc77c56eb..9d848b9a7268 100644 --- a/python/sglang/srt/managers/router/manager.py +++ b/python/sglang/srt/managers/router/manager.py @@ -4,10 +4,10 @@ import uvloop import zmq import zmq.asyncio +from sglang.srt.backend_config import GLOBAL_BACKEND_CONFIG from sglang.srt.managers.router.model_rpc import ModelRpcClient from sglang.srt.server_args import PortArgs, ServerArgs from sglang.srt.utils import get_exception_traceback -from sglang.srt.backend_config import GLOBAL_BACKEND_CONFIG asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())