Skip to content

Optimize uvicorn startup command #25041

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hnyls2002 merged 1 commit into from
May 12, 2026
+4 −17
Merged

Optimize uvicorn startup command #25041

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions python/sglang/srt/entrypoints/http_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,6 @@
MultiTokenizerRouter,
TokenizerWorker,
get_main_process_id,
monkey_patch_uvicorn_multiprocessing,
read_from_shared_memory,
write_data_for_multi_tokenizer,
)
Expand Down Expand Up @@ -2282,7 +2281,6 @@ async def _run_with_ssl_refresh():
"level": "INFO",
"propagate": False,
}
monkey_patch_uvicorn_multiprocessing()

if server_args.enable_ssl_refresh:
logger.warning(
Expand All @@ -2298,6 +2296,7 @@ async def _run_with_ssl_refresh():
root_path=server_args.fastapi_root_path,
log_level=server_args.log_level_http or server_args.log_level,
timeout_keep_alive=envs.SGLANG_TIMEOUT_KEEP_ALIVE.get(),
timeout_worker_healthcheck=envs.SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT.get(),
loop="uvloop",
workers=server_args.tokenizer_worker_num,
ssl_keyfile=server_args.ssl_keyfile,
Expand Down
3 changes: 3 additions & 0 deletions python/sglang/srt/environ.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,9 @@ class Envs:

# HTTP Server
SGLANG_TIMEOUT_KEEP_ALIVE = EnvInt(5)
# Uvicorn multiprocess supervisor pings each worker on this interval; default 5s is
# too short when many workers cold-start and load tokenizers in parallel.
SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT = EnvInt(10)

# HTTP/2 Server
SGLANG_GRANIAN_PARENT_PID = EnvInt(None)
Expand Down
15 changes: 0 additions & 15 deletions python/sglang/srt/managers/multi_tokenizer_mixin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import pickle
import sys
import threading
from functools import partialmethod
from multiprocessing import shared_memory
from typing import TYPE_CHECKING, Any, Dict, Optional, Union

Expand Down Expand Up @@ -591,20 +590,6 @@ def write_data_for_multi_tokenizer(
return args_shm


def monkey_patch_uvicorn_multiprocessing(timeout: float = 10):
"""Monkey patch uvicorn multiprocessing is_alive timeout"""
# from default 5s -> 10s
try:
from uvicorn.supervisors.multiprocess import Process

Process.is_alive = partialmethod(Process.is_alive, timeout=timeout)

except ImportError:
logger.warning(
"uvicorn.supervisors.multiprocess not found, skipping monkey patch"
)


class SenderWrapper:
def __init__(self, port_args: PortArgs, send_to_scheduler: zmq.Socket):
self.port_args = port_args
Expand Down
Loading