Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions .github/workflows/pr-test-rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,35 @@ jobs:
- id: check
uses: ./.github/actions/check-ci

python-lint:
needs: check-ci
if: needs.check-ci.outputs.should_run == 'true'
runs-on: k8s-runner-cpu
permissions:
contents: read
steps:
- uses: actions/checkout@v6

- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.13"

- name: Install linting tools
run: pip install ruff mypy

- name: Ruff check
run: ruff check e2e_test/ bindings/python/ scripts/

- name: Ruff format check
run: ruff format --check e2e_test/ bindings/python/ scripts/

- name: Mypy (e2e_test)
run: mypy e2e_test/ --config-file mypy.ini

- name: Mypy (bindings/python)
run: mypy bindings/python/ --config-file mypy.ini

build-wheel:
needs: check-ci
if: needs.check-ci.outputs.should_run == 'true'
Expand Down Expand Up @@ -506,7 +535,7 @@ jobs:
path: benchmark_go_bindings/

finish:
needs: [check-ci, build-wheel, python-unit-tests, unit-tests, gateway-e2e, go-unit-tests, go-bindings-e2e]
needs: [check-ci, python-lint, build-wheel, python-unit-tests, unit-tests, gateway-e2e, go-unit-tests, go-bindings-e2e]
if: always()
runs-on: k8s-runner-cpu
permissions: {}
Expand All @@ -516,7 +545,7 @@ jobs:
if [[ "${{ needs.check-ci.outputs.should_run }}" != "true" ]]; then
echo "CI was skipped (external contributor without run-ci label)"
exit 0
elif [[ "${{ needs.build-wheel.result }}" == "failure" || "${{ needs.unit-tests.result }}" == "failure" || "${{ needs.gateway-e2e.result }}" == "failure" || "${{ needs.go-unit-tests.result }}" == "failure" || "${{ needs.go-bindings-e2e.result }}" == "failure" ]]; then
elif [[ "${{ needs.python-lint.result }}" == "failure" || "${{ needs.build-wheel.result }}" == "failure" || "${{ needs.unit-tests.result }}" == "failure" || "${{ needs.gateway-e2e.result }}" == "failure" || "${{ needs.go-unit-tests.result }}" == "failure" || "${{ needs.go-bindings-e2e.result }}" == "failure" ]]; then
echo "One or more jobs failed"
exit 1
else
Expand Down
8 changes: 7 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
default_stages: [pre-commit, pre-push, manual]
exclude: ^(bindings/golang/|bindings/python/|e2e_test/)

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
Expand Down Expand Up @@ -52,3 +51,10 @@ repos:
language: system
types: [rust]
pass_filenames: false

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.0
Comment thread
CatherineSue marked this conversation as resolved.
Resolved
hooks:
- id: ruff
args: [--fix]
- id: ruff-format
7 changes: 3 additions & 4 deletions bindings/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,13 @@ authors = [
{name = "Chang Su", email = "mckvtl@gmail.com"},
{name = "Keyang Ru", email = "rukeyang@gmail.com"}
]
requires-python = ">=3.9"
requires-python = ">=3.12"
readme = "../../README.md"
license = { text = "Apache-2.0" }
classifiers = [
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Rust",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
Comment thread
CatherineSue marked this conversation as resolved.
Resolved
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
Expand All @@ -34,7 +31,9 @@ dependencies = [

[project.optional-dependencies]
dev = [
"mypy",
"requests>=2.25.0",
"ruff",
"pytest>=7.0.0",
]

Expand Down
6 changes: 2 additions & 4 deletions bindings/python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from setuptools import setup

with_rust = os.environ.get("SMG_BUILD_WITH_RUST", None)
with_rust = with_rust is None or (not with_rust.lower() in ["0", "false", "no"])
with_rust = with_rust is None or (with_rust.lower() not in ["0", "false", "no"])

rust_extensions = []
if with_rust:
Expand All @@ -18,9 +18,7 @@
)
)
else:
warnings.warn(
"Building 'smg' without Rust support. Performance may be degraded."
)
warnings.warn("Building 'smg' without Rust support. Performance may be degraded.")

setup(
rust_extensions=rust_extensions,
Expand Down
9 changes: 4 additions & 5 deletions bindings/python/src/smg/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import argparse
import os
import sys
from typing import List, Optional

from smg.smg_rs import (
get_verbose_version_string,
Expand All @@ -33,23 +32,23 @@ def create_parser() -> argparse.ArgumentParser:
subparsers = parser.add_subparsers(dest="command", help="Available commands")

# Launch router subcommand
launch_parser = subparsers.add_parser(
subparsers.add_parser(
"launch",
help="Launch router only (requires existing worker URLs)",
description="Launch the Shepherd router with existing worker instances",
add_help=False, # Let router handle --help
)

# Launch server + router subcommand
server_parser = subparsers.add_parser(
subparsers.add_parser(
"server",
help="Launch router and server processes together",
description="Launch both Shepherd router and server processes",
add_help=False, # Let server handle --help
)

# Serve subcommand (two-pass parsing with lazy backend import)
serve_parser = subparsers.add_parser(
subparsers.add_parser(
"serve",
help="Launch backend worker(s) + gateway router",
description="Launch inference backend workers and gateway router",
Expand All @@ -59,7 +58,7 @@ def create_parser() -> argparse.ArgumentParser:
return parser


def main(argv: Optional[List[str]] = None) -> None:
def main(argv: list[str] | None = None) -> None:
"""Main CLI entry point."""
if argv is None:
argv = sys.argv[1:]
Expand Down
6 changes: 3 additions & 3 deletions bindings/python/src/smg/launch_router.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import argparse
import logging
import sys
from typing import List, Optional

import setproctitle

from smg.router_args import RouterArgs

logger = logging.getLogger("router")
Expand All @@ -15,7 +15,7 @@
logger.warning("Rust Router is not installed")


def launch_router(args: argparse.Namespace) -> Optional[Router]:
def launch_router(args: argparse.Namespace) -> Router | None:
"""
Launch the SMG router with the configuration from parsed arguments.

Expand Down Expand Up @@ -53,7 +53,7 @@ class CustomHelpFormatter(
pass


def parse_router_args(args: List[str]) -> RouterArgs:
def parse_router_args(args: list[str]) -> RouterArgs:
"""Parse command line arguments and return RouterArgs instance."""
parser = argparse.ArgumentParser(
description="""SMG Router - High-performance request distribution across worker nodes
Expand Down
43 changes: 15 additions & 28 deletions bindings/python/src/smg/launch_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,14 @@
import signal
import sys
import time
from typing import List

import requests
from setproctitle import setproctitle
from smg.launch_router import RouterArgs, launch_router

from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_port_available

from smg.launch_router import RouterArgs, launch_router


def setup_logger():
logger = logging.getLogger("router")
Expand All @@ -42,8 +41,10 @@ def run_server(server_args, dp_rank):
"""
Note:

1. Without os.setpgrp(), all processes share the same PGID. When you press Ctrl+C, the terminal sends SIGINT to all processes in the group simultaneously.
This can cause leaf processes to terminate first, which messes up the cleaning order and produces orphaned processes.
1. Without os.setpgrp(), all processes share the same PGID. When you press Ctrl+C,
the terminal sends SIGINT to all processes in the group simultaneously.
This can cause leaf processes to terminate first, which messes up the cleaning order
and produces orphaned processes.

Terminal (PGID=100)
└── Main Python Process (PGID=100)
Expand Down Expand Up @@ -83,9 +84,7 @@ def run_server(server_args, dp_rank):
launch_server(server_args)


def launch_server_process(
server_args: ServerArgs, worker_port: int, dp_id: int
) -> mp.Process:
def launch_server_process(server_args: ServerArgs, worker_port: int, dp_id: int) -> mp.Process:
"""Launch a single server process with the given args and port."""
server_args = copy.deepcopy(server_args)
server_args.port = worker_port
Expand Down Expand Up @@ -113,7 +112,7 @@ def wait_for_server_health(host: str, port: int, timeout: int = 300) -> bool:
return False


def find_available_ports(base_port: int, count: int) -> List[int]:
def find_available_ports(base_port: int, count: int) -> list[int]:
"""Find consecutive available ports starting from base_port."""
available_ports = []
current_port = base_port
Expand All @@ -126,7 +125,7 @@ def find_available_ports(base_port: int, count: int) -> List[int]:
return available_ports


def cleanup_processes(processes: List[mp.Process]):
def cleanup_processes(processes: list[mp.Process]):
for process in processes:
logger.info(f"Terminating process group {process.pid}")
try:
Expand All @@ -139,9 +138,7 @@ def cleanup_processes(processes: List[mp.Process]):
for process in processes:
process.join(timeout=5)
if process.is_alive():
logger.warning(
f"Process {process.pid} did not terminate gracefully, forcing kill"
)
logger.warning(f"Process {process.pid} did not terminate gracefully, forcing kill")
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
Expand All @@ -154,9 +151,7 @@ def main():
# CUDA runtime isn't fork-safe, which can lead to subtle bugs or crashes
mp.set_start_method("spawn")

parser = argparse.ArgumentParser(
description="Launch SGLang router and server processes"
)
parser = argparse.ArgumentParser(description="Launch SGLang router and server processes")

ServerArgs.add_cli_args(parser)
RouterArgs.add_cli_args(parser, use_router_prefix=True, exclude_host_port=True)
Expand All @@ -173,9 +168,7 @@ def main():
router_args = RouterArgs.from_cli_args(args, use_router_prefix=True)

# Find available ports for workers
worker_ports = find_available_ports(
args.router_dp_worker_base_port, server_args.dp_size
)
worker_ports = find_available_ports(args.router_dp_worker_base_port, server_args.dp_size)

# Start server processes
server_processes = []
Expand All @@ -186,19 +179,13 @@ def main():
server_processes.append(proc)

signal.signal(signal.SIGINT, lambda sig, frame: cleanup_processes(server_processes))
signal.signal(
signal.SIGTERM, lambda sig, frame: cleanup_processes(server_processes)
)
signal.signal(
signal.SIGQUIT, lambda sig, frame: cleanup_processes(server_processes)
)
signal.signal(signal.SIGTERM, lambda sig, frame: cleanup_processes(server_processes))
signal.signal(signal.SIGQUIT, lambda sig, frame: cleanup_processes(server_processes))

Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
# Update router args with worker URLs
# Use grpc:// protocol if server is in gRPC mode, otherwise http://
protocol = "grpc" if server_args.grpc_mode else "http"
router_args.worker_urls = [
f"{protocol}://{server_args.host}:{port}" for port in worker_ports
]
router_args.worker_urls = [f"{protocol}://{server_args.host}:{port}" for port in worker_ports]

# Start the router
try:
Expand Down
Loading
Loading