diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 727080c001..67188d40d6 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -832,8 +832,22 @@ jobs: export CGO_LDFLAGS="-L$(pwd)/bindings/golang/target/release" export LD_LIBRARY_PATH="$(pwd)/bindings/golang/target/release:$LD_LIBRARY_PATH" SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1 ROUTER_LOCAL_MODEL_PATH="/models" \ + E2E_LOG_DIR=e2e-logs \ pytest --reruns 2 --reruns-delay 5 e2e_test/bindings_go -s -vv + - name: Worker failure diagnostics + if: failure() || cancelled() + run: bash scripts/ci_dump_worker_logs.sh e2e-logs "e2e-worker-logs-go-bindings" + + - name: Upload worker logs + if: failure() || cancelled() + uses: actions/upload-artifact@v7 + with: + name: e2e-worker-logs-go-bindings + path: e2e-logs/ + retention-days: 7 + if-no-files-found: ignore + go-bindings-benchmark: name: go-bindings-benchmark needs: build-wheel diff --git a/.github/workflows/release-sglang-docker.yml b/.github/workflows/release-sglang-docker.yml index ed8a33bfc5..56b47b76b7 100644 --- a/.github/workflows/release-sglang-docker.yml +++ b/.github/workflows/release-sglang-docker.yml @@ -2,7 +2,7 @@ name: Release SMG+SGLang Docker Image run-name: >- SMG+SGLang | - ${{ github.event_name == 'push' && '3-version matrix' || format('base={0}', inputs.base_image_ref || 'lmsysorg/sglang:v0.5.9') }} | + ${{ github.event_name == 'push' && 'base=lmsysorg/sglang:v0.5.10' || format('base={0}', inputs.base_image_ref || 'lmsysorg/sglang:v0.5.10') }} | engine=${{ inputs.sglang_commit || 'latest' }} | smg=${{ github.event_name == 'pull_request' && 'main' || inputs.smg_commit || 'v1.4.0' }} | ${{ github.event_name == 'pull_request' && 'dry-run |' || '' }} @@ -23,8 +23,8 @@ on: workflow_dispatch: inputs: base_image_ref: - description: 'Base image (e.g. lmsysorg/sglang:v0.5.9)' - default: 'lmsysorg/sglang:v0.5.9' + description: 'Base image (e.g. lmsysorg/sglang:v0.5.10)' + default: 'lmsysorg/sglang:v0.5.10' required: true type: string sglang_repo: @@ -47,7 +47,7 @@ on: default: 'v1.4.0' type: string tag: - description: 'Override image tag (e.g. v1.4.0-sglang-v0.5.9)' + description: 'Override image tag (e.g. v1.4.0-sglang-v0.5.10)' required: false type: string @@ -60,7 +60,7 @@ jobs: strategy: fail-fast: false matrix: - base_image: ${{ github.event_name == 'push' && fromJSON('["lmsysorg/sglang:v0.5.9","lmsysorg/sglang:v0.5.8","lmsysorg/sglang:v0.5.7"]') || fromJSON(format('["{0}"]', inputs.base_image_ref || 'lmsysorg/sglang:v0.5.9')) }} + base_image: ${{ github.event_name == 'push' && fromJSON('["lmsysorg/sglang:v0.5.10"]') || fromJSON(format('["{0}"]', inputs.base_image_ref || 'lmsysorg/sglang:v0.5.10')) }} uses: ./.github/workflows/_build-engine-image.yml with: engine: sglang diff --git a/bindings/golang/internal/grpc/client_grpc.go b/bindings/golang/internal/grpc/client_grpc.go index d7b4aa0da1..d5d8c8cacc 100644 --- a/bindings/golang/internal/grpc/client_grpc.go +++ b/bindings/golang/internal/grpc/client_grpc.go @@ -148,10 +148,12 @@ func (c *GrpcClient) CreateChatCompletionStream(ctx context.Context, reqJSON str // Set sampling parameters samplingParams := &proto.SamplingParams{ - Temperature: 1.0, - TopP: 1.0, - TopK: -1, - SkipSpecialTokens: true, + Temperature: 1.0, + TopP: 1.0, + TopK: -1, + RepetitionPenalty: 1.0, + SkipSpecialTokens: true, + SpacesBetweenSpecialTokens: true, } if temp, ok := reqMap["temperature"].(float64); ok { diff --git a/docker/engine.Dockerfile b/docker/engine.Dockerfile index 8f6cacdb92..ccd9f6f6c4 100644 --- a/docker/engine.Dockerfile +++ b/docker/engine.Dockerfile @@ -10,7 +10,7 @@ # SMG_COMMIT - commit/ref for SMG_REPO ("latest" = HEAD) # # Usage: -# docker build --build-arg BASE_IMAGE_REF=lmsysorg/sglang:v0.5.9 \ +# docker build --build-arg BASE_IMAGE_REF=lmsysorg/sglang:v0.5.10 \ # --build-arg ENGINE=sglang \ # --build-arg SMG_REPO=https://github.com/lightseekorg/smg \ # --build-arg SMG_COMMIT=v1.1.0 \ diff --git a/e2e_test/chat_completions/test_multimodal.py b/e2e_test/chat_completions/test_multimodal.py index bfe9bc2004..6ab0930d18 100644 --- a/e2e_test/chat_completions/test_multimodal.py +++ b/e2e_test/chat_completions/test_multimodal.py @@ -44,7 +44,7 @@ def _make_image_content(image_source: str) -> dict: # ============================================================================= -@pytest.mark.engine("vllm") +@pytest.mark.engine("vllm", "sglang") @pytest.mark.gpu(1) @pytest.mark.e2e @pytest.mark.model("Qwen/Qwen3-VL-8B-Instruct") @@ -170,7 +170,7 @@ def test_multi_images_mixed(self, model, setup_backend): # ============================================================================= -@pytest.mark.engine("vllm") +@pytest.mark.engine("vllm", "sglang") @pytest.mark.gpu(4) @pytest.mark.e2e @pytest.mark.model("meta-llama/Llama-4-Scout-17B-16E-Instruct") diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 3951eef0b7..3fd1eb2cdf 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -149,6 +149,7 @@ def _resolve_model_path(hf_path: str) -> str: "--cuda-graph-max-bs=256", "--max-running-requests=300", "--mem-fraction-static=0.85", + "--enable-multimodal", ], "vllm_args": [ "--max-model-len=196608", diff --git a/grpc_servicer/pyproject.toml b/grpc_servicer/pyproject.toml index 0f37d04ec7..3f04ba0b83 100644 --- a/grpc_servicer/pyproject.toml +++ b/grpc_servicer/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ [project.optional-dependencies] vllm = ["vllm>=0.19.0"] -sglang = ["sglang>=0.5.10rc0"] +sglang = ["sglang>=0.5.10"] [project.urls] Homepage = "https://github.com/lightseekorg/smg" diff --git a/grpc_servicer/smg_grpc_servicer/sglang/servicer.py b/grpc_servicer/smg_grpc_servicer/sglang/servicer.py index d330216ca4..04b727a6e9 100644 --- a/grpc_servicer/smg_grpc_servicer/sglang/servicer.py +++ b/grpc_servicer/smg_grpc_servicer/sglang/servicer.py @@ -37,7 +37,11 @@ TokenizedEmbeddingReqInput, TokenizedGenerateReqInput, ) -from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) from sglang.srt.sampling.sampling_params import SamplingParams as SGLSamplingParams from sglang.srt.server_args import ServerArgs from sglang.utils import get_exception_traceback @@ -351,7 +355,7 @@ async def HealthCheck( rid=rid, input_text="", input_ids=[0], - image_inputs={"mm_items": []}, + image_inputs=None, token_type_ids=[0], sampling_params=sampling_params, ) @@ -768,8 +772,8 @@ def _decode_tensor_data(tensor_data): arr = np.frombuffer(tensor_data.data, dtype=np_dtype).reshape(shape) return torch.from_numpy(arr) - def _parse_mm_inputs(self, mm_proto) -> dict: - """Parse proto MultimodalInputs into the mm_inputs dict expected by scheduler.""" + def _parse_mm_inputs(self, mm_proto) -> MultimodalInputs: + """Parse proto MultimodalInputs into the mm_inputs expected by scheduler.""" # Decode pixel_values from typed TensorData field pixel_values = self._decode_tensor_data(mm_proto.pixel_values) @@ -795,10 +799,10 @@ def _parse_mm_inputs(self, mm_proto) -> dict: offsets=offsets, ) - result = {"mm_items": [mm_item]} + result = MultimodalInputs(mm_items=[mm_item]) if mm_proto.HasField("im_token_id"): - result["im_token_id"] = mm_proto.im_token_id + result.im_token_id = mm_proto.im_token_id return result @@ -827,7 +831,7 @@ def _convert_embed_request( rid=grpc_req.request_id, input_text=input_text, input_ids=input_ids, - image_inputs={"mm_items": []}, + image_inputs=None, token_type_ids=list(grpc_req.token_type_ids), sampling_params=sampling_params, ) diff --git a/scripts/ci_install_sglang.sh b/scripts/ci_install_sglang.sh index e407dccbc5..193cf6c803 100755 --- a/scripts/ci_install_sglang.sh +++ b/scripts/ci_install_sglang.sh @@ -40,13 +40,8 @@ else fi # Install SGLang with all dependencies -# TODO(changsu): Revert to `uv pip install "sglang[all]"` once sglang releases -# a stable version (>= 0.5.10) that includes the smg_grpc_servicer migration -# (sgl-project/sglang#20478). The rc0 is needed because the old built-in -# grpc_server.py references proto types (EmbedError, EmbedComplete) that were -# removed from our proto definitions. echo "Installing SGLang..." -uv pip install --prerelease=allow "sglang[all]==0.5.10rc0" +uv pip install --prerelease=allow "sglang[all]==0.5.10" # Install flashinfer-jit-cache: sglang bundles flashinfer_python but only for attention ops. # Multi-GPU models need trtllm_comm kernels (fused allreduce + layernorm) which FlashInfer diff --git a/scripts/installation/install-sglang.sh b/scripts/installation/install-sglang.sh index 95ae449a9b..7502cae8e9 100755 --- a/scripts/installation/install-sglang.sh +++ b/scripts/installation/install-sglang.sh @@ -8,7 +8,3 @@ set -e SGL_SRC="${1:-/tmp/sglang-src}" cd "${SGL_SRC}/python" pip install --no-deps --force-reinstall --editable . - -# Pin gRPC packages to 1.78.0 — grpcio 1.78.1 was yanked from PyPI -# TODO: remove when new SGLang 0.5.10 is released -pip install --force-reinstall grpcio==1.78.0 grpcio-reflection==1.78.0 grpcio-tools==1.78.0 grpcio-health-checking==1.78.0