diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 0ae6097d9b67..eb6040c8a5dd 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -122,6 +122,67 @@ jobs: run: | python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_vlms + nightly-test-multimodal-server-1-gpu: + if: github.repository == 'sgl-project/sglang' + runs-on: 1-gpu-runner + strategy: + fail-fast: false + max-parallel: 5 + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh diffusion + pip install slack_sdk + + - name: Run diffusion server tests + env: + SGLANG_DIFFUSION_SLACK_TOKEN: ${{ secrets.SGLANG_DIFFUSION_SLACK_TOKEN }} + GITHUB_RUN_ID: ${{ github.run_id }} + + timeout-minutes: 60 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite.py \ + --suite 1-gpu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 + + + nightly-test-multimodal-server-2-gpu: + if: github.repository == 'sgl-project/sglang' + runs-on: 2-gpu-runner + strategy: + fail-fast: false + max-parallel: 5 + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh diffusion + pip install slack_sdk + + - name: Run diffusion server tests + env: + SGLANG_DIFFUSION_SLACK_TOKEN: ${{ secrets.SGLANG_DIFFUSION_SLACK_TOKEN }} + GITHUB_RUN_ID: ${{ github.run_id }} + + timeout-minutes: 60 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite.py \ + --suite 2-gpu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 + nightly-test-1-gpu: if: github.repository == 'sgl-project/sglang' runs-on: 1-gpu-runner @@ -235,6 +296,8 @@ jobs: - nightly-test-perf-text-models - nightly-test-eval-vlms - nightly-test-perf-vlms + - nightly-test-multimodal-server-1-gpu + - nightly-test-multimodal-server-2-gpu - nightly-test-1-gpu - nightly-test-4-gpu - nightly-test-8-gpu-h200 diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index c7bf6fbadba3..3f7b5ad3f1bd 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -7,6 +7,7 @@ from __future__ import annotations +import base64 import os import time from pathlib import Path @@ -32,6 +33,7 @@ PerformanceSummary, ScenarioConfig, ) +from sglang.multimodal_gen.test.slackbot import upload_file_to_slack from sglang.multimodal_gen.test.test_utils import ( get_dynamic_server_port, read_perf_logs, @@ -225,6 +227,19 @@ def _create_and_download_video( resp = client.videos.download_content(video_id=video_id) # type: ignore[attr-defined] content = resp.read() validate_openai_video(content) + + tmp_path = f"{video_id}.mp4" + with open(tmp_path, "wb") as f: + f.write(content) + upload_file_to_slack( + case_id=case.id, + model=case.model_path, + prompt=case.prompt, + file_path=tmp_path, + origin_file_path=case.image_path, + ) + os.remove(tmp_path) + return video_id # for all tests, seconds = case.seconds or fallback 4 seconds @@ -248,6 +263,19 @@ def generate_image() -> str: ) result = response.parse() validate_image(result.data[0].b64_json) + + img_data = base64.b64decode(result.data[0].b64_json) + tmp_path = f"{result.created}.png" + with open(tmp_path, "wb") as f: + f.write(img_data) + upload_file_to_slack( + case_id=case.id, + model=case.model_path, + prompt=case.prompt, + file_path=tmp_path, + ) + os.remove(tmp_path) + return str(result.created) def generate_image_edit() -> str: @@ -276,6 +304,20 @@ def generate_image_edit() -> str: result = response.parse() validate_image(result.data[0].b64_json) + + img_data = base64.b64decode(result.data[0].b64_json) + tmp_path = f"{rid}.png" + with open(tmp_path, "wb") as f: + f.write(img_data) + upload_file_to_slack( + case_id=case.id, + model=case.model_path, + prompt=case.edit_prompt, + file_path=tmp_path, + origin_file_path=case.image_path, + ) + os.remove(tmp_path) + return rid # ------------------------- diff --git a/python/sglang/multimodal_gen/test/slackbot.py b/python/sglang/multimodal_gen/test/slackbot.py new file mode 100644 index 000000000000..2af0abc7546e --- /dev/null +++ b/python/sglang/multimodal_gen/test/slackbot.py @@ -0,0 +1,66 @@ +import logging +import os +import tempfile +from urllib.parse import urlparse +from urllib.request import urlopen + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def upload_file_to_slack( + case_id: str = None, + model: str = None, + prompt: str = None, + file_path: str = None, + origin_file_path: str = None, +) -> bool: + temp_path = None + try: + from slack_sdk import WebClient + + run_id = os.getenv("GITHUB_RUN_ID", "local") + + token = os.environ.get("SGLANG_DIFFUSION_SLACK_TOKEN") + if not token: + logger.info(f"Slack upload failed: no token") + return False + + if not file_path or not os.path.exists(file_path): + logger.info(f"Slack upload failed: no file path") + return False + + if origin_file_path and origin_file_path.startswith(("http", "https")): + suffix = os.path.splitext(urlparse(origin_file_path).path)[1] or ".tmp" + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf: + with urlopen(origin_file_path) as response: + tf.write(response.read()) + temp_path = tf.name + origin_file_path = temp_path + + uploads = [{"file": file_path, "title": "Generated Image"}] + if origin_file_path and os.path.exists(origin_file_path): + uploads.insert(0, {"file": origin_file_path, "title": "Original Image"}) + + message = ( + f"*GitHub Run ID:* {run_id}\n" + f"*Case ID:* `{case_id}`\n" + f"*Model:* `{model}`\n" + f"*Prompt:* {prompt}" + ) + + WebClient(token=token).files_upload_v2( + channel="C0A02NDF7UY", + file_uploads=uploads, + initial_comment=message, + ) + + logger.info(f"File uploaded successfully: {os.path.basename(file_path)}") + return True + + except Exception as e: + logger.info(f"Slack upload failed: {e}") + return False + finally: + if temp_path and os.path.exists(temp_path): + os.remove(temp_path)