[diffusion] CI: send diffusion nightly-test outputs to slack for correctness monitoring

.github/workflows/nightly-test.yml

@@ -122,6 +122,67 @@ jobs:
         run: |
           python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_vlms
 
+  nightly-test-multimodal-server-1-gpu:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: 1-gpu-runner
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix:
+        part: [0, 1]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/ci_install_dependency.sh diffusion
+          pip install slack_sdk
+
+      - name: Run diffusion server tests
+        env:
+          SGLANG_DIFFUSION_SLACK_TOKEN: ${{ secrets.SGLANG_DIFFUSION_SLACK_TOKEN }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+
+        timeout-minutes: 60
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite 1-gpu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions 2
+
+
+  nightly-test-multimodal-server-2-gpu:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: 2-gpu-runner
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix:
+        part: [0, 1]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/ci_install_dependency.sh diffusion
+          pip install slack_sdk
+
+      - name: Run diffusion server tests
+        env:
+          SGLANG_DIFFUSION_SLACK_TOKEN: ${{ secrets.SGLANG_DIFFUSION_SLACK_TOKEN }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+
+        timeout-minutes: 60
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite 2-gpu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions 2
+
   nightly-test-1-gpu:
     if: github.repository == 'sgl-project/sglang'
     runs-on: 1-gpu-runner
@@ -235,6 +296,8 @@ jobs:
       - nightly-test-perf-text-models
       - nightly-test-eval-vlms
       - nightly-test-perf-vlms
+      - nightly-test-multimodal-server-1-gpu
+      - nightly-test-multimodal-server-2-gpu
       - nightly-test-1-gpu
       - nightly-test-4-gpu
       - nightly-test-8-gpu-h200

python/sglang/multimodal_gen/test/server/test_server_common.py

@@ -7,6 +7,7 @@
 
 from __future__ import annotations
 
+import base64
 import os
 import time
 from pathlib import Path
@@ -32,6 +33,7 @@
     PerformanceSummary,
     ScenarioConfig,
 )
+from sglang.multimodal_gen.test.slackbot import upload_file_to_slack
 from sglang.multimodal_gen.test.test_utils import (
     get_dynamic_server_port,
     read_perf_logs,
@@ -225,6 +227,19 @@ def _create_and_download_video(
             resp = client.videos.download_content(video_id=video_id)  # type: ignore[attr-defined]
             content = resp.read()
             validate_openai_video(content)
+
+            tmp_path = f"{video_id}.mp4"
+            with open(tmp_path, "wb") as f:
+                f.write(content)
+            upload_file_to_slack(
+                case_id=case.id,
+                model=case.model_path,
+                prompt=case.prompt,
+                file_path=tmp_path,
+                origin_file_path=case.image_path,
+            )
+            os.remove(tmp_path)
+
             return video_id
 
         # for all tests, seconds = case.seconds or fallback 4 seconds
@@ -248,6 +263,19 @@ def generate_image() -> str:
             )
             result = response.parse()
             validate_image(result.data[0].b64_json)
+
+            img_data = base64.b64decode(result.data[0].b64_json)
+            tmp_path = f"{result.created}.png"
+            with open(tmp_path, "wb") as f:
+                f.write(img_data)
+            upload_file_to_slack(
+                case_id=case.id,
+                model=case.model_path,
+                prompt=case.prompt,
+                file_path=tmp_path,
+            )
+            os.remove(tmp_path)
+
             return str(result.created)
 
         def generate_image_edit() -> str:
@@ -276,6 +304,20 @@ def generate_image_edit() -> str:
 
             result = response.parse()
             validate_image(result.data[0].b64_json)
+
+            img_data = base64.b64decode(result.data[0].b64_json)
+            tmp_path = f"{rid}.png"
+            with open(tmp_path, "wb") as f:
+                f.write(img_data)
+            upload_file_to_slack(
+                case_id=case.id,
+                model=case.model_path,
+                prompt=case.edit_prompt,
+                file_path=tmp_path,
+                origin_file_path=case.image_path,
+            )
+            os.remove(tmp_path)
+
             return rid
 
         # -------------------------

python/sglang/multimodal_gen/test/slackbot.py

@@ -0,0 +1,66 @@
+import logging
+import os
+import tempfile
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def upload_file_to_slack(
+    case_id: str = None,
+    model: str = None,
+    prompt: str = None,
+    file_path: str = None,
+    origin_file_path: str = None,
+) -> bool:
+    temp_path = None
+    try:
+        from slack_sdk import WebClient
+
+        run_id = os.getenv("GITHUB_RUN_ID", "local")
+
+        token = os.environ.get("SGLANG_DIFFUSION_SLACK_TOKEN")
+        if not token:
+            logger.info(f"Slack upload failed: no token")
+            return False
+
+        if not file_path or not os.path.exists(file_path):
+            logger.info(f"Slack upload failed: no file path")
+            return False
+
+        if origin_file_path and origin_file_path.startswith(("http", "https")):
+            suffix = os.path.splitext(urlparse(origin_file_path).path)[1] or ".tmp"
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
+                with urlopen(origin_file_path) as response:
+                    tf.write(response.read())
+                temp_path = tf.name
+                origin_file_path = temp_path
+
+        uploads = [{"file": file_path, "title": "Generated Image"}]
+        if origin_file_path and os.path.exists(origin_file_path):
+            uploads.insert(0, {"file": origin_file_path, "title": "Original Image"})
+
+        message = (
+            f"*GitHub Run ID:* {run_id}\n"
+            f"*Case ID:* `{case_id}`\n"
+            f"*Model:* `{model}`\n"
+            f"*Prompt:* {prompt}"
+        )
+
+        WebClient(token=token).files_upload_v2(
+            channel="C0A02NDF7UY",
+            file_uploads=uploads,
+            initial_comment=message,
+        )
+
+        logger.info(f"File uploaded successfully: {os.path.basename(file_path)}")
+        return True
+
+    except Exception as e:
+        logger.info(f"Slack upload failed: {e}")
+        return False
+    finally:
+        if temp_path and os.path.exists(temp_path):
+            os.remove(temp_path)