From b1ea86feb749e806cbb43c933ce23644b8f94351 Mon Sep 17 00:00:00 2001 From: Prozac614 Date: Tue, 24 Feb 2026 08:44:56 +0000 Subject: [PATCH 1/2] Add CI consistency validation using CLIP similarity - Add consistency_threshold.json for per-case threshold config - Add CLIP-based consistency checking utilities: get_clip_model, compute_clip_embedding, compute_clip_similarity, compare_with_gt, load_gt_embeddings, gt_exists, get_clip_threshold, ConsistencyResult - Add _validate_consistency() method to test_server_common.py - Call consistency validation after performance validation in test_diffusion_generation --- .../test/server/consistency_threshold.json | 5 + .../test/server/test_server_common.py | 124 +++++++ .../sglang/multimodal_gen/test/test_utils.py | 326 ++++++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 python/sglang/multimodal_gen/test/server/consistency_threshold.json diff --git a/python/sglang/multimodal_gen/test/server/consistency_threshold.json b/python/sglang/multimodal_gen/test/server/consistency_threshold.json new file mode 100644 index 000000000000..8d8557199c93 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/consistency_threshold.json @@ -0,0 +1,5 @@ +{ + "cases": {}, + "default_clip_threshold_image": 0.92, + "default_clip_threshold_video": 0.90 +} diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index 690c5ba62636..bcdb36c8448c 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -35,8 +35,15 @@ ) from sglang.multimodal_gen.test.test_utils import ( _consistency_gt_filenames, + _get_consistency_gt_dir, + compare_with_gt, extract_key_frames_from_video, + get_clip_threshold, + get_consistency_gt_candidates, get_dynamic_server_port, + gt_exists, + image_bytes_to_numpy, + load_gt_embeddings, wait_for_req_perf_record, ) @@ -417,6 +424,120 @@ def _dump_baseline_for_testcase( """ logger.error(output) + def _validate_consistency( + self, + case: DiffusionTestCase, + content: bytes, + ) -> None: + """Validate output consistency against ground truth using CLIP similarity. + + Args: + case: Test case configuration + content: Generated content bytes (image or video) + + Raises: + pytest.fail: If consistency check fails (GT exists but doesn't match) + + Note: + If GT doesn't exist, the test is skipped (not failed) with a message + to run the GT generation workflow after PR is merged. + + Environment Variables: + SGLANG_SKIP_CONSISTENCY: Set to "1" to skip consistency checks + """ + # Allow skipping consistency checks for debugging/development + if os.environ.get("SGLANG_SKIP_CONSISTENCY", "0") == "1": + logger.info( + f"[Consistency] Skipping consistency check for {case.id} (SGLANG_SKIP_CONSISTENCY=1)" + ) + return + + # Skip if content is empty (e.g., video generation timed out) + if not content: + logger.warning( + f"[Consistency] Skipping consistency check for {case.id}: " + "content is empty (generation may have timed out)" + ) + return + + num_gpus = case.server_args.num_gpus + is_video = case.server_args.modality == "video" + + # Check GT exists - if not, fail with instructions to add image(s) to sgl-test-files + output_format = case.sampling_params.output_format + if not gt_exists( + case.id, num_gpus, is_video=is_video, output_format=output_format + ): + if _get_consistency_gt_dir() is not None: + names = ", ".join( + get_consistency_gt_candidates( + case.id, num_gpus, is_video, output_format + ) + ) + else: + names = ", ".join( + _consistency_gt_filenames( + case.id, num_gpus, is_video, output_format + ) + ) + error_msg = f""" +--- MISSING GROUND TRUTH DETECTED --- +GT image(s) not found for '{case.id}'. + +Add the expected file(s) to sgl-test-files in diffusion-ci/consistency_gt/ with naming (n=num_gpus). + Image: {case.id}_{{n}}gpu. (ext from output_format: png, jpg, webp) + Video: {case.id}_{{n}}gpu_frame_0.png, {case.id}_{{n}}gpu_frame_mid.png, {case.id}_{{n}}gpu_frame_last.png + +For this case, expected file(s): {names} + +Repository: https://github.com/sgl-project/sgl-test-files (path: diffusion-ci/consistency_gt/) + +(Optional) Per-case override in consistency_threshold.json: "cases": {{ "{case.id}": {{ "clip_threshold": 0.92 }} }} +""" + logger.error(error_msg) + pytest.fail( + f"GT not found for {case.id}. See logs for instructions to add GT." + ) + + # Load GT embeddings (format matches case; PIL converts to RGB for CLIP) + gt_embeddings = load_gt_embeddings( + case.id, num_gpus, is_video=is_video, output_format=output_format + ) + + # Convert output to frames + if is_video: + output_frames = extract_key_frames_from_video(content) + else: + output_frames = [image_bytes_to_numpy(content)] + + threshold = get_clip_threshold(case) + + # Compare frames with GT embeddings using CLIP similarity + result = compare_with_gt( + output_frames=output_frames, + gt_embeddings=gt_embeddings, + threshold=threshold, + case_id=case.id, + ) + + if not result.passed: + # Build detailed failure message + failed_frames = [ + f" Frame {d['frame_index']}: similarity={d['similarity']:.4f}" + for d in result.frame_details + if not d["passed"] + ] + pytest.fail( + f"Consistency check failed for {case.id}:\n" + f" Min similarity: {result.min_similarity:.4f}\n" + f" Threshold: {result.threshold}\n" + f" Failed frames:\n" + "\n".join(failed_frames) + ) + + logger.info( + f"[Consistency] {case.id}: PASSED (min_similarity={result.min_similarity:.4f})" + ) + def _save_gt_output( self, case: DiffusionTestCase, @@ -874,6 +995,9 @@ def test_diffusion_generation( self._test_v1_models_endpoint(diffusion_server, case) self._test_t2v_rejects_input_reference(diffusion_server, case) + # Validation 2: Consistency (reuse the same content) + self._validate_consistency(case, content) + # LoRA API functionality test with E2E validation (only for LoRA-enabled cases) if case.server_args.lora_path or case.server_args.dynamic_lora_path: self._test_lora_api_functionality(diffusion_server, case, generate_fn) diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 62e738a81b24..6815091f7f2b 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -7,12 +7,15 @@ import subprocess import tempfile import time +from dataclasses import dataclass from pathlib import Path +from typing import Any from urllib.parse import urljoin import cv2 import httpx import numpy as np +import requests from PIL import Image from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var @@ -21,6 +24,7 @@ RequestPerfRecord, get_diffusion_perf_log_dir, ) +from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase logger = init_logger(__name__) @@ -86,6 +90,17 @@ def print_divider(length: int, char: str = "-"): print(char * length) +# --- Consistency testing: GT from sgl-test-files or local SGLANG_CONSISTENCY_GT_DIR --- +SGL_TEST_FILES_CONSISTENCY_GT_BASE = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt" +CONSISTENCY_THRESHOLD_JSON_PATH = ( + Path(__file__).resolve().parent / "server" / "consistency_threshold.json" +) +CLIP_MODEL_NAME = "openai/clip-vit-large-patch14" +DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92 +DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90 +_clip_model_cache: dict[str, Any] = {} + + def is_image_url(image_path: str | Path | None) -> bool: """Check if image_path is a URL.""" if image_path is None: @@ -640,3 +655,314 @@ def image_bytes_to_numpy(image_bytes: bytes) -> np.ndarray: """Convert image bytes to numpy array.""" img = Image.open(io.BytesIO(image_bytes)).convert("RGB") return np.array(img) + + +# --- Consistency testing (GT from sgl-test-files, embeddings computed on the fly) --- + + +def _load_threshold_json() -> dict[str, Any]: + """Load consistency_threshold.json; returns {} if missing.""" + if not CONSISTENCY_THRESHOLD_JSON_PATH.exists(): + return {} + with CONSISTENCY_THRESHOLD_JSON_PATH.open("r", encoding="utf-8") as f: + return json.load(f) + + +def get_clip_threshold( + case: DiffusionTestCase, + metadata: dict[str, Any] | None = None, +) -> float: + """ + Get CLIP similarity threshold for a consistency test case. + Uses consistency_threshold.json: default_clip_threshold_image/video and + optional per-case override in cases..clip_threshold. + """ + if metadata is None: + metadata = _load_threshold_json() + case_meta = metadata.get("cases", {}).get(case.id, {}) + is_video = case.server_args.modality == "video" + default = ( + metadata.get("default_clip_threshold_video", DEFAULT_CLIP_THRESHOLD_VIDEO) + if is_video + else metadata.get("default_clip_threshold_image", DEFAULT_CLIP_THRESHOLD_IMAGE) + ) + return float(case_meta.get("clip_threshold", default)) + + +@dataclass +class ConsistencyResult: + """Result of a consistency comparison.""" + + case_id: str + passed: bool + similarity_scores: list[float] + min_similarity: float + threshold: float + frame_details: list[dict[str, Any]] + + +def get_clip_model() -> tuple[Any, Any]: + """ + Get CLIP model and processor (lazy loading with singleton pattern). + + Returns: + Tuple of (model, processor) + + Raises: + ImportError: If transformers is not installed + """ + global _clip_model_cache + + if "model" not in _clip_model_cache: + try: + import torch + from transformers import CLIPModel, CLIPProcessor + except ImportError: + raise ImportError( + "transformers and torch are required for CLIP consistency check. " + "Install with: pip install transformers torch" + ) + + logger.info(f"Loading CLIP model: {CLIP_MODEL_NAME}") + processor = CLIPProcessor.from_pretrained(CLIP_MODEL_NAME) + model = CLIPModel.from_pretrained(CLIP_MODEL_NAME) + + device = "cuda" if torch.cuda.is_available() else "cpu" + model = model.to(device) + model.eval() + + _clip_model_cache["model"] = model + _clip_model_cache["processor"] = processor + _clip_model_cache["device"] = device + logger.info(f"CLIP model loaded on {device}") + + return _clip_model_cache["model"], _clip_model_cache["processor"] + + +def compute_clip_embedding(image: np.ndarray) -> np.ndarray: + """ + Compute CLIP embedding for a single image. + + Args: + image: numpy array (H, W, C) in RGB format + + Returns: + 768-dimensional numpy array (L2 normalized) + """ + try: + import torch + except ImportError: + raise ImportError( + "torch is required for CLIP consistency check. " + "Install with: pip install torch" + ) + + model, processor = get_clip_model() + device = _clip_model_cache["device"] + + pil_image = Image.fromarray(image) + inputs = processor(images=pil_image, return_tensors="pt") + inputs = {k: v.to(device) for k, v in inputs.items()} + + with torch.no_grad(): + image_features = model.get_image_features(**inputs) + image_features = image_features / image_features.norm(dim=-1, keepdim=True) + + return image_features.cpu().numpy().flatten() + + +def compute_clip_similarity(emb1: np.ndarray, emb2: np.ndarray) -> float: + """ + Compute cosine similarity between two CLIP embeddings. + """ + similarity = np.dot(emb1, emb2) + return float(similarity) + + +def get_consistency_gt_candidates( + case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None +) -> list[str]: + """Return list of GT filenames to try when using SGLANG_CONSISTENCY_GT_DIR. + + For image: [base.png, base.jpg, base.webp] in preferred-first order so that + GT can be stored as .png, .jpg, or .webp. For video: the 3 frame filenames + (no extension fallback). + """ + n = num_gpus + if is_video: + return [ + f"{case_id}_{n}gpu_frame_0.png", + f"{case_id}_{n}gpu_frame_mid.png", + f"{case_id}_{n}gpu_frame_last.png", + ] + base = f"{case_id}_{n}gpu" + preferred = output_format_to_ext(output_format) + exts = [preferred] + [e for e in ("png", "jpg", "webp") if e != preferred] + return [f"{base}.{e}" for e in exts] + + +def _get_consistency_gt_dir() -> Path | None: + """If SGLANG_CONSISTENCY_GT_DIR is set, return that Path; else None (use remote).""" + d = os.environ.get("SGLANG_CONSISTENCY_GT_DIR") + if not d: + return None + return Path(d).resolve() + + +def load_gt_embeddings( + case_id: str, + num_gpus: int, + is_video: bool = False, + output_format: str | None = None, +) -> list[np.ndarray]: + """ + Load ground truth by downloading image(s) from sgl-test-files or reading from + SGLANG_CONSISTENCY_GT_DIR, then compute CLIP embeddings. + Format (png/jpg/webp) follows case output_format; PIL converts to RGB for CLIP. + """ + filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format) + embeddings = [] + + gt_dir = _get_consistency_gt_dir() + if gt_dir is not None: + candidates = get_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ) + if is_video: + for fn in candidates: + path = gt_dir / fn + if not path.exists(): + raise FileNotFoundError(f"GT image not found: {path}") + arr = np.array(Image.open(path).convert("RGB")) + emb = compute_clip_embedding(arr) + embeddings.append(emb) + else: + path = None + for fn in candidates: + p = gt_dir / fn + if p.exists(): + path = p + break + if path is None: + raise FileNotFoundError( + f"GT image not found in {gt_dir}. Tried: {', '.join(candidates)}" + ) + arr = np.array(Image.open(path).convert("RGB")) + emb = compute_clip_embedding(arr) + embeddings.append(emb) + logger.info( + f"Loaded {len(embeddings)} GT embeddings for {case_id} from {gt_dir}" + ) + else: + for fn in filenames: + url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{fn}" + resp = requests.get(url, timeout=30) + if resp.status_code != 200: + raise FileNotFoundError(f"GT image not found: {url}") + + arr = np.array(Image.open(io.BytesIO(resp.content)).convert("RGB")) + emb = compute_clip_embedding(arr) + embeddings.append(emb) + + logger.info( + f"Loaded {len(embeddings)} GT embeddings for {case_id} from sgl-test-files" + ) + return embeddings + + +def gt_exists( + case_id: str, + num_gpus: int, + is_video: bool = False, + output_format: str | None = None, +) -> bool: + """Check if GT image(s) exist (sgl-test-files or SGLANG_CONSISTENCY_GT_DIR).""" + gt_dir = _get_consistency_gt_dir() + if gt_dir is not None: + candidates = get_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ) + if is_video: + return all((gt_dir / c).exists() for c in candidates) + return any((gt_dir / c).exists() for c in candidates) + + filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format) + fn = filenames[0] + url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{fn}" + try: + r = requests.head(url, timeout=10) + return r.status_code == 200 + except Exception: + return False + + +def compare_with_gt( + output_frames: list[np.ndarray], + gt_embeddings: list[np.ndarray], + threshold: float, + case_id: str, +) -> ConsistencyResult: + """ + Compare output frames with ground truth embeddings using CLIP similarity. + """ + if len(output_frames) != len(gt_embeddings): + raise ValueError( + f"Frame count mismatch: output={len(output_frames)}, gt={len(gt_embeddings)}" + ) + + similarity_scores = [] + frame_details = [] + + for i, (out_frame, gt_emb) in enumerate(zip(output_frames, gt_embeddings)): + out_emb = compute_clip_embedding(out_frame) + similarity = compute_clip_similarity(out_emb, gt_emb) + similarity_scores.append(similarity) + frame_details.append( + { + "frame_index": i, + "similarity": similarity, + "passed": similarity >= threshold, + "output_shape": out_frame.shape, + } + ) + + min_similarity = min(similarity_scores) + passed = all(s >= threshold for s in similarity_scores) + + result = ConsistencyResult( + case_id=case_id, + passed=passed, + similarity_scores=similarity_scores, + min_similarity=min_similarity, + threshold=threshold, + frame_details=frame_details, + ) + + status = "PASSED" if passed else "FAILED" + print(f"\n{'='*60}") + print(f"[CLIP Consistency] {case_id}: {status}") + print(f" Threshold: {threshold}") + print(f" Min similarity: {min_similarity:.4f}") + print(f" Frame details:") + for detail in frame_details: + frame_status = "✓" if detail["passed"] else "✗" + print( + f" Frame {detail['frame_index']}: similarity={detail['similarity']:.4f} {frame_status}" + ) + print(f"{'='*60}\n") + + if passed: + logger.info( + f"[Consistency] {case_id}: PASSED (min_similarity={min_similarity:.4f}, threshold={threshold})" + ) + else: + logger.error( + f"[Consistency] {case_id}: FAILED (min_similarity={min_similarity:.4f}, threshold={threshold})" + ) + for detail in frame_details: + if not detail["passed"]: + logger.error( + f" Frame {detail['frame_index']}: similarity={detail['similarity']:.4f} < {threshold}" + ) + + return result From 8bc033e7f51cbd0b8728e03862cfb0e4abae0ce0 Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Sun, 8 Mar 2026 19:19:27 +0000 Subject: [PATCH 2/2] upd --- .../test/server/consistency_threshold.json | 1 + .../test/server/test_server_common.py | 61 ++++---- .../sglang/multimodal_gen/test/test_utils.py | 133 ++++++++++++------ 3 files changed, 114 insertions(+), 81 deletions(-) diff --git a/python/sglang/multimodal_gen/test/server/consistency_threshold.json b/python/sglang/multimodal_gen/test/server/consistency_threshold.json index 8d8557199c93..52dc86670475 100644 --- a/python/sglang/multimodal_gen/test/server/consistency_threshold.json +++ b/python/sglang/multimodal_gen/test/server/consistency_threshold.json @@ -1,4 +1,5 @@ { + "diffusers_version": "0.36.0", "cases": {}, "default_clip_threshold_image": 0.92, "default_clip_threshold_video": 0.90 diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index bcdb36c8448c..a977c66a9fb3 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -35,7 +35,7 @@ ) from sglang.multimodal_gen.test.test_utils import ( _consistency_gt_filenames, - _get_consistency_gt_dir, + check_diffusers_version_match, compare_with_gt, extract_key_frames_from_video, get_clip_threshold, @@ -439,8 +439,8 @@ def _validate_consistency( pytest.fail: If consistency check fails (GT exists but doesn't match) Note: - If GT doesn't exist, the test is skipped (not failed) with a message - to run the GT generation workflow after PR is merged. + If GT doesn't exist, the test is skipped (not failed) so that new + cases don't block CI. Run the GT generation workflow after merging. Environment Variables: SGLANG_SKIP_CONSISTENCY: Set to "1" to skip consistency checks @@ -460,49 +460,40 @@ def _validate_consistency( ) return + # Warn if diffusers version doesn't match GT generation version + check_diffusers_version_match() + num_gpus = case.server_args.num_gpus is_video = case.server_args.modality == "video" - # Check GT exists - if not, fail with instructions to add image(s) to sgl-test-files + # Check GT exists - if not, skip with instructions to generate GT output_format = case.sampling_params.output_format if not gt_exists( case.id, num_gpus, is_video=is_video, output_format=output_format ): - if _get_consistency_gt_dir() is not None: - names = ", ".join( - get_consistency_gt_candidates( - case.id, num_gpus, is_video, output_format - ) - ) - else: - names = ", ".join( - _consistency_gt_filenames( - case.id, num_gpus, is_video, output_format - ) + names = ", ".join( + get_consistency_gt_candidates( + case.id, num_gpus, is_video, output_format ) - error_msg = f""" ---- MISSING GROUND TRUTH DETECTED --- -GT image(s) not found for '{case.id}'. - -Add the expected file(s) to sgl-test-files in diffusion-ci/consistency_gt/ with naming (n=num_gpus). - Image: {case.id}_{{n}}gpu. (ext from output_format: png, jpg, webp) - Video: {case.id}_{{n}}gpu_frame_0.png, {case.id}_{{n}}gpu_frame_mid.png, {case.id}_{{n}}gpu_frame_last.png - -For this case, expected file(s): {names} - -Repository: https://github.com/sgl-project/sgl-test-files (path: diffusion-ci/consistency_gt/) - -(Optional) Per-case override in consistency_threshold.json: "cases": {{ "{case.id}": {{ "clip_threshold": 0.92 }} }} -""" - logger.error(error_msg) - pytest.fail( - f"GT not found for {case.id}. See logs for instructions to add GT." + ) + logger.warning( + f"[Consistency] GT not found for '{case.id}'. " + f"Expected file(s): {names}. " + f"Run the 'Diffusion CI Ground Truth Generation' workflow after merging." + ) + pytest.skip( + f"GT not found for {case.id}. " + f"Run GT generation workflow to enable consistency check." ) # Load GT embeddings (format matches case; PIL converts to RGB for CLIP) - gt_embeddings = load_gt_embeddings( - case.id, num_gpus, is_video=is_video, output_format=output_format - ) + try: + gt_embeddings = load_gt_embeddings( + case.id, num_gpus, is_video=is_video, output_format=output_format + ) + except Exception as e: + logger.warning(f"[Consistency] Failed to load GT for {case.id}: {e}") + pytest.skip(f"Failed to load GT for {case.id}: {e}") # Convert output to frames if is_video: diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 6815091f7f2b..493f51a3664d 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -99,6 +99,7 @@ def print_divider(length: int, char: str = "-"): DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92 DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90 _clip_model_cache: dict[str, Any] = {} +_diffusers_version_warned: bool = False def is_image_url(image_path: str | Path | None) -> bool: @@ -668,6 +669,33 @@ def _load_threshold_json() -> dict[str, Any]: return json.load(f) +def check_diffusers_version_match() -> None: + """Warn once if the installed diffusers version doesn't match GT generation version.""" + global _diffusers_version_warned + if _diffusers_version_warned: + return + _diffusers_version_warned = True + + metadata = _load_threshold_json() + gt_version = metadata.get("diffusers_version") + if not gt_version: + return + + try: + import diffusers + + installed = diffusers.__version__ + except ImportError: + return + + if installed != gt_version: + logger.warning( + f"[Consistency] Diffusers version mismatch: GT was generated with " + f"{gt_version}, but {installed} is installed. Consider regenerating GT " + f"via the 'Diffusion CI Ground Truth Generation' workflow." + ) + + def get_clip_threshold( case: DiffusionTestCase, metadata: dict[str, Any] | None = None, @@ -714,14 +742,8 @@ def get_clip_model() -> tuple[Any, Any]: global _clip_model_cache if "model" not in _clip_model_cache: - try: - import torch - from transformers import CLIPModel, CLIPProcessor - except ImportError: - raise ImportError( - "transformers and torch are required for CLIP consistency check. " - "Install with: pip install transformers torch" - ) + import torch + from transformers import CLIPModel, CLIPProcessor logger.info(f"Loading CLIP model: {CLIP_MODEL_NAME}") processor = CLIPProcessor.from_pretrained(CLIP_MODEL_NAME) @@ -749,13 +771,7 @@ def compute_clip_embedding(image: np.ndarray) -> np.ndarray: Returns: 768-dimensional numpy array (L2 normalized) """ - try: - import torch - except ImportError: - raise ImportError( - "torch is required for CLIP consistency check. " - "Install with: pip install torch" - ) + import torch model, processor = get_clip_model() device = _clip_model_cache["device"] @@ -820,7 +836,6 @@ def load_gt_embeddings( SGLANG_CONSISTENCY_GT_DIR, then compute CLIP embeddings. Format (png/jpg/webp) follows case output_format; PIL converts to RGB for CLIP. """ - filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format) embeddings = [] gt_dir = _get_consistency_gt_dir() @@ -854,15 +869,34 @@ def load_gt_embeddings( f"Loaded {len(embeddings)} GT embeddings for {case_id} from {gt_dir}" ) else: - for fn in filenames: - url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{fn}" - resp = requests.get(url, timeout=30) - if resp.status_code != 200: - raise FileNotFoundError(f"GT image not found: {url}") - - arr = np.array(Image.open(io.BytesIO(resp.content)).convert("RGB")) - emb = compute_clip_embedding(arr) - embeddings.append(emb) + candidates = get_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ) + if is_video: + # Video: download all 3 frame files + for fn in candidates: + url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{fn}" + resp = requests.get(url, timeout=30) + if resp.status_code != 200: + raise FileNotFoundError(f"GT image not found: {url}") + arr = np.array(Image.open(io.BytesIO(resp.content)).convert("RGB")) + embeddings.append(compute_clip_embedding(arr)) + else: + # Image: try each extension until one succeeds + content = None + for fn in candidates: + url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{fn}" + resp = requests.get(url, timeout=30) + if resp.status_code == 200: + content = resp.content + break + if content is None: + raise FileNotFoundError( + f"GT image not found in sgl-test-files. " + f"Tried: {', '.join(candidates)}" + ) + arr = np.array(Image.open(io.BytesIO(content)).convert("RGB")) + embeddings.append(compute_clip_embedding(arr)) logger.info( f"Loaded {len(embeddings)} GT embeddings for {case_id} from sgl-test-files" @@ -870,6 +904,33 @@ def load_gt_embeddings( return embeddings +def _remote_gt_url( + case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None +) -> str | None: + """Find the first existing remote GT URL, trying multiple extensions for images.""" + candidates = get_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ) + if is_video: + # For video, all 3 frame files must exist; just check the first one + url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{candidates[0]}" + try: + r = requests.head(url, timeout=10) + return url if r.status_code == 200 else None + except Exception: + return None + # For images, try each extension candidate + for fn in candidates: + url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{fn}" + try: + r = requests.head(url, timeout=10) + if r.status_code == 200: + return url + except Exception: + continue + return None + + def gt_exists( case_id: str, num_gpus: int, @@ -886,14 +947,7 @@ def gt_exists( return all((gt_dir / c).exists() for c in candidates) return any((gt_dir / c).exists() for c in candidates) - filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format) - fn = filenames[0] - url = f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{fn}" - try: - r = requests.head(url, timeout=10) - return r.status_code == 200 - except Exception: - return False + return _remote_gt_url(case_id, num_gpus, is_video, output_format) is not None def compare_with_gt( @@ -938,19 +992,6 @@ def compare_with_gt( frame_details=frame_details, ) - status = "PASSED" if passed else "FAILED" - print(f"\n{'='*60}") - print(f"[CLIP Consistency] {case_id}: {status}") - print(f" Threshold: {threshold}") - print(f" Min similarity: {min_similarity:.4f}") - print(f" Frame details:") - for detail in frame_details: - frame_status = "✓" if detail["passed"] else "✗" - print( - f" Frame {detail['frame_index']}: similarity={detail['similarity']:.4f} {frame_status}" - ) - print(f"{'='*60}\n") - if passed: logger.info( f"[Consistency] {case_id}: PASSED (min_similarity={min_similarity:.4f}, threshold={threshold})"