diff --git a/python/sglang/benchmark/datasets/longbench_v2.py b/python/sglang/benchmark/datasets/longbench_v2.py index e8a64295798f..5368bd286fa6 100644 --- a/python/sglang/benchmark/datasets/longbench_v2.py +++ b/python/sglang/benchmark/datasets/longbench_v2.py @@ -1,3 +1,4 @@ +import json import random from argparse import Namespace from dataclasses import dataclass @@ -8,9 +9,24 @@ from sglang.benchmark.datasets.common import BaseDataset, DatasetRow LONGBENCH_V2_REPO_ID = "THUDM/LongBench-v2" +# Pinned to a commit so a cached copy loads without any Hub API call. +LONGBENCH_V2_REVISION = "2b48e494f2c7a2f0af81aae178e05c7e1dde0fe9" LONGBENCH_V2_DEFAULT_OUTPUT_LEN = 10 # answer letter + short explanation +def load_longbench_v2_examples() -> List[dict]: + from huggingface_hub import hf_hub_download + + path = hf_hub_download( + LONGBENCH_V2_REPO_ID, + "data.json", + repo_type="dataset", + revision=LONGBENCH_V2_REVISION, + ) + with open(path) as f: + return json.load(f) + + def _format_prompt(example: dict) -> str: return ( f"{example['context']}\n\n" @@ -78,10 +94,7 @@ def sample_longbench_v2_requests( with open(dataset_path) as f: examples = [json.loads(line) for line in f if line.strip()] else: - from datasets import load_dataset - - ds = load_dataset(LONGBENCH_V2_REPO_ID, split="train") - examples = list(ds) + examples = load_longbench_v2_examples() random.shuffle(examples) diff --git a/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py index 64a3d200faa6..28f90b22a606 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py @@ -65,6 +65,7 @@ from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE +from sglang.srt.utils.common import retry_on_hub_rate_limit logger = init_logger(__name__) @@ -158,9 +159,11 @@ def _resolve_shape_dir( "Local path %s not found, downloading from HuggingFace Hub", local_path, ) - downloaded = snapshot_download( - repo_id=model_path, - allow_patterns=[f"{subfolder}/**"], + downloaded = retry_on_hub_rate_limit( + lambda: snapshot_download( + repo_id=model_path, + allow_patterns=[f"{subfolder}/**"], + ) ) local_path = os.path.join(downloaded, subfolder) @@ -205,9 +208,11 @@ def _resolve_model_subfolder( "Local path %s not found, downloading from HuggingFace Hub", local_path, ) - downloaded = snapshot_download( - repo_id=model_path, - allow_patterns=[f"{subfolder}/**"], + downloaded = retry_on_hub_rate_limit( + lambda: snapshot_download( + repo_id=model_path, + allow_patterns=[f"{subfolder}/**"], + ) ) local_path = os.path.join(downloaded, subfolder) diff --git a/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py b/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py index f32afd668fdd..f1884383744c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py @@ -33,6 +33,7 @@ verify_model_config_and_directory, ) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.srt.utils.common import retry_on_hub_rate_limit logger = init_logger(__name__) @@ -70,12 +71,14 @@ def _resolve_ideogram4_distilled_components_path() -> str: # fal's model cards explicitly reference the NF4 Diffusers repository for # these shared components. Download only those components: its conditional # and unconditional base transformers are unused by distilled variants. - return snapshot_download( - repo_id=_IDEOGRAM4_DISTILLED_COMPONENTS_MODEL, - revision=_IDEOGRAM4_DISTILLED_COMPONENTS_REVISION, - allow_patterns=_IDEOGRAM4_DISTILLED_COMPONENT_PATTERNS, - ignore_patterns=["*.onnx", "*.msgpack"], - max_workers=8, + return retry_on_hub_rate_limit( + lambda: snapshot_download( + repo_id=_IDEOGRAM4_DISTILLED_COMPONENTS_MODEL, + revision=_IDEOGRAM4_DISTILLED_COMPONENTS_REVISION, + allow_patterns=_IDEOGRAM4_DISTILLED_COMPONENT_PATTERNS, + ignore_patterns=["*.onnx", "*.msgpack"], + max_workers=8, + ) ) @@ -289,11 +292,13 @@ def _resolve_distilled_transformer_path(self) -> str: model_path = ( self.model_path if os.path.exists(self.model_path) - else snapshot_download( - repo_id=self.model_path, - allow_patterns=["transformer/*"], - ignore_patterns=["*.onnx", "*.msgpack"], - max_workers=8, + else retry_on_hub_rate_limit( + lambda: snapshot_download( + repo_id=self.model_path, + allow_patterns=["transformer/*"], + ignore_patterns=["*.onnx", "*.msgpack"], + max_workers=8, + ) ) ) self._distilled_transformer_path = os.path.join(model_path, "transformer") diff --git a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py index 8c049c3e07cb..cbf86fecd1c0 100644 --- a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py +++ b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py @@ -20,6 +20,7 @@ from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.loader.weight_utils import get_lock from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.srt.utils.common import retry_on_hub_rate_limit from sglang.utils import load_diffusion_overlay_registry_from_env logger = init_logger(__name__) @@ -470,11 +471,13 @@ def download_overlay_metadata( overlay_repo_id, ) return str( - snapshot_download_fn( - repo_id=overlay_repo_id, - allow_patterns=MODEL_OVERLAY_METADATA_PATTERNS, - revision=revision, - max_workers=4, + retry_on_hub_rate_limit( + lambda: snapshot_download_fn( + repo_id=overlay_repo_id, + allow_patterns=MODEL_OVERLAY_METADATA_PATTERNS, + revision=revision, + max_workers=4, + ) ) ) diff --git a/python/sglang/multimodal_gen/runtime/weights/source.py b/python/sglang/multimodal_gen/runtime/weights/source.py index 68a1e49f11bd..0471bb7728eb 100644 --- a/python/sglang/multimodal_gen/runtime/weights/source.py +++ b/python/sglang/multimodal_gen/runtime/weights/source.py @@ -14,6 +14,8 @@ from huggingface_hub import HfApi, hf_hub_download from huggingface_hub.utils import validate_repo_id +from sglang.srt.utils.common import retry_on_hub_rate_limit + WeightSourceKind = Literal["local", "huggingface"] _WEIGHT_SUFFIXES = (".safetensors", ".gguf", ".bin", ".pt", ".pth", ".ckpt") @@ -274,9 +276,8 @@ def resolve_weight_inventory(source: WeightSource) -> WeightInventory: ) assert source.repo_id is not None - model_info = HfApi().model_info( - source.repo_id, - revision=source.revision, + model_info = retry_on_hub_rate_limit( + lambda: HfApi().model_info(source.repo_id, revision=source.revision) ) files = tuple(sibling.rfilename for sibling in model_info.siblings) return WeightInventory( diff --git a/python/sglang/srt/arg_groups/model_path_hook.py b/python/sglang/srt/arg_groups/model_path_hook.py index 0093fac31515..605b128e3580 100644 --- a/python/sglang/srt/arg_groups/model_path_hook.py +++ b/python/sglang/srt/arg_groups/model_path_hook.py @@ -15,7 +15,7 @@ resolving_view, run_post_process_pass, ) -from sglang.srt.utils.common import is_remote_url +from sglang.srt.utils.common import find_local_repo_dir, is_remote_url from sglang.srt.utils.hf_transformers_utils import check_gguf_file from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri @@ -347,10 +347,25 @@ def _check_format(has_params, has_consolidated, has_hf_weights) -> bool: ), ) + snapshot_dir = find_local_repo_dir(cfg.model_path, revision=cfg.revision) + if snapshot_dir is not None: + # A cache may hold only some files, so trust it only when what it has decides. + has_params = os.path.exists(os.path.join(snapshot_dir, "params.json")) + has_hf_weights = bool( + glob.glob(os.path.join(snapshot_dir, "model*.safetensors")) + ) + if has_params and name_matches: + return True + if has_hf_weights and not name_matches: + return False + try: from huggingface_hub import HfApi - files = {s.rfilename for s in HfApi().model_info(cfg.model_path).siblings} + files = { + s.rfilename + for s in HfApi().model_info(cfg.model_path, revision=cfg.revision).siblings + } return _check_format( has_params="params.json" in files, has_consolidated=any( diff --git a/python/sglang/srt/lora/lora_config.py b/python/sglang/srt/lora/lora_config.py index 1187a77d95ef..a2d3331452a4 100644 --- a/python/sglang/srt/lora/lora_config.py +++ b/python/sglang/srt/lora/lora_config.py @@ -17,7 +17,9 @@ import os from typing import Dict, Optional -from huggingface_hub import snapshot_download +from huggingface_hub import hf_hub_download + +from sglang.srt.utils.common import download_hf_file_if_exists logger = logging.getLogger(__name__) @@ -76,27 +78,25 @@ def get_lora_config(self, dummy=False): if dummy: raise NotImplementedError() else: + config_name = "adapter_config.json" if not os.path.isdir(self.path): - weights_dir = snapshot_download(self.path, allow_patterns=["*.json"]) + config_path = hf_hub_download(repo_id=self.path, filename=config_name) else: - weights_dir = self.path - config_name = "adapter_config.json" - with open(os.path.join(weights_dir, config_name), "r") as f: + config_path = os.path.join(self.path, config_name) + with open(config_path, "r") as f: return json.load(f) def get_added_tokens_config(self): """Load added tokens from the LoRA adapter if the file exists.""" - # Determine the weights directory if not os.path.isdir(self.path): - weights_dir = snapshot_download(self.path, allow_patterns=["*.json"]) + added_tokens_path = download_hf_file_if_exists( + repo_id=self.path, filename="added_tokens.json" + ) else: - weights_dir = self.path - - # Construct the path to added_tokens.json - added_tokens_path = os.path.join(weights_dir, "added_tokens.json") + added_tokens_path = os.path.join(self.path, "added_tokens.json") # Return None if the file doesn't exist (optional for standard LoRA adapters) - if not os.path.exists(added_tokens_path): + if added_tokens_path is None or not os.path.exists(added_tokens_path): return None # Load and return the added tokens diff --git a/python/sglang/srt/model_loader/ci_weight_validation.py b/python/sglang/srt/model_loader/ci_weight_validation.py index 3d22e75b39ff..c5c655feded8 100644 --- a/python/sglang/srt/model_loader/ci_weight_validation.py +++ b/python/sglang/srt/model_loader/ci_weight_validation.py @@ -18,7 +18,11 @@ import safetensors -from sglang.srt.utils import log_info_on_rank0 +from sglang.srt.utils import ( + find_local_repo_dir, + log_info_on_rank0, + retry_on_hub_rate_limit, +) logger = logging.getLogger(__name__) @@ -152,11 +156,20 @@ def validate_cache_lightweight( custom_files = set() for key, value in auto_map.items(): if isinstance(value, str) and "." in value: - module_name = value.split(".")[0] - custom_files.add(f"{module_name}.py") + # "owner/repo--module.Class" loads its code from another repo. + code_repo, _, reference = value.rpartition("--") + code_dir = ( + find_local_repo_dir(code_repo) + if code_repo + else snapshot_dir + ) + if code_dir is None: + return False + module_name = reference.split(".")[0] + custom_files.add(os.path.join(code_dir, f"{module_name}.py")) - for custom_file in custom_files: - custom_file_path = os.path.join(snapshot_dir, custom_file) + for custom_file_path in custom_files: + custom_file = os.path.basename(custom_file_path) if not os.path.exists(custom_file_path): logger.debug( "Custom module file not in snapshot: %s for %s", @@ -782,19 +795,21 @@ def __init__(self, *args, **kwargs): hf_folder = None for attempt in range(max_retries): try: - hf_folder = snapshot_download( - model_name_or_path, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - cache_dir=cache_dir, - tqdm_class=DisabledTqdm, - revision=revision, - local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, - # Force single-threaded downloads to prevent race conditions - # on NFS. HF hub defaults to max_workers=8, which can cause - # .incomplete file conflicts when multiple threads operate - # on the same files - max_workers=1, + hf_folder = retry_on_hub_rate_limit( + lambda: snapshot_download( + model_name_or_path, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + cache_dir=cache_dir, + tqdm_class=DisabledTqdm, + revision=revision, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + # Force single-threaded downloads to prevent race conditions + # on NFS. HF hub defaults to max_workers=8, which can cause + # .incomplete file conflicts when multiple threads operate + # on the same files + max_workers=1, + ) ) except (FileNotFoundError, OSError) as e: # Race condition: .incomplete file was moved/deleted by another diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index b0f8813b8763..bd20ed8d8c17 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -8,7 +8,6 @@ import collections import concurrent.futures import dataclasses -import fnmatch import gc import glob import hashlib @@ -67,7 +66,6 @@ init_empty_weights = None get_max_memory = None -from huggingface_hub import HfApi, hf_hub_download from torch import nn from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer from transformers.utils import SAFE_WEIGHTS_INDEX_NAME @@ -98,7 +96,7 @@ get_model_architecture, set_default_torch_dtype, ) -from sglang.srt.utils.common import is_cuda_alike +from sglang.srt.utils.common import download_hf_file_if_exists, is_cuda_alike # Constants for memory management DEFAULT_GPU_MEMORY_FRACTION_FOR_CALIBRATION = ( @@ -2603,13 +2601,11 @@ def _get_config_file(self, qlora_adapter: str) -> str: if os.path.exists(config_file_path): break else: - hf_api = HfApi() - repo_files = hf_api.list_repo_files(repo_id=qlora_adapter) for file in self.possible_config_file_names: - if file in repo_files: - config_file_path = hf_hub_download( - repo_id=qlora_adapter, filename=file - ) + config_file_path = download_hf_file_if_exists( + repo_id=qlora_adapter, filename=file + ) + if config_file_path: break if not config_file_path: @@ -2634,19 +2630,18 @@ def _get_weight_files( if weight_files: return weight_files, pattern else: - hf_api = HfApi() - repo_files = hf_api.list_repo_files(repo_id=model_name_or_path) + # Downloads only the first pattern the repo has, checking the cache first. + hf_folder = download_weights_from_hf( + model_name_or_path, + self.load_config.download_dir, + allowed_patterns, + revision, + ignore_patterns=self.load_config.ignore_patterns, + ) for pattern in allowed_patterns: - matching_files = fnmatch.filter(repo_files, pattern) - if matching_files: - hf_folder = download_weights_from_hf( - model_name_or_path, - self.load_config.download_dir, - [pattern], - revision, - ignore_patterns=self.load_config.ignore_patterns, - ) - return glob.glob(os.path.join(hf_folder, pattern)), pattern + weight_files = glob.glob(os.path.join(hf_folder, pattern)) + if weight_files: + return weight_files, pattern raise RuntimeError(f"No model weights found in: `{model_name_or_path}`") diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index e3cca9b705ce..274d6506b842 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -36,7 +36,6 @@ import safetensors.torch import torch from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download -from huggingface_hub.errors import HfHubHTTPError from pydantic import BaseModel, ConfigDict, ValidationInfo, model_validator from tqdm.auto import tqdm @@ -67,6 +66,7 @@ is_hip, log_info_on_rank0, print_warning_once, + retry_on_hub_rate_limit, ) from sglang.srt.utils.common import is_cuda_alike from sglang.utils import is_in_ci @@ -323,13 +323,15 @@ def get_quant_config( if not is_local: # Download the config files. with get_lock(model_name_or_path, load_config.download_dir): - hf_folder = snapshot_download( - model_name_or_path, - revision=model_config.revision, - allow_patterns="*.json", - cache_dir=load_config.download_dir, - local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, - tqdm_class=DisabledTqdm, + hf_folder = retry_on_hub_rate_limit( + lambda: snapshot_download( + model_name_or_path, + revision=model_config.revision, + allow_patterns="*.json", + cache_dir=load_config.download_dir, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + tqdm_class=DisabledTqdm, + ) ) else: hf_folder = model_name_or_path @@ -515,21 +517,9 @@ def _find_local_hf_snapshot_dir_unlocked( # Check custom cache_dir (if provided) if cache_dir: try: - repo_folder = os.path.join( - cache_dir, - huggingface_hub.constants.REPO_ID_SEPARATOR.join( - ["models", *model_name_or_path.split("/")] - ), + found_local_snapshot_dir = find_local_repo_dir( + model_name_or_path, revision=revision, cache_dir=cache_dir ) - rev_to_use = revision - ref_path = os.path.join(repo_folder, "refs", revision or "main") - if os.path.isfile(ref_path): - with open(ref_path) as f: - rev_to_use = f.read().strip() - if rev_to_use: - rev_dir = os.path.join(repo_folder, "snapshots", rev_to_use) - if os.path.isdir(rev_dir): - found_local_snapshot_dir = rev_dir except Exception as e: logger.warning( "Failed to find local snapshot in custom cache_dir %s: %s", @@ -664,18 +654,9 @@ def download_weights_from_hf( if not huggingface_hub.constants.HF_HUB_OFFLINE: # Before we download we look at what is available: fs = HfFileSystem() - try: - file_list = fs.ls(model_name_or_path, detail=False, revision=revision) - except HfHubHTTPError as e: - # Fail open (e.g. a 429 rate limit): pick the format from the local - # snapshot; snapshot_download below re-raises errors it cannot recover. - logger.warning( - "Listing %s on the Hub failed, using the local snapshot: %s", - model_name_or_path, - e, - ) - local_dir = find_local_repo_dir(model_name_or_path, revision) - file_list = os.listdir(local_dir) if local_dir else [] + file_list = retry_on_hub_rate_limit( + lambda: fs.ls(model_name_or_path, detail=False, revision=revision) + ) # depending on what is available we download different things for pattern in allow_patterns: diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index 63944d61975d..739f5edbc62b 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -1017,7 +1017,11 @@ def __init__( ) inkling_quant_config = get_quantization_config( - SimpleNamespace(hf_config=self.config, model_path=get_model().model_path) + SimpleNamespace( + hf_config=self.config, + model_path=get_model().model_path, + revision=get_model().revision, + ) ) if inkling_quant_config is not None: quant_config = inkling_quant_config diff --git a/python/sglang/srt/models/inkling_common/quantization/config.py b/python/sglang/srt/models/inkling_common/quantization/config.py index cf16df690e2c..fe15ef914bb8 100644 --- a/python/sglang/srt/models/inkling_common/quantization/config.py +++ b/python/sglang/srt/models/inkling_common/quantization/config.py @@ -7,7 +7,6 @@ import huggingface_hub import torch -from huggingface_hub import snapshot_download from sglang.srt.configs.model_config import ModelConfig from sglang.srt.layers.quantization.base_config import ( @@ -19,6 +18,7 @@ ModelOptFp4LinearMethod, ) from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils.common import download_hf_file_if_exists from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri logger = logging.getLogger(__name__) @@ -45,22 +45,21 @@ def _get_raw_quant_config( model_name_or_path = model_config.model_path - # A local path holds hf_quant_config.json directly; anything else resolves its - # JSON configs first. An object-storage URL is neither a directory nor a valid - # repo id, so it needs its own branch before snapshot_download. + # An object-storage URL is neither a directory nor a valid repo id. if os.path.isdir(model_name_or_path): - hf_folder = model_name_or_path + quant_config_file = os.path.join(model_name_or_path, "hf_quant_config.json") elif is_runai_obj_uri(model_name_or_path): hf_folder = ObjectStorageModel.download_and_get_path(model_name_or_path) + quant_config_file = os.path.join(hf_folder, "hf_quant_config.json") else: - hf_folder = snapshot_download( - model_name_or_path, - allow_patterns="*.json", + quant_config_file = download_hf_file_if_exists( + repo_id=model_name_or_path, + filename="hf_quant_config.json", + revision=model_config.revision, local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, ) - quant_config_file = os.path.join(hf_folder, "hf_quant_config.json") - if not os.path.exists(quant_config_file): + if quant_config_file is None or not os.path.exists(quant_config_file): return None with open(quant_config_file) as f: config = json.load(f) diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index e39826167f8b..1f3dd315d24f 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Callable, List, Literal, Optional, Tuple import torch -from huggingface_hub import snapshot_download +from huggingface_hub import hf_hub_download from sglang.kernels.ops.sampling import softmax as sampling_softmax from sglang.kernels.ops.speculative.cache_locs import ( @@ -726,19 +726,17 @@ def load_token_map(token_map_path: str) -> List[int]: if os.path.exists(cached_repo_path): cache_dir = cached_repo_path - if cache_dir is None: - if envs.SGLANG_USE_MODELSCOPE.get(): - from modelscope.hub.snapshot_download import ( - snapshot_download as download_func, - ) - else: - download_func = snapshot_download - cache_dir = download_func( - repo_id, - ignore_patterns=["*.bin", "*.safetensors"], - ) + if cache_dir is None and not envs.SGLANG_USE_MODELSCOPE.get(): + token_map_path = hf_hub_download(repo_id=repo_id, filename=file_name) + else: + if cache_dir is None: + from modelscope.hub.snapshot_download import snapshot_download - token_map_path = os.path.join(cache_dir, file_name) + cache_dir = snapshot_download( + repo_id, + ignore_patterns=["*.bin", "*.safetensors"], + ) + token_map_path = os.path.join(cache_dir, file_name) hot_token_id = torch.load(token_map_path, weights_only=True) return torch.tensor(hot_token_id, dtype=torch.int64) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 0d22705f49ef..041857ef3c8f 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -3941,12 +3941,22 @@ def _exists(name): from huggingface_hub import HfFileSystem fs = HfFileSystem() + snapshot_dir = find_local_repo_dir(model_path) + + def _cached(name): + if snapshot_dir is None: + return None + path = os.path.join(snapshot_dir, name) + return path if os.path.exists(path) else None def _open(name): - return fs.open(f"{model_path}/{name}", "rb") + path = _cached(name) + return ( + open(path, "rb") if path else fs.open(f"{model_path}/{name}", "rb") + ) def _exists(name): - return fs.exists(f"{model_path}/{name}") + return _cached(name) is not None or fs.exists(f"{model_path}/{name}") if _exists("model.safetensors.index.json"): with _open("model.safetensors.index.json") as f: @@ -4247,17 +4257,18 @@ def get_eager_max_batch_size(max_batch_size: int) -> int: return ceil_align(max_batch_size, get_cp_padding_align_size()) -def find_local_repo_dir(repo_id: str, revision: Optional[str] = None) -> Optional[str]: +def find_local_repo_dir( + repo_id: str, revision: Optional[str] = None, cache_dir: Optional[str] = None +) -> Optional[str]: import huggingface_hub as hf # Build cache path cache_path = os.path.join( - hf.constants.HF_HUB_CACHE, + cache_dir or hf.constants.HF_HUB_CACHE, hf.constants.REPO_ID_SEPARATOR.join(["models", *repo_id.split("/")]), ) - # A branch or tag name (default "main") maps to a commit through refs/; - # snapshots/ is keyed by commit only. + # Snapshots are named by commit; a branch or tag maps to one through refs/. ref_path = os.path.join(cache_path, "refs", revision or "main") if os.path.isfile(ref_path): with open(ref_path) as f: @@ -4272,6 +4283,59 @@ def find_local_repo_dir(repo_id: str, revision: Optional[str] = None) -> Optiona return None +def _hub_rate_limit_response(error: Optional[BaseException]): + from huggingface_hub.errors import HfHubHTTPError + + # snapshot_download re-raises an uncached 429 as LocalEntryNotFoundError. + while error is not None: + if ( + isinstance(error, HfHubHTTPError) + and error.response is not None + and error.response.status_code == 429 + ): + return error.response + error = error.__cause__ + return None + + +def retry_on_hub_rate_limit(fn: Callable[[], T], max_wait_s: float = 600) -> T: + from huggingface_hub.utils import parse_ratelimit_headers + + deadline = time.monotonic() + max_wait_s + while True: + try: + return fn() + except Exception as e: + response = _hub_rate_limit_response(e) + remaining_s = deadline - time.monotonic() + if response is None or remaining_s <= 0: + raise + rate_limit = parse_ratelimit_headers(response.headers) + # Arbitrary fallback for a 429 without a RateLimit header. + wait_s = rate_limit.reset_in_seconds + 1 if rate_limit else 30 + logger.warning( + "HF Hub rate limit hit, retrying in %ds: %s", wait_s, response.url + ) + time.sleep(min(wait_s, remaining_s)) + + +def download_hf_file_if_exists(repo_id: str, filename: str, **kwargs) -> Optional[str]: + import huggingface_hub + from huggingface_hub.errors import LocalEntryNotFoundError, RemoteEntryNotFoundError + + try: + return huggingface_hub.hf_hub_download( + repo_id=repo_id, filename=filename, **kwargs + ) + except RemoteEntryNotFoundError: + return None + except LocalEntryNotFoundError: + # Offline, a file missing from the cache is absent; online, the Hub failed. + if huggingface_hub.constants.HF_HUB_OFFLINE or kwargs.get("local_files_only"): + return None + raise + + def read_system_prompt_from_file(model_name: str) -> str: """Read system prompt from a file in the HuggingFace cache directory. diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py index 5c90ddfd93df..0d51f8d05dc8 100644 --- a/python/sglang/srt/utils/hf_transformers/common.py +++ b/python/sglang/srt/utils/hf_transformers/common.py @@ -96,7 +96,12 @@ from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config from sglang.srt.configs.deepseek_v41 import DEEPSEEK_V41_CONFIG_CLASSES from sglang.srt.configs.internvl import InternVLChatConfig -from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset +from sglang.srt.utils import ( + get_bool_env_var, + logger, + lru_cache_frozenset, + retry_on_hub_rate_limit, +) from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri from ..hf_transformers_patches import normalize_rope_scaling_compat @@ -341,7 +346,9 @@ def download_from_hf( if not allow_patterns: allow_patterns = ["*.json", "*.bin", "*.model"] - return snapshot_download(model_path, allow_patterns=allow_patterns) + return retry_on_hub_rate_limit( + lambda: snapshot_download(model_path, allow_patterns=allow_patterns) + ) def resolve_runai_obj_uri(model_name_or_path: str) -> str: @@ -457,6 +464,8 @@ def resolve_hf_gguf_reference( if len(parts) != 2: return None + if _cached_file_exists(model, filename="config.json", revision=revision): + return None from huggingface_hub import HfApi diff --git a/python/sglang/srt/utils/hf_transformers/mistral_utils.py b/python/sglang/srt/utils/hf_transformers/mistral_utils.py index 48e8be00bcdc..c7255eb6d52b 100644 --- a/python/sglang/srt/utils/hf_transformers/mistral_utils.py +++ b/python/sglang/srt/utils/hf_transformers/mistral_utils.py @@ -421,7 +421,6 @@ def wrap_as_pixtral(processor, config): _MISTRAL_COMMON_REJECTED_KWARGS = frozenset( { "trust_remote_code", - "tokenizer_revision", "use_fast", "_from_auto", "clean_up_tokenization_spaces", diff --git a/python/sglang/srt/utils/hf_transformers/processor.py b/python/sglang/srt/utils/hf_transformers/processor.py index f129ecf59967..039f281fdf48 100644 --- a/python/sglang/srt/utils/hf_transformers/processor.py +++ b/python/sglang/srt/utils/hf_transformers/processor.py @@ -214,7 +214,7 @@ def get_processor( *args, tokenizer_mode: str = "auto", trust_remote_code: bool = False, - tokenizer_revision: Optional[str] = None, + revision: Optional[str] = None, use_fast: Optional[bool] = None, image_processor_backend: Optional[str] = None, tokenizer_backend: str = "huggingface", @@ -226,7 +226,6 @@ def get_processor( _ensure_fastokens_patched() - revision = kwargs.pop("revision", tokenizer_revision) image_processor_backend = _normalize_image_processor_backend( image_processor_backend, use_fast ) @@ -369,7 +368,7 @@ def get_processor( tokenizer_name, tokenizer_mode=tokenizer_mode, trust_remote_code=trust_remote_code, - tokenizer_revision=revision, + revision=revision, tokenizer_backend=tokenizer_backend, ) if isinstance(processor, PreTrainedTokenizerBase): diff --git a/python/sglang/srt/utils/hf_transformers/tokenizer.py b/python/sglang/srt/utils/hf_transformers/tokenizer.py index 9a76031b0b8e..9ab193210956 100644 --- a/python/sglang/srt/utils/hf_transformers/tokenizer.py +++ b/python/sglang/srt/utils/hf_transformers/tokenizer.py @@ -63,7 +63,7 @@ def _load_tokenizer_by_declared_class(tokenizer_name, *args, **kwargs): import transformers try: - revision = kwargs.get("revision") or kwargs.get("tokenizer_revision") + revision = kwargs.get("revision") config_file = _resolve_local_or_cached_file( tokenizer_name, "tokenizer_config.json", revision ) @@ -472,7 +472,7 @@ def get_tokenizer( *args, tokenizer_mode: str = "auto", trust_remote_code: bool = False, - tokenizer_revision: Optional[str] = None, + revision: Optional[str] = None, tokenizer_backend: str = "huggingface", **kwargs, ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: @@ -512,13 +512,13 @@ def get_tokenizer( common_kwargs = dict( trust_remote_code=trust_remote_code, - tokenizer_revision=tokenizer_revision, + revision=revision, clean_up_tokenization_spaces=False, **kwargs, ) try: - if is_bare_tekken_checkpoint(tokenizer_name, tokenizer_revision): + if is_bare_tekken_checkpoint(tokenizer_name, revision): from transformers.tokenization_mistral_common import ( MistralCommonTokenizer, ) @@ -532,7 +532,7 @@ def get_tokenizer( ) tokenizer = MistralCommonTokenizer.from_pretrained( - tokenizer_name, revision=tokenizer_revision + tokenizer_name, revision=revision ) else: tokenizer = _auto_tokenizer_from_pretrained( @@ -550,7 +550,7 @@ def get_tokenizer( tokenizer_name, *args, **common_kwargs ) - return _apply_post_load_fixes(tokenizer, tokenizer_name, tokenizer_revision) + return _apply_post_load_fixes(tokenizer, tokenizer_name, revision) except Exception as e: if tokenizer_backend == "fastokens": raise RuntimeError( diff --git a/python/sglang/srt/utils/hf_transformers_patches.py b/python/sglang/srt/utils/hf_transformers_patches.py index 3e945992d37b..f4a3149bd3f7 100644 --- a/python/sglang/srt/utils/hf_transformers_patches.py +++ b/python/sglang/srt/utils/hf_transformers_patches.py @@ -70,8 +70,9 @@ def apply_all(): # v5 general patches _ensure_is_torch_fx_available_compat() - # CI-only: neutralize HF API calls inside tokenizer from_pretrained + # CI-only: avoid HF API calls during from_pretrained patch_is_base_mistral_in_ci() + _patch_transformers_hub_calls_in_ci() logger.debug("transformers compatibility patches applied") @@ -464,3 +465,85 @@ def _noop_patch_mistral_regex(cls, tokenizer, *args, **kwargs): logger.info("CI: patched _patch_mistral_regex to skip HF API calls") _is_base_mistral_patched = True + + +def _patch_transformers_hub_calls_in_ci(): + # Serve transformers' per-load Hub API calls from the local cache when it has them. + from sglang.srt.environ import envs + + if not envs.SGLANG_IS_IN_CI.get(): + return + + from huggingface_hub import get_cached_repo_tree + from huggingface_hub.errors import CachedRepoTreeNotFoundError + from huggingface_hub.hf_api import RepoFolder + from transformers.utils.hub import hf_api + + # transformers' own HfApi instance; huggingface_hub downloads are unaffected. + api = hf_api() + resolve_revision = api.resolve_revision + list_repo_files = api.list_repo_files + list_repo_tree = api.list_repo_tree + + def _resolve_revision_from_cache(*args, **kwargs): + # The cached ref still pins a load to one commit; a miss stays unresolved. + return resolve_revision(*args, **{**kwargs, "local_files_only": True}) + + def _cached_repo_files(repo_id, revision, repo_type): + # The saved full listing; a snapshot folder holds only what was downloaded. + try: + return get_cached_repo_tree(repo_id, repo_type=repo_type, revision=revision) + except CachedRepoTreeNotFoundError: + return None + + def _list_repo_files(repo_id, *, revision=None, repo_type=None, token=None): + files = _cached_repo_files( + repo_id=repo_id, revision=revision, repo_type=repo_type + ) + if files is None: + return list_repo_files( + repo_id=repo_id, revision=revision, repo_type=repo_type, token=token + ) + return [entry.path for entry in files] + + def _list_repo_tree( + repo_id, + path_in_repo=None, + *, + recursive=False, + revision=None, + repo_type=None, + **kwargs, + ): + files = _cached_repo_files( + repo_id=repo_id, revision=revision, repo_type=repo_type + ) + if files is None: + return list_repo_tree( + repo_id=repo_id, + path_in_repo=path_in_repo, + recursive=recursive, + revision=revision, + repo_type=repo_type, + **kwargs, + ) + prefix = f"{path_in_repo.strip('/')}/" if path_in_repo else "" + entries, folders = [], set() + for entry in files: + if not entry.path.startswith(prefix): + continue + *dirs, _ = entry.path[len(prefix) :].split("/") + if recursive: + entries.append(entry) + folders.update( + prefix + "/".join(dirs[:depth]) for depth in range(1, len(dirs) + 1) + ) + elif dirs: + folders.add(prefix + dirs[0]) + else: + entries.append(entry) + return entries + [RepoFolder(path=path, oid=None) for path in sorted(folders)] + + api.resolve_revision = _resolve_revision_from_cache + api.list_repo_files = _list_repo_files + api.list_repo_tree = _list_repo_tree diff --git a/python/sglang/test/kl_test_utils.py b/python/sglang/test/kl_test_utils.py index 11cff0316a50..2e3bb244164e 100644 --- a/python/sglang/test/kl_test_utils.py +++ b/python/sglang/test/kl_test_utils.py @@ -2,27 +2,16 @@ import json import os import random -import time import numpy as np import requests +from sglang.benchmark.datasets.longbench_v2 import load_longbench_v2_examples from sglang.srt.utils.hf_transformers_utils import get_tokenizer -# LongBench V2 dataset configuration -# Reference: https://github.com/THUDM/LongBench -LONGBENCH_V2_DATASET = "THUDM/LongBench-v2" -LONGBENCH_V2_SPLIT = "train" DEFAULT_NUM_SAMPLES = 48 # Number of samples to use DEFAULT_PROMPT_TOKENS = 3000 # Maximum number of tokens to use -# Outside the repo, where CI's checkout clean would wipe it before every job. -CACHE_DIR = os.path.join( - os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), - "sglang", - "longbench", -) -# Fail the test instead of letting a slow stream run out the CI job. -DOWNLOAD_TIMEOUT_S = 300 +CACHE_DIR = os.path.join(os.path.dirname(__file__), ".longbench_cache") # In-memory cache for the current session _cached_input_ids = {} @@ -70,42 +59,24 @@ def get_input_ids( print(f"Loaded {len(input_ids)} prompts from cache") return input_ids - # Download from HuggingFace using streaming - try: - from datasets import load_dataset - except ImportError as exc: - raise ImportError( - "Please install the 'datasets' package: pip install datasets" - ) from exc - tokenizer = get_tokenizer(tokenizer_path, trust_remote_code=trust_remote_code) - print(f"Downloading {num_samples} samples from LongBench V2 (streaming)...") - dataset = load_dataset( - LONGBENCH_V2_DATASET, split=LONGBENCH_V2_SPLIT, streaming=True - ) + print(f"Loading {num_samples} samples from LongBench V2...") + dataset = load_longbench_v2_examples() input_ids = [] - deadline = time.monotonic() + DOWNLOAD_TIMEOUT_S for example in dataset: if len(input_ids) >= num_samples: break - if time.monotonic() > deadline: - raise TimeoutError( - f"LongBench V2 download exceeded {DOWNLOAD_TIMEOUT_S}s after " - f"{len(input_ids)}/{num_samples} samples" - ) text = format_longbench_v2_example(example) tokens = tokenizer.encode(text) # Truncate to a random length between 0.5x and 1.5x of max_prompt_tokens truncate_len = int(max_prompt_tokens * random.uniform(0.5, 1.5)) input_ids.append(tokens[:truncate_len]) - # Save to local cache; concurrent jobs may share the directory. - tmp_file = f"{cache_file}.{os.getpid()}.tmp" - with open(tmp_file, "w") as f: + # Save to local cache + with open(cache_file, "w") as f: json.dump(input_ids, f) - os.replace(tmp_file, cache_file) print(f"Saved {len(input_ids)} prompts to cache: {cache_file}") # Also cache in memory diff --git a/python/sglang/test/simple_eval_mmmu_vlm.py b/python/sglang/test/simple_eval_mmmu_vlm.py index e05885e9d739..1a60aadfbd7d 100644 --- a/python/sglang/test/simple_eval_mmmu_vlm.py +++ b/python/sglang/test/simple_eval_mmmu_vlm.py @@ -11,6 +11,7 @@ from typing import List, Optional, Tuple from datasets import concatenate_datasets, load_dataset +from huggingface_hub import hf_hub_download from PIL import Image from sglang.test import simple_eval_common as common @@ -23,6 +24,9 @@ map_with_progress, ) +# Pinned to a commit so cached files load without any Hub API call. +MMMU_REVISION = "876ce5cb130f7f7e290ce4d9984357737d4db5cf" + class MMMUVLMEval(Eval): DOMAIN_CAT2SUB_CAT = { @@ -100,7 +104,13 @@ def _prepare_mmmu_samples(self, k: int) -> List[dict]: datasets = [] for subj in subjects: try: - d = load_dataset("MMMU/MMMU", subj, split="validation") + path = hf_hub_download( + "MMMU/MMMU", + f"{subj}/validation-00000-of-00001.parquet", + repo_type="dataset", + revision=MMMU_REVISION, + ) + d = load_dataset("parquet", data_files=path, split="train") # attach subject info via transform d = d.add_column("__subject__", [subj] * len(d)) datasets.append(d) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 162706cbba49..c4cb97c6ee35 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -396,6 +396,36 @@ def watcher(): return stop +def _get_launch_arg(other_args: list[str], flag: str) -> Optional[str]: + for i, arg in enumerate(other_args): + if arg == flag and i + 1 < len(other_args): + return str(other_args[i + 1]) + if isinstance(arg, str) and arg.startswith(f"{flag}="): + return arg.split("=", 1)[1] + return None + + +def _draft_model_cache_complete(draft: Optional[str], revision: Optional[str]) -> bool: + from sglang.srt.model_loader.weight_utils import _check_index_files_exist + from sglang.srt.utils import find_local_repo_dir + + if draft is None or os.path.isdir(draft): + return True + snapshot_dir = find_local_repo_dir(draft, revision=revision) + if snapshot_dir is None: + return False + # Draft repos often ship no tokenizer; only the config and weights must be cached. + snapshot = Path(snapshot_dir) + has_weights = any( + any(snapshot.glob(pattern)) for pattern in ("*.safetensors", "*.bin", "*.pt") + ) + return ( + (snapshot / "config.json").exists() + and has_weights + and _check_index_files_exist(snapshot_dir)[0] + ) + + def _try_enable_offline_mode_if_cache_complete( model_name_or_path: str, env: dict, other_args: Optional[list[str]] = None ) -> Optional[str]: @@ -426,8 +456,19 @@ def _try_enable_offline_mode_if_cache_complete( requires_hf_quant_config = True break + if not _draft_model_cache_complete( + draft=_get_launch_arg(other_args, flag="--speculative-draft-model-path"), + revision=_get_launch_arg(other_args, flag="--speculative-draft-model-revision"), + ): + print( + f"CI_OFFLINE: Draft model cache incomplete, will use online mode - {model_name_or_path}" + ) + return None + marker_path = _validate_cache_for_offline( - model_name_or_path, requires_hf_quant_config + model_name_or_path, + requires_hf_quant_config=requires_hf_quant_config, + revision=_get_launch_arg(other_args, flag="--revision"), ) if marker_path is not None: env["HF_HUB_OFFLINE"] = "1" @@ -435,7 +476,9 @@ def _try_enable_offline_mode_if_cache_complete( def _validate_cache_for_offline( - model_name_or_path: str, requires_hf_quant_config: bool + model_name_or_path: str, + requires_hf_quant_config: bool, + revision: Optional[str] = None, ) -> Optional[str]: """Return the per-run marker path if the model's local cache is complete enough to load offline, else None. @@ -457,7 +500,7 @@ def _validate_cache_for_offline( # Try to find local snapshot try: - snapshot_dir = find_local_repo_dir(model_name_or_path, revision=None) + snapshot_dir = find_local_repo_dir(model_name_or_path, revision=revision) if not snapshot_dir or not os.path.isdir(snapshot_dir): return None except Exception: @@ -531,12 +574,16 @@ def _hf_offline_if_cache_complete(engine_kwargs: dict): model_path = server_args.model_path tokenizer_path = server_args.tokenizer_path draft_path = server_args.speculative_draft_model_path + revision = server_args.revision + draft_revision = server_args.speculative_draft_model_revision quantization = server_args.quantization uses_lora = server_args.enable_lora or bool(server_args.lora_paths) else: model_path = engine_kwargs.get("model_path") tokenizer_path = engine_kwargs.get("tokenizer_path") draft_path = engine_kwargs.get("speculative_draft_model_path") + revision = engine_kwargs.get("revision") + draft_revision = engine_kwargs.get("speculative_draft_model_revision") quantization = engine_kwargs.get("quantization") uses_lora = bool( engine_kwargs.get("enable_lora") or engine_kwargs.get("lora_paths") @@ -554,14 +601,21 @@ def _hf_offline_if_cache_complete(engine_kwargs: dict): ] repo_ids = { path - for path in (model_path, tokenizer_path, draft_path) + for path in (model_path, tokenizer_path) if path is not None and not os.path.isdir(path) } - if not repo_ids or not all( - _validate_cache_for_offline( - repo_id, requires_hf_quant_config and repo_id == model_path + if ( + not repo_ids + or not all( + _validate_cache_for_offline( + repo_id, + requires_hf_quant_config=requires_hf_quant_config + and repo_id == model_path, + revision=revision, + ) + for repo_id in repo_ids ) - for repo_id in repo_ids + or not _draft_model_cache_complete(draft=draft_path, revision=draft_revision) ): yield return