diff --git a/python/sglang/srt/model_loader/ci_weight_validation.py b/python/sglang/srt/model_loader/ci_weight_validation.py index 98abdd3cb1c3..3d22e75b39ff 100644 --- a/python/sglang/srt/model_loader/ci_weight_validation.py +++ b/python/sglang/srt/model_loader/ci_weight_validation.py @@ -412,8 +412,11 @@ def _validate_sharded_model( corrupted_files.append(f) if group_info["suffix"] == "safetensors": + # Qwen3.5 names shards `model.safetensors-00001-of-00014.safetensors`, + # so the prefix may already end with the suffix. + index_stem = group_info["prefix"].removesuffix(".safetensors") index_file = os.path.join( - snapshot_dir, f"{group_info['prefix']}.safetensors.index.json" + snapshot_dir, f"{index_stem}.safetensors.index.json" ) if not os.path.exists(index_file): return ( diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index 6b274b97afc0..e3cca9b705ce 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -36,6 +36,7 @@ import safetensors.torch import torch from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download +from huggingface_hub.errors import HfHubHTTPError from pydantic import BaseModel, ConfigDict, ValidationInfo, model_validator from tqdm.auto import tqdm @@ -521,11 +522,10 @@ def _find_local_hf_snapshot_dir_unlocked( ), ) rev_to_use = revision - if not rev_to_use: - ref_main = os.path.join(repo_folder, "refs", "main") - if os.path.isfile(ref_main): - with open(ref_main) as f: - rev_to_use = f.read().strip() + ref_path = os.path.join(repo_folder, "refs", revision or "main") + if os.path.isfile(ref_path): + with open(ref_path) as f: + rev_to_use = f.read().strip() if rev_to_use: rev_dir = os.path.join(repo_folder, "snapshots", rev_to_use) if os.path.isdir(rev_dir): @@ -664,7 +664,18 @@ def download_weights_from_hf( if not huggingface_hub.constants.HF_HUB_OFFLINE: # Before we download we look at what is available: fs = HfFileSystem() - file_list = fs.ls(model_name_or_path, detail=False, revision=revision) + try: + file_list = fs.ls(model_name_or_path, detail=False, revision=revision) + except HfHubHTTPError as e: + # Fail open (e.g. a 429 rate limit): pick the format from the local + # snapshot; snapshot_download below re-raises errors it cannot recover. + logger.warning( + "Listing %s on the Hub failed, using the local snapshot: %s", + model_name_or_path, + e, + ) + local_dir = find_local_repo_dir(model_name_or_path, revision) + file_list = os.listdir(local_dir) if local_dir else [] # depending on what is available we download different things for pattern in allow_patterns: diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 5d320f3bf8c1..0d22705f49ef 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -4256,12 +4256,12 @@ def find_local_repo_dir(repo_id: str, revision: Optional[str] = None) -> Optiona hf.constants.REPO_ID_SEPARATOR.join(["models", *repo_id.split("/")]), ) - # Get revision from main ref if not specified - if not revision: - ref_path = os.path.join(cache_path, "refs", "main") - if os.path.isfile(ref_path): - with open(ref_path) as f: - revision = f.read().strip() + # A branch or tag name (default "main") maps to a commit through refs/; + # snapshots/ is keyed by commit only. + ref_path = os.path.join(cache_path, "refs", revision or "main") + if os.path.isfile(ref_path): + with open(ref_path) as f: + revision = f.read().strip() # List files from revision directory if revision: diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 1d745a118e00..162706cbba49 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -401,18 +401,7 @@ def _try_enable_offline_mode_if_cache_complete( ) -> Optional[str]: """Set HF_HUB_OFFLINE=1 in `env` if the model cache validates; return the per-run marker path, or None if offline mode was not enabled. - - Markers are per-run and not shared across runners. A marker is only a hint; - the current launch's requirements (e.g. hf_quant_config.json) are revalidated. """ - from sglang.srt.model_loader.ci_weight_validation import ( - _get_per_run_marker_path, - _read_per_run_marker, - _write_per_run_marker, - validate_cache_lightweight, - ) - from sglang.srt.utils import find_local_repo_dir - other_args = other_args or [] # Skip offline mode for LoRA scenarios (dynamic adapter loading may need online access) @@ -428,6 +417,40 @@ def _try_enable_offline_mode_if_cache_complete( ) return None + # Detect before the marker check so the current launch's requirements are known. + requires_hf_quant_config = False + for i, arg in enumerate(other_args): + if arg == "--quantization" and i + 1 < len(other_args): + quant_value = other_args[i + 1].lower() + if quant_value in ["modelopt_fp4", "modelopt_fp8", "modelopt"]: + requires_hf_quant_config = True + break + + marker_path = _validate_cache_for_offline( + model_name_or_path, requires_hf_quant_config + ) + if marker_path is not None: + env["HF_HUB_OFFLINE"] = "1" + return marker_path + + +def _validate_cache_for_offline( + model_name_or_path: str, requires_hf_quant_config: bool +) -> Optional[str]: + """Return the per-run marker path if the model's local cache is complete + enough to load offline, else None. + + Markers are per-run and not shared across runners. A marker is only a hint; + the current launch's requirements (e.g. hf_quant_config.json) are revalidated. + """ + from sglang.srt.model_loader.ci_weight_validation import ( + _get_per_run_marker_path, + _read_per_run_marker, + _write_per_run_marker, + validate_cache_lightweight, + ) + from sglang.srt.utils import find_local_repo_dir + # Skip if already a local path if os.path.isdir(model_name_or_path): return None @@ -440,15 +463,6 @@ def _try_enable_offline_mode_if_cache_complete( except Exception: return None - # Detect before the marker check so the current launch's requirements are known. - requires_hf_quant_config = False - for i, arg in enumerate(other_args): - if arg == "--quantization" and i + 1 < len(other_args): - quant_value = other_args[i + 1].lower() - if quant_value in ["modelopt_fp4", "modelopt_fp8", "modelopt"]: - requires_hf_quant_config = True - break - # Check per-run marker (fast hint - snapshot validated earlier in this run) per_run_marker = _read_per_run_marker(snapshot_dir) if per_run_marker is not None: @@ -466,7 +480,6 @@ def _try_enable_offline_mode_if_cache_complete( return None # Marker exists and current validation passed - env["HF_HUB_OFFLINE"] = "1" marker_path = _get_per_run_marker_path(snapshot_dir) print( f"CI_OFFLINE: Per-run marker found and current validation passed " @@ -487,9 +500,6 @@ def _try_enable_offline_mode_if_cache_complete( ) return None - # Validation passed - enable offline mode and write per-run marker - env["HF_HUB_OFFLINE"] = "1" - # Write per-run marker for subsequent tests in this run _write_per_run_marker(snapshot_dir, model_name_or_path) @@ -506,6 +516,94 @@ def _try_enable_offline_mode_if_cache_complete( return marker_path +@contextlib.contextmanager +def _hf_offline_if_cache_complete(engine_kwargs: dict): + """Run an in-process Engine startup with HF_HUB_OFFLINE=1 when every Hub + repo it loads validates in the local cache. + + A scheduler that fails at startup kills the test process, so there is no + online retry as in popen_launch_server; any doubt keeps the startup online. + """ + import huggingface_hub.constants + + if "server_args" in engine_kwargs: + server_args = engine_kwargs["server_args"] + model_path = server_args.model_path + tokenizer_path = server_args.tokenizer_path + draft_path = server_args.speculative_draft_model_path + quantization = server_args.quantization + uses_lora = server_args.enable_lora or bool(server_args.lora_paths) + else: + model_path = engine_kwargs.get("model_path") + tokenizer_path = engine_kwargs.get("tokenizer_path") + draft_path = engine_kwargs.get("speculative_draft_model_path") + quantization = engine_kwargs.get("quantization") + uses_lora = bool( + engine_kwargs.get("enable_lora") or engine_kwargs.get("lora_paths") + ) + + # Adapters can be loaded by repo id after startup, from an offline scheduler. + if uses_lora or os.environ.get("HF_HUB_OFFLINE") == "1": + yield + return + + requires_hf_quant_config = quantization in [ + "modelopt_fp4", + "modelopt_fp8", + "modelopt", + ] + repo_ids = { + path + for path in (model_path, tokenizer_path, draft_path) + if path is not None and not os.path.isdir(path) + } + if not repo_ids or not all( + _validate_cache_for_offline( + repo_id, requires_hf_quant_config and repo_id == model_path + ) + for repo_id in repo_ids + ): + yield + return + + # Spawned children read the env at import; this process already imported + # huggingface_hub, which reads it once into a module constant. + prev_env = os.environ.get("HF_HUB_OFFLINE") + prev_constant = huggingface_hub.constants.HF_HUB_OFFLINE + os.environ["HF_HUB_OFFLINE"] = "1" + huggingface_hub.constants.HF_HUB_OFFLINE = True + try: + yield + finally: + if prev_env is None: + os.environ.pop("HF_HUB_OFFLINE", None) + else: + os.environ["HF_HUB_OFFLINE"] = prev_env + huggingface_hub.constants.HF_HUB_OFFLINE = prev_constant + + +def _install_ci_engine_offline_hook(): + """In CI, wrap Engine.__init__ once per process with _hf_offline_if_cache_complete.""" + try: + from sglang.srt.entrypoints.engine import Engine + except ImportError as e: + # A test that never builds an Engine must not fail on this. + print(f"CI_OFFLINE: Engine import failed, skip the offline hook: {e}") + return + + orig_init = Engine.__init__ + if getattr(orig_init, "_ci_offline_wrapped", False): + return + + @wraps(orig_init) + def init(self, **kwargs): + with _hf_offline_if_cache_complete(kwargs): + orig_init(self, **kwargs) + + init._ci_offline_wrapped = True + Engine.__init__ = init + + def _create_clean_subprocess_env(env: dict) -> dict: """Create a clean subprocess environment without internal CI keys.""" child_env = env.copy() @@ -2301,6 +2399,8 @@ def __init_subclass__(cls, **kwargs): def safe_setUpClass(klass): try: _wait_for_gpu_idle_in_ci() + if is_in_ci(): + _install_ci_engine_offline_hook() orig_func(klass) except Exception: # Best-effort cleanup; suppress teardown errors so the diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 86930292c6e3..b66b50e10e2f 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -820,17 +820,6 @@ install_extra_deps() { mark_step_done "${FUNCNAME[0]}" } -install_test_tools() { - # Download kernels from kernels community - kernels download python || true - kernels lock python || true - [ -e "${HOME}/.cache/sglang" ] && [ ! -d "${HOME}/.cache/sglang" ] && rm -f "${HOME}/.cache/sglang" - mkdir -p "${HOME}/.cache/sglang/" - mv python/kernels.lock "${HOME}/.cache/sglang/" || true - - mark_step_done "${FUNCNAME[0]}" -} - prepare_runner() { bash "${SCRIPT_DIR}/prepare_runner.sh" @@ -950,7 +939,6 @@ main() { download_flashinfer_cache stabilize_flashinfer_jit_paths install_extra_deps - install_test_tools install_nccl prepare_runner setup_ld_library_path diff --git a/test/manual/test_weight_validation.py b/test/manual/test_weight_validation.py index c4d8bc1b5f60..c516174c96bf 100644 --- a/test/manual/test_weight_validation.py +++ b/test/manual/test_weight_validation.py @@ -104,6 +104,26 @@ def test_validate_sharded_model_all_present(self): self.assertIsNone(error_msg) self.assertEqual(corrupted_files, []) + def test_validate_sharded_model_suffixed_shard_prefix(self): + """Shards named `model.safetensors-0000N-of-0000M.safetensors` (Qwen3.5) + pair with `model.safetensors.index.json`.""" + with tempfile.TemporaryDirectory() as tmpdir: + names = [f"model.safetensors-0000{i}-of-00002.safetensors" for i in [1, 2]] + header = b'{"__metadata__":{}}' + for name in names: + with open(os.path.join(tmpdir, name), "wb") as f: + f.write(struct.pack("