From 98c65a3928662d02d45bcc4f9425c5a754469653 Mon Sep 17 00:00:00 2001 From: Injae Ryou Date: Thu, 9 Apr 2026 23:56:00 +0900 Subject: [PATCH 1/2] [GGUF] Support non-standard quant types with prefix (e.g. UD-Q4_K_XL) Signed-off-by: Injae Ryou --- vllm/transformers_utils/gguf_utils.py | 41 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/vllm/transformers_utils/gguf_utils.py b/vllm/transformers_utils/gguf_utils.py index 3faa5ee60e9f..7708378ee13b 100644 --- a/vllm/transformers_utils/gguf_utils.py +++ b/vllm/transformers_utils/gguf_utils.py @@ -40,15 +40,48 @@ def check_gguf_file(model: str | PathLike) -> bool: @cache def is_remote_gguf(model: str | Path) -> bool: - """Check if the model is a remote GGUF model.""" + """Check if the model is a remote GGUF model. + + Recognizes two forms: + 1. Standard: ``repo_id:quant_type`` where *quant_type* is a known + GGML quantization type (e.g. ``Q4_K_M``). + 2. Non-standard: ``repo_id:quant_type`` where *quant_type* contains + a known GGML type with extra prefixes (e.g. ``UD-Q4_K_XL``). + A warning is logged and actual file existence is validated later + during download. + """ pattern = r"^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*:[A-Za-z0-9_+-]+$" model = str(model) if re.fullmatch(pattern, model): _, quant_type = model.rsplit(":", 1) - return is_valid_gguf_quant_type(quant_type) + if is_valid_gguf_quant_type(quant_type): + return True + if is_nonstandard_gguf_quant_type(quant_type): + logger.warning( + "Non-standard GGUF quant type '%s' detected.", + quant_type, + ) + return True return False +def is_nonstandard_gguf_quant_type(quant_type: str) -> bool: + """Check if a non-standard quant type contains a known GGML type. + + Splits the quant type by the last ``-`` and checks whether the + trailing part is a standard GGML type. For example:: + + UD-Q4_K_XL → rsplit → ["UD", "Q4_K_XL"] → Q4_K_XL valid ✓ + UD-IQ4_NL → rsplit → ["UD", "IQ4_NL"] → IQ4_NL valid ✓ + Custom-UD-Q4_K → rsplit → ["Custom-UD", "Q4_K"] → Q4_K valid ✓ + RANDOM → no "-" → False + """ + if "-" not in quant_type: + return False + _, remainder = quant_type.rsplit("-", 1) + return is_valid_gguf_quant_type(remainder) + + # Common suffixes used in GGUF file naming conventions # e.g., Q4_K_M, Q3_K_S, Q5_K_L, Q2_K_XL _GGUF_QUANT_SUFFIXES = ("_M", "_S", "_L", "_XL", "_XS", "_XXS") @@ -84,7 +117,9 @@ def split_remote_gguf(model: str | Path) -> tuple[str, str]: f"Wrong GGUF model or invalid GGUF quant type: {model}.\n" "- It should be in repo_id:quant_type format.\n" f"- Valid base quant types: {GGMLQuantizationType._member_names_}\n" - f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}", + f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}\n" + "- Non-standard GGUF quant types also supported: " + "dash-separated prefixes (e.g. UD-Q4_K_XL, Custom-Q8_0)", ) From eed38429d34a81003e56490fea8f910fd8e85c62 Mon Sep 17 00:00:00 2001 From: Injae Ryou Date: Thu, 9 Apr 2026 23:58:00 +0900 Subject: [PATCH 2/2] [GGUF] Add tests for non-standard quant type validation Signed-off-by: Injae Ryou --- tests/transformers_utils/test_utils.py | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/transformers_utils/test_utils.py b/tests/transformers_utils/test_utils.py index 485c2efff77f..94dd014c929f 100644 --- a/tests/transformers_utils/test_utils.py +++ b/tests/transformers_utils/test_utils.py @@ -81,6 +81,25 @@ def test_is_remote_gguf_extended_quant_types(self): assert not is_remote_gguf("repo/model:INVALID_M") assert not is_remote_gguf("repo/model:Q9_K_M") + def test_is_remote_gguf_nonstandard_quant_type(self): + """Test is_remote_gguf with non-standard quant types containing + a known GGML type.""" + # Non-standard quant types with known GGML type after prefix + assert is_remote_gguf("unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL") + assert is_remote_gguf("user/Model:UD-Q4_K_M") + assert is_remote_gguf("user/SomeModel:Custom-Q8_0") + + # Exact GGML type after prefix (no suffix stripping needed) + assert is_remote_gguf("user/Model-GGUF:UD-IQ4_NL") + assert is_remote_gguf("user/Model-GGUF:UD-Q8_0") + + # Completely unknown quant types should still fail + assert not is_remote_gguf("repo/model:TOTALLY-RANDOM") + assert not is_remote_gguf("user/Model:UD-INVALID") + + # No dash separator → not recognized as prefixed + assert not is_remote_gguf("repo/model:UDIQ4NL") + def test_is_remote_gguf_without_colon(self): """Test is_remote_gguf without colon.""" assert not is_remote_gguf("repo/model") @@ -143,6 +162,14 @@ def test_split_remote_gguf_extended_quant_types(self): assert repo_id == "repo/model" assert quant_type == "Q3_K_S" + def test_split_remote_gguf_nonstandard_quant_type(self): + """Test split_remote_gguf with non-standard quant types in GGUF repos.""" + repo_id, quant_type = split_remote_gguf( + "unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL" + ) + assert repo_id == "unsloth/Qwen3.5-35B-A3B-GGUF" + assert quant_type == "UD-Q4_K_XL" + def test_split_remote_gguf_with_path_object(self): """Test split_remote_gguf with Path object.""" repo_id, quant_type = split_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))