From a8061ea5143c0486886c1fa79e816580a692a48e Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Thu, 16 Apr 2026 14:01:33 -0700 Subject: [PATCH 1/5] Add quant mode for qwen3.5 --- src/python/py/models/builder.py | 6 +++++ src/python/py/models/builders/qwen.py | 35 +++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index 0cc75acdd3..9c3c90fe8c 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -71,6 +71,12 @@ def check_extra_options(kv_pairs, execution_provider): "disable_qkv_fusion", "prune_lm_head", ] + + # Validate quant_mode if provided + if "quant_mode" in kv_pairs: + valid_modes = {"default", "hybrid", "int4"} + if kv_pairs["quant_mode"] not in valid_modes: + raise ValueError(f"quant_mode must be one of {valid_modes}, got '{kv_pairs['quant_mode']}'") for key in bools: if key in kv_pairs: if kv_pairs[key] in {"false", "False", "0"}: diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 0e228bb000..210854cb92 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -1019,15 +1019,34 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # # Linear attention recurrence accumulates errors across the full sequence, # unlike softmax attention which normalizes per-step. + # + # Control via extra_options quant_mode or QWEN35_QUANT_MODE env var: + # "default" - INT8 for all linear attn + MLP layers (original, most accurate) + # "hybrid" - INT8 for linear attn projections only, INT4 for MLPs (balanced) + # "int4" - INT4 for everything (fastest, may degrade quality) + import os + quant_mode = extra_options.get("quant_mode", "") or os.environ.get("QWEN35_QUANT_MODE", "") + if not quant_mode: + quant_mode = "default" + int8_nodes = {} - for i, lt in enumerate(self.layer_types): - if lt == "linear_attention": - # All linear attention projections: INT8 - for proj in ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj"): - int8_nodes[f"/model/layers.{i}/linear_attn/{proj}/MatMul"] = {"bits": 8} - # MLP projections in linear attention layers: INT8 - for proj in ("gate_proj", "up_proj", "down_proj"): - int8_nodes[f"/model/layers.{i}/mlp/{proj}/MatMul"] = {"bits": 8} + if quant_mode == "default": + for i, lt in enumerate(self.layer_types): + if lt == "linear_attention": + # All linear attention projections: INT8 + for proj in ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj"): + int8_nodes[f"/model/layers.{i}/linear_attn/{proj}/MatMul"] = {"bits": 8} + # MLP projections in linear attention layers: INT8 + for proj in ("gate_proj", "up_proj", "down_proj"): + int8_nodes[f"/model/layers.{i}/mlp/{proj}/MatMul"] = {"bits": 8} + elif quant_mode == "hybrid": + for i, lt in enumerate(self.layer_types): + if lt == "linear_attention": + # Only linear attention projections: INT8 (recurrence-sensitive) + for proj in ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj"): + int8_nodes[f"/model/layers.{i}/linear_attn/{proj}/MatMul"] = {"bits": 8} + # MLP layers stay INT4 (feedforward, no error accumulation) + # quant_mode == "int4": no overrides, everything INT4 if int8_nodes: algo_config = self.quant_attrs["int4"].get("algo_config") From eed8c2f85f48a6b672f0e54338552060e1b33e2d Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Thu, 23 Apr 2026 01:38:16 +0000 Subject: [PATCH 2/5] Address comments --- src/python/py/models/builder.py | 18 +++++++++++++++--- src/python/py/models/builders/qwen.py | 10 +++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index 9c3c90fe8c..65738f5f3d 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -72,11 +72,23 @@ def check_extra_options(kv_pairs, execution_provider): "prune_lm_head", ] - # Validate quant_mode if provided + # Validate quant_mode if provided. Empty or whitespace-only values are + # treated as unset so downstream builders can apply their default behavior. if "quant_mode" in kv_pairs: valid_modes = {"default", "hybrid", "int4"} - if kv_pairs["quant_mode"] not in valid_modes: - raise ValueError(f"quant_mode must be one of {valid_modes}, got '{kv_pairs['quant_mode']}'") + quant_mode = kv_pairs["quant_mode"] + if isinstance(quant_mode, str): + quant_mode = quant_mode.strip() + + if not quant_mode: + del kv_pairs["quant_mode"] + elif quant_mode not in valid_modes: + valid_modes_display = ", ".join(sorted(valid_modes)) + raise ValueError( + f"quant_mode must be one of {valid_modes_display}, got '{kv_pairs['quant_mode']}'" + ) + else: + kv_pairs["quant_mode"] = quant_mode for key in bools: if key in kv_pairs: if kv_pairs[key] in {"false", "False", "0"}: diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 210854cb92..cfcac2db05 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -1020,14 +1020,14 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # Linear attention recurrence accumulates errors across the full sequence, # unlike softmax attention which normalizes per-step. # - # Control via extra_options quant_mode or QWEN35_QUANT_MODE env var: + # Control via extra_options quant_mode: # "default" - INT8 for all linear attn + MLP layers (original, most accurate) # "hybrid" - INT8 for linear attn projections only, INT4 for MLPs (balanced) # "int4" - INT4 for everything (fastest, may degrade quality) - import os - quant_mode = extra_options.get("quant_mode", "") or os.environ.get("QWEN35_QUANT_MODE", "") - if not quant_mode: - quant_mode = "default" + quant_mode = extra_options.get("quant_mode", "").strip().lower() or "default" + valid_modes = {"default", "hybrid", "int4"} + if quant_mode not in valid_modes: + raise ValueError(f"quant_mode must be one of {', '.join(sorted(valid_modes))}, got '{quant_mode}'") int8_nodes = {} if quant_mode == "default": From 20d0786c32d45ec23a9366de0c90e00e7f602b85 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Thu, 23 Apr 2026 01:43:38 +0000 Subject: [PATCH 3/5] Clean up the code --- src/python/py/models/builders/qwen.py | 29 ++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index cfcac2db05..3d9d36b2aa 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -1021,32 +1021,25 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # unlike softmax attention which normalizes per-step. # # Control via extra_options quant_mode: - # "default" - INT8 for all linear attn + MLP layers (original, most accurate) + # "default" - INT8 for all linear attn + MLP layers (most accurate) # "hybrid" - INT8 for linear attn projections only, INT4 for MLPs (balanced) # "int4" - INT4 for everything (fastest, may degrade quality) quant_mode = extra_options.get("quant_mode", "").strip().lower() or "default" - valid_modes = {"default", "hybrid", "int4"} - if quant_mode not in valid_modes: - raise ValueError(f"quant_mode must be one of {', '.join(sorted(valid_modes))}, got '{quant_mode}'") + if quant_mode not in ("default", "hybrid", "int4"): + raise ValueError(f"quant_mode must be one of default, hybrid, int4, got '{quant_mode}'") + + linear_attn_projs = ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj") + mlp_projs = ("gate_proj", "up_proj", "down_proj") int8_nodes = {} - if quant_mode == "default": - for i, lt in enumerate(self.layer_types): - if lt == "linear_attention": - # All linear attention projections: INT8 - for proj in ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj"): - int8_nodes[f"/model/layers.{i}/linear_attn/{proj}/MatMul"] = {"bits": 8} - # MLP projections in linear attention layers: INT8 - for proj in ("gate_proj", "up_proj", "down_proj"): - int8_nodes[f"/model/layers.{i}/mlp/{proj}/MatMul"] = {"bits": 8} - elif quant_mode == "hybrid": + if quant_mode in ("default", "hybrid"): for i, lt in enumerate(self.layer_types): if lt == "linear_attention": - # Only linear attention projections: INT8 (recurrence-sensitive) - for proj in ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj"): + for proj in linear_attn_projs: int8_nodes[f"/model/layers.{i}/linear_attn/{proj}/MatMul"] = {"bits": 8} - # MLP layers stay INT4 (feedforward, no error accumulation) - # quant_mode == "int4": no overrides, everything INT4 + if quant_mode == "default": + for proj in mlp_projs: + int8_nodes[f"/model/layers.{i}/mlp/{proj}/MatMul"] = {"bits": 8} if int8_nodes: algo_config = self.quant_attrs["int4"].get("algo_config") From 461615dc3e4d93365061352d588609952c786e82 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Thu, 23 Apr 2026 04:04:43 +0000 Subject: [PATCH 4/5] fix comments --- src/python/py/models/builder.py | 10 ++++++---- src/python/py/models/builders/qwen.py | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index 65738f5f3d..de2745137a 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -51,6 +51,9 @@ ) +VALID_QUANT_MODES = frozenset({"default", "hybrid", "int4"}) + + def check_extra_options(kv_pairs, execution_provider): """ Check key-value pairs and set values correctly @@ -75,15 +78,14 @@ def check_extra_options(kv_pairs, execution_provider): # Validate quant_mode if provided. Empty or whitespace-only values are # treated as unset so downstream builders can apply their default behavior. if "quant_mode" in kv_pairs: - valid_modes = {"default", "hybrid", "int4"} quant_mode = kv_pairs["quant_mode"] if isinstance(quant_mode, str): - quant_mode = quant_mode.strip() + quant_mode = quant_mode.strip().lower() if not quant_mode: del kv_pairs["quant_mode"] - elif quant_mode not in valid_modes: - valid_modes_display = ", ".join(sorted(valid_modes)) + elif quant_mode not in VALID_QUANT_MODES: + valid_modes_display = ", ".join(sorted(VALID_QUANT_MODES)) raise ValueError( f"quant_mode must be one of {valid_modes_display}, got '{kv_pairs['quant_mode']}'" ) diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 3d9d36b2aa..5831adc9e7 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -15,6 +15,8 @@ Qwen3VLForConditionalGeneration, ) +from builder import VALID_QUANT_MODES + from .base import Model @@ -1025,8 +1027,8 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # "hybrid" - INT8 for linear attn projections only, INT4 for MLPs (balanced) # "int4" - INT4 for everything (fastest, may degrade quality) quant_mode = extra_options.get("quant_mode", "").strip().lower() or "default" - if quant_mode not in ("default", "hybrid", "int4"): - raise ValueError(f"quant_mode must be one of default, hybrid, int4, got '{quant_mode}'") + if quant_mode not in VALID_QUANT_MODES: + raise ValueError(f"quant_mode must be one of {', '.join(sorted(VALID_QUANT_MODES))}, got '{quant_mode}'") linear_attn_projs = ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj") mlp_projs = ("gate_proj", "up_proj", "down_proj") From ed7d2ead19aebdfee04b6e5744318af14c253037 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Thu, 23 Apr 2026 04:36:31 +0000 Subject: [PATCH 5/5] fix pipeline, remove circular dependency --- src/python/py/models/builders/qwen.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 5831adc9e7..3d9d36b2aa 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -15,8 +15,6 @@ Qwen3VLForConditionalGeneration, ) -from builder import VALID_QUANT_MODES - from .base import Model @@ -1027,8 +1025,8 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # "hybrid" - INT8 for linear attn projections only, INT4 for MLPs (balanced) # "int4" - INT4 for everything (fastest, may degrade quality) quant_mode = extra_options.get("quant_mode", "").strip().lower() or "default" - if quant_mode not in VALID_QUANT_MODES: - raise ValueError(f"quant_mode must be one of {', '.join(sorted(VALID_QUANT_MODES))}, got '{quant_mode}'") + if quant_mode not in ("default", "hybrid", "int4"): + raise ValueError(f"quant_mode must be one of default, hybrid, int4, got '{quant_mode}'") linear_attn_projs = ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj") mlp_projs = ("gate_proj", "up_proj", "down_proj")