From e2fab0b55a5dd56e91a44e9576c783f221fe19bb Mon Sep 17 00:00:00 2001 From: Jianhui Dai Date: Wed, 22 Apr 2026 09:35:06 +0800 Subject: [PATCH 1/2] Make linear attention INT8 quantization opt-in via extra_options Change Qwen3.5 mixed-precision quantization for linear attention layers from always-on to opt-in. Users can enable INT8 promotion of linear attention nodes by passing linear_attention_int8=true in extra_options. --- src/python/py/models/builders/qwen.py | 39 +++++++++++++-------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 0e228bb000..333cb39585 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -1013,29 +1013,28 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # Disable fused RoPE in attention op - we apply mRoPE manually self.attention_attrs["use_rope_in_attn"] = False - # Mixed-precision quantization for linear attention layers. - # Baseline: whole model INT4. Override linear attention layer nodes - # to INT8 for better accuracy with modest size increase. + # Optional mixed-precision quantization for linear attention layers. + # When enabled via extra_options linear_attention_int8=true, promotes + # linear attention layer nodes from INT4 to INT8 for better accuracy. # # Linear attention recurrence accumulates errors across the full sequence, # unlike softmax attention which normalizes per-step. - int8_nodes = {} - for i, lt in enumerate(self.layer_types): - if lt == "linear_attention": - # All linear attention projections: INT8 - for proj in ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj"): - int8_nodes[f"/model/layers.{i}/linear_attn/{proj}/MatMul"] = {"bits": 8} - # MLP projections in linear attention layers: INT8 - for proj in ("gate_proj", "up_proj", "down_proj"): - int8_nodes[f"/model/layers.{i}/mlp/{proj}/MatMul"] = {"bits": 8} - - if int8_nodes: - algo_config = self.quant_attrs["int4"].get("algo_config") - if algo_config is not None and hasattr(algo_config, "customized_weight_config"): - algo_config.customized_weight_config.update(int8_nodes) - else: - algo_config = RTNWeightOnlyQuantConfig(customized_weight_config=int8_nodes) - self.quant_attrs["int4"]["algo_config"] = algo_config + if extra_options.get("linear_attention_int8", "false").lower() == "true": + int8_nodes = {} + for i, lt in enumerate(self.layer_types): + if lt == "linear_attention": + for proj in ("in_proj_a", "in_proj_b", "in_proj_qkv", "in_proj_z", "out_proj"): + int8_nodes[f"/model/layers.{i}/linear_attn/{proj}/MatMul"] = {"bits": 8} + for proj in ("gate_proj", "up_proj", "down_proj"): + int8_nodes[f"/model/layers.{i}/mlp/{proj}/MatMul"] = {"bits": 8} + + if int8_nodes: + algo_config = self.quant_attrs["int4"].get("algo_config") + if algo_config is not None and hasattr(algo_config, "customized_weight_config"): + algo_config.customized_weight_config.update(int8_nodes) + else: + algo_config = RTNWeightOnlyQuantConfig(customized_weight_config=int8_nodes) + self.quant_attrs["int4"]["algo_config"] = algo_config # Replace standard KV cache I/O with hybrid cache I/O self._setup_hybrid_cache_io() From 4bb4842f743239fc6f12aecf3673badc5e51a2d1 Mon Sep 17 00:00:00 2001 From: Jianhui Dai Date: Wed, 22 Apr 2026 15:34:52 +0800 Subject: [PATCH 2/2] Update src/python/py/models/builders/qwen.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/python/py/models/builders/qwen.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 333cb39585..f63160dc9f 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -1019,7 +1019,16 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # # Linear attention recurrence accumulates errors across the full sequence, # unlike softmax attention which normalizes per-step. - if extra_options.get("linear_attention_int8", "false").lower() == "true": + linear_attention_int8 = extra_options.get("linear_attention_int8", False) + if isinstance(linear_attention_int8, bool): + enable_linear_attention_int8 = linear_attention_int8 + elif isinstance(linear_attention_int8, str): + linear_attention_int8 = linear_attention_int8.lower() + enable_linear_attention_int8 = linear_attention_int8 in {"true", "1"} + else: + enable_linear_attention_int8 = False + + if enable_linear_attention_int8: int8_nodes = {} for i, lt in enumerate(self.layer_types): if lt == "linear_attention":