Skip to content

Reject QDQ Gemm→QGemm fusion when alpha != 1 with bias #28131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tianleiwu merged 8 commits into from
Jun 3, 2026
Merged

Reject QDQ Gemm→QGemm fusion when alpha != 1 with bias #28131

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e8c90e9
Reject QDQ Gemm→QGemm fusion when alpha != 1 with bias (#28130)
elwhyjay Apr 19, 2026
f983383
Trim redundant no-bias alpha_not_one cases to reduce ASAN memory pres…
elwhyjay May 19, 2026
bf7bc2d
Merge branch 'microsoft:main' into fix/qdq-gemm-alpha-28130
elwhyjay May 19, 2026
f031046
Skip TestDequantizeLinearNoAxis variants under ASan and restore alpha…
elwhyjay May 21, 2026
c6a9c6c
Revert ASan guard
elwhyjay May 27, 2026
12b7ec6
Merge remote-tracking branch 'upstream/main' into fix/qdq-gemm-alpha-…
elwhyjay May 27, 2026
987cb8c
Reduce QDQ Gemm alpha test matrix
elwhyjay Jun 2, 2026
ea615ec
Limit QDQ Gemm alpha test shape variants
elwhyjay Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -835,6 +835,13 @@ bool GemmNodeGroupSelector::Check(const GraphViewer& graph_viewer, const Node& n
return true;
}

// When bias is present, QGemm folds bias into the int32 accumulator before
// applying the alpha*sa*sb output scale, which would incorrectly scale the
// bias by alpha. Require alpha==1 and beta==1 so the fused path is exact.
if (node.GetAttributes().at("alpha").f() != 1.0) {
return false;
}

if (node.GetAttributes().at("beta").f() != 1.0) { // beta needs to be 1.0
return false;
}
Expand Down
14 changes: 12 additions & 2 deletions onnxruntime/test/optimizer/qdq_transformer_fastmath_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,8 @@ TEST(QDQTransformerTests, MatMul_S8S8U8_DisableFastMath) {
}

template <typename Input1Type, typename Input2Type, typename OutputType, typename BiasType = int32_t>
void QDQTransformerGemmTests(bool has_output_q, bool has_bias, bool beta_not_one = false, bool disable_fastmath = false) {
void QDQTransformerGemmTests(bool has_output_q, bool has_bias, bool beta_not_one = false,
bool disable_fastmath = false, bool alpha_not_one = false) {
auto test_case = [&](const std::vector<int64_t>& input1_shape, const std::vector<int64_t>& input2_shape,
bool use_contrib_qdq = false) {
auto build_test_case = [&](ModelTestBuilder& builder) {
Expand Down Expand Up @@ -396,12 +397,17 @@ void QDQTransformerGemmTests(bool has_output_q, bool has_bias, bool beta_not_one
if (beta_not_one) {
gemm_node->AddAttribute("beta", 2.0f);
}

if (alpha_not_one) {
gemm_node->AddAttribute("alpha", 2.0f);
}
};

auto check_binary_op_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
const QDQOpKeys qdq_keys = GetQDQOpKeys(use_contrib_qdq);
if ((!has_output_q || std::is_same_v<Input1Type, OutputType>) && (!has_bias || (std::is_same_v<BiasType, int32_t> && !beta_not_one)) &&
if ((!has_output_q || std::is_same_v<Input1Type, OutputType>) &&
(!has_bias || (std::is_same_v<BiasType, int32_t> && !beta_not_one && !alpha_not_one)) &&
(std::is_same_v<Input1Type, uint8_t> || std::is_same_v<Input2Type, int8_t>)) {
EXPECT_EQ(op_to_count["com.microsoft.QGemm"], 1);
EXPECT_EQ(op_to_count["Gemm"], 0);
Expand Down Expand Up @@ -490,6 +496,10 @@ void QDQTransformerGemmTests() {
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(false, true, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, true, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(false, false, false, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(false, true, false, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, false, false, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, true, false, false, true);
// dummy test to disable the fastmath session
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, true, true, true);
}
Expand Down
14 changes: 12 additions & 2 deletions onnxruntime/test/optimizer/qdq_transformer_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,8 @@ TEST(QDQTransformerTests, MatMul_S8S8U8) {
}

template <typename Input1Type, typename Input2Type, typename OutputType, typename BiasType = int32_t>
void QDQTransformerGemmTests(bool has_output_q, bool has_bias, bool beta_not_one = false) {
void QDQTransformerGemmTests(bool has_output_q, bool has_bias, bool beta_not_one = false,
bool alpha_not_one = false) {
auto test_case = [&](const std::vector<int64_t>& input1_shape, const std::vector<int64_t>& input2_shape,
bool use_contrib_qdq = false) {
auto build_test_case = [&](ModelTestBuilder& builder) {
Expand Down Expand Up @@ -791,12 +792,17 @@ void QDQTransformerGemmTests(bool has_output_q, bool has_bias, bool beta_not_one
if (beta_not_one) {
gemm_node->AddAttribute("beta", 2.0f);
}

if (alpha_not_one) {
gemm_node->AddAttribute("alpha", 2.0f);
}
};

auto check_binary_op_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
const QDQOpKeys qdq_keys = GetQDQOpKeys(use_contrib_qdq);
if ((!has_output_q || std::is_same_v<Input1Type, OutputType>) && (!has_bias || (std::is_same_v<BiasType, int32_t> && !beta_not_one)) &&
if ((!has_output_q || std::is_same_v<Input1Type, OutputType>) &&
(!has_bias || (std::is_same_v<BiasType, int32_t> && !beta_not_one && !alpha_not_one)) &&
(std::is_same_v<Input1Type, uint8_t> || std::is_same_v<Input2Type, int8_t>)) {
EXPECT_EQ(op_to_count["com.microsoft.QGemm"], 1);
EXPECT_EQ(op_to_count["Gemm"], 0);
Expand Down Expand Up @@ -860,6 +866,10 @@ void QDQTransformerGemmTests() {
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(false, true, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, true, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(false, false, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(false, true, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, false, false, true);
QDQTransformerGemmTests<Input1Type, Input2Type, OutputType, BiasType>(true, true, false, true);
}

TEST(QDQTransformerTests, Gemm_U8U8U8) {
Expand Down
3 changes: 3 additions & 0 deletions onnxruntime/test/optimizer/transpose_optimizer_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3763,6 +3763,8 @@ TEST(TransposeOptimizerTests, TestDequantizeLinearNoAxis) {
std::optional<ONNX_NAMESPACE::AttributeProto> no_axis; // Empty axis value will not be set.

RunDequantizeLinearTestCase<uint8_t>(zp_input_shape, zp_value_shape, no_axis, kOnnxDomain);
// Skip remaining variants under ASan to stay under the 8 GB allocator limit.
#if !defined(__SANITIZE_ADDRESS__)
RunDequantizeLinearTestCase<uint16_t>(zp_input_shape, zp_value_shape, no_axis, kOnnxDomain);
RunDequantizeLinearTestCase<int16_t>(zp_input_shape, zp_value_shape, no_axis, kOnnxDomain);
#if !defined(DISABLE_CONTRIB_OPS)
Expand All @@ -3771,6 +3773,7 @@ TEST(TransposeOptimizerTests, TestDequantizeLinearNoAxis) {
RunDequantizeLinearTestCase<uint16_t>(zp_input_shape, zp_value_shape, no_axis, kMSDomain);
RunDequantizeLinearTestCase<int16_t>(zp_input_shape, zp_value_shape, no_axis, kMSDomain);
#endif
#endif
}

TEST(TransposeOptimizerTests, TestCast) {
Expand Down
Loading