From 894b6b2611d0b0879bbd3d9c8f6a9399e0081509 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 4 Dec 2021 19:10:02 +0300 Subject: [PATCH 1/4] Handle X % 2 == 0 on arm64 --- src/coreclr/jit/morph.cpp | 49 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index f8019ed88eec81..fe6f66bacf5b63 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -13371,9 +13371,58 @@ GenTree* Compiler::fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp) ssize_t modValue = op1op2->AsIntCon()->IconValue(); if (isPow2(modValue)) { + JITDUMP("\nTransforming:\n"); + DISPTREE(cmp); + op1->SetOper(GT_AND); // Change % => & op1op2->AsIntConCommon()->SetIconValue(modValue - 1); // Change c => c - 1 fgUpdateConstTreeValueNumber(op1op2); + + JITDUMP("\ninto:\n"); + DISPTREE(cmp); + } + } + } + } + + // Now let's do the same but for a more complicated input: + // (X - ((X / C1) << C2)) != 0 + // where C1 is a power-of-two number and 1 << C2 equals C3. + // It is especially important for ARM64 where "X % POT_CNS ==/!= 0" is transformed to this pattern early + if (fgGlobalMorph && op2->IsIntegralConst(0) && op1->OperIs(GT_SUB) && !op1->gtOverflow() && varTypeIsIntegral(op1)) + { + GenTree* lsh = op1->gtGetOp2(); + if (lsh->OperIs(GT_LSH) && lsh->gtGetOp2()->IsCnsIntOrI() && lsh->gtGetOp1()->OperIs(GT_DIV)) + { + GenTreeOp* div = lsh->gtGetOp1()->AsOp(); + const ssize_t c1 = lsh->gtGetOp2()->AsIntCon()->IconValue(); + if (div->OperIs(GT_DIV) && div->gtGetOp2()->IsCnsIntOrI()) + { + const ssize_t c2 = div->gtGetOp2()->AsIntCon()->IconValue(); + + // Same X is used twice, in the second case it can be a COMMA + if (op1->gtGetOp1()->OperIsLeaf() && + GenTree::Compare(op1->gtGetOp1(), div->gtGetOp1()->gtEffectiveVal(), false)) + { + if ((c2 > 1) && isPow2(c2) && ((1ULL << (size_t)c1) == (size_t)c2)) + { + JITDUMP("\nTransforming:\n"); + DISPTREE(cmp); + + DEBUG_DESTROY_NODE(lsh->gtGetOp2()); + DEBUG_DESTROY_NODE(op1->gtGetOp1()); + DEBUG_DESTROY_NODE(lsh); + DEBUG_DESTROY_NODE(op1); + + // Transform to "X & (C2 - 1) ==/!= 0" + div->SetOper(GT_AND); + div->gtGetOp2()->AsIntConCommon()->SetIconValue(c2 - 1); + fgUpdateConstTreeValueNumber(div->gtGetOp2()); + cmp->gtOp1 = op1 = div; + + JITDUMP("\ninto:\n"); + DISPTREE(cmp); + } } } } From bbccf1ef09909c65e9e60ad105a4c0d483375737 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 4 Dec 2021 19:38:13 +0300 Subject: [PATCH 2/4] remove fgUpdateConstTreeValueNumber --- src/coreclr/jit/morph.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index fe6f66bacf5b63..e00623288ed743 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -13417,7 +13417,6 @@ GenTree* Compiler::fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp) // Transform to "X & (C2 - 1) ==/!= 0" div->SetOper(GT_AND); div->gtGetOp2()->AsIntConCommon()->SetIconValue(c2 - 1); - fgUpdateConstTreeValueNumber(div->gtGetOp2()); cmp->gtOp1 = op1 = div; JITDUMP("\ninto:\n"); From 6e97a3f7bb7c51e0a7b1855103da7d004dc81cc0 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 4 Dec 2021 19:57:36 +0300 Subject: [PATCH 3/4] Move (a % c) ==/!= 0 transformation to pre-order --- src/coreclr/jit/morph.cpp | 102 +++++++++++--------------------------- 1 file changed, 30 insertions(+), 72 deletions(-) diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index e00623288ed743..9fff5ee1d2a471 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -11447,6 +11447,36 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac) { return fgMorphTree(optimizedTree); } + + // Pattern-matching optimization: + // (a % c) ==/!= 0 + // for power-of-2 constant `c` + // => + // a & (c - 1) ==/!= 0 + // For integer `a`, even if negative. + if (opts.OptimizationEnabled() && !optValnumCSE_phase) + { + assert(tree->OperIs(GT_EQ, GT_NE)); + if (op1->OperIs(GT_MOD) && varTypeIsIntegral(op1) && op2->IsIntegralConst(0)) + { + GenTree* op1op2 = op1->AsOp()->gtOp2; + if (op1op2->IsCnsIntOrI()) + { + ssize_t modValue = op1op2->AsIntCon()->IconValue(); + if (isPow2(modValue)) + { + JITDUMP("\nTransforming:\n"); + DISPTREE(tree); + + op1->SetOper(GT_AND); // Change % => & + op1op2->AsIntConCommon()->SetIconValue(modValue - 1); // Change c => c - 1 + + JITDUMP("\ninto:\n"); + DISPTREE(tree); + } + } + } + } } FALLTHROUGH; @@ -13355,78 +13385,6 @@ GenTree* Compiler::fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp) GenTree* op1 = cmp->gtGetOp1(); GenTreeIntConCommon* op2 = cmp->gtGetOp2()->AsIntConCommon(); - // Pattern-matching optimization: - // (a % c) ==/!= 0 - // for power-of-2 constant `c` - // => - // a & (c - 1) ==/!= 0 - // For integer `a`, even if negative. - if (opts.OptimizationEnabled()) - { - if (op1->OperIs(GT_MOD) && varTypeIsIntegral(op1) && op2->IsIntegralConst(0)) - { - GenTree* op1op2 = op1->AsOp()->gtOp2; - if (op1op2->IsCnsIntOrI()) - { - ssize_t modValue = op1op2->AsIntCon()->IconValue(); - if (isPow2(modValue)) - { - JITDUMP("\nTransforming:\n"); - DISPTREE(cmp); - - op1->SetOper(GT_AND); // Change % => & - op1op2->AsIntConCommon()->SetIconValue(modValue - 1); // Change c => c - 1 - fgUpdateConstTreeValueNumber(op1op2); - - JITDUMP("\ninto:\n"); - DISPTREE(cmp); - } - } - } - } - - // Now let's do the same but for a more complicated input: - // (X - ((X / C1) << C2)) != 0 - // where C1 is a power-of-two number and 1 << C2 equals C3. - // It is especially important for ARM64 where "X % POT_CNS ==/!= 0" is transformed to this pattern early - if (fgGlobalMorph && op2->IsIntegralConst(0) && op1->OperIs(GT_SUB) && !op1->gtOverflow() && varTypeIsIntegral(op1)) - { - GenTree* lsh = op1->gtGetOp2(); - if (lsh->OperIs(GT_LSH) && lsh->gtGetOp2()->IsCnsIntOrI() && lsh->gtGetOp1()->OperIs(GT_DIV)) - { - GenTreeOp* div = lsh->gtGetOp1()->AsOp(); - const ssize_t c1 = lsh->gtGetOp2()->AsIntCon()->IconValue(); - if (div->OperIs(GT_DIV) && div->gtGetOp2()->IsCnsIntOrI()) - { - const ssize_t c2 = div->gtGetOp2()->AsIntCon()->IconValue(); - - // Same X is used twice, in the second case it can be a COMMA - if (op1->gtGetOp1()->OperIsLeaf() && - GenTree::Compare(op1->gtGetOp1(), div->gtGetOp1()->gtEffectiveVal(), false)) - { - if ((c2 > 1) && isPow2(c2) && ((1ULL << (size_t)c1) == (size_t)c2)) - { - JITDUMP("\nTransforming:\n"); - DISPTREE(cmp); - - DEBUG_DESTROY_NODE(lsh->gtGetOp2()); - DEBUG_DESTROY_NODE(op1->gtGetOp1()); - DEBUG_DESTROY_NODE(lsh); - DEBUG_DESTROY_NODE(op1); - - // Transform to "X & (C2 - 1) ==/!= 0" - div->SetOper(GT_AND); - div->gtGetOp2()->AsIntConCommon()->SetIconValue(c2 - 1); - cmp->gtOp1 = op1 = div; - - JITDUMP("\ninto:\n"); - DISPTREE(cmp); - } - } - } - } - } - // Check for "(expr +/- icon1) ==/!= (non-zero-icon2)". if (op2->IsCnsIntOrI() && (op2->IconValue() != 0)) { From 1c301b56bcca7a965cdb6cd8e9e856591c298f4f Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 4 Dec 2021 20:31:18 +0300 Subject: [PATCH 4/4] Address feedback --- src/coreclr/jit/morph.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 9fff5ee1d2a471..b344b6d9ef9f09 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -11462,7 +11462,7 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac) GenTree* op1op2 = op1->AsOp()->gtOp2; if (op1op2->IsCnsIntOrI()) { - ssize_t modValue = op1op2->AsIntCon()->IconValue(); + const ssize_t modValue = op1op2->AsIntCon()->IconValue(); if (isPow2(modValue)) { JITDUMP("\nTransforming:\n"); @@ -11470,6 +11470,7 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac) op1->SetOper(GT_AND); // Change % => & op1op2->AsIntConCommon()->SetIconValue(modValue - 1); // Change c => c - 1 + fgUpdateConstTreeValueNumber(op1op2); JITDUMP("\ninto:\n"); DISPTREE(tree);