diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index bfc43527dcfff7..36e35ba3ecc190 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6696,15 +6696,6 @@ class Compiler GenTree* fgMorphHWIntrinsicOptional(GenTreeHWIntrinsic* tree); GenTree* fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node); GenTree* fgOptimizeHWIntrinsicAssociative(GenTreeHWIntrinsic* node); -#if defined(FEATURE_MASKED_HW_INTRINSICS) - GenTreeHWIntrinsic* fgOptimizeForMaskedIntrinsic(GenTreeHWIntrinsic* node); -#endif // FEATURE_MASKED_HW_INTRINSICS -#ifdef TARGET_ARM64 - bool canMorphVectorOperandToMask(GenTree* node); - bool canMorphAllVectorOperandsToMasks(GenTreeHWIntrinsic* node); - GenTree* doMorphVectorOperandToMask(GenTree* node, GenTreeHWIntrinsic* parent); - GenTreeHWIntrinsic* fgMorphTryUseAllMaskVariant(GenTreeHWIntrinsic* node); -#endif // TARGET_ARM64 #endif // FEATURE_HW_INTRINSICS GenTree* fgOptimizeCommutativeArithmetic(GenTreeOp* tree); GenTree* fgOptimizeRelationalComparisonWithCasts(GenTreeOp* cmp); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 988c146742b9e1..3bde090b88d7a5 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -32112,10 +32112,8 @@ bool GenTree::CanDivOrModPossiblyOverflow(Compiler* comp) const #if defined(FEATURE_HW_INTRINSICS) GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) { - if (!opts.Tier0OptimizationEnabled()) - { - return tree; - } + assert(!optValnumCSE_phase); + assert(opts.Tier0OptimizationEnabled()); NamedIntrinsic ni = tree->GetHWIntrinsicId(); var_types retType = tree->TypeGet(); @@ -32254,6 +32252,126 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) // We shouldn't find AND_NOT nodes since it should only be produced in lowering assert(oper != GT_AND_NOT); +#if defined(FEATURE_MASKED_HW_INTRINSICS) && defined(TARGET_XARCH) + if (GenTreeHWIntrinsic::OperIsBitwiseHWIntrinsic(oper)) + { + // Comparisons that produce masks lead to more verbose trees than + // necessary in many scenarios due to requiring a CvtMaskToVector + // node to be inserted over them and this can block various opts + // that are dependent on tree height and similar. So we want to + // fold the unnecessary back and forth conversions away where possible. + + genTreeOps effectiveOper = oper; + GenTree* actualOp2 = op2; + + if (oper == GT_NOT) + { + assert(op2 == nullptr); + op2 = op1; + } + + // We need both operands to be ConvertMaskToVector in + // order to optimize this to a direct mask operation + + if (op1->OperIsConvertMaskToVector()) + { + if (!op2->OperIsHWIntrinsic()) + { + if ((oper == GT_XOR) && op2->IsVectorAllBitsSet()) + { + // We want to explicitly recognize op1 ^ AllBitsSet as + // some platforms don't have direct support for ~op1 + + effectiveOper = GT_NOT; + op2 = op1; + } + } + + if (op2->OperIsConvertMaskToVector()) + { + GenTreeHWIntrinsic* cvtOp1 = op1->AsHWIntrinsic(); + GenTreeHWIntrinsic* cvtOp2 = op2->AsHWIntrinsic(); + + unsigned simdBaseTypeSize = genTypeSize(simdBaseType); + + if ((genTypeSize(cvtOp1->GetSimdBaseType()) == simdBaseTypeSize) && + (genTypeSize(cvtOp2->GetSimdBaseType()) == simdBaseTypeSize)) + { + // We need both operands to be the same kind of mask; otherwise + // the bitwise operation can differ in how it performs + + NamedIntrinsic maskIntrinsicId = NI_Illegal; + + switch (effectiveOper) + { + case GT_AND: + { + maskIntrinsicId = NI_AVX512_AndMask; + break; + } + + case GT_NOT: + { + maskIntrinsicId = NI_AVX512_NotMask; + break; + } + + case GT_OR: + { + maskIntrinsicId = NI_AVX512_OrMask; + break; + } + + case GT_XOR: + { + maskIntrinsicId = NI_AVX512_XorMask; + break; + } + + default: + { + unreached(); + } + } + + assert(maskIntrinsicId != NI_Illegal); + + if (effectiveOper == oper) + { + tree->ChangeHWIntrinsicId(maskIntrinsicId); + tree->Op(1) = cvtOp1->Op(1); + } + else + { + assert(effectiveOper == GT_NOT); + tree->ResetHWIntrinsicId(maskIntrinsicId, this, cvtOp1->Op(1)); + tree->gtFlags &= ~GTF_REVERSE_OPS; + } + + tree->gtType = TYP_MASK; + DEBUG_DESTROY_NODE(op1); + + if (effectiveOper != GT_NOT) + { + tree->Op(2) = cvtOp2->Op(1); + } + + if (actualOp2 != nullptr) + { + DEBUG_DESTROY_NODE(actualOp2); + } + tree->SetMorphed(this); + + tree = gtNewSimdCvtMaskToVectorNode(retType, tree, simdBaseJitType, simdSize)->AsHWIntrinsic(); + tree->SetMorphed(this); + + return tree; + } + } + } + } +#endif // FEATURE_MASKED_HW_INTRINSICS && TARGET_XARCH + switch (ni) { // There's certain IR simplifications that are possible and which @@ -32830,10 +32948,28 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) oper = GT_NONE; } + // For mask nodes in particular, the foldings below are done under the presumption + // that we only produce something like `AddMask(op1, op2)` if op1 and op2 are compatible + // masks. On xarch, for example, this means that it'd be adding 8, 16, 32, or 64-bits + // together with the same size. We wouldn't ever encounter something like an 8 and 16 bit + // masks being added. This ensures that we don't end up with a case where folding would + // cause a different result to be produced, such as because the remaining upper bits are + // no longer zeroed. + switch (oper) { case GT_ADD: { + if (varTypeIsMask(retType)) + { + // Handle `x + 0 == x` and `0 + x == x` + if (cnsNode->IsMaskZero()) + { + resultNode = otherNode; + } + break; + } + if (varTypeIsFloating(simdBaseType)) { // Handle `x + NaN == NaN` and `NaN + x == NaN` @@ -32867,6 +33003,23 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) case GT_AND: { + if (varTypeIsMask(retType)) + { + // Handle `x & 0 == 0` and `0 & x == 0` + if (cnsNode->IsMaskZero()) + { + resultNode = otherNode; + break; + } + + // Handle `x & AllBitsSet == x` and `AllBitsSet & x == x` + if (cnsNode->IsMaskAllBitsSet()) + { + resultNode = otherNode; + } + break; + } + // Handle `x & 0 == 0` and `0 & x == 0` if (cnsNode->IsVectorZero()) { @@ -33100,6 +33253,23 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) case GT_OR: { + if (varTypeIsMask(retType)) + { + // Handle `x | 0 == x` and `0 | x == x` + if (cnsNode->IsMaskZero()) + { + resultNode = otherNode; + break; + } + + // Handle `x | AllBitsSet == AllBitsSet` and `AllBitsSet | x == AllBitsSet` + if (cnsNode->IsMaskAllBitsSet()) + { + resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); + } + break; + } + // Handle `x | 0 == x` and `0 | x == x` if (cnsNode->IsVectorZero()) { @@ -33127,6 +33297,27 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) // Handle `x >> 0 == x` and `0 >> x == 0` // Handle `x >>> 0 == x` and `0 >>> x == 0` + if (varTypeIsMask(retType)) + { + if (cnsNode->IsMaskZero()) + { + if (cnsNode == op2) + { + resultNode = otherNode; + } + else + { + resultNode = gtWrapWithSideEffects(cnsNode, otherNode, GTF_ALL_EFFECT); + } + } + else if (cnsNode->IsIntegralConst(0)) + { + assert(cnsNode == op2); + resultNode = otherNode; + } + break; + } + if (cnsNode->IsVectorZero()) { if (cnsNode == op2) @@ -33172,7 +33363,17 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) case GT_XOR: { - // Handle `x | 0 == x` and `0 | x == x` + if (varTypeIsMask(retType)) + { + // Handle `x ^ 0 == x` and `0 ^ x == x` + if (cnsNode->IsMaskZero()) + { + resultNode = otherNode; + } + break; + } + + // Handle `x ^ 0 == x` and `0 ^ x == x` if (cnsNode->IsVectorZero()) { resultNode = otherNode; @@ -33341,7 +33542,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) } else { - assert(!op1->IsTrueMask(simdBaseType) && !op1->IsFalseMask()); + assert(!op1->IsTrueMask(simdBaseType) && !op1->IsMaskZero()); } #endif @@ -33359,7 +33560,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) return op2; } - if (op1->IsVectorZero() || op1->IsFalseMask()) + if (op1->IsVectorZero() || op1->IsMaskZero()) { return gtWrapWithSideEffects(op3, op2, GTF_ALL_EFFECT); } diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 982dd41c611ec3..921a34fa057fbb 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -1803,8 +1803,9 @@ struct GenTree inline bool IsVectorCreate() const; inline bool IsVectorAllBitsSet() const; inline bool IsVectorBroadcast(var_types simdBaseType) const; + inline bool IsMaskZero() const; + inline bool IsMaskAllBitsSet() const; inline bool IsTrueMask(var_types simdBaseType) const; - inline bool IsFalseMask() const; inline uint64_t GetIntegralVectorConstElement(size_t index, var_types simdBaseType); @@ -9629,6 +9630,42 @@ inline bool GenTree::IsVectorBroadcast(var_types simdBaseType) const return false; } +//------------------------------------------------------------------- +// IsMaskZero: returns true if this node is a mask constant with all bits zero. +// +// Returns: +// True if this node is a mask constant with all bits zero +// +inline bool GenTree::IsMaskZero() const +{ +#if defined(FEATURE_MASKED_HW_INTRINSICS) + if (IsCnsMsk()) + { + return AsMskCon()->IsZero(); + } +#endif // FEATURE_MASKED_HW_INTRINSICS + + return false; +} + +//------------------------------------------------------------------- +// IsMaskAllBitsSet: returns true if this node is a mask constant with all bits set. +// +// Returns: +// True if this node is a mask constant with all bits set +// +inline bool GenTree::IsMaskAllBitsSet() const +{ +#if defined(FEATURE_MASKED_HW_INTRINSICS) + if (IsCnsMsk()) + { + return AsMskCon()->IsAllBitsSet(); + } +#endif // FEATURE_MASKED_HW_INTRINSICS + + return false; +} + //------------------------------------------------------------------------ // IsTrueMask: Is the given node a true mask // @@ -9655,23 +9692,6 @@ inline bool GenTree::IsTrueMask(var_types simdBaseType) const return false; } -//------------------------------------------------------------------------ -// IsFalseMask: Is the given node a false mask -// -// Returns true if the node is a false mask, ie all zeros -// -inline bool GenTree::IsFalseMask() const -{ -#ifdef TARGET_ARM64 - if (IsCnsMsk()) - { - return AsMskCon()->IsZero(); - } -#endif - - return false; -} - //------------------------------------------------------------------- // GetIntegralVectorConstElement: Gets the value of a given element in an integral vector constant // diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 1d69b329e760bd..c949410a3fdf9f 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -3942,7 +3942,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) GenTree* op3 = intrin.op3; // Handle op1 - if (op1->IsFalseMask()) + if (op1->IsMaskZero()) { // When we are merging with zero, we can specialize // and avoid instantiating the vector constant. diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index fe0cc627eed7f2..7030985785d605 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -10058,300 +10058,9 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node) } } -#if defined(FEATURE_MASKED_HW_INTRINSICS) - GenTreeHWIntrinsic* maskedIntrinsic = fgOptimizeForMaskedIntrinsic(node); - - if (maskedIntrinsic != nullptr) - { - node = maskedIntrinsic; - node->SetMorphed(this); - } -#endif // FEATURE_MASKED_HW_INTRINSICS - return node; } -#if defined(FEATURE_MASKED_HW_INTRINSICS) -//------------------------------------------------------------------------ -// fgOptimizeForMaskedIntrinsic: Tries to recognize intrinsics that are operating -// on mask types and morphs the tree to use intrinsics -// better suited to this. -// -// Arguments: -// node - the hardware intrinsic tree to try and optimize. -// This tree will be mutated if it is possible to optimize the tree. -// -// Return Value: -// The optimized tree, nullptr if no change was made. -// -GenTreeHWIntrinsic* Compiler::fgOptimizeForMaskedIntrinsic(GenTreeHWIntrinsic* node) -{ -#if defined(TARGET_XARCH) - bool isScalar = false; - genTreeOps actualOper = node->GetOperForHWIntrinsicId(&isScalar); - genTreeOps oper = actualOper; - var_types retType = node->TypeGet(); - CorInfoType simdBaseJitType = node->GetSimdBaseJitType(); - var_types simdBaseType = node->GetSimdBaseType(); - unsigned simdSize = node->GetSimdSize(); - - // We shouldn't find AND_NOT, OR_NOT or XOR_NOT nodes since it should only be produced in lowering - assert((oper != GT_AND_NOT) && (oper != GT_OR_NOT) && (oper != GT_XOR_NOT)); - - if (GenTreeHWIntrinsic::OperIsBitwiseHWIntrinsic(oper)) - { - GenTree* op1 = node->Op(1); - - GenTree* op2; - GenTree* actualOp2; - - if (oper == GT_NOT) - { - op2 = op1; - actualOp2 = nullptr; - } - else - { - op2 = node->Op(2); - actualOp2 = op2; - } - - // We need both operands to be ConvertMaskToVector in - // order to optimize this to a direct mask operation - - if (!op1->OperIsConvertMaskToVector()) - { - return nullptr; - } - - if (!op2->OperIsHWIntrinsic()) - { - if ((oper != GT_XOR) || !op2->IsVectorAllBitsSet()) - { - return nullptr; - } - - // We want to explicitly recognize op1 ^ AllBitsSet as - // some platforms don't have direct support for ~op1 - - oper = GT_NOT; - op2 = op1; - } - - GenTreeHWIntrinsic* cvtOp1 = op1->AsHWIntrinsic(); - GenTreeHWIntrinsic* cvtOp2 = op2->AsHWIntrinsic(); - - if (!cvtOp2->OperIsConvertMaskToVector()) - { - return nullptr; - } - - unsigned simdBaseTypeSize = genTypeSize(node->GetSimdBaseType()); - - if ((genTypeSize(cvtOp1->GetSimdBaseType()) != simdBaseTypeSize) || - (genTypeSize(cvtOp2->GetSimdBaseType()) != simdBaseTypeSize)) - { - // We need both operands to be the same kind of mask; otherwise - // the bitwise operation can differ in how it performs - return nullptr; - } - - NamedIntrinsic maskIntrinsicId = NI_Illegal; - - switch (oper) - { - case GT_AND: - { - maskIntrinsicId = NI_AVX512_AndMask; - break; - } - - case GT_NOT: - { - maskIntrinsicId = NI_AVX512_NotMask; - break; - } - - case GT_OR: - { - maskIntrinsicId = NI_AVX512_OrMask; - break; - } - - case GT_XOR: - { - maskIntrinsicId = NI_AVX512_XorMask; - break; - } - - default: - { - unreached(); - } - } - - if (maskIntrinsicId == NI_Illegal) - { - return nullptr; - } - - if (oper == actualOper) - { - node->ChangeHWIntrinsicId(maskIntrinsicId); - node->Op(1) = cvtOp1->Op(1); - } - else - { - assert(oper == GT_NOT); - node->ResetHWIntrinsicId(maskIntrinsicId, this, cvtOp1->Op(1)); - node->gtFlags &= ~GTF_REVERSE_OPS; - } - - node->gtType = TYP_MASK; - DEBUG_DESTROY_NODE(op1); - - if (oper != GT_NOT) - { - assert(actualOp2 != nullptr); - node->Op(2) = cvtOp2->Op(1); - } - - if (actualOp2 != nullptr) - { - DEBUG_DESTROY_NODE(actualOp2); - } - - node->SetMorphed(this); - node = gtNewSimdCvtMaskToVectorNode(retType, node, simdBaseJitType, simdSize)->AsHWIntrinsic(); - node->SetMorphed(this); - return node; - } -#elif defined(TARGET_ARM64) - // TODO-SVE: This optimisation is too naive. It needs to calculate the full cost of the instruction - // vs using the predicate version, taking into account all input arguements and all uses - // of the result. - // return fgMorphTryUseAllMaskVariant(node); -#else -#error Unsupported platform -#endif - return nullptr; -} - -#ifdef TARGET_ARM64 -//------------------------------------------------------------------------ -// canMorphVectorOperandToMask: Can this vector operand be converted to a -// node with type TYP_MASK easily? -// -bool Compiler::canMorphVectorOperandToMask(GenTree* node) -{ - return varTypeIsMask(node) || node->OperIsConvertMaskToVector() || node->IsVectorZero(); -} - -//------------------------------------------------------------------------ -// canMorphAllVectorOperandsToMasks: Can all vector operands to this node -// be converted to a node with type -// TYP_MASK easily? -// -bool Compiler::canMorphAllVectorOperandsToMasks(GenTreeHWIntrinsic* node) -{ - bool allMaskConversions = true; - for (size_t i = 1; i <= node->GetOperandCount() && allMaskConversions; i++) - { - allMaskConversions &= canMorphVectorOperandToMask(node->Op(i)); - } - - return allMaskConversions; -} - -//------------------------------------------------------------------------ -// doMorphVectorOperandToMask: Morph a vector node that is close to a mask -// node into a mask node. -// -// Return value: -// The morphed tree, or nullptr if the transform is not applicable. -// -GenTree* Compiler::doMorphVectorOperandToMask(GenTree* node, GenTreeHWIntrinsic* parent) -{ - if (varTypeIsMask(node)) - { - // Already a mask, nothing to do. - return node; - } - else if (node->OperIsConvertMaskToVector()) - { - // Replace node with op1. - return node->AsHWIntrinsic()->Op(1); - } - else if (node->IsVectorZero()) - { - // Morph the vector of zeroes into mask of zeroes. - GenTree* mask = gtNewSimdFalseMaskByteNode(); - mask->SetMorphed(this); - return mask; - } - - return nullptr; -} - -//----------------------------------------------------------------------------------------------------- -// fgMorphTryUseAllMaskVariant: For NamedIntrinsics that have a variant where all operands are -// mask nodes. If all operands to this node are 'suggesting' that they -// originate closely from a mask, but are of vector types, then morph the -// operands as appropriate to use mask types instead. 'Suggesting' -// is defined by the canMorphVectorOperandToMask function. -// -// Arguments: -// tree - The HWIntrinsic to try and optimize. -// -// Return Value: -// The fully morphed tree if a change was made, else nullptr. -// -GenTreeHWIntrinsic* Compiler::fgMorphTryUseAllMaskVariant(GenTreeHWIntrinsic* node) -{ - if (HWIntrinsicInfo::HasAllMaskVariant(node->GetHWIntrinsicId())) - { - NamedIntrinsic maskVariant = HWIntrinsicInfo::GetMaskVariant(node->GetHWIntrinsicId()); - - // As some intrinsics have many variants, check that the count of operands on the node - // matches the number of operands required for the mask variant of the intrinsic. The mask - // variant of the intrinsic must have a fixed number of operands. - int numArgs = HWIntrinsicInfo::lookupNumArgs(maskVariant); - assert(numArgs >= 0); - if (node->GetOperandCount() == (size_t)numArgs) - { - // We're sure it will work at this point, so perform the pattern match on operands. - if (canMorphAllVectorOperandsToMasks(node)) - { - switch (node->GetOperandCount()) - { - case 1: - node->ResetHWIntrinsicId(maskVariant, doMorphVectorOperandToMask(node->Op(1), node)); - break; - case 2: - node->ResetHWIntrinsicId(maskVariant, doMorphVectorOperandToMask(node->Op(1), node), - doMorphVectorOperandToMask(node->Op(2), node)); - break; - case 3: - node->ResetHWIntrinsicId(maskVariant, this, doMorphVectorOperandToMask(node->Op(1), node), - doMorphVectorOperandToMask(node->Op(2), node), - doMorphVectorOperandToMask(node->Op(3), node)); - break; - default: - unreached(); - } - - node->gtType = TYP_MASK; - return node; - } - } - } - - return nullptr; -} -#endif // TARGET_ARM64 - -#endif // FEATURE_MASKED_HW_INTRINSICS - //------------------------------------------------------------------------ // fgOptimizeHWIntrinsicAssociative: Morph an associative GenTreeHWIntrinsic tree. // @@ -10368,12 +10077,12 @@ GenTree* Compiler::fgOptimizeHWIntrinsicAssociative(GenTreeHWIntrinsic* tree) assert(opts.OptimizationEnabled()); NamedIntrinsic intrinsicId = tree->GetHWIntrinsicId(); - var_types simdType = tree->TypeGet(); + var_types retType = tree->TypeGet(); CorInfoType simdBaseJitType = tree->GetSimdBaseJitType(); var_types simdBaseType = tree->GetSimdBaseType(); unsigned simdSize = tree->GetSimdSize(); - if (!varTypeIsSIMD(simdType)) + if (!varTypeIsSIMD(retType) && !varTypeIsMask(retType)) { return nullptr; } @@ -10434,7 +10143,7 @@ GenTree* Compiler::fgOptimizeHWIntrinsicAssociative(GenTreeHWIntrinsic* tree) return nullptr; } - if (!intrinOp1->Op(2)->IsCnsVec() || !tree->Op(2)->IsCnsVec()) + if (!intrinOp1->Op(2)->OperIsConst() || !tree->Op(2)->OperIsConst()) { return nullptr; } @@ -10447,17 +10156,18 @@ GenTree* Compiler::fgOptimizeHWIntrinsicAssociative(GenTreeHWIntrinsic* tree) return nullptr; } - GenTreeVecCon* cns1 = intrinOp1->Op(2)->AsVecCon(); - GenTreeVecCon* cns2 = tree->Op(2)->AsVecCon(); + GenTree* cns1 = intrinOp1->Op(2); + GenTree* cns2 = tree->Op(2); - assert(cns1->TypeIs(simdType)); - assert(cns2->TypeIs(simdType)); + assert(cns1->TypeIs(retType)); + assert(cns2->TypeIs(retType)); - GenTree* res = gtNewSimdHWIntrinsicNode(simdType, cns1, cns2, intrinsicId, simdBaseJitType, simdSize); + GenTree* res = gtNewSimdHWIntrinsicNode(retType, cns1, cns2, intrinsicId, simdBaseJitType, simdSize); res = gtFoldExprHWIntrinsic(res->AsHWIntrinsic()); assert(res == cns1); - assert(res->IsCnsVec()); + assert(res->OperIsConst()); + assert(res->TypeIs(retType)); if (effectiveOp1 != op1) {