diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 9281f55471ec78..885008f00b520f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3646,6 +3646,10 @@ class Compiler GenTree* gtFoldTypeCompare(GenTree* tree); GenTree* gtFoldTypeEqualityCall(bool isEq, GenTree* op1, GenTree* op2); +#if defined(FEATURE_HW_INTRINSICS) + GenTree* gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree); +#endif // FEATURE_HW_INTRINSICS + // Options to control behavior of gtTryRemoveBoxUpstreamEffects enum BoxRemovalOptions { diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 000def1e8f1eb3..145436a9c4c6ab 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -13599,6 +13599,14 @@ GenTree* Compiler::gtFoldExpr(GenTree* tree) { return gtFoldExprConditional(tree); } + +#if defined(FEATURE_HW_INTRINSICS) + if (tree->OperIsHWIntrinsic()) + { + return gtFoldExprHWIntrinsic(tree->AsHWIntrinsic()); + } +#endif // FEATURE_HW_INTRINSICS + return tree; } @@ -15176,6 +15184,313 @@ GenTree* Compiler::gtOptimizeEnumHasFlag(GenTree* thisOp, GenTree* flagOp) return cmpTree; } +#if defined(FEATURE_HW_INTRINSICS) +GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) +{ + assert(tree->OperIsHWIntrinsic()); + + // NOTE: MinOpts() is always true for Tier0 so we have to check explicit flags instead. + // To be fixed in https://github.com/dotnet/runtime/pull/77465 + const bool tier0opts = !opts.compDbgCode && !opts.jitFlags->IsSet(JitFlags::JIT_FLAG_MIN_OPT); + if (!tier0opts) + { + return tree; + } + + GenTree* op1 = nullptr; + GenTree* op2 = nullptr; + size_t opCount = tree->GetOperandCount(); + + switch (opCount) + { + case 2: + { + op2 = tree->Op(2); + FALLTHROUGH; + } + + case 1: + { + op1 = tree->Op(1); + break; + } + + default: + { + return tree; + } + } + + bool isScalar = false; + genTreeOps oper = tree->HWOperGet(&isScalar); + + if (oper == GT_NONE) + { + return tree; + } + +#if defined(TARGET_XARCH) + if (oper == GT_AND_NOT) + { + std::swap(op1, op2); + } +#endif // TARGET_XARCH + + GenTree* result = tree; + + var_types simdType = tree->TypeGet(); + var_types simdBaseType = tree->GetSimdBaseType(); + CorInfoType simdBaseJitType = tree->GetSimdBaseJitType(); + unsigned int simdSize = tree->GetSimdSize(); + + GenTreeVecCon* cnsNode = nullptr; + GenTree* otherNode = nullptr; + + if (op1->IsVectorConst()) + { + if (op2 == nullptr) + { + op1->AsVecCon()->EvaluateUnaryInPlace(oper, isScalar, simdType, simdBaseType); + result = op1; + } + else if (op2->IsVectorConst()) + { + op1->AsVecCon()->EvaluateBinaryInPlace(oper, isScalar, simdType, simdBaseType, op2->AsVecCon()); + result = op1; + } + else + { + cnsNode = op1->AsVecCon(); + otherNode = op2; + } + } + else if ((op2 != nullptr) && op2->IsVectorConst()) + { + cnsNode = op2->AsVecCon(); + otherNode = op1; + } + + if (cnsNode != nullptr) + { + bool otherNodeHasSideEffects = (otherNode->gtFlags & GTF_SIDE_EFFECT) != 0; + + switch (oper) + { + case GT_ADD: + { + if (varTypeIsFloating(simdBaseType)) + { + // Not safe for floating-point when x == -0.0 + break; + } + + // Handle `x + 0 == x` and `0 + x == x` + if (cnsNode->IsZero()) + { + result = otherNode; + } + break; + } + + case GT_AND: + { + // Handle `x & 0 == 0` and `0 & x == 0` + if (cnsNode->IsZero()) + { + if (!otherNodeHasSideEffects) + { + // We need to preserve side effects when they exist + result = cnsNode; + } + break; + } + + // Handle `x & AllBitsSet == x` and `AllBitsSet & x == x` + if (cnsNode->IsAllBitsSet()) + { + result = otherNode; + } + break; + } + + case GT_AND_NOT: + { + // Handle `x & ~0 == x` and `0 & ~x == 0` + if (cnsNode->IsZero()) + { + if (cnsNode == op1) + { + if (!otherNodeHasSideEffects) + { + // We need to preserve side effects when they exist + result = cnsNode; + } + break; + } + else + { + result = otherNode; + } + break; + } + + // Handle `x & ~AllBitsSet == 0` + if (cnsNode->IsAllBitsSet() && (cnsNode == op2)) + { + if (!otherNodeHasSideEffects) + { + // We need to preserve side effects when they exist + result = cnsNode; + } + } + break; + } + + case GT_DIV: + { + // Handle `x / 1 == x`. + // This is safe for all floats since we do not fault for sNaN + + if (cnsNode != op2) + { + break; + } + + if (!cnsNode->IsBroadcast(simdType, simdBaseType)) + { + break; + } + + if (varTypeIsFloating(simdBaseType)) + { + if (cnsNode->ToScalarDouble(simdBaseType) == 1.0) + { + result = otherNode; + } + } + else if (cnsNode->ToScalarUInt64(simdBaseType) == 1) + { + result = otherNode; + } + break; + } + + case GT_MUL: + { + if (!varTypeIsFloating(simdBaseType)) + { + // Handle `x * 0 == 0` and `0 * x == 0` + // Not safe for floating-point when x == -0.0, NaN, +Inf, -Inf + if (cnsNode->IsZero()) + { + if ((otherNode->gtFlags & GTF_SIDE_EFFECT) == 0) + { + // We need to preserve side effects when they exist + result = cnsNode; + } + break; + } + } + + // Handle `x * 1 == x` and `1 * x == x` + // This is safe for all floats since we do not fault for sNaN + + if (!cnsNode->IsBroadcast(simdType, simdBaseType)) + { + break; + } + + if (varTypeIsFloating(simdBaseType)) + { + if (cnsNode->ToScalarDouble(simdBaseType) == 1.0) + { + result = otherNode; + } + } + else if (cnsNode->ToScalarUInt64(simdBaseType) == 1) + { + result = otherNode; + } + break; + } + + case GT_OR: + { + // Handle `x | 0 == x` and `0 | x == x` + if (cnsNode->IsZero()) + { + result = otherNode; + break; + } + + // Handle `x | AllBitsSet == AllBitsSet` and `AllBitsSet | x == AllBitsSet` + if (cnsNode->IsAllBitsSet()) + { + if ((otherNode->gtFlags & GTF_SIDE_EFFECT) == 0) + { + // We need to preserve side effects when they exist + result = cnsNode; + } + } + break; + } + + case GT_SUB: + { + if (varTypeIsFloating(simdBaseType)) + { + // Not safe for floating-point when x == -0.0 + break; + } + + // Handle `x - 0 == x` + if ((op2 == cnsNode) && cnsNode->IsZero()) + { + result = otherNode; + } + break; + } + + case GT_XOR: + { + // Handle `x | 0 == x` and `0 | x == x` + if (cnsNode->IsZero()) + { + result = otherNode; + } + break; + } + + default: + { + break; + } + } + } + + if (result != tree) + { + JITDUMP("\nFolding hwintrinsic:\n"); + DISPTREE(tree); + + if (result->IsVectorConst()) + { + if (vnStore != nullptr) + { + fgValueNumberTreeConst(result); + } + + // Make sure no side effect flags are set on this constant node. + result->gtFlags &= ~GTF_ALL_EFFECT; + } + + JITDUMP("Transformed into:\n"); + DISPTREE(result); + } + return result; +} +#endif // FEATURE_HW_INTRINSICS + /***************************************************************************** * * Fold the given constant tree. @@ -18353,6 +18668,184 @@ bool Compiler::IsValidForShuffle(GenTreeVecCon* vecCon, unsigned simdSize, var_t return true; } + +//------------------------------------------------------------------------ +// GenTreeVecCon::EvaluateUnaryInPlace: Evaluates this constant using the given operation +// +// Arguments: +// oper - the operation to use in the evaluation +// scalar - true if this is a scalar operation; otherwise, false +// simdType - the size of the constant being checked +// baseType - the base type of the constant being checked +// +void GenTreeVecCon::EvaluateUnaryInPlace(genTreeOps oper, bool scalar, var_types simdType, var_types baseType) +{ + switch (simdType) + { + case TYP_SIMD8: + { + simd8_t result = {}; + EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd8Val); + gtSimd8Val = result; + break; + } + + case TYP_SIMD12: + { + simd12_t result = {}; + EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd12Val); + gtSimd12Val = result; + break; + } + + case TYP_SIMD16: + { + simd16_t result = {}; + EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd16Val); + gtSimd16Val = result; + break; + } + +#if defined(TARGET_XARCH) + case TYP_SIMD32: + { + simd32_t result = {}; + EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd32Val); + gtSimd32Val = result; + break; + } + + case TYP_SIMD64: + { + simd64_t result = {}; + EvaluateUnarySimd(oper, scalar, baseType, &result, gtSimd64Val); + gtSimd64Val = result; + break; + } +#endif // TARGET_XARCH + + default: + { + unreached(); + } + } +} + +//------------------------------------------------------------------------ +// GenTreeVecCon::EvaluateUnaryInPlace: Evaluates this constant using the given operation +// +// Arguments: +// oper - the operation to use in the evaluation +// scalar - true if this is a scalar operation; otherwise, false +// simdType - the size of the constant being checked +// baseType - the base type of the constant being checked +// other - the other vector constant to use in the evaluation +// +void GenTreeVecCon::EvaluateBinaryInPlace( + genTreeOps oper, bool scalar, var_types simdType, var_types baseType, GenTreeVecCon* other) +{ + switch (simdType) + { + case TYP_SIMD8: + { + simd8_t result = {}; + EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd8Val, other->gtSimd8Val); + gtSimd8Val = result; + break; + } + + case TYP_SIMD12: + { + simd12_t result = {}; + EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd12Val, other->gtSimd12Val); + gtSimd12Val = result; + break; + } + + case TYP_SIMD16: + { + simd16_t result = {}; + EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd16Val, other->gtSimd16Val); + gtSimd16Val = result; + break; + } + +#if defined(TARGET_XARCH) + case TYP_SIMD32: + { + simd32_t result = {}; + EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd32Val, other->gtSimd32Val); + gtSimd32Val = result; + break; + } + + case TYP_SIMD64: + { + simd64_t result = {}; + EvaluateBinarySimd(oper, scalar, baseType, &result, gtSimd64Val, other->gtSimd64Val); + gtSimd64Val = result; + break; + } +#endif // TARGET_XARCH + + default: + { + unreached(); + } + } +} + +//------------------------------------------------------------------------ +// GenTreeVecCon::IsBroadcast: Determines if this vector constant is a broadcast +// +// Arguments: +// simdType - the size of the constant being checked +// simdBaseType - the base type of the constant being checked +// +// Returns: +// true if the constant represents a broadcast value; otherwise, false +// +bool GenTreeVecCon::IsBroadcast(var_types simdType, var_types simdBaseType) const +{ + assert(varTypeIsSIMD(simdType)); + assert(varTypeIsArithmetic(simdBaseType)); + + int elementCount = ElementCount(genTypeSize(simdType), simdBaseType); + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + { + return ElementsAreSame(>SimdVal.u8[0], elementCount); + } + + case TYP_SHORT: + case TYP_USHORT: + { + return ElementsAreSame(>SimdVal.u16[0], elementCount); + } + + case TYP_FLOAT: + case TYP_INT: + case TYP_UINT: + { + return ElementsAreSame(>SimdVal.u32[0], elementCount); + } + + case TYP_DOUBLE: + case TYP_LONG: + case TYP_ULONG: + { + return ElementsAreSame(>SimdVal.u64[0], elementCount); + } + + default: + { + return false; + } + } +} #endif // FEATURE_HW_INTRINSICS*/ //------------------------------------------------------------------------ @@ -27058,8 +27551,9 @@ bool GenTreeHWIntrinsic::OperIsCreateScalarUnsafe() const // bool GenTreeHWIntrinsic::OperIsBitwiseHWIntrinsic() const { - genTreeOps Oper = HWOperGet(); - return Oper == GT_AND || Oper == GT_OR || Oper == GT_XOR || Oper == GT_AND_NOT; + bool isScalar = false; + genTreeOps oper = HWOperGet(&isScalar); + return (oper == GT_AND) || (oper == GT_AND_NOT) || (oper == GT_NOT) || (oper == GT_OR) || (oper == GT_XOR); } //------------------------------------------------------------------------ @@ -27388,8 +27882,10 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId) //------------------------------------------------------------------------------ // HWOperGet : Returns Oper based on the HWIntrinsicId // -genTreeOps GenTreeHWIntrinsic::HWOperGet() const +genTreeOps GenTreeHWIntrinsic::HWOperGet(bool* isScalar) const { + *isScalar = false; + switch (GetHWIntrinsicId()) { #if defined(TARGET_XARCH) @@ -27448,11 +27944,148 @@ genTreeOps GenTreeHWIntrinsic::HWOperGet() const case NI_AVX2_AndNot: case NI_AVX512F_AndNot: case NI_AVX512DQ_AndNot: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_BitwiseClear: +#endif { return GT_AND_NOT; } + +#if defined(TARGET_XARCH) + case NI_SSE_Add: + case NI_SSE2_Add: + case NI_AVX_Add: + case NI_AVX2_Add: + case NI_AVX512F_Add: + case NI_AVX512BW_Add: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_Add: + case NI_AdvSimd_Arm64_Add: #endif - // TODO: Handle other cases + { + return GT_ADD; + } + +#if defined(TARGET_XARCH) + case NI_SSE_AddScalar: + case NI_SSE2_AddScalar: + case NI_AVX512F_AddScalar: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_AddScalar: +#endif + { + *isScalar = true; + return GT_ADD; + } + +#if defined(TARGET_XARCH) + case NI_SSE_Divide: + case NI_SSE2_Divide: + case NI_AVX_Divide: + case NI_AVX512F_Divide: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_Arm64_Divide: +#endif + { + return GT_DIV; + } + +#if defined(TARGET_XARCH) + case NI_SSE_DivideScalar: + case NI_SSE2_DivideScalar: + case NI_AVX512F_DivideScalar: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_DivideScalar: +#endif + { + *isScalar = true; + return GT_DIV; + } + +#if defined(TARGET_XARCH) + case NI_SSE_Multiply: + case NI_SSE2_MultiplyLow: + case NI_SSE41_MultiplyLow: + case NI_AVX_Multiply: + case NI_AVX2_MultiplyLow: + case NI_AVX512F_MultiplyLow: + case NI_AVX512BW_MultiplyLow: + case NI_AVX512DQ_MultiplyLow: + case NI_AVX512DQ_VL_MultiplyLow: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_Multiply: + case NI_AdvSimd_Arm64_Multiply: +#endif + { + return GT_MUL; + } + +#if defined(TARGET_XARCH) + case NI_SSE2_Multiply: + case NI_AVX512F_Multiply: + { + if (varTypeIsFloating(GetSimdBaseType())) + { + return GT_MUL; + } + return GT_NONE; + } +#endif + +#if defined(TARGET_XARCH) + case NI_SSE_MultiplyScalar: + case NI_SSE2_MultiplyScalar: + case NI_AVX512F_MultiplyScalar: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_MultiplyScalar: +#endif + { + *isScalar = true; + return GT_MUL; + } + +#if defined(TARGET_ARM64) + case NI_AdvSimd_Negate: + case NI_AdvSimd_Arm64_Negate: + { + return GT_NEG; + } + + case NI_AdvSimd_NegateScalar: + case NI_AdvSimd_Arm64_NegateScalar: + { + *isScalar = true; + return GT_NEG; + } +#endif + + +#if defined(TARGET_XARCH) + case NI_SSE_Subtract: + case NI_SSE2_Subtract: + case NI_AVX_Subtract: + case NI_AVX2_Subtract: + case NI_AVX512F_Subtract: + case NI_AVX512BW_Subtract: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_Subtract: + case NI_AdvSimd_Arm64_Subtract: +#endif + { + return GT_SUB; + } + +#if defined(TARGET_XARCH) + case NI_SSE_SubtractScalar: + case NI_SSE2_SubtractScalar: + case NI_AVX512F_SubtractScalar: +#elif defined(TARGET_ARM64) + case NI_AdvSimd_SubtractScalar: +#endif + { + *isScalar = true; + return GT_SUB; + } default: { @@ -28278,8 +28911,13 @@ uint8_t GenTreeHWIntrinsic::GetTernaryControlByte(GenTreeHWIntrinsic* second) co const uint8_t B = 0xCC; const uint8_t C = 0xAA; - genTreeOps firstOper = HWOperGet(); - genTreeOps secondOper = second->HWOperGet(); + bool isScalar = false; + + genTreeOps firstOper = HWOperGet(&isScalar); + assert(!isScalar); + + genTreeOps secondOper = second->HWOperGet(&isScalar); + assert(!isScalar); uint8_t AB = 0; uint8_t ABC = 0; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 578b2f43b349be..cbf0c38e4a3c66 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -6632,7 +6632,7 @@ struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic static bool Equals(GenTreeHWIntrinsic* op1, GenTreeHWIntrinsic* op2); - genTreeOps HWOperGet() const; + genTreeOps HWOperGet(bool* isScalar) const; private: void SetHWIntrinsicId(NamedIntrinsic intrinsicId); @@ -6876,6 +6876,10 @@ struct GenTreeVecCon : public GenTree #endif // FEATURE_HW_INTRINSICS + void EvaluateUnaryInPlace(genTreeOps oper, bool scalar, var_types simdType, var_types baseType); + void EvaluateBinaryInPlace( + genTreeOps oper, bool scalar, var_types simdType, var_types baseType, GenTreeVecCon* other); + bool IsAllBitsSet() const { switch (gtType) @@ -6921,6 +6925,8 @@ struct GenTreeVecCon : public GenTree } } + bool IsBroadcast(var_types simdType, var_types simdBaseType) const; + static bool Equals(const GenTreeVecCon* left, const GenTreeVecCon* right) { var_types gtType = left->TypeGet(); @@ -7018,6 +7024,89 @@ struct GenTreeVecCon : public GenTree } } + double ToScalarDouble(var_types simdBaseType) const + { + switch (simdBaseType) + { + case TYP_FLOAT: + { + return gtSimdVal.f32[0]; + } + + case TYP_DOUBLE: + { + return gtSimdVal.f64[0]; + } + + default: + { + unreached(); + } + } + } + + int64_t ToScalarInt64(var_types simdBaseType) const + { + switch (simdBaseType) + { + case TYP_BYTE: + { + return gtSimdVal.i8[0]; + } + + case TYP_SHORT: + { + return gtSimdVal.i16[0]; + } + + case TYP_INT: + { + return gtSimdVal.i32[0]; + } + + case TYP_LONG: + { + return gtSimdVal.i64[0]; + } + + default: + { + unreached(); + } + } + } + + uint64_t ToScalarUInt64(var_types simdBaseType) const + { + switch (simdBaseType) + { + case TYP_UBYTE: + { + return gtSimdVal.u8[0]; + } + + case TYP_USHORT: + { + return gtSimdVal.u16[0]; + } + + case TYP_UINT: + { + return gtSimdVal.u32[0]; + } + + case TYP_ULONG: + { + return gtSimdVal.u64[0]; + } + + default: + { + unreached(); + } + } + } + GenTreeVecCon(var_types type) : GenTree(GT_CNS_VEC, type) { diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 6c445351f78fe8..870b2b57d9b2f9 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -3070,12 +3070,17 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig R2RARG(entryPoint), mustExpand); - if (mustExpand && (hwintrinsic == nullptr)) + if (hwintrinsic == nullptr) { - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); + if (mustExpand) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); + } + return nullptr; } - return hwintrinsic; + // Fold result, if possible + return gtFoldExpr(hwintrinsic); } else { @@ -3083,7 +3088,16 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, if (isIntrinsic) { - return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis, mustExpand); + GenTree* hwintrinsic = impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis, mustExpand); + + if (hwintrinsic == nullptr) + { + assert(!mustExpand); + return nullptr; + } + + // Fold result, if possible + return gtFoldExpr(hwintrinsic); } } } diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 2599c7a8b4d84c..f869d5f36843ac 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2179,7 +2179,8 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } - if (second->AsHWIntrinsic()->HWOperGet() == GT_AND_NOT) + bool isScalar = false; + if ((second->AsHWIntrinsic()->HWOperGet(&isScalar) == GT_AND_NOT) || isScalar) { // currently ANDNOT logic cannot be optimized by the ternary node. break; @@ -9189,50 +9190,22 @@ void Lowering::TryFoldCnsVecForEmbeddedBroadcast(GenTreeHWIntrinsic* parentNode, { simdType = Compiler::getSIMDTypeForSize(simdSize); } - int elementCount = GenTreeVecCon::ElementCount(genTypeSize(simdType), simdBaseType); - switch (simdBaseType) + if (varTypeIsSmall(simdBaseType)) { - case TYP_FLOAT: - case TYP_INT: - case TYP_UINT: - { - uint32_t firstElement = static_cast(childNode->gtSimdVal.u32[0]); - for (int i = 1; i < elementCount; i++) - { - uint32_t elementToCheck = static_cast(childNode->gtSimdVal.u32[i]); - if (firstElement != elementToCheck) - { - isCreatedFromScalar = false; - break; - } - } - break; - } - - case TYP_DOUBLE: -#if defined(TARGET_AMD64) - case TYP_LONG: - case TYP_ULONG: -#endif // TARGET_AMD64 - { - uint64_t firstElement = static_cast(childNode->gtSimdVal.u64[0]); - for (int i = 1; i < elementCount; i++) - { - uint64_t elementToCheck = static_cast(childNode->gtSimdVal.u64[i]); - if (firstElement != elementToCheck) - { - isCreatedFromScalar = false; - break; - } - } - break; - } - - default: - isCreatedFromScalar = false; - break; + isCreatedFromScalar = false; + } +#ifndef TARGET_64BIT + else if (varTypeIsLong(simdBaseType)) + { + isCreatedFromScalar = false; } +#endif // TARGET_64BIT + else + { + isCreatedFromScalar = childNode->IsBroadcast(simdType, simdBaseType); + } + if (isCreatedFromScalar) { NamedIntrinsic broadcastName = NI_AVX2_BroadcastScalarToVector128; diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 89fae6f436709e..152c9a249a29e4 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -10887,11 +10887,19 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node) } } - // Transforms: - // 1.(~v1 & v2) to VectorXxx.AndNot(v1, v2) - // 2.(v1 & (~v2)) to VectorXxx.AndNot(v2, v1) - switch (node->HWOperGet()) + bool isScalar = false; + genTreeOps oper = node->HWOperGet(&isScalar); + + if (isScalar) + { + return node; + } + + switch (oper) { + // Transforms: + // 1.(~v1 & v2) to VectorXxx.AndNot(v1, v2) + // 2.(v1 & (~v2)) to VectorXxx.AndNot(v2, v1) case GT_AND: { GenTree* op1 = node->Op(1); @@ -10903,7 +10911,12 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node) { // Try handle: ~op1 & op2 GenTreeHWIntrinsic* hw = op1->AsHWIntrinsic(); - genTreeOps hwOper = hw->HWOperGet(); + genTreeOps hwOper = hw->HWOperGet(&isScalar); + + if (isScalar) + { + return node; + } if (hwOper == GT_NOT) { @@ -10932,7 +10945,12 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node) { // Try handle: op1 & ~op2 GenTreeHWIntrinsic* hw = op2->AsHWIntrinsic(); - genTreeOps hwOper = hw->HWOperGet(); + genTreeOps hwOper = hw->HWOperGet(&isScalar); + + if (isScalar) + { + return node; + } if (hwOper == GT_NOT) { @@ -11955,8 +11973,6 @@ GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree, bool* optAssertionPropD // GenTree* Compiler::fgMorphMultiOp(GenTreeMultiOp* multiOp) { - gtUpdateNodeOperSideEffects(multiOp); - bool dontCseConstArguments = false; #if defined(FEATURE_HW_INTRINSICS) // Opportunistically, avoid unexpected CSE for hw intrinsics with IMM arguments @@ -11979,12 +11995,10 @@ GenTree* Compiler::fgMorphMultiOp(GenTreeMultiOp* multiOp) for (GenTree** use : multiOp->UseEdges()) { - *use = fgMorphTree(*use); - + *use = fgMorphTree(*use); GenTree* operand = *use; - multiOp->gtFlags |= (operand->gtFlags & GTF_ALL_EFFECT); - if (dontCseConstArguments && operand->OperIsConst()) + if (dontCseConstArguments && operand->IsCnsIntOrI()) { operand->SetDoNotCSE(); } @@ -12003,10 +12017,31 @@ GenTree* Compiler::fgMorphMultiOp(GenTreeMultiOp* multiOp) } } + gtUpdateNodeOperSideEffects(multiOp); + + for (GenTree** use : multiOp->UseEdges()) + { + GenTree* operand = *use; + multiOp->AddAllEffectsFlags(operand); + } + #if defined(FEATURE_HW_INTRINSICS) - if (opts.OptimizationEnabled() && multiOp->OperIs(GT_HWINTRINSIC)) + if (opts.OptimizationEnabled() && multiOp->OperIsHWIntrinsic()) { - GenTreeHWIntrinsic* hw = multiOp->AsHWIntrinsic(); + // Try to fold it, maybe we get lucky, + GenTree* foldedTree = gtFoldExpr(multiOp); + + if (foldedTree != multiOp) + { + assert(!fgIsCommaThrow(foldedTree)); + return foldedTree; + } + else if (!foldedTree->OperIsHWIntrinsic()) + { + return foldedTree; + } + + GenTreeHWIntrinsic* hw = foldedTree->AsHWIntrinsic(); // Move constant vectors from op1 to op2 for commutative and compare operations if ((hw->GetOperandCount() == 2) && hw->Op(1)->IsVectorConst() && diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 58a5285b2e3500..1179bee5345e55 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -7235,16 +7235,23 @@ ValueNum EvaluateSimdGetElement(ValueNumStore* vns, var_types type, var_types ba } } -ValueNum ValueNumStore::EvalHWIntrinsicFunUnary(var_types type, - var_types baseType, - NamedIntrinsic ni, - VNFunc func, - ValueNum arg0VN, - bool encodeResultType, - ValueNum resultTypeVN) +ValueNum ValueNumStore::EvalHWIntrinsicFunUnary( + GenTreeHWIntrinsic* tree, VNFunc func, ValueNum arg0VN, bool encodeResultType, ValueNum resultTypeVN) { + var_types type = tree->TypeGet(); + var_types baseType = tree->GetSimdBaseType(); + NamedIntrinsic ni = tree->GetHWIntrinsicId(); + if (IsVNConstant(arg0VN)) { + bool isScalar = false; + genTreeOps oper = tree->HWOperGet(&isScalar); + + if (oper != GT_NONE) + { + return EvaluateUnarySimd(this, oper, isScalar, type, baseType, arg0VN); + } + switch (ni) { #ifdef TARGET_ARM64 @@ -7303,23 +7310,6 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunUnary(var_types type, return VNForLongCon(static_cast(result)); } - - case NI_AdvSimd_Negate: - case NI_AdvSimd_Arm64_Negate: - { - return EvaluateUnarySimd(this, GT_NEG, /* scalar */ false, type, baseType, arg0VN); - } - - case NI_AdvSimd_NegateScalar: - case NI_AdvSimd_Arm64_NegateScalar: - { - return EvaluateUnarySimd(this, GT_NEG, /* scalar */ true, type, baseType, arg0VN); - } - - case NI_AdvSimd_Not: - { - return EvaluateUnarySimd(this, GT_NOT, /* scalar */ false, type, baseType, arg0VN); - } #endif // TARGET_ARM64 #if defined(TARGET_XARCH) @@ -7453,15 +7443,17 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunUnary(var_types type, return VNForFunc(type, func, arg0VN); } -ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, - var_types baseType, - NamedIntrinsic ni, - VNFunc func, - ValueNum arg0VN, - ValueNum arg1VN, - bool encodeResultType, - ValueNum resultTypeVN) +ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(GenTreeHWIntrinsic* tree, + VNFunc func, + ValueNum arg0VN, + ValueNum arg1VN, + bool encodeResultType, + ValueNum resultTypeVN) { + var_types type = tree->TypeGet(); + var_types baseType = tree->GetSimdBaseType(); + NamedIntrinsic ni = tree->GetHWIntrinsicId(); + ValueNum cnsVN = NoVN; ValueNum argVN = NoVN; @@ -7488,87 +7480,23 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, { assert(IsVNConstant(arg0VN) && IsVNConstant(arg1VN)); - switch (ni) - { -#ifdef TARGET_ARM64 - case NI_AdvSimd_Add: - case NI_AdvSimd_Arm64_Add: -#else - case NI_SSE_Add: - case NI_SSE2_Add: - case NI_AVX_Add: - case NI_AVX2_Add: - case NI_AVX512F_Add: - case NI_AVX512BW_Add: -#endif - { - return EvaluateBinarySimd(this, GT_ADD, /* scalar */ false, type, baseType, arg0VN, arg1VN); - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_AddScalar: -#else - case NI_SSE_AddScalar: - case NI_SSE2_AddScalar: -#endif - { - return EvaluateBinarySimd(this, GT_ADD, /* scalar */ true, type, baseType, arg0VN, arg1VN); - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_And: -#else - case NI_SSE_And: - case NI_SSE2_And: - case NI_AVX_And: - case NI_AVX2_And: - case NI_AVX512F_And: - case NI_AVX512DQ_And: -#endif - { - return EvaluateBinarySimd(this, GT_AND, /* scalar */ false, type, baseType, arg0VN, arg1VN); - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_BitwiseClear: - { - return EvaluateBinarySimd(this, GT_AND_NOT, /* scalar */ false, type, baseType, arg0VN, arg1VN); - } -#else - case NI_SSE_AndNot: - case NI_SSE2_AndNot: - case NI_AVX_AndNot: - case NI_AVX2_AndNot: - case NI_AVX512F_AndNot: - case NI_AVX512DQ_AndNot: - { - // xarch does: ~arg0VN & arg1VN - return EvaluateBinarySimd(this, GT_AND_NOT, /* scalar */ false, type, baseType, arg1VN, arg0VN); - } -#endif + bool isScalar = false; + genTreeOps oper = tree->HWOperGet(&isScalar); -#ifdef TARGET_ARM64 - case NI_AdvSimd_Arm64_Divide: -#else - case NI_SSE_Divide: - case NI_SSE2_Divide: - case NI_AVX_Divide: - case NI_AVX512F_Divide: -#endif + if (oper != GT_NONE) + { +#if defined(TARGET_XARCH) + if (oper == GT_AND_NOT) { - return EvaluateBinarySimd(this, GT_DIV, /* scalar */ false, type, baseType, arg0VN, arg1VN); + std::swap(arg0VN, arg1VN); } +#endif // TARGET_XARCH -#ifdef TARGET_ARM64 - case NI_AdvSimd_DivideScalar: -#else - case NI_SSE_DivideScalar: - case NI_SSE2_DivideScalar: -#endif - { - return EvaluateBinarySimd(this, GT_DIV, /* scalar */ true, type, baseType, arg0VN, arg1VN); - } + return EvaluateBinarySimd(this, oper, isScalar, type, baseType, arg0VN, arg1VN); + } + switch (ni) + { case NI_Vector128_GetElement: #ifdef TARGET_ARM64 case NI_Vector64_GetElement: @@ -7593,61 +7521,6 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, } #endif -#ifdef TARGET_XARCH - case NI_AVX512F_Multiply: - { - if (!varTypeIsFloating(baseType)) - { - // We don't support this for integrals since it returns a different size than the input - break; - } - FALLTHROUGH; - } -#endif // TARGET_XARCH - -#ifdef TARGET_ARM64 - case NI_AdvSimd_Multiply: - case NI_AdvSimd_Arm64_Multiply: -#else - case NI_SSE_Multiply: - case NI_SSE2_Multiply: - case NI_SSE2_MultiplyLow: - case NI_SSE41_MultiplyLow: - case NI_AVX_Multiply: - case NI_AVX2_MultiplyLow: - case NI_AVX512F_MultiplyLow: - case NI_AVX512BW_MultiplyLow: - case NI_AVX512DQ_MultiplyLow: - case NI_AVX512DQ_VL_MultiplyLow: -#endif - { - return EvaluateBinarySimd(this, GT_MUL, /* scalar */ false, type, baseType, arg0VN, arg1VN); - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_MultiplyScalar: -#else - case NI_SSE_MultiplyScalar: - case NI_SSE2_MultiplyScalar: -#endif - { - return EvaluateBinarySimd(this, GT_MUL, /* scalar */ true, type, baseType, arg0VN, arg1VN); - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_Or: -#else - case NI_SSE_Or: - case NI_SSE2_Or: - case NI_AVX_Or: - case NI_AVX2_Or: - case NI_AVX512F_Or: - case NI_AVX512DQ_Or: -#endif - { - return EvaluateBinarySimd(this, GT_OR, /* scalar */ false, type, baseType, arg0VN, arg1VN); - } - #ifdef TARGET_XARCH case NI_AVX512F_RotateLeft: case NI_AVX512F_VL_RotateLeft: @@ -7791,64 +7664,24 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, } #endif // TARGET_ARM64 -#ifdef TARGET_ARM64 - case NI_AdvSimd_Subtract: - case NI_AdvSimd_Arm64_Subtract: -#else - case NI_SSE_Subtract: - case NI_SSE2_Subtract: - case NI_AVX_Subtract: - case NI_AVX2_Subtract: - case NI_AVX512F_Subtract: - case NI_AVX512BW_Subtract: -#endif - { - return EvaluateBinarySimd(this, GT_SUB, /* scalar */ false, type, baseType, arg0VN, arg1VN); - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_SubtractScalar: -#else - case NI_SSE_SubtractScalar: - case NI_SSE2_SubtractScalar: -#endif - { - return EvaluateBinarySimd(this, GT_SUB, /* scalar */ true, type, baseType, arg0VN, arg1VN); - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_Xor: -#else - case NI_SSE_Xor: - case NI_SSE2_Xor: - case NI_AVX_Xor: - case NI_AVX2_Xor: - case NI_AVX512F_Xor: - case NI_AVX512DQ_Xor: -#endif - { - return EvaluateBinarySimd(this, GT_XOR, /* scalar */ false, type, baseType, arg0VN, arg1VN); - } - default: break; } } else if (cnsVN != NoVN) { - switch (ni) + bool isScalar = false; + genTreeOps oper = tree->HWOperGet(&isScalar); + + if (isScalar) { -#ifdef TARGET_ARM64 - case NI_AdvSimd_Add: - case NI_AdvSimd_Arm64_Add: -#else - case NI_SSE_Add: - case NI_SSE2_Add: - case NI_AVX_Add: - case NI_AVX2_Add: - case NI_AVX512F_Add: - case NI_AVX512BW_Add: -#endif + // We don't support folding scalars today + oper = GT_NONE; + } + + switch (oper) + { + case GT_ADD: { if (varTypeIsFloating(baseType)) { @@ -7866,16 +7699,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, break; } -#ifdef TARGET_ARM64 - case NI_AdvSimd_And: -#else - case NI_SSE_And: - case NI_SSE2_And: - case NI_AVX_And: - case NI_AVX2_And: - case NI_AVX512F_And: - case NI_AVX512DQ_And: -#endif + case GT_AND: { // Handle `x & 0 == 0` and `0 & x == 0` ValueNum zeroVN = VNZeroForType(type); @@ -7885,7 +7709,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, return zeroVN; } - // Handle `x & ~0 == x` and `~0 & x == x` + // Handle `x & AllBitsSet == x` and `AllBitsSet & x == x` ValueNum allBitsVN = VNAllBitsForType(type); if (cnsVN == allBitsVN) @@ -7895,57 +7719,38 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, break; } -#ifdef TARGET_ARM64 - case NI_AdvSimd_BitwiseClear: -#else - case NI_SSE_AndNot: - case NI_SSE2_AndNot: - case NI_AVX_AndNot: - case NI_AVX2_AndNot: - case NI_AVX512F_AndNot: - case NI_AVX512DQ_AndNot: + case GT_AND_NOT: { -#ifdef TARGET_ARM64 - if (cnsVN == arg0VN) - { - // arm64 preserves the args, so we can only handle `x & ~cns` - break; - } -#else - if (cnsVN == arg1VN) - { - // xarch swaps the args, so we can only handle `~cns & x` - break; - } -#endif +#if defined(TARGET_XARCH) + std::swap(arg0VN, arg1VN); +#endif // TARGET_XARCH - // Handle `x & ~0 == x` + // Handle `x & ~0 == x` and `0 & ~x == 0` ValueNum zeroVN = VNZeroForType(type); if (cnsVN == zeroVN) { + if (cnsVN == arg0VN) + { + return zeroVN; + } return argVN; } - // Handle `x & 0 == 0` + // Handle `x & ~AllBitsSet == 0` ValueNum allBitsVN = VNAllBitsForType(type); if (cnsVN == allBitsVN) { - return zeroVN; + if (cnsVN == arg1VN) + { + return zeroVN; + } } break; } -#endif -#ifdef TARGET_ARM64 - case NI_AdvSimd_Arm64_Divide: -#else - case NI_SSE_Divide: - case NI_SSE2_Divide: - case NI_AVX_Divide: - case NI_AVX512F_Divide: -#endif + case GT_DIV: { // Handle `x / 1 == x`. // This is safe for all floats since we do not fault for sNaN @@ -7967,63 +7772,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, break; } -#ifdef TARGET_ARM64 - case NI_AdvSimd_MultiplyByScalar: - case NI_AdvSimd_Arm64_MultiplyByScalar: - { - if (!varTypeIsFloating(baseType)) - { - // Handle `x * 0 == 0` and `0 * x == 0` - // Not safe for floating-point when x == -0.0, NaN, +Inf, -Inf - ValueNum zeroVN = VNZeroForType(TypeOfVN(cnsVN)); - - if (cnsVN == zeroVN) - { - return VNZeroForType(type); - } - } - - assert((TypeOfVN(arg0VN) == type) && (TypeOfVN(arg1VN) == TYP_SIMD8)); - - // Handle x * 1 => x, but only if the scalar RHS is <1, ...>. - if (IsVNConstant(arg1VN)) - { - if (EvaluateSimdGetElement(this, TYP_SIMD8, baseType, arg1VN, 0) == VNOneForType(baseType)) - { - return arg0VN; - } - } - break; - } -#endif - -#ifdef TARGET_XARCH - case NI_AVX512F_Multiply: - { - if (!varTypeIsFloating(baseType)) - { - // We don't support this for integrals since it returns a different size than the input - break; - } - FALLTHROUGH; - } -#endif // TARGET_XARCH - -#ifdef TARGET_ARM64 - case NI_AdvSimd_Multiply: - case NI_AdvSimd_Arm64_Multiply: -#else - case NI_SSE_Multiply: - case NI_SSE2_Multiply: - case NI_SSE2_MultiplyLow: - case NI_SSE41_MultiplyLow: - case NI_AVX_Multiply: - case NI_AVX2_MultiplyLow: - case NI_AVX512F_MultiplyLow: - case NI_AVX512BW_MultiplyLow: - case NI_AVX512DQ_MultiplyLow: - case NI_AVX512DQ_VL_MultiplyLow: -#endif + case GT_MUL: { if (!varTypeIsFloating(baseType)) { @@ -8057,16 +7806,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, break; } -#ifdef TARGET_ARM64 - case NI_AdvSimd_Or: -#else - case NI_SSE_Or: - case NI_SSE2_Or: - case NI_AVX_Or: - case NI_AVX2_Or: - case NI_AVX512F_Or: - case NI_AVX512DQ_Or: -#endif + case GT_OR: { // Handle `x | 0 == x` and `0 | x == x` ValueNum zeroVN = VNZeroForType(type); @@ -8086,6 +7826,74 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, break; } + case GT_SUB: + { + if (varTypeIsFloating(baseType)) + { + // Not safe for floating-point when x == -0.0 + break; + } + + // Handle `x - 0 == x` + ValueNum zeroVN = VNZeroForType(type); + + if (arg1VN == zeroVN) + { + return argVN; + } + break; + } + + case GT_XOR: + { + // Handle `x ^ 0 == x` and `0 ^ x == x` + ValueNum zeroVN = VNZeroForType(type); + + if (cnsVN == zeroVN) + { + return argVN; + } + break; + } + + default: + { + break; + } + } + + switch (ni) + { +#ifdef TARGET_ARM64 + case NI_AdvSimd_MultiplyByScalar: + case NI_AdvSimd_Arm64_MultiplyByScalar: + { + if (!varTypeIsFloating(baseType)) + { + // Handle `x * 0 == 0` and `0 * x == 0` + // Not safe for floating-point when x == -0.0, NaN, +Inf, -Inf + ValueNum zeroVN = VNZeroForType(TypeOfVN(cnsVN)); + + if (cnsVN == zeroVN) + { + return VNZeroForType(type); + } + } + + assert((TypeOfVN(arg0VN) == type) && (TypeOfVN(arg1VN) == TYP_SIMD8)); + + // Handle x * 1 => x, but only if the scalar RHS is <1, ...>. + if (IsVNConstant(arg1VN)) + { + if (EvaluateSimdGetElement(this, TYP_SIMD8, baseType, arg1VN, 0) == VNOneForType(baseType)) + { + return arg0VN; + } + } + break; + } +#endif + #ifdef TARGET_ARM64 case NI_AdvSimd_ShiftLeftLogical: case NI_AdvSimd_ShiftRightArithmetic: @@ -8118,119 +7926,41 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, break; } -#ifdef TARGET_ARM64 - case NI_AdvSimd_Subtract: - case NI_AdvSimd_Arm64_Subtract: -#else - case NI_SSE_Subtract: - case NI_SSE2_Subtract: - case NI_AVX_Subtract: - case NI_AVX2_Subtract: - case NI_AVX512F_Subtract: - case NI_AVX512BW_Subtract: -#endif - { - if (varTypeIsFloating(baseType)) - { - // Not safe for floating-point when x == -0.0 - break; - } - - // Handle `x - 0 == x` - ValueNum zeroVN = VNZeroForType(type); - - if (arg1VN == zeroVN) - { - return argVN; - } - break; - } - -#ifdef TARGET_ARM64 - case NI_AdvSimd_Xor: -#else - case NI_SSE_Xor: - case NI_SSE2_Xor: - case NI_AVX_Xor: - case NI_AVX2_Xor: - case NI_AVX512F_Xor: - case NI_AVX512DQ_Xor: -#endif - { - // Handle `x | 0 == x` and `0 | x == x` - ValueNum zeroVN = VNZeroForType(type); - - if (cnsVN == zeroVN) - { - return argVN; - } - break; - } - default: break; } } else if (arg0VN == arg1VN) - { - switch (ni) + {bool isScalar = false; + genTreeOps oper = tree->HWOperGet(&isScalar); + + if (isScalar) { -#ifdef TARGET_ARM64 - case NI_AdvSimd_And: -#else - case NI_SSE_And: - case NI_SSE2_And: - case NI_AVX_And: - case NI_AVX2_And: - case NI_AVX512F_And: - case NI_AVX512DQ_And: -#endif + // We don't support folding scalars today + oper = GT_NONE; + } + + switch (oper) + { + case GT_AND: { // Handle `x & x == x` return arg0VN; } -#ifdef TARGET_ARM64 - case NI_AdvSimd_BitwiseClear: -#else - case NI_SSE_AndNot: - case NI_SSE2_AndNot: - case NI_AVX_AndNot: - case NI_AVX2_AndNot: - case NI_AVX512F_AndNot: - case NI_AVX512DQ_AndNot: + case GT_AND_NOT: { // Handle `x & ~x == 0` return VNZeroForType(type); } -#endif -#ifdef TARGET_ARM64 - case NI_AdvSimd_Or: -#else - case NI_SSE_Or: - case NI_SSE2_Or: - case NI_AVX_Or: - case NI_AVX2_Or: - case NI_AVX512F_Or: - case NI_AVX512DQ_Or: -#endif + case GT_OR: { // Handle `x | x == x` return arg0VN; } -#ifdef TARGET_ARM64 - case NI_AdvSimd_Subtract: - case NI_AdvSimd_Arm64_Subtract: -#else - case NI_SSE_Subtract: - case NI_SSE2_Subtract: - case NI_AVX_Subtract: - case NI_AVX2_Subtract: - case NI_AVX512F_Subtract: - case NI_AVX512BW_Subtract: -#endif + case GT_SUB: { if (varTypeIsFloating(baseType)) { @@ -8242,16 +7972,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary(var_types type, return VNZeroForType(type); } -#ifdef TARGET_ARM64 - case NI_AdvSimd_Xor: -#else - case NI_SSE_Xor: - case NI_SSE2_Xor: - case NI_AVX_Xor: - case NI_AVX2_Xor: - case NI_AVX512F_Xor: - case NI_AVX512DQ_Xor: -#endif + case GT_XOR: { // Handle `x ^ x == 0` return VNZeroForType(type); @@ -8315,16 +8036,18 @@ ValueNum EvaluateSimdFloatWithElement(ValueNumStore* vns, var_types type, ValueN } } -ValueNum ValueNumStore::EvalHWIntrinsicFunTernary(var_types type, - var_types baseType, - NamedIntrinsic ni, - VNFunc func, - ValueNum arg0VN, - ValueNum arg1VN, - ValueNum arg2VN, - bool encodeResultType, - ValueNum resultTypeVN) +ValueNum ValueNumStore::EvalHWIntrinsicFunTernary(GenTreeHWIntrinsic* tree, + VNFunc func, + ValueNum arg0VN, + ValueNum arg1VN, + ValueNum arg2VN, + bool encodeResultType, + ValueNum resultTypeVN) { + var_types type = tree->TypeGet(); + var_types baseType = tree->GetSimdBaseType(); + NamedIntrinsic ni = tree->GetHWIntrinsicId(); + if (IsVNConstant(arg0VN) && IsVNConstant(arg1VN) && IsVNConstant(arg2VN)) { switch (ni) @@ -12103,12 +11826,10 @@ void Compiler::fgValueNumberHWIntrinsic(GenTreeHWIntrinsic* tree) if (opCount == 1) { - ValueNum normalLVN = vnStore->EvalHWIntrinsicFunUnary(tree->TypeGet(), tree->GetSimdBaseType(), - intrinsicId, func, op1vnp.GetLiberal(), - encodeResultType, resultTypeVNPair.GetLiberal()); + ValueNum normalLVN = vnStore->EvalHWIntrinsicFunUnary(tree, func, op1vnp.GetLiberal(), encodeResultType, + resultTypeVNPair.GetLiberal()); ValueNum normalCVN = - vnStore->EvalHWIntrinsicFunUnary(tree->TypeGet(), tree->GetSimdBaseType(), intrinsicId, func, - op1vnp.GetConservative(), encodeResultType, + vnStore->EvalHWIntrinsicFunUnary(tree, func, op1vnp.GetConservative(), encodeResultType, resultTypeVNPair.GetConservative()); normalPair = ValueNumPair(normalLVN, normalCVN); @@ -12123,13 +11844,11 @@ void Compiler::fgValueNumberHWIntrinsic(GenTreeHWIntrinsic* tree) if (opCount == 2) { ValueNum normalLVN = - vnStore->EvalHWIntrinsicFunBinary(tree->TypeGet(), tree->GetSimdBaseType(), intrinsicId, func, - op1vnp.GetLiberal(), op2vnp.GetLiberal(), encodeResultType, - resultTypeVNPair.GetLiberal()); - ValueNum normalCVN = - vnStore->EvalHWIntrinsicFunBinary(tree->TypeGet(), tree->GetSimdBaseType(), intrinsicId, func, - op1vnp.GetConservative(), op2vnp.GetConservative(), - encodeResultType, resultTypeVNPair.GetConservative()); + vnStore->EvalHWIntrinsicFunBinary(tree, func, op1vnp.GetLiberal(), op2vnp.GetLiberal(), + encodeResultType, resultTypeVNPair.GetLiberal()); + ValueNum normalCVN = vnStore->EvalHWIntrinsicFunBinary(tree, func, op1vnp.GetConservative(), + op2vnp.GetConservative(), encodeResultType, + resultTypeVNPair.GetConservative()); normalPair = ValueNumPair(normalLVN, normalCVN); excSetPair = vnStore->VNPExcSetUnion(op1Xvnp, op2Xvnp); @@ -12143,15 +11862,13 @@ void Compiler::fgValueNumberHWIntrinsic(GenTreeHWIntrinsic* tree) getOperandVNs(tree->Op(3), &op3vnp, &op3Xvnp); ValueNum normalLVN = - vnStore->EvalHWIntrinsicFunTernary(tree->TypeGet(), tree->GetSimdBaseType(), intrinsicId, func, - op1vnp.GetLiberal(), op2vnp.GetLiberal(), + vnStore->EvalHWIntrinsicFunTernary(tree, func, op1vnp.GetLiberal(), op2vnp.GetLiberal(), op3vnp.GetLiberal(), encodeResultType, resultTypeVNPair.GetLiberal()); ValueNum normalCVN = - vnStore->EvalHWIntrinsicFunTernary(tree->TypeGet(), tree->GetSimdBaseType(), intrinsicId, func, - op1vnp.GetConservative(), op2vnp.GetConservative(), - op3vnp.GetConservative(), encodeResultType, - resultTypeVNPair.GetConservative()); + vnStore->EvalHWIntrinsicFunTernary(tree, func, op1vnp.GetConservative(), + op2vnp.GetConservative(), op3vnp.GetConservative(), + encodeResultType, resultTypeVNPair.GetConservative()); normalPair = ValueNumPair(normalLVN, normalCVN); diff --git a/src/coreclr/jit/valuenum.h b/src/coreclr/jit/valuenum.h index 6a5032cd79ed74..72a0166357f4f6 100644 --- a/src/coreclr/jit/valuenum.h +++ b/src/coreclr/jit/valuenum.h @@ -1211,32 +1211,25 @@ class ValueNumStore EvalMathFuncBinary(typ, mthFunc, arg0VNP.GetConservative(), arg1VNP.GetConservative())); } - ValueNum EvalHWIntrinsicFunUnary(var_types type, - var_types baseType, - NamedIntrinsic ni, - VNFunc func, - ValueNum arg0VN, - bool encodeResultType, - ValueNum resultTypeVN); - - ValueNum EvalHWIntrinsicFunBinary(var_types type, - var_types baseType, - NamedIntrinsic ni, - VNFunc func, - ValueNum arg0VN, - ValueNum arg1VN, - bool encodeResultType, - ValueNum resultTypeVN); - - ValueNum EvalHWIntrinsicFunTernary(var_types type, - var_types baseType, - NamedIntrinsic ni, - VNFunc func, - ValueNum arg0VN, - ValueNum arg1VN, - ValueNum arg2VN, - bool encodeResultType, - ValueNum resultTypeVN); +#if defined(FEATURE_HW_INTRINSICS) + ValueNum EvalHWIntrinsicFunUnary( + GenTreeHWIntrinsic* tree, VNFunc func, ValueNum arg0VN, bool encodeResultType, ValueNum resultTypeVN); + + ValueNum EvalHWIntrinsicFunBinary(GenTreeHWIntrinsic* tree, + VNFunc func, + ValueNum arg0VN, + ValueNum arg1VN, + bool encodeResultType, + ValueNum resultTypeVN); + + ValueNum EvalHWIntrinsicFunTernary(GenTreeHWIntrinsic* tree, + VNFunc func, + ValueNum arg0VN, + ValueNum arg1VN, + ValueNum arg2VN, + bool encodeResultType, + ValueNum resultTypeVN); +#endif // FEATURE_HW_INTRINSICS // Returns "true" iff "vn" represents a function application. bool IsVNFunc(ValueNum vn);