Skip to content

Revert "Arm64 SVE: Optimise zero/allbits vectors the same as masks (#… #116889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Revert "Arm64 SVE: Optimise zero/allbits vectors the same as masks (#… #116889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 0 additions & 44 deletions src/coreclr/jit/codegenarm64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2338,50 +2338,6 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre

break;
}

case GT_CNS_MSK:
{
GenTreeMskCon* mask = tree->AsMskCon();
emitter* emit = GetEmitter();

// Try every type until a match is found

if (mask->IsZero())
{
emit->emitInsSve_R(INS_sve_pfalse, EA_SCALABLE, targetReg, INS_OPTS_SCALABLE_B);
break;
}

insOpts opt = INS_OPTS_SCALABLE_B;
SveMaskPattern pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_BYTE, mask->gtSimdMaskVal);

if (pat == SveMaskPatternNone)
{
opt = INS_OPTS_SCALABLE_H;
pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_SHORT, mask->gtSimdMaskVal);
}

if (pat == SveMaskPatternNone)
{
opt = INS_OPTS_SCALABLE_S;
pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_INT, mask->gtSimdMaskVal);
}

if (pat == SveMaskPatternNone)
{
opt = INS_OPTS_SCALABLE_D;
pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_LONG, mask->gtSimdMaskVal);
}

// Should only ever create constant masks for valid patterns.
if (pat == SveMaskPatternNone)
{
unreached();
}

emit->emitIns_R_PATTERN(INS_sve_ptrue, EA_SCALABLE, targetReg, opt, (insSvePattern)pat);
break;
}
#endif // FEATURE_SIMD

default:
Expand Down
5 changes: 2 additions & 3 deletions src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3146,8 +3146,8 @@ class Compiler
var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);

#if defined(TARGET_ARM64)
GenTree* gtNewSimdAllTrueMaskNode(CorInfoType simdBaseJitType);
GenTree* gtNewSimdFalseMaskByteNode();
GenTree* gtNewSimdAllTrueMaskNode(CorInfoType simdBaseJitType, unsigned simdSize);
GenTree* gtNewSimdFalseMaskByteNode(unsigned simdSize);
#endif

GenTree* gtNewSimdBinOpNode(genTreeOps op,
Expand Down Expand Up @@ -3707,7 +3707,6 @@ class Compiler

#if defined(FEATURE_HW_INTRINSICS)
GenTree* gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree);
GenTreeMskCon* gtFoldExprConvertVecCnsToMask(GenTreeHWIntrinsic* tree, GenTreeVecCon* vecCon);
#endif // FEATURE_HW_INTRINSICS

// Options to control behavior of gtTryRemoveBoxUpstreamEffects
Expand Down
144 changes: 53 additions & 91 deletions src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21945,8 +21945,8 @@ GenTree* Compiler::gtNewSimdCvtVectorToMaskNode(var_types type,
#if defined(TARGET_XARCH)
return gtNewSimdHWIntrinsicNode(TYP_MASK, op1, NI_AVX512_ConvertVectorToMask, simdBaseJitType, simdSize);
#elif defined(TARGET_ARM64)
// ConvertVectorToMask uses cmpne which requires an embedded mask.
GenTree* trueMask = gtNewSimdHWIntrinsicNode(TYP_MASK, NI_Sve_ConversionTrueMask, simdBaseJitType, simdSize);
// We use cmpne which requires an embedded mask.
GenTree* trueMask = gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize);
return gtNewSimdHWIntrinsicNode(TYP_MASK, trueMask, op1, NI_Sve_ConvertVectorToMask, simdBaseJitType, simdSize);
#else
#error Unsupported platform
Expand Down Expand Up @@ -32028,7 +32028,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
}

#if defined(FEATURE_MASKED_HW_INTRINSICS)
// Fold ConvertMaskToVector(ConvertVectorToMask(vec)) to vec
if (tree->OperIsConvertMaskToVector())
{
GenTree* op = op1;
Expand Down Expand Up @@ -32061,7 +32060,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
}
}

// Fold ConvertVectorToMask(ConvertMaskToVector(mask)) to mask
if (tree->OperIsConvertVectorToMask())
{
GenTree* op = op1;
Expand All @@ -32070,9 +32068,11 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
#if defined(TARGET_XARCH)
tryHandle = op->OperIsHWIntrinsic();
#elif defined(TARGET_ARM64)
assert(op->OperIsHWIntrinsic(NI_Sve_ConversionTrueMask));
op = op2;
tryHandle = op->OperIsHWIntrinsic();
if (op->OperIsHWIntrinsic(NI_Sve_CreateTrueMaskAll))
{
op = op2;
tryHandle = op->OperIsHWIntrinsic();
}
#endif // TARGET_ARM64

if (tryHandle)
Expand Down Expand Up @@ -32158,12 +32158,53 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)

resultNode = gtNewVconNode(retType, &simdVal);
}
#if defined(TARGET_XARCH)
else if (tree->OperIsConvertVectorToMask())
{
resultNode = gtFoldExprConvertVecCnsToMask(tree, cnsNode->AsVecCon());
}
GenTreeVecCon* vecCon = cnsNode->AsVecCon();
GenTreeMskCon* mskCon = gtNewMskConNode(retType);

switch (vecCon->TypeGet())
{
case TYP_SIMD8:
{
EvaluateSimdCvtVectorToMask<simd8_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd8Val);
break;
}

case TYP_SIMD12:
{
EvaluateSimdCvtVectorToMask<simd12_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd12Val);
break;
}

case TYP_SIMD16:
{
EvaluateSimdCvtVectorToMask<simd16_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd16Val);
break;
}

#if defined(TARGET_XARCH)
case TYP_SIMD32:
{
EvaluateSimdCvtVectorToMask<simd32_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd32Val);
break;
}

case TYP_SIMD64:
{
EvaluateSimdCvtVectorToMask<simd64_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd64Val);
break;
}
#endif // TARGET_XARCH

default:
{
unreached();
}
}

resultNode = mskCon;
}
#endif // FEATURE_MASKED_HW_INTRINSICS
else
{
Expand Down Expand Up @@ -33006,10 +33047,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
switch (ni)
{
#ifdef TARGET_ARM64
case NI_Sve_ConvertVectorToMask:
resultNode = gtFoldExprConvertVecCnsToMask(tree, cnsNode->AsVecCon());
break;

case NI_AdvSimd_MultiplyByScalar:
case NI_AdvSimd_Arm64_MultiplyByScalar:
{
Expand Down Expand Up @@ -33151,18 +33188,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
break;
}

#if defined(TARGET_ARM64)
if (ni == NI_Sve_ConditionalSelect)
{
assert(!op1->IsVectorAllBitsSet() && !op1->IsVectorZero());
}
else
{
assert(!op1->IsTrueMask(simdBaseType) && !op1->IsFalseMask());
}
#endif

if (op1->IsVectorAllBitsSet() || op1->IsTrueMask(simdBaseType))
if (op1->IsVectorAllBitsSet() || op1->IsMaskAllBitsSet())
{
if ((op3->gtFlags & GTF_SIDE_EFFECT) != 0)
{
Expand All @@ -33176,7 +33202,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
return op2;
}

if (op1->IsVectorZero() || op1->IsFalseMask())
if (op1->IsVectorZero())
{
return gtWrapWithSideEffects(op3, op2, GTF_ALL_EFFECT);
}
Expand Down Expand Up @@ -33228,70 +33254,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
}
return resultNode;
}

//------------------------------------------------------------------------------
// gtFoldExprConvertVecCnsToMask: Folds a constant vector plus conversion to
// mask into a constant mask.
//
// Arguments:
// tree - The convert vector to mask node
// vecCon - The vector constant converted by the convert
//
// Return Value:
// Returns a constant mask
//
GenTreeMskCon* Compiler::gtFoldExprConvertVecCnsToMask(GenTreeHWIntrinsic* tree, GenTreeVecCon* vecCon)
{
assert(tree->OperIsConvertVectorToMask());
assert(vecCon == tree->Op(1) || vecCon == tree->Op(2));

var_types retType = tree->TypeGet();
var_types simdBaseType = tree->GetSimdBaseType();
GenTreeMskCon* mskCon = gtNewMskConNode(retType);

switch (vecCon->TypeGet())
{
case TYP_SIMD8:
{
EvaluateSimdCvtVectorToMask<simd8_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd8Val);
break;
}

case TYP_SIMD12:
{
EvaluateSimdCvtVectorToMask<simd12_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd12Val);
break;
}

case TYP_SIMD16:
{
EvaluateSimdCvtVectorToMask<simd16_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd16Val);
break;
}

#if defined(TARGET_XARCH)
case TYP_SIMD32:
{
EvaluateSimdCvtVectorToMask<simd32_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd32Val);
break;
}

case TYP_SIMD64:
{
EvaluateSimdCvtVectorToMask<simd64_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd64Val);
break;
}
#endif // TARGET_XARCH

default:
{
unreached();
}
}

return mskCon;
}

#endif // FEATURE_HW_INTRINSICS

//------------------------------------------------------------------------
Expand Down
64 changes: 36 additions & 28 deletions src/coreclr/jit/gentree.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1802,8 +1802,8 @@ struct GenTree
inline bool IsVectorCreate() const;
inline bool IsVectorAllBitsSet() const;
inline bool IsVectorBroadcast(var_types simdBaseType) const;
inline bool IsTrueMask(var_types simdBaseType) const;
inline bool IsFalseMask() const;
inline bool IsMaskAllBitsSet() const;
inline bool IsMaskZero() const;

inline uint64_t GetIntegralVectorConstElement(size_t index, var_types simdBaseType);

Expand Down Expand Up @@ -9550,46 +9550,54 @@ inline bool GenTree::IsVectorBroadcast(var_types simdBaseType) const
return false;
}

//------------------------------------------------------------------------
// IsTrueMask: Is the given node a true mask
//
// Arguments:
// simdBaseType - the base type of the mask
//
// Returns true if the node is a true mask for the given simdBaseType.
//
// Note that a byte true mask (1111...) is different to an int true mask
// (10001000...), therefore the simdBaseType of the mask needs to be
// taken into account.
//
inline bool GenTree::IsTrueMask(var_types simdBaseType) const
inline bool GenTree::IsMaskAllBitsSet() const
{
#ifdef TARGET_ARM64
// TODO-SVE: For agnostic VL, vector type may not be simd16_t
static_assert_no_msg(AreContiguous(NI_Sve_CreateTrueMaskByte, NI_Sve_CreateTrueMaskDouble,
NI_Sve_CreateTrueMaskInt16, NI_Sve_CreateTrueMaskInt32,
NI_Sve_CreateTrueMaskInt64, NI_Sve_CreateTrueMaskSByte,
NI_Sve_CreateTrueMaskSingle, NI_Sve_CreateTrueMaskUInt16,
NI_Sve_CreateTrueMaskUInt32, NI_Sve_CreateTrueMaskUInt64));

if (IsCnsMsk())
if (OperIsHWIntrinsic())
{
return SveMaskPatternAll == EvaluateSimdMaskToPattern<simd16_t>(simdBaseType, AsMskCon()->gtSimdMaskVal);
NamedIntrinsic id = AsHWIntrinsic()->GetHWIntrinsicId();
if (id == NI_Sve_ConvertMaskToVector)
{
GenTree* op1 = AsHWIntrinsic()->Op(1);
assert(op1->OperIsHWIntrinsic());
id = op1->AsHWIntrinsic()->GetHWIntrinsicId();
}
return ((id == NI_Sve_CreateTrueMaskAll) ||
((id >= NI_Sve_CreateTrueMaskByte) && (id <= NI_Sve_CreateTrueMaskUInt64)));
}
#endif

#endif
return false;
}

//------------------------------------------------------------------------
// IsFalseMask: Is the given node a false mask
//
// Returns true if the node is a false mask, ie all zeros
//
inline bool GenTree::IsFalseMask() const
inline bool GenTree::IsMaskZero() const
{
#ifdef TARGET_ARM64
if (IsCnsMsk())
static_assert_no_msg(AreContiguous(NI_Sve_CreateFalseMaskByte, NI_Sve_CreateFalseMaskDouble,
NI_Sve_CreateFalseMaskInt16, NI_Sve_CreateFalseMaskInt32,
NI_Sve_CreateFalseMaskInt64, NI_Sve_CreateFalseMaskSByte,
NI_Sve_CreateFalseMaskSingle, NI_Sve_CreateFalseMaskUInt16,
NI_Sve_CreateFalseMaskUInt32, NI_Sve_CreateFalseMaskUInt64));

if (OperIsHWIntrinsic())
{
return AsMskCon()->IsZero();
NamedIntrinsic id = AsHWIntrinsic()->GetHWIntrinsicId();
if (id == NI_Sve_ConvertMaskToVector)
{
GenTree* op1 = AsHWIntrinsic()->Op(1);
assert(op1->OperIsHWIntrinsic());
id = op1->AsHWIntrinsic()->GetHWIntrinsicId();
}
return ((id >= NI_Sve_CreateFalseMaskByte) && (id <= NI_Sve_CreateFalseMaskUInt64));
}
#endif

#endif
return false;
}

Expand Down
Loading
Loading