diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 90105004bb4be5..0406bc4a6e19a2 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -6237,10 +6237,12 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_D); // REVW .D, /M, .D // IF_SVE_CV_3A - theEmitter->emitIns_R_R_R(INS_sve_splice, EA_SCALABLE, REG_V0, REG_P0, REG_V30, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // SPLICE ., , {., .} - theEmitter->emitIns_R_R_R(INS_sve_splice, EA_SCALABLE, REG_V3, REG_P7, REG_V27, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // SPLICE ., , {., .} + // TODO-SVE: Currently not supporting the constructive version of splice. Uncomment the tests on closing + // https://github.com/dotnet/runtime/issues/103850. + // theEmitter->emitIns_R_R_R(INS_sve_splice, EA_SCALABLE, REG_V0,REG_P0, REG_V30, INS_OPTS_SCALABLE_B, + // INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // SPLICE ., , {., .} + // theEmitter->emitIns_R_R_R(INS_sve_splice, EA_SCALABLE, REG_V3, REG_P7, REG_V27, INS_OPTS_SCALABLE_D, + // INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // SPLICE ., , {., .} // IF_SVE_CV_3B theEmitter->emitIns_R_R_R(INS_sve_splice, EA_SCALABLE, REG_V1, REG_P1, REG_V29, diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 99e86bb38c15c0..1c4e183f51403f 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -3858,6 +3858,9 @@ void emitter::emitInsSve_R_R_R(instruction ins, assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); + // TODO-SVE: We currently support only the destructive version of splice. Remove the following assert when + // the constructive version is added, as described in https://github.com/dotnet/runtime/issues/103850. + assert(sopt != INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); fmt = (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) ? IF_SVE_CV_3A : IF_SVE_CV_3B; break; @@ -10294,7 +10297,7 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) + case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (constructive) case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V<4, 0>(id->idReg1()); // ddddd @@ -13257,7 +13260,7 @@ void emitter::emitInsSveSanityCheck(instrDesc* id) assert(isScalableVectorSize(id->idOpSize())); break; - case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) + case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (constructive) case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) assert(isScalableVectorSize(id->idOpSize())); // xx assert(insOptsScalableStandard(id->idInsOpt())); @@ -14944,7 +14947,7 @@ void emitter::emitDispInsSveHelp(instrDesc* id) break; // ., , {., .} - case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) + case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (constructive) emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), id->idInsOpt(), true); // VVV emitDispSveConsecutiveRegList(id->idReg3(), insGetSveReg1ListSize(ins), id->idInsOpt(), false); // nnnnn @@ -16805,7 +16808,7 @@ void emitter::getInsSveExecutionCharacteristics(instrDesc* id, insExecutionChara result.insLatency = PERFSCORE_LATENCY_140C; break; - case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) + case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (constructive) case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) result.insLatency = PERFSCORE_LATENCY_3C; result.insThroughput = PERFSCORE_THROUGHPUT_1C; diff --git a/src/coreclr/jit/emitfmtsarm64sve.h b/src/coreclr/jit/emitfmtsarm64sve.h index 91b409bc7f4fd7..662126efb331df 100644 --- a/src/coreclr/jit/emitfmtsarm64sve.h +++ b/src/coreclr/jit/emitfmtsarm64sve.h @@ -221,7 +221,7 @@ IF_DEF(SVE_CR_3A, IS_NONE, NONE) // SVE_CR_3A ........xx...... ...gggnnnnnddd IF_DEF(SVE_CS_3A, IS_NONE, NONE) // SVE_CS_3A ........xx...... ...gggnnnnnddddd -- SVE extract element to general register IF_DEF(SVE_CT_3A, IS_NONE, NONE) // SVE_CT_3A ................ ...gggnnnnnddddd -- SVE reverse doublewords IF_DEF(SVE_CU_3A, IS_NONE, NONE) // SVE_CU_3A ........xx...... ...gggnnnnnddddd -- SVE reverse within elements -IF_DEF(SVE_CV_3A, IS_NONE, NONE) // SVE_CV_3A ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) +IF_DEF(SVE_CV_3A, IS_NONE, NONE) // SVE_CV_3A ........xx...... ...VVVnnnnnddddd -- SVE vector splice (constructive) IF_DEF(SVE_CV_3B, IS_NONE, NONE) // SVE_CV_3B ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) IF_DEF(SVE_CW_4A, IS_NONE, NONE) // SVE_CW_4A ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) IF_DEF(SVE_CX_4A, IS_NONE, NONE) // SVE_CX_4A ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index a729a20fba9174..5a6649eeaafbd4 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -853,15 +853,31 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case 3: if (isRMW) { - if (targetReg != op1Reg) + if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) { - assert(targetReg != op2Reg); - assert(targetReg != op3Reg); + if (targetReg != op2Reg) + { + assert(targetReg != op1Reg); + assert(targetReg != op3Reg); - GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, - /* canSkip */ true); + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op2Reg, + /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op3Reg, opt); + } + else + { + if (targetReg != op1Reg) + { + assert(targetReg != op2Reg); + assert(targetReg != op3Reg); + + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, + /* canSkip */ true); + } + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, opt); } - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, opt); } else { diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 9eb583e5ff4da2..62b8ab7dd60dac 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -198,6 +198,7 @@ HARDWARE_INTRINSIC(Sve, SignExtend32, HARDWARE_INTRINSIC(Sve, SignExtend8, -1, -1, false, {INS_invalid, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, SignExtendWideningLower, -1, 1, true, {INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Sve, SignExtendWideningUpper, -1, 1, true, {INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Splice, -1, 3, true, {INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, Sqrt, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fsqrt, INS_sve_fsqrt}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, StoreAndZip, -1, 3, true, {INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, StoreNarrowing, -1, 3, true, {INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_invalid, INS_invalid}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 133bbd52b985d9..c7084efc96a8e7 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1508,6 +1508,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou const bool isRMW = intrinsicTree->isRMWHWIntrinsic(compiler); bool tgtPrefOp1 = false; + bool tgtPrefOp2 = false; bool delayFreeMultiple = false; if (intrin.op1 != nullptr) { @@ -1562,9 +1563,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou // If we have an RMW intrinsic or an intrinsic with simple move semantic between two SIMD registers, // we want to preference op1Reg to the target if op1 is not contained. - if (isRMW || simdRegToSimdRegMove) + + if ((isRMW || simdRegToSimdRegMove)) { - tgtPrefOp1 = !intrin.op1->isContained(); + if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) + { + assert(!simdRegToSimdRegMove); + // Prefer op2Reg for the masked operation as mask would be the op1Reg + tgtPrefOp2 = !intrin.op1->isContained(); + } + else + { + tgtPrefOp1 = !intrin.op1->isContained(); + } } if (delayFreeMultiple) @@ -1947,6 +1958,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou (argNum == lowVectorOperandNum) ? lowVectorCandidates : RBM_NONE); } } + else if (tgtPrefOp2) + { + if (!intrin.op2->isContained()) + { + assert(tgtPrefUse == nullptr); + tgtPrefUse2 = BuildUse(intrin.op2); + srcCount++; + } + else + { + srcCount += BuildOperandUses(intrin.op2); + } + } else { switch (intrin.id) @@ -1990,12 +2014,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { SingleTypeRegSet candidates = lowVectorOperandNum == 3 ? lowVectorCandidates : RBM_NONE; - srcCount += isRMW ? BuildDelayFreeUses(intrin.op3, intrin.op1, candidates) - : BuildOperandUses(intrin.op3, candidates); + if (isRMW) + { + srcCount += BuildDelayFreeUses(intrin.op3, (tgtPrefOp2 ? intrin.op2 : intrin.op1), candidates); + } + else + { + srcCount += BuildOperandUses(intrin.op3, candidates); + } if (intrin.op4 != nullptr) { assert(lowVectorOperandNum != 4); + assert(!tgtPrefOp2); srcCount += isRMW ? BuildDelayFreeUses(intrin.op4, intrin.op1) : BuildOperandUses(intrin.op4); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 844fb507b0a373..771e9901dfa024 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -5496,6 +5496,69 @@ internal Arm64() { } public static unsafe Vector SignExtend8(Vector value) { throw new PlatformNotSupportedException(); } + /// Splice two vectors under predicate control + + /// + /// svuint8_t svsplice[_u8](svbool_t pg, svuint8_t op1, svuint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svsplice[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svsplice[_s16](svbool_t pg, svint16_t op1, svint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svsplice[_s32](svbool_t pg, svint32_t op1, svint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svsplice[_s64](svbool_t pg, svint64_t op1, svint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint8_t svsplice[_s8](svbool_t pg, svint8_t op1, svint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svsplice[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svsplice[_u16](svbool_t pg, svuint16_t op1, svuint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svsplice[_u32](svbool_t pg, svuint32_t op1, svuint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svsplice[_u64](svbool_t pg, svuint64_t op1, svuint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// Sqrt : Square root /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index a5cb2e3d09a368..bf6d02924f09e8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -5580,6 +5580,69 @@ internal Arm64() { } public static unsafe Vector SignExtendWideningUpper(Vector value) => SignExtendWideningUpper(value); + /// Splice two vectors under predicate control + + /// + /// svuint8_t svsplice[_u8](svbool_t pg, svuint8_t op1, svuint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svfloat64_t svsplice[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint16_t svsplice[_s16](svbool_t pg, svint16_t op1, svint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint32_t svsplice[_s32](svbool_t pg, svint32_t op1, svint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint64_t svsplice[_s64](svbool_t pg, svint64_t op1, svint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint8_t svsplice[_s8](svbool_t pg, svint8_t op1, svint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svfloat32_t svsplice[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svuint16_t svsplice[_u16](svbool_t pg, svuint16_t op1, svuint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svuint32_t svsplice[_u32](svbool_t pg, svuint32_t op1, svuint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svuint64_t svsplice[_u64](svbool_t pg, svuint64_t op1, svuint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// Sqrt : Square root /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 8f9fad52775b25..0dc97c4a1102c0 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4989,6 +4989,17 @@ internal Arm64() { } public static System.Numerics.Vector SignExtendWideningUpper(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector SignExtendWideningUpper(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Sqrt(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector Sqrt(System.Numerics.Vector value) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 348e781453f1cf..d0157d8c755907 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -186,6 +186,7 @@ ("_SveTernOpTestTemplate.template", "SveVecTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveTernOpFirstArgTestTemplate.template", "SveVecTernOpFirstArgTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveImmTernOpTestTemplate.template", "SveVecImmTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), + ("_SveTernOpMaskedOpTestTemplate.template", "SveVecTernOpMaskedTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveImmTernOpFirstArgTestTemplate.template", "SveVecImmTernOpFirstArgTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveImm2UnaryOpTestTemplate.template", "SveVecImm2UnOpTest.template", new Dictionary { ["TemplateName"] = "Imm", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }), ("_SveMinimalUnaryOpTestTemplate.template", "SveVecReduceUnOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = VecReduceOpTest_ValidationLogic }), @@ -3745,6 +3746,17 @@ ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Sqrt_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Sqrt", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "result[i] != Helpers.Sqrt(firstOp[i])", ["GetIterResult"] = "Helpers.Sqrt(leftOp[i])"}), ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Sqrt_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Sqrt", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "result[i] != Helpers.Sqrt(firstOp[i])", ["GetIterResult"] = "Helpers.Sqrt(leftOp[i])"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = "BitConverter.DoubleToInt64Bits"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve_Splice_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Subtract_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Subtract", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "Helpers.Subtract(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Subtract(left[i], right[i])"}), ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Subtract_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Subtract", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "Helpers.Subtract(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Subtract(left[i], right[i])"}), ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Subtract_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Subtract", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(sbyte)TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "Helpers.Subtract(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Subtract(left[i], right[i])"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 7ae4e080edbbd1..d60a01a3a0fc53 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -7045,5 +7045,265 @@ public static double LoadDoubleFromByteArray(byte[] array, ulong offset) return BitConverter.Int64BitsToDouble(ret); } + public static Byte Splice(Byte[] first, Byte[] second, Byte[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static double Splice(double[] first, double[] second, double[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (Double.IsNaN(maskArray[i]) || maskArray[i] > 0.0d) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static float Splice(float[] first, float[] second, float[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0.0f) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static Int16 Splice(Int16[] first, Int16[] second, Int16[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static Int32 Splice(Int32[] first, Int32[] second, Int32[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static Int64 Splice(Int64[] first, Int64[] second, Int64[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static SByte Splice(SByte[] first, SByte[] second, SByte[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static UInt16 Splice(UInt16[] first, UInt16[] second, UInt16[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static UInt32 Splice(UInt32[] first, UInt32[] second, UInt32[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static ulong Splice(ulong[] first, ulong[] second, ulong[] maskArray, int index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + } } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template new file mode 100644 index 00000000000000..84935d765458b7 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template @@ -0,0 +1,446 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm._{Isa} +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {Method}Test__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({Isa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {Method}Test__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] outArray; + private byte[] maskArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle outHandle; + private GCHandle maskHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {RetBaseType}[] outArray, {Op1BaseType}[] maskArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + int sizeOfmaskArray = maskArray.Length * Unsafe.SizeOf<{Op1BaseType}>(); + + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfoutArray || (alignment * 2) < sizeOfmaskArray) + { + throw new ArgumentException("Invalid value of alignment"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + this.maskArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + this.maskHandle = GCHandle.Alloc(this.maskArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(maskArrayPtr), ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), (uint)sizeOfmaskArray); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* maskArrayPtr => Align((byte*)(maskHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + outHandle.Free(); + maskHandle.Free(); + } + + public static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + + private struct TestStruct + { + public Vector<{Op1BaseType}> _fld1; + public Vector<{Op2BaseType}> _fld2; + public Vector<{Op1BaseType}> _storeMask; + private GCHandle _outHandle; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf>()); + for (var i = 0; i < MaskElementCount; i++) { _maskData[i] = ({Op1BaseType})(Helpers.getMask{Op1BaseType}()); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref testStruct._storeMask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf>()); + + ulong alignment = (ulong)LargestVectorSize; + byte[] _outArray = new byte[alignment * 2]; + testStruct._outHandle = GCHandle.Alloc(_outArray, GCHandleType.Pinned); + + return testStruct; + } + + public void* _outArrayPtr => DataTable.Align((byte*)(_outHandle.AddrOfPinnedObject().ToPointer()), (ulong)LargestVectorSize); + + public void Dispose() + { + _outHandle.Free(); + } + + public void RunStructFldScenario({Method}Test__{TestName} testClass) + { + var result = {Isa}.{Method}(_storeMask, _fld1, _fld2); + + testClass.ValidateResult(_fld1, _fld2, result, _storeMask); + } + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_FalseOp)); + ConditionalSelectScenario(_mask, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_ZeroOp)); + ConditionalSelectScenario(_mask, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(mask, op1, op2), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, op2, falseOp, _dataTable.outArrayPtr); + } + + private static readonly int LargestVectorSize = 64; + + private static readonly int Op1ElementCount = Unsafe.SizeOf>() / sizeof({Op1BaseType}); + private static readonly int Op2ElementCount = Unsafe.SizeOf>() / sizeof({Op2BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf>() / sizeof({RetBaseType}); + private static readonly int MaskElementCount = Unsafe.SizeOf>() / sizeof({Op1BaseType}); + + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[MaskElementCount]; + + private Vector<{Op1BaseType}> _fld1; + private Vector<{Op2BaseType}> _fld2; + private Vector<{Op1BaseType}> _mask; + private Vector<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public {Method}Test__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + for (var i = 0; i < MaskElementCount; i++) { _maskData[i] = ({Op1BaseType})(Helpers.getMask{Op1BaseType}()); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + + _dataTable = new DataTable(_data1, _data2, new {RetBaseType}[RetElementCount], _maskData, LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}(Unsafe.Read>(_dataTable.maskArrayPtr), Unsafe.Read>(_dataTable.inArray1Ptr), Unsafe.Read>(_dataTable.inArray2Ptr)); + + Unsafe.Write(_dataTable.outArrayPtr, (Vector<{RetBaseType}>)result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr, _dataTable.maskArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + Vector<{Op1BaseType}> loadMask = {Isa}.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}({LoadIsa}.LoadVector(loadMask, ({Op1BaseType}*)(_dataTable.maskArrayPtr)), {LoadIsa}.LoadVector(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), {LoadIsa}.LoadVector(loadMask, ({Op2BaseType}*)(_dataTable.inArray2Ptr))); + + Unsafe.Write(_dataTable.outArrayPtr, (Vector<{RetBaseType}>)result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr, _dataTable.maskArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var first = Unsafe.Read>(_dataTable.inArray1Ptr); + var second = Unsafe.Read>(_dataTable.inArray2Ptr); + var mask = Unsafe.Read>(_dataTable.maskArrayPtr); + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof(Vector<{Op1BaseType}>), typeof(Vector<{Op1BaseType}>), typeof(Vector<{Op2BaseType}>) }) + .Invoke(null, new object[] { mask, + first, second + }); + Unsafe.Write(_dataTable.outArrayPtr, (Vector<{RetBaseType}>)result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr, _dataTable.maskArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + Vector<{Op1BaseType}> op1 = Unsafe.Read>(_dataTable.inArray1Ptr); + Vector<{Op2BaseType}> op2 = Unsafe.Read>(_dataTable.inArray2Ptr); + Vector<{RetBaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); + + var result = {Isa}.{Method}(storeMask, op1, op2); + + ValidateResult(op1, op2, result, storeMask); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + Vector<{Op1BaseType}> op1 = Unsafe.Read>(_dataTable.inArray1Ptr); + Vector<{Op2BaseType}> op2 = Unsafe.Read>(_dataTable.inArray2Ptr); + Vector<{RetBaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); + + var result = {Isa}.{Method}(storeMask, op1, op2); + + ValidateResult(op1, op2, result, storeMask); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._storeMask, test._fld1, test._fld2); + + ValidateResult(test._fld1, test._fld2, result, test._storeMask); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult(Vector<{Op1BaseType}> maskOp, Vector<{Op1BaseType}> leftOp, Vector<{Op2BaseType}> rightOp, Vector<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] right = new {Op2BaseType}[Op2ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref right[0]), rightOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf>()); + + bool succeeded = true; + + {TemplateValidationLogicForCndSel} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}<{Op1BaseType}>(Vector<{Op1BaseType}>, Vector<{Op1BaseType}>, Vector<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateResult(Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{RetBaseType}> result, Vector<{Op1BaseType}> storeMask, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), storeMask); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), result); + + ValidateResult(inArray1, inArray2, outArray, maskArray, method); + } + + private void ValidateResult(void* op1, void* op2, void* result, void* mask, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), ref Unsafe.AsRef(mask), (uint)Unsafe.SizeOf>()); + + ValidateResult(inArray1, inArray2, outArray, maskArray, method); + } + + private void ValidateResult({Op1BaseType}[] first, {Op2BaseType}[] second, {RetBaseType}[] result, {Op1BaseType}[] maskArray, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>(Vector<{Op1BaseType}>, Vector<{Op1BaseType}>, Vector<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" first: ({string.Join(", ", first)})"); + TestLibrary.TestFramework.LogInformation($" second: ({string.Join(", ", second)})"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", maskArray)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +}