diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 83e367392ac9c7..cc21be120234a6 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -18690,6 +18690,701 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) // "predicated using the same governing predicate register and source element size as this instruction." assert(firstId->idInsOpt() == secondId->idInsOpt()); } + + // The following instructions cannot use predicated movprfx, else the behaviour will be unpredictable. + switch (secondId->idIns()) + { + case INS_sve_sqdecd: + case INS_sve_sqdech: + case INS_sve_sqdecw: + case INS_sve_sqincd: + case INS_sve_sqinch: + case INS_sve_sqincw: + case INS_sve_uqdecd: + case INS_sve_uqdech: + case INS_sve_uqdecw: + case INS_sve_uqincd: + case INS_sve_uqinch: + case INS_sve_uqincw: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_BP_1A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_smlalb: + case INS_sve_smlalt: + case INS_sve_smlslb: + case INS_sve_smlslt: + case INS_sve_umlalb: + case INS_sve_umlalt: + case INS_sve_umlslb: + case INS_sve_umlslt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EL_3A: + case IF_SVE_FG_3A: + case IF_SVE_FG_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_add: + case INS_sve_sqadd: + case INS_sve_sqsub: + case INS_sve_sub: + case INS_sve_subr: + case INS_sve_uqadd: + case INS_sve_uqsub: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EC_1A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_and: + case INS_sve_bic: + case INS_sve_eon: + case INS_sve_eor: + case INS_sve_orn: + case INS_sve_orr: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_BS_1A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_bcax: + case INS_sve_bsl: + case INS_sve_bsl1n: + case INS_sve_bsl2n: + case INS_sve_eor3: + case INS_sve_nbsl: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_AV_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + case INS_sve_fmlslb: + case INS_sve_fmlslt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GZ_3A: + case IF_SVE_HB_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_decd: + case INS_sve_dech: + case INS_sve_decw: + case INS_sve_incd: + case INS_sve_inch: + case INS_sve_incw: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_BN_1A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_sabalb: + case INS_sve_sabalt: + case INS_sve_sqdmlalbt: + case INS_sve_sqdmlslbt: + case INS_sve_uabalb: + case INS_sve_uabalt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EL_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_addp: + case INS_sve_smaxp: + case INS_sve_sminp: + case INS_sve_umaxp: + case INS_sve_uminp: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_AA_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_eorbt: + case INS_sve_eortb: + case INS_sve_fclamp: + case INS_sve_sclamp: + case INS_sve_uclamp: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_AT_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_faddp: + case INS_sve_fmaxnmp: + case INS_sve_fmaxp: + case INS_sve_fminnmp: + case INS_sve_fminp: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GR_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_adclb: + case INS_sve_adclt: + case INS_sve_sbclb: + case INS_sve_sbclt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_FY_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_fmlallbb: + case INS_sve_fmlallbt: + case INS_sve_fmlalltb: + case INS_sve_fmlalltt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GO_3A: + case IF_SVE_HC_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_smax: + case INS_sve_smin: + case INS_sve_umax: + case INS_sve_umin: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_ED_1A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_sqdecp: + case INS_sve_sqincp: + case INS_sve_uqdecp: + case INS_sve_uqincp: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_DP_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_sqdmlalb: + case INS_sve_sqdmlalt: + case INS_sve_sqdmlslb: + case INS_sve_sqdmlslt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EL_3A: + case IF_SVE_FJ_3A: + case IF_SVE_FJ_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_srsra: + case INS_sve_ssra: + case INS_sve_ursra: + case INS_sve_usra: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_FU_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_smmla: + case INS_sve_ummla: + case INS_sve_usmmla: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_FO_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_bfmla: + case INS_sve_bfmls: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GU_3C: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_cadd: + case INS_sve_sqcadd: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_FV_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_clasta: + case INS_sve_clastb: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_CM_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_decp: + case INS_sve_incp: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_DN_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_fmla: + case INS_sve_fmls: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GU_3A: + case IF_SVE_GU_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_fmlalb: + case INS_sve_fmlalt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GM_3A: + case IF_SVE_GN_3A: + case IF_SVE_GZ_3A: + case IF_SVE_HB_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_mla: + case INS_sve_mls: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_FF_3A: + case IF_SVE_FF_3B: + case IF_SVE_FF_3C: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_saba: + case INS_sve_uaba: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_FW_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_sdot: + case INS_sve_udot: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EF_3A: + case IF_SVE_EG_3A: + case IF_SVE_EH_3A: + case IF_SVE_EY_3A: + case IF_SVE_EY_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EM_3A: + case IF_SVE_FK_3A: + case IF_SVE_FK_3B: + case IF_SVE_FK_3C: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_bfclamp: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GW_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_bfdot: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GY_3B: + case IF_SVE_HA_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_bfmmla: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_HD_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_cdot: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EJ_3A: + case IF_SVE_FA_3A: + case IF_SVE_FA_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_cmla: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EK_3A: + case IF_SVE_FB_3A: + case IF_SVE_FB_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_extq: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_BY_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_fcmla: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GV_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_fdot: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_GY_3A: + case IF_SVE_GY_3B: + case IF_SVE_GY_3B_D: + case IF_SVE_HA_3A: + case IF_SVE_HA_3A_E: + case IF_SVE_HA_3A_F: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_fmmla: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_HD_3A_A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_ftmad: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_HN_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_insr: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_CC_2A: + case IF_SVE_CD_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_madpt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EW_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_mlapt: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EW_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_mul: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EE_1A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_revd: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_CT_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_sqrdcmlah: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EK_3A: + case IF_SVE_FC_3A: + case IF_SVE_FC_3B: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_sudot: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EZ_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_usdot: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_EI_3A: + case IF_SVE_EZ_3A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case INS_sve_xar: + { + switch (secondId->idInsFmt()) + { + case IF_SVE_AW_2A: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + default: + break; + } } #endif // DEBUG diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 13c7163c4fcbb1..81ecbc07fa1bd1 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -783,6 +783,21 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitInsHelper(targetReg, maskReg, embMaskOp2Reg); break; + case NI_Sve2_AddPairwise: + case NI_Sve2_MaxNumberPairwise: + case NI_Sve2_MaxPairwise: + case NI_Sve2_MinNumberPairwise: + case NI_Sve2_MinPairwise: + // These instructions have unpredictable behaviour when using predicated movprfx, + // so the unpredicated variant must be used here. + assert(!intrin.op3->isContained() && falseReg != REG_NA); + GetEmitter()->emitIns_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, embMaskOp1Reg); + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, + embOpt, sopt); + GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, + falseReg, opt); + break; + default: assert(targetReg != embMaskOp2Reg); diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index f1d450661bdbcd..a75814b38d3cc4 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -4061,8 +4061,23 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) // When we are merging with zero, we can specialize // and avoid instantiating the vector constant. // Do this only if op1 was AllTrueMask - MakeSrcContained(node, op3); - LABELEDDISPTREERANGE("Contained false mask op3 in ConditionalSelect", BlockRange(), op3); + switch (op2->AsHWIntrinsic()->GetHWIntrinsicId()) + { + case NI_Sve2_AddPairwise: + case NI_Sve2_MaxNumberPairwise: + case NI_Sve2_MaxPairwise: + case NI_Sve2_MinNumberPairwise: + case NI_Sve2_MinPairwise: + // This is an edge case where these instructions have unpredictable behaviour when + // using predicated movprfx, so the unpredicated variant must be used here. This + // prevents us from performing this optimization as we will need the constant vector + // for masking the result. + break; + + default: + MakeSrcContained(node, op3); + LABELEDDISPTREERANGE("Contained false mask op3 in ConditionalSelect", BlockRange(), op3); + } } break;