diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h index c760fe7ef99dd..c705936b43ab5 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -227,6 +227,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector { bool selectScaleOffset(MachineOperand &Root, Register &Offset, bool IsSigned) const; + bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset, int64_t *Offset, bool *ScaleOffset) const; InstructionSelector::ComplexRendererFns diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 1a13b2226ecd6..b26121ccd980f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -2094,9 +2094,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .clampScalar(0, S32, S64) .lower(); - getActionDefinitionsBuilder({G_ROTR, G_ROTL}) - .scalarize(0) - .lower(); + getActionDefinitionsBuilder(G_ROTR).legalFor({S32}).scalarize(0).lower(); + + getActionDefinitionsBuilder(G_ROTL).scalarize(0).lower(); auto &FSHRActionDefs = getActionDefinitionsBuilder(G_FSHR); FSHRActionDefs.legalFor({{S32, S32}}) diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index 7ed026ee5f69e..e16bfe3610a77 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -4131,6 +4131,12 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_AMDGPU_SMED3: case AMDGPU::G_AMDGPU_FMED3: return getDefaultMappingVOP(MI); + case AMDGPU::G_ROTR: + case AMDGPU::G_ROTL: { + if (isSALUMapping(MI)) + return getDefaultMappingSOP(MI); + return getDefaultMappingVOP(MI); + } case AMDGPU::G_UMULH: case AMDGPU::G_SMULH: { if (Subtarget.hasScalarMulHiInsts() && isSALUMapping(MI)) diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 6cc9b3cc67530..3769e5cb3b891 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -2672,6 +2672,14 @@ def : AMDGPUPat < $src1), sub1) >; +def : GCNPat<(UniformBinFrag i32:$src0, i32:$src1), + (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src0, sub0, $src0, sub1), (S_AND_B32 $src1, (i32 31))), sub0)) +>; + +def : GCNPat<(UniformBinFrag i32:$src0, (i32 ShiftAmt32Imm:$src1)), + (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src0, sub0, $src0, sub1), $src1), sub0)) +>; + let True16Predicate = NotHasTrue16BitInsts in { let SubtargetPredicate = isNotGFX9Plus in { def : ROTRPattern ; diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-rotl-rotr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-rotl-rotr.mir index 7fdee12315754..9610caa1f2012 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-rotl-rotr.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-rotl-rotr.mir @@ -181,8 +181,8 @@ body: | ; GFX-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1 ; GFX-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]] - ; GFX-NEXT: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY]], [[SUB]](s32) - ; GFX-NEXT: $sgpr0 = COPY [[FSHR]](s32) + ; GFX-NEXT: [[ROTR:%[0-9]+]]:_(s32) = G_ROTR [[COPY]], [[SUB]](s32) + ; GFX-NEXT: $sgpr0 = COPY [[ROTR]](s32) %0:_(s32) = COPY $sgpr0 %1:_(s32) = COPY $sgpr1 %2:_(s32) = G_ROTL %0, %1(s32) @@ -301,14 +301,14 @@ body: | ; GFX-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>) ; GFX-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 ; GFX-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV4]] - ; GFX-NEXT: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV]], [[SUB]](s32) + ; GFX-NEXT: [[ROTR:%[0-9]+]]:_(s32) = G_ROTR [[UV]], [[SUB]](s32) ; GFX-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV5]] - ; GFX-NEXT: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV1]], [[SUB1]](s32) + ; GFX-NEXT: [[ROTR1:%[0-9]+]]:_(s32) = G_ROTR [[UV1]], [[SUB1]](s32) ; GFX-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV6]] - ; GFX-NEXT: [[FSHR2:%[0-9]+]]:_(s32) = G_FSHR [[UV2]], [[UV2]], [[SUB2]](s32) + ; GFX-NEXT: [[ROTR2:%[0-9]+]]:_(s32) = G_ROTR [[UV2]], [[SUB2]](s32) ; GFX-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV7]] - ; GFX-NEXT: [[FSHR3:%[0-9]+]]:_(s32) = G_FSHR [[UV3]], [[UV3]], [[SUB3]](s32) - ; GFX-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FSHR]](s32), [[FSHR1]](s32), [[FSHR2]](s32), [[FSHR3]](s32) + ; GFX-NEXT: [[ROTR3:%[0-9]+]]:_(s32) = G_ROTR [[UV3]], [[SUB3]](s32) + ; GFX-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[ROTR]](s32), [[ROTR1]](s32), [[ROTR2]](s32), [[ROTR3]](s32) ; GFX-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>) %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 %1:_(<4 x s32>) = COPY $sgpr4_sgpr5_sgpr6_sgpr7 @@ -391,8 +391,8 @@ body: | ; GFX-NEXT: {{ $}} ; GFX-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0 ; GFX-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1 - ; GFX-NEXT: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY]], [[COPY1]](s32) - ; GFX-NEXT: $sgpr0 = COPY [[FSHR]](s32) + ; GFX-NEXT: [[ROTR:%[0-9]+]]:_(s32) = G_ROTR [[COPY]], [[COPY1]](s32) + ; GFX-NEXT: $sgpr0 = COPY [[ROTR]](s32) %0:_(s32) = COPY $sgpr0 %1:_(s32) = COPY $sgpr1 %2:_(s32) = G_ROTR %0, %1(s32) @@ -452,11 +452,11 @@ body: | ; GFX-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr4_sgpr5_sgpr6_sgpr7 ; GFX-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>) ; GFX-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>) - ; GFX-NEXT: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV]], [[UV4]](s32) - ; GFX-NEXT: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV1]], [[UV5]](s32) - ; GFX-NEXT: [[FSHR2:%[0-9]+]]:_(s32) = G_FSHR [[UV2]], [[UV2]], [[UV6]](s32) - ; GFX-NEXT: [[FSHR3:%[0-9]+]]:_(s32) = G_FSHR [[UV3]], [[UV3]], [[UV7]](s32) - ; GFX-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FSHR]](s32), [[FSHR1]](s32), [[FSHR2]](s32), [[FSHR3]](s32) + ; GFX-NEXT: [[ROTR:%[0-9]+]]:_(s32) = G_ROTR [[UV]], [[UV4]](s32) + ; GFX-NEXT: [[ROTR1:%[0-9]+]]:_(s32) = G_ROTR [[UV1]], [[UV5]](s32) + ; GFX-NEXT: [[ROTR2:%[0-9]+]]:_(s32) = G_ROTR [[UV2]], [[UV6]](s32) + ; GFX-NEXT: [[ROTR3:%[0-9]+]]:_(s32) = G_ROTR [[UV3]], [[UV7]](s32) + ; GFX-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[ROTR]](s32), [[ROTR1]](s32), [[ROTR2]](s32), [[ROTR3]](s32) ; GFX-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>) %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 %1:_(<4 x s32>) = COPY $sgpr4_sgpr5_sgpr6_sgpr7 diff --git a/llvm/test/CodeGen/AMDGPU/rotl.ll b/llvm/test/CodeGen/AMDGPU/rotl.ll index 0a1d15bf945f9..a7fcb6439703a 100644 --- a/llvm/test/CodeGen/AMDGPU/rotl.ll +++ b/llvm/test/CodeGen/AMDGPU/rotl.ll @@ -25,12 +25,14 @@ define amdgpu_kernel void @rotl_i32(ptr addrspace(1) %in, i32 %x, i32 %y) { ; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; SI-NEXT: s_mov_b32 s7, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_sub_i32 s3, 32, s3 +; SI-NEXT: s_sub_i32 s4, 32, s3 +; SI-NEXT: s_mov_b32 s3, s2 +; SI-NEXT: s_and_b32 s4, s4, 31 +; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 ; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_mov_b32 s4, s0 ; SI-NEXT: s_mov_b32 s5, s1 -; SI-NEXT: v_mov_b32_e32 v0, s3 -; SI-NEXT: v_alignbit_b32 v0, s2, s2, v0 +; SI-NEXT: v_mov_b32_e32 v0, s2 ; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -38,11 +40,13 @@ define amdgpu_kernel void @rotl_i32(ptr addrspace(1) %in, i32 %x, i32 %y) { ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_sub_i32 s3, 32, s3 -; GFX8-NEXT: v_mov_b32_e32 v0, s3 -; GFX8-NEXT: v_alignbit_b32 v2, s2, s2, v0 +; GFX8-NEXT: s_sub_i32 s4, 32, s3 +; GFX8-NEXT: s_mov_b32 s3, s2 +; GFX8-NEXT: s_and_b32 s4, s4, 31 +; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 +; GFX8-NEXT: v_mov_b32_e32 v2, s2 ; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: s_endpgm ; @@ -52,18 +56,24 @@ define amdgpu_kernel void @rotl_i32(ptr addrspace(1) %in, i32 %x, i32 %y) { ; GFX10-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_sub_i32 s3, 32, s3 -; GFX10-NEXT: v_alignbit_b32 v1, s2, s2, s3 +; GFX10-NEXT: s_and_b32 s4, s3, 31 +; GFX10-NEXT: s_mov_b32 s3, s2 +; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 +; GFX10-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: rotl_i32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v0, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_sub_i32 s3, 32, s3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: s_and_b32 s4, s3, 31 +; GFX11-NEXT: s_mov_b32 s3, s2 +; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: v_alignbit_b32 v1, s2, s2, s3 +; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-NEXT: s_endpgm entry: @@ -97,14 +107,19 @@ define amdgpu_kernel void @rotl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb ; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9 ; SI-NEXT: s_mov_b32 s7, 0xf000 -; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_sub_i32 s3, 32, s3 -; SI-NEXT: s_sub_i32 s2, 32, s2 -; SI-NEXT: v_mov_b32_e32 v0, s3 -; SI-NEXT: v_alignbit_b32 v1, s1, s1, v0 -; SI-NEXT: v_mov_b32_e32 v0, s2 -; SI-NEXT: v_alignbit_b32 v0, s0, s0, v0 +; SI-NEXT: s_sub_i32 s6, 32, s2 +; SI-NEXT: s_sub_i32 s8, 32, s3 +; SI-NEXT: s_mov_b32 s2, s1 +; SI-NEXT: s_mov_b32 s3, s1 +; SI-NEXT: s_mov_b32 s1, s0 +; SI-NEXT: s_and_b32 s8, s8, 31 +; SI-NEXT: s_and_b32 s6, s6, 31 +; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s8 +; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s6 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: v_mov_b32_e32 v1, s2 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -113,13 +128,18 @@ define amdgpu_kernel void @rotl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_sub_i32 s2, 32, s2 -; GFX8-NEXT: s_sub_i32 s3, 32, s3 -; GFX8-NEXT: v_mov_b32_e32 v0, s3 -; GFX8-NEXT: v_mov_b32_e32 v2, s2 -; GFX8-NEXT: v_alignbit_b32 v1, s1, s1, v0 -; GFX8-NEXT: v_alignbit_b32 v0, s0, s0, v2 +; GFX8-NEXT: s_sub_i32 s7, 32, s3 +; GFX8-NEXT: s_sub_i32 s6, 32, s2 +; GFX8-NEXT: s_mov_b32 s2, s1 +; GFX8-NEXT: s_mov_b32 s3, s1 +; GFX8-NEXT: s_and_b32 s1, s7, 31 +; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s1 +; GFX8-NEXT: s_and_b32 s3, s6, 31 +; GFX8-NEXT: s_mov_b32 s1, s0 +; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s3 ; GFX8-NEXT: v_mov_b32_e32 v2, s4 +; GFX8-NEXT: v_mov_b32_e32 v0, s0 +; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: v_mov_b32_e32 v3, s5 ; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GFX8-NEXT: s_endpgm @@ -131,10 +151,17 @@ define amdgpu_kernel void @rotl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_sub_i32 s3, 32, s3 -; GFX10-NEXT: s_sub_i32 s2, 32, s2 -; GFX10-NEXT: v_alignbit_b32 v1, s1, s1, s3 -; GFX10-NEXT: v_alignbit_b32 v0, s0, s0, s2 +; GFX10-NEXT: s_sub_i32 s4, 32, s2 +; GFX10-NEXT: s_sub_i32 s5, 32, s3 +; GFX10-NEXT: s_mov_b32 s2, s1 +; GFX10-NEXT: s_mov_b32 s3, s1 +; GFX10-NEXT: s_mov_b32 s1, s0 +; GFX10-NEXT: s_and_b32 s4, s4, 31 +; GFX10-NEXT: s_and_b32 s5, s5, 31 +; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s4 +; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s5 +; GFX10-NEXT: v_mov_b32_e32 v0, s0 +; GFX10-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] ; GFX10-NEXT: s_endpgm ; @@ -143,12 +170,19 @@ define amdgpu_kernel void @rotl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c ; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v2, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_sub_i32 s3, 32, s3 -; GFX11-NEXT: s_sub_i32 s2, 32, s2 -; GFX11-NEXT: v_alignbit_b32 v1, s1, s1, s3 -; GFX11-NEXT: v_alignbit_b32 v0, s0, s0, s2 +; GFX11-NEXT: s_sub_i32 s6, 32, s2 +; GFX11-NEXT: s_sub_i32 s7, 32, s3 +; GFX11-NEXT: s_mov_b32 s2, s1 +; GFX11-NEXT: s_mov_b32 s3, s1 +; GFX11-NEXT: s_mov_b32 s1, s0 +; GFX11-NEXT: s_and_b32 s6, s6, 31 +; GFX11-NEXT: s_and_b32 s7, s7, 31 +; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s6 +; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s7 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2 +; GFX11-NEXT: v_mov_b32_e32 v0, s0 ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5] ; GFX11-NEXT: s_endpgm entry: @@ -188,20 +222,30 @@ define amdgpu_kernel void @rotl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd ; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 -; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_sub_i32 s4, 32, s12 -; SI-NEXT: s_sub_i32 s5, 32, s13 -; SI-NEXT: s_sub_i32 s6, 32, s15 -; SI-NEXT: s_sub_i32 s7, 32, s14 -; SI-NEXT: v_mov_b32_e32 v0, s6 -; SI-NEXT: v_alignbit_b32 v3, s11, s11, v0 -; SI-NEXT: v_mov_b32_e32 v0, s7 -; SI-NEXT: v_alignbit_b32 v2, s10, s10, v0 -; SI-NEXT: v_mov_b32_e32 v0, s5 -; SI-NEXT: v_alignbit_b32 v1, s9, s9, v0 -; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: v_alignbit_b32 v0, s8, s8, v0 +; SI-NEXT: s_sub_i32 s2, 32, s12 +; SI-NEXT: s_sub_i32 s12, 32, s13 +; SI-NEXT: s_sub_i32 s13, 32, s14 +; SI-NEXT: s_sub_i32 s14, 32, s15 +; SI-NEXT: s_mov_b32 s4, s11 +; SI-NEXT: s_mov_b32 s5, s11 +; SI-NEXT: s_mov_b32 s11, s10 +; SI-NEXT: s_mov_b32 s6, s9 +; SI-NEXT: s_mov_b32 s7, s9 +; SI-NEXT: s_mov_b32 s9, s8 +; SI-NEXT: s_and_b32 s14, s14, 31 +; SI-NEXT: s_and_b32 s13, s13, 31 +; SI-NEXT: s_and_b32 s12, s12, 31 +; SI-NEXT: s_and_b32 s2, s2, 31 +; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], s14 +; SI-NEXT: s_lshr_b64 s[10:11], s[10:11], s13 +; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], s12 +; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], s2 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: v_mov_b32_e32 v1, s6 +; SI-NEXT: v_mov_b32_e32 v2, s10 +; SI-NEXT: v_mov_b32_e32 v3, s4 ; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -210,19 +254,29 @@ define amdgpu_kernel void @rotl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_sub_i32 s5, 32, s15 +; GFX8-NEXT: s_sub_i32 s2, 32, s15 +; GFX8-NEXT: s_and_b32 s5, s2, 31 +; GFX8-NEXT: s_mov_b32 s2, s11 +; GFX8-NEXT: s_mov_b32 s3, s11 ; GFX8-NEXT: s_sub_i32 s4, 32, s14 -; GFX8-NEXT: v_mov_b32_e32 v0, s5 -; GFX8-NEXT: s_sub_i32 s3, 32, s13 -; GFX8-NEXT: v_alignbit_b32 v3, s11, s11, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s4 -; GFX8-NEXT: s_sub_i32 s2, 32, s12 -; GFX8-NEXT: v_alignbit_b32 v2, s10, s10, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s3 -; GFX8-NEXT: v_alignbit_b32 v1, s9, s9, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s2 +; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s5 +; GFX8-NEXT: s_sub_i32 s6, 32, s13 +; GFX8-NEXT: s_and_b32 s3, s4, 31 +; GFX8-NEXT: s_mov_b32 s11, s10 +; GFX8-NEXT: s_sub_i32 s12, 32, s12 +; GFX8-NEXT: s_lshr_b64 s[4:5], s[10:11], s3 +; GFX8-NEXT: s_and_b32 s3, s6, 31 +; GFX8-NEXT: s_mov_b32 s6, s9 +; GFX8-NEXT: s_mov_b32 s7, s9 +; GFX8-NEXT: s_lshr_b64 s[6:7], s[6:7], s3 +; GFX8-NEXT: s_and_b32 s3, s12, 31 +; GFX8-NEXT: s_mov_b32 s9, s8 +; GFX8-NEXT: s_lshr_b64 s[8:9], s[8:9], s3 ; GFX8-NEXT: v_mov_b32_e32 v5, s1 -; GFX8-NEXT: v_alignbit_b32 v0, s8, s8, v0 +; GFX8-NEXT: v_mov_b32_e32 v0, s8 +; GFX8-NEXT: v_mov_b32_e32 v1, s6 +; GFX8-NEXT: v_mov_b32_e32 v2, s4 +; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_mov_b32_e32 v4, s0 ; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NEXT: s_endpgm @@ -234,14 +288,28 @@ define amdgpu_kernel void @rotl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v4, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_sub_i32 s2, 32, s12 -; GFX10-NEXT: s_sub_i32 s3, 32, s13 -; GFX10-NEXT: s_sub_i32 s4, 32, s15 -; GFX10-NEXT: s_sub_i32 s5, 32, s14 -; GFX10-NEXT: v_alignbit_b32 v3, s11, s11, s4 -; GFX10-NEXT: v_alignbit_b32 v2, s10, s10, s5 -; GFX10-NEXT: v_alignbit_b32 v1, s9, s9, s3 -; GFX10-NEXT: v_alignbit_b32 v0, s8, s8, s2 +; GFX10-NEXT: s_sub_i32 s6, 32, s12 +; GFX10-NEXT: s_sub_i32 s7, 32, s13 +; GFX10-NEXT: s_sub_i32 s12, 32, s14 +; GFX10-NEXT: s_sub_i32 s13, 32, s15 +; GFX10-NEXT: s_mov_b32 s2, s11 +; GFX10-NEXT: s_mov_b32 s3, s11 +; GFX10-NEXT: s_mov_b32 s11, s10 +; GFX10-NEXT: s_mov_b32 s4, s9 +; GFX10-NEXT: s_mov_b32 s5, s9 +; GFX10-NEXT: s_mov_b32 s9, s8 +; GFX10-NEXT: s_and_b32 s13, s13, 31 +; GFX10-NEXT: s_and_b32 s12, s12, 31 +; GFX10-NEXT: s_and_b32 s14, s7, 31 +; GFX10-NEXT: s_and_b32 s15, s6, 31 +; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s13 +; GFX10-NEXT: s_lshr_b64 s[6:7], s[10:11], s12 +; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s15 +; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s14 +; GFX10-NEXT: v_mov_b32_e32 v0, s8 +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: v_mov_b32_e32 v2, s6 +; GFX10-NEXT: v_mov_b32_e32 v3, s2 ; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -250,16 +318,29 @@ define amdgpu_kernel void @rotl_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 ; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v4, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_sub_i32 s2, 32, s12 -; GFX11-NEXT: s_sub_i32 s3, 32, s13 -; GFX11-NEXT: s_sub_i32 s4, 32, s15 -; GFX11-NEXT: s_sub_i32 s5, 32, s14 -; GFX11-NEXT: v_alignbit_b32 v3, s11, s11, s4 -; GFX11-NEXT: v_alignbit_b32 v2, s10, s10, s5 -; GFX11-NEXT: v_alignbit_b32 v1, s9, s9, s3 -; GFX11-NEXT: v_alignbit_b32 v0, s8, s8, s2 +; GFX11-NEXT: s_sub_i32 s6, 32, s12 +; GFX11-NEXT: s_sub_i32 s7, 32, s13 +; GFX11-NEXT: s_sub_i32 s12, 32, s14 +; GFX11-NEXT: s_sub_i32 s13, 32, s15 +; GFX11-NEXT: s_mov_b32 s2, s11 +; GFX11-NEXT: s_mov_b32 s3, s11 +; GFX11-NEXT: s_mov_b32 s11, s10 +; GFX11-NEXT: s_mov_b32 s4, s9 +; GFX11-NEXT: s_mov_b32 s5, s9 +; GFX11-NEXT: s_mov_b32 s9, s8 +; GFX11-NEXT: s_and_b32 s13, s13, 31 +; GFX11-NEXT: s_and_b32 s12, s12, 31 +; GFX11-NEXT: s_and_b32 s14, s7, 31 +; GFX11-NEXT: s_and_b32 s15, s6, 31 +; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s13 +; GFX11-NEXT: s_lshr_b64 s[6:7], s[10:11], s12 +; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s15 +; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s14 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4 +; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2 +; GFX11-NEXT: v_mov_b32_e32 v2, s6 ; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1] ; GFX11-NEXT: s_endpgm entry: diff --git a/llvm/test/CodeGen/AMDGPU/rotr.ll b/llvm/test/CodeGen/AMDGPU/rotr.ll index 403a556688091..199433734529b 100644 --- a/llvm/test/CodeGen/AMDGPU/rotr.ll +++ b/llvm/test/CodeGen/AMDGPU/rotr.ll @@ -1,10 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 ; RUN: llc -mtriple=r600 -mcpu=redwood < %s | FileCheck --check-prefixes=R600 %s -; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefixes=SI %s -; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=GFX8 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s +; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s +; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12 %s define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) { ; R600-LABEL: rotr_i32: @@ -22,12 +23,14 @@ define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) { ; SI: ; %bb.0: ; %entry ; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; SI-NEXT: s_mov_b32 s7, 0xf000 -; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: s_and_b32 s4, s3, 31 +; SI-NEXT: s_mov_b32 s3, s2 +; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 +; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_mov_b32 s4, s0 ; SI-NEXT: s_mov_b32 s5, s1 -; SI-NEXT: v_mov_b32_e32 v0, s3 -; SI-NEXT: v_alignbit_b32 v0, s2, s2, v0 +; SI-NEXT: v_mov_b32_e32 v0, s2 ; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -35,10 +38,12 @@ define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) { ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v0, s3 -; GFX8-NEXT: v_alignbit_b32 v2, s2, s2, v0 +; GFX8-NEXT: s_and_b32 s4, s3, 31 +; GFX8-NEXT: s_mov_b32 s3, s2 +; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 +; GFX8-NEXT: v_mov_b32_e32 v2, s2 ; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: s_endpgm ; @@ -47,18 +52,36 @@ define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) { ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_alignbit_b32 v1, s2, s2, s3 +; GFX10-NEXT: s_and_b32 s4, s3, 31 +; GFX10-NEXT: s_mov_b32 s3, s2 +; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 +; GFX10-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: rotr_i32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v0, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_alignbit_b32 v1, s2, s2, s3 +; GFX11-NEXT: s_and_b32 s4, s3, 31 +; GFX11-NEXT: s_mov_b32 s3, s2 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 +; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: rotr_i32: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_and_b32 s4, s3, 31 +; GFX12-NEXT: s_mov_b32 s3, s2 +; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX12-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 +; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 +; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX12-NEXT: s_endpgm entry: %tmp0 = sub i32 32, %y %tmp1 = shl i32 %x, %tmp0 @@ -86,12 +109,17 @@ define amdgpu_kernel void @rotr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb ; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9 ; SI-NEXT: s_mov_b32 s7, 0xf000 -; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s3 -; SI-NEXT: v_alignbit_b32 v1, s1, s1, v0 -; SI-NEXT: v_mov_b32_e32 v0, s2 -; SI-NEXT: v_alignbit_b32 v0, s0, s0, v0 +; SI-NEXT: s_and_b32 s3, s3, 31 +; SI-NEXT: s_mov_b32 s8, s1 +; SI-NEXT: s_mov_b32 s9, s1 +; SI-NEXT: s_and_b32 s6, s2, 31 +; SI-NEXT: s_mov_b32 s1, s0 +; SI-NEXT: s_lshr_b64 s[2:3], s[8:9], s3 +; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s6 +; SI-NEXT: s_mov_b32 s6, -1 +; SI-NEXT: v_mov_b32_e32 v0, s0 +; SI-NEXT: v_mov_b32_e32 v1, s2 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -100,11 +128,16 @@ define amdgpu_kernel void @rotr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v0, s3 -; GFX8-NEXT: v_mov_b32_e32 v2, s2 -; GFX8-NEXT: v_alignbit_b32 v1, s1, s1, v0 -; GFX8-NEXT: v_alignbit_b32 v0, s0, s0, v2 +; GFX8-NEXT: s_and_b32 s3, s3, 31 +; GFX8-NEXT: s_mov_b32 s6, s1 +; GFX8-NEXT: s_mov_b32 s7, s1 +; GFX8-NEXT: s_and_b32 s8, s2, 31 +; GFX8-NEXT: s_mov_b32 s1, s0 +; GFX8-NEXT: s_lshr_b64 s[2:3], s[6:7], s3 +; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s8 ; GFX8-NEXT: v_mov_b32_e32 v2, s4 +; GFX8-NEXT: v_mov_b32_e32 v0, s0 +; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: v_mov_b32_e32 v3, s5 ; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GFX8-NEXT: s_endpgm @@ -116,8 +149,15 @@ define amdgpu_kernel void @rotr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_alignbit_b32 v1, s1, s1, s3 -; GFX10-NEXT: v_alignbit_b32 v0, s0, s0, s2 +; GFX10-NEXT: s_mov_b32 s4, s1 +; GFX10-NEXT: s_mov_b32 s5, s1 +; GFX10-NEXT: s_and_b32 s2, s2, 31 +; GFX10-NEXT: s_mov_b32 s1, s0 +; GFX10-NEXT: s_and_b32 s3, s3, 31 +; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 +; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s3 +; GFX10-NEXT: v_mov_b32_e32 v0, s0 +; GFX10-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] ; GFX10-NEXT: s_endpgm ; @@ -126,12 +166,38 @@ define amdgpu_kernel void @rotr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c ; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v2, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_alignbit_b32 v1, s1, s1, s3 -; GFX11-NEXT: v_alignbit_b32 v0, s0, s0, s2 +; GFX11-NEXT: s_mov_b32 s6, s1 +; GFX11-NEXT: s_mov_b32 s7, s1 +; GFX11-NEXT: s_and_b32 s2, s2, 31 +; GFX11-NEXT: s_mov_b32 s1, s0 +; GFX11-NEXT: s_and_b32 s3, s3, 31 +; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 +; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], s3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2 +; GFX11-NEXT: v_mov_b32_e32 v0, s0 ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5] ; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: rotr_v2i32: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c +; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_mov_b32 s6, s1 +; GFX12-NEXT: s_mov_b32 s7, s1 +; GFX12-NEXT: s_and_b32 s2, s2, 31 +; GFX12-NEXT: s_mov_b32 s1, s0 +; GFX12-NEXT: s_and_b32 s3, s3, 31 +; GFX12-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 +; GFX12-NEXT: s_lshr_b64 s[2:3], s[6:7], s3 +; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2 +; GFX12-NEXT: v_mov_b32_e32 v0, s0 +; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5] +; GFX12-NEXT: s_endpgm entry: %tmp0 = sub <2 x i32> , %y %tmp1 = shl <2 x i32> %x, %tmp0 @@ -161,16 +227,26 @@ define amdgpu_kernel void @rotr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd ; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 -; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s15 -; SI-NEXT: v_alignbit_b32 v3, s11, s11, v0 -; SI-NEXT: v_mov_b32_e32 v0, s14 -; SI-NEXT: v_alignbit_b32 v2, s10, s10, v0 -; SI-NEXT: v_mov_b32_e32 v0, s13 -; SI-NEXT: v_alignbit_b32 v1, s9, s9, v0 -; SI-NEXT: v_mov_b32_e32 v0, s12 -; SI-NEXT: v_alignbit_b32 v0, s8, s8, v0 +; SI-NEXT: s_and_b32 s2, s15, 31 +; SI-NEXT: s_mov_b32 s4, s11 +; SI-NEXT: s_mov_b32 s5, s11 +; SI-NEXT: s_and_b32 s14, s14, 31 +; SI-NEXT: s_mov_b32 s11, s10 +; SI-NEXT: s_and_b32 s13, s13, 31 +; SI-NEXT: s_mov_b32 s6, s9 +; SI-NEXT: s_mov_b32 s7, s9 +; SI-NEXT: s_and_b32 s12, s12, 31 +; SI-NEXT: s_mov_b32 s9, s8 +; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], s2 +; SI-NEXT: s_lshr_b64 s[10:11], s[10:11], s14 +; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], s13 +; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], s12 +; SI-NEXT: s_mov_b32 s2, -1 +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: v_mov_b32_e32 v1, s6 +; SI-NEXT: v_mov_b32_e32 v2, s10 +; SI-NEXT: v_mov_b32_e32 v3, s4 ; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -179,15 +255,25 @@ define amdgpu_kernel void @rotr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v0, s15 -; GFX8-NEXT: v_mov_b32_e32 v1, s14 -; GFX8-NEXT: v_mov_b32_e32 v4, s13 -; GFX8-NEXT: v_alignbit_b32 v3, s11, s11, v0 -; GFX8-NEXT: v_alignbit_b32 v2, s10, s10, v1 -; GFX8-NEXT: v_alignbit_b32 v1, s9, s9, v4 -; GFX8-NEXT: v_mov_b32_e32 v0, s12 +; GFX8-NEXT: s_and_b32 s4, s15, 31 +; GFX8-NEXT: s_mov_b32 s2, s11 +; GFX8-NEXT: s_mov_b32 s3, s11 +; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 +; GFX8-NEXT: s_and_b32 s3, s13, 31 +; GFX8-NEXT: s_mov_b32 s6, s9 +; GFX8-NEXT: s_mov_b32 s7, s9 +; GFX8-NEXT: s_and_b32 s5, s14, 31 +; GFX8-NEXT: s_mov_b32 s11, s10 +; GFX8-NEXT: s_lshr_b64 s[6:7], s[6:7], s3 +; GFX8-NEXT: s_and_b32 s3, s12, 31 +; GFX8-NEXT: s_mov_b32 s9, s8 +; GFX8-NEXT: s_lshr_b64 s[4:5], s[10:11], s5 +; GFX8-NEXT: s_lshr_b64 s[8:9], s[8:9], s3 ; GFX8-NEXT: v_mov_b32_e32 v5, s1 -; GFX8-NEXT: v_alignbit_b32 v0, s8, s8, v0 +; GFX8-NEXT: v_mov_b32_e32 v0, s8 +; GFX8-NEXT: v_mov_b32_e32 v1, s6 +; GFX8-NEXT: v_mov_b32_e32 v2, s4 +; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_mov_b32_e32 v4, s0 ; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NEXT: s_endpgm @@ -199,10 +285,24 @@ define amdgpu_kernel void @rotr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v4, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_alignbit_b32 v3, s11, s11, s15 -; GFX10-NEXT: v_alignbit_b32 v2, s10, s10, s14 -; GFX10-NEXT: v_alignbit_b32 v1, s9, s9, s13 -; GFX10-NEXT: v_alignbit_b32 v0, s8, s8, s12 +; GFX10-NEXT: s_and_b32 s6, s15, 31 +; GFX10-NEXT: s_mov_b32 s2, s11 +; GFX10-NEXT: s_mov_b32 s3, s11 +; GFX10-NEXT: s_and_b32 s7, s14, 31 +; GFX10-NEXT: s_mov_b32 s11, s10 +; GFX10-NEXT: s_and_b32 s13, s13, 31 +; GFX10-NEXT: s_mov_b32 s4, s9 +; GFX10-NEXT: s_mov_b32 s5, s9 +; GFX10-NEXT: s_and_b32 s12, s12, 31 +; GFX10-NEXT: s_mov_b32 s9, s8 +; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s6 +; GFX10-NEXT: s_lshr_b64 s[6:7], s[10:11], s7 +; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s12 +; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s13 +; GFX10-NEXT: v_mov_b32_e32 v0, s8 +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: v_mov_b32_e32 v2, s6 +; GFX10-NEXT: v_mov_b32_e32 v3, s2 ; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -211,14 +311,54 @@ define amdgpu_kernel void @rotr_v4i32(ptr addrspace(1) %in, <4 x i32> %x, <4 x i ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 ; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v4, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_alignbit_b32 v3, s11, s11, s15 -; GFX11-NEXT: v_alignbit_b32 v2, s10, s10, s14 -; GFX11-NEXT: v_alignbit_b32 v1, s9, s9, s13 -; GFX11-NEXT: v_alignbit_b32 v0, s8, s8, s12 +; GFX11-NEXT: s_and_b32 s6, s15, 31 +; GFX11-NEXT: s_mov_b32 s2, s11 +; GFX11-NEXT: s_mov_b32 s3, s11 +; GFX11-NEXT: s_and_b32 s7, s14, 31 +; GFX11-NEXT: s_mov_b32 s11, s10 +; GFX11-NEXT: s_and_b32 s13, s13, 31 +; GFX11-NEXT: s_mov_b32 s4, s9 +; GFX11-NEXT: s_mov_b32 s5, s9 +; GFX11-NEXT: s_and_b32 s12, s12, 31 +; GFX11-NEXT: s_mov_b32 s9, s8 +; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s6 +; GFX11-NEXT: s_lshr_b64 s[6:7], s[10:11], s7 +; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s12 +; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s13 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4 +; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2 +; GFX11-NEXT: v_mov_b32_e32 v2, s6 ; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1] ; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: rotr_v4i32: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 +; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_and_b32 s6, s15, 31 +; GFX12-NEXT: s_mov_b32 s2, s11 +; GFX12-NEXT: s_mov_b32 s3, s11 +; GFX12-NEXT: s_and_b32 s7, s14, 31 +; GFX12-NEXT: s_mov_b32 s11, s10 +; GFX12-NEXT: s_and_b32 s13, s13, 31 +; GFX12-NEXT: s_mov_b32 s4, s9 +; GFX12-NEXT: s_mov_b32 s5, s9 +; GFX12-NEXT: s_and_b32 s12, s12, 31 +; GFX12-NEXT: s_mov_b32 s9, s8 +; GFX12-NEXT: s_lshr_b64 s[2:3], s[2:3], s6 +; GFX12-NEXT: s_lshr_b64 s[6:7], s[10:11], s7 +; GFX12-NEXT: s_lshr_b64 s[8:9], s[8:9], s12 +; GFX12-NEXT: s_lshr_b64 s[4:5], s[4:5], s13 +; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s4 +; GFX12-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s2 +; GFX12-NEXT: v_mov_b32_e32 v2, s6 +; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1] +; GFX12-NEXT: s_endpgm entry: %tmp0 = sub <4 x i32> , %y %tmp1 = shl <4 x i32> %x, %tmp0 @@ -258,23 +398,44 @@ define amdgpu_kernel void @rotr_v8i32(ptr addrspace(1) %in, <8 x i32> %x, <8 x i ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s19 -; SI-NEXT: v_alignbit_b32 v3, s11, s11, v0 -; SI-NEXT: v_mov_b32_e32 v0, s18 -; SI-NEXT: v_alignbit_b32 v2, s10, s10, v0 -; SI-NEXT: v_mov_b32_e32 v0, s17 -; SI-NEXT: v_alignbit_b32 v1, s9, s9, v0 -; SI-NEXT: v_mov_b32_e32 v0, s16 -; SI-NEXT: v_alignbit_b32 v0, s8, s8, v0 -; SI-NEXT: v_mov_b32_e32 v4, s23 -; SI-NEXT: v_alignbit_b32 v7, s15, s15, v4 -; SI-NEXT: v_mov_b32_e32 v4, s22 -; SI-NEXT: v_alignbit_b32 v6, s14, s14, v4 -; SI-NEXT: v_mov_b32_e32 v4, s21 -; SI-NEXT: v_alignbit_b32 v5, s13, s13, v4 -; SI-NEXT: v_mov_b32_e32 v4, s20 -; SI-NEXT: v_alignbit_b32 v4, s12, s12, v4 -; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16 +; SI-NEXT: s_and_b32 s24, s19, 31 +; SI-NEXT: s_mov_b32 s4, s11 +; SI-NEXT: s_mov_b32 s5, s11 +; SI-NEXT: s_and_b32 s25, s18, 31 +; SI-NEXT: s_mov_b32 s11, s10 +; SI-NEXT: s_and_b32 s26, s17, 31 +; SI-NEXT: s_mov_b32 s6, s9 +; SI-NEXT: s_mov_b32 s7, s9 +; SI-NEXT: s_and_b32 s27, s16, 31 +; SI-NEXT: s_mov_b32 s9, s8 +; SI-NEXT: s_and_b32 s23, s23, 31 +; SI-NEXT: s_mov_b32 s16, s15 +; SI-NEXT: s_mov_b32 s17, s15 +; SI-NEXT: s_and_b32 s22, s22, 31 +; SI-NEXT: s_mov_b32 s15, s14 +; SI-NEXT: s_and_b32 s21, s21, 31 +; SI-NEXT: s_mov_b32 s18, s13 +; SI-NEXT: s_mov_b32 s19, s13 +; SI-NEXT: s_and_b32 s20, s20, 31 +; SI-NEXT: s_mov_b32 s13, s12 +; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], s24 +; SI-NEXT: s_lshr_b64 s[10:11], s[10:11], s25 +; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], s26 +; SI-NEXT: s_lshr_b64 s[16:17], s[16:17], s23 +; SI-NEXT: s_lshr_b64 s[14:15], s[14:15], s22 +; SI-NEXT: s_lshr_b64 s[18:19], s[18:19], s21 +; SI-NEXT: s_lshr_b64 s[12:13], s[12:13], s20 +; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], s27 +; SI-NEXT: v_mov_b32_e32 v0, s12 +; SI-NEXT: v_mov_b32_e32 v1, s18 +; SI-NEXT: v_mov_b32_e32 v2, s14 +; SI-NEXT: v_mov_b32_e32 v3, s16 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16 +; SI-NEXT: s_waitcnt expcnt(0) +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: v_mov_b32_e32 v1, s6 +; SI-NEXT: v_mov_b32_e32 v2, s10 +; SI-NEXT: v_mov_b32_e32 v3, s4 ; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -283,28 +444,48 @@ define amdgpu_kernel void @rotr_v8i32(ptr addrspace(1) %in, <8 x i32> %x, <8 x i ; GFX8-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x44 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v1, s18 -; GFX8-NEXT: v_mov_b32_e32 v4, s17 -; GFX8-NEXT: v_alignbit_b32 v2, s10, s10, v1 -; GFX8-NEXT: v_alignbit_b32 v1, s9, s9, v4 -; GFX8-NEXT: v_mov_b32_e32 v4, s23 -; GFX8-NEXT: v_alignbit_b32 v7, s15, s15, v4 -; GFX8-NEXT: v_mov_b32_e32 v4, s22 -; GFX8-NEXT: s_add_u32 s2, s0, 16 -; GFX8-NEXT: v_alignbit_b32 v6, s14, s14, v4 -; GFX8-NEXT: v_mov_b32_e32 v4, s21 -; GFX8-NEXT: s_addc_u32 s3, s1, 0 -; GFX8-NEXT: v_alignbit_b32 v5, s13, s13, v4 -; GFX8-NEXT: v_mov_b32_e32 v4, s20 -; GFX8-NEXT: v_mov_b32_e32 v9, s3 -; GFX8-NEXT: v_mov_b32_e32 v0, s19 -; GFX8-NEXT: v_alignbit_b32 v4, s12, s12, v4 -; GFX8-NEXT: v_mov_b32_e32 v8, s2 -; GFX8-NEXT: v_alignbit_b32 v3, s11, s11, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s16 -; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[4:7] -; GFX8-NEXT: v_alignbit_b32 v0, s8, s8, v0 +; GFX8-NEXT: s_and_b32 s4, s19, 31 +; GFX8-NEXT: s_mov_b32 s2, s11 +; GFX8-NEXT: s_mov_b32 s3, s11 +; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4 +; GFX8-NEXT: s_and_b32 s3, s17, 31 +; GFX8-NEXT: s_mov_b32 s6, s9 +; GFX8-NEXT: s_mov_b32 s7, s9 +; GFX8-NEXT: s_and_b32 s5, s18, 31 +; GFX8-NEXT: s_mov_b32 s11, s10 +; GFX8-NEXT: s_lshr_b64 s[6:7], s[6:7], s3 +; GFX8-NEXT: s_and_b32 s3, s16, 31 +; GFX8-NEXT: s_mov_b32 s9, s8 +; GFX8-NEXT: s_lshr_b64 s[4:5], s[10:11], s5 +; GFX8-NEXT: s_lshr_b64 s[8:9], s[8:9], s3 +; GFX8-NEXT: s_and_b32 s3, s23, 31 +; GFX8-NEXT: s_mov_b32 s10, s15 +; GFX8-NEXT: s_mov_b32 s11, s15 +; GFX8-NEXT: s_lshr_b64 s[10:11], s[10:11], s3 +; GFX8-NEXT: s_and_b32 s3, s22, 31 +; GFX8-NEXT: s_mov_b32 s15, s14 +; GFX8-NEXT: s_lshr_b64 s[14:15], s[14:15], s3 +; GFX8-NEXT: s_and_b32 s3, s21, 31 +; GFX8-NEXT: s_mov_b32 s16, s13 +; GFX8-NEXT: s_mov_b32 s17, s13 +; GFX8-NEXT: s_lshr_b64 s[16:17], s[16:17], s3 +; GFX8-NEXT: s_and_b32 s3, s20, 31 +; GFX8-NEXT: s_mov_b32 s13, s12 +; GFX8-NEXT: s_lshr_b64 s[12:13], s[12:13], s3 +; GFX8-NEXT: v_mov_b32_e32 v3, s10 +; GFX8-NEXT: s_add_u32 s10, s0, 16 +; GFX8-NEXT: s_addc_u32 s11, s1, 0 +; GFX8-NEXT: v_mov_b32_e32 v4, s10 +; GFX8-NEXT: v_mov_b32_e32 v0, s12 +; GFX8-NEXT: v_mov_b32_e32 v1, s16 +; GFX8-NEXT: v_mov_b32_e32 v2, s14 +; GFX8-NEXT: v_mov_b32_e32 v5, s11 +; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NEXT: v_mov_b32_e32 v5, s1 +; GFX8-NEXT: v_mov_b32_e32 v0, s8 +; GFX8-NEXT: v_mov_b32_e32 v1, s6 +; GFX8-NEXT: v_mov_b32_e32 v2, s4 +; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_mov_b32_e32 v4, s0 ; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NEXT: s_endpgm @@ -316,16 +497,44 @@ define amdgpu_kernel void @rotr_v8i32(ptr addrspace(1) %in, <8 x i32> %x, <8 x i ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v8, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_alignbit_b32 v7, s15, s15, s23 -; GFX10-NEXT: v_alignbit_b32 v6, s14, s14, s22 -; GFX10-NEXT: v_alignbit_b32 v5, s13, s13, s21 -; GFX10-NEXT: v_alignbit_b32 v4, s12, s12, s20 -; GFX10-NEXT: v_alignbit_b32 v3, s11, s11, s19 -; GFX10-NEXT: v_alignbit_b32 v2, s10, s10, s18 -; GFX10-NEXT: v_alignbit_b32 v1, s9, s9, s17 -; GFX10-NEXT: v_alignbit_b32 v0, s8, s8, s16 -; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 -; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX10-NEXT: s_and_b32 s19, s19, 31 +; GFX10-NEXT: s_mov_b32 s2, s11 +; GFX10-NEXT: s_mov_b32 s3, s11 +; GFX10-NEXT: s_and_b32 s17, s17, 31 +; GFX10-NEXT: s_mov_b32 s4, s9 +; GFX10-NEXT: s_mov_b32 s5, s9 +; GFX10-NEXT: s_and_b32 s16, s16, 31 +; GFX10-NEXT: s_mov_b32 s9, s8 +; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s19 +; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s17 +; GFX10-NEXT: s_and_b32 s23, s23, 31 +; GFX10-NEXT: s_mov_b32 s6, s15 +; GFX10-NEXT: s_mov_b32 s7, s15 +; GFX10-NEXT: s_and_b32 s22, s22, 31 +; GFX10-NEXT: s_mov_b32 s15, s14 +; GFX10-NEXT: s_lshr_b64 s[8:9], s[8:9], s16 +; GFX10-NEXT: s_mov_b32 s16, s13 +; GFX10-NEXT: s_mov_b32 s17, s13 +; GFX10-NEXT: s_and_b32 s3, s20, 31 +; GFX10-NEXT: s_mov_b32 s13, s12 +; GFX10-NEXT: s_and_b32 s5, s21, 31 +; GFX10-NEXT: s_and_b32 s18, s18, 31 +; GFX10-NEXT: s_mov_b32 s11, s10 +; GFX10-NEXT: s_lshr_b64 s[6:7], s[6:7], s23 +; GFX10-NEXT: s_lshr_b64 s[14:15], s[14:15], s22 +; GFX10-NEXT: s_lshr_b64 s[12:13], s[12:13], s3 +; GFX10-NEXT: s_lshr_b64 s[16:17], s[16:17], s5 +; GFX10-NEXT: s_lshr_b64 s[10:11], s[10:11], s18 +; GFX10-NEXT: v_mov_b32_e32 v0, s12 +; GFX10-NEXT: v_mov_b32_e32 v1, s16 +; GFX10-NEXT: v_mov_b32_e32 v2, s14 +; GFX10-NEXT: v_mov_b32_e32 v3, s6 +; GFX10-NEXT: v_mov_b32_e32 v4, s8 +; GFX10-NEXT: v_mov_b32_e32 v5, s4 +; GFX10-NEXT: v_mov_b32_e32 v6, s10 +; GFX10-NEXT: v_mov_b32_e32 v7, s2 +; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 +; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: rotr_v8i32: @@ -333,20 +542,88 @@ define amdgpu_kernel void @rotr_v8i32(ptr addrspace(1) %in, <8 x i32> %x, <8 x i ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_load_b512 s[8:23], s[4:5], 0x44 ; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v8, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_alignbit_b32 v7, s15, s15, s23 -; GFX11-NEXT: v_alignbit_b32 v6, s14, s14, s22 -; GFX11-NEXT: v_alignbit_b32 v5, s13, s13, s21 -; GFX11-NEXT: v_alignbit_b32 v4, s12, s12, s20 -; GFX11-NEXT: v_alignbit_b32 v3, s11, s11, s19 -; GFX11-NEXT: v_alignbit_b32 v2, s10, s10, s18 -; GFX11-NEXT: v_alignbit_b32 v1, s9, s9, s17 -; GFX11-NEXT: v_alignbit_b32 v0, s8, s8, s16 +; GFX11-NEXT: s_and_b32 s19, s19, 31 +; GFX11-NEXT: s_mov_b32 s2, s11 +; GFX11-NEXT: s_mov_b32 s3, s11 +; GFX11-NEXT: s_and_b32 s17, s17, 31 +; GFX11-NEXT: s_mov_b32 s4, s9 +; GFX11-NEXT: s_mov_b32 s5, s9 +; GFX11-NEXT: s_and_b32 s16, s16, 31 +; GFX11-NEXT: s_mov_b32 s9, s8 +; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s19 +; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s17 +; GFX11-NEXT: s_and_b32 s23, s23, 31 +; GFX11-NEXT: s_mov_b32 s6, s15 +; GFX11-NEXT: s_mov_b32 s7, s15 +; GFX11-NEXT: s_and_b32 s22, s22, 31 +; GFX11-NEXT: s_mov_b32 s15, s14 +; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s16 +; GFX11-NEXT: s_mov_b32 s16, s13 +; GFX11-NEXT: s_mov_b32 s17, s13 +; GFX11-NEXT: s_and_b32 s3, s20, 31 +; GFX11-NEXT: s_mov_b32 s13, s12 +; GFX11-NEXT: s_and_b32 s5, s21, 31 +; GFX11-NEXT: s_and_b32 s18, s18, 31 +; GFX11-NEXT: s_mov_b32 s11, s10 +; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s23 +; GFX11-NEXT: s_lshr_b64 s[14:15], s[14:15], s22 +; GFX11-NEXT: s_lshr_b64 s[12:13], s[12:13], s3 +; GFX11-NEXT: s_lshr_b64 s[16:17], s[16:17], s5 +; GFX11-NEXT: s_lshr_b64 s[10:11], s[10:11], s18 +; GFX11-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s16 +; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s6 +; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v5, s4 +; GFX11-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s2 +; GFX11-NEXT: v_mov_b32_e32 v6, s10 ; GFX11-NEXT: s_clause 0x1 -; GFX11-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16 -; GFX11-NEXT: global_store_b128 v8, v[0:3], s[0:1] +; GFX11-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16 +; GFX11-NEXT: global_store_b128 v8, v[4:7], s[0:1] ; GFX11-NEXT: s_endpgm +; +; GFX12-LABEL: rotr_v8i32: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b512 s[8:23], s[4:5], 0x44 +; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_and_b32 s19, s19, 31 +; GFX12-NEXT: s_mov_b32 s2, s11 +; GFX12-NEXT: s_mov_b32 s3, s11 +; GFX12-NEXT: s_and_b32 s17, s17, 31 +; GFX12-NEXT: s_mov_b32 s4, s9 +; GFX12-NEXT: s_mov_b32 s5, s9 +; GFX12-NEXT: s_and_b32 s16, s16, 31 +; GFX12-NEXT: s_mov_b32 s9, s8 +; GFX12-NEXT: s_lshr_b64 s[2:3], s[2:3], s19 +; GFX12-NEXT: s_lshr_b64 s[4:5], s[4:5], s17 +; GFX12-NEXT: s_and_b32 s23, s23, 31 +; GFX12-NEXT: s_mov_b32 s6, s15 +; GFX12-NEXT: s_mov_b32 s7, s15 +; GFX12-NEXT: s_and_b32 s22, s22, 31 +; GFX12-NEXT: s_mov_b32 s15, s14 +; GFX12-NEXT: s_lshr_b64 s[8:9], s[8:9], s16 +; GFX12-NEXT: s_mov_b32 s16, s13 +; GFX12-NEXT: s_mov_b32 s17, s13 +; GFX12-NEXT: s_and_b32 s3, s20, 31 +; GFX12-NEXT: s_mov_b32 s13, s12 +; GFX12-NEXT: s_and_b32 s5, s21, 31 +; GFX12-NEXT: s_and_b32 s18, s18, 31 +; GFX12-NEXT: s_mov_b32 s11, s10 +; GFX12-NEXT: s_lshr_b64 s[6:7], s[6:7], s23 +; GFX12-NEXT: s_lshr_b64 s[14:15], s[14:15], s22 +; GFX12-NEXT: s_lshr_b64 s[12:13], s[12:13], s3 +; GFX12-NEXT: s_lshr_b64 s[16:17], s[16:17], s5 +; GFX12-NEXT: s_lshr_b64 s[10:11], s[10:11], s18 +; GFX12-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v1, s16 +; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s6 +; GFX12-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v5, s4 +; GFX12-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s2 +; GFX12-NEXT: v_mov_b32_e32 v6, s10 +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16 +; GFX12-NEXT: global_store_b128 v8, v[4:7], s[0:1] +; GFX12-NEXT: s_endpgm entry: %tmp0 = sub <8 x i32> , %y %tmp1 = shl <8 x i32> %x, %tmp0 @@ -484,6 +761,25 @@ define void @test_rotr_i16(ptr addrspace(1) nocapture readonly %sourceA, ptr add ; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v2, v0 ; GFX11-FAKE16-NEXT: global_store_b16 v[4:5], v0, off offset:8 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-LABEL: test_rotr_i16: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: global_load_u16 v2, v[2:3], off offset:48 +; GFX12-NEXT: global_load_u16 v0, v[0:1], off offset:32 +; GFX12-NEXT: s_wait_loadcnt 0x1 +; GFX12-NEXT: v_sub_nc_u16 v1, 0, v2 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b16 v2, v2, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b16 v0, v1, v0 +; GFX12-NEXT: v_or_b32_e32 v0, v2, v0 +; GFX12-NEXT: global_store_b16 v[4:5], v0, off offset:8 +; GFX12-NEXT: s_setpc_b64 s[30:31] entry: %arrayidx = getelementptr inbounds i16, ptr addrspace(1) %sourceA, i64 16 %a = load i16, ptr addrspace(1) %arrayidx