diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 9d3e707cccaaa..0bb4c73f88283 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -496,6 +496,14 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos, unsigned Width); + template + bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) { + return SelectCVTFixedPosRecipOperandVec(N, FixedPos, FloatWidth); + } + + bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos, + unsigned Width); + bool SelectCMP_SWAP(SDNode *N); bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, @@ -4147,14 +4155,11 @@ static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, return false; } -bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, - unsigned RegWidth) { - return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth, - /*isReciprocal*/ false); -} - -bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, - unsigned RegWidth) { +static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG, + SDValue N, + SDValue &FixedPos, + unsigned RegWidth, + bool isReciprocal) { if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) && N.getValueType().getScalarSizeInBits() == N.getOperand(0).getValueType().getScalarSizeInBits()) @@ -4192,8 +4197,8 @@ bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, return false; } - if (unsigned FBits = CheckFixedPointOperandConstant(FVal, RegWidth, - /*isReciprocal*/ false)) { + if (unsigned FBits = + CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) { FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32); return true; } @@ -4201,6 +4206,25 @@ bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, return false; } +bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, + unsigned RegWidth) { + return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth, + /*isReciprocal*/ false); +} + +bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, + unsigned RegWidth) { + return checkCVTFixedPointOperandWithFBitsForVectors( + CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false); +} + +bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N, + SDValue &FixedPos, + unsigned RegWidth) { + return checkCVTFixedPointOperandWithFBitsForVectors( + CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true); +} + bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos, unsigned RegWidth) { diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 08512f6ed8df1..cd931615c79cc 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -9265,6 +9265,53 @@ defm : FCVTPat; defm : FCVTPat; } +// fmul(sitofp(x), 1/2^N) -> scvtf(x, N), fmul(uitofp(x), 1/2^N) -> ucvtf(x, N) +class fixedpoint_recip_vec_f64 + : ComplexPattern">; +class fixedpoint_recip_vec_f32 + : ComplexPattern">; +class fixedpoint_recip_vec_f16 + : ComplexPattern">; +def fixedpoint_recip_vec_xform : SDNodeXForm; +def gi_fixedpoint_recip_vec_xform + : GICustomOperandRenderer<"renderFixedPointRecipXForm">, + GISDNodeXFormEquiv; + +def fixedpoint_recip_v2f64 : fixedpoint_recip_vec_f64; +def fixedpoint_recip_v2f32 : fixedpoint_recip_vec_f32; +def fixedpoint_recip_v4f32 : fixedpoint_recip_vec_f32; +def fixedpoint_recip_v4f16 : fixedpoint_recip_vec_f16; +def fixedpoint_recip_v8f16 : fixedpoint_recip_vec_f16; + +def gi_fixedpoint_recip_v2f64 + : GIComplexOperandMatcher, + GIComplexPatternEquiv; +def gi_fixedpoint_recip_v2f32 + : GIComplexOperandMatcher, + GIComplexPatternEquiv; +def gi_fixedpoint_recip_v4f32 + : GIComplexOperandMatcher, + GIComplexPatternEquiv; +def gi_fixedpoint_recip_v4f16 + : GIComplexOperandMatcher, + GIComplexPatternEquiv; +def gi_fixedpoint_recip_v8f16 + : GIComplexOperandMatcher, + GIComplexPatternEquiv; + +multiclass CVTFRecipPat { + def : Pat<(FVT (fmul (sint_to_fp (IVT RC:$Rn)), fixedpoint:$scale)), + (!cast("SCVTF"#IVT#"_shift") RC:$Rn, + (fixedpoint_recip_vec_xform fixedpoint:$scale))>; + def : Pat<(FVT (fmul (uint_to_fp (IVT RC:$Rn)), fixedpoint:$scale)), + (!cast("UCVTF"#IVT#"_shift") RC:$Rn, + (fixedpoint_recip_vec_xform fixedpoint:$scale))>; +} + // X << 1 ==> X + X class SHLToADDPat : Pat<(ty (AArch64vshl (ty regtype:$Rn), (i32 1))), @@ -9319,6 +9366,16 @@ defm USHR : SIMDVectorRShiftBHSD<1, 0b00000, "ushr", AArch64vlshr>; defm USRA : SIMDVectorRShiftBHSDTied<1, 0b00010, "usra", TriOpFrag<(add_like node:$LHS, (AArch64vlshr node:$MHS, node:$RHS))> >; +let Predicates = [HasNEON] in { +defm : CVTFRecipPat; +defm : CVTFRecipPat; +defm : CVTFRecipPat; +} +let Predicates = [HasNEON, HasFullFP16] in { +defm : CVTFRecipPat; +defm : CVTFRecipPat; +} + def VImm0080: PatLeaf<(AArch64movi_shift (i32 128), (i32 0))>; def VImm00008000: PatLeaf<(AArch64movi_shift (i32 128), (i32 8))>; def VImm0000000080000000: PatLeaf<(AArch64NvCast (v2f64 (fneg (AArch64NvCast (v4i32 (AArch64movi_shift (i32 128), (i32 24)))))))>; diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 2fa0fca176c88..743d1c6421bf4 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -484,9 +484,14 @@ class AArch64InstructionSelector : public InstructionSelector { ComplexRendererFns selectCVTFixedPointVec(MachineOperand &Root) const; ComplexRendererFns - selectCVTFixedPointVecBase(const MachineOperand &Root) const; + selectCVTFixedPosRecipOperandVec(MachineOperand &Root) const; + ComplexRendererFns + selectCVTFixedPointVecBase(const MachineOperand &Root, + bool isReciprocal = false) const; void renderFixedPointXForm(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx = -1) const; + void renderFixedPointRecipXForm(MachineInstrBuilder &MIB, + const MachineInstr &MI, int OpIdx = -1) const; void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx = -1) const; @@ -7853,7 +7858,7 @@ AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root) const { InstructionSelector::ComplexRendererFns AArch64InstructionSelector::selectCVTFixedPointVecBase( - const MachineOperand &Root) const { + const MachineOperand &Root, bool isReciprocal) const { if (!Root.isReg()) return std::nullopt; const MachineRegisterInfo &MRI = @@ -7882,8 +7887,8 @@ AArch64InstructionSelector::selectCVTFixedPointVecBase( default: return std::nullopt; }; - if (unsigned FBits = CheckFixedPointOperandConstant(FVal, RegWidth, - /*isReciprocal*/ false)) + if (unsigned FBits = + CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(FBits); }}}; return std::nullopt; @@ -7891,7 +7896,13 @@ AArch64InstructionSelector::selectCVTFixedPointVecBase( InstructionSelector::ComplexRendererFns AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root) const { - return selectCVTFixedPointVecBase(Root); + return selectCVTFixedPointVecBase(Root, /*isReciprocal*/ false); +} + +InstructionSelector::ComplexRendererFns +AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec( + MachineOperand &Root) const { + return selectCVTFixedPointVecBase(Root, /*isReciprocal*/ true); } void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB, @@ -7901,12 +7912,21 @@ void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB, // should be able to reuse the Renderers already calculated by // selectCVTFixedPointVecBase. InstructionSelector::ComplexRendererFns Renderer = - selectCVTFixedPointVecBase(MI.getOperand(2)); + selectCVTFixedPointVecBase(MI.getOperand(2), /*isReciprocal*/ false); assert((Renderer && Renderer->size() == 1) && "Expected selectCVTFixedPointVec to provide a function\n"); (Renderer->front())(MIB); } +void AArch64InstructionSelector::renderFixedPointRecipXForm( + MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { + InstructionSelector::ComplexRendererFns Renderer = + selectCVTFixedPointVecBase(MI.getOperand(2), /*isReciprocal*/ true); + assert((Renderer && Renderer->size() == 1) && + "Expected selectCVTFixedPosRecipOperandVec to provide a function\n"); + (Renderer->front())(MIB); +} + void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { diff --git a/llvm/test/CodeGen/AArch64/scvtf-div-mul-combine.ll b/llvm/test/CodeGen/AArch64/scvtf-div-mul-combine.ll new file mode 100644 index 0000000000000..db2fd518afcee --- /dev/null +++ b/llvm/test/CodeGen/AArch64/scvtf-div-mul-combine.ll @@ -0,0 +1,474 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-neon-syntax=apple -mattr=+fullfp16 -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-neon-syntax=apple -mattr=+fullfp16 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI + +;; ---- sitofp + fdiv (power-of-2 divisors) ---- + +define <2 x float> @sitofp_v2f32_fdiv_2(<2 x i32> %in) { +; CHECK-SD-LABEL: sitofp_v2f32_fdiv_2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.2s v0, v0, #1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v2f32_fdiv_2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2s v1, #64, lsl #24 +; CHECK-GI-NEXT: scvtf.2s v0, v0 +; CHECK-GI-NEXT: fdiv.2s v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = sitofp <2 x i32> %in to <2 x float> + %div = fdiv <2 x float> %conv, + ret <2 x float> %div +} + +define <4 x float> @sitofp_v4f32_fdiv_4(<4 x i32> %in) { +; CHECK-SD-LABEL: sitofp_v4f32_fdiv_4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.4s v0, v0, #2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v4f32_fdiv_4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov.4s v1, #4.00000000 +; CHECK-GI-NEXT: scvtf.4s v0, v0 +; CHECK-GI-NEXT: fdiv.4s v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = sitofp <4 x i32> %in to <4 x float> + %div = fdiv <4 x float> %conv, + ret <4 x float> %div +} + +define <2 x double> @sitofp_v2f64_fdiv_16(<2 x i64> %in) { +; CHECK-SD-LABEL: sitofp_v2f64_fdiv_16: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.2d v0, v0, #4 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v2f64_fdiv_16: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov.2d v1, #16.00000000 +; CHECK-GI-NEXT: scvtf.2d v0, v0 +; CHECK-GI-NEXT: fdiv.2d v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = sitofp <2 x i64> %in to <2 x double> + %div = fdiv <2 x double> %conv, + ret <2 x double> %div +} + +define <4 x half> @sitofp_v4f16_fdiv_8(<4 x i16> %in) { +; CHECK-SD-LABEL: sitofp_v4f16_fdiv_8: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.4h v0, v0, #3 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v4f16_fdiv_8: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.4h v1, #72, lsl #8 +; CHECK-GI-NEXT: scvtf.4h v0, v0 +; CHECK-GI-NEXT: fdiv.4h v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = sitofp <4 x i16> %in to <4 x half> + %div = fdiv <4 x half> %conv, + ret <4 x half> %div +} + +define <8 x half> @sitofp_v8f16_fdiv_4(<8 x i16> %in) { +; CHECK-SD-LABEL: sitofp_v8f16_fdiv_4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.8h v0, v0, #2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v8f16_fdiv_4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.8h v1, #68, lsl #8 +; CHECK-GI-NEXT: scvtf.8h v0, v0 +; CHECK-GI-NEXT: fdiv.8h v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = sitofp <8 x i16> %in to <8 x half> + %div = fdiv <8 x half> %conv, + ret <8 x half> %div +} + +;; ---- sitofp + fmul (reciprocal power-of-2 multipliers) ---- + +define <2 x float> @sitofp_v2f32_fmul_half(<2 x i32> %in) { +; CHECK-LABEL: sitofp_v2f32_fmul_half: +; CHECK: // %bb.0: +; CHECK-NEXT: scvtf.2s v0, v0, #1 +; CHECK-NEXT: ret + %conv = sitofp <2 x i32> %in to <2 x float> + %mul = fmul <2 x float> %conv, + ret <2 x float> %mul +} + +define <4 x float> @sitofp_v4f32_fmul_quarter(<4 x i32> %in) { +; CHECK-LABEL: sitofp_v4f32_fmul_quarter: +; CHECK: // %bb.0: +; CHECK-NEXT: scvtf.4s v0, v0, #2 +; CHECK-NEXT: ret + %conv = sitofp <4 x i32> %in to <4 x float> + %mul = fmul <4 x float> %conv, + ret <4 x float> %mul +} + +define <2 x double> @sitofp_v2f64_fmul_eighth(<2 x i64> %in) { +; CHECK-LABEL: sitofp_v2f64_fmul_eighth: +; CHECK: // %bb.0: +; CHECK-NEXT: scvtf.2d v0, v0, #3 +; CHECK-NEXT: ret + %conv = sitofp <2 x i64> %in to <2 x double> + %mul = fmul <2 x double> %conv, + ret <2 x double> %mul +} + +define <4 x half> @sitofp_v4f16_fmul_half(<4 x i16> %in) { +; CHECK-LABEL: sitofp_v4f16_fmul_half: +; CHECK: // %bb.0: +; CHECK-NEXT: scvtf.4h v0, v0, #1 +; CHECK-NEXT: ret + %conv = sitofp <4 x i16> %in to <4 x half> + %mul = fmul <4 x half> %conv, + ret <4 x half> %mul +} + +define <8 x half> @sitofp_v8f16_fmul_quarter(<8 x i16> %in) { +; CHECK-LABEL: sitofp_v8f16_fmul_quarter: +; CHECK: // %bb.0: +; CHECK-NEXT: scvtf.8h v0, v0, #2 +; CHECK-NEXT: ret + %conv = sitofp <8 x i16> %in to <8 x half> + %mul = fmul <8 x half> %conv, + ret <8 x half> %mul +} + +;; ---- uitofp + fdiv (unsigned) ---- + +define <2 x float> @uitofp_v2f32_fdiv_2(<2 x i32> %in) { +; CHECK-SD-LABEL: uitofp_v2f32_fdiv_2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: ucvtf.2s v0, v0, #1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: uitofp_v2f32_fdiv_2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2s v1, #64, lsl #24 +; CHECK-GI-NEXT: ucvtf.2s v0, v0 +; CHECK-GI-NEXT: fdiv.2s v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = uitofp <2 x i32> %in to <2 x float> + %div = fdiv <2 x float> %conv, + ret <2 x float> %div +} + +define <4 x float> @uitofp_v4f32_fdiv_4(<4 x i32> %in) { +; CHECK-SD-LABEL: uitofp_v4f32_fdiv_4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: ucvtf.4s v0, v0, #2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: uitofp_v4f32_fdiv_4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov.4s v1, #4.00000000 +; CHECK-GI-NEXT: ucvtf.4s v0, v0 +; CHECK-GI-NEXT: fdiv.4s v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = uitofp <4 x i32> %in to <4 x float> + %div = fdiv <4 x float> %conv, + ret <4 x float> %div +} + +define <2 x double> @uitofp_v2f64_fdiv_16(<2 x i64> %in) { +; CHECK-SD-LABEL: uitofp_v2f64_fdiv_16: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: ucvtf.2d v0, v0, #4 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: uitofp_v2f64_fdiv_16: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov.2d v1, #16.00000000 +; CHECK-GI-NEXT: ucvtf.2d v0, v0 +; CHECK-GI-NEXT: fdiv.2d v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = uitofp <2 x i64> %in to <2 x double> + %div = fdiv <2 x double> %conv, + ret <2 x double> %div +} + +define <4 x half> @uitofp_v4f16_fdiv_8(<4 x i16> %in) { +; CHECK-SD-LABEL: uitofp_v4f16_fdiv_8: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: ucvtf.4h v0, v0, #3 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: uitofp_v4f16_fdiv_8: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.4h v1, #72, lsl #8 +; CHECK-GI-NEXT: ucvtf.4h v0, v0 +; CHECK-GI-NEXT: fdiv.4h v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = uitofp <4 x i16> %in to <4 x half> + %div = fdiv <4 x half> %conv, + ret <4 x half> %div +} + +define <8 x half> @uitofp_v8f16_fdiv_4(<8 x i16> %in) { +; CHECK-SD-LABEL: uitofp_v8f16_fdiv_4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: ucvtf.8h v0, v0, #2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: uitofp_v8f16_fdiv_4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.8h v1, #68, lsl #8 +; CHECK-GI-NEXT: ucvtf.8h v0, v0 +; CHECK-GI-NEXT: fdiv.8h v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = uitofp <8 x i16> %in to <8 x half> + %div = fdiv <8 x half> %conv, + ret <8 x half> %div +} + +;; ---- uitofp + fmul (unsigned, reciprocal) ---- + +define <2 x float> @uitofp_v2f32_fmul_half(<2 x i32> %in) { +; CHECK-LABEL: uitofp_v2f32_fmul_half: +; CHECK: // %bb.0: +; CHECK-NEXT: ucvtf.2s v0, v0, #1 +; CHECK-NEXT: ret + %conv = uitofp <2 x i32> %in to <2 x float> + %mul = fmul <2 x float> %conv, + ret <2 x float> %mul +} + +define <4 x float> @uitofp_v4f32_fmul_quarter(<4 x i32> %in) { +; CHECK-LABEL: uitofp_v4f32_fmul_quarter: +; CHECK: // %bb.0: +; CHECK-NEXT: ucvtf.4s v0, v0, #2 +; CHECK-NEXT: ret + %conv = uitofp <4 x i32> %in to <4 x float> + %mul = fmul <4 x float> %conv, + ret <4 x float> %mul +} + +define <2 x double> @uitofp_v2f64_fmul_eighth(<2 x i64> %in) { +; CHECK-LABEL: uitofp_v2f64_fmul_eighth: +; CHECK: // %bb.0: +; CHECK-NEXT: ucvtf.2d v0, v0, #3 +; CHECK-NEXT: ret + %conv = uitofp <2 x i64> %in to <2 x double> + %mul = fmul <2 x double> %conv, + ret <2 x double> %mul +} + +define <4 x half> @uitofp_v4f16_fmul_half(<4 x i16> %in) { +; CHECK-LABEL: uitofp_v4f16_fmul_half: +; CHECK: // %bb.0: +; CHECK-NEXT: ucvtf.4h v0, v0, #1 +; CHECK-NEXT: ret + %conv = uitofp <4 x i16> %in to <4 x half> + %mul = fmul <4 x half> %conv, + ret <4 x half> %mul +} + +define <8 x half> @uitofp_v8f16_fmul_quarter(<8 x i16> %in) { +; CHECK-LABEL: uitofp_v8f16_fmul_quarter: +; CHECK: // %bb.0: +; CHECK-NEXT: ucvtf.8h v0, v0, #2 +; CHECK-NEXT: ret + %conv = uitofp <8 x i16> %in to <8 x half> + %mul = fmul <8 x half> %conv, + ret <8 x half> %mul +} + +;; ---- boundary: max valid shift per element type ---- + +define <2 x float> @sitofp_v2f32_fdiv_max(<2 x i32> %in) { +; CHECK-SD-LABEL: sitofp_v2f32_fdiv_max: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.2s v0, v0, #32 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v2f32_fdiv_max: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: scvtf.2s v0, v0 +; CHECK-GI-NEXT: adrp x8, .LCPI20_0 +; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI20_0] +; CHECK-GI-NEXT: fdiv.2s v0, v0, v1 +; CHECK-GI-NEXT: ret + ; 2^32 = 4294967296.0 — max valid shift for i32 + %conv = sitofp <2 x i32> %in to <2 x float> + %div = fdiv <2 x float> %conv, + ret <2 x float> %div +} + +define <2 x double> @sitofp_v2f64_fdiv_max(<2 x i64> %in) { +; CHECK-SD-LABEL: sitofp_v2f64_fdiv_max: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.2d v0, v0, #64 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v2f64_fdiv_max: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: scvtf.2d v0, v0 +; CHECK-GI-NEXT: adrp x8, .LCPI21_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI21_0] +; CHECK-GI-NEXT: fdiv.2d v0, v0, v1 +; CHECK-GI-NEXT: ret + ; 2^64 — max valid shift for i64 + %conv = sitofp <2 x i64> %in to <2 x double> + %div = fdiv <2 x double> %conv, + ret <2 x double> %div +} + +define <4 x half> @sitofp_v4f16_fdiv_max(<4 x i16> %in) { +; CHECK-SD-LABEL: sitofp_v4f16_fdiv_max: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: movi d1, #0000000000000000 +; CHECK-SD-NEXT: scvtf.4h v0, v0 +; CHECK-SD-NEXT: fmul.4h v0, v0, v1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v4f16_fdiv_max: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.4h v1, #124, lsl #8 +; CHECK-GI-NEXT: scvtf.4h v0, v0 +; CHECK-GI-NEXT: fdiv.4h v0, v0, v1 +; CHECK-GI-NEXT: ret + ; 2^16 = 65536.0 — max valid shift for i16 (but not representable in f16) + ; f16 max is 65504, so 65536.0 overflows to inf. This should NOT match. + %conv = sitofp <4 x i16> %in to <4 x half> + %div = fdiv <4 x half> %conv, + ret <4 x half> %div +} + +define <4 x half> @sitofp_v4f16_fdiv_1024(<4 x i16> %in) { +; CHECK-SD-LABEL: sitofp_v4f16_fdiv_1024: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: scvtf.4h v0, v0, #10 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: sitofp_v4f16_fdiv_1024: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.4h v1, #100, lsl #8 +; CHECK-GI-NEXT: scvtf.4h v0, v0 +; CHECK-GI-NEXT: fdiv.4h v0, v0, v1 +; CHECK-GI-NEXT: ret + ; 2^10 = 1024.0 — shift=10, near upper end of f16 range + %conv = sitofp <4 x i16> %in to <4 x half> + %div = fdiv <4 x half> %conv, + ret <4 x half> %div +} + +;; ---- negative: out-of-range shift ---- + +define <2 x float> @neg_sitofp_v2f32_fdiv_too_large(<2 x i32> %in) { +; CHECK-SD-LABEL: neg_sitofp_v2f32_fdiv_too_large: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: movi.2s v1, #47, lsl #24 +; CHECK-SD-NEXT: scvtf.2s v0, v0 +; CHECK-SD-NEXT: fmul.2s v0, v0, v1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: neg_sitofp_v2f32_fdiv_too_large: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2s v1, #80, lsl #24 +; CHECK-GI-NEXT: scvtf.2s v0, v0 +; CHECK-GI-NEXT: fdiv.2s v0, v0, v1 +; CHECK-GI-NEXT: ret + ; 2^33 — exceeds i32 range, should NOT match + %conv = sitofp <2 x i32> %in to <2 x float> + %div = fdiv <2 x float> %conv, + ret <2 x float> %div +} + +define <2 x double> @neg_sitofp_v2f64_fdiv_too_large(<2 x i64> %in) { +; CHECK-SD-LABEL: neg_sitofp_v2f64_fdiv_too_large: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: mov x8, #4314448443020935168 // =0x3be0000000000000 +; CHECK-SD-NEXT: scvtf.2d v0, v0 +; CHECK-SD-NEXT: dup.2d v1, x8 +; CHECK-SD-NEXT: fmul.2d v0, v0, v1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: neg_sitofp_v2f64_fdiv_too_large: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: scvtf.2d v0, v0 +; CHECK-GI-NEXT: adrp x8, .LCPI25_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI25_0] +; CHECK-GI-NEXT: fdiv.2d v0, v0, v1 +; CHECK-GI-NEXT: ret + ; 2^65 — exceeds i64 range, should NOT match + %conv = sitofp <2 x i64> %in to <2 x double> + %div = fdiv <2 x double> %conv, + ret <2 x double> %div +} + +;; ---- negative: non-power-of-2 ---- + +define <2 x float> @neg_v2f32_fmul_non_pow2(<2 x i32> %in) { +; CHECK-SD-LABEL: neg_v2f32_fmul_non_pow2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: mov w8, #36704 // =0x8f60 +; CHECK-SD-NEXT: scvtf.2s v0, v0 +; CHECK-SD-NEXT: movk w8, #9666, lsl #16 +; CHECK-SD-NEXT: dup.2s v1, w8 +; CHECK-SD-NEXT: fmul.2s v0, v0, v1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: neg_v2f32_fmul_non_pow2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: scvtf.2s v0, v0 +; CHECK-GI-NEXT: adrp x8, .LCPI26_0 +; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI26_0] +; CHECK-GI-NEXT: fmul.2s v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = sitofp <2 x i32> %in to <2 x float> + %mul = fmul <2 x float> %conv, + ret <2 x float> %mul +} + +define <4 x float> @neg_v4f32_fmul_non_pow2(<4 x i32> %in) { +; CHECK-SD-LABEL: neg_v4f32_fmul_non_pow2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: mov w8, #36704 // =0x8f60 +; CHECK-SD-NEXT: scvtf.4s v0, v0 +; CHECK-SD-NEXT: movk w8, #9666, lsl #16 +; CHECK-SD-NEXT: dup.4s v1, w8 +; CHECK-SD-NEXT: fmul.4s v0, v0, v1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: neg_v4f32_fmul_non_pow2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: scvtf.4s v0, v0 +; CHECK-GI-NEXT: adrp x8, .LCPI27_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI27_0] +; CHECK-GI-NEXT: fmul.4s v0, v0, v1 +; CHECK-GI-NEXT: ret + %conv = sitofp <4 x i32> %in to <4 x float> + %mul = fmul <4 x float> %conv, + ret <4 x float> %mul +} + +define <2 x double> @neg_v2f64_fmul_non_pow2(<2 x i64> %in) { +; CHECK-SD-LABEL: neg_v2f64_fmul_non_pow2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: mov x8, #5243 // =0x147b +; CHECK-SD-NEXT: scvtf.2d v0, v0 +; CHECK-SD-NEXT: movk x8, #18350, lsl #16 +; CHECK-SD-NEXT: movk x8, #31457, lsl #32 +; CHECK-SD-NEXT: movk x8, #16276, lsl #48 +; CHECK-SD-NEXT: dup.2d v1, x8 +; CHECK-SD-NEXT: fmul.2d v0, v0, v1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: neg_v2f64_fmul_non_pow2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: scvtf.2d v0, v0 +; CHECK-GI-NEXT: adrp x8, .LCPI28_0 +; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI28_0] +; CHECK-GI-NEXT: fmul.2d v0, v0, v1[0] +; CHECK-GI-NEXT: ret + %conv = sitofp <2 x i64> %in to <2 x double> + %mul = fmul <2 x double> %conv, + ret <2 x double> %mul +}