Skip to content

[AArch64] Optimize vector fmul(sitofp/uitofp, 1/2^N) -> scvtf/ucvtf #141480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davemgreen merged 1 commit into from
Apr 26, 2026
Merged

[AArch64] Optimize vector fmul(sitofp/uitofp, 1/2^N) -> scvtf/ucvtf #141480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 34 additions & 10 deletions llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,14 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos,
unsigned Width);

template <unsigned FloatWidth>
bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) {
return SelectCVTFixedPosRecipOperandVec(N, FixedPos, FloatWidth);
}

bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos,
unsigned Width);

bool SelectCMP_SWAP(SDNode *N);

bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
Expand Down Expand Up @@ -4147,14 +4155,11 @@ static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N,
return false;
}

bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
unsigned RegWidth) {
return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
/*isReciprocal*/ false);
}

bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
unsigned RegWidth) {
static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG,
SDValue N,
SDValue &FixedPos,
unsigned RegWidth,
bool isReciprocal) {
if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) &&
N.getValueType().getScalarSizeInBits() ==
N.getOperand(0).getValueType().getScalarSizeInBits())
Expand Down Expand Up @@ -4192,15 +4197,34 @@ bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
return false;
}

if (unsigned FBits = CheckFixedPointOperandConstant(FVal, RegWidth,
/*isReciprocal*/ false)) {
if (unsigned FBits =
CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32);
return true;
}

return false;
}

bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
unsigned RegWidth) {
return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
/*isReciprocal*/ false);
}

bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
unsigned RegWidth) {
return checkCVTFixedPointOperandWithFBitsForVectors(
CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false);
}

bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N,
SDValue &FixedPos,
unsigned RegWidth) {
return checkCVTFixedPointOperandWithFBitsForVectors(
CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true);
}

bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N,
SDValue &FixedPos,
unsigned RegWidth) {
Expand Down
57 changes: 57 additions & 0 deletions llvm/lib/Target/AArch64/AArch64InstrInfo.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9265,6 +9265,53 @@ defm : FCVTPat<v4f16, v4i16, i16, V64, fixedpoint_v4f16>;
defm : FCVTPat<v8f16, v8i16, i16, V128, fixedpoint_v8f16>;
}

// fmul(sitofp(x), 1/2^N) -> scvtf(x, N), fmul(uitofp(x), 1/2^N) -> ucvtf(x, N)
class fixedpoint_recip_vec_f64<ValueType FloatVT>
: ComplexPattern<FloatVT, 1, "SelectCVTFixedPosRecipOperandVec<64>">;
class fixedpoint_recip_vec_f32<ValueType FloatVT>
: ComplexPattern<FloatVT, 1, "SelectCVTFixedPosRecipOperandVec<32>">;
class fixedpoint_recip_vec_f16<ValueType FloatVT>
: ComplexPattern<FloatVT, 1, "SelectCVTFixedPosRecipOperandVec<16>">;
def fixedpoint_recip_vec_xform : SDNodeXForm<timm, [{
(void)N;
return V;
}]>;
def gi_fixedpoint_recip_vec_xform
: GICustomOperandRenderer<"renderFixedPointRecipXForm">,
GISDNodeXFormEquiv<fixedpoint_recip_vec_xform>;

def fixedpoint_recip_v2f64 : fixedpoint_recip_vec_f64<v2f64>;
def fixedpoint_recip_v2f32 : fixedpoint_recip_vec_f32<v2f32>;
def fixedpoint_recip_v4f32 : fixedpoint_recip_vec_f32<v4f32>;
def fixedpoint_recip_v4f16 : fixedpoint_recip_vec_f16<v4f16>;
def fixedpoint_recip_v8f16 : fixedpoint_recip_vec_f16<v8f16>;

def gi_fixedpoint_recip_v2f64
: GIComplexOperandMatcher<v2s64, "selectCVTFixedPosRecipOperandVec">,
GIComplexPatternEquiv<fixedpoint_recip_v2f64>;
def gi_fixedpoint_recip_v2f32
: GIComplexOperandMatcher<v2s32, "selectCVTFixedPosRecipOperandVec">,
GIComplexPatternEquiv<fixedpoint_recip_v2f32>;
def gi_fixedpoint_recip_v4f32
: GIComplexOperandMatcher<v4s32, "selectCVTFixedPosRecipOperandVec">,
GIComplexPatternEquiv<fixedpoint_recip_v4f32>;
def gi_fixedpoint_recip_v4f16
: GIComplexOperandMatcher<v4s16, "selectCVTFixedPosRecipOperandVec">,
GIComplexPatternEquiv<fixedpoint_recip_v4f16>;
def gi_fixedpoint_recip_v8f16
: GIComplexOperandMatcher<v8s16, "selectCVTFixedPosRecipOperandVec">,
GIComplexPatternEquiv<fixedpoint_recip_v8f16>;

multiclass CVTFRecipPat<ValueType FVT, ValueType IVT,
RegisterOperand RC, ComplexPattern fixedpoint> {
def : Pat<(FVT (fmul (sint_to_fp (IVT RC:$Rn)), fixedpoint:$scale)),
(!cast<Instruction>("SCVTF"#IVT#"_shift") RC:$Rn,
(fixedpoint_recip_vec_xform fixedpoint:$scale))>;
def : Pat<(FVT (fmul (uint_to_fp (IVT RC:$Rn)), fixedpoint:$scale)),
(!cast<Instruction>("UCVTF"#IVT#"_shift") RC:$Rn,
(fixedpoint_recip_vec_xform fixedpoint:$scale))>;
}

// X << 1 ==> X + X
class SHLToADDPat<ValueType ty, RegisterClass regtype>
: Pat<(ty (AArch64vshl (ty regtype:$Rn), (i32 1))),
Expand Down Expand Up @@ -9319,6 +9366,16 @@ defm USHR : SIMDVectorRShiftBHSD<1, 0b00000, "ushr", AArch64vlshr>;
defm USRA : SIMDVectorRShiftBHSDTied<1, 0b00010, "usra",
TriOpFrag<(add_like node:$LHS, (AArch64vlshr node:$MHS, node:$RHS))> >;

let Predicates = [HasNEON] in {
defm : CVTFRecipPat<v2f64, v2i64, V128, fixedpoint_recip_v2f64>;
defm : CVTFRecipPat<v2f32, v2i32, V64, fixedpoint_recip_v2f32>;
defm : CVTFRecipPat<v4f32, v4i32, V128, fixedpoint_recip_v4f32>;
}
let Predicates = [HasNEON, HasFullFP16] in {
defm : CVTFRecipPat<v4f16, v4i16, V64, fixedpoint_recip_v4f16>;
defm : CVTFRecipPat<v8f16, v8i16, V128, fixedpoint_recip_v8f16>;
}

def VImm0080: PatLeaf<(AArch64movi_shift (i32 128), (i32 0))>;
def VImm00008000: PatLeaf<(AArch64movi_shift (i32 128), (i32 8))>;
def VImm0000000080000000: PatLeaf<(AArch64NvCast (v2f64 (fneg (AArch64NvCast (v4i32 (AArch64movi_shift (i32 128), (i32 24)))))))>;
Expand Down
32 changes: 26 additions & 6 deletions llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,9 +484,14 @@ class AArch64InstructionSelector : public InstructionSelector {

ComplexRendererFns selectCVTFixedPointVec(MachineOperand &Root) const;
ComplexRendererFns
selectCVTFixedPointVecBase(const MachineOperand &Root) const;
selectCVTFixedPosRecipOperandVec(MachineOperand &Root) const;
ComplexRendererFns
selectCVTFixedPointVecBase(const MachineOperand &Root,
bool isReciprocal = false) const;
void renderFixedPointXForm(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx = -1) const;
void renderFixedPointRecipXForm(MachineInstrBuilder &MIB,
const MachineInstr &MI, int OpIdx = -1) const;

void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx = -1) const;
Expand Down Expand Up @@ -7853,7 +7858,7 @@ AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root) const {

InstructionSelector::ComplexRendererFns
AArch64InstructionSelector::selectCVTFixedPointVecBase(
const MachineOperand &Root) const {
const MachineOperand &Root, bool isReciprocal) const {
if (!Root.isReg())
return std::nullopt;
const MachineRegisterInfo &MRI =
Expand Down Expand Up @@ -7882,16 +7887,22 @@ AArch64InstructionSelector::selectCVTFixedPointVecBase(
default:
return std::nullopt;
};
if (unsigned FBits = CheckFixedPointOperandConstant(FVal, RegWidth,
/*isReciprocal*/ false))
if (unsigned FBits =
CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal))
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(FBits); }}};

return std::nullopt;
}

InstructionSelector::ComplexRendererFns
AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root) const {
return selectCVTFixedPointVecBase(Root);
return selectCVTFixedPointVecBase(Root, /*isReciprocal*/ false);
}

InstructionSelector::ComplexRendererFns
AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec(
MachineOperand &Root) const {
return selectCVTFixedPointVecBase(Root, /*isReciprocal*/ true);
}

void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
Expand All @@ -7901,12 +7912,21 @@ void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
// should be able to reuse the Renderers already calculated by
// selectCVTFixedPointVecBase.
InstructionSelector::ComplexRendererFns Renderer =
selectCVTFixedPointVecBase(MI.getOperand(2));
selectCVTFixedPointVecBase(MI.getOperand(2), /*isReciprocal*/ false);
assert((Renderer && Renderer->size() == 1) &&
"Expected selectCVTFixedPointVec to provide a function\n");
(Renderer->front())(MIB);
}

void AArch64InstructionSelector::renderFixedPointRecipXForm(
MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
InstructionSelector::ComplexRendererFns Renderer =
selectCVTFixedPointVecBase(MI.getOperand(2), /*isReciprocal*/ true);
assert((Renderer && Renderer->size() == 1) &&
"Expected selectCVTFixedPosRecipOperandVec to provide a function\n");
(Renderer->front())(MIB);
}

void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const {
Expand Down
Loading