diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 21afcbefdf719..3696625133090 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1261,10 +1261,32 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { EVT ExtVT = EVT::getEVT(Dst); EVT LoadVT = EVT::getEVT(Src); unsigned LType = - ((Opcode == Instruction::ZExt) ? ISD::ZEXTLOAD : ISD::SEXTLOAD); - if (DstLT.first == SrcLT.first && - TLI->isLoadExtLegal(LType, ExtVT, LoadVT)) - return 0; + ((Opcode == Instruction::ZExt) ? ISD::ZEXTLOAD : ISD::SEXTLOAD); + if (I) { + if (auto *LI = dyn_cast(I->getOperand(0))) { + if (DstLT.first == SrcLT.first && + TLI->isLoadLegal(ExtVT, LoadVT, LI->getAlign(), + LI->getPointerAddressSpace(), LType, false)) + return 0; + } else if (auto *II = dyn_cast(I->getOperand(0))) { + switch (II->getIntrinsicID()) { + case Intrinsic::masked_load: { + Type *PtrType = II->getArgOperand(0)->getType(); + assert(PtrType->isPointerTy()); + + if (DstLT.first == SrcLT.first && + TLI->isLoadLegal( + ExtVT, LoadVT, II->getParamAlign(0).valueOrOne(), + PtrType->getPointerAddressSpace(), LType, false)) + return 0; + + break; + } + default: + break; + } + } + } } break; case Instruction::AddrSpaceCast: @@ -1556,7 +1578,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { if (Opcode == Instruction::Store) LA = getTLI()->getTruncStoreAction(LT.second, MemVT); else - LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT); + LA = getTLI()->getLoadAction(LT.second, MemVT, Alignment, AddressSpace, + ISD::EXTLOAD, false); if (LA != TargetLowering::Legal && LA != TargetLowering::Custom) { // This is a vector load/store for some illegal type that is scalarized. diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index ada4ffd3bcc89..73ac6643b8ed2 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -31,6 +31,7 @@ #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/LibcallLoweringInfo.h" #include "llvm/CodeGen/LowLevelTypeUtils.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RuntimeLibcallUtil.h" #include "llvm/CodeGen/SelectionDAG.h" @@ -1511,49 +1512,25 @@ class LLVM_ABI TargetLoweringBase { /// Return how this load with extension should be treated: either it is legal, /// needs to be promoted to a larger size, needs to be expanded to some other /// code sequence, or the target has a custom expander for it. - LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT, - EVT MemVT) const { - if (ValVT.isExtended() || MemVT.isExtended()) return Expand; - unsigned ValI = (unsigned) ValVT.getSimpleVT().SimpleTy; - unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy; - assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::VALUETYPE_SIZE && - MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!"); - unsigned Shift = 4 * ExtType; - return (LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf); - } + virtual LegalizeAction getLoadAction(EVT ValVT, EVT MemVT, Align Alignment, + unsigned AddrSpace, unsigned ExtType, + bool Atomic) const; /// Return true if the specified load with extension is legal on this target. - bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const { - return getLoadExtAction(ExtType, ValVT, MemVT) == Legal; + bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, + unsigned ExtType, bool Atomic) const { + return getLoadAction(ValVT, MemVT, Alignment, AddrSpace, ExtType, Atomic) == + Legal; } /// Return true if the specified load with extension is legal or custom /// on this target. - bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const { - return getLoadExtAction(ExtType, ValVT, MemVT) == Legal || - getLoadExtAction(ExtType, ValVT, MemVT) == Custom; - } - - /// Same as getLoadExtAction, but for atomic loads. - LegalizeAction getAtomicLoadExtAction(unsigned ExtType, EVT ValVT, - EVT MemVT) const { - if (ValVT.isExtended() || MemVT.isExtended()) return Expand; - unsigned ValI = (unsigned)ValVT.getSimpleVT().SimpleTy; - unsigned MemI = (unsigned)MemVT.getSimpleVT().SimpleTy; - assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::VALUETYPE_SIZE && - MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!"); - unsigned Shift = 4 * ExtType; + bool isLoadLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment, + unsigned AddrSpace, unsigned ExtType, + bool Atomic) const { LegalizeAction Action = - (LegalizeAction)((AtomicLoadExtActions[ValI][MemI] >> Shift) & 0xf); - assert((Action == Legal || Action == Expand) && - "Unsupported atomic load extension action."); - return Action; - } - - /// Return true if the specified atomic load with extension is legal on - /// this target. - bool isAtomicLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const { - return getAtomicLoadExtAction(ExtType, ValVT, MemVT) == Legal; + getLoadAction(ValVT, MemVT, Alignment, AddrSpace, ExtType, Atomic); + return Action == Legal || Action == Custom; } /// Return how this store with truncation should be treated: either it is @@ -3182,7 +3159,8 @@ class LLVM_ABI TargetLoweringBase { LType = ISD::SEXTLOAD; } - return isLoadExtLegal(LType, VT, LoadVT); + return isLoadLegal(VT, LoadVT, Load->getAlign(), + Load->getPointerAddressSpace(), LType, false); } /// Return true if any actual instruction that defines a value of type FromTy diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index bf56ffd3b4b7b..e537d113c377f 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -7495,12 +7495,12 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) { uint32_t ActiveBits = DemandBits.getActiveBits(); // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the - // target even if isLoadExtLegal says an i1 EXTLOAD is valid. For example, - // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but - // (and (load x) 1) is not matched as a single instruction, rather as a LDR - // followed by an AND. + // target even if isLoadLegal says an i1 EXTLOAD is valid. For example, + // for the AArch64 target isLoadLegal(i32, i1, ..., ZEXTLOAD, false) returns + // true, but (and (load x) 1) is not matched as a single instruction, rather + // as a LDR followed by an AND. // TODO: Look into removing this restriction by fixing backends to either - // return false for isLoadExtLegal for i1 or have them select this pattern to + // return false for isLoadLegal for i1 or have them select this pattern to // a single instruction. // // Also avoid hoisting if we didn't see any ands with the exact DemandBits @@ -7515,7 +7515,8 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) { // Reject cases that won't be matched as extloads. if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() || - !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT)) + !TLI->isLoadLegal(LoadResultVT, TruncVT, Load->getAlign(), + Load->getPointerAddressSpace(), ISD::ZEXTLOAD, false)) return false; IRBuilder<> Builder(Load->getNextNode()); diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 58c59628342c7..4a1b02c294cd7 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -7047,7 +7047,8 @@ bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN, if (ExtVT == LoadedVT && (!LegalOperations || - TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) { + TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(), + LoadN->getAddressSpace(), ISD::ZEXTLOAD, false))) { // ZEXTLOAD will match without needing to change the size of the value being // loaded. return true; @@ -7063,7 +7064,8 @@ bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN, return false; if (LegalOperations && - !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT)) + !TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(), + LoadN->getAddressSpace(), ISD::ZEXTLOAD, false)) return false; if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT, /*ByteOffset=*/0)) @@ -7126,7 +7128,8 @@ bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST, return false; if (LegalOperations && - !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT)) + !TLI.isLoadLegal(Load->getValueType(0), MemVT, Load->getAlign(), + Load->getAddressSpace(), ExtType, false)) return false; // For the transform to be legal, the load must produce only two values @@ -7638,7 +7641,8 @@ SDValue DAGCombiner::visitAND(SDNode *N) { ConstantSDNode *Splat = isConstOrConstSplat(N1, true, true); if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat) { EVT MemVT = MLoad->getMemoryVT(); - if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT)) { + if (TLI.isLoadLegal(VT, MemVT, MLoad->getAlign(), + MLoad->getAddressSpace(), ISD::ZEXTLOAD, false)) { // For this AND to be a zero extension of the masked load the elements // of the BuildVec must mask the bottom bits of the extended element // type @@ -7785,9 +7789,9 @@ SDValue DAGCombiner::visitAND(SDNode *N) { // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is // actually legal and isn't going to get expanded, else this is a false // optimisation. - bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD, - Load->getValueType(0), - Load->getMemoryVT()); + bool CanZextLoadProfitably = TLI.isLoadLegal( + Load->getValueType(0), Load->getMemoryVT(), Load->getAlign(), + Load->getAddressSpace(), ISD::ZEXTLOAD, false); // Resize the constant to the same size as the original memory access before // extension. If it is still the AllOnesValue then this AND is completely @@ -7979,7 +7983,8 @@ SDValue DAGCombiner::visitAND(SDNode *N) { APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize); if (DAG.MaskedValueIsZero(N1, ExtBits) && ((!LegalOperations && LN0->isSimple()) || - TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { + TLI.isLoadLegal(VT, MemVT, LN0->getAlign(), LN0->getAddressSpace(), + ISD::ZEXTLOAD, false))) { SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), LN0->getBasePtr(), MemVT, LN0->getMemOperand()); @@ -9901,10 +9906,16 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) { // Before legalize we can introduce too wide illegal loads which will be later // split into legal sized loads. This enables us to combine i64 load by i8 // patterns to a couple of i32 loads on 32 bit targets. - if (LegalOperations && - !TLI.isLoadExtLegal(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, VT, - MemVT)) - return SDValue(); + if (LegalOperations) { + for (auto *L : Loads) { + if (!TLI.isLoadLegal(VT, MemVT, L->getAlign(), L->getAddressSpace(), + NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, + false)) { + + return SDValue(); + } + } + } // Check if the bytes of the OR we are looking at match with either big or // little endian value load @@ -13782,20 +13793,26 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) { unsigned WideWidth = WideVT.getScalarSizeInBits(); bool IsSigned = isSignedIntSetCC(CC); auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD; - if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && - SetCCWidth != 1 && SetCCWidth < WideWidth && - TLI.isLoadExtLegalOrCustom(LoadExtOpcode, WideVT, NarrowVT) && + if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && SetCCWidth != 1 && + SetCCWidth < WideWidth && TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) { - // Both compare operands can be widened for free. The LHS can use an - // extended load, and the RHS is a constant: - // vselect (ext (setcc load(X), C)), N1, N2 --> - // vselect (setcc extload(X), C'), N1, N2 - auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; - SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS); - SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS); - EVT WideSetCCVT = getSetCCResultType(WideVT); - SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC); - return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2); + LoadSDNode *Ld = cast(LHS); + + if (TLI.isLoadLegalOrCustom(WideVT, NarrowVT, Ld->getAlign(), + Ld->getAddressSpace(), LoadExtOpcode, + false)) { + // Both compare operands can be widened for free. The LHS can use an + // extended load, and the RHS is a constant: + // vselect (ext (setcc load(X), C)), N1, N2 --> + // vselect (setcc extload(X), C'), N1, N2 + auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS); + SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS); + EVT WideSetCCVT = getSetCCResultType(WideVT); + SDValue WideSetCC = + DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC); + return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2); + } } } @@ -14230,8 +14247,10 @@ static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI, // Combine2), so we should conservatively check the OperationAction. LoadSDNode *Load1 = cast(Op1); LoadSDNode *Load2 = cast(Op2); - if (!TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load1->getMemoryVT()) || - !TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load2->getMemoryVT()) || + if (!TLI.isLoadLegal(VT, Load1->getMemoryVT(), Load1->getAlign(), + Load1->getAddressSpace(), ExtLoadOpcode, false) || + !TLI.isLoadLegal(VT, Load2->getMemoryVT(), Load2->getAlign(), + Load2->getAddressSpace(), ExtLoadOpcode, false) || (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes && TLI.getOperationAction(ISD::VSELECT, VT) != TargetLowering::Legal)) return SDValue(); @@ -14455,13 +14474,15 @@ SDValue DAGCombiner::CombineExtLoad(SDNode *N) { // Try to split the vector types to get down to legal types. EVT SplitSrcVT = SrcVT; EVT SplitDstVT = DstVT; - while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) && + while (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(), + LN0->getAddressSpace(), ExtType, false) && SplitSrcVT.getVectorNumElements() > 1) { SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first; SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first; } - if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT)) + if (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(), + LN0->getAddressSpace(), ExtType, false)) return SDValue(); assert(!DstVT.isScalableVector() && "Unexpected scalable vector type"); @@ -14534,7 +14555,8 @@ SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) { return SDValue(); LoadSDNode *Load = cast(N1.getOperand(0)); EVT MemVT = Load->getMemoryVT(); - if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || + if (!TLI.isLoadLegal(VT, MemVT, Load->getAlign(), Load->getAddressSpace(), + ISD::ZEXTLOAD, false) || Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed()) return SDValue(); @@ -14647,7 +14669,8 @@ static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, EVT MemVT = OldExtLoad->getMemoryVT(); if ((LegalOperations || !OldExtLoad->isSimple() || VT.isVector()) && - !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT)) + !TLI.isLoadLegal(VT, MemVT, OldExtLoad->getAlign(), + OldExtLoad->getAddressSpace(), ExtLoadType, false)) return SDValue(); SDLoc DL(OldExtLoad); @@ -14707,7 +14730,8 @@ static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, // code generation being the result of that target's implementation of // isVectorLoadExtDesirable(). if ((LegalOperations || VT.isFixedLengthVector() || !Load->isSimple()) && - !TLI.isLoadExtLegal(ExtLoadType, VT, Load->getValueType(0))) + !TLI.isLoadLegal(VT, Load->getValueType(0), Load->getAlign(), + Load->getAddressSpace(), ExtLoadType, false)) return {}; bool DoXform = true; @@ -14763,7 +14787,8 @@ tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, return SDValue(); if ((LegalOperations || !cast(N0)->isSimple()) && - !TLI.isLoadExtLegalOrCustom(ExtLoadType, VT, Ld->getValueType(0))) + !TLI.isLoadLegalOrCustom(VT, Ld->getValueType(0), Ld->getAlign(), + Ld->getAddressSpace(), ExtLoadType, false)) return SDValue(); if (!TLI.isVectorLoadExtDesirable(SDValue(N, 0))) @@ -14788,7 +14813,8 @@ static SDValue tryToFoldExtOfAtomicLoad(SelectionDAG &DAG, if (!ALoad || ALoad->getOpcode() != ISD::ATOMIC_LOAD) return {}; EVT MemoryVT = ALoad->getMemoryVT(); - if (!TLI.isAtomicLoadExtLegal(ExtLoadType, VT, MemoryVT)) + if (!TLI.isLoadLegal(VT, MemoryVT, ALoad->getAlign(), + ALoad->getAddressSpace(), ExtLoadType, true)) return {}; // Can't fold into ALoad if it is already extending differently. ISD::LoadExtType ALoadExtTy = ALoad->getExtensionType(); @@ -14905,9 +14931,14 @@ SDValue DAGCombiner::foldSextSetcc(SDNode *N) { // legal {z/s}ext-load. // TODO: Allow widening of an existing {z/s}ext-load? if (!(ISD::isNON_EXTLoad(V.getNode()) && - ISD::isUNINDEXEDLoad(V.getNode()) && - cast(V)->isSimple() && - TLI.isLoadExtLegal(LoadOpcode, VT, V.getValueType()))) + ISD::isUNINDEXEDLoad(V.getNode()))) + return false; + + LoadSDNode *Ld = cast(V.getNode()); + + if (!Ld->isSimple() || + !TLI.isLoadLegal(VT, V.getValueType(), Ld->getAlign(), + Ld->getAddressSpace(), LoadOpcode, false)) return false; // Non-chain users of this value must either be the setcc in this @@ -15104,8 +15135,9 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { LoadSDNode *LN00 = cast(N0.getOperand(0)); EVT MemVT = LN00->getMemoryVT(); - if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && - LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) { + if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(), LN00->getAddressSpace(), + ISD::SEXTLOAD, false) && + LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) { SmallVector SetCCs; bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), ISD::SIGN_EXTEND, SetCCs, TLI); @@ -15422,7 +15454,8 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { LoadSDNode *LN00 = cast(N0.getOperand(0)); EVT MemVT = LN00->getMemoryVT(); - if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && + if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(), LN00->getAddressSpace(), + ISD::ZEXTLOAD, false) && LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) { bool DoXform = true; SmallVector SetCCs; @@ -15652,31 +15685,33 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) return foldedExt; } else if (ISD::isNON_EXTLoad(N0.getNode()) && - ISD::isUNINDEXEDLoad(N0.getNode()) && - TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { - bool DoXform = true; - SmallVector SetCCs; - if (!N0.hasOneUse()) - DoXform = - ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); - if (DoXform) { - LoadSDNode *LN0 = cast(N0); - SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(), - LN0->getBasePtr(), N0.getValueType(), - LN0->getMemOperand()); - ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND); - // If the load value is used only by N, replace it via CombineTo N. - bool NoReplaceTrunc = N0.hasOneUse(); - CombineTo(N, ExtLoad); - if (NoReplaceTrunc) { - DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1)); - recursivelyDeleteUnusedNodes(LN0); - } else { - SDValue Trunc = - DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); - CombineTo(LN0, Trunc, ExtLoad.getValue(1)); + ISD::isUNINDEXEDLoad(N0.getNode())) { + LoadSDNode *LN0 = cast(N0); + if (TLI.isLoadLegalOrCustom(VT, N0.getValueType(), LN0->getAlign(), + LN0->getAddressSpace(), ISD::EXTLOAD, false)) { + bool DoXform = true; + SmallVector SetCCs; + if (!N0.hasOneUse()) + DoXform = + ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); + if (DoXform) { + SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(), + LN0->getBasePtr(), N0.getValueType(), + LN0->getMemOperand()); + ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND); + // If the load value is used only by N, replace it via CombineTo N. + bool NoReplaceTrunc = N0.hasOneUse(); + CombineTo(N, ExtLoad); + if (NoReplaceTrunc) { + DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1)); + recursivelyDeleteUnusedNodes(LN0); + } else { + SDValue Trunc = + DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); + CombineTo(LN0, Trunc, ExtLoad.getValue(1)); + } + return SDValue(N, 0); // Return N so it doesn't get rechecked! } - return SDValue(N, 0); // Return N so it doesn't get rechecked! } } @@ -15688,7 +15723,9 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { LoadSDNode *LN0 = cast(N0); ISD::LoadExtType ExtType = LN0->getExtensionType(); EVT MemVT = LN0->getMemoryVT(); - if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) { + if (!LegalOperations || + TLI.isLoadLegal(VT, MemVT, LN0->getAlign(), LN0->getAddressSpace(), + ExtType, false)) { SDValue ExtLoad = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), LN0->getBasePtr(), MemVT, LN0->getMemOperand()); @@ -16002,7 +16039,8 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) { EVT::getIntegerVT(*DAG.getContext(), ShiftMask.countr_one()); // If the mask is smaller, recompute the type. if ((ExtVT.getScalarSizeInBits() > MaskedVT.getScalarSizeInBits()) && - TLI.isLoadExtLegal(ExtType, SRL.getValueType(), MaskedVT)) + TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(), + LN->getAddressSpace(), ExtType, false)) ExtVT = MaskedVT; } else if (ExtType == ISD::ZEXTLOAD && ShiftMask.isShiftedMask(Offset, ActiveBits) && @@ -16011,7 +16049,8 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) { // If the mask is shifted we can use a narrower load and a shl to insert // the trailing zeros. if (((Offset + ActiveBits) <= ExtVT.getScalarSizeInBits()) && - TLI.isLoadExtLegal(ExtType, SRL.getValueType(), MaskedVT)) { + TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(), + LN->getAddressSpace(), ExtType, false)) { ExtVT = MaskedVT; ShAmt = Offset + ShAmt; ShiftedOffset = Offset; @@ -16233,33 +16272,37 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { // If sextload is not supported by target, we can only do the combine when // load has one use. Doing otherwise can block folding the extload with other // extends that the target does support. - if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && - ExtVT == cast(N0)->getMemoryVT() && - ((!LegalOperations && cast(N0)->isSimple() && - N0.hasOneUse()) || - TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { + if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { auto *LN0 = cast(N0); - SDValue ExtLoad = - DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(), - LN0->getBasePtr(), ExtVT, LN0->getMemOperand()); - CombineTo(N, ExtLoad); - CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); - AddToWorklist(ExtLoad.getNode()); - return SDValue(N, 0); // Return N so it doesn't get rechecked! + if (ExtVT == LN0->getMemoryVT() && + ((!LegalOperations && LN0->isSimple() && N0.hasOneUse()) || + TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(), LN0->getAddressSpace(), + ISD::SEXTLOAD, false))) { + SDValue ExtLoad = + DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(), + LN0->getBasePtr(), ExtVT, LN0->getMemOperand()); + CombineTo(N, ExtLoad); + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + AddToWorklist(ExtLoad.getNode()); + return SDValue(N, 0); // Return N so it doesn't get rechecked! + } } // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use - if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && - N0.hasOneUse() && ExtVT == cast(N0)->getMemoryVT() && - ((!LegalOperations && cast(N0)->isSimple()) && - TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { + if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { auto *LN0 = cast(N0); - SDValue ExtLoad = - DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(), - LN0->getBasePtr(), ExtVT, LN0->getMemOperand()); - CombineTo(N, ExtLoad); - CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); - return SDValue(N, 0); // Return N so it doesn't get rechecked! + + if (N0.hasOneUse() && ExtVT == LN0->getMemoryVT() && + ((!LegalOperations && LN0->isSimple()) && + TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(), LN0->getAddressSpace(), + ISD::SEXTLOAD, false))) { + SDValue ExtLoad = + DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(), + LN0->getBasePtr(), ExtVT, LN0->getMemOperand()); + CombineTo(N, ExtLoad); + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! + } } // fold (sext_inreg (masked_load x)) -> (sext_masked_load x) @@ -16268,7 +16311,8 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { if (auto *Ld = dyn_cast(Frozen ? N0.getOperand(0) : N0)) { if (ExtVT == Ld->getMemoryVT() && Ld->hasNUsesOfValue(1, 0) && Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD && - TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) { + TLI.isLoadLegal(VT, ExtVT, Ld->getAlign(), Ld->getAddressSpace(), + ISD::SEXTLOAD, false)) { SDValue ExtMaskedLoad = DAG.getMaskedLoad( VT, DL, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(), @@ -19726,20 +19770,21 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) { } // fold (fpext (load x)) -> (fpext (fptrunc (extload x))) - if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && - TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { + if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse()) { LoadSDNode *LN0 = cast(N0); - SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, - LN0->getChain(), - LN0->getBasePtr(), N0.getValueType(), - LN0->getMemOperand()); - CombineTo(N, ExtLoad); - CombineTo( - N0.getNode(), - DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), ExtLoad, - DAG.getIntPtrConstant(1, SDLoc(N0), /*isTarget=*/true)), - ExtLoad.getValue(1)); - return SDValue(N, 0); // Return N so it doesn't get rechecked! + if (TLI.isLoadLegalOrCustom(VT, N0.getValueType(), LN0->getAlign(), + LN0->getAddressSpace(), ISD::EXTLOAD, false)) { + SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(), + LN0->getBasePtr(), N0.getValueType(), + LN0->getMemOperand()); + CombineTo(N, ExtLoad); + CombineTo( + N0.getNode(), + DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), ExtLoad, + DAG.getIntPtrConstant(1, SDLoc(N0), /*isTarget=*/true)), + ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! + } } if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N)) @@ -22829,9 +22874,15 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl &StoreNodes, if (TLI.isTruncStoreLegal(LegalizedStoredValTy, StoreTy) && TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG.getMachineFunction()) && - TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValTy, StoreTy) && - TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy, StoreTy) && - TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) && + TLI.isLoadLegal(LegalizedStoredValTy, StoreTy, + FirstLoad->getAlign(), FirstLoad->getAddressSpace(), + ISD::ZEXTLOAD, false) && + TLI.isLoadLegal(LegalizedStoredValTy, StoreTy, + FirstLoad->getAlign(), FirstLoad->getAddressSpace(), + ISD::SEXTLOAD, false) && + TLI.isLoadLegal(LegalizedStoredValTy, StoreTy, + FirstLoad->getAlign(), FirstLoad->getAddressSpace(), + ISD::EXTLOAD, false) && TLI.allowsMemoryAccess(Context, DL, StoreTy, *FirstInChain->getMemOperand(), &IsFastSt) && IsFastSt && diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 9ab0c9d475e94..de853fe448576 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -352,7 +352,13 @@ SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { if (ConstantFPSDNode::isValueValidForType(SVT, APF) && // Only do this if the target has a native EXTLOAD instruction from // smaller type. - TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && + TLI.isLoadLegal( + OrigVT, SVT, + Align(DAG.getDataLayout().getPrefTypeAlign( + SVT.getTypeForEVT(*DAG.getContext()))), + MachinePointerInfo::getConstantPool(DAG.getMachineFunction()) + .getAddrSpace(), + ISD::EXTLOAD, false) && TLI.ShouldShrinkFPConstant(OrigVT)) { Type *SType = SVT.getTypeForEVT(*DAG.getContext()); LLVMC = cast(ConstantFoldCastOperand( @@ -744,8 +750,9 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { // nice to have an effective generic way of getting these benefits... // Until such a way is found, don't insist on promoting i1 here. (SrcVT != MVT::i1 || - TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == - TargetLowering::Promote)) { + TLI.getLoadAction(Node->getValueType(0), MVT::i1, LD->getAlign(), + LD->getAddressSpace(), ExtType, + false) == TargetLowering::Promote)) { // Promote to a byte-sized load if not loading an integral number of // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. unsigned NewWidth = SrcVT.getStoreSizeInBits(); @@ -855,9 +862,11 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { Chain = Ch; } else { bool isCustom = false; - switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), - SrcVT.getSimpleVT())) { - default: llvm_unreachable("This action is not supported yet!"); + switch (TLI.getLoadAction(Node->getValueType(0), SrcVT.getSimpleVT(), + LD->getAlign(), LD->getAddressSpace(), ExtType, + false)) { + default: + llvm_unreachable("This action is not supported yet!"); case TargetLowering::Custom: isCustom = true; [[fallthrough]]; @@ -884,13 +893,15 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { case TargetLowering::Expand: { EVT DestVT = Node->getValueType(0); - if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { + if (!TLI.isLoadLegal(DestVT, SrcVT, LD->getAlign(), LD->getAddressSpace(), + ISD::EXTLOAD, false)) { // If the source type is not legal, see if there is a legal extload to // an intermediate type that we can then extend further. EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) && (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT? - TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) { + TLI.isLoadLegal(LoadVT, SrcVT, LD->getAlign(), + LD->getAddressSpace(), ExtType, false))) { // If we are loading a legal type, this is a non-extload followed by a // full extend. ISD::LoadExtType MidExtType = @@ -1853,7 +1864,9 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, if ((SrcVT.bitsGT(SlotVT) && !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) || (SlotVT.bitsLT(DestVT) && - !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT))) + !TLI.isLoadLegalOrCustom(DestVT, SlotVT, DestAlign, + DAG.getDataLayout().getAllocaAddrSpace(), + ISD::EXTLOAD, false))) return SDValue(); // Create the stack frame object. diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 81184f709bd8c..c2937247b695c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -298,7 +298,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { ISD::LoadExtType ExtType = LD->getExtensionType(); EVT LoadedVT = LD->getMemoryVT(); if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD) - Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT); + Action = TLI.getLoadAction(LD->getValueType(0), LoadedVT, LD->getAlign(), + LD->getAddressSpace(), ExtType, false); break; } case ISD::STORE: { diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index a10b93063dc27..823d26ac29674 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -12731,9 +12731,11 @@ SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT, if (ResultVT.bitsGT(VecEltVT)) { // If the result type of vextract is wider than the load, then issue an // extending load instead. - ISD::LoadExtType ExtType = isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, VecEltVT) - ? ISD::ZEXTLOAD - : ISD::EXTLOAD; + ISD::LoadExtType ExtType = + isLoadLegal(ResultVT, VecEltVT, Alignment, + OriginalLoad->getAddressSpace(), ISD::ZEXTLOAD, false) + ? ISD::ZEXTLOAD + : ISD::EXTLOAD; Load = DAG.getExtLoad(ExtType, DL, ResultVT, OriginalLoad->getChain(), NewPtr, MPI, VecEltVT, Alignment, OriginalLoad->getMemOperand()->getFlags(), diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index c6707162d9b9e..9b7246b622762 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -20,6 +20,7 @@ #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/Analysis.h" +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -2029,6 +2030,29 @@ MVT TargetLoweringBase::getPreferredSwitchConditionType(LLVMContext &Context, return getRegisterType(Context, ConditionVT); } +TargetLoweringBase::LegalizeAction +TargetLoweringBase::getLoadAction(EVT ValVT, EVT MemVT, Align Alignment, + unsigned AddrSpace, unsigned ExtType, + bool Atomic) const { + if (ValVT.isExtended() || MemVT.isExtended()) + return Expand; + unsigned ValI = (unsigned)ValVT.getSimpleVT().SimpleTy; + unsigned MemI = (unsigned)MemVT.getSimpleVT().SimpleTy; + assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::VALUETYPE_SIZE && + MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!"); + unsigned Shift = 4 * ExtType; + + if (Atomic) { + LegalizeAction Action = + (LegalizeAction)((AtomicLoadExtActions[ValI][MemI] >> Shift) & 0xf); + assert((Action == Legal || Action == Expand) && + "Unsupported atomic load extension action."); + return Action; + } + + return (LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf); +} + /// Get the EVTs and ArgFlags collections that represent the legalized return /// type of the given function. This does not require a DAG or a return value, /// and is suitable for use before any DAGs for the function are constructed. diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 842655d0ca0e9..030e87b86206d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -7045,7 +7045,8 @@ bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const { // results in just one set of predicate unpacks at the start, instead of // multiple sets of vector unpacks after each load. if (auto *Ld = dyn_cast(ExtVal->getOperand(0))) { - if (!isLoadExtLegalOrCustom(ISD::ZEXTLOAD, ExtVT, Ld->getValueType(0))) { + if (!isLoadLegalOrCustom(ExtVT, Ld->getValueType(0), Ld->getAlign(), + Ld->getAddressSpace(), ISD::ZEXTLOAD, false)) { // Disable extending masked loads for fixed-width for now, since the code // quality doesn't look great. if (!ExtVT.isScalableVector()) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 3ec2bf9b19360..b3d18ef4f9abd 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -61640,7 +61640,8 @@ static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG, ? ISD::SEXTLOAD : ISD::ZEXTLOAD; EVT MemVT = VT.changeVectorElementType(*DAG.getContext(), SVT); - if (TLI.isLoadExtLegal(Ext, VT, MemVT)) { + if (TLI.isLoadLegal(VT, MemVT, Ld->getAlign(), Ld->getAddressSpace(), Ext, + false)) { SDValue Load = DAG.getExtLoad( Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(), MemVT, Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());