diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h index 660670ccdcd75..4762494e6ccb7 100644 --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -17,7 +17,7 @@ // scheduled. Targets can override the DAG builder and scheduler without // replacing the pass as follows: // -// ScheduleDAGInstrs *PassConfig:: +// ScheduleDAGInstrs *TargetMachine:: // createMachineScheduler(MachineSchedContext *C) { // return new CustomMachineScheduler(C); // } @@ -29,7 +29,7 @@ // plugin an alternate MachineSchedStrategy. The strategy is responsible for // selecting the highest priority node from the list: // -// ScheduleDAGInstrs *PassConfig:: +// ScheduleDAGInstrs *TargetMachine:: // createMachineScheduler(MachineSchedContext *C) { // return new ScheduleDAGMILive(C, CustomStrategy(C)); // } @@ -39,7 +39,7 @@ // can adjust dependencies based on target-specific knowledge or add weak edges // to aid heuristics: // -// ScheduleDAGInstrs *PassConfig:: +// ScheduleDAGInstrs *TargetMachine:: // createMachineScheduler(MachineSchedContext *C) { // ScheduleDAGMI *DAG = createGenericSchedLive(C); // DAG->addMutation(new CustomDAGMutation(...)); @@ -137,7 +137,7 @@ struct MachineSchedContext { MachineFunction *MF = nullptr; const MachineLoopInfo *MLI = nullptr; const MachineDominatorTree *MDT = nullptr; - const TargetPassConfig *PassConfig = nullptr; + const TargetMachine *TM = nullptr; AAResults *AA = nullptr; LiveIntervals *LIS = nullptr; diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index a91cb0d4f603b..c8eba71c9bb0a 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -1566,7 +1566,7 @@ class TargetInstrInfo : public MCInstrInfo { /// DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); /// or /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); - /// to TargetPassConfig::createMachineScheduler() to have an effect. + /// to TargetMachine::createMachineScheduler() to have an effect. /// /// \p BaseOps1 and \p BaseOps2 are memory operands of two memory operations. /// \p Offset1 and \p Offset2 are the byte offsets for the memory diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h index 66c79c74f2be5..1af7267fa9ecf 100644 --- a/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -22,9 +22,7 @@ namespace llvm { class TargetMachine; -struct MachineSchedContext; class PassConfigImpl; -class ScheduleDAGInstrs; class CSEConfigBase; class PassInstrumentationCallbacks; @@ -300,27 +298,6 @@ class TargetPassConfig : public ImmutablePass { /// Fully developed targets will not generally override this. virtual void addMachinePasses(); - /// Create an instance of ScheduleDAGInstrs to be run within the standard - /// MachineScheduler pass for this function and target at the current - /// optimization level. - /// - /// This can also be used to plug a new MachineSchedStrategy into an instance - /// of the standard ScheduleDAGMI: - /// return new ScheduleDAGMI(C, std::make_unique(C), /*RemoveKillFlags=*/false) - /// - /// Return NULL to select the default (generic) machine scheduler. - virtual ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const { - return nullptr; - } - - /// Similar to createMachineScheduler but used when postRA machine scheduling - /// is enabled. - virtual ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const { - return nullptr; - } - /// printAndVerify - Add a pass to dump then verify the machine function, if /// those steps are enabled. void printAndVerify(const std::string &Banner); diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index fe1dbbd44f8eb..b1ec0b9c07c17 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -39,6 +39,7 @@ using ModulePassManager = PassManager; class Function; class GlobalValue; class MachineModuleInfoWrapperPass; +struct MachineSchedContext; class Mangler; class MCAsmInfo; class MCContext; @@ -50,6 +51,7 @@ class raw_pwrite_stream; class PassBuilder; class PassInstrumentationCallbacks; struct PerFunctionMIParsingState; +class ScheduleDAGInstrs; class SMDiagnostic; class SMRange; class Target; @@ -147,6 +149,28 @@ class TargetMachine { return nullptr; } + /// Create an instance of ScheduleDAGInstrs to be run within the standard + /// MachineScheduler pass for this function and target at the current + /// optimization level. + /// + /// This can also be used to plug a new MachineSchedStrategy into an instance + /// of the standard ScheduleDAGMI: + /// return new ScheduleDAGMI(C, std::make_unique(C), + /// /*RemoveKillFlags=*/false) + /// + /// Return NULL to select the default (generic) machine scheduler. + virtual ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const { + return nullptr; + } + + /// Similar to createMachineScheduler but used when postRA machine scheduling + /// is enabled. + virtual ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const { + return nullptr; + } + /// Allocate and return a default initialized instance of the YAML /// representation for the MachineFunctionInfo. virtual yaml::MachineFunctionInfo *createDefaultFuncInfoYAML() const { diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index 0d5dc96159003..dbf320f88fd65 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -511,7 +511,7 @@ bool MachinePipeliner::runWindowScheduler(MachineLoop &L) { Context.MF = MF; Context.MLI = MLI; Context.MDT = MDT; - Context.PassConfig = &getAnalysis(); + Context.TM = &getAnalysis().getTM(); Context.AA = &getAnalysis().getAAResults(); Context.LIS = &getAnalysis().getLIS(); Context.RegClassInfo->runOnMachineFunction(*MF); diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index 393530f56cc27..da46129ebaa04 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -58,6 +58,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" #include #include #include @@ -392,8 +393,11 @@ ScheduleDAGInstrs *MachineScheduler::createMachineScheduler() { if (Ctor != useDefaultMachineSched) return Ctor(this); + const TargetMachine &TM = + getAnalysis().getTM(); + // Get the default scheduler set by the target for this function. - ScheduleDAGInstrs *Scheduler = PassConfig->createMachineScheduler(this); + ScheduleDAGInstrs *Scheduler = TM.createMachineScheduler(this); if (Scheduler) return Scheduler; @@ -405,8 +409,10 @@ ScheduleDAGInstrs *MachineScheduler::createMachineScheduler() { /// the caller. We don't have a command line option to override the postRA /// scheduler. The Target must configure it. ScheduleDAGInstrs *PostMachineScheduler::createPostMachineScheduler() { + const TargetMachine &TM = + getAnalysis().getTM(); // Get the postRA scheduler set by the target for this function. - ScheduleDAGInstrs *Scheduler = PassConfig->createPostMachineScheduler(this); + ScheduleDAGInstrs *Scheduler = TM.createPostMachineScheduler(this); if (Scheduler) return Scheduler; @@ -446,7 +452,6 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { MF = &mf; MLI = &getAnalysis().getLI(); MDT = &getAnalysis().getDomTree(); - PassConfig = &getAnalysis(); AA = &getAnalysis().getAAResults(); LIS = &getAnalysis().getLIS(); @@ -484,7 +489,6 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) { // Initialize the context of the pass. MF = &mf; MLI = &getAnalysis().getLI(); - PassConfig = &getAnalysis(); AA = &getAnalysis().getAAResults(); if (VerifyScheduling) diff --git a/llvm/lib/CodeGen/WindowScheduler.cpp b/llvm/lib/CodeGen/WindowScheduler.cpp index e7fc0d9a3d254..379740cae78d5 100644 --- a/llvm/lib/CodeGen/WindowScheduler.cpp +++ b/llvm/lib/CodeGen/WindowScheduler.cpp @@ -45,6 +45,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/TimeProfiler.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; @@ -167,7 +168,7 @@ WindowScheduler::createMachineScheduler(bool OnlyBuildGraph) { ? new ScheduleDAGMI( Context, std::make_unique(Context), true) - : Context->PassConfig->createMachineScheduler(Context); + : Context->TM->createMachineScheduler(Context); } bool WindowScheduler::initialize() { diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.h b/llvm/lib/Target/AArch64/AArch64MacroFusion.h index 2999e7a8aa909..62da0545875b6 100644 --- a/llvm/lib/Target/AArch64/AArch64MacroFusion.h +++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.h @@ -20,7 +20,7 @@ namespace llvm { /// Note that you have to add: /// DAG.addMutation(createAArch64MacroFusionDAGMutation()); -/// to AArch64PassConfig::createMachineScheduler() to have an effect. +/// to AArch64TargetMachine::createMachineScheduler() to have an effect. std::unique_ptr createAArch64MacroFusionDAGMutation(); } // llvm diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index 07f072446081a..d10a0c0a08f89 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -480,6 +480,33 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const { return I.get(); } +ScheduleDAGInstrs * +AArch64TargetMachine::createMachineScheduler(MachineSchedContext *C) const { + const AArch64Subtarget &ST = C->MF->getSubtarget(); + ScheduleDAGMILive *DAG = createGenericSchedLive(C); + DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); + DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); + if (ST.hasFusion()) + DAG->addMutation(createAArch64MacroFusionDAGMutation()); + return DAG; +} + +ScheduleDAGInstrs * +AArch64TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { + const AArch64Subtarget &ST = C->MF->getSubtarget(); + ScheduleDAGMI *DAG = + new ScheduleDAGMI(C, std::make_unique(C), + /* RemoveKillFlags=*/true); + if (ST.hasFusion()) { + // Run the Macro Fusion after RA again since literals are expanded from + // pseudos then (v. addPreSched2()). + DAG->addMutation(createAArch64MacroFusionDAGMutation()); + return DAG; + } + + return DAG; +} + void AArch64leTargetMachine::anchor() { } AArch64leTargetMachine::AArch64leTargetMachine( @@ -512,33 +539,6 @@ class AArch64PassConfig : public TargetPassConfig { return getTM(); } - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override { - const AArch64Subtarget &ST = C->MF->getSubtarget(); - ScheduleDAGMILive *DAG = createGenericSchedLive(C); - DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); - DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); - if (ST.hasFusion()) - DAG->addMutation(createAArch64MacroFusionDAGMutation()); - return DAG; - } - - ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const override { - const AArch64Subtarget &ST = C->MF->getSubtarget(); - ScheduleDAGMI *DAG = - new ScheduleDAGMI(C, std::make_unique(C), - /* RemoveKillFlags=*/true); - if (ST.hasFusion()) { - // Run the Macro Fusion after RA again since literals are expanded from - // pseudos then (v. addPreSched2()). - DAG->addMutation(createAArch64MacroFusionDAGMutation()); - return DAG; - } - - return DAG; - } - void addIRPasses() override; bool addPreISel() override; void addCodeGenPrepare() override; diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h index 621adb380dbcd..f8ba41f215430 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h @@ -70,6 +70,11 @@ class AArch64TargetMachine : public CodeGenTargetMachineImpl { bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override { return getPointerSize(SrcAS) == getPointerSize(DestAS); } + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; + + ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const override; private: bool isLittle; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h index ad198a301dbe4..8aee23d824e97 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h @@ -16,7 +16,7 @@ namespace llvm { /// Note that you have to add: /// DAG.addMutation(createAMDGPUMacroFusionDAGMutation()); -/// to AMDGPUPassConfig::createMachineScheduler() to have an effect. +/// to AMDGPUTargetMachine::createMachineScheduler() to have an effect. std::unique_ptr createAMDGPUMacroFusionDAGMutation(); } // llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 96062b30fc012..8d7e4fb5e88eb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -729,6 +729,16 @@ StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const { : getTargetFeatureString(); } +llvm::ScheduleDAGInstrs * +AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const { + const GCNSubtarget &ST = C->MF->getSubtarget(); + ScheduleDAGMILive *DAG = createGenericSchedLive(C); + DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); + if (ST.shouldClusterStores()) + DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); + return DAG; +} + /// Predicate for Internalize pass. static bool mustPreserveGV(const GlobalValue &GV) { if (const Function *F = dyn_cast(&GV)) @@ -1046,6 +1056,43 @@ Error GCNTargetMachine::buildCodeGenPipeline( return CGPB.buildPipeline(MPM, Out, DwoOut, FileType); } +ScheduleDAGInstrs * +GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const { + const GCNSubtarget &ST = C->MF->getSubtarget(); + if (ST.enableSIScheduler()) + return createSIMachineScheduler(C); + + Attribute SchedStrategyAttr = + C->MF->getFunction().getFnAttribute("amdgpu-sched-strategy"); + StringRef SchedStrategy = SchedStrategyAttr.isValid() + ? SchedStrategyAttr.getValueAsString() + : AMDGPUSchedStrategy; + + if (SchedStrategy == "max-ilp") + return createGCNMaxILPMachineScheduler(C); + + if (SchedStrategy == "max-memory-clause") + return createGCNMaxMemoryClauseMachineScheduler(C); + + return createGCNMaxOccupancyMachineScheduler(C); +} + +ScheduleDAGInstrs * +GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { + ScheduleDAGMI *DAG = + new GCNPostScheduleDAGMILive(C, std::make_unique(C), + /*RemoveKillFlags=*/true); + const GCNSubtarget &ST = C->MF->getSubtarget(); + DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); + if (ST.shouldClusterStores()) + DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); + DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::PostRA)); + if ((EnableVOPD.getNumOccurrences() || + getOptLevel() >= CodeGenOptLevel::Less) && + EnableVOPD) + DAG->addMutation(createVOPDPairingMutation()); + return DAG; +} //===----------------------------------------------------------------------===// // AMDGPU Legacy Pass Setup //===----------------------------------------------------------------------===// @@ -1071,25 +1118,6 @@ class GCNPassConfig final : public AMDGPUPassConfig { return getTM(); } - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override; - - ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const override { - ScheduleDAGMI *DAG = new GCNPostScheduleDAGMILive( - C, std::make_unique(C), - /*RemoveKillFlags=*/true); - const GCNSubtarget &ST = C->MF->getSubtarget(); - DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); - if (ST.shouldClusterStores()) - DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); - DAG->addMutation( - createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::PostRA)); - if (isPassEnabled(EnableVOPD, CodeGenOptLevel::Less)) - DAG->addMutation(createVOPDPairingMutation()); - return DAG; - } - bool addPreISel() override; void addMachineSSAOptimization() override; bool addILPOpts() override; @@ -1316,41 +1344,10 @@ bool AMDGPUPassConfig::addGCPasses() { return false; } -llvm::ScheduleDAGInstrs * -AMDGPUPassConfig::createMachineScheduler(MachineSchedContext *C) const { - const GCNSubtarget &ST = C->MF->getSubtarget(); - ScheduleDAGMILive *DAG = createGenericSchedLive(C); - DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); - if (ST.shouldClusterStores()) - DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); - return DAG; -} - //===----------------------------------------------------------------------===// // GCN Legacy Pass Setup //===----------------------------------------------------------------------===// -ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler( - MachineSchedContext *C) const { - const GCNSubtarget &ST = C->MF->getSubtarget(); - if (ST.enableSIScheduler()) - return createSIMachineScheduler(C); - - Attribute SchedStrategyAttr = - C->MF->getFunction().getFnAttribute("amdgpu-sched-strategy"); - StringRef SchedStrategy = SchedStrategyAttr.isValid() - ? SchedStrategyAttr.getValueAsString() - : AMDGPUSchedStrategy; - - if (SchedStrategy == "max-ilp") - return createGCNMaxILPMachineScheduler(C); - - if (SchedStrategy == "max-memory-clause") - return createGCNMaxMemoryClauseMachineScheduler(C); - - return createGCNMaxOccupancyMachineScheduler(C); -} - bool GCNPassConfig::addPreISel() { AMDGPUPassConfig::addPreISel(); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h index 24b4da3a68f67..1455494d0ef7d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h @@ -71,6 +71,8 @@ class AMDGPUTargetMachine : public CodeGenTargetMachineImpl { bool splitModule(Module &M, unsigned NumParts, function_ref MPart)> ModuleCallback) override; + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; }; //===----------------------------------------------------------------------===// @@ -115,6 +117,10 @@ class GCNTargetMachine final : public AMDGPUTargetMachine { PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) const override; + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; + ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const override; }; //===----------------------------------------------------------------------===// @@ -128,10 +134,6 @@ class AMDGPUPassConfig : public TargetPassConfig { AMDGPUTargetMachine &getAMDGPUTargetMachine() const { return getTM(); } - - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override; - void addEarlyCSEOrGVNPass(); void addStraightLineScalarOptimizationPasses(); void addIRPasses() override; diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp index a9e5327d2c6c8..10552a1f0b1bc 100644 --- a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp @@ -90,17 +90,17 @@ R600TargetMachine::getTargetTransformInfo(const Function &F) const { return TargetTransformInfo(R600TTIImpl(this, F)); } +ScheduleDAGInstrs * +R600TargetMachine::createMachineScheduler(MachineSchedContext *C) const { + return createR600MachineScheduler(C); +} + namespace { class R600PassConfig final : public AMDGPUPassConfig { public: R600PassConfig(TargetMachine &TM, PassManagerBase &PM) : AMDGPUPassConfig(TM, PM) {} - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override { - return createR600MachineScheduler(C); - } - bool addPreISel() override; bool addInstSelector() override; void addPreRegAlloc() override; diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.h b/llvm/lib/Target/AMDGPU/R600TargetMachine.h index b7f123a07a9c1..eb4cb91cb704d 100644 --- a/llvm/lib/Target/AMDGPU/R600TargetMachine.h +++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.h @@ -53,6 +53,8 @@ class R600TargetMachine final : public AMDGPUTargetMachine { MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override; + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; }; //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/ARM/ARMLatencyMutations.h b/llvm/lib/Target/ARM/ARMLatencyMutations.h index a4b8de0be51f7..b22025aedf854 100644 --- a/llvm/lib/Target/ARM/ARMLatencyMutations.h +++ b/llvm/lib/Target/ARM/ARMLatencyMutations.h @@ -47,7 +47,7 @@ class ARMOverrideBypasses : public ScheduleDAGMutation { /// Note that you have to add: /// DAG.addMutation(createARMLatencyMutation(ST, AA)); -/// to ARMPassConfig::createMachineScheduler() to have an effect. +/// to ARMTargetMachine::createMachineScheduler() to have an effect. std::unique_ptr createARMLatencyMutations(const class ARMSubtarget &, AAResults *AA); diff --git a/llvm/lib/Target/ARM/ARMMacroFusion.h b/llvm/lib/Target/ARM/ARMMacroFusion.h index 4896a4a2544db..b8642dc536c89 100644 --- a/llvm/lib/Target/ARM/ARMMacroFusion.h +++ b/llvm/lib/Target/ARM/ARMMacroFusion.h @@ -20,7 +20,7 @@ namespace llvm { /// Note that you have to add: /// DAG.addMutation(createARMMacroFusionDAGMutation()); -/// to ARMPassConfig::createMachineScheduler() to have an effect. +/// to ARMTargetMachine::createMachineScheduler() to have an effect. std::unique_ptr createARMMacroFusionDAGMutation(); } // llvm diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp index 8e7b756b67905..98bdf310dea91 100644 --- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp +++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp @@ -325,6 +325,28 @@ ARMBaseTargetMachine::getTargetTransformInfo(const Function &F) const { return TargetTransformInfo(ARMTTIImpl(this, F)); } +ScheduleDAGInstrs * +ARMBaseTargetMachine::createMachineScheduler(MachineSchedContext *C) const { + ScheduleDAGMILive *DAG = createGenericSchedLive(C); + // add DAG Mutations here. + const ARMSubtarget &ST = C->MF->getSubtarget(); + if (ST.hasFusion()) + DAG->addMutation(createARMMacroFusionDAGMutation()); + return DAG; +} + +ScheduleDAGInstrs * +ARMBaseTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { + ScheduleDAGMI *DAG = createGenericSchedPostRA(C); + // add DAG Mutations here. + const ARMSubtarget &ST = C->MF->getSubtarget(); + if (ST.hasFusion()) + DAG->addMutation(createARMMacroFusionDAGMutation()); + if (auto Mutation = createARMLatencyMutations(ST, C->AA)) + DAG->addMutation(std::move(Mutation)); + return DAG; +} + ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, @@ -353,28 +375,6 @@ class ARMPassConfig : public TargetPassConfig { return getTM(); } - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override { - ScheduleDAGMILive *DAG = createGenericSchedLive(C); - // add DAG Mutations here. - const ARMSubtarget &ST = C->MF->getSubtarget(); - if (ST.hasFusion()) - DAG->addMutation(createARMMacroFusionDAGMutation()); - return DAG; - } - - ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const override { - ScheduleDAGMI *DAG = createGenericSchedPostRA(C); - // add DAG Mutations here. - const ARMSubtarget &ST = C->MF->getSubtarget(); - if (ST.hasFusion()) - DAG->addMutation(createARMMacroFusionDAGMutation()); - if (auto Mutation = createARMLatencyMutations(ST, C->AA)) - DAG->addMutation(std::move(Mutation)); - return DAG; - } - void addIRPasses() override; void addCodeGenPrepare() override; bool addPreISel() override; diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h index 94d48fcae15eb..99fd817c81f89 100644 --- a/llvm/lib/Target/ARM/ARMTargetMachine.h +++ b/llvm/lib/Target/ARM/ARMTargetMachine.h @@ -96,6 +96,10 @@ class ARMBaseTargetMachine : public CodeGenTargetMachineImpl { PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) const override; + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; + ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const override; }; /// ARM/Thumb little endian target machine. diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index a97bc1985c614..ff4cb3cc5a881 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -351,6 +351,11 @@ MachineFunctionInfo *HexagonTargetMachine::createMachineFunctionInfo( HexagonTargetMachine::~HexagonTargetMachine() = default; +ScheduleDAGInstrs * +HexagonTargetMachine::createMachineScheduler(MachineSchedContext *C) const { + return createVLIWMachineSched(C); +} + namespace { /// Hexagon Code Generator Pass Configuration Options. class HexagonPassConfig : public TargetPassConfig { @@ -362,11 +367,6 @@ class HexagonPassConfig : public TargetPassConfig { return getTM(); } - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override { - return createVLIWMachineSched(C); - } - void addIRPasses() override; bool addInstSelector() override; void addPreRegAlloc() override; diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h index 65f5b6e7e8eb7..48e0c08c0cab2 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h @@ -50,6 +50,8 @@ class HexagonTargetMachine : public CodeGenTargetMachineImpl { bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override { return true; } + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; }; } // end namespace llvm diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.h b/llvm/lib/Target/PowerPC/PPCMacroFusion.h index cbf49ee779ceb..22d15cac2ef55 100644 --- a/llvm/lib/Target/PowerPC/PPCMacroFusion.h +++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.h @@ -20,7 +20,7 @@ namespace llvm { /// Note that you have to add: /// DAG.addMutation(createPowerPCMacroFusionDAGMutation()); -/// to PPCPassConfig::createMachineScheduler() to have an effect. +/// to PPCTargetMachine::createMachineScheduler() to have an effect. std::unique_ptr createPowerPCMacroFusionDAGMutation(); } // llvm diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp index b1ad041bde790..5ee13a92cf993 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp @@ -403,6 +403,16 @@ PPCTargetMachine::getSubtargetImpl(const Function &F) const { return I.get(); } +ScheduleDAGInstrs * +PPCTargetMachine::createMachineScheduler(MachineSchedContext *C) const { + return createPPCMachineScheduler(C); +} + +ScheduleDAGInstrs * +PPCTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { + return createPPCPostMachineScheduler(C); +} + //===----------------------------------------------------------------------===// // Pass Pipeline Configuration //===----------------------------------------------------------------------===// @@ -438,15 +448,6 @@ class PPCPassConfig : public TargetPassConfig { bool addLegalizeMachineIR() override; bool addRegBankSelect() override; bool addGlobalInstructionSelect() override; - - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override { - return createPPCMachineScheduler(C); - } - ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const override { - return createPPCPostMachineScheduler(C); - } }; } // end anonymous namespace diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.h b/llvm/lib/Target/PowerPC/PPCTargetMachine.h index 026bf2f26a816..cb02b446fadb3 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetMachine.h +++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.h @@ -63,6 +63,10 @@ class PPCTargetMachine final : public CodeGenTargetMachineImpl { MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override; + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; + ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const override; bool isELFv2ABI() const { return TargetABI == PPC_ABI_ELFv2; } bool hasGlibcHWCAPAccess() const { return HasGlibcHWCAPAccess; } diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp index dde808ad90413..3e4949232298e 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp @@ -287,6 +287,39 @@ bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS, return true; } +ScheduleDAGInstrs * +RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const { + ScheduleDAGMILive *DAG = nullptr; + if (EnableMISchedLoadStoreClustering) { + DAG = createGenericSchedLive(C); + DAG->addMutation(createLoadClusterDAGMutation( + DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); + DAG->addMutation(createStoreClusterDAGMutation( + DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); + } + + const RISCVSubtarget &ST = C->MF->getSubtarget(); + if (!DisableVectorMaskMutation && ST.hasVInstructions()) { + DAG = DAG ? DAG : createGenericSchedLive(C); + DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI)); + } + return DAG; +} + +ScheduleDAGInstrs * +RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { + ScheduleDAGMI *DAG = nullptr; + if (EnablePostMISchedLoadStoreClustering) { + DAG = createGenericSchedPostRA(C); + DAG->addMutation(createLoadClusterDAGMutation( + DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); + DAG->addMutation(createStoreClusterDAGMutation( + DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); + } + + return DAG; +} + namespace { class RVVRegisterRegAlloc : public RegisterRegAllocBase { @@ -360,39 +393,6 @@ class RISCVPassConfig : public TargetPassConfig { return getTM(); } - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override { - ScheduleDAGMILive *DAG = nullptr; - if (EnableMISchedLoadStoreClustering) { - DAG = createGenericSchedLive(C); - DAG->addMutation(createLoadClusterDAGMutation( - DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); - DAG->addMutation(createStoreClusterDAGMutation( - DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); - } - - const RISCVSubtarget &ST = C->MF->getSubtarget(); - if (!DisableVectorMaskMutation && ST.hasVInstructions()) { - DAG = DAG ? DAG : createGenericSchedLive(C); - DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI)); - } - return DAG; - } - - ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const override { - ScheduleDAGMI *DAG = nullptr; - if (EnablePostMISchedLoadStoreClustering) { - DAG = createGenericSchedPostRA(C); - DAG->addMutation(createLoadClusterDAGMutation( - DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); - DAG->addMutation(createStoreClusterDAGMutation( - DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true)); - } - - return DAG; - } - void addIRPasses() override; bool addPreISel() override; void addCodeGenPrepare() override; diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.h b/llvm/lib/Target/RISCV/RISCVTargetMachine.h index b1610e3f81eba..c85c2b3d0e611 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetMachine.h +++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.h @@ -59,6 +59,10 @@ class RISCVTargetMachine : public CodeGenTargetMachineImpl { SMDiagnostic &Error, SMRange &SourceRange) const override; void registerPassBuilderCallbacks(PassBuilder &PB) override; + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; + ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const override; }; std::unique_ptr diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp index 9000df2f6be0f..092515ee197a8 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp @@ -205,6 +205,12 @@ SystemZTargetMachine::getSubtargetImpl(const Function &F) const { return I.get(); } +ScheduleDAGInstrs * +SystemZTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { + return new ScheduleDAGMI(C, std::make_unique(C), + /*RemoveKillFlags=*/true); +} + namespace { /// SystemZ Code Generator Pass Configuration Options. @@ -217,13 +223,6 @@ class SystemZPassConfig : public TargetPassConfig { return getTM(); } - ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const override { - return new ScheduleDAGMI(C, - std::make_unique(C), - /*RemoveKillFlags=*/true); - } - void addIRPasses() override; bool addInstSelector() override; bool addILPOpts() override; diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.h b/llvm/lib/Target/SystemZ/SystemZTargetMachine.h index e8eeb85647b83..cced57a40ede0 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.h @@ -55,6 +55,8 @@ class SystemZTargetMachine : public CodeGenTargetMachineImpl { MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override; + ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const override; bool targetSchedulesPostRAScheduling() const override { return true; }; }; diff --git a/llvm/lib/Target/X86/X86MacroFusion.h b/llvm/lib/Target/X86/X86MacroFusion.h index 05388b275ca3f..ee07977664caf 100644 --- a/llvm/lib/Target/X86/X86MacroFusion.h +++ b/llvm/lib/Target/X86/X86MacroFusion.h @@ -22,7 +22,7 @@ class ScheduleDAGMutation; /// Note that you have to add: /// DAG.addMutation(createX86MacroFusionDAGMutation()); -/// to X86PassConfig::createMachineScheduler() to have an effect. +/// to X86TargetMachine::createMachineScheduler() to have an effect. std::unique_ptr createX86MacroFusionDAGMutation(); diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 20dfdd27b33df..0430279b88984 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -374,6 +374,20 @@ bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS, void X86TargetMachine::reset() { SubtargetMap.clear(); } +ScheduleDAGInstrs * +X86TargetMachine::createMachineScheduler(MachineSchedContext *C) const { + ScheduleDAGMILive *DAG = createGenericSchedLive(C); + DAG->addMutation(createX86MacroFusionDAGMutation()); + return DAG; +} + +ScheduleDAGInstrs * +X86TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { + ScheduleDAGMI *DAG = createGenericSchedPostRA(C); + DAG->addMutation(createX86MacroFusionDAGMutation()); + return DAG; +} + //===----------------------------------------------------------------------===// // X86 TTI query. //===----------------------------------------------------------------------===// @@ -399,20 +413,6 @@ class X86PassConfig : public TargetPassConfig { return getTM(); } - ScheduleDAGInstrs * - createMachineScheduler(MachineSchedContext *C) const override { - ScheduleDAGMILive *DAG = createGenericSchedLive(C); - DAG->addMutation(createX86MacroFusionDAGMutation()); - return DAG; - } - - ScheduleDAGInstrs * - createPostMachineScheduler(MachineSchedContext *C) const override { - ScheduleDAGMI *DAG = createGenericSchedPostRA(C); - DAG->addMutation(createX86MacroFusionDAGMutation()); - return DAG; - } - void addIRPasses() override; bool addInstSelector() override; bool addIRTranslator() override; diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h index b8d84a8d20032..ced0a9c71fdd8 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -79,6 +79,10 @@ class X86TargetMachine final : public CodeGenTargetMachineImpl { bool isJIT() const { return IsJIT; } bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override; + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override; + ScheduleDAGInstrs * + createPostMachineScheduler(MachineSchedContext *C) const override; }; } // end namespace llvm