Index: machinelicm_clangformat_modv1.cpp =================================================================== --- machinelicm_clangformat_modv1.cpp +++ machinelicm_clangformat_modv1.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -75,6 +76,25 @@ cl::desc("Hoist invariant stores"), cl::init(true), cl::Hidden); +static cl::opt +BlockFrequencyRatioThreshold("block-freq-ratio-threshold", + cl::desc("Block frequency ratio threshold to " + "disable instruction hoisting"), + cl::init(100), cl::Hidden); + +enum NotHoistDueToBlockHotnessType { Disable, Enable_PGO, Enable }; + +static cl::opt +NotHoistDueToBlockHotness("not-hoist-due-to-block-hotness", + cl::desc("Disable instruction hoist due to block hotness"), + cl::init(Disable), cl::Hidden, + cl::values(clEnumValN(Disable, "disable", + "disable the feature"), + clEnumValN(Enable_PGO, "enable-pgo", + "enable the feature when using profile data"), + clEnumValN(Enable, "enable", + "enable the feature with/without profile data"))); + STATISTIC(NumHoisted, "Number of machine instructions hoisted out of loops"); STATISTIC(NumLowRP, @@ -87,6 +107,8 @@ "Number of machine instructions hoisted out of loops post regalloc"); STATISTIC(NumStoreConst, "Number of stores of const phys reg hoisted out of loops"); +STATISTIC(NumNotHoistedDueToHotness, + "Number of instructions not hoisted due to block frequency hotness"); namespace { @@ -101,6 +123,7 @@ // Various analyses that we use... AliasAnalysis *AA; // Alias analysis info. + MachineBlockFrequencyInfo *MBFI; // Machine block frequncy info MachineLoopInfo *MLI; // Current MachineLoopInfo MachineDominatorTree *DT; // Machine dominator tree for the cur loop @@ -150,6 +173,8 @@ void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); + if (NotHoistDueToBlockHotness != Disable) + AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.addPreserved(); @@ -246,6 +271,9 @@ void InitCSEMap(MachineBasicBlock *BB); + bool IsHoistingFromColdToHotBlock(MachineBasicBlock *CurBB, + MachineBasicBlock *Preheader); + MachineBasicBlock *getCurPreheader(); }; @@ -276,6 +304,7 @@ INITIALIZE_PASS_BEGIN(MachineLICM, DEBUG_TYPE, "Machine Loop Invariant Code Motion", false, false) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_END(MachineLICM, DEBUG_TYPE, @@ -284,6 +313,7 @@ INITIALIZE_PASS_BEGIN(EarlyMachineLICM, "early-machinelicm", "Early Machine Loop Invariant Code Motion", false, false) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_END(EarlyMachineLICM, "early-machinelicm", @@ -334,6 +364,8 @@ } // Get our Loop information... + if (NotHoistDueToBlockHotness != Disable) + MBFI = &getAnalysis(); MLI = &getAnalysis(); DT = &getAnalysis(); AA = &getAnalysis().getAAResults(); @@ -1434,6 +1466,15 @@ /// that are safe to hoist, this instruction is called to do the dirty work. /// It returns true if the instruction is hoisted. bool MachineLICMBase::Hoist(MachineInstr *MI, MachineBasicBlock *Preheader) { + MachineBasicBlock *CurBB = MI->getParent(); + bool hasProfileData = CurBB->getParent()->getFunction().hasProfileData(); + + // Disable the instruction hoisting due to block hotness + if ((NotHoistDueToBlockHotness == Enable || + (NotHoistDueToBlockHotness == Enable_PGO && hasProfileData)) && + IsHoistingFromColdToHotBlock(CurBB, Preheader)) + return false; + // First check whether we should hoist this instruction. if (!IsLoopInvariantInst(*MI) || !IsProfitableToHoist(*MI)) { // If not, try unfolding a hoistable load. @@ -1528,3 +1569,23 @@ return CurPreheader; } +bool MachineLICMBase::IsHoistingFromColdToHotBlock(MachineBasicBlock *CurBB, + MachineBasicBlock *Preheader) { + // Parse source and target basic block frequency from MBFI + uint64_t SrcBF = MBFI->getBlockFreq(CurBB).getFrequency(); + uint64_t DstBF = MBFI->getBlockFreq(Preheader).getFrequency(); + + // Disable the hoisting if source block frequency is zero + if (!SrcBF) + return true; + + double Ratio = (double)DstBF / SrcBF; + // Compare the block frequency ratio with the threshold + if (Ratio > BlockFrequencyRatioThreshold) { + ++NumNotHoistedDueToHotness; + return true; + } + + return false; +} +