Index: llvm/include/llvm/Analysis/InlineCost.h =================================================================== --- llvm/include/llvm/Analysis/InlineCost.h +++ llvm/include/llvm/Analysis/InlineCost.h @@ -15,6 +15,7 @@ #define LLVM_ANALYSIS_INLINECOST_H #include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LoopInfo.h" #include #include @@ -110,6 +111,7 @@ /// Also note that calling this function *dynamically* computes the cost of /// inlining the callsite. It is an expensive, heavyweight call. InlineCost getInlineCost(CallSite CS, int DefaultThreshold, + const DenseMap &SCCLI, TargetTransformInfo &CalleeTTI, AssumptionCacheTracker *ACT); @@ -119,6 +121,7 @@ /// parameter in all other respects. // InlineCost getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, + const DenseMap &SCCLI, TargetTransformInfo &CalleeTTI, AssumptionCacheTracker *ACT); Index: llvm/include/llvm/Transforms/IPO/InlinerPass.h =================================================================== --- llvm/include/llvm/Transforms/IPO/InlinerPass.h +++ llvm/include/llvm/Transforms/IPO/InlinerPass.h @@ -18,6 +18,7 @@ #define LLVM_TRANSFORMS_IPO_INLINERPASS_H #include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LoopInfo.h" namespace llvm { class AssumptionCacheTracker; @@ -72,6 +73,7 @@ protected: AssumptionCacheTracker *ACT; + DenseMap SCCLoopInfo; }; } // End llvm namespace Index: llvm/lib/Analysis/InlineCost.cpp =================================================================== --- llvm/lib/Analysis/InlineCost.cpp +++ llvm/lib/Analysis/InlineCost.cpp @@ -76,7 +76,7 @@ /// The cache of @llvm.assume intrinsics. AssumptionCacheTracker *ACT; - + const DenseMap &SCCLoopInfo; // The called function. Function &F; @@ -194,11 +194,13 @@ public: CallAnalyzer(const TargetTransformInfo &TTI, AssumptionCacheTracker *ACT, - Function &Callee, int Threshold, CallSite CSArg) - : TTI(TTI), ACT(ACT), F(Callee), CandidateCS(CSArg), Threshold(Threshold), - Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), - ExposesReturnsTwice(false), HasDynamicAlloca(false), - ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), + Function &Callee, int Threshold, CallSite CSArg, + const DenseMap &SLI) + : TTI(TTI), ACT(ACT), SCCLoopInfo(SLI), F(Callee), CandidateCS(CSArg), + Threshold(Threshold), Cost(0), IsCallerRecursive(false), + IsRecursiveCall(false), ExposesReturnsTwice(false), + HasDynamicAlloca(false), ContainsNoDuplicateCall(false), + HasReturn(false), HasIndirectBr(false), HasFrameEscape(false), AllocatedSize(0), NumInstructions(0), NumVectorInstructions(0), FiftyPercentVectorBonus(0), TenPercentVectorBonus(0), VectorBonus(0), NumConstantArgs(0), @@ -912,7 +914,8 @@ // during devirtualization and so we want to give it a hefty bonus for // inlining, but cap that bonus in the event that inlining wouldn't pan // out. Pretend to inline the function, with a custom threshold. - CallAnalyzer CA(TTI, ACT, *F, InlineConstants::IndirectCallThreshold, CS); + CallAnalyzer CA(TTI, ACT, *F, InlineConstants::IndirectCallThreshold, CS, + SCCLoopInfo); if (CA.analyzeCall(CS)) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. @@ -1241,6 +1244,15 @@ if (F.getCallingConv() == CallingConv::Cold) Cost += InlineConstants::ColdccPenalty; + // If the CallSite is inside a loop then give some bonus. + BasicBlock *BB = CS.getParent(); + auto LIter = SCCLoopInfo.find(BB->getParent()); + if (LIter != SCCLoopInfo.end()) { + const LoopInfo& LI = LIter->second; + if (LI.getLoopFor(BB)) + Threshold *= 2; + } + // Check if we're done. This can happen due to bonuses and penalties. if (Cost > Threshold) return false; @@ -1432,10 +1444,11 @@ } InlineCost llvm::getInlineCost(CallSite CS, int DefaultThreshold, + const DenseMap &SCCLI, TargetTransformInfo &CalleeTTI, AssumptionCacheTracker *ACT) { - return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI, - ACT); + return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, SCCLI, + CalleeTTI, ACT); } int llvm::computeThresholdFromOptLevels(unsigned OptLevel, @@ -1453,6 +1466,7 @@ InlineCost llvm::getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, + const DenseMap &SCCLI, TargetTransformInfo &CalleeTTI, AssumptionCacheTracker *ACT) { @@ -1487,7 +1501,7 @@ DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "...\n"); - CallAnalyzer CA(CalleeTTI, ACT, *Callee, DefaultThreshold, CS); + CallAnalyzer CA(CalleeTTI, ACT, *Callee, DefaultThreshold, CS, SCCLI); bool ShouldInline = CA.analyzeCall(CS); DEBUG(CA.dump()); Index: llvm/lib/Transforms/IPO/InlineSimple.cpp =================================================================== --- llvm/lib/Transforms/IPO/InlineSimple.cpp +++ llvm/lib/Transforms/IPO/InlineSimple.cpp @@ -60,7 +60,7 @@ InlineCost getInlineCost(CallSite CS) override { Function *Callee = CS.getCalledFunction(); TargetTransformInfo &TTI = TTIWP->getTTI(*Callee); - return llvm::getInlineCost(CS, DefaultThreshold, TTI, ACT); + return llvm::getInlineCost(CS, DefaultThreshold, SCCLoopInfo, TTI, ACT); } bool runOnSCC(CallGraphSCC &SCC) override; Index: llvm/lib/Transforms/IPO/Inliner.cpp =================================================================== --- llvm/lib/Transforms/IPO/Inliner.cpp +++ llvm/lib/Transforms/IPO/Inliner.cpp @@ -362,6 +362,8 @@ auto &TLI = getAnalysis().getTLI(); SmallPtrSet SCCFunctions; + DominatorTreeAnalysis DTA; + DEBUG(dbgs() << "Inliner visiting SCC:"); for (CallGraphNode *Node : SCC) { Function *F = Node->getFunction(); @@ -398,7 +400,8 @@ if (Function *Callee = CS.getCalledFunction()) if (Callee->isDeclaration()) continue; - + + SCCLoopInfo.insert(std::make_pair(F, LoopInfo(DTA.run(*F)))); CallSites.push_back(std::make_pair(CS, -1)); } } @@ -484,6 +487,10 @@ " will not be inlined into " + Caller->getName())); continue; + } else { + // Erase the loop info for the caller as a function was inlined. + // FIXME: Should we recompute the loop info for Caller. + SCCLoopInfo.erase(Caller); } ++NumInlined;