Index: include/llvm/Analysis/InlineCost.h =================================================================== --- include/llvm/Analysis/InlineCost.h +++ include/llvm/Analysis/InlineCost.h @@ -27,16 +27,26 @@ class TargetTransformInfo; namespace InlineConstants { - // Various magic constants used to adjust heuristics. - const int InstrCost = 5; - const int IndirectCallThreshold = 100; - const int CallPenalty = 25; - const int LastCallToStaticBonus = -15000; - const int ColdccPenalty = 2000; - const int NoreturnPenalty = 10000; - /// Do not inline functions which allocate this many bytes on the stack - /// when the caller is recursive. - const unsigned TotalAllocaSizeRecursiveCaller = 1024; +// Various thresholds used by inline cost analysis. +// Use when optsize (-Os) is specified. +const int OptSizeThreshold = 75; + +// Use when minsize (-Oz) is specified. +const int OptMinSizeThreshold = 25; + +// Use when -O[34] is specified. +const int OptAggressiveThreshold = 275; + +// Various magic constants used to adjust heuristics. +const int InstrCost = 5; +const int IndirectCallThreshold = 100; +const int CallPenalty = 25; +const int LastCallToStaticBonus = -15000; +const int ColdccPenalty = 2000; +const int NoreturnPenalty = 10000; +/// Do not inline functions which allocate this many bytes on the stack +/// when the caller is recursive. +const unsigned TotalAllocaSizeRecursiveCaller = 1024; } /// \brief Represents the cost of inlining a function. @@ -99,6 +109,15 @@ int getCostDelta() const { return Threshold - getCost(); } }; +/// Knobs to tune inline cost analysis. These are populated based on commandline +/// options. +struct InlineKnobs { + int HintThreshold; + Optional ColdThreshold; + Optional OptSizeThreshold; + Optional OptMinSizeThreshold; +}; + /// \brief Get an InlineCost object representing the cost of inlining this /// callsite. /// @@ -112,7 +131,8 @@ /// inlining the callsite. It is an expensive, heavyweight call. InlineCost getInlineCost(CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, - AssumptionCacheTracker *ACT, ProfileSummaryInfo *PSI); + AssumptionCacheTracker *ACT, ProfileSummaryInfo *PSI, + const InlineKnobs &Knobs); /// \brief Get an InlineCost with the callee explicitly specified. /// This allows you to calculate the cost of inlining a function via a @@ -121,12 +141,8 @@ // InlineCost getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, TargetTransformInfo &CalleeTTI, - AssumptionCacheTracker *ACT, ProfileSummaryInfo *PSI); - -int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel); - -/// \brief Return the default value of -inline-threshold. -int getDefaultInlineThreshold(); + AssumptionCacheTracker *ACT, ProfileSummaryInfo *PSI, + const InlineKnobs &Knobs); /// \brief Minimal filter to detect invalid constructs for inlining. bool isInlineViable(Function &Callee); Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -40,32 +40,6 @@ STATISTIC(NumCallsAnalyzed, "Number of call sites analyzed"); -// Threshold to use when optsize is specified (and there is no -// -inline-threshold). -const int OptSizeThreshold = 75; - -// Threshold to use when -Oz is specified (and there is no -inline-threshold). -const int OptMinSizeThreshold = 25; - -// Threshold to use when -O[34] is specified (and there is no -// -inline-threshold). -const int OptAggressiveThreshold = 275; - -static cl::opt DefaultInlineThreshold( - "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, - cl::desc("Control the amount of inlining to perform (default = 225)")); - -static cl::opt HintThreshold( - "inlinehint-threshold", cl::Hidden, cl::init(325), - cl::desc("Threshold for inlining functions with inline hint")); - -// We introduce this threshold to help performance of instrumentation based -// PGO before we actually hook up inliner with analysis passes such as BPI and -// BFI. -static cl::opt ColdThreshold( - "inlinecold-threshold", cl::Hidden, cl::init(225), - cl::desc("Threshold for inlining functions with cold attribute")); - namespace { class CallAnalyzer : public InstVisitor { @@ -89,6 +63,9 @@ // easily cacheable. Instead, use the cover function paramHasAttr. CallSite CandidateCS; + // Tunable knobs that control the analysis. + const InlineKnobs &Knobs; + int Threshold; int Cost; @@ -205,9 +182,9 @@ public: CallAnalyzer(const TargetTransformInfo &TTI, AssumptionCacheTracker *ACT, ProfileSummaryInfo *PSI, Function &Callee, int Threshold, - CallSite CSArg) + CallSite CSArg, const InlineKnobs &Knobs) : TTI(TTI), ACT(ACT), PSI(PSI), F(Callee), CandidateCS(CSArg), - Threshold(Threshold), Cost(0), IsCallerRecursive(false), + Knobs(Knobs), Threshold(Threshold), Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), HasFrameEscape(false), @@ -611,6 +588,10 @@ return true; } +static int Min(int A, Optional B) { + return B ? std::min(A, B.getValue()) : A; +} + void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // If no size growth is allowed for this inlining, set Threshold to 0. if (!allowSizeGrowth(CS)) { @@ -619,35 +600,26 @@ } Function *Caller = CS.getCaller(); - if (DefaultInlineThreshold.getNumOccurrences() > 0) { - // Explicitly specified -inline-threhold overrides the threshold passed to - // CallAnalyzer's constructor. - Threshold = DefaultInlineThreshold; - } else { - // If -inline-threshold is not given, listen to the optsize and minsize - // attributes when they would decrease the threshold. - if (Caller->optForMinSize() && OptMinSizeThreshold < Threshold) - Threshold = OptMinSizeThreshold; - else if (Caller->optForSize() && OptSizeThreshold < Threshold) - Threshold = OptSizeThreshold; - } + // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available + // and reduce the threshold if the caller has the necessary attribute. + if (Caller->optForMinSize()) + Threshold = Min(Threshold, Knobs.OptMinSizeThreshold); + else if (Caller->optForSize()) + Threshold = Min(Threshold, Knobs.OptSizeThreshold); // Listen to the inlinehint attribute or profile based hotness information // when it would increase the threshold and the caller does not need to // minimize its size. bool InlineHint = Callee.hasFnAttribute(Attribute::InlineHint) || PSI->isHotFunction(&Callee); - if (InlineHint && HintThreshold > Threshold && !Caller->optForMinSize()) - Threshold = HintThreshold; + if (InlineHint && !Caller->optForMinSize()) + Threshold = std::max(Threshold, Knobs.HintThreshold); bool ColdCallee = PSI->isColdFunction(&Callee); - // Command line argument for DefaultInlineThreshold will override the default - // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, - // do not use the default cold threshold even if it is smaller. - if ((DefaultInlineThreshold.getNumOccurrences() == 0 || - ColdThreshold.getNumOccurrences() > 0) && - ColdCallee && ColdThreshold < Threshold) - Threshold = ColdThreshold; + // For cold callees, use the ColdThreshold knob if it is available and reduces + // the threshold. + if (ColdCallee) + Threshold = Min(Threshold, Knobs.ColdThreshold); // Finally, take the target-specific inlining threshold multiplier into // account. @@ -949,8 +921,8 @@ // during devirtualization and so we want to give it a hefty bonus for // inlining, but cap that bonus in the event that inlining wouldn't pan // out. Pretend to inline the function, with a custom threshold. - CallAnalyzer CA(TTI, ACT, PSI, *F, InlineConstants::IndirectCallThreshold, - CS); + CallAnalyzer CA(TTI, ACT, PSI, *F, InlineConstants::IndirectCallThreshold, CS, + Knobs); if (CA.analyzeCall(CS)) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. @@ -1439,29 +1411,16 @@ InlineCost llvm::getInlineCost(CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, AssumptionCacheTracker *ACT, - ProfileSummaryInfo *PSI) { + ProfileSummaryInfo *PSI, + const InlineKnobs &Knobs) { return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI, - ACT, PSI); -} - -int llvm::computeThresholdFromOptLevels(unsigned OptLevel, - unsigned SizeOptLevel) { - if (OptLevel > 2) - return OptAggressiveThreshold; - if (SizeOptLevel == 1) // -Os - return OptSizeThreshold; - if (SizeOptLevel == 2) // -Oz - return OptMinSizeThreshold; - return DefaultInlineThreshold; + ACT, PSI, Knobs); } -int llvm::getDefaultInlineThreshold() { return DefaultInlineThreshold; } - -InlineCost llvm::getInlineCost(CallSite CS, Function *Callee, - int DefaultThreshold, - TargetTransformInfo &CalleeTTI, - AssumptionCacheTracker *ACT, - ProfileSummaryInfo *PSI) { +InlineCost +llvm::getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, + TargetTransformInfo &CalleeTTI, AssumptionCacheTracker *ACT, + ProfileSummaryInfo *PSI, const InlineKnobs &Knobs) { // Cannot inline indirect calls. if (!Callee) @@ -1495,7 +1454,7 @@ DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "...\n"); - CallAnalyzer CA(CalleeTTI, ACT, PSI, *Callee, DefaultThreshold, CS); + CallAnalyzer CA(CalleeTTI, ACT, PSI, *Callee, DefaultThreshold, CS, Knobs); bool ShouldInline = CA.analyzeCall(CS); DEBUG(CA.dump()); Index: lib/Transforms/IPO/InlineSimple.cpp =================================================================== --- lib/Transforms/IPO/InlineSimple.cpp +++ lib/Transforms/IPO/InlineSimple.cpp @@ -31,6 +31,21 @@ #define DEBUG_TYPE "inline" +static cl::opt DefaultInlineThreshold( + "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, + cl::desc("Control the amount of inlining to perform (default = 225)")); + +static cl::opt HintThreshold( + "inlinehint-threshold", cl::Hidden, cl::init(325), + cl::desc("Threshold for inlining functions with inline hint")); + +// We introduce this threshold to help performance of instrumentation based +// PGO before we actually hook up inliner with analysis passes such as BPI and +// BFI. +static cl::opt ColdThreshold( + "inlinecold-threshold", cl::Hidden, cl::init(225), + cl::desc("Threshold for inlining functions with cold attribute")); + namespace { /// \brief Actual inliner pass implementation. @@ -41,18 +56,23 @@ class SimpleInliner : public Inliner { // This field is populated based on one of the following: // * optimization or size-optimization levels, - // * the --inline-threshold flag, or - // * a user specified value. + // * a value passed to createFunctionInliningPass function, or + // * the -inline-threshold flag. + // If the -inline-threshold flag is explicitly specified, it overrides the + // other two ways of specifiying this threshold. int DefaultThreshold; + InlineKnobs Knobs; + public: SimpleInliner() - : Inliner(ID), DefaultThreshold(llvm::getDefaultInlineThreshold()) { + : Inliner(ID), DefaultThreshold(DefaultInlineThreshold), + Knobs(getInlineKnobs()) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } explicit SimpleInliner(int Threshold) - : Inliner(ID), DefaultThreshold(Threshold) { + : Inliner(ID), DefaultThreshold(Threshold), Knobs(getInlineKnobs()) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } @@ -61,7 +81,7 @@ InlineCost getInlineCost(CallSite CS) override { Function *Callee = CS.getCalledFunction(); TargetTransformInfo &TTI = TTIWP->getTTI(*Callee); - return llvm::getInlineCost(CS, DefaultThreshold, TTI, ACT, PSI); + return llvm::getInlineCost(CS, DefaultThreshold, TTI, ACT, PSI, Knobs); } bool runOnSCC(CallGraphSCC &SCC) override; @@ -69,6 +89,31 @@ private: TargetTransformInfoWrapperPass *TTIWP; + + // Generate the knobs to tune the inline cost analysis based on commandline + // options. + InlineKnobs getInlineKnobs() { + InlineKnobs Knobs; + // Set the HintThreshold knob from the -inlinehint-threshold. + Knobs.HintThreshold = HintThreshold; + + // Set the OptMinSizeThreshold and OptSizeThreshold knobs only if the + // -inlinehint-threshold commandline option is not explicitly given. If that + // option is present, then its value applies even for callees with size and + // minsize attributes. + // If the -inline-threshold is not specified, set the ColdThreshold from the + // -inlinecold-threshold even if it is not explicitly passed. If + // -inline-threshold is specified, then -inlinecold-threshold needs to be + // explicitly specified to set the ColdThreshold knob + if (DefaultInlineThreshold.getNumOccurrences() == 0) { + Knobs.OptMinSizeThreshold = InlineConstants::OptMinSizeThreshold; + Knobs.OptSizeThreshold = InlineConstants::OptSizeThreshold; + Knobs.ColdThreshold = ColdThreshold; + } else if (ColdThreshold.getNumOccurrences() > 0) { + Knobs.ColdThreshold = ColdThreshold; + } + return Knobs; + } }; } // end anonymous namespace @@ -84,16 +129,36 @@ INITIALIZE_PASS_END(SimpleInliner, "inline", "Function Integration/Inlining", false, false) -Pass *llvm::createFunctionInliningPass() { return new SimpleInliner(); } +Pass *llvm::createFunctionInliningPass() { + return new SimpleInliner(DefaultInlineThreshold); +} Pass *llvm::createFunctionInliningPass(int Threshold) { - return new SimpleInliner(Threshold); + // Explicitly specified -inline-threshold overrides Threshold. + return new SimpleInliner(DefaultInlineThreshold.getNumOccurrences() > 0 + ? DefaultInlineThreshold + : Threshold); +} + +static int computeThresholdFromOptLevels(unsigned OptLevel, + unsigned SizeOptLevel) { + if (OptLevel > 2) + return InlineConstants::OptAggressiveThreshold; + if (SizeOptLevel == 1) // -Os + return InlineConstants::OptSizeThreshold; + if (SizeOptLevel == 2) // -Oz + return InlineConstants::OptMinSizeThreshold; + return DefaultInlineThreshold; } Pass *llvm::createFunctionInliningPass(unsigned OptLevel, unsigned SizeOptLevel) { + // Explicitly specified -inline-threshold overrides threshold generated based + // on opt/size levels. return new SimpleInliner( - llvm::computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); + DefaultInlineThreshold.getNumOccurrences() > 0 + ? DefaultInlineThreshold + : computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); } bool SimpleInliner::runOnSCC(CallGraphSCC &SCC) {