Index: include/llvm/Analysis/InlineCost.h =================================================================== --- include/llvm/Analysis/InlineCost.h +++ include/llvm/Analysis/InlineCost.h @@ -28,16 +28,26 @@ class TargetTransformInfo; namespace InlineConstants { - // Various magic constants used to adjust heuristics. - const int InstrCost = 5; - const int IndirectCallThreshold = 100; - const int CallPenalty = 25; - const int LastCallToStaticBonus = -15000; - const int ColdccPenalty = 2000; - const int NoreturnPenalty = 10000; - /// Do not inline functions which allocate this many bytes on the stack - /// when the caller is recursive. - const unsigned TotalAllocaSizeRecursiveCaller = 1024; +// Various thresholds used by inline cost analysis. +// Use when optsize (-Os) is specified. +const int OptSizeThreshold = 75; + +// Use when minsize (-Oz) is specified. +const int OptMinSizeThreshold = 25; + +// Use when -O[34] is specified. +const int OptAggressiveThreshold = 275; + +// Various magic constants used to adjust heuristics. +const int InstrCost = 5; +const int IndirectCallThreshold = 100; +const int CallPenalty = 25; +const int LastCallToStaticBonus = -15000; +const int ColdccPenalty = 2000; +const int NoreturnPenalty = 10000; +/// Do not inline functions which allocate this many bytes on the stack +/// when the caller is recursive. +const unsigned TotalAllocaSizeRecursiveCaller = 1024; } /// \brief Represents the cost of inlining a function. @@ -100,6 +110,20 @@ int getCostDelta() const { return Threshold - getCost(); } }; +/// Params to tune inline cost analysis. These are populated based on +/// commandline +/// options. +struct InlineParams { + /// Threshold to use for callees with inline hint. + int HintThreshold; + /// Threshold to use for cold callees. + Optional ColdThreshold; + /// Threshold to use when the caller is optimized for size. + Optional OptSizeThreshold; + /// Threshold to use when the caller is optimized for minsize. + Optional OptMinSizeThreshold; +}; + /// \brief Get an InlineCost object representing the cost of inlining this /// callsite. /// @@ -114,7 +138,7 @@ InlineCost getInlineCost(CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI); + ProfileSummaryInfo *PSI, const InlineParams &Params); /// \brief Get an InlineCost with the callee explicitly specified. /// This allows you to calculate the cost of inlining a function via a @@ -125,12 +149,7 @@ getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI); - -int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel); - -/// \brief Return the default value of -inline-threshold. -int getDefaultInlineThreshold(); + ProfileSummaryInfo *PSI, const InlineParams &Params); /// \brief Minimal filter to detect invalid constructs for inlining. bool isInlineViable(Function &Callee); Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -40,32 +40,6 @@ STATISTIC(NumCallsAnalyzed, "Number of call sites analyzed"); -// Threshold to use when optsize is specified (and there is no -// -inline-threshold). -const int OptSizeThreshold = 75; - -// Threshold to use when -Oz is specified (and there is no -inline-threshold). -const int OptMinSizeThreshold = 25; - -// Threshold to use when -O[34] is specified (and there is no -// -inline-threshold). -const int OptAggressiveThreshold = 275; - -static cl::opt DefaultInlineThreshold( - "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, - cl::desc("Control the amount of inlining to perform (default = 225)")); - -static cl::opt HintThreshold( - "inlinehint-threshold", cl::Hidden, cl::init(325), - cl::desc("Threshold for inlining functions with inline hint")); - -// We introduce this threshold to help performance of instrumentation based -// PGO before we actually hook up inliner with analysis passes such as BPI and -// BFI. -static cl::opt ColdThreshold( - "inlinecold-threshold", cl::Hidden, cl::init(225), - cl::desc("Threshold for inlining functions with cold attribute")); - namespace { class CallAnalyzer : public InstVisitor { @@ -89,6 +63,9 @@ // easily cacheable. Instead, use the cover function paramHasAttr. CallSite CandidateCS; + // Tunable knobs that control the analysis. + const InlineParams &Params; + int Threshold; int Cost; @@ -206,9 +183,9 @@ CallAnalyzer(const TargetTransformInfo &TTI, std::function &GetAssumptionCache, ProfileSummaryInfo *PSI, Function &Callee, int Threshold, - CallSite CSArg) + CallSite CSArg, const InlineParams &Params) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), PSI(PSI), F(Callee), - CandidateCS(CSArg), Threshold(Threshold), Cost(0), + CandidateCS(CSArg), Params(Params), Threshold(Threshold), Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), @@ -613,6 +590,10 @@ return true; } +static int Min(int A, Optional B) { + return B ? std::min(A, B.getValue()) : A; +} + void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // If no size growth is allowed for this inlining, set Threshold to 0. if (!allowSizeGrowth(CS)) { @@ -621,18 +602,12 @@ } Function *Caller = CS.getCaller(); - if (DefaultInlineThreshold.getNumOccurrences() > 0) { - // Explicitly specified -inline-threhold overrides the threshold passed to - // CallAnalyzer's constructor. - Threshold = DefaultInlineThreshold; - } else { - // If -inline-threshold is not given, listen to the optsize and minsize - // attributes when they would decrease the threshold. - if (Caller->optForMinSize() && OptMinSizeThreshold < Threshold) - Threshold = OptMinSizeThreshold; - else if (Caller->optForSize() && OptSizeThreshold < Threshold) - Threshold = OptSizeThreshold; - } + // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available + // and reduce the threshold if the caller has the necessary attribute. + if (Caller->optForMinSize()) + Threshold = Min(Threshold, Params.OptMinSizeThreshold); + else if (Caller->optForSize()) + Threshold = Min(Threshold, Params.OptSizeThreshold); bool HotCallsite = false; uint64_t TotalWeight; @@ -646,17 +621,14 @@ bool InlineHint = Callee.hasFnAttribute(Attribute::InlineHint) || PSI->isHotFunction(&Callee) || HotCallsite; - if (InlineHint && HintThreshold > Threshold && !Caller->optForMinSize()) - Threshold = HintThreshold; + if (InlineHint && !Caller->optForMinSize()) + Threshold = std::max(Threshold, Params.HintThreshold); bool ColdCallee = PSI->isColdFunction(&Callee); - // Command line argument for DefaultInlineThreshold will override the default - // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, - // do not use the default cold threshold even if it is smaller. - if ((DefaultInlineThreshold.getNumOccurrences() == 0 || - ColdThreshold.getNumOccurrences() > 0) && - ColdCallee && ColdThreshold < Threshold) - Threshold = ColdThreshold; + // For cold callees, use the ColdThreshold knob if it is available and reduces + // the threshold. + if (ColdCallee) + Threshold = Min(Threshold, Params.ColdThreshold); // Finally, take the target-specific inlining threshold multiplier into // account. @@ -959,7 +931,7 @@ // inlining, but cap that bonus in the event that inlining wouldn't pan // out. Pretend to inline the function, with a custom threshold. CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F, - InlineConstants::IndirectCallThreshold, CS); + InlineConstants::IndirectCallThreshold, CS, Params); if (CA.analyzeCall(CS)) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. @@ -1447,29 +1419,16 @@ InlineCost llvm::getInlineCost( CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI) { + ProfileSummaryInfo *PSI, const InlineParams &Params) { return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI, - GetAssumptionCache, PSI); + GetAssumptionCache, PSI, Params); } -int llvm::computeThresholdFromOptLevels(unsigned OptLevel, - unsigned SizeOptLevel) { - if (OptLevel > 2) - return OptAggressiveThreshold; - if (SizeOptLevel == 1) // -Os - return OptSizeThreshold; - if (SizeOptLevel == 2) // -Oz - return OptMinSizeThreshold; - return DefaultInlineThreshold; -} - -int llvm::getDefaultInlineThreshold() { return DefaultInlineThreshold; } - InlineCost llvm::getInlineCost( CallSite CS, Function *Callee, int DefaultThreshold, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI) { + ProfileSummaryInfo *PSI, const InlineParams &Params) { // Cannot inline indirect calls. if (!Callee) @@ -1503,7 +1462,8 @@ DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "...\n"); - CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, DefaultThreshold, CS); + CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, DefaultThreshold, + CS, Params); bool ShouldInline = CA.analyzeCall(CS); DEBUG(CA.dump()); Index: lib/Transforms/IPO/InlineSimple.cpp =================================================================== --- lib/Transforms/IPO/InlineSimple.cpp +++ lib/Transforms/IPO/InlineSimple.cpp @@ -31,6 +31,21 @@ #define DEBUG_TYPE "inline" +static cl::opt DefaultInlineThreshold( + "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, + cl::desc("Control the amount of inlining to perform (default = 225)")); + +static cl::opt HintThreshold( + "inlinehint-threshold", cl::Hidden, cl::init(325), + cl::desc("Threshold for inlining functions with inline hint")); + +// We introduce this threshold to help performance of instrumentation based +// PGO before we actually hook up inliner with analysis passes such as BPI and +// BFI. +static cl::opt ColdThreshold( + "inlinecold-threshold", cl::Hidden, cl::init(225), + cl::desc("Threshold for inlining functions with cold attribute")); + namespace { /// \brief Actual inliner pass implementation. @@ -41,18 +56,23 @@ class SimpleInliner : public Inliner { // This field is populated based on one of the following: // * optimization or size-optimization levels, - // * the --inline-threshold flag, or - // * a user specified value. + // * a value passed to createFunctionInliningPass function, or + // * the -inline-threshold flag. + // If the -inline-threshold flag is explicitly specified, it overrides the + // other two ways of specifiying this threshold. int DefaultThreshold; + InlineParams Params; + public: SimpleInliner() - : Inliner(ID), DefaultThreshold(llvm::getDefaultInlineThreshold()) { + : Inliner(ID), DefaultThreshold(DefaultInlineThreshold), + Params(getInlineParams()) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } explicit SimpleInliner(int Threshold) - : Inliner(ID), DefaultThreshold(Threshold) { + : Inliner(ID), DefaultThreshold(Threshold), Params(getInlineParams()) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } @@ -66,7 +86,7 @@ return ACT->getAssumptionCache(F); }; return llvm::getInlineCost(CS, DefaultThreshold, TTI, GetAssumptionCache, - PSI); + PSI, Params); } bool runOnSCC(CallGraphSCC &SCC) override; @@ -74,6 +94,31 @@ private: TargetTransformInfoWrapperPass *TTIWP; + + // Generate the knobs to tune the inline cost analysis based on commandline + // options. + InlineParams getInlineParams() { + InlineParams Params; + // Set the HintThreshold knob from the -inlinehint-threshold. + Params.HintThreshold = HintThreshold; + + // Set the OptMinSizeThreshold and OptSizeThreshold knobs only if the + // -inlinehint-threshold commandline option is not explicitly given. If that + // option is present, then its value applies even for callees with size and + // minsize attributes. + // If the -inline-threshold is not specified, set the ColdThreshold from the + // -inlinecold-threshold even if it is not explicitly passed. If + // -inline-threshold is specified, then -inlinecold-threshold needs to be + // explicitly specified to set the ColdThreshold knob + if (DefaultInlineThreshold.getNumOccurrences() == 0) { + Params.OptMinSizeThreshold = InlineConstants::OptMinSizeThreshold; + Params.OptSizeThreshold = InlineConstants::OptSizeThreshold; + Params.ColdThreshold = ColdThreshold; + } else if (ColdThreshold.getNumOccurrences() > 0) { + Params.ColdThreshold = ColdThreshold; + } + return Params; + } }; } // end anonymous namespace @@ -89,16 +134,36 @@ INITIALIZE_PASS_END(SimpleInliner, "inline", "Function Integration/Inlining", false, false) -Pass *llvm::createFunctionInliningPass() { return new SimpleInliner(); } +Pass *llvm::createFunctionInliningPass() { + return new SimpleInliner(DefaultInlineThreshold); +} Pass *llvm::createFunctionInliningPass(int Threshold) { - return new SimpleInliner(Threshold); + // Explicitly specified -inline-threshold overrides Threshold. + return new SimpleInliner(DefaultInlineThreshold.getNumOccurrences() > 0 + ? DefaultInlineThreshold + : Threshold); +} + +static int computeThresholdFromOptLevels(unsigned OptLevel, + unsigned SizeOptLevel) { + if (OptLevel > 2) + return InlineConstants::OptAggressiveThreshold; + if (SizeOptLevel == 1) // -Os + return InlineConstants::OptSizeThreshold; + if (SizeOptLevel == 2) // -Oz + return InlineConstants::OptMinSizeThreshold; + return DefaultInlineThreshold; } Pass *llvm::createFunctionInliningPass(unsigned OptLevel, unsigned SizeOptLevel) { + // Explicitly specified -inline-threshold overrides threshold generated based + // on opt/size levels. return new SimpleInliner( - llvm::computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); + DefaultInlineThreshold.getNumOccurrences() > 0 + ? DefaultInlineThreshold + : computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); } bool SimpleInliner::runOnSCC(CallGraphSCC &SCC) {