Index: include/llvm/Analysis/InlineCost.h =================================================================== --- include/llvm/Analysis/InlineCost.h +++ include/llvm/Analysis/InlineCost.h @@ -28,16 +28,26 @@ class TargetTransformInfo; namespace InlineConstants { - // Various magic constants used to adjust heuristics. - const int InstrCost = 5; - const int IndirectCallThreshold = 100; - const int CallPenalty = 25; - const int LastCallToStaticBonus = -15000; - const int ColdccPenalty = 2000; - const int NoreturnPenalty = 10000; - /// Do not inline functions which allocate this many bytes on the stack - /// when the caller is recursive. - const unsigned TotalAllocaSizeRecursiveCaller = 1024; +// Various thresholds used by inline cost analysis. +// Use when optsize (-Os) is specified. +const int OptSizeThreshold = 75; + +// Use when minsize (-Oz) is specified. +const int OptMinSizeThreshold = 25; + +// Use when -O[34] is specified. +const int OptAggressiveThreshold = 275; + +// Various magic constants used to adjust heuristics. +const int InstrCost = 5; +const int IndirectCallThreshold = 100; +const int CallPenalty = 25; +const int LastCallToStaticBonus = -15000; +const int ColdccPenalty = 2000; +const int NoreturnPenalty = 10000; +/// Do not inline functions which allocate this many bytes on the stack +/// when the caller is recursive. +const unsigned TotalAllocaSizeRecursiveCaller = 1024; } /// \brief Represents the cost of inlining a function. @@ -100,6 +110,22 @@ int getCostDelta() const { return Threshold - getCost(); } }; +/// Params to tune inline cost analysis. These are populated based on +/// commandline +/// options. +struct InlineParams { + /// The default threshold to start with for a callee. + int DefaultThreshold; + /// Threshold to use for callees with inline hint. + int HintThreshold; + /// Threshold to use for cold callees. + Optional ColdThreshold; + /// Threshold to use when the caller is optimized for size. + Optional OptSizeThreshold; + /// Threshold to use when the caller is optimized for minsize. + Optional OptMinSizeThreshold; +}; + /// \brief Get an InlineCost object representing the cost of inlining this /// callsite. /// @@ -112,9 +138,9 @@ /// Also note that calling this function *dynamically* computes the cost of /// inlining the callsite. It is an expensive, heavyweight call. InlineCost -getInlineCost(CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, +getInlineCost(CallSite CS, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI); + ProfileSummaryInfo *PSI, const InlineParams &Params); /// \brief Get an InlineCost with the callee explicitly specified. /// This allows you to calculate the cost of inlining a function via a @@ -122,15 +148,9 @@ /// parameter in all other respects. // InlineCost -getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, - TargetTransformInfo &CalleeTTI, +getInlineCost(CallSite CS, Function *Callee, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI); - -int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel); - -/// \brief Return the default value of -inline-threshold. -int getDefaultInlineThreshold(); + ProfileSummaryInfo *PSI, const InlineParams &Params); /// \brief Minimal filter to detect invalid constructs for inlining. bool isInlineViable(Function &Callee); Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -40,32 +40,6 @@ STATISTIC(NumCallsAnalyzed, "Number of call sites analyzed"); -// Threshold to use when optsize is specified (and there is no -// -inline-threshold). -const int OptSizeThreshold = 75; - -// Threshold to use when -Oz is specified (and there is no -inline-threshold). -const int OptMinSizeThreshold = 25; - -// Threshold to use when -O[34] is specified (and there is no -// -inline-threshold). -const int OptAggressiveThreshold = 275; - -static cl::opt DefaultInlineThreshold( - "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, - cl::desc("Control the amount of inlining to perform (default = 225)")); - -static cl::opt HintThreshold( - "inlinehint-threshold", cl::Hidden, cl::init(325), - cl::desc("Threshold for inlining functions with inline hint")); - -// We introduce this threshold to help performance of instrumentation based -// PGO before we actually hook up inliner with analysis passes such as BPI and -// BFI. -static cl::opt ColdThreshold( - "inlinecold-threshold", cl::Hidden, cl::init(225), - cl::desc("Threshold for inlining functions with cold attribute")); - namespace { class CallAnalyzer : public InstVisitor { @@ -89,6 +63,9 @@ // easily cacheable. Instead, use the cover function paramHasAttr. CallSite CandidateCS; + // Tunable parameters that control the analysis. + const InlineParams &Params; + int Threshold; int Cost; @@ -205,11 +182,11 @@ public: CallAnalyzer(const TargetTransformInfo &TTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI, Function &Callee, int Threshold, - CallSite CSArg) + ProfileSummaryInfo *PSI, Function &Callee, CallSite CSArg, + const InlineParams &Params) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), PSI(PSI), F(Callee), - CandidateCS(CSArg), Threshold(Threshold), Cost(0), - IsCallerRecursive(false), IsRecursiveCall(false), + CandidateCS(CSArg), Params(Params), Threshold(Params.DefaultThreshold), + Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), HasFrameEscape(false), AllocatedSize(0), NumInstructions(0), @@ -613,6 +590,10 @@ return true; } +static int Min(int A, Optional B) { + return B ? std::min(A, B.getValue()) : A; +} + void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // If no size growth is allowed for this inlining, set Threshold to 0. if (!allowSizeGrowth(CS)) { @@ -621,18 +602,12 @@ } Function *Caller = CS.getCaller(); - if (DefaultInlineThreshold.getNumOccurrences() > 0) { - // Explicitly specified -inline-threhold overrides the threshold passed to - // CallAnalyzer's constructor. - Threshold = DefaultInlineThreshold; - } else { - // If -inline-threshold is not given, listen to the optsize and minsize - // attributes when they would decrease the threshold. - if (Caller->optForMinSize() && OptMinSizeThreshold < Threshold) - Threshold = OptMinSizeThreshold; - else if (Caller->optForSize() && OptSizeThreshold < Threshold) - Threshold = OptSizeThreshold; - } + // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available + // and reduce the threshold if the caller has the necessary attribute. + if (Caller->optForMinSize()) + Threshold = Min(Threshold, Params.OptMinSizeThreshold); + else if (Caller->optForSize()) + Threshold = Min(Threshold, Params.OptSizeThreshold); bool HotCallsite = false; uint64_t TotalWeight; @@ -646,17 +621,14 @@ bool InlineHint = Callee.hasFnAttribute(Attribute::InlineHint) || PSI->isHotFunction(&Callee) || HotCallsite; - if (InlineHint && HintThreshold > Threshold && !Caller->optForMinSize()) - Threshold = HintThreshold; + if (InlineHint && !Caller->optForMinSize()) + Threshold = std::max(Threshold, Params.HintThreshold); bool ColdCallee = PSI->isColdFunction(&Callee); - // Command line argument for DefaultInlineThreshold will override the default - // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, - // do not use the default cold threshold even if it is smaller. - if ((DefaultInlineThreshold.getNumOccurrences() == 0 || - ColdThreshold.getNumOccurrences() > 0) && - ColdCallee && ColdThreshold < Threshold) - Threshold = ColdThreshold; + // For cold callees, use the ColdThreshold knob if it is available and reduces + // the threshold. + if (ColdCallee) + Threshold = Min(Threshold, Params.ColdThreshold); // Finally, take the target-specific inlining threshold multiplier into // account. @@ -958,8 +930,9 @@ // during devirtualization and so we want to give it a hefty bonus for // inlining, but cap that bonus in the event that inlining wouldn't pan // out. Pretend to inline the function, with a custom threshold. - CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F, - InlineConstants::IndirectCallThreshold, CS); + InlineParams IndirectCallParams(Params); + IndirectCallParams.DefaultThreshold = InlineConstants::IndirectCallThreshold; + CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F, CS, Params); if (CA.analyzeCall(CS)) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. @@ -1445,31 +1418,17 @@ } InlineCost llvm::getInlineCost( - CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, + CallSite CS, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI) { - return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI, - GetAssumptionCache, PSI); + ProfileSummaryInfo *PSI, const InlineParams &Params) { + return getInlineCost(CS, CS.getCalledFunction(), CalleeTTI, + GetAssumptionCache, PSI, Params); } -int llvm::computeThresholdFromOptLevels(unsigned OptLevel, - unsigned SizeOptLevel) { - if (OptLevel > 2) - return OptAggressiveThreshold; - if (SizeOptLevel == 1) // -Os - return OptSizeThreshold; - if (SizeOptLevel == 2) // -Oz - return OptMinSizeThreshold; - return DefaultInlineThreshold; -} - -int llvm::getDefaultInlineThreshold() { return DefaultInlineThreshold; } - InlineCost llvm::getInlineCost( - CallSite CS, Function *Callee, int DefaultThreshold, - TargetTransformInfo &CalleeTTI, + CallSite CS, Function *Callee, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI) { + ProfileSummaryInfo *PSI, const InlineParams &Params) { // Cannot inline indirect calls. if (!Callee) @@ -1503,7 +1462,7 @@ DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "...\n"); - CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, DefaultThreshold, CS); + CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, CS, Params); bool ShouldInline = CA.analyzeCall(CS); DEBUG(CA.dump()); Index: lib/Transforms/IPO/InlineSimple.cpp =================================================================== --- lib/Transforms/IPO/InlineSimple.cpp +++ lib/Transforms/IPO/InlineSimple.cpp @@ -31,6 +31,21 @@ #define DEBUG_TYPE "inline" +static cl::opt DefaultInlineThreshold( + "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, + cl::desc("Control the amount of inlining to perform (default = 225)")); + +static cl::opt HintThreshold( + "inlinehint-threshold", cl::Hidden, cl::init(325), + cl::desc("Threshold for inlining functions with inline hint")); + +// We introduce this threshold to help performance of instrumentation based +// PGO before we actually hook up inliner with analysis passes such as BPI and +// BFI. +static cl::opt ColdThreshold( + "inlinecold-threshold", cl::Hidden, cl::init(225), + cl::desc("Threshold for inlining functions with cold attribute")); + namespace { /// \brief Actual inliner pass implementation. @@ -39,20 +54,17 @@ /// inliner pass and the always inliner pass. The two passes use different cost /// analyses to determine when to inline. class SimpleInliner : public Inliner { - // This field is populated based on one of the following: - // * optimization or size-optimization levels, - // * the --inline-threshold flag, or - // * a user specified value. - int DefaultThreshold; + + InlineParams Params; public: SimpleInliner() - : Inliner(ID), DefaultThreshold(llvm::getDefaultInlineThreshold()) { + : Inliner(ID), Params(getInlineParams(DefaultInlineThreshold)) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } explicit SimpleInliner(int Threshold) - : Inliner(ID), DefaultThreshold(Threshold) { + : Inliner(ID), Params(getInlineParams(Threshold)) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } @@ -65,8 +77,7 @@ Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); }; - return llvm::getInlineCost(CS, DefaultThreshold, TTI, GetAssumptionCache, - PSI); + return llvm::getInlineCost(CS, TTI, GetAssumptionCache, PSI, Params); } bool runOnSCC(CallGraphSCC &SCC) override; @@ -74,6 +85,44 @@ private: TargetTransformInfoWrapperPass *TTIWP; + + // Generate the knobs to tune the inline cost analysis based on commandline + // options. + InlineParams getInlineParams(int Threshold) { + InlineParams Params; + + // This field is the threshold to use for a callee by default. This is + // derived from one or more of: + // * optimization or size-optimization levels, + // * a value passed to createFunctionInliningPass function, or + // * the -inline-threshold flag. + // If the -inline-threshold flag is explicitly specified, that is used + // irrespective of anything else. + if (DefaultInlineThreshold.getNumOccurrences() > 0) + Params.DefaultThreshold = DefaultInlineThreshold; + else + Params.DefaultThreshold = Threshold; + + // Set the HintThreshold knob from the -inlinehint-threshold. + Params.HintThreshold = HintThreshold; + + // Set the OptMinSizeThreshold and OptSizeThreshold knobs only if the + // -inlinehint-threshold commandline option is not explicitly given. If that + // option is present, then its value applies even for callees with size and + // minsize attributes. + // If the -inline-threshold is not specified, set the ColdThreshold from the + // -inlinecold-threshold even if it is not explicitly passed. If + // -inline-threshold is specified, then -inlinecold-threshold needs to be + // explicitly specified to set the ColdThreshold knob + if (DefaultInlineThreshold.getNumOccurrences() == 0) { + Params.OptMinSizeThreshold = InlineConstants::OptMinSizeThreshold; + Params.OptSizeThreshold = InlineConstants::OptSizeThreshold; + Params.ColdThreshold = ColdThreshold; + } else if (ColdThreshold.getNumOccurrences() > 0) { + Params.ColdThreshold = ColdThreshold; + } + return Params; + } }; } // end anonymous namespace @@ -95,10 +144,21 @@ return new SimpleInliner(Threshold); } +static int computeThresholdFromOptLevels(unsigned OptLevel, + unsigned SizeOptLevel) { + if (OptLevel > 2) + return InlineConstants::OptAggressiveThreshold; + if (SizeOptLevel == 1) // -Os + return InlineConstants::OptSizeThreshold; + if (SizeOptLevel == 2) // -Oz + return InlineConstants::OptMinSizeThreshold; + return DefaultInlineThreshold; +} + Pass *llvm::createFunctionInliningPass(unsigned OptLevel, unsigned SizeOptLevel) { return new SimpleInliner( - llvm::computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); + computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); } bool SimpleInliner::runOnSCC(CallGraphSCC &SCC) {