Index: include/llvm/Analysis/InlineCost.h =================================================================== --- include/llvm/Analysis/InlineCost.h +++ include/llvm/Analysis/InlineCost.h @@ -28,16 +28,26 @@ class TargetTransformInfo; namespace InlineConstants { - // Various magic constants used to adjust heuristics. - const int InstrCost = 5; - const int IndirectCallThreshold = 100; - const int CallPenalty = 25; - const int LastCallToStaticBonus = -15000; - const int ColdccPenalty = 2000; - const int NoreturnPenalty = 10000; - /// Do not inline functions which allocate this many bytes on the stack - /// when the caller is recursive. - const unsigned TotalAllocaSizeRecursiveCaller = 1024; +// Various thresholds used by inline cost analysis. +// Use when optsize (-Os) is specified. +const int OptSizeThreshold = 75; + +// Use when minsize (-Oz) is specified. +const int OptMinSizeThreshold = 25; + +// Use when -O3 is specified. +const int OptAggressiveThreshold = 275; + +// Various magic constants used to adjust heuristics. +const int InstrCost = 5; +const int IndirectCallThreshold = 100; +const int CallPenalty = 25; +const int LastCallToStaticBonus = -15000; +const int ColdccPenalty = 2000; +const int NoreturnPenalty = 10000; +/// Do not inline functions which allocate this many bytes on the stack +/// when the caller is recursive. +const unsigned TotalAllocaSizeRecursiveCaller = 1024; } /// \brief Represents the cost of inlining a function. @@ -100,6 +110,24 @@ int getCostDelta() const { return Threshold - getCost(); } }; +/// Thresholds to tune inline cost analysis. These are populated based on +/// commandline options by the clients of inline cost analysis. The inline cost +/// analysis decides the condition to apply a threshold and applies it. +/// Otherwise, DefaultThreshold is used. If a threshold is Optional, it is +/// applied only when it has a valid value. +struct InlineParams { + /// The default threshold to start with for a callee. + int DefaultThreshold; + /// Threshold to use for callees with inline hint. + int HintThreshold; + /// Threshold to use for cold callees. + Optional ColdThreshold; + /// Threshold to use when the caller is optimized for size. + Optional OptSizeThreshold; + /// Threshold to use when the caller is optimized for minsize. + Optional OptMinSizeThreshold; +}; + /// \brief Get an InlineCost object representing the cost of inlining this /// callsite. /// @@ -112,9 +140,9 @@ /// Also note that calling this function *dynamically* computes the cost of /// inlining the callsite. It is an expensive, heavyweight call. InlineCost -getInlineCost(CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, +getInlineCost(CallSite CS, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI); + ProfileSummaryInfo *PSI, const InlineParams &Params); /// \brief Get an InlineCost with the callee explicitly specified. /// This allows you to calculate the cost of inlining a function via a @@ -122,15 +150,9 @@ /// parameter in all other respects. // InlineCost -getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, - TargetTransformInfo &CalleeTTI, +getInlineCost(CallSite CS, Function *Callee, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI); - -int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel); - -/// \brief Return the default value of -inline-threshold. -int getDefaultInlineThreshold(); + ProfileSummaryInfo *PSI, const InlineParams &Params); /// \brief Minimal filter to detect invalid constructs for inlining. bool isInlineViable(Function &Callee); Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -40,31 +40,22 @@ STATISTIC(NumCallsAnalyzed, "Number of call sites analyzed"); -// Threshold to use when optsize is specified (and there is no -// -inline-threshold). -const int OptSizeThreshold = 75; - -// Threshold to use when -Oz is specified (and there is no -inline-threshold). -const int OptMinSizeThreshold = 25; - -// Threshold to use when -O[34] is specified (and there is no -// -inline-threshold). -const int OptAggressiveThreshold = 275; - -static cl::opt DefaultInlineThreshold( +namespace llvm { +cl::opt InlineThreshold( "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, cl::desc("Control the amount of inlining to perform (default = 225)")); -static cl::opt HintThreshold( +cl::opt HintThreshold( "inlinehint-threshold", cl::Hidden, cl::init(325), cl::desc("Threshold for inlining functions with inline hint")); // We introduce this threshold to help performance of instrumentation based // PGO before we actually hook up inliner with analysis passes such as BPI and // BFI. -static cl::opt ColdThreshold( +cl::opt ColdThreshold( "inlinecold-threshold", cl::Hidden, cl::init(225), cl::desc("Threshold for inlining functions with cold attribute")); +} namespace { @@ -89,6 +80,9 @@ // easily cacheable. Instead, use the cover function paramHasAttr. CallSite CandidateCS; + // Tunable parameters that control the analysis. + const InlineParams &Params; + int Threshold; int Cost; @@ -205,11 +199,11 @@ public: CallAnalyzer(const TargetTransformInfo &TTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI, Function &Callee, int Threshold, - CallSite CSArg) + ProfileSummaryInfo *PSI, Function &Callee, CallSite CSArg, + const InlineParams &Params) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), PSI(PSI), F(Callee), - CandidateCS(CSArg), Threshold(Threshold), Cost(0), - IsCallerRecursive(false), IsRecursiveCall(false), + CandidateCS(CSArg), Params(Params), Threshold(Params.DefaultThreshold), + Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), HasFrameEscape(false), AllocatedSize(0), NumInstructions(0), @@ -621,18 +615,18 @@ } Function *Caller = CS.getCaller(); - if (DefaultInlineThreshold.getNumOccurrences() > 0) { - // Explicitly specified -inline-threhold overrides the threshold passed to - // CallAnalyzer's constructor. - Threshold = DefaultInlineThreshold; - } else { - // If -inline-threshold is not given, listen to the optsize and minsize - // attributes when they would decrease the threshold. - if (Caller->optForMinSize() && OptMinSizeThreshold < Threshold) - Threshold = OptMinSizeThreshold; - else if (Caller->optForSize() && OptSizeThreshold < Threshold) - Threshold = OptSizeThreshold; - } + + // return min(A, B) if B is valid. + auto minIfValid = [](int A, Optional B) { + return B ? std::min(A, B.getValue()) : A; + }; + + // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available + // and reduce the threshold if the caller has the necessary attribute. + if (Caller->optForMinSize()) + Threshold = minIfValid(Threshold, Params.OptMinSizeThreshold); + else if (Caller->optForSize()) + Threshold = minIfValid(Threshold, Params.OptSizeThreshold); bool HotCallsite = false; uint64_t TotalWeight; @@ -646,17 +640,14 @@ bool InlineHint = Callee.hasFnAttribute(Attribute::InlineHint) || PSI->isHotFunction(&Callee) || HotCallsite; - if (InlineHint && HintThreshold > Threshold && !Caller->optForMinSize()) - Threshold = HintThreshold; + if (InlineHint && !Caller->optForMinSize()) + Threshold = std::max(Threshold, Params.HintThreshold); bool ColdCallee = PSI->isColdFunction(&Callee); - // Command line argument for DefaultInlineThreshold will override the default - // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, - // do not use the default cold threshold even if it is smaller. - if ((DefaultInlineThreshold.getNumOccurrences() == 0 || - ColdThreshold.getNumOccurrences() > 0) && - ColdCallee && ColdThreshold < Threshold) - Threshold = ColdThreshold; + // For cold callees, use the ColdThreshold knob if it is available and reduces + // the threshold. + if (ColdCallee) + Threshold = minIfValid(Threshold, Params.ColdThreshold); // Finally, take the target-specific inlining threshold multiplier into // account. @@ -958,8 +949,9 @@ // during devirtualization and so we want to give it a hefty bonus for // inlining, but cap that bonus in the event that inlining wouldn't pan // out. Pretend to inline the function, with a custom threshold. - CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F, - InlineConstants::IndirectCallThreshold, CS); + InlineParams IndirectCallParams(Params); + IndirectCallParams.DefaultThreshold = InlineConstants::IndirectCallThreshold; + CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F, CS, Params); if (CA.analyzeCall(CS)) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. @@ -1445,31 +1437,17 @@ } InlineCost llvm::getInlineCost( - CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, + CallSite CS, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI) { - return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI, - GetAssumptionCache, PSI); + ProfileSummaryInfo *PSI, const InlineParams &Params) { + return getInlineCost(CS, CS.getCalledFunction(), CalleeTTI, + GetAssumptionCache, PSI, Params); } -int llvm::computeThresholdFromOptLevels(unsigned OptLevel, - unsigned SizeOptLevel) { - if (OptLevel > 2) - return OptAggressiveThreshold; - if (SizeOptLevel == 1) // -Os - return OptSizeThreshold; - if (SizeOptLevel == 2) // -Oz - return OptMinSizeThreshold; - return DefaultInlineThreshold; -} - -int llvm::getDefaultInlineThreshold() { return DefaultInlineThreshold; } - InlineCost llvm::getInlineCost( - CallSite CS, Function *Callee, int DefaultThreshold, - TargetTransformInfo &CalleeTTI, + CallSite CS, Function *Callee, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI) { + ProfileSummaryInfo *PSI, const InlineParams &Params) { // Cannot inline indirect calls. if (!Callee) @@ -1503,7 +1481,7 @@ DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "...\n"); - CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, DefaultThreshold, CS); + CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, CS, Params); bool ShouldInline = CA.analyzeCall(CS); DEBUG(CA.dump()); Index: lib/Transforms/IPO/InlineSimple.cpp =================================================================== --- lib/Transforms/IPO/InlineSimple.cpp +++ lib/Transforms/IPO/InlineSimple.cpp @@ -31,6 +31,10 @@ #define DEBUG_TYPE "inline" +namespace llvm { +extern cl::opt InlineThreshold, HintThreshold, ColdThreshold; +} + namespace { /// \brief Actual inliner pass implementation. @@ -39,20 +43,17 @@ /// inliner pass and the always inliner pass. The two passes use different cost /// analyses to determine when to inline. class SimpleInliner : public Inliner { - // This field is populated based on one of the following: - // * optimization or size-optimization levels, - // * the --inline-threshold flag, or - // * a user specified value. - int DefaultThreshold; + + InlineParams Params; public: SimpleInliner() - : Inliner(ID), DefaultThreshold(llvm::getDefaultInlineThreshold()) { + : Inliner(ID), Params(getInlineParams(llvm::InlineThreshold)) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } explicit SimpleInliner(int Threshold) - : Inliner(ID), DefaultThreshold(Threshold) { + : Inliner(ID), Params(getInlineParams(Threshold)) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } @@ -65,8 +66,7 @@ Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); }; - return llvm::getInlineCost(CS, DefaultThreshold, TTI, GetAssumptionCache, - PSI); + return llvm::getInlineCost(CS, TTI, GetAssumptionCache, PSI, Params); } bool runOnSCC(CallGraphSCC &SCC) override; @@ -74,6 +74,44 @@ private: TargetTransformInfoWrapperPass *TTIWP; + + // Generate the parameters to tune the inline cost analysis based on + // commandline options. + InlineParams getInlineParams(int Threshold) { + InlineParams Params; + + // This field is the threshold to use for a callee by default. This is + // derived from one or more of: + // * optimization or size-optimization levels, + // * a value passed to createFunctionInliningPass function, or + // * the -inline-threshold flag. + // If the -inline-threshold flag is explicitly specified, that is used + // irrespective of anything else. + if (llvm::InlineThreshold.getNumOccurrences() > 0) + Params.DefaultThreshold = llvm::InlineThreshold; + else + Params.DefaultThreshold = Threshold; + + // Set the HintThreshold knob from the -inlinehint-threshold. + Params.HintThreshold = llvm::HintThreshold; + + // Set the OptMinSizeThreshold and OptSizeThreshold params only if the + // -inlinehint-threshold commandline option is not explicitly given. If that + // option is present, then its value applies even for callees with size and + // minsize attributes. + // If the -inline-threshold is not specified, set the ColdThreshold from the + // -inlinecold-threshold even if it is not explicitly passed. If + // -inline-threshold is specified, then -inlinecold-threshold needs to be + // explicitly specified to set the ColdThreshold knob + if (llvm::InlineThreshold.getNumOccurrences() == 0) { + Params.OptMinSizeThreshold = InlineConstants::OptMinSizeThreshold; + Params.OptSizeThreshold = InlineConstants::OptSizeThreshold; + Params.ColdThreshold = llvm::ColdThreshold; + } else if (ColdThreshold.getNumOccurrences() > 0) { + Params.ColdThreshold = llvm::ColdThreshold; + } + return Params; + } }; } // end anonymous namespace @@ -95,10 +133,21 @@ return new SimpleInliner(Threshold); } +static int computeThresholdFromOptLevels(unsigned OptLevel, + unsigned SizeOptLevel) { + if (OptLevel > 2) + return InlineConstants::OptAggressiveThreshold; + if (SizeOptLevel == 1) // -Os + return InlineConstants::OptSizeThreshold; + if (SizeOptLevel == 2) // -Oz + return InlineConstants::OptMinSizeThreshold; + return llvm::InlineThreshold; +} + Pass *llvm::createFunctionInliningPass(unsigned OptLevel, unsigned SizeOptLevel) { return new SimpleInliner( - llvm::computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); + computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); } bool SimpleInliner::runOnSCC(CallGraphSCC &SCC) {